]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/aarch64/tcg-target.inc.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg/i386: Implement vector minmax arithmetic
[mirror_qemu.git] / tcg / aarch64 / tcg-target.inc.c
CommitLineData
4a136e0a
CF		 1/*
2 * Initial TCG Implementation for aarch64
3 *
4 * Copyright (c) 2013 Huawei Technologies Duesseldorf GmbH
5 * Written by Claudio Fontana
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or
8 * (at your option) any later version.
9 *
10 * See the COPYING file in the top-level directory for details.
11 */
12
55129955 13#include "tcg-pool.inc.c"
4a136e0a
CF		 14#include "qemu/bitops.h"
15
7763ffa0
RH		 16/* We're going to re-use TCGType in setting of the SF bit, which controls
17 the size of the operation performed. If we know the values match, it
18 makes things much cleaner. */
19QEMU_BUILD_BUG_ON(TCG_TYPE_I32 != 0 || TCG_TYPE_I64 != 1);
20
8d8fdbae 21#ifdef CONFIG_DEBUG_TCG
4a136e0a 22static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
14e4c1e2
RH		 23 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
24 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
25 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
26 "x24", "x25", "x26", "x27", "x28", "fp", "x30", "sp",
27
28 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
29 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
30 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
31 "v24", "v25", "v26", "v27", "v28", "fp", "v30", "v31",
4a136e0a 32};
8d8fdbae 33#endif /* CONFIG_DEBUG_TCG */
4a136e0a
CF		 34
35static const int tcg_target_reg_alloc_order[] = {
36 TCG_REG_X20, TCG_REG_X21, TCG_REG_X22, TCG_REG_X23,
37 TCG_REG_X24, TCG_REG_X25, TCG_REG_X26, TCG_REG_X27,
b76f21a7 38 TCG_REG_X28, /* we will reserve this for guest_base if configured */
4a136e0a 39
d82b78e4
RH		 40 TCG_REG_X8, TCG_REG_X9, TCG_REG_X10, TCG_REG_X11,
41 TCG_REG_X12, TCG_REG_X13, TCG_REG_X14, TCG_REG_X15,
4a136e0a
CF		 42 TCG_REG_X16, TCG_REG_X17,
43
4a136e0a
CF		 44 TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
45 TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7,
46
d82b78e4
RH		 47 /* X18 reserved by system */
48 /* X19 reserved for AREG0 */
49 /* X29 reserved as fp */
50 /* X30 reserved as temporary */
14e4c1e2
RH		 51
52 TCG_REG_V0, TCG_REG_V1, TCG_REG_V2, TCG_REG_V3,
53 TCG_REG_V4, TCG_REG_V5, TCG_REG_V6, TCG_REG_V7,
54 /* V8 - V15 are call-saved, and skipped. */
55 TCG_REG_V16, TCG_REG_V17, TCG_REG_V18, TCG_REG_V19,
56 TCG_REG_V20, TCG_REG_V21, TCG_REG_V22, TCG_REG_V23,
57 TCG_REG_V24, TCG_REG_V25, TCG_REG_V26, TCG_REG_V27,
58 TCG_REG_V28, TCG_REG_V29, TCG_REG_V30, TCG_REG_V31,
4a136e0a
CF		 59};
60
61static const int tcg_target_call_iarg_regs[8] = {
62 TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
63 TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7
64};
65static const int tcg_target_call_oarg_regs[1] = {
66 TCG_REG_X0
67};
68
d82b78e4 69#define TCG_REG_TMP TCG_REG_X30
14e4c1e2 70#define TCG_VEC_TMP TCG_REG_V31
4a136e0a 71
6a91c7c9 72#ifndef CONFIG_SOFTMMU
352bcb0a
RH		 73/* Note that XZR cannot be encoded in the address base register slot,
74 as that actaully encodes SP. So if we need to zero-extend the guest
75 address, via the address index register slot, we need to load even
76 a zero guest base into a register. */
77#define USE_GUEST_BASE (guest_base != 0 || TARGET_LONG_BITS == 32)
4cbea598 78#define TCG_REG_GUEST_BASE TCG_REG_X28
6a91c7c9
JK		 79#endif
80
214bfe83 81static inline bool reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
4a136e0a 82{
8587c30c 83 ptrdiff_t offset = target - code_ptr;
214bfe83
RH		 84 if (offset == sextract64(offset, 0, 26)) {
85 /* read instruction, mask away previous PC_REL26 parameter contents,
86 set the proper offset, then write back the instruction. */
87 *code_ptr = deposit32(*code_ptr, 0, 26, offset);
88 return true;
89 }
90 return false;
4a136e0a
CF		 91}
92
214bfe83 93static inline bool reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
4a136e0a 94{
8587c30c 95 ptrdiff_t offset = target - code_ptr;
214bfe83
RH		 96 if (offset == sextract64(offset, 0, 19)) {
97 *code_ptr = deposit32(*code_ptr, 5, 19, offset);
98 return true;
99 }
100 return false;
4a136e0a
CF		 101}
102
6ac17786 103static inline bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 104 intptr_t value, intptr_t addend)
4a136e0a 105{
eabb7b91 106 tcg_debug_assert(addend == 0);
4a136e0a
CF		 107 switch (type) {
108 case R_AARCH64_JUMP26:
109 case R_AARCH64_CALL26:
214bfe83 110 return reloc_pc26(code_ptr, (tcg_insn_unit *)value);
4a136e0a 111 case R_AARCH64_CONDBR19:
214bfe83 112 return reloc_pc19(code_ptr, (tcg_insn_unit *)value);
4a136e0a 113 default:
214bfe83 114 g_assert_not_reached();
4a136e0a
CF		 115 }
116}
117
170bf931
RH		 118#define TCG_CT_CONST_AIMM 0x100
119#define TCG_CT_CONST_LIMM 0x200
120#define TCG_CT_CONST_ZERO 0x400
121#define TCG_CT_CONST_MONE 0x800
90f1cd91 122
4a136e0a 123/* parse target specific constraints */
069ea736
RH		 124static const char *target_parse_constraint(TCGArgConstraint *ct,
125 const char *ct_str, TCGType type)
4a136e0a 126{
069ea736 127 switch (*ct_str++) {
14e4c1e2 128 case 'r': /* general registers */
4a136e0a 129 ct->ct |= TCG_CT_REG;
14e4c1e2
RH		 130 ct->u.regs |= 0xffffffffu;
131 break;
132 case 'w': /* advsimd registers */
133 ct->ct |= TCG_CT_REG;
134 ct->u.regs |= 0xffffffff00000000ull;
4a136e0a
CF		 135 break;
136 case 'l': /* qemu_ld / qemu_st address, data_reg */
137 ct->ct |= TCG_CT_REG;
f46934df 138 ct->u.regs = 0xffffffffu;
4a136e0a
CF		 139#ifdef CONFIG_SOFTMMU
140 /* x0 and x1 will be overwritten when reading the tlb entry,
141 and x2, and x3 for helper args, better to avoid using them. */
142 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X0);
143 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X1);
144 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X2);
145 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X3);
146#endif
147 break;
90f1cd91
RH		 148 case 'A': /* Valid for arithmetic immediate (positive or negative). */
149 ct->ct |= TCG_CT_CONST_AIMM;
150 break;
e029f293
RH		 151 case 'L': /* Valid for logical immediate. */
152 ct->ct |= TCG_CT_CONST_LIMM;
153 break;
c6e929e7
RH		 154 case 'M': /* minus one */
155 ct->ct |= TCG_CT_CONST_MONE;
156 break;
04ce397b
RH		 157 case 'Z': /* zero */
158 ct->ct |= TCG_CT_CONST_ZERO;
159 break;
4a136e0a 160 default:
069ea736 161 return NULL;
4a136e0a 162 }
069ea736 163 return ct_str;
4a136e0a
CF		 164}
165
14e4c1e2 166/* Match a constant valid for addition (12-bit, optionally shifted). */
90f1cd91
RH		 167static inline bool is_aimm(uint64_t val)
168{
169 return (val & ~0xfff) == 0 || (val & ~0xfff000) == 0;
170}
171
14e4c1e2 172/* Match a constant valid for logical operations. */
e029f293
RH		 173static inline bool is_limm(uint64_t val)
174{
175 /* Taking a simplified view of the logical immediates for now, ignoring
176 the replication that can happen across the field. Match bit patterns
177 of the forms
178 0....01....1
179 0..01..10..0
180 and their inverses. */
181
182 /* Make things easier below, by testing the form with msb clear. */
183 if ((int64_t)val < 0) {
184 val = ~val;
185 }
186 if (val == 0) {
187 return false;
188 }
189 val += val & -val;
190 return (val & (val - 1)) == 0;
191}
192
14e4c1e2
RH		 193/* Match a constant that is valid for vectors. */
194static bool is_fimm(uint64_t v64, int *op, int *cmode, int *imm8)
195{
196 int i;
197
198 *op = 0;
199 /* Match replication across 8 bits. */
200 if (v64 == dup_const(MO_8, v64)) {
201 *cmode = 0xe;
202 *imm8 = v64 & 0xff;
203 return true;
204 }
205 /* Match replication across 16 bits. */
206 if (v64 == dup_const(MO_16, v64)) {
207 uint16_t v16 = v64;
208
209 if (v16 == (v16 & 0xff)) {
210 *cmode = 0x8;
211 *imm8 = v16 & 0xff;
212 return true;
213 } else if (v16 == (v16 & 0xff00)) {
214 *cmode = 0xa;
215 *imm8 = v16 >> 8;
216 return true;
217 }
218 }
219 /* Match replication across 32 bits. */
220 if (v64 == dup_const(MO_32, v64)) {
221 uint32_t v32 = v64;
222
223 if (v32 == (v32 & 0xff)) {
224 *cmode = 0x0;
225 *imm8 = v32 & 0xff;
226 return true;
227 } else if (v32 == (v32 & 0xff00)) {
228 *cmode = 0x2;
229 *imm8 = (v32 >> 8) & 0xff;
230 return true;
231 } else if (v32 == (v32 & 0xff0000)) {
232 *cmode = 0x4;
233 *imm8 = (v32 >> 16) & 0xff;
234 return true;
235 } else if (v32 == (v32 & 0xff000000)) {
236 *cmode = 0x6;
237 *imm8 = v32 >> 24;
238 return true;
239 } else if ((v32 & 0xffff00ff) == 0xff) {
240 *cmode = 0xc;
241 *imm8 = (v32 >> 8) & 0xff;
242 return true;
243 } else if ((v32 & 0xff00ffff) == 0xffff) {
244 *cmode = 0xd;
245 *imm8 = (v32 >> 16) & 0xff;
246 return true;
247 }
248 /* Match forms of a float32. */
249 if (extract32(v32, 0, 19) == 0
250 && (extract32(v32, 25, 6) == 0x20
251 || extract32(v32, 25, 6) == 0x1f)) {
252 *cmode = 0xf;
253 *imm8 = (extract32(v32, 31, 1) << 7)
254 | (extract32(v32, 25, 1) << 6)
255 | extract32(v32, 19, 6);
256 return true;
257 }
258 }
259 /* Match forms of a float64. */
260 if (extract64(v64, 0, 48) == 0
261 && (extract64(v64, 54, 9) == 0x100
262 || extract64(v64, 54, 9) == 0x0ff)) {
263 *cmode = 0xf;
264 *op = 1;
265 *imm8 = (extract64(v64, 63, 1) << 7)
266 | (extract64(v64, 54, 1) << 6)
267 | extract64(v64, 48, 6);
268 return true;
269 }
270 /* Match bytes of 0x00 and 0xff. */
271 for (i = 0; i < 64; i += 8) {
272 uint64_t byte = extract64(v64, i, 8);
273 if (byte != 0 && byte != 0xff) {
274 break;
275 }
276 }
277 if (i == 64) {
278 *cmode = 0xe;
279 *op = 1;
280 *imm8 = (extract64(v64, 0, 1) << 0)
281 | (extract64(v64, 8, 1) << 1)
282 | (extract64(v64, 16, 1) << 2)
283 | (extract64(v64, 24, 1) << 3)
284 | (extract64(v64, 32, 1) << 4)
285 | (extract64(v64, 40, 1) << 5)
286 | (extract64(v64, 48, 1) << 6)
287 | (extract64(v64, 56, 1) << 7);
288 return true;
289 }
290 return false;
291}
292
f6c6afc1 293static int tcg_target_const_match(tcg_target_long val, TCGType type,
90f1cd91 294 const TCGArgConstraint *arg_ct)
4a136e0a
CF		 295{
296 int ct = arg_ct->ct;
297
298 if (ct & TCG_CT_CONST) {
299 return 1;
300 }
170bf931 301 if (type == TCG_TYPE_I32) {
90f1cd91
RH		 302 val = (int32_t)val;
303 }
304 if ((ct & TCG_CT_CONST_AIMM) && (is_aimm(val) || is_aimm(-val))) {
305 return 1;
306 }
e029f293
RH		 307 if ((ct & TCG_CT_CONST_LIMM) && is_limm(val)) {
308 return 1;
309 }
04ce397b
RH		 310 if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
311 return 1;
312 }
c6e929e7
RH		 313 if ((ct & TCG_CT_CONST_MONE) && val == -1) {
314 return 1;
315 }
4a136e0a
CF		 316
317 return 0;
318}
319
320enum aarch64_cond_code {
321 COND_EQ = 0x0,
322 COND_NE = 0x1,
323 COND_CS = 0x2, /* Unsigned greater or equal */
324 COND_HS = COND_CS, /* ALIAS greater or equal */
325 COND_CC = 0x3, /* Unsigned less than */
326 COND_LO = COND_CC, /* ALIAS Lower */
327 COND_MI = 0x4, /* Negative */
328 COND_PL = 0x5, /* Zero or greater */
329 COND_VS = 0x6, /* Overflow */
330 COND_VC = 0x7, /* No overflow */
331 COND_HI = 0x8, /* Unsigned greater than */
332 COND_LS = 0x9, /* Unsigned less or equal */
333 COND_GE = 0xa,
334 COND_LT = 0xb,
335 COND_GT = 0xc,
336 COND_LE = 0xd,
337 COND_AL = 0xe,
338 COND_NV = 0xf, /* behaves like COND_AL here */
339};
340
341static const enum aarch64_cond_code tcg_cond_to_aarch64[] = {
342 [TCG_COND_EQ] = COND_EQ,
343 [TCG_COND_NE] = COND_NE,
344 [TCG_COND_LT] = COND_LT,
345 [TCG_COND_GE] = COND_GE,
346 [TCG_COND_LE] = COND_LE,
347 [TCG_COND_GT] = COND_GT,
348 /* unsigned */
349 [TCG_COND_LTU] = COND_LO,
350 [TCG_COND_GTU] = COND_HI,
351 [TCG_COND_GEU] = COND_HS,
352 [TCG_COND_LEU] = COND_LS,
353};
354
3d4299f4
RH		 355typedef enum {
356 LDST_ST = 0, /* store */
357 LDST_LD = 1, /* load */
358 LDST_LD_S_X = 2, /* load and sign-extend into Xt */
359 LDST_LD_S_W = 3, /* load and sign-extend into Wt */
360} AArch64LdstType;
4a136e0a 361
50573c66
RH		 362/* We encode the format of the insn into the beginning of the name, so that
363 we can have the preprocessor help "typecheck" the insn vs the output
364 function. Arm didn't provide us with nice names for the formats, so we
365 use the section number of the architecture reference manual in which the
366 instruction group is described. */
367typedef enum {
3d9e69a2
RH		 368 /* Compare and branch (immediate). */
369 I3201_CBZ = 0x34000000,
370 I3201_CBNZ = 0x35000000,
371
81d8a5ee
RH		 372 /* Conditional branch (immediate). */
373 I3202_B_C = 0x54000000,
374
375 /* Unconditional branch (immediate). */
376 I3206_B = 0x14000000,
377 I3206_BL = 0x94000000,
378
379 /* Unconditional branch (register). */
380 I3207_BR = 0xd61f0000,
381 I3207_BLR = 0xd63f0000,
382 I3207_RET = 0xd65f0000,
383
2acee8b2
PK		 384 /* Load literal for loading the address at pc-relative offset */
385 I3305_LDR = 0x58000000,
14e4c1e2
RH		 386 I3305_LDR_v64 = 0x5c000000,
387 I3305_LDR_v128 = 0x9c000000,
388
3d4299f4
RH		 389 /* Load/store register. Described here as 3.3.12, but the helper
390 that emits them can transform to 3.3.10 or 3.3.13. */
391 I3312_STRB = 0x38000000 | LDST_ST << 22 | MO_8 << 30,
392 I3312_STRH = 0x38000000 | LDST_ST << 22 | MO_16 << 30,
393 I3312_STRW = 0x38000000 | LDST_ST << 22 | MO_32 << 30,
394 I3312_STRX = 0x38000000 | LDST_ST << 22 | MO_64 << 30,
395
396 I3312_LDRB = 0x38000000 | LDST_LD << 22 | MO_8 << 30,
397 I3312_LDRH = 0x38000000 | LDST_LD << 22 | MO_16 << 30,
398 I3312_LDRW = 0x38000000 | LDST_LD << 22 | MO_32 << 30,
399 I3312_LDRX = 0x38000000 | LDST_LD << 22 | MO_64 << 30,
400
401 I3312_LDRSBW = 0x38000000 | LDST_LD_S_W << 22 | MO_8 << 30,
402 I3312_LDRSHW = 0x38000000 | LDST_LD_S_W << 22 | MO_16 << 30,
403
404 I3312_LDRSBX = 0x38000000 | LDST_LD_S_X << 22 | MO_8 << 30,
405 I3312_LDRSHX = 0x38000000 | LDST_LD_S_X << 22 | MO_16 << 30,
406 I3312_LDRSWX = 0x38000000 | LDST_LD_S_X << 22 | MO_32 << 30,
407
14e4c1e2
RH		 408 I3312_LDRVS = 0x3c000000 | LDST_LD << 22 | MO_32 << 30,
409 I3312_STRVS = 0x3c000000 | LDST_ST << 22 | MO_32 << 30,
410
411 I3312_LDRVD = 0x3c000000 | LDST_LD << 22 | MO_64 << 30,
412 I3312_STRVD = 0x3c000000 | LDST_ST << 22 | MO_64 << 30,
413
414 I3312_LDRVQ = 0x3c000000 | 3 << 22 | 0 << 30,
415 I3312_STRVQ = 0x3c000000 | 2 << 22 | 0 << 30,
416
6c0f0c0f 417 I3312_TO_I3310 = 0x00200800,
3d4299f4
RH		 418 I3312_TO_I3313 = 0x01000000,
419
95f72aa9
RH		 420 /* Load/store register pair instructions. */
421 I3314_LDP = 0x28400000,
422 I3314_STP = 0x28000000,
423
096c46c0
RH		 424 /* Add/subtract immediate instructions. */
425 I3401_ADDI = 0x11000000,
426 I3401_ADDSI = 0x31000000,
427 I3401_SUBI = 0x51000000,
428 I3401_SUBSI = 0x71000000,
429
b3c56df7
RH		 430 /* Bitfield instructions. */
431 I3402_BFM = 0x33000000,
432 I3402_SBFM = 0x13000000,
433 I3402_UBFM = 0x53000000,
434
435 /* Extract instruction. */
436 I3403_EXTR = 0x13800000,
437
e029f293
RH		 438 /* Logical immediate instructions. */
439 I3404_ANDI = 0x12000000,
440 I3404_ORRI = 0x32000000,
441 I3404_EORI = 0x52000000,
442
582ab779
RH		 443 /* Move wide immediate instructions. */
444 I3405_MOVN = 0x12800000,
445 I3405_MOVZ = 0x52800000,
446 I3405_MOVK = 0x72800000,
447
c6e310d9
RH		 448 /* PC relative addressing instructions. */
449 I3406_ADR = 0x10000000,
450 I3406_ADRP = 0x90000000,
451
50573c66
RH		 452 /* Add/subtract shifted register instructions (without a shift). */
453 I3502_ADD = 0x0b000000,
454 I3502_ADDS = 0x2b000000,
455 I3502_SUB = 0x4b000000,
456 I3502_SUBS = 0x6b000000,
457
458 /* Add/subtract shifted register instructions (with a shift). */
459 I3502S_ADD_LSL = I3502_ADD,
460
c6e929e7
RH		 461 /* Add/subtract with carry instructions. */
462 I3503_ADC = 0x1a000000,
463 I3503_SBC = 0x5a000000,
464
04ce397b
RH		 465 /* Conditional select instructions. */
466 I3506_CSEL = 0x1a800000,
467 I3506_CSINC = 0x1a800400,
53c76c19
RH		 468 I3506_CSINV = 0x5a800000,
469 I3506_CSNEG = 0x5a800400,
04ce397b 470
edd8824c 471 /* Data-processing (1 source) instructions. */
53c76c19
RH		 472 I3507_CLZ = 0x5ac01000,
473 I3507_RBIT = 0x5ac00000,
edd8824c
RH		 474 I3507_REV16 = 0x5ac00400,
475 I3507_REV32 = 0x5ac00800,
476 I3507_REV64 = 0x5ac00c00,
477
df9351e3
RH		 478 /* Data-processing (2 source) instructions. */
479 I3508_LSLV = 0x1ac02000,
480 I3508_LSRV = 0x1ac02400,
481 I3508_ASRV = 0x1ac02800,
482 I3508_RORV = 0x1ac02c00,
1fcc9ddf
RH		 483 I3508_SMULH = 0x9b407c00,
484 I3508_UMULH = 0x9bc07c00,
8678b71c
RH		 485 I3508_UDIV = 0x1ac00800,
486 I3508_SDIV = 0x1ac00c00,
487
488 /* Data-processing (3 source) instructions. */
489 I3509_MADD = 0x1b000000,
490 I3509_MSUB = 0x1b008000,
df9351e3 491
50573c66
RH		 492 /* Logical shifted register instructions (without a shift). */
493 I3510_AND = 0x0a000000,
14b155dd 494 I3510_BIC = 0x0a200000,
50573c66 495 I3510_ORR = 0x2a000000,
14b155dd 496 I3510_ORN = 0x2a200000,
50573c66 497 I3510_EOR = 0x4a000000,
14b155dd 498 I3510_EON = 0x4a200000,
50573c66 499 I3510_ANDS = 0x6a000000,
c7a59c2a 500
14e4c1e2
RH		 501 /* AdvSIMD copy */
502 I3605_DUP = 0x0e000400,
503 I3605_INS = 0x4e001c00,
504 I3605_UMOV = 0x0e003c00,
505
506 /* AdvSIMD modified immediate */
507 I3606_MOVI = 0x0f000400,
508
509 /* AdvSIMD shift by immediate */
510 I3614_SSHR = 0x0f000400,
511 I3614_SSRA = 0x0f001400,
512 I3614_SHL = 0x0f005400,
513 I3614_USHR = 0x2f000400,
514 I3614_USRA = 0x2f001400,
515
516 /* AdvSIMD three same. */
517 I3616_ADD = 0x0e208400,
518 I3616_AND = 0x0e201c00,
519 I3616_BIC = 0x0e601c00,
520 I3616_EOR = 0x2e201c00,
521 I3616_MUL = 0x0e209c00,
522 I3616_ORR = 0x0ea01c00,
523 I3616_ORN = 0x0ee01c00,
524 I3616_SUB = 0x2e208400,
525 I3616_CMGT = 0x0e203400,
526 I3616_CMGE = 0x0e203c00,
527 I3616_CMTST = 0x0e208c00,
528 I3616_CMHI = 0x2e203400,
529 I3616_CMHS = 0x2e203c00,
530 I3616_CMEQ = 0x2e208c00,
531
532 /* AdvSIMD two-reg misc. */
533 I3617_CMGT0 = 0x0e208800,
534 I3617_CMEQ0 = 0x0e209800,
535 I3617_CMLT0 = 0x0e20a800,
536 I3617_CMGE0 = 0x2e208800,
537 I3617_CMLE0 = 0x2e20a800,
538 I3617_NOT = 0x2e205800,
539 I3617_NEG = 0x2e20b800,
540
c7a59c2a 541 /* System instructions. */
14e4c1e2 542 NOP = 0xd503201f,
c7a59c2a
PK		 543 DMB_ISH = 0xd50338bf,
544 DMB_LD = 0x00000100,
545 DMB_ST = 0x00000200,
50573c66 546} AArch64Insn;
4a136e0a 547
4a136e0a
CF		 548static inline uint32_t tcg_in32(TCGContext *s)
549{
550 uint32_t v = *(uint32_t *)s->code_ptr;
551 return v;
552}
553
50573c66
RH		 554/* Emit an opcode with "type-checking" of the format. */
555#define tcg_out_insn(S, FMT, OP, ...) \
556 glue(tcg_out_insn_,FMT)(S, glue(glue(glue(I,FMT),_),OP), ## __VA_ARGS__)
557
2acee8b2
PK		 558static void tcg_out_insn_3305(TCGContext *s, AArch64Insn insn, int imm19, TCGReg rt)
559{
560 tcg_out32(s, insn | (imm19 & 0x7ffff) << 5 | rt);
561}
562
3d9e69a2
RH		 563static void tcg_out_insn_3201(TCGContext *s, AArch64Insn insn, TCGType ext,
564 TCGReg rt, int imm19)
565{
566 tcg_out32(s, insn | ext << 31 | (imm19 & 0x7ffff) << 5 | rt);
567}
568
81d8a5ee
RH		 569static void tcg_out_insn_3202(TCGContext *s, AArch64Insn insn,
570 TCGCond c, int imm19)
571{
572 tcg_out32(s, insn | tcg_cond_to_aarch64[c] | (imm19 & 0x7ffff) << 5);
573}
574
575static void tcg_out_insn_3206(TCGContext *s, AArch64Insn insn, int imm26)
576{
577 tcg_out32(s, insn | (imm26 & 0x03ffffff));
578}
579
580static void tcg_out_insn_3207(TCGContext *s, AArch64Insn insn, TCGReg rn)
581{
582 tcg_out32(s, insn | rn << 5);
583}
584
95f72aa9
RH		 585static void tcg_out_insn_3314(TCGContext *s, AArch64Insn insn,
586 TCGReg r1, TCGReg r2, TCGReg rn,
587 tcg_target_long ofs, bool pre, bool w)
588{
589 insn |= 1u << 31; /* ext */
590 insn |= pre << 24;
591 insn |= w << 23;
592
eabb7b91 593 tcg_debug_assert(ofs >= -0x200 && ofs < 0x200 && (ofs & 7) == 0);
95f72aa9
RH		 594 insn |= (ofs & (0x7f << 3)) << (15 - 3);
595
596 tcg_out32(s, insn | r2 << 10 | rn << 5 | r1);
597}
598
096c46c0
RH		 599static void tcg_out_insn_3401(TCGContext *s, AArch64Insn insn, TCGType ext,
600 TCGReg rd, TCGReg rn, uint64_t aimm)
601{
602 if (aimm > 0xfff) {
eabb7b91 603 tcg_debug_assert((aimm & 0xfff) == 0);
096c46c0 604 aimm >>= 12;
eabb7b91 605 tcg_debug_assert(aimm <= 0xfff);
096c46c0
RH		 606 aimm |= 1 << 12; /* apply LSL 12 */
607 }
608 tcg_out32(s, insn | ext << 31 | aimm << 10 | rn << 5 | rd);
609}
610
e029f293
RH		 611/* This function can be used for both 3.4.2 (Bitfield) and 3.4.4
612 (Logical immediate). Both insn groups have N, IMMR and IMMS fields
613 that feed the DecodeBitMasks pseudo function. */
614static void tcg_out_insn_3402(TCGContext *s, AArch64Insn insn, TCGType ext,
615 TCGReg rd, TCGReg rn, int n, int immr, int imms)
616{
617 tcg_out32(s, insn | ext << 31 | n << 22 | immr << 16 | imms << 10
618 | rn << 5 | rd);
619}
620
621#define tcg_out_insn_3404 tcg_out_insn_3402
622
b3c56df7
RH		 623static void tcg_out_insn_3403(TCGContext *s, AArch64Insn insn, TCGType ext,
624 TCGReg rd, TCGReg rn, TCGReg rm, int imms)
625{
626 tcg_out32(s, insn | ext << 31 | ext << 22 | rm << 16 | imms << 10
627 | rn << 5 | rd);
628}
629
582ab779
RH		 630/* This function is used for the Move (wide immediate) instruction group.
631 Note that SHIFT is a full shift count, not the 2 bit HW field. */
632static void tcg_out_insn_3405(TCGContext *s, AArch64Insn insn, TCGType ext,
633 TCGReg rd, uint16_t half, unsigned shift)
634{
eabb7b91 635 tcg_debug_assert((shift & ~0x30) == 0);
582ab779
RH		 636 tcg_out32(s, insn | ext << 31 | shift << (21 - 4) | half << 5 | rd);
637}
638
c6e310d9
RH		 639static void tcg_out_insn_3406(TCGContext *s, AArch64Insn insn,
640 TCGReg rd, int64_t disp)
641{
642 tcg_out32(s, insn | (disp & 3) << 29 | (disp & 0x1ffffc) << (5 - 2) | rd);
643}
644
50573c66
RH		 645/* This function is for both 3.5.2 (Add/Subtract shifted register), for
646 the rare occasion when we actually want to supply a shift amount. */
647static inline void tcg_out_insn_3502S(TCGContext *s, AArch64Insn insn,
648 TCGType ext, TCGReg rd, TCGReg rn,
649 TCGReg rm, int imm6)
650{
651 tcg_out32(s, insn | ext << 31 | rm << 16 | imm6 << 10 | rn << 5 | rd);
652}
653
654/* This function is for 3.5.2 (Add/subtract shifted register),
655 and 3.5.10 (Logical shifted register), for the vast majorty of cases
656 when we don't want to apply a shift. Thus it can also be used for
657 3.5.3 (Add/subtract with carry) and 3.5.8 (Data processing 2 source). */
658static void tcg_out_insn_3502(TCGContext *s, AArch64Insn insn, TCGType ext,
659 TCGReg rd, TCGReg rn, TCGReg rm)
660{
661 tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd);
662}
663
664#define tcg_out_insn_3503 tcg_out_insn_3502
665#define tcg_out_insn_3508 tcg_out_insn_3502
666#define tcg_out_insn_3510 tcg_out_insn_3502
667
04ce397b
RH		 668static void tcg_out_insn_3506(TCGContext *s, AArch64Insn insn, TCGType ext,
669 TCGReg rd, TCGReg rn, TCGReg rm, TCGCond c)
670{
671 tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd
672 | tcg_cond_to_aarch64[c] << 12);
673}
674
edd8824c
RH		 675static void tcg_out_insn_3507(TCGContext *s, AArch64Insn insn, TCGType ext,
676 TCGReg rd, TCGReg rn)
677{
678 tcg_out32(s, insn | ext << 31 | rn << 5 | rd);
679}
680
8678b71c
RH		 681static void tcg_out_insn_3509(TCGContext *s, AArch64Insn insn, TCGType ext,
682 TCGReg rd, TCGReg rn, TCGReg rm, TCGReg ra)
683{
684 tcg_out32(s, insn | ext << 31 | rm << 16 | ra << 10 | rn << 5 | rd);
685}
686
14e4c1e2
RH		 687static void tcg_out_insn_3605(TCGContext *s, AArch64Insn insn, bool q,
688 TCGReg rd, TCGReg rn, int dst_idx, int src_idx)
689{
690 /* Note that bit 11 set means general register input. Therefore
691 we can handle both register sets with one function. */
692 tcg_out32(s, insn | q << 30 | (dst_idx << 16) | (src_idx << 11)
693 | (rd & 0x1f) | (~rn & 0x20) << 6 | (rn & 0x1f) << 5);
694}
695
696static void tcg_out_insn_3606(TCGContext *s, AArch64Insn insn, bool q,
697 TCGReg rd, bool op, int cmode, uint8_t imm8)
698{
699 tcg_out32(s, insn | q << 30 | op << 29 | cmode << 12 | (rd & 0x1f)
700 | (imm8 & 0xe0) << (16 - 5) | (imm8 & 0x1f) << 5);
701}
702
703static void tcg_out_insn_3614(TCGContext *s, AArch64Insn insn, bool q,
704 TCGReg rd, TCGReg rn, unsigned immhb)
705{
706 tcg_out32(s, insn | q << 30 | immhb << 16
707 | (rn & 0x1f) << 5 | (rd & 0x1f));
708}
709
710static void tcg_out_insn_3616(TCGContext *s, AArch64Insn insn, bool q,
711 unsigned size, TCGReg rd, TCGReg rn, TCGReg rm)
712{
713 tcg_out32(s, insn | q << 30 | (size << 22) | (rm & 0x1f) << 16
714 | (rn & 0x1f) << 5 | (rd & 0x1f));
715}
716
717static void tcg_out_insn_3617(TCGContext *s, AArch64Insn insn, bool q,
718 unsigned size, TCGReg rd, TCGReg rn)
719{
720 tcg_out32(s, insn | q << 30 | (size << 22)
721 | (rn & 0x1f) << 5 | (rd & 0x1f));
722}
723
3d4299f4 724static void tcg_out_insn_3310(TCGContext *s, AArch64Insn insn,
6c0f0c0f
PB		 725 TCGReg rd, TCGReg base, TCGType ext,
726 TCGReg regoff)
3d4299f4
RH		 727{
728 /* Note the AArch64Insn constants above are for C3.3.12. Adjust. */
6c0f0c0f 729 tcg_out32(s, insn | I3312_TO_I3310 | regoff << 16 |
14e4c1e2 730 0x4000 | ext << 13 | base << 5 | (rd & 0x1f));
3d4299f4 731}
50573c66 732
3d4299f4
RH		 733static void tcg_out_insn_3312(TCGContext *s, AArch64Insn insn,
734 TCGReg rd, TCGReg rn, intptr_t offset)
4a136e0a 735{
14e4c1e2 736 tcg_out32(s, insn | (offset & 0x1ff) << 12 | rn << 5 | (rd & 0x1f));
4a136e0a
CF		 737}
738
3d4299f4
RH		 739static void tcg_out_insn_3313(TCGContext *s, AArch64Insn insn,
740 TCGReg rd, TCGReg rn, uintptr_t scaled_uimm)
b1f6dc0d 741{
3d4299f4 742 /* Note the AArch64Insn constants above are for C3.3.12. Adjust. */
14e4c1e2
RH		 743 tcg_out32(s, insn | I3312_TO_I3313 | scaled_uimm << 10
744 | rn << 5 | (rd & 0x1f));
b1f6dc0d
CF		 745}
746
7d11fc7c
RH		 747/* Register to register move using ORR (shifted register with no shift). */
748static void tcg_out_movr(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rm)
4a136e0a 749{
7d11fc7c
RH		 750 tcg_out_insn(s, 3510, ORR, ext, rd, TCG_REG_XZR, rm);
751}
752
753/* Register to register move using ADDI (move to/from SP). */
754static void tcg_out_movr_sp(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rn)
755{
756 tcg_out_insn(s, 3401, ADDI, ext, rd, rn, 0);
4a136e0a
CF		 757}
758
4ec4f0bd
RH		 759/* This function is used for the Logical (immediate) instruction group.
760 The value of LIMM must satisfy IS_LIMM. See the comment above about
761 only supporting simplified logical immediates. */
762static void tcg_out_logicali(TCGContext *s, AArch64Insn insn, TCGType ext,
763 TCGReg rd, TCGReg rn, uint64_t limm)
764{
765 unsigned h, l, r, c;
766
eabb7b91 767 tcg_debug_assert(is_limm(limm));
4ec4f0bd
RH		 768
769 h = clz64(limm);
770 l = ctz64(limm);
771 if (l == 0) {
772 r = 0; /* form 0....01....1 */
773 c = ctz64(~limm) - 1;
774 if (h == 0) {
775 r = clz64(~limm); /* form 1..10..01..1 */
776 c += r;
777 }
778 } else {
779 r = 64 - l; /* form 1....10....0 or 0..01..10..0 */
780 c = r - h - 1;
781 }
782 if (ext == TCG_TYPE_I32) {
783 r &= 31;
784 c &= 31;
785 }
786
787 tcg_out_insn_3404(s, insn, ext, rd, rn, ext, r, c);
788}
789
14e4c1e2
RH		 790static void tcg_out_dupi_vec(TCGContext *s, TCGType type,
791 TCGReg rd, uint64_t v64)
792{
793 int op, cmode, imm8;
794
795 if (is_fimm(v64, &op, &cmode, &imm8)) {
796 tcg_out_insn(s, 3606, MOVI, type == TCG_TYPE_V128, rd, op, cmode, imm8);
797 } else if (type == TCG_TYPE_V128) {
798 new_pool_l2(s, R_AARCH64_CONDBR19, s->code_ptr, 0, v64, v64);
799 tcg_out_insn(s, 3305, LDR_v128, 0, rd);
800 } else {
801 new_pool_label(s, v64, R_AARCH64_CONDBR19, s->code_ptr, 0);
802 tcg_out_insn(s, 3305, LDR_v64, 0, rd);
803 }
804}
805
582ab779
RH		 806static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
807 tcg_target_long value)
4a136e0a 808{
dfeb5fe7
RH		 809 tcg_target_long svalue = value;
810 tcg_target_long ivalue = ~value;
55129955
RH		 811 tcg_target_long t0, t1, t2;
812 int s0, s1;
813 AArch64Insn opc;
dfeb5fe7 814
14e4c1e2
RH		 815 switch (type) {
816 case TCG_TYPE_I32:
817 case TCG_TYPE_I64:
818 tcg_debug_assert(rd < 32);
819 break;
820
821 case TCG_TYPE_V64:
822 case TCG_TYPE_V128:
823 tcg_debug_assert(rd >= 32);
824 tcg_out_dupi_vec(s, type, rd, value);
825 return;
826
827 default:
828 g_assert_not_reached();
829 }
830
dfeb5fe7
RH		 831 /* For 32-bit values, discard potential garbage in value. For 64-bit
832 values within [2**31, 2**32-1], we can create smaller sequences by
833 interpreting this as a negative 32-bit number, while ensuring that
834 the high 32 bits are cleared by setting SF=0. */
835 if (type == TCG_TYPE_I32 || (value & ~0xffffffffull) == 0) {
836 svalue = (int32_t)value;
582ab779 837 value = (uint32_t)value;
dfeb5fe7
RH		 838 ivalue = (uint32_t)ivalue;
839 type = TCG_TYPE_I32;
840 }
841
d8918df5
RH		 842 /* Speed things up by handling the common case of small positive
843 and negative values specially. */
844 if ((value & ~0xffffull) == 0) {
845 tcg_out_insn(s, 3405, MOVZ, type, rd, value, 0);
846 return;
847 } else if ((ivalue & ~0xffffull) == 0) {
848 tcg_out_insn(s, 3405, MOVN, type, rd, ivalue, 0);
849 return;
850 }
851
4ec4f0bd
RH		 852 /* Check for bitfield immediates. For the benefit of 32-bit quantities,
853 use the sign-extended value. That lets us match rotated values such
854 as 0xff0000ff with the same 64-bit logic matching 0xffffffffff0000ff. */
855 if (is_limm(svalue)) {
856 tcg_out_logicali(s, I3404_ORRI, type, rd, TCG_REG_XZR, svalue);
857 return;
858 }
859
c6e310d9
RH		 860 /* Look for host pointer values within 4G of the PC. This happens
861 often when loading pointers to QEMU's own data structures. */
862 if (type == TCG_TYPE_I64) {
cc74d332
RH		 863 tcg_target_long disp = value - (intptr_t)s->code_ptr;
864 if (disp == sextract64(disp, 0, 21)) {
865 tcg_out_insn(s, 3406, ADR, rd, disp);
866 return;
867 }
868 disp = (value >> 12) - ((intptr_t)s->code_ptr >> 12);
c6e310d9
RH		 869 if (disp == sextract64(disp, 0, 21)) {
870 tcg_out_insn(s, 3406, ADRP, rd, disp);
871 if (value & 0xfff) {
872 tcg_out_insn(s, 3401, ADDI, type, rd, rd, value & 0xfff);
873 }
874 return;
875 }
876 }
877
55129955
RH		 878 /* Would it take fewer insns to begin with MOVN? */
879 if (ctpop64(value) >= 32) {
880 t0 = ivalue;
881 opc = I3405_MOVN;
8cf9a3d3 882 } else {
55129955
RH		 883 t0 = value;
884 opc = I3405_MOVZ;
885 }
886 s0 = ctz64(t0) & (63 & -16);
887 t1 = t0 & ~(0xffffUL << s0);
888 s1 = ctz64(t1) & (63 & -16);
889 t2 = t1 & ~(0xffffUL << s1);
890 if (t2 == 0) {
891 tcg_out_insn_3405(s, opc, type, rd, t0 >> s0, s0);
892 if (t1 != 0) {
893 tcg_out_insn(s, 3405, MOVK, type, rd, value >> s1, s1);
8cf9a3d3 894 }
55129955 895 return;
dfeb5fe7 896 }
55129955
RH		 897
898 /* For more than 2 insns, dump it into the constant pool. */
899 new_pool_label(s, value, R_AARCH64_CONDBR19, s->code_ptr, 0);
900 tcg_out_insn(s, 3305, LDR, 0, rd);
4a136e0a
CF		 901}
902
3d4299f4
RH		 903/* Define something more legible for general use. */
904#define tcg_out_ldst_r tcg_out_insn_3310
4a136e0a 905
14e4c1e2
RH		 906static void tcg_out_ldst(TCGContext *s, AArch64Insn insn, TCGReg rd,
907 TCGReg rn, intptr_t offset, int lgsize)
4a136e0a 908{
3d4299f4
RH		 909 /* If the offset is naturally aligned and in range, then we can
910 use the scaled uimm12 encoding */
14e4c1e2
RH		 911 if (offset >= 0 && !(offset & ((1 << lgsize) - 1))) {
912 uintptr_t scaled_uimm = offset >> lgsize;
3d4299f4
RH		 913 if (scaled_uimm <= 0xfff) {
914 tcg_out_insn_3313(s, insn, rd, rn, scaled_uimm);
915 return;
b1f6dc0d
CF		 916 }
917 }
918
a056c9fa
RH		 919 /* Small signed offsets can use the unscaled encoding. */
920 if (offset >= -256 && offset < 256) {
921 tcg_out_insn_3312(s, insn, rd, rn, offset);
922 return;
923 }
924
3d4299f4 925 /* Worst-case scenario, move offset to temp register, use reg offset. */
b1f6dc0d 926 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, offset);
6c0f0c0f 927 tcg_out_ldst_r(s, insn, rd, rn, TCG_TYPE_I64, TCG_REG_TMP);
4a136e0a
CF		 928}
929
14e4c1e2 930static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
4a136e0a 931{
14e4c1e2
RH		 932 if (ret == arg) {
933 return;
934 }
935 switch (type) {
936 case TCG_TYPE_I32:
937 case TCG_TYPE_I64:
938 if (ret < 32 && arg < 32) {
939 tcg_out_movr(s, type, ret, arg);
940 break;
941 } else if (ret < 32) {
942 tcg_out_insn(s, 3605, UMOV, type, ret, arg, 0, 0);
943 break;
944 } else if (arg < 32) {
945 tcg_out_insn(s, 3605, INS, 0, ret, arg, 4 << type, 0);
946 break;
947 }
948 /* FALLTHRU */
949
950 case TCG_TYPE_V64:
951 tcg_debug_assert(ret >= 32 && arg >= 32);
952 tcg_out_insn(s, 3616, ORR, 0, 0, ret, arg, arg);
953 break;
954 case TCG_TYPE_V128:
955 tcg_debug_assert(ret >= 32 && arg >= 32);
956 tcg_out_insn(s, 3616, ORR, 1, 0, ret, arg, arg);
957 break;
958
959 default:
960 g_assert_not_reached();
4a136e0a
CF		 961 }
962}
963
14e4c1e2
RH		 964static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
965 TCGReg base, intptr_t ofs)
4a136e0a 966{
14e4c1e2
RH		 967 AArch64Insn insn;
968 int lgsz;
969
970 switch (type) {
971 case TCG_TYPE_I32:
972 insn = (ret < 32 ? I3312_LDRW : I3312_LDRVS);
973 lgsz = 2;
974 break;
975 case TCG_TYPE_I64:
976 insn = (ret < 32 ? I3312_LDRX : I3312_LDRVD);
977 lgsz = 3;
978 break;
979 case TCG_TYPE_V64:
980 insn = I3312_LDRVD;
981 lgsz = 3;
982 break;
983 case TCG_TYPE_V128:
984 insn = I3312_LDRVQ;
985 lgsz = 4;
986 break;
987 default:
988 g_assert_not_reached();
989 }
990 tcg_out_ldst(s, insn, ret, base, ofs, lgsz);
4a136e0a
CF		 991}
992
14e4c1e2
RH		 993static void tcg_out_st(TCGContext *s, TCGType type, TCGReg src,
994 TCGReg base, intptr_t ofs)
4a136e0a 995{
14e4c1e2
RH		 996 AArch64Insn insn;
997 int lgsz;
998
999 switch (type) {
1000 case TCG_TYPE_I32:
1001 insn = (src < 32 ? I3312_STRW : I3312_STRVS);
1002 lgsz = 2;
1003 break;
1004 case TCG_TYPE_I64:
1005 insn = (src < 32 ? I3312_STRX : I3312_STRVD);
1006 lgsz = 3;
1007 break;
1008 case TCG_TYPE_V64:
1009 insn = I3312_STRVD;
1010 lgsz = 3;
1011 break;
1012 case TCG_TYPE_V128:
1013 insn = I3312_STRVQ;
1014 lgsz = 4;
1015 break;
1016 default:
1017 g_assert_not_reached();
1018 }
1019 tcg_out_ldst(s, insn, src, base, ofs, lgsz);
4a136e0a
CF		 1020}
1021
59d7c14e
RH		 1022static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1023 TCGReg base, intptr_t ofs)
1024{
14e4c1e2 1025 if (type <= TCG_TYPE_I64 && val == 0) {
59d7c14e
RH		 1026 tcg_out_st(s, type, TCG_REG_XZR, base, ofs);
1027 return true;
1028 }
1029 return false;
1030}
1031
b3c56df7
RH		 1032static inline void tcg_out_bfm(TCGContext *s, TCGType ext, TCGReg rd,
1033 TCGReg rn, unsigned int a, unsigned int b)
1034{
1035 tcg_out_insn(s, 3402, BFM, ext, rd, rn, ext, a, b);
1036}
1037
7763ffa0
RH		 1038static inline void tcg_out_ubfm(TCGContext *s, TCGType ext, TCGReg rd,
1039 TCGReg rn, unsigned int a, unsigned int b)
4a136e0a 1040{
b3c56df7 1041 tcg_out_insn(s, 3402, UBFM, ext, rd, rn, ext, a, b);
4a136e0a
CF		 1042}
1043
7763ffa0
RH		 1044static inline void tcg_out_sbfm(TCGContext *s, TCGType ext, TCGReg rd,
1045 TCGReg rn, unsigned int a, unsigned int b)
4a136e0a 1046{
b3c56df7 1047 tcg_out_insn(s, 3402, SBFM, ext, rd, rn, ext, a, b);
4a136e0a
CF		 1048}
1049
7763ffa0 1050static inline void tcg_out_extr(TCGContext *s, TCGType ext, TCGReg rd,
4a136e0a
CF		 1051 TCGReg rn, TCGReg rm, unsigned int a)
1052{
b3c56df7 1053 tcg_out_insn(s, 3403, EXTR, ext, rd, rn, rm, a);
4a136e0a
CF		 1054}
1055
7763ffa0 1056static inline void tcg_out_shl(TCGContext *s, TCGType ext,
4a136e0a
CF		 1057 TCGReg rd, TCGReg rn, unsigned int m)
1058{
b3c56df7
RH		 1059 int bits = ext ? 64 : 32;
1060 int max = bits - 1;
4a136e0a
CF		 1061 tcg_out_ubfm(s, ext, rd, rn, bits - (m & max), max - (m & max));
1062}
1063
7763ffa0 1064static inline void tcg_out_shr(TCGContext *s, TCGType ext,
4a136e0a
CF		 1065 TCGReg rd, TCGReg rn, unsigned int m)
1066{
1067 int max = ext ? 63 : 31;
1068 tcg_out_ubfm(s, ext, rd, rn, m & max, max);
1069}
1070
7763ffa0 1071static inline void tcg_out_sar(TCGContext *s, TCGType ext,
4a136e0a
CF		 1072 TCGReg rd, TCGReg rn, unsigned int m)
1073{
1074 int max = ext ? 63 : 31;
1075 tcg_out_sbfm(s, ext, rd, rn, m & max, max);
1076}
1077
7763ffa0 1078static inline void tcg_out_rotr(TCGContext *s, TCGType ext,
4a136e0a
CF		 1079 TCGReg rd, TCGReg rn, unsigned int m)
1080{
1081 int max = ext ? 63 : 31;
1082 tcg_out_extr(s, ext, rd, rn, rn, m & max);
1083}
1084
7763ffa0 1085static inline void tcg_out_rotl(TCGContext *s, TCGType ext,
4a136e0a
CF		 1086 TCGReg rd, TCGReg rn, unsigned int m)
1087{
b3c56df7
RH		 1088 int bits = ext ? 64 : 32;
1089 int max = bits - 1;
4a136e0a
CF		 1090 tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
1091}
1092
b3c56df7
RH		 1093static inline void tcg_out_dep(TCGContext *s, TCGType ext, TCGReg rd,
1094 TCGReg rn, unsigned lsb, unsigned width)
1095{
1096 unsigned size = ext ? 64 : 32;
1097 unsigned a = (size - lsb) & (size - 1);
1098 unsigned b = width - 1;
1099 tcg_out_bfm(s, ext, rd, rn, a, b);
1100}
1101
90f1cd91
RH		 1102static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a,
1103 tcg_target_long b, bool const_b)
4a136e0a 1104{
90f1cd91
RH		 1105 if (const_b) {
1106 /* Using CMP or CMN aliases. */
1107 if (b >= 0) {
1108 tcg_out_insn(s, 3401, SUBSI, ext, TCG_REG_XZR, a, b);
1109 } else {
1110 tcg_out_insn(s, 3401, ADDSI, ext, TCG_REG_XZR, a, -b);
1111 }
1112 } else {
1113 /* Using CMP alias SUBS wzr, Wn, Wm */
1114 tcg_out_insn(s, 3502, SUBS, ext, TCG_REG_XZR, a, b);
1115 }
4a136e0a
CF		 1116}
1117
8587c30c 1118static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
4a136e0a 1119{
8587c30c 1120 ptrdiff_t offset = target - s->code_ptr;
eabb7b91 1121 tcg_debug_assert(offset == sextract64(offset, 0, 26));
81d8a5ee 1122 tcg_out_insn(s, 3206, B, offset);
4a136e0a
CF		 1123}
1124
23b7aa1d
PK		 1125static inline void tcg_out_goto_long(TCGContext *s, tcg_insn_unit *target)
1126{
1127 ptrdiff_t offset = target - s->code_ptr;
1128 if (offset == sextract64(offset, 0, 26)) {
1129 tcg_out_insn(s, 3206, BL, offset);
1130 } else {
1131 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target);
1132 tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
1133 }
1134}
1135
4a136e0a
CF		 1136static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
1137{
81d8a5ee 1138 tcg_out_insn(s, 3207, BLR, reg);
4a136e0a
CF		 1139}
1140
8587c30c 1141static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
4a136e0a 1142{
8587c30c
RH		 1143 ptrdiff_t offset = target - s->code_ptr;
1144 if (offset == sextract64(offset, 0, 26)) {
81d8a5ee 1145 tcg_out_insn(s, 3206, BL, offset);
8587c30c
RH		 1146 } else {
1147 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target);
1148 tcg_out_callr(s, TCG_REG_TMP);
4a136e0a
CF		 1149 }
1150}
1151
a8583393
RH		 1152void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
1153 uintptr_t addr)
4a136e0a 1154{
b68686bd
PK		 1155 tcg_insn_unit i1, i2;
1156 TCGType rt = TCG_TYPE_I64;
1157 TCGReg rd = TCG_REG_TMP;
1158 uint64_t pair;
4a136e0a 1159
b68686bd
PK		 1160 ptrdiff_t offset = addr - jmp_addr;
1161
1162 if (offset == sextract64(offset, 0, 26)) {
1163 i1 = I3206_B | ((offset >> 2) & 0x3ffffff);
1164 i2 = NOP;
1165 } else {
1166 offset = (addr >> 12) - (jmp_addr >> 12);
1167
1168 /* patch ADRP */
1169 i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd;
1170 /* patch ADDI */
1171 i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd;
1172 }
1173 pair = (uint64_t)i2 << 32 | i1;
1174 atomic_set((uint64_t *)jmp_addr, pair);
1175 flush_icache_range(jmp_addr, jmp_addr + 8);
4a136e0a
CF		 1176}
1177
bec16311 1178static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l)
4a136e0a 1179{
4a136e0a 1180 if (!l->has_value) {
bec16311 1181 tcg_out_reloc(s, s->code_ptr, R_AARCH64_JUMP26, l, 0);
733589b3 1182 tcg_out_insn(s, 3206, B, 0);
4a136e0a 1183 } else {
8587c30c 1184 tcg_out_goto(s, l->u.value_ptr);
4a136e0a
CF		 1185 }
1186}
1187
dc1eccd6 1188static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a,
bec16311 1189 TCGArg b, bool b_const, TCGLabel *l)
4a136e0a 1190{
cae1f6f3 1191 intptr_t offset;
3d9e69a2 1192 bool need_cmp;
cae1f6f3 1193
3d9e69a2
RH		 1194 if (b_const && b == 0 && (c == TCG_COND_EQ || c == TCG_COND_NE)) {
1195 need_cmp = false;
1196 } else {
1197 need_cmp = true;
1198 tcg_out_cmp(s, ext, a, b, b_const);
1199 }
4a136e0a
CF		 1200
1201 if (!l->has_value) {
bec16311 1202 tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0);
cae1f6f3 1203 offset = tcg_in32(s) >> 5;
4a136e0a 1204 } else {
8587c30c 1205 offset = l->u.value_ptr - s->code_ptr;
eabb7b91 1206 tcg_debug_assert(offset == sextract64(offset, 0, 19));
4a136e0a 1207 }
cae1f6f3 1208
3d9e69a2
RH		 1209 if (need_cmp) {
1210 tcg_out_insn(s, 3202, B_C, c, offset);
1211 } else if (c == TCG_COND_EQ) {
1212 tcg_out_insn(s, 3201, CBZ, ext, a, offset);
1213 } else {
1214 tcg_out_insn(s, 3201, CBNZ, ext, a, offset);
1215 }
4a136e0a
CF		 1216}
1217
edd8824c 1218static inline void tcg_out_rev64(TCGContext *s, TCGReg rd, TCGReg rn)
9c4a059d 1219{
edd8824c 1220 tcg_out_insn(s, 3507, REV64, TCG_TYPE_I64, rd, rn);
9c4a059d
CF		 1221}
1222
edd8824c 1223static inline void tcg_out_rev32(TCGContext *s, TCGReg rd, TCGReg rn)
9c4a059d 1224{
edd8824c
RH		 1225 tcg_out_insn(s, 3507, REV32, TCG_TYPE_I32, rd, rn);
1226}
1227
1228static inline void tcg_out_rev16(TCGContext *s, TCGReg rd, TCGReg rn)
1229{
1230 tcg_out_insn(s, 3507, REV16, TCG_TYPE_I32, rd, rn);
9c4a059d
CF		 1231}
1232
929f8b55 1233static inline void tcg_out_sxt(TCGContext *s, TCGType ext, TCGMemOp s_bits,
31f1275b
CF		 1234 TCGReg rd, TCGReg rn)
1235{
b3c56df7 1236 /* Using ALIASes SXTB, SXTH, SXTW, of SBFM Xd, Xn, #0, #7|15|31 */
929f8b55 1237 int bits = (8 << s_bits) - 1;
31f1275b
CF		 1238 tcg_out_sbfm(s, ext, rd, rn, 0, bits);
1239}
1240
929f8b55 1241static inline void tcg_out_uxt(TCGContext *s, TCGMemOp s_bits,
31f1275b
CF		 1242 TCGReg rd, TCGReg rn)
1243{
b3c56df7 1244 /* Using ALIASes UXTB, UXTH of UBFM Wd, Wn, #0, #7|15 */
929f8b55 1245 int bits = (8 << s_bits) - 1;
31f1275b
CF		 1246 tcg_out_ubfm(s, 0, rd, rn, 0, bits);
1247}
1248
90f1cd91
RH		 1249static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
1250 TCGReg rn, int64_t aimm)
1251{
1252 if (aimm >= 0) {
1253 tcg_out_insn(s, 3401, ADDI, ext, rd, rn, aimm);
1254 } else {
1255 tcg_out_insn(s, 3401, SUBI, ext, rd, rn, -aimm);
1256 }
1257}
1258
dc1eccd6 1259static inline void tcg_out_addsub2(TCGContext *s, TCGType ext, TCGReg rl,
c6e929e7
RH		 1260 TCGReg rh, TCGReg al, TCGReg ah,
1261 tcg_target_long bl, tcg_target_long bh,
1262 bool const_bl, bool const_bh, bool sub)
1263{
1264 TCGReg orig_rl = rl;
1265 AArch64Insn insn;
1266
1267 if (rl == ah || (!const_bh && rl == bh)) {
1268 rl = TCG_REG_TMP;
1269 }
1270
1271 if (const_bl) {
1272 insn = I3401_ADDSI;
1273 if ((bl < 0) ^ sub) {
1274 insn = I3401_SUBSI;
1275 bl = -bl;
1276 }
b1eb20da
RH		 1277 if (unlikely(al == TCG_REG_XZR)) {
1278 /* ??? We want to allow al to be zero for the benefit of
1279 negation via subtraction. However, that leaves open the
1280 possibility of adding 0+const in the low part, and the
1281 immediate add instructions encode XSP not XZR. Don't try
1282 anything more elaborate here than loading another zero. */
1283 al = TCG_REG_TMP;
1284 tcg_out_movi(s, ext, al, 0);
1285 }
c6e929e7
RH		 1286 tcg_out_insn_3401(s, insn, ext, rl, al, bl);
1287 } else {
1288 tcg_out_insn_3502(s, sub ? I3502_SUBS : I3502_ADDS, ext, rl, al, bl);
1289 }
1290
1291 insn = I3503_ADC;
1292 if (const_bh) {
1293 /* Note that the only two constants we support are 0 and -1, and
1294 that SBC = rn + ~rm + c, so adc -1 is sbc 0, and vice-versa. */
1295 if ((bh != 0) ^ sub) {
1296 insn = I3503_SBC;
1297 }
1298 bh = TCG_REG_XZR;
1299 } else if (sub) {
1300 insn = I3503_SBC;
1301 }
1302 tcg_out_insn_3503(s, insn, ext, rh, ah, bh);
1303
b825025f 1304 tcg_out_mov(s, ext, orig_rl, rl);
c6e929e7
RH		 1305}
1306
c7a59c2a
PK		 1307static inline void tcg_out_mb(TCGContext *s, TCGArg a0)
1308{
1309 static const uint32_t sync[] = {
1310 [0 ... TCG_MO_ALL] = DMB_ISH | DMB_LD | DMB_ST,
1311 [TCG_MO_ST_ST] = DMB_ISH | DMB_ST,
1312 [TCG_MO_LD_LD] = DMB_ISH | DMB_LD,
1313 [TCG_MO_LD_ST] = DMB_ISH | DMB_LD,
1314 [TCG_MO_LD_ST | TCG_MO_LD_LD] = DMB_ISH | DMB_LD,
1315 };
1316 tcg_out32(s, sync[a0 & TCG_MO_ALL]);
1317}
1318
53c76c19
RH		 1319static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d,
1320 TCGReg a0, TCGArg b, bool const_b, bool is_ctz)
1321{
1322 TCGReg a1 = a0;
1323 if (is_ctz) {
1324 a1 = TCG_REG_TMP;
1325 tcg_out_insn(s, 3507, RBIT, ext, a1, a0);
1326 }
1327 if (const_b && b == (ext ? 64 : 32)) {
1328 tcg_out_insn(s, 3507, CLZ, ext, d, a1);
1329 } else {
1330 AArch64Insn sel = I3506_CSEL;
1331
1332 tcg_out_cmp(s, ext, a0, 0, 1);
1333 tcg_out_insn(s, 3507, CLZ, ext, TCG_REG_TMP, a1);
1334
1335 if (const_b) {
1336 if (b == -1) {
1337 b = TCG_REG_XZR;
1338 sel = I3506_CSINV;
1339 } else if (b == 0) {
1340 b = TCG_REG_XZR;
1341 } else {
1342 tcg_out_movi(s, ext, d, b);
1343 b = d;
1344 }
1345 }
1346 tcg_out_insn_3506(s, sel, ext, d, TCG_REG_TMP, b, TCG_COND_NE);
1347 }
1348}
1349
4a136e0a 1350#ifdef CONFIG_SOFTMMU
659ef5cb
RH		 1351#include "tcg-ldst.inc.c"
1352
023261ef 1353/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
3972ef6f 1354 * TCGMemOpIdx oi, uintptr_t ra)
023261ef 1355 */
8587c30c 1356static void * const qemu_ld_helpers[16] = {
de61d14f
RH		 1357 [MO_UB] = helper_ret_ldub_mmu,
1358 [MO_LEUW] = helper_le_lduw_mmu,
1359 [MO_LEUL] = helper_le_ldul_mmu,
1360 [MO_LEQ] = helper_le_ldq_mmu,
1361 [MO_BEUW] = helper_be_lduw_mmu,
1362 [MO_BEUL] = helper_be_ldul_mmu,
1363 [MO_BEQ] = helper_be_ldq_mmu,
4a136e0a
CF		 1364};
1365
023261ef 1366/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
3972ef6f
RH		 1367 * uintxx_t val, TCGMemOpIdx oi,
1368 * uintptr_t ra)
023261ef 1369 */
8587c30c 1370static void * const qemu_st_helpers[16] = {
de61d14f
RH		 1371 [MO_UB] = helper_ret_stb_mmu,
1372 [MO_LEUW] = helper_le_stw_mmu,
1373 [MO_LEUL] = helper_le_stl_mmu,
1374 [MO_LEQ] = helper_le_stq_mmu,
1375 [MO_BEUW] = helper_be_stw_mmu,
1376 [MO_BEUL] = helper_be_stl_mmu,
1377 [MO_BEQ] = helper_be_stq_mmu,
4a136e0a
CF		 1378};
1379
8587c30c 1380static inline void tcg_out_adr(TCGContext *s, TCGReg rd, void *target)
dc0c8aaf 1381{
8587c30c 1382 ptrdiff_t offset = tcg_pcrel_diff(s, target);
eabb7b91 1383 tcg_debug_assert(offset == sextract64(offset, 0, 21));
8587c30c 1384 tcg_out_insn(s, 3406, ADR, rd, offset);
dc0c8aaf
RH		 1385}
1386
c6d8ed24
JK		 1387static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1388{
3972ef6f
RH		 1389 TCGMemOpIdx oi = lb->oi;
1390 TCGMemOp opc = get_memop(oi);
929f8b55
RH		 1391 TCGMemOp size = opc & MO_SIZE;
1392
214bfe83
RH		 1393 bool ok = reloc_pc19(lb->label_ptr[0], s->code_ptr);
1394 tcg_debug_assert(ok);
017a86f7 1395
3972ef6f 1396 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
b825025f 1397 tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
3972ef6f 1398 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, oi);
8587c30c 1399 tcg_out_adr(s, TCG_REG_X3, lb->raddr);
2b7ec66f 1400 tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
929f8b55 1401 if (opc & MO_SIGN) {
9c53889b 1402 tcg_out_sxt(s, lb->type, size, lb->datalo_reg, TCG_REG_X0);
c6d8ed24 1403 } else {
b825025f 1404 tcg_out_mov(s, size == MO_64, lb->datalo_reg, TCG_REG_X0);
c6d8ed24
JK		 1405 }
1406
8587c30c 1407 tcg_out_goto(s, lb->raddr);
c6d8ed24
JK		 1408}
1409
1410static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1411{
3972ef6f
RH		 1412 TCGMemOpIdx oi = lb->oi;
1413 TCGMemOp opc = get_memop(oi);
de61d14f 1414 TCGMemOp size = opc & MO_SIZE;
929f8b55 1415
214bfe83
RH		 1416 bool ok = reloc_pc19(lb->label_ptr[0], s->code_ptr);
1417 tcg_debug_assert(ok);
c6d8ed24 1418
3972ef6f 1419 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
b825025f
RH		 1420 tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
1421 tcg_out_mov(s, size == MO_64, TCG_REG_X2, lb->datalo_reg);
3972ef6f 1422 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, oi);
8587c30c 1423 tcg_out_adr(s, TCG_REG_X4, lb->raddr);
2b7ec66f 1424 tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
8587c30c 1425 tcg_out_goto(s, lb->raddr);
c6d8ed24
JK		 1426}
1427
3972ef6f 1428static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
9c53889b 1429 TCGType ext, TCGReg data_reg, TCGReg addr_reg,
3972ef6f 1430 tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
c6d8ed24 1431{
9ecefc84 1432 TCGLabelQemuLdst *label = new_ldst_label(s);
c6d8ed24 1433
c6d8ed24 1434 label->is_ld = is_ld;
3972ef6f 1435 label->oi = oi;
9c53889b 1436 label->type = ext;
c6d8ed24
JK		 1437 label->datalo_reg = data_reg;
1438 label->addrlo_reg = addr_reg;
c6d8ed24
JK		 1439 label->raddr = raddr;
1440 label->label_ptr[0] = label_ptr;
1441}
1442
1443/* Load and compare a TLB entry, emitting the conditional jump to the
1444 slow path for the failure case, which will be patched later when finalizing
1445 the slow path. Generated code returns the host addend in X1,
1446 clobbers X0,X2,X3,TMP. */
9ee14902 1447static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp opc,
8587c30c
RH		 1448 tcg_insn_unit **label_ptr, int mem_index,
1449 bool is_read)
c6d8ed24 1450{
c6d8ed24
JK		 1451 int tlb_offset = is_read ?
1452 offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
1453 : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
85aa8081
RH		 1454 unsigned a_bits = get_alignment_bits(opc);
1455 unsigned s_bits = opc & MO_SIZE;
1456 unsigned a_mask = (1u << a_bits) - 1;
1457 unsigned s_mask = (1u << s_bits) - 1;
9ee14902
RH		 1458 TCGReg base = TCG_AREG0, x3;
1459 uint64_t tlb_mask;
1460
1461 /* For aligned accesses, we check the first byte and include the alignment
1462 bits within the address. For unaligned access, we check that we don't
1463 cross pages using the address of the last byte of the access. */
85aa8081 1464 if (a_bits >= s_bits) {
9ee14902
RH		 1465 x3 = addr_reg;
1466 } else {
1467 tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64,
85aa8081 1468 TCG_REG_X3, addr_reg, s_mask - a_mask);
9ee14902
RH		 1469 x3 = TCG_REG_X3;
1470 }
85aa8081 1471 tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
6f472467 1472
c6d8ed24
JK		 1473 /* Extract the TLB index from the address into X0.
1474 X0<CPU_TLB_BITS:0> =
1475 addr_reg<TARGET_PAGE_BITS+CPU_TLB_BITS:TARGET_PAGE_BITS> */
6f472467 1476 tcg_out_ubfm(s, TARGET_LONG_BITS == 64, TCG_REG_X0, addr_reg,
c6d8ed24 1477 TARGET_PAGE_BITS, TARGET_PAGE_BITS + CPU_TLB_BITS);
6f472467 1478
9ee14902
RH		 1479 /* Store the page mask part of the address into X3. */
1480 tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64,
1481 TCG_REG_X3, x3, tlb_mask);
6f472467 1482
c6d8ed24 1483 /* Add any "high bits" from the tlb offset to the env address into X2,
096c46c0 1484 to take advantage of the LSL12 form of the ADDI instruction.
c6d8ed24 1485 X2 = env + (tlb_offset & 0xfff000) */
6f472467
RH		 1486 if (tlb_offset & 0xfff000) {
1487 tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_X2, base,
1488 tlb_offset & 0xfff000);
1489 base = TCG_REG_X2;
1490 }
1491
c6d8ed24
JK		 1492 /* Merge the tlb index contribution into X2.
1493 X2 = X2 + (X0 << CPU_TLB_ENTRY_BITS) */
6f472467 1494 tcg_out_insn(s, 3502S, ADD_LSL, TCG_TYPE_I64, TCG_REG_X2, base,
50573c66 1495 TCG_REG_X0, CPU_TLB_ENTRY_BITS);
6f472467 1496
c6d8ed24
JK		 1497 /* Merge "low bits" from tlb offset, load the tlb comparator into X0.
1498 X0 = load [X2 + (tlb_offset & 0x000fff)] */
3d4299f4 1499 tcg_out_ldst(s, TARGET_LONG_BITS == 32 ? I3312_LDRW : I3312_LDRX,
14e4c1e2
RH		 1500 TCG_REG_X0, TCG_REG_X2, tlb_offset & 0xfff,
1501 TARGET_LONG_BITS == 32 ? 2 : 3);
6f472467 1502
c6d8ed24
JK		 1503 /* Load the tlb addend. Do that early to avoid stalling.
1504 X1 = load [X2 + (tlb_offset & 0xfff) + offsetof(addend)] */
3d4299f4 1505 tcg_out_ldst(s, I3312_LDRX, TCG_REG_X1, TCG_REG_X2,
c6d8ed24
JK		 1506 (tlb_offset & 0xfff) + (offsetof(CPUTLBEntry, addend)) -
1507 (is_read ? offsetof(CPUTLBEntry, addr_read)
14e4c1e2 1508 : offsetof(CPUTLBEntry, addr_write)), 3);
6f472467 1509
c6d8ed24 1510 /* Perform the address comparison. */
90f1cd91 1511 tcg_out_cmp(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, TCG_REG_X3, 0);
6f472467 1512
c6d8ed24 1513 /* If not equal, we jump to the slow path. */
6f472467 1514 *label_ptr = s->code_ptr;
733589b3 1515 tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
c6d8ed24
JK		 1516}
1517
1518#endif /* CONFIG_SOFTMMU */
6a91c7c9 1519
9c53889b 1520static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp memop, TCGType ext,
ffc63728
PB		 1521 TCGReg data_r, TCGReg addr_r,
1522 TCGType otype, TCGReg off_r)
6a91c7c9 1523{
9e4177ad
RH		 1524 const TCGMemOp bswap = memop & MO_BSWAP;
1525
1526 switch (memop & MO_SSIZE) {
1527 case MO_UB:
6c0f0c0f 1528 tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, otype, off_r);
6a91c7c9 1529 break;
9e4177ad 1530 case MO_SB:
9c53889b 1531 tcg_out_ldst_r(s, ext ? I3312_LDRSBX : I3312_LDRSBW,
6c0f0c0f 1532 data_r, addr_r, otype, off_r);
6a91c7c9 1533 break;
9e4177ad 1534 case MO_UW:
6c0f0c0f 1535 tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r);
9e4177ad 1536 if (bswap) {
edd8824c 1537 tcg_out_rev16(s, data_r, data_r);
6a91c7c9
JK		 1538 }
1539 break;
9e4177ad
RH		 1540 case MO_SW:
1541 if (bswap) {
6c0f0c0f 1542 tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r);
edd8824c 1543 tcg_out_rev16(s, data_r, data_r);
9c53889b 1544 tcg_out_sxt(s, ext, MO_16, data_r, data_r);
6a91c7c9 1545 } else {
6c0f0c0f
PB		 1546 tcg_out_ldst_r(s, (ext ? I3312_LDRSHX : I3312_LDRSHW),
1547 data_r, addr_r, otype, off_r);
6a91c7c9
JK		 1548 }
1549 break;
9e4177ad 1550 case MO_UL:
6c0f0c0f 1551 tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r);
9e4177ad 1552 if (bswap) {
edd8824c 1553 tcg_out_rev32(s, data_r, data_r);
6a91c7c9
JK		 1554 }
1555 break;
9e4177ad
RH		 1556 case MO_SL:
1557 if (bswap) {
6c0f0c0f 1558 tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r);
edd8824c 1559 tcg_out_rev32(s, data_r, data_r);
929f8b55 1560 tcg_out_sxt(s, TCG_TYPE_I64, MO_32, data_r, data_r);
6a91c7c9 1561 } else {
6c0f0c0f 1562 tcg_out_ldst_r(s, I3312_LDRSWX, data_r, addr_r, otype, off_r);
6a91c7c9
JK		 1563 }
1564 break;
9e4177ad 1565 case MO_Q:
6c0f0c0f 1566 tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, otype, off_r);
9e4177ad 1567 if (bswap) {
edd8824c 1568 tcg_out_rev64(s, data_r, data_r);
6a91c7c9
JK		 1569 }
1570 break;
1571 default:
1572 tcg_abort();
1573 }
1574}
1575
9e4177ad 1576static void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp memop,
ffc63728
PB		 1577 TCGReg data_r, TCGReg addr_r,
1578 TCGType otype, TCGReg off_r)
6a91c7c9 1579{
9e4177ad
RH		 1580 const TCGMemOp bswap = memop & MO_BSWAP;
1581
1582 switch (memop & MO_SIZE) {
1583 case MO_8:
6c0f0c0f 1584 tcg_out_ldst_r(s, I3312_STRB, data_r, addr_r, otype, off_r);
6a91c7c9 1585 break;
9e4177ad 1586 case MO_16:
e81864a1 1587 if (bswap && data_r != TCG_REG_XZR) {
edd8824c 1588 tcg_out_rev16(s, TCG_REG_TMP, data_r);
9e4177ad 1589 data_r = TCG_REG_TMP;
6a91c7c9 1590 }
6c0f0c0f 1591 tcg_out_ldst_r(s, I3312_STRH, data_r, addr_r, otype, off_r);
6a91c7c9 1592 break;
9e4177ad 1593 case MO_32:
e81864a1 1594 if (bswap && data_r != TCG_REG_XZR) {
edd8824c 1595 tcg_out_rev32(s, TCG_REG_TMP, data_r);
9e4177ad 1596 data_r = TCG_REG_TMP;
6a91c7c9 1597 }
6c0f0c0f 1598 tcg_out_ldst_r(s, I3312_STRW, data_r, addr_r, otype, off_r);
6a91c7c9 1599 break;
9e4177ad 1600 case MO_64:
e81864a1 1601 if (bswap && data_r != TCG_REG_XZR) {
edd8824c 1602 tcg_out_rev64(s, TCG_REG_TMP, data_r);
9e4177ad 1603 data_r = TCG_REG_TMP;
6a91c7c9 1604 }
6c0f0c0f 1605 tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, otype, off_r);
6a91c7c9
JK		 1606 break;
1607 default:
1608 tcg_abort();
1609 }
1610}
4a136e0a 1611
667b1cdd 1612static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
59227d5d 1613 TCGMemOpIdx oi, TCGType ext)
4a136e0a 1614{
59227d5d 1615 TCGMemOp memop = get_memop(oi);
80adb8fc 1616 const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
4a136e0a 1617#ifdef CONFIG_SOFTMMU
59227d5d 1618 unsigned mem_index = get_mmuidx(oi);
8587c30c 1619 tcg_insn_unit *label_ptr;
4a136e0a 1620
9ee14902 1621 tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 1);
80adb8fc
RH		 1622 tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
1623 TCG_REG_X1, otype, addr_reg);
3972ef6f
RH		 1624 add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg,
1625 s->code_ptr, label_ptr);
4a136e0a 1626#else /* !CONFIG_SOFTMMU */
352bcb0a
RH		 1627 if (USE_GUEST_BASE) {
1628 tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
1629 TCG_REG_GUEST_BASE, otype, addr_reg);
1630 } else {
1631 tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
1632 addr_reg, TCG_TYPE_I64, TCG_REG_XZR);
1633 }
6a91c7c9 1634#endif /* CONFIG_SOFTMMU */
4a136e0a
CF		 1635}
1636
667b1cdd 1637static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
59227d5d 1638 TCGMemOpIdx oi)
4a136e0a 1639{
59227d5d 1640 TCGMemOp memop = get_memop(oi);
80adb8fc 1641 const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
4a136e0a 1642#ifdef CONFIG_SOFTMMU
59227d5d 1643 unsigned mem_index = get_mmuidx(oi);
8587c30c 1644 tcg_insn_unit *label_ptr;
4a136e0a 1645
9ee14902 1646 tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 0);
80adb8fc
RH		 1647 tcg_out_qemu_st_direct(s, memop, data_reg,
1648 TCG_REG_X1, otype, addr_reg);
9ee14902
RH		 1649 add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64,
1650 data_reg, addr_reg, s->code_ptr, label_ptr);
4a136e0a 1651#else /* !CONFIG_SOFTMMU */
352bcb0a
RH		 1652 if (USE_GUEST_BASE) {
1653 tcg_out_qemu_st_direct(s, memop, data_reg,
1654 TCG_REG_GUEST_BASE, otype, addr_reg);
1655 } else {
1656 tcg_out_qemu_st_direct(s, memop, data_reg,
1657 addr_reg, TCG_TYPE_I64, TCG_REG_XZR);
1658 }
6a91c7c9 1659#endif /* CONFIG_SOFTMMU */
4a136e0a
CF		 1660}
1661
8587c30c 1662static tcg_insn_unit *tb_ret_addr;
4a136e0a 1663
4a136e0a 1664static void tcg_out_op(TCGContext *s, TCGOpcode opc,
8d8db193
RH		 1665 const TCGArg args[TCG_MAX_OP_ARGS],
1666 const int const_args[TCG_MAX_OP_ARGS])
4a136e0a 1667{
f0293414
RH		 1668 /* 99% of the time, we can signal the use of extension registers
1669 by looking to see if the opcode handles 64-bit data. */
1670 TCGType ext = (tcg_op_defs[opc].flags & TCG_OPF_64BIT) != 0;
4a136e0a 1671
8d8db193
RH		 1672 /* Hoist the loads of the most common arguments. */
1673 TCGArg a0 = args[0];
1674 TCGArg a1 = args[1];
1675 TCGArg a2 = args[2];
1676 int c2 = const_args[2];
1677
04ce397b
RH		 1678 /* Some operands are defined with "rZ" constraint, a register or
1679 the zero register. These need not actually test args[I] == 0. */
1680#define REG0(I) (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
1681
4a136e0a
CF		 1682 switch (opc) {
1683 case INDEX_op_exit_tb:
b19f0c2e
RH		 1684 /* Reuse the zeroing that exists for goto_ptr. */
1685 if (a0 == 0) {
23b7aa1d 1686 tcg_out_goto_long(s, s->code_gen_epilogue);
b19f0c2e
RH		 1687 } else {
1688 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
23b7aa1d 1689 tcg_out_goto_long(s, tb_ret_addr);
b19f0c2e 1690 }
4a136e0a
CF		 1691 break;
1692
1693 case INDEX_op_goto_tb:
2acee8b2 1694 if (s->tb_jmp_insn_offset != NULL) {
a8583393 1695 /* TCG_TARGET_HAS_direct_jump */
2acee8b2
PK		 1696 /* Ensure that ADRP+ADD are 8-byte aligned so that an atomic
1697 write can be used to patch the target address. */
1698 if ((uintptr_t)s->code_ptr & 7) {
1699 tcg_out32(s, NOP);
1700 }
1701 s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
1702 /* actual branch destination will be patched by
a8583393 1703 tb_target_set_jmp_target later. */
2acee8b2
PK		 1704 tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0);
1705 tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0);
1706 } else {
a8583393 1707 /* !TCG_TARGET_HAS_direct_jump */
2acee8b2
PK		 1708 tcg_debug_assert(s->tb_jmp_target_addr != NULL);
1709 intptr_t offset = tcg_pcrel_diff(s, (s->tb_jmp_target_addr + a0)) >> 2;
1710 tcg_out_insn(s, 3305, LDR, offset, TCG_REG_TMP);
b68686bd 1711 }
b68686bd 1712 tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
9f754620 1713 set_jmp_reset_offset(s, a0);
4a136e0a
CF		 1714 break;
1715
b19f0c2e
RH		 1716 case INDEX_op_goto_ptr:
1717 tcg_out_insn(s, 3207, BR, a0);
1718 break;
1719
4a136e0a 1720 case INDEX_op_br:
bec16311 1721 tcg_out_goto_label(s, arg_label(a0));
4a136e0a
CF		 1722 break;
1723
4a136e0a 1724 case INDEX_op_ld8u_i32:
4a136e0a 1725 case INDEX_op_ld8u_i64:
14e4c1e2 1726 tcg_out_ldst(s, I3312_LDRB, a0, a1, a2, 0);
dc73dfd4
RH		 1727 break;
1728 case INDEX_op_ld8s_i32:
14e4c1e2 1729 tcg_out_ldst(s, I3312_LDRSBW, a0, a1, a2, 0);
dc73dfd4 1730 break;
4a136e0a 1731 case INDEX_op_ld8s_i64:
14e4c1e2 1732 tcg_out_ldst(s, I3312_LDRSBX, a0, a1, a2, 0);
dc73dfd4
RH		 1733 break;
1734 case INDEX_op_ld16u_i32:
4a136e0a 1735 case INDEX_op_ld16u_i64:
14e4c1e2 1736 tcg_out_ldst(s, I3312_LDRH, a0, a1, a2, 1);
dc73dfd4
RH		 1737 break;
1738 case INDEX_op_ld16s_i32:
14e4c1e2 1739 tcg_out_ldst(s, I3312_LDRSHW, a0, a1, a2, 1);
dc73dfd4 1740 break;
4a136e0a 1741 case INDEX_op_ld16s_i64:
14e4c1e2 1742 tcg_out_ldst(s, I3312_LDRSHX, a0, a1, a2, 1);
dc73dfd4
RH		 1743 break;
1744 case INDEX_op_ld_i32:
4a136e0a 1745 case INDEX_op_ld32u_i64:
14e4c1e2 1746 tcg_out_ldst(s, I3312_LDRW, a0, a1, a2, 2);
dc73dfd4 1747 break;
4a136e0a 1748 case INDEX_op_ld32s_i64:
14e4c1e2 1749 tcg_out_ldst(s, I3312_LDRSWX, a0, a1, a2, 2);
e81864a1 1750 break;
dc73dfd4 1751 case INDEX_op_ld_i64:
14e4c1e2 1752 tcg_out_ldst(s, I3312_LDRX, a0, a1, a2, 3);
dc73dfd4
RH		 1753 break;
1754
4a136e0a
CF		 1755 case INDEX_op_st8_i32:
1756 case INDEX_op_st8_i64:
14e4c1e2 1757 tcg_out_ldst(s, I3312_STRB, REG0(0), a1, a2, 0);
dc73dfd4 1758 break;
4a136e0a
CF		 1759 case INDEX_op_st16_i32:
1760 case INDEX_op_st16_i64:
14e4c1e2 1761 tcg_out_ldst(s, I3312_STRH, REG0(0), a1, a2, 1);
dc73dfd4
RH		 1762 break;
1763 case INDEX_op_st_i32:
4a136e0a 1764 case INDEX_op_st32_i64:
14e4c1e2 1765 tcg_out_ldst(s, I3312_STRW, REG0(0), a1, a2, 2);
dc73dfd4
RH		 1766 break;
1767 case INDEX_op_st_i64:
14e4c1e2 1768 tcg_out_ldst(s, I3312_STRX, REG0(0), a1, a2, 3);
4a136e0a
CF		 1769 break;
1770
4a136e0a 1771 case INDEX_op_add_i32:
90f1cd91
RH		 1772 a2 = (int32_t)a2;
1773 /* FALLTHRU */
1774 case INDEX_op_add_i64:
1775 if (c2) {
1776 tcg_out_addsubi(s, ext, a0, a1, a2);
1777 } else {
1778 tcg_out_insn(s, 3502, ADD, ext, a0, a1, a2);
1779 }
4a136e0a
CF		 1780 break;
1781
4a136e0a 1782 case INDEX_op_sub_i32:
90f1cd91
RH		 1783 a2 = (int32_t)a2;
1784 /* FALLTHRU */
1785 case INDEX_op_sub_i64:
1786 if (c2) {
1787 tcg_out_addsubi(s, ext, a0, a1, -a2);
1788 } else {
1789 tcg_out_insn(s, 3502, SUB, ext, a0, a1, a2);
1790 }
4a136e0a
CF		 1791 break;
1792
14b155dd
RH		 1793 case INDEX_op_neg_i64:
1794 case INDEX_op_neg_i32:
1795 tcg_out_insn(s, 3502, SUB, ext, a0, TCG_REG_XZR, a1);
1796 break;
1797
4a136e0a 1798 case INDEX_op_and_i32:
e029f293
RH		 1799 a2 = (int32_t)a2;
1800 /* FALLTHRU */
1801 case INDEX_op_and_i64:
1802 if (c2) {
1803 tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, a2);
1804 } else {
1805 tcg_out_insn(s, 3510, AND, ext, a0, a1, a2);
1806 }
4a136e0a
CF		 1807 break;
1808
14b155dd
RH		 1809 case INDEX_op_andc_i32:
1810 a2 = (int32_t)a2;
1811 /* FALLTHRU */
1812 case INDEX_op_andc_i64:
1813 if (c2) {
1814 tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, ~a2);
1815 } else {
1816 tcg_out_insn(s, 3510, BIC, ext, a0, a1, a2);
1817 }
1818 break;
1819
4a136e0a 1820 case INDEX_op_or_i32:
e029f293
RH		 1821 a2 = (int32_t)a2;
1822 /* FALLTHRU */
1823 case INDEX_op_or_i64:
1824 if (c2) {
1825 tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, a2);
1826 } else {
1827 tcg_out_insn(s, 3510, ORR, ext, a0, a1, a2);
1828 }
4a136e0a
CF		 1829 break;
1830
14b155dd
RH		 1831 case INDEX_op_orc_i32:
1832 a2 = (int32_t)a2;
1833 /* FALLTHRU */
1834 case INDEX_op_orc_i64:
1835 if (c2) {
1836 tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, ~a2);
1837 } else {
1838 tcg_out_insn(s, 3510, ORN, ext, a0, a1, a2);
1839 }
1840 break;
1841
4a136e0a 1842 case INDEX_op_xor_i32:
e029f293
RH		 1843 a2 = (int32_t)a2;
1844 /* FALLTHRU */
1845 case INDEX_op_xor_i64:
1846 if (c2) {
1847 tcg_out_logicali(s, I3404_EORI, ext, a0, a1, a2);
1848 } else {
1849 tcg_out_insn(s, 3510, EOR, ext, a0, a1, a2);
1850 }
4a136e0a
CF		 1851 break;
1852
14b155dd
RH		 1853 case INDEX_op_eqv_i32:
1854 a2 = (int32_t)a2;
1855 /* FALLTHRU */
1856 case INDEX_op_eqv_i64:
1857 if (c2) {
1858 tcg_out_logicali(s, I3404_EORI, ext, a0, a1, ~a2);
1859 } else {
1860 tcg_out_insn(s, 3510, EON, ext, a0, a1, a2);
1861 }
1862 break;
1863
1864 case INDEX_op_not_i64:
1865 case INDEX_op_not_i32:
1866 tcg_out_insn(s, 3510, ORN, ext, a0, TCG_REG_XZR, a1);
1867 break;
1868
4a136e0a 1869 case INDEX_op_mul_i64:
4a136e0a 1870 case INDEX_op_mul_i32:
8678b71c
RH		 1871 tcg_out_insn(s, 3509, MADD, ext, a0, a1, a2, TCG_REG_XZR);
1872 break;
1873
1874 case INDEX_op_div_i64:
1875 case INDEX_op_div_i32:
1876 tcg_out_insn(s, 3508, SDIV, ext, a0, a1, a2);
1877 break;
1878 case INDEX_op_divu_i64:
1879 case INDEX_op_divu_i32:
1880 tcg_out_insn(s, 3508, UDIV, ext, a0, a1, a2);
1881 break;
1882
1883 case INDEX_op_rem_i64:
1884 case INDEX_op_rem_i32:
1885 tcg_out_insn(s, 3508, SDIV, ext, TCG_REG_TMP, a1, a2);
1886 tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
1887 break;
1888 case INDEX_op_remu_i64:
1889 case INDEX_op_remu_i32:
1890 tcg_out_insn(s, 3508, UDIV, ext, TCG_REG_TMP, a1, a2);
1891 tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
4a136e0a
CF		 1892 break;
1893
1894 case INDEX_op_shl_i64:
4a136e0a 1895 case INDEX_op_shl_i32:
df9351e3 1896 if (c2) {
8d8db193 1897 tcg_out_shl(s, ext, a0, a1, a2);
df9351e3
RH		 1898 } else {
1899 tcg_out_insn(s, 3508, LSLV, ext, a0, a1, a2);
4a136e0a
CF		 1900 }
1901 break;
1902
1903 case INDEX_op_shr_i64:
4a136e0a 1904 case INDEX_op_shr_i32:
df9351e3 1905 if (c2) {
8d8db193 1906 tcg_out_shr(s, ext, a0, a1, a2);
df9351e3
RH		 1907 } else {
1908 tcg_out_insn(s, 3508, LSRV, ext, a0, a1, a2);
4a136e0a
CF		 1909 }
1910 break;
1911
1912 case INDEX_op_sar_i64:
4a136e0a 1913 case INDEX_op_sar_i32:
df9351e3 1914 if (c2) {
8d8db193 1915 tcg_out_sar(s, ext, a0, a1, a2);
df9351e3
RH		 1916 } else {
1917 tcg_out_insn(s, 3508, ASRV, ext, a0, a1, a2);
4a136e0a
CF		 1918 }
1919 break;
1920
1921 case INDEX_op_rotr_i64:
4a136e0a 1922 case INDEX_op_rotr_i32:
df9351e3 1923 if (c2) {
8d8db193 1924 tcg_out_rotr(s, ext, a0, a1, a2);
df9351e3
RH		 1925 } else {
1926 tcg_out_insn(s, 3508, RORV, ext, a0, a1, a2);
4a136e0a
CF		 1927 }
1928 break;
1929
1930 case INDEX_op_rotl_i64:
df9351e3
RH		 1931 case INDEX_op_rotl_i32:
1932 if (c2) {
8d8db193 1933 tcg_out_rotl(s, ext, a0, a1, a2);
4a136e0a 1934 } else {
50573c66 1935 tcg_out_insn(s, 3502, SUB, 0, TCG_REG_TMP, TCG_REG_XZR, a2);
df9351e3 1936 tcg_out_insn(s, 3508, RORV, ext, a0, a1, TCG_REG_TMP);
4a136e0a
CF		 1937 }
1938 break;
1939
53c76c19
RH		 1940 case INDEX_op_clz_i64:
1941 case INDEX_op_clz_i32:
1942 tcg_out_cltz(s, ext, a0, a1, a2, c2, false);
1943 break;
1944 case INDEX_op_ctz_i64:
1945 case INDEX_op_ctz_i32:
1946 tcg_out_cltz(s, ext, a0, a1, a2, c2, true);
1947 break;
1948
8d8db193 1949 case INDEX_op_brcond_i32:
90f1cd91
RH		 1950 a1 = (int32_t)a1;
1951 /* FALLTHRU */
1952 case INDEX_op_brcond_i64:
bec16311 1953 tcg_out_brcond(s, ext, a2, a0, a1, const_args[1], arg_label(args[3]));
4a136e0a
CF		 1954 break;
1955
4a136e0a 1956 case INDEX_op_setcond_i32:
90f1cd91
RH		 1957 a2 = (int32_t)a2;
1958 /* FALLTHRU */
1959 case INDEX_op_setcond_i64:
1960 tcg_out_cmp(s, ext, a1, a2, c2);
ed7a0aa8
RH		 1961 /* Use CSET alias of CSINC Wd, WZR, WZR, invert(cond). */
1962 tcg_out_insn(s, 3506, CSINC, TCG_TYPE_I32, a0, TCG_REG_XZR,
1963 TCG_REG_XZR, tcg_invert_cond(args[3]));
4a136e0a
CF		 1964 break;
1965
04ce397b
RH		 1966 case INDEX_op_movcond_i32:
1967 a2 = (int32_t)a2;
1968 /* FALLTHRU */
1969 case INDEX_op_movcond_i64:
1970 tcg_out_cmp(s, ext, a1, a2, c2);
1971 tcg_out_insn(s, 3506, CSEL, ext, a0, REG0(3), REG0(4), args[5]);
1972 break;
1973
de61d14f
RH		 1974 case INDEX_op_qemu_ld_i32:
1975 case INDEX_op_qemu_ld_i64:
59227d5d 1976 tcg_out_qemu_ld(s, a0, a1, a2, ext);
4a136e0a 1977 break;
de61d14f
RH		 1978 case INDEX_op_qemu_st_i32:
1979 case INDEX_op_qemu_st_i64:
59227d5d 1980 tcg_out_qemu_st(s, REG0(0), a1, a2);
4a136e0a
CF		 1981 break;
1982
f0293414 1983 case INDEX_op_bswap64_i64:
edd8824c
RH		 1984 tcg_out_rev64(s, a0, a1);
1985 break;
1986 case INDEX_op_bswap32_i64:
9c4a059d 1987 case INDEX_op_bswap32_i32:
edd8824c 1988 tcg_out_rev32(s, a0, a1);
9c4a059d
CF		 1989 break;
1990 case INDEX_op_bswap16_i64:
1991 case INDEX_op_bswap16_i32:
edd8824c 1992 tcg_out_rev16(s, a0, a1);
9c4a059d
CF		 1993 break;
1994
31f1275b 1995 case INDEX_op_ext8s_i64:
31f1275b 1996 case INDEX_op_ext8s_i32:
929f8b55 1997 tcg_out_sxt(s, ext, MO_8, a0, a1);
31f1275b
CF		 1998 break;
1999 case INDEX_op_ext16s_i64:
31f1275b 2000 case INDEX_op_ext16s_i32:
929f8b55 2001 tcg_out_sxt(s, ext, MO_16, a0, a1);
31f1275b 2002 break;
4f2331e5 2003 case INDEX_op_ext_i32_i64:
31f1275b 2004 case INDEX_op_ext32s_i64:
929f8b55 2005 tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
31f1275b
CF		 2006 break;
2007 case INDEX_op_ext8u_i64:
2008 case INDEX_op_ext8u_i32:
929f8b55 2009 tcg_out_uxt(s, MO_8, a0, a1);
31f1275b
CF		 2010 break;
2011 case INDEX_op_ext16u_i64:
2012 case INDEX_op_ext16u_i32:
929f8b55 2013 tcg_out_uxt(s, MO_16, a0, a1);
31f1275b 2014 break;
4f2331e5 2015 case INDEX_op_extu_i32_i64:
31f1275b 2016 case INDEX_op_ext32u_i64:
929f8b55 2017 tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
31f1275b
CF		 2018 break;
2019
b3c56df7
RH		 2020 case INDEX_op_deposit_i64:
2021 case INDEX_op_deposit_i32:
2022 tcg_out_dep(s, ext, a0, REG0(2), args[3], args[4]);
2023 break;
2024
e2179f94
RH		 2025 case INDEX_op_extract_i64:
2026 case INDEX_op_extract_i32:
2027 tcg_out_ubfm(s, ext, a0, a1, a2, a2 + args[3] - 1);
2028 break;
2029
2030 case INDEX_op_sextract_i64:
2031 case INDEX_op_sextract_i32:
2032 tcg_out_sbfm(s, ext, a0, a1, a2, a2 + args[3] - 1);
2033 break;
2034
c6e929e7
RH		 2035 case INDEX_op_add2_i32:
2036 tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
2037 (int32_t)args[4], args[5], const_args[4],
2038 const_args[5], false);
2039 break;
2040 case INDEX_op_add2_i64:
2041 tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
2042 args[5], const_args[4], const_args[5], false);
2043 break;
2044 case INDEX_op_sub2_i32:
2045 tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
2046 (int32_t)args[4], args[5], const_args[4],
2047 const_args[5], true);
2048 break;
2049 case INDEX_op_sub2_i64:
2050 tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
2051 args[5], const_args[4], const_args[5], true);
2052 break;
2053
1fcc9ddf
RH		 2054 case INDEX_op_muluh_i64:
2055 tcg_out_insn(s, 3508, UMULH, TCG_TYPE_I64, a0, a1, a2);
2056 break;
2057 case INDEX_op_mulsh_i64:
2058 tcg_out_insn(s, 3508, SMULH, TCG_TYPE_I64, a0, a1, a2);
2059 break;
2060
c7a59c2a
PK		 2061 case INDEX_op_mb:
2062 tcg_out_mb(s, a0);
2063 break;
2064
96d0ee7f 2065 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
a51a6b6a 2066 case INDEX_op_mov_i64:
14e4c1e2 2067 case INDEX_op_mov_vec:
96d0ee7f 2068 case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */
a51a6b6a 2069 case INDEX_op_movi_i64:
14e4c1e2 2070 case INDEX_op_dupi_vec:
96d0ee7f 2071 case INDEX_op_call: /* Always emitted via tcg_out_call. */
4a136e0a 2072 default:
14e4c1e2 2073 g_assert_not_reached();
4a136e0a 2074 }
04ce397b
RH		 2075
2076#undef REG0
4a136e0a
CF		 2077}
2078
14e4c1e2
RH		 2079static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2080 unsigned vecl, unsigned vece,
2081 const TCGArg *args, const int *const_args)
2082{
2083 static const AArch64Insn cmp_insn[16] = {
2084 [TCG_COND_EQ] = I3616_CMEQ,
2085 [TCG_COND_GT] = I3616_CMGT,
2086 [TCG_COND_GE] = I3616_CMGE,
2087 [TCG_COND_GTU] = I3616_CMHI,
2088 [TCG_COND_GEU] = I3616_CMHS,
2089 };
2090 static const AArch64Insn cmp0_insn[16] = {
2091 [TCG_COND_EQ] = I3617_CMEQ0,
2092 [TCG_COND_GT] = I3617_CMGT0,
2093 [TCG_COND_GE] = I3617_CMGE0,
2094 [TCG_COND_LT] = I3617_CMLT0,
2095 [TCG_COND_LE] = I3617_CMLE0,
2096 };
2097
2098 TCGType type = vecl + TCG_TYPE_V64;
2099 unsigned is_q = vecl;
2100 TCGArg a0, a1, a2;
2101
2102 a0 = args[0];
2103 a1 = args[1];
2104 a2 = args[2];
2105
2106 switch (opc) {
2107 case INDEX_op_ld_vec:
2108 tcg_out_ld(s, type, a0, a1, a2);
2109 break;
2110 case INDEX_op_st_vec:
2111 tcg_out_st(s, type, a0, a1, a2);
2112 break;
2113 case INDEX_op_add_vec:
2114 tcg_out_insn(s, 3616, ADD, is_q, vece, a0, a1, a2);
2115 break;
2116 case INDEX_op_sub_vec:
2117 tcg_out_insn(s, 3616, SUB, is_q, vece, a0, a1, a2);
2118 break;
2119 case INDEX_op_mul_vec:
2120 tcg_out_insn(s, 3616, MUL, is_q, vece, a0, a1, a2);
2121 break;
2122 case INDEX_op_neg_vec:
2123 tcg_out_insn(s, 3617, NEG, is_q, vece, a0, a1);
2124 break;
2125 case INDEX_op_and_vec:
2126 tcg_out_insn(s, 3616, AND, is_q, 0, a0, a1, a2);
2127 break;
2128 case INDEX_op_or_vec:
2129 tcg_out_insn(s, 3616, ORR, is_q, 0, a0, a1, a2);
2130 break;
2131 case INDEX_op_xor_vec:
2132 tcg_out_insn(s, 3616, EOR, is_q, 0, a0, a1, a2);
2133 break;
2134 case INDEX_op_andc_vec:
2135 tcg_out_insn(s, 3616, BIC, is_q, 0, a0, a1, a2);
2136 break;
2137 case INDEX_op_orc_vec:
2138 tcg_out_insn(s, 3616, ORN, is_q, 0, a0, a1, a2);
2139 break;
2140 case INDEX_op_not_vec:
2141 tcg_out_insn(s, 3617, NOT, is_q, 0, a0, a1);
2142 break;
2143 case INDEX_op_dup_vec:
2144 tcg_out_insn(s, 3605, DUP, is_q, a0, a1, 1 << vece, 0);
2145 break;
2146 case INDEX_op_shli_vec:
2147 tcg_out_insn(s, 3614, SHL, is_q, a0, a1, a2 + (8 << vece));
2148 break;
2149 case INDEX_op_shri_vec:
2150 tcg_out_insn(s, 3614, USHR, is_q, a0, a1, (16 << vece) - a2);
2151 break;
2152 case INDEX_op_sari_vec:
2153 tcg_out_insn(s, 3614, SSHR, is_q, a0, a1, (16 << vece) - a2);
2154 break;
2155 case INDEX_op_cmp_vec:
2156 {
2157 TCGCond cond = args[3];
2158 AArch64Insn insn;
2159
2160 if (cond == TCG_COND_NE) {
2161 if (const_args[2]) {
2162 tcg_out_insn(s, 3616, CMTST, is_q, vece, a0, a1, a1);
2163 } else {
2164 tcg_out_insn(s, 3616, CMEQ, is_q, vece, a0, a1, a2);
2165 tcg_out_insn(s, 3617, NOT, is_q, 0, a0, a0);
2166 }
2167 } else {
2168 if (const_args[2]) {
2169 insn = cmp0_insn[cond];
2170 if (insn) {
2171 tcg_out_insn_3617(s, insn, is_q, vece, a0, a1);
2172 break;
2173 }
2174 tcg_out_dupi_vec(s, type, TCG_VEC_TMP, 0);
2175 a2 = TCG_VEC_TMP;
2176 }
2177 insn = cmp_insn[cond];
2178 if (insn == 0) {
2179 TCGArg t;
2180 t = a1, a1 = a2, a2 = t;
2181 cond = tcg_swap_cond(cond);
2182 insn = cmp_insn[cond];
2183 tcg_debug_assert(insn != 0);
2184 }
2185 tcg_out_insn_3616(s, insn, is_q, vece, a0, a1, a2);
2186 }
2187 }
2188 break;
2189 default:
2190 g_assert_not_reached();
2191 }
2192}
2193
2194int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
2195{
2196 switch (opc) {
2197 case INDEX_op_add_vec:
2198 case INDEX_op_sub_vec:
14e4c1e2
RH		 2199 case INDEX_op_and_vec:
2200 case INDEX_op_or_vec:
2201 case INDEX_op_xor_vec:
2202 case INDEX_op_andc_vec:
2203 case INDEX_op_orc_vec:
2204 case INDEX_op_neg_vec:
2205 case INDEX_op_not_vec:
2206 case INDEX_op_cmp_vec:
2207 case INDEX_op_shli_vec:
2208 case INDEX_op_shri_vec:
2209 case INDEX_op_sari_vec:
2210 return 1;
e65a5f22
AB		 2211 case INDEX_op_mul_vec:
2212 return vece < MO_64;
14e4c1e2
RH		 2213
2214 default:
2215 return 0;
2216 }
2217}
2218
2219void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
2220 TCGArg a0, ...)
2221{
2222}
2223
f69d277e
RH		 2224static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
2225{
1897cc2e
RH		 2226 static const TCGTargetOpDef r = { .args_ct_str = { "r" } };
2227 static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } };
14e4c1e2
RH		 2228 static const TCGTargetOpDef w_w = { .args_ct_str = { "w", "w" } };
2229 static const TCGTargetOpDef w_r = { .args_ct_str = { "w", "r" } };
2230 static const TCGTargetOpDef w_wr = { .args_ct_str = { "w", "wr" } };
1897cc2e
RH		 2231 static const TCGTargetOpDef r_l = { .args_ct_str = { "r", "l" } };
2232 static const TCGTargetOpDef r_rA = { .args_ct_str = { "r", "rA" } };
2233 static const TCGTargetOpDef rZ_r = { .args_ct_str = { "rZ", "r" } };
2234 static const TCGTargetOpDef lZ_l = { .args_ct_str = { "lZ", "l" } };
2235 static const TCGTargetOpDef r_r_r = { .args_ct_str = { "r", "r", "r" } };
14e4c1e2
RH		 2236 static const TCGTargetOpDef w_w_w = { .args_ct_str = { "w", "w", "w" } };
2237 static const TCGTargetOpDef w_w_wZ = { .args_ct_str = { "w", "w", "wZ" } };
1897cc2e
RH		 2238 static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } };
2239 static const TCGTargetOpDef r_r_rA = { .args_ct_str = { "r", "r", "rA" } };
2240 static const TCGTargetOpDef r_r_rL = { .args_ct_str = { "r", "r", "rL" } };
2241 static const TCGTargetOpDef r_r_rAL
2242 = { .args_ct_str = { "r", "r", "rAL" } };
2243 static const TCGTargetOpDef dep
2244 = { .args_ct_str = { "r", "0", "rZ" } };
2245 static const TCGTargetOpDef movc
2246 = { .args_ct_str = { "r", "r", "rA", "rZ", "rZ" } };
2247 static const TCGTargetOpDef add2
2248 = { .args_ct_str = { "r", "r", "rZ", "rZ", "rA", "rMZ" } };
2249
2250 switch (op) {
2251 case INDEX_op_goto_ptr:
2252 return &r;
f69d277e 2253
1897cc2e
RH		 2254 case INDEX_op_ld8u_i32:
2255 case INDEX_op_ld8s_i32:
2256 case INDEX_op_ld16u_i32:
2257 case INDEX_op_ld16s_i32:
2258 case INDEX_op_ld_i32:
2259 case INDEX_op_ld8u_i64:
2260 case INDEX_op_ld8s_i64:
2261 case INDEX_op_ld16u_i64:
2262 case INDEX_op_ld16s_i64:
2263 case INDEX_op_ld32u_i64:
2264 case INDEX_op_ld32s_i64:
2265 case INDEX_op_ld_i64:
2266 case INDEX_op_neg_i32:
2267 case INDEX_op_neg_i64:
2268 case INDEX_op_not_i32:
2269 case INDEX_op_not_i64:
2270 case INDEX_op_bswap16_i32:
2271 case INDEX_op_bswap32_i32:
2272 case INDEX_op_bswap16_i64:
2273 case INDEX_op_bswap32_i64:
2274 case INDEX_op_bswap64_i64:
2275 case INDEX_op_ext8s_i32:
2276 case INDEX_op_ext16s_i32:
2277 case INDEX_op_ext8u_i32:
2278 case INDEX_op_ext16u_i32:
2279 case INDEX_op_ext8s_i64:
2280 case INDEX_op_ext16s_i64:
2281 case INDEX_op_ext32s_i64:
2282 case INDEX_op_ext8u_i64:
2283 case INDEX_op_ext16u_i64:
2284 case INDEX_op_ext32u_i64:
2285 case INDEX_op_ext_i32_i64:
2286 case INDEX_op_extu_i32_i64:
2287 case INDEX_op_extract_i32:
2288 case INDEX_op_extract_i64:
2289 case INDEX_op_sextract_i32:
2290 case INDEX_op_sextract_i64:
2291 return &r_r;
2292
2293 case INDEX_op_st8_i32:
2294 case INDEX_op_st16_i32:
2295 case INDEX_op_st_i32:
2296 case INDEX_op_st8_i64:
2297 case INDEX_op_st16_i64:
2298 case INDEX_op_st32_i64:
2299 case INDEX_op_st_i64:
2300 return &rZ_r;
2301
2302 case INDEX_op_add_i32:
2303 case INDEX_op_add_i64:
2304 case INDEX_op_sub_i32:
2305 case INDEX_op_sub_i64:
2306 case INDEX_op_setcond_i32:
2307 case INDEX_op_setcond_i64:
2308 return &r_r_rA;
2309
2310 case INDEX_op_mul_i32:
2311 case INDEX_op_mul_i64:
2312 case INDEX_op_div_i32:
2313 case INDEX_op_div_i64:
2314 case INDEX_op_divu_i32:
2315 case INDEX_op_divu_i64:
2316 case INDEX_op_rem_i32:
2317 case INDEX_op_rem_i64:
2318 case INDEX_op_remu_i32:
2319 case INDEX_op_remu_i64:
2320 case INDEX_op_muluh_i64:
2321 case INDEX_op_mulsh_i64:
2322 return &r_r_r;
2323
2324 case INDEX_op_and_i32:
2325 case INDEX_op_and_i64:
2326 case INDEX_op_or_i32:
2327 case INDEX_op_or_i64:
2328 case INDEX_op_xor_i32:
2329 case INDEX_op_xor_i64:
2330 case INDEX_op_andc_i32:
2331 case INDEX_op_andc_i64:
2332 case INDEX_op_orc_i32:
2333 case INDEX_op_orc_i64:
2334 case INDEX_op_eqv_i32:
2335 case INDEX_op_eqv_i64:
2336 return &r_r_rL;
2337
2338 case INDEX_op_shl_i32:
2339 case INDEX_op_shr_i32:
2340 case INDEX_op_sar_i32:
2341 case INDEX_op_rotl_i32:
2342 case INDEX_op_rotr_i32:
2343 case INDEX_op_shl_i64:
2344 case INDEX_op_shr_i64:
2345 case INDEX_op_sar_i64:
2346 case INDEX_op_rotl_i64:
2347 case INDEX_op_rotr_i64:
2348 return &r_r_ri;
2349
2350 case INDEX_op_clz_i32:
2351 case INDEX_op_ctz_i32:
2352 case INDEX_op_clz_i64:
2353 case INDEX_op_ctz_i64:
2354 return &r_r_rAL;
2355
2356 case INDEX_op_brcond_i32:
2357 case INDEX_op_brcond_i64:
2358 return &r_rA;
2359
2360 case INDEX_op_movcond_i32:
2361 case INDEX_op_movcond_i64:
2362 return &movc;
2363
2364 case INDEX_op_qemu_ld_i32:
2365 case INDEX_op_qemu_ld_i64:
2366 return &r_l;
2367 case INDEX_op_qemu_st_i32:
2368 case INDEX_op_qemu_st_i64:
2369 return &lZ_l;
2370
2371 case INDEX_op_deposit_i32:
2372 case INDEX_op_deposit_i64:
2373 return &dep;
2374
2375 case INDEX_op_add2_i32:
2376 case INDEX_op_add2_i64:
2377 case INDEX_op_sub2_i32:
2378 case INDEX_op_sub2_i64:
2379 return &add2;
2380
14e4c1e2
RH		 2381 case INDEX_op_add_vec:
2382 case INDEX_op_sub_vec:
2383 case INDEX_op_mul_vec:
2384 case INDEX_op_and_vec:
2385 case INDEX_op_or_vec:
2386 case INDEX_op_xor_vec:
2387 case INDEX_op_andc_vec:
2388 case INDEX_op_orc_vec:
2389 return &w_w_w;
2390 case INDEX_op_not_vec:
2391 case INDEX_op_neg_vec:
2392 case INDEX_op_shli_vec:
2393 case INDEX_op_shri_vec:
2394 case INDEX_op_sari_vec:
2395 return &w_w;
2396 case INDEX_op_ld_vec:
2397 case INDEX_op_st_vec:
2398 return &w_r;
2399 case INDEX_op_dup_vec:
2400 return &w_wr;
2401 case INDEX_op_cmp_vec:
2402 return &w_w_wZ;
2403
1897cc2e
RH		 2404 default:
2405 return NULL;
f69d277e 2406 }
f69d277e
RH		 2407}
2408
4a136e0a
CF		 2409static void tcg_target_init(TCGContext *s)
2410{
f46934df
RH		 2411 tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffffu;
2412 tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffffu;
14e4c1e2
RH		 2413 tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
2414 tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
f46934df 2415
14e4c1e2 2416 tcg_target_call_clobber_regs = -1ull;
f46934df
RH		 2417 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X19);
2418 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X20);
2419 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X21);
2420 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X22);
2421 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X23);
2422 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X24);
2423 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X25);
2424 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X26);
2425 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X27);
2426 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X28);
2427 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X29);
14e4c1e2
RH		 2428 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V8);
2429 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V9);
2430 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V10);
2431 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V11);
2432 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V12);
2433 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V13);
2434 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V14);
2435 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V15);
4a136e0a 2436
ccb1bb66 2437 s->reserved_regs = 0;
4a136e0a
CF		 2438 tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);
2439 tcg_regset_set_reg(s->reserved_regs, TCG_REG_FP);
2440 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP);
2441 tcg_regset_set_reg(s->reserved_regs, TCG_REG_X18); /* platform register */
14e4c1e2 2442 tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP);
4a136e0a
CF		 2443}
2444
38d195aa
RH		 2445/* Saving pairs: (X19, X20) .. (X27, X28), (X29(fp), X30(lr)). */
2446#define PUSH_SIZE ((30 - 19 + 1) * 8)
2447
2448#define FRAME_SIZE \
2449 ((PUSH_SIZE \
2450 + TCG_STATIC_CALL_ARGS_SIZE \
2451 + CPU_TEMP_BUF_NLONGS * sizeof(long) \
2452 + TCG_TARGET_STACK_ALIGN - 1) \
2453 & ~(TCG_TARGET_STACK_ALIGN - 1))
2454
2455/* We're expecting a 2 byte uleb128 encoded value. */
2456QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
2457
2458/* We're expecting to use a single ADDI insn. */
2459QEMU_BUILD_BUG_ON(FRAME_SIZE - PUSH_SIZE > 0xfff);
2460
4a136e0a
CF		 2461static void tcg_target_qemu_prologue(TCGContext *s)
2462{
4a136e0a
CF		 2463 TCGReg r;
2464
95f72aa9
RH		 2465 /* Push (FP, LR) and allocate space for all saved registers. */
2466 tcg_out_insn(s, 3314, STP, TCG_REG_FP, TCG_REG_LR,
38d195aa 2467 TCG_REG_SP, -PUSH_SIZE, 1, 1);
4a136e0a 2468
d82b78e4 2469 /* Set up frame pointer for canonical unwinding. */
929f8b55 2470 tcg_out_movr_sp(s, TCG_TYPE_I64, TCG_REG_FP, TCG_REG_SP);
4a136e0a 2471
d82b78e4 2472 /* Store callee-preserved regs x19..x28. */
4a136e0a 2473 for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
95f72aa9
RH		 2474 int ofs = (r - TCG_REG_X19 + 2) * 8;
2475 tcg_out_insn(s, 3314, STP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
4a136e0a
CF		 2476 }
2477
096c46c0
RH		 2478 /* Make stack space for TCG locals. */
2479 tcg_out_insn(s, 3401, SUBI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
38d195aa 2480 FRAME_SIZE - PUSH_SIZE);
096c46c0 2481
95f72aa9 2482 /* Inform TCG about how to find TCG locals with register, offset, size. */
4a136e0a
CF		 2483 tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
2484 CPU_TEMP_BUF_NLONGS * sizeof(long));
2485
4cbea598 2486#if !defined(CONFIG_SOFTMMU)
352bcb0a 2487 if (USE_GUEST_BASE) {
b76f21a7 2488 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base);
6a91c7c9
JK		 2489 tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE);
2490 }
2491#endif
2492
4a136e0a 2493 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
81d8a5ee 2494 tcg_out_insn(s, 3207, BR, tcg_target_call_iarg_regs[1]);
4a136e0a 2495
b19f0c2e
RH		 2496 /*
2497 * Return path for goto_ptr. Set return value to 0, a-la exit_tb,
2498 * and fall through to the rest of the epilogue.
2499 */
2500 s->code_gen_epilogue = s->code_ptr;
2501 tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_X0, 0);
2502
2503 /* TB epilogue */
4a136e0a
CF		 2504 tb_ret_addr = s->code_ptr;
2505
096c46c0
RH		 2506 /* Remove TCG locals stack space. */
2507 tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
38d195aa 2508 FRAME_SIZE - PUSH_SIZE);
4a136e0a 2509
95f72aa9 2510 /* Restore registers x19..x28. */
4a136e0a 2511 for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
95f72aa9
RH		 2512 int ofs = (r - TCG_REG_X19 + 2) * 8;
2513 tcg_out_insn(s, 3314, LDP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
4a136e0a
CF		 2514 }
2515
95f72aa9
RH		 2516 /* Pop (FP, LR), restore SP to previous frame. */
2517 tcg_out_insn(s, 3314, LDP, TCG_REG_FP, TCG_REG_LR,
38d195aa 2518 TCG_REG_SP, PUSH_SIZE, 0, 1);
81d8a5ee 2519 tcg_out_insn(s, 3207, RET, TCG_REG_LR);
4a136e0a 2520}
38d195aa 2521
55129955
RH		 2522static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
2523{
2524 int i;
2525 for (i = 0; i < count; ++i) {
2526 p[i] = NOP;
2527 }
2528}
2529
38d195aa 2530typedef struct {
3d9bddb3 2531 DebugFrameHeader h;
38d195aa
RH		 2532 uint8_t fde_def_cfa[4];
2533 uint8_t fde_reg_ofs[24];
2534} DebugFrame;
2535
2536#define ELF_HOST_MACHINE EM_AARCH64
2537
3d9bddb3
RH		 2538static const DebugFrame debug_frame = {
2539 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
2540 .h.cie.id = -1,
2541 .h.cie.version = 1,
2542 .h.cie.code_align = 1,
2543 .h.cie.data_align = 0x78, /* sleb128 -8 */
2544 .h.cie.return_column = TCG_REG_LR,
38d195aa
RH		 2545
2546 /* Total FDE size does not include the "len" member. */
3d9bddb3 2547 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
38d195aa
RH		 2548
2549 .fde_def_cfa = {
2550 12, TCG_REG_SP, /* DW_CFA_def_cfa sp, ... */
2551 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
2552 (FRAME_SIZE >> 7)
2553 },
2554 .fde_reg_ofs = {
2555 0x80 + 28, 1, /* DW_CFA_offset, x28, -8 */
2556 0x80 + 27, 2, /* DW_CFA_offset, x27, -16 */
2557 0x80 + 26, 3, /* DW_CFA_offset, x26, -24 */
2558 0x80 + 25, 4, /* DW_CFA_offset, x25, -32 */
2559 0x80 + 24, 5, /* DW_CFA_offset, x24, -40 */
2560 0x80 + 23, 6, /* DW_CFA_offset, x23, -48 */
2561 0x80 + 22, 7, /* DW_CFA_offset, x22, -56 */
2562 0x80 + 21, 8, /* DW_CFA_offset, x21, -64 */
2563 0x80 + 20, 9, /* DW_CFA_offset, x20, -72 */
2564 0x80 + 19, 10, /* DW_CFA_offset, x1p, -80 */
2565 0x80 + 30, 11, /* DW_CFA_offset, lr, -88 */
2566 0x80 + 29, 12, /* DW_CFA_offset, fp, -96 */
2567 }
2568};
2569
2570void tcg_register_jit(void *buf, size_t buf_size)
2571{
38d195aa
RH		 2572 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
2573}
QEMU mirror