]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/aarch64/tcg-target.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg-aarch64: Use symbolic names for branches
[mirror_qemu.git] / tcg / aarch64 / tcg-target.c
CommitLineData
4a136e0a
CF		 1/*
2 * Initial TCG Implementation for aarch64
3 *
4 * Copyright (c) 2013 Huawei Technologies Duesseldorf GmbH
5 * Written by Claudio Fontana
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or
8 * (at your option) any later version.
9 *
10 * See the COPYING file in the top-level directory for details.
11 */
12
9ecefc84 13#include "tcg-be-ldst.h"
4a136e0a
CF		 14#include "qemu/bitops.h"
15
7763ffa0
RH		 16/* We're going to re-use TCGType in setting of the SF bit, which controls
17 the size of the operation performed. If we know the values match, it
18 makes things much cleaner. */
19QEMU_BUILD_BUG_ON(TCG_TYPE_I32 != 0 || TCG_TYPE_I64 != 1);
20
4a136e0a
CF		 21#ifndef NDEBUG
22static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
23 "%x0", "%x1", "%x2", "%x3", "%x4", "%x5", "%x6", "%x7",
24 "%x8", "%x9", "%x10", "%x11", "%x12", "%x13", "%x14", "%x15",
25 "%x16", "%x17", "%x18", "%x19", "%x20", "%x21", "%x22", "%x23",
26 "%x24", "%x25", "%x26", "%x27", "%x28",
27 "%fp", /* frame pointer */
28 "%lr", /* link register */
29 "%sp", /* stack pointer */
30};
31#endif /* NDEBUG */
32
6a91c7c9
JK		 33#ifdef TARGET_WORDS_BIGENDIAN
34 #define TCG_LDST_BSWAP 1
35#else
36 #define TCG_LDST_BSWAP 0
37#endif
38
4a136e0a
CF		 39static const int tcg_target_reg_alloc_order[] = {
40 TCG_REG_X20, TCG_REG_X21, TCG_REG_X22, TCG_REG_X23,
41 TCG_REG_X24, TCG_REG_X25, TCG_REG_X26, TCG_REG_X27,
6a91c7c9 42 TCG_REG_X28, /* we will reserve this for GUEST_BASE if configured */
4a136e0a
CF		 43
44 TCG_REG_X9, TCG_REG_X10, TCG_REG_X11, TCG_REG_X12,
45 TCG_REG_X13, TCG_REG_X14, TCG_REG_X15,
46 TCG_REG_X16, TCG_REG_X17,
47
48 TCG_REG_X18, TCG_REG_X19, /* will not use these, see tcg_target_init */
49
50 TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
51 TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7,
52
53 TCG_REG_X8, /* will not use, see tcg_target_init */
54};
55
56static const int tcg_target_call_iarg_regs[8] = {
57 TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
58 TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7
59};
60static const int tcg_target_call_oarg_regs[1] = {
61 TCG_REG_X0
62};
63
64#define TCG_REG_TMP TCG_REG_X8
65
6a91c7c9
JK		 66#ifndef CONFIG_SOFTMMU
67# if defined(CONFIG_USE_GUEST_BASE)
68# define TCG_REG_GUEST_BASE TCG_REG_X28
69# else
70# define TCG_REG_GUEST_BASE TCG_REG_XZR
71# endif
72#endif
73
017a86f7 74static inline void reloc_pc26(void *code_ptr, intptr_t target)
4a136e0a 75{
017a86f7 76 intptr_t offset = (target - (intptr_t)code_ptr) / 4;
4a136e0a
CF		 77 /* read instruction, mask away previous PC_REL26 parameter contents,
78 set the proper offset, then write back the instruction. */
017a86f7 79 uint32_t insn = *(uint32_t *)code_ptr;
4a136e0a
CF		 80 insn = deposit32(insn, 0, 26, offset);
81 *(uint32_t *)code_ptr = insn;
82}
83
017a86f7 84static inline void reloc_pc19(void *code_ptr, intptr_t target)
4a136e0a 85{
017a86f7 86 intptr_t offset = (target - (intptr_t)code_ptr) / 4;
4a136e0a
CF		 87 /* read instruction, mask away previous PC_REL19 parameter contents,
88 set the proper offset, then write back the instruction. */
017a86f7 89 uint32_t insn = *(uint32_t *)code_ptr;
4a136e0a
CF		 90 insn = deposit32(insn, 5, 19, offset);
91 *(uint32_t *)code_ptr = insn;
92}
93
94static inline void patch_reloc(uint8_t *code_ptr, int type,
2ba7fae2 95 intptr_t value, intptr_t addend)
4a136e0a
CF		 96{
97 value += addend;
98
99 switch (type) {
100 case R_AARCH64_JUMP26:
101 case R_AARCH64_CALL26:
102 reloc_pc26(code_ptr, value);
103 break;
104 case R_AARCH64_CONDBR19:
105 reloc_pc19(code_ptr, value);
106 break;
107
108 default:
109 tcg_abort();
110 }
111}
112
90f1cd91
RH		 113#define TCG_CT_CONST_IS32 0x100
114#define TCG_CT_CONST_AIMM 0x200
e029f293 115#define TCG_CT_CONST_LIMM 0x400
04ce397b 116#define TCG_CT_CONST_ZERO 0x800
c6e929e7 117#define TCG_CT_CONST_MONE 0x1000
90f1cd91 118
4a136e0a
CF		 119/* parse target specific constraints */
120static int target_parse_constraint(TCGArgConstraint *ct,
121 const char **pct_str)
122{
123 const char *ct_str = *pct_str;
124
125 switch (ct_str[0]) {
126 case 'r':
127 ct->ct |= TCG_CT_REG;
128 tcg_regset_set32(ct->u.regs, 0, (1ULL << TCG_TARGET_NB_REGS) - 1);
129 break;
130 case 'l': /* qemu_ld / qemu_st address, data_reg */
131 ct->ct |= TCG_CT_REG;
132 tcg_regset_set32(ct->u.regs, 0, (1ULL << TCG_TARGET_NB_REGS) - 1);
133#ifdef CONFIG_SOFTMMU
134 /* x0 and x1 will be overwritten when reading the tlb entry,
135 and x2, and x3 for helper args, better to avoid using them. */
136 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X0);
137 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X1);
138 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X2);
139 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X3);
140#endif
141 break;
90f1cd91
RH		 142 case 'w': /* The operand should be considered 32-bit. */
143 ct->ct |= TCG_CT_CONST_IS32;
144 break;
145 case 'A': /* Valid for arithmetic immediate (positive or negative). */
146 ct->ct |= TCG_CT_CONST_AIMM;
147 break;
e029f293
RH		 148 case 'L': /* Valid for logical immediate. */
149 ct->ct |= TCG_CT_CONST_LIMM;
150 break;
c6e929e7
RH		 151 case 'M': /* minus one */
152 ct->ct |= TCG_CT_CONST_MONE;
153 break;
04ce397b
RH		 154 case 'Z': /* zero */
155 ct->ct |= TCG_CT_CONST_ZERO;
156 break;
4a136e0a
CF		 157 default:
158 return -1;
159 }
160
161 ct_str++;
162 *pct_str = ct_str;
163 return 0;
164}
165
90f1cd91
RH		 166static inline bool is_aimm(uint64_t val)
167{
168 return (val & ~0xfff) == 0 || (val & ~0xfff000) == 0;
169}
170
e029f293
RH		 171static inline bool is_limm(uint64_t val)
172{
173 /* Taking a simplified view of the logical immediates for now, ignoring
174 the replication that can happen across the field. Match bit patterns
175 of the forms
176 0....01....1
177 0..01..10..0
178 and their inverses. */
179
180 /* Make things easier below, by testing the form with msb clear. */
181 if ((int64_t)val < 0) {
182 val = ~val;
183 }
184 if (val == 0) {
185 return false;
186 }
187 val += val & -val;
188 return (val & (val - 1)) == 0;
189}
190
90f1cd91
RH		 191static int tcg_target_const_match(tcg_target_long val,
192 const TCGArgConstraint *arg_ct)
4a136e0a
CF		 193{
194 int ct = arg_ct->ct;
195
196 if (ct & TCG_CT_CONST) {
197 return 1;
198 }
90f1cd91
RH		 199 if (ct & TCG_CT_CONST_IS32) {
200 val = (int32_t)val;
201 }
202 if ((ct & TCG_CT_CONST_AIMM) && (is_aimm(val) || is_aimm(-val))) {
203 return 1;
204 }
e029f293
RH		 205 if ((ct & TCG_CT_CONST_LIMM) && is_limm(val)) {
206 return 1;
207 }
04ce397b
RH		 208 if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
209 return 1;
210 }
c6e929e7
RH		 211 if ((ct & TCG_CT_CONST_MONE) && val == -1) {
212 return 1;
213 }
4a136e0a
CF		 214
215 return 0;
216}
217
218enum aarch64_cond_code {
219 COND_EQ = 0x0,
220 COND_NE = 0x1,
221 COND_CS = 0x2, /* Unsigned greater or equal */
222 COND_HS = COND_CS, /* ALIAS greater or equal */
223 COND_CC = 0x3, /* Unsigned less than */
224 COND_LO = COND_CC, /* ALIAS Lower */
225 COND_MI = 0x4, /* Negative */
226 COND_PL = 0x5, /* Zero or greater */
227 COND_VS = 0x6, /* Overflow */
228 COND_VC = 0x7, /* No overflow */
229 COND_HI = 0x8, /* Unsigned greater than */
230 COND_LS = 0x9, /* Unsigned less or equal */
231 COND_GE = 0xa,
232 COND_LT = 0xb,
233 COND_GT = 0xc,
234 COND_LE = 0xd,
235 COND_AL = 0xe,
236 COND_NV = 0xf, /* behaves like COND_AL here */
237};
238
239static const enum aarch64_cond_code tcg_cond_to_aarch64[] = {
240 [TCG_COND_EQ] = COND_EQ,
241 [TCG_COND_NE] = COND_NE,
242 [TCG_COND_LT] = COND_LT,
243 [TCG_COND_GE] = COND_GE,
244 [TCG_COND_LE] = COND_LE,
245 [TCG_COND_GT] = COND_GT,
246 /* unsigned */
247 [TCG_COND_LTU] = COND_LO,
248 [TCG_COND_GTU] = COND_HI,
249 [TCG_COND_GEU] = COND_HS,
250 [TCG_COND_LEU] = COND_LS,
251};
252
253/* opcodes for LDR / STR instructions with base + simm9 addressing */
254enum aarch64_ldst_op_data { /* size of the data moved */
255 LDST_8 = 0x38,
256 LDST_16 = 0x78,
257 LDST_32 = 0xb8,
258 LDST_64 = 0xf8,
259};
260enum aarch64_ldst_op_type { /* type of operation */
261 LDST_ST = 0x0, /* store */
262 LDST_LD = 0x4, /* load */
263 LDST_LD_S_X = 0x8, /* load and sign-extend into Xt */
264 LDST_LD_S_W = 0xc, /* load and sign-extend into Wt */
265};
266
50573c66
RH		 267/* We encode the format of the insn into the beginning of the name, so that
268 we can have the preprocessor help "typecheck" the insn vs the output
269 function. Arm didn't provide us with nice names for the formats, so we
270 use the section number of the architecture reference manual in which the
271 instruction group is described. */
272typedef enum {
81d8a5ee
RH		 273 /* Conditional branch (immediate). */
274 I3202_B_C = 0x54000000,
275
276 /* Unconditional branch (immediate). */
277 I3206_B = 0x14000000,
278 I3206_BL = 0x94000000,
279
280 /* Unconditional branch (register). */
281 I3207_BR = 0xd61f0000,
282 I3207_BLR = 0xd63f0000,
283 I3207_RET = 0xd65f0000,
284
096c46c0
RH		 285 /* Add/subtract immediate instructions. */
286 I3401_ADDI = 0x11000000,
287 I3401_ADDSI = 0x31000000,
288 I3401_SUBI = 0x51000000,
289 I3401_SUBSI = 0x71000000,
290
b3c56df7
RH		 291 /* Bitfield instructions. */
292 I3402_BFM = 0x33000000,
293 I3402_SBFM = 0x13000000,
294 I3402_UBFM = 0x53000000,
295
296 /* Extract instruction. */
297 I3403_EXTR = 0x13800000,
298
e029f293
RH		 299 /* Logical immediate instructions. */
300 I3404_ANDI = 0x12000000,
301 I3404_ORRI = 0x32000000,
302 I3404_EORI = 0x52000000,
303
582ab779
RH		 304 /* Move wide immediate instructions. */
305 I3405_MOVN = 0x12800000,
306 I3405_MOVZ = 0x52800000,
307 I3405_MOVK = 0x72800000,
308
c6e310d9
RH		 309 /* PC relative addressing instructions. */
310 I3406_ADR = 0x10000000,
311 I3406_ADRP = 0x90000000,
312
50573c66
RH		 313 /* Add/subtract shifted register instructions (without a shift). */
314 I3502_ADD = 0x0b000000,
315 I3502_ADDS = 0x2b000000,
316 I3502_SUB = 0x4b000000,
317 I3502_SUBS = 0x6b000000,
318
319 /* Add/subtract shifted register instructions (with a shift). */
320 I3502S_ADD_LSL = I3502_ADD,
321
c6e929e7
RH		 322 /* Add/subtract with carry instructions. */
323 I3503_ADC = 0x1a000000,
324 I3503_SBC = 0x5a000000,
325
04ce397b
RH		 326 /* Conditional select instructions. */
327 I3506_CSEL = 0x1a800000,
328 I3506_CSINC = 0x1a800400,
329
df9351e3
RH		 330 /* Data-processing (2 source) instructions. */
331 I3508_LSLV = 0x1ac02000,
332 I3508_LSRV = 0x1ac02400,
333 I3508_ASRV = 0x1ac02800,
334 I3508_RORV = 0x1ac02c00,
1fcc9ddf
RH		 335 I3508_SMULH = 0x9b407c00,
336 I3508_UMULH = 0x9bc07c00,
8678b71c
RH		 337 I3508_UDIV = 0x1ac00800,
338 I3508_SDIV = 0x1ac00c00,
339
340 /* Data-processing (3 source) instructions. */
341 I3509_MADD = 0x1b000000,
342 I3509_MSUB = 0x1b008000,
df9351e3 343
50573c66
RH		 344 /* Logical shifted register instructions (without a shift). */
345 I3510_AND = 0x0a000000,
14b155dd 346 I3510_BIC = 0x0a200000,
50573c66 347 I3510_ORR = 0x2a000000,
14b155dd 348 I3510_ORN = 0x2a200000,
50573c66 349 I3510_EOR = 0x4a000000,
14b155dd 350 I3510_EON = 0x4a200000,
50573c66
RH		 351 I3510_ANDS = 0x6a000000,
352} AArch64Insn;
4a136e0a 353
4a136e0a
CF		 354static inline enum aarch64_ldst_op_data
355aarch64_ldst_get_data(TCGOpcode tcg_op)
356{
357 switch (tcg_op) {
358 case INDEX_op_ld8u_i32:
359 case INDEX_op_ld8s_i32:
360 case INDEX_op_ld8u_i64:
361 case INDEX_op_ld8s_i64:
362 case INDEX_op_st8_i32:
363 case INDEX_op_st8_i64:
364 return LDST_8;
365
366 case INDEX_op_ld16u_i32:
367 case INDEX_op_ld16s_i32:
368 case INDEX_op_ld16u_i64:
369 case INDEX_op_ld16s_i64:
370 case INDEX_op_st16_i32:
371 case INDEX_op_st16_i64:
372 return LDST_16;
373
374 case INDEX_op_ld_i32:
375 case INDEX_op_st_i32:
376 case INDEX_op_ld32u_i64:
377 case INDEX_op_ld32s_i64:
378 case INDEX_op_st32_i64:
379 return LDST_32;
380
381 case INDEX_op_ld_i64:
382 case INDEX_op_st_i64:
383 return LDST_64;
384
385 default:
386 tcg_abort();
387 }
388}
389
390static inline enum aarch64_ldst_op_type
391aarch64_ldst_get_type(TCGOpcode tcg_op)
392{
393 switch (tcg_op) {
394 case INDEX_op_st8_i32:
395 case INDEX_op_st16_i32:
396 case INDEX_op_st8_i64:
397 case INDEX_op_st16_i64:
398 case INDEX_op_st_i32:
399 case INDEX_op_st32_i64:
400 case INDEX_op_st_i64:
401 return LDST_ST;
402
403 case INDEX_op_ld8u_i32:
404 case INDEX_op_ld16u_i32:
405 case INDEX_op_ld8u_i64:
406 case INDEX_op_ld16u_i64:
407 case INDEX_op_ld_i32:
408 case INDEX_op_ld32u_i64:
409 case INDEX_op_ld_i64:
410 return LDST_LD;
411
412 case INDEX_op_ld8s_i32:
413 case INDEX_op_ld16s_i32:
414 return LDST_LD_S_W;
415
416 case INDEX_op_ld8s_i64:
417 case INDEX_op_ld16s_i64:
418 case INDEX_op_ld32s_i64:
419 return LDST_LD_S_X;
420
421 default:
422 tcg_abort();
423 }
424}
425
426static inline uint32_t tcg_in32(TCGContext *s)
427{
428 uint32_t v = *(uint32_t *)s->code_ptr;
429 return v;
430}
431
50573c66
RH		 432/* Emit an opcode with "type-checking" of the format. */
433#define tcg_out_insn(S, FMT, OP, ...) \
434 glue(tcg_out_insn_,FMT)(S, glue(glue(glue(I,FMT),_),OP), ## __VA_ARGS__)
435
81d8a5ee
RH		 436static void tcg_out_insn_3202(TCGContext *s, AArch64Insn insn,
437 TCGCond c, int imm19)
438{
439 tcg_out32(s, insn | tcg_cond_to_aarch64[c] | (imm19 & 0x7ffff) << 5);
440}
441
442static void tcg_out_insn_3206(TCGContext *s, AArch64Insn insn, int imm26)
443{
444 tcg_out32(s, insn | (imm26 & 0x03ffffff));
445}
446
447static void tcg_out_insn_3207(TCGContext *s, AArch64Insn insn, TCGReg rn)
448{
449 tcg_out32(s, insn | rn << 5);
450}
451
096c46c0
RH		 452static void tcg_out_insn_3401(TCGContext *s, AArch64Insn insn, TCGType ext,
453 TCGReg rd, TCGReg rn, uint64_t aimm)
454{
455 if (aimm > 0xfff) {
456 assert((aimm & 0xfff) == 0);
457 aimm >>= 12;
458 assert(aimm <= 0xfff);
459 aimm |= 1 << 12; /* apply LSL 12 */
460 }
461 tcg_out32(s, insn | ext << 31 | aimm << 10 | rn << 5 | rd);
462}
463
e029f293
RH		 464/* This function can be used for both 3.4.2 (Bitfield) and 3.4.4
465 (Logical immediate). Both insn groups have N, IMMR and IMMS fields
466 that feed the DecodeBitMasks pseudo function. */
467static void tcg_out_insn_3402(TCGContext *s, AArch64Insn insn, TCGType ext,
468 TCGReg rd, TCGReg rn, int n, int immr, int imms)
469{
470 tcg_out32(s, insn | ext << 31 | n << 22 | immr << 16 | imms << 10
471 | rn << 5 | rd);
472}
473
474#define tcg_out_insn_3404 tcg_out_insn_3402
475
b3c56df7
RH		 476static void tcg_out_insn_3403(TCGContext *s, AArch64Insn insn, TCGType ext,
477 TCGReg rd, TCGReg rn, TCGReg rm, int imms)
478{
479 tcg_out32(s, insn | ext << 31 | ext << 22 | rm << 16 | imms << 10
480 | rn << 5 | rd);
481}
482
582ab779
RH		 483/* This function is used for the Move (wide immediate) instruction group.
484 Note that SHIFT is a full shift count, not the 2 bit HW field. */
485static void tcg_out_insn_3405(TCGContext *s, AArch64Insn insn, TCGType ext,
486 TCGReg rd, uint16_t half, unsigned shift)
487{
488 assert((shift & ~0x30) == 0);
489 tcg_out32(s, insn | ext << 31 | shift << (21 - 4) | half << 5 | rd);
490}
491
c6e310d9
RH		 492static void tcg_out_insn_3406(TCGContext *s, AArch64Insn insn,
493 TCGReg rd, int64_t disp)
494{
495 tcg_out32(s, insn | (disp & 3) << 29 | (disp & 0x1ffffc) << (5 - 2) | rd);
496}
497
50573c66
RH		 498/* This function is for both 3.5.2 (Add/Subtract shifted register), for
499 the rare occasion when we actually want to supply a shift amount. */
500static inline void tcg_out_insn_3502S(TCGContext *s, AArch64Insn insn,
501 TCGType ext, TCGReg rd, TCGReg rn,
502 TCGReg rm, int imm6)
503{
504 tcg_out32(s, insn | ext << 31 | rm << 16 | imm6 << 10 | rn << 5 | rd);
505}
506
507/* This function is for 3.5.2 (Add/subtract shifted register),
508 and 3.5.10 (Logical shifted register), for the vast majorty of cases
509 when we don't want to apply a shift. Thus it can also be used for
510 3.5.3 (Add/subtract with carry) and 3.5.8 (Data processing 2 source). */
511static void tcg_out_insn_3502(TCGContext *s, AArch64Insn insn, TCGType ext,
512 TCGReg rd, TCGReg rn, TCGReg rm)
513{
514 tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd);
515}
516
517#define tcg_out_insn_3503 tcg_out_insn_3502
518#define tcg_out_insn_3508 tcg_out_insn_3502
519#define tcg_out_insn_3510 tcg_out_insn_3502
520
04ce397b
RH		 521static void tcg_out_insn_3506(TCGContext *s, AArch64Insn insn, TCGType ext,
522 TCGReg rd, TCGReg rn, TCGReg rm, TCGCond c)
523{
524 tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd
525 | tcg_cond_to_aarch64[c] << 12);
526}
527
8678b71c
RH		 528static void tcg_out_insn_3509(TCGContext *s, AArch64Insn insn, TCGType ext,
529 TCGReg rd, TCGReg rn, TCGReg rm, TCGReg ra)
530{
531 tcg_out32(s, insn | ext << 31 | rm << 16 | ra << 10 | rn << 5 | rd);
532}
533
50573c66 534
4a136e0a
CF		 535static inline void tcg_out_ldst_9(TCGContext *s,
536 enum aarch64_ldst_op_data op_data,
537 enum aarch64_ldst_op_type op_type,
8bf56493 538 TCGReg rd, TCGReg rn, intptr_t offset)
4a136e0a
CF		 539{
540 /* use LDUR with BASE register with 9bit signed unscaled offset */
523fdc08
RH		 541 tcg_out32(s, op_data << 24 | op_type << 20
542 | (offset & 0x1ff) << 12 | rn << 5 | rd);
4a136e0a
CF		 543}
544
b1f6dc0d
CF		 545/* tcg_out_ldst_12 expects a scaled unsigned immediate offset */
546static inline void tcg_out_ldst_12(TCGContext *s,
547 enum aarch64_ldst_op_data op_data,
548 enum aarch64_ldst_op_type op_type,
549 TCGReg rd, TCGReg rn,
550 tcg_target_ulong scaled_uimm)
551{
552 tcg_out32(s, (op_data | 1) << 24
553 | op_type << 20 | scaled_uimm << 10 | rn << 5 | rd);
554}
555
7d11fc7c
RH		 556/* Register to register move using ORR (shifted register with no shift). */
557static void tcg_out_movr(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rm)
4a136e0a 558{
7d11fc7c
RH		 559 tcg_out_insn(s, 3510, ORR, ext, rd, TCG_REG_XZR, rm);
560}
561
562/* Register to register move using ADDI (move to/from SP). */
563static void tcg_out_movr_sp(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rn)
564{
565 tcg_out_insn(s, 3401, ADDI, ext, rd, rn, 0);
4a136e0a
CF		 566}
567
4ec4f0bd
RH		 568/* This function is used for the Logical (immediate) instruction group.
569 The value of LIMM must satisfy IS_LIMM. See the comment above about
570 only supporting simplified logical immediates. */
571static void tcg_out_logicali(TCGContext *s, AArch64Insn insn, TCGType ext,
572 TCGReg rd, TCGReg rn, uint64_t limm)
573{
574 unsigned h, l, r, c;
575
576 assert(is_limm(limm));
577
578 h = clz64(limm);
579 l = ctz64(limm);
580 if (l == 0) {
581 r = 0; /* form 0....01....1 */
582 c = ctz64(~limm) - 1;
583 if (h == 0) {
584 r = clz64(~limm); /* form 1..10..01..1 */
585 c += r;
586 }
587 } else {
588 r = 64 - l; /* form 1....10....0 or 0..01..10..0 */
589 c = r - h - 1;
590 }
591 if (ext == TCG_TYPE_I32) {
592 r &= 31;
593 c &= 31;
594 }
595
596 tcg_out_insn_3404(s, insn, ext, rd, rn, ext, r, c);
597}
598
582ab779
RH		 599static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
600 tcg_target_long value)
4a136e0a 601{
582ab779 602 AArch64Insn insn;
dfeb5fe7
RH		 603 int i, wantinv, shift;
604 tcg_target_long svalue = value;
605 tcg_target_long ivalue = ~value;
606 tcg_target_long imask;
607
608 /* For 32-bit values, discard potential garbage in value. For 64-bit
609 values within [2**31, 2**32-1], we can create smaller sequences by
610 interpreting this as a negative 32-bit number, while ensuring that
611 the high 32 bits are cleared by setting SF=0. */
612 if (type == TCG_TYPE_I32 || (value & ~0xffffffffull) == 0) {
613 svalue = (int32_t)value;
582ab779 614 value = (uint32_t)value;
dfeb5fe7
RH		 615 ivalue = (uint32_t)ivalue;
616 type = TCG_TYPE_I32;
617 }
618
d8918df5
RH		 619 /* Speed things up by handling the common case of small positive
620 and negative values specially. */
621 if ((value & ~0xffffull) == 0) {
622 tcg_out_insn(s, 3405, MOVZ, type, rd, value, 0);
623 return;
624 } else if ((ivalue & ~0xffffull) == 0) {
625 tcg_out_insn(s, 3405, MOVN, type, rd, ivalue, 0);
626 return;
627 }
628
4ec4f0bd
RH		 629 /* Check for bitfield immediates. For the benefit of 32-bit quantities,
630 use the sign-extended value. That lets us match rotated values such
631 as 0xff0000ff with the same 64-bit logic matching 0xffffffffff0000ff. */
632 if (is_limm(svalue)) {
633 tcg_out_logicali(s, I3404_ORRI, type, rd, TCG_REG_XZR, svalue);
634 return;
635 }
636
c6e310d9
RH		 637 /* Look for host pointer values within 4G of the PC. This happens
638 often when loading pointers to QEMU's own data structures. */
639 if (type == TCG_TYPE_I64) {
640 tcg_target_long disp = (value >> 12) - ((intptr_t)s->code_ptr >> 12);
641 if (disp == sextract64(disp, 0, 21)) {
642 tcg_out_insn(s, 3406, ADRP, rd, disp);
643 if (value & 0xfff) {
644 tcg_out_insn(s, 3401, ADDI, type, rd, rd, value & 0xfff);
645 }
646 return;
647 }
648 }
649
dfeb5fe7
RH		 650 /* Would it take fewer insns to begin with MOVN? For the value and its
651 inverse, count the number of 16-bit lanes that are 0. */
652 for (i = wantinv = imask = 0; i < 64; i += 16) {
653 tcg_target_long mask = 0xffffull << i;
654 if ((value & mask) == 0) {
655 wantinv -= 1;
656 }
657 if ((ivalue & mask) == 0) {
658 wantinv += 1;
659 imask |= mask;
660 }
582ab779
RH		 661 }
662
dfeb5fe7 663 /* If we had more 0xffff than 0x0000, invert VALUE and use MOVN. */
582ab779 664 insn = I3405_MOVZ;
dfeb5fe7
RH		 665 if (wantinv > 0) {
666 value = ivalue;
667 insn = I3405_MOVN;
668 }
669
670 /* Find the lowest lane that is not 0x0000. */
671 shift = ctz64(value) & (63 & -16);
672 tcg_out_insn_3405(s, insn, type, rd, value >> shift, shift);
673
674 if (wantinv > 0) {
675 /* Re-invert the value, so MOVK sees non-inverted bits. */
676 value = ~value;
677 /* Clear out all the 0xffff lanes. */
678 value ^= imask;
679 }
680 /* Clear out the lane that we just set. */
681 value &= ~(0xffffUL << shift);
682
683 /* Iterate until all lanes have been set, and thus cleared from VALUE. */
684 while (value) {
685 shift = ctz64(value) & (63 & -16);
686 tcg_out_insn(s, 3405, MOVK, type, rd, value >> shift, shift);
4a136e0a 687 value &= ~(0xffffUL << shift);
dfeb5fe7 688 }
4a136e0a
CF		 689}
690
4a136e0a
CF		 691static inline void tcg_out_ldst_r(TCGContext *s,
692 enum aarch64_ldst_op_data op_data,
693 enum aarch64_ldst_op_type op_type,
694 TCGReg rd, TCGReg base, TCGReg regoff)
695{
696 /* load from memory to register using base + 64bit register offset */
697 /* using f.e. STR Wt, [Xn, Xm] 0xb8600800|(regoff << 16)|(base << 5)|rd */
698 /* the 0x6000 is for the "no extend field" */
699 tcg_out32(s, 0x00206800
700 | op_data << 24 | op_type << 20 | regoff << 16 | base << 5 | rd);
701}
702
703/* solve the whole ldst problem */
704static inline void tcg_out_ldst(TCGContext *s, enum aarch64_ldst_op_data data,
705 enum aarch64_ldst_op_type type,
8bf56493 706 TCGReg rd, TCGReg rn, intptr_t offset)
4a136e0a
CF		 707{
708 if (offset >= -256 && offset < 256) {
709 tcg_out_ldst_9(s, data, type, rd, rn, offset);
b1f6dc0d 710 return;
4a136e0a 711 }
b1f6dc0d
CF		 712
713 if (offset >= 256) {
714 /* if the offset is naturally aligned and in range,
715 then we can use the scaled uimm12 encoding */
716 unsigned int s_bits = data >> 6;
717 if (!(offset & ((1 << s_bits) - 1))) {
718 tcg_target_ulong scaled_uimm = offset >> s_bits;
719 if (scaled_uimm <= 0xfff) {
720 tcg_out_ldst_12(s, data, type, rd, rn, scaled_uimm);
721 return;
722 }
723 }
724 }
725
726 /* worst-case scenario, move offset to temp register, use reg offset */
727 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, offset);
728 tcg_out_ldst_r(s, data, type, rd, rn, TCG_REG_TMP);
4a136e0a
CF		 729}
730
4a136e0a
CF		 731static inline void tcg_out_mov(TCGContext *s,
732 TCGType type, TCGReg ret, TCGReg arg)
733{
734 if (ret != arg) {
929f8b55 735 tcg_out_movr(s, type, ret, arg);
4a136e0a
CF		 736 }
737}
738
739static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
a05b5b9b 740 TCGReg arg1, intptr_t arg2)
4a136e0a
CF		 741{
742 tcg_out_ldst(s, (type == TCG_TYPE_I64) ? LDST_64 : LDST_32, LDST_LD,
743 arg, arg1, arg2);
744}
745
746static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
a05b5b9b 747 TCGReg arg1, intptr_t arg2)
4a136e0a
CF		 748{
749 tcg_out_ldst(s, (type == TCG_TYPE_I64) ? LDST_64 : LDST_32, LDST_ST,
750 arg, arg1, arg2);
751}
752
b3c56df7
RH		 753static inline void tcg_out_bfm(TCGContext *s, TCGType ext, TCGReg rd,
754 TCGReg rn, unsigned int a, unsigned int b)
755{
756 tcg_out_insn(s, 3402, BFM, ext, rd, rn, ext, a, b);
757}
758
7763ffa0
RH		 759static inline void tcg_out_ubfm(TCGContext *s, TCGType ext, TCGReg rd,
760 TCGReg rn, unsigned int a, unsigned int b)
4a136e0a 761{
b3c56df7 762 tcg_out_insn(s, 3402, UBFM, ext, rd, rn, ext, a, b);
4a136e0a
CF		 763}
764
7763ffa0
RH		 765static inline void tcg_out_sbfm(TCGContext *s, TCGType ext, TCGReg rd,
766 TCGReg rn, unsigned int a, unsigned int b)
4a136e0a 767{
b3c56df7 768 tcg_out_insn(s, 3402, SBFM, ext, rd, rn, ext, a, b);
4a136e0a
CF		 769}
770
7763ffa0 771static inline void tcg_out_extr(TCGContext *s, TCGType ext, TCGReg rd,
4a136e0a
CF		 772 TCGReg rn, TCGReg rm, unsigned int a)
773{
b3c56df7 774 tcg_out_insn(s, 3403, EXTR, ext, rd, rn, rm, a);
4a136e0a
CF		 775}
776
7763ffa0 777static inline void tcg_out_shl(TCGContext *s, TCGType ext,
4a136e0a
CF		 778 TCGReg rd, TCGReg rn, unsigned int m)
779{
b3c56df7
RH		 780 int bits = ext ? 64 : 32;
781 int max = bits - 1;
4a136e0a
CF		 782 tcg_out_ubfm(s, ext, rd, rn, bits - (m & max), max - (m & max));
783}
784
7763ffa0 785static inline void tcg_out_shr(TCGContext *s, TCGType ext,
4a136e0a
CF		 786 TCGReg rd, TCGReg rn, unsigned int m)
787{
788 int max = ext ? 63 : 31;
789 tcg_out_ubfm(s, ext, rd, rn, m & max, max);
790}
791
7763ffa0 792static inline void tcg_out_sar(TCGContext *s, TCGType ext,
4a136e0a
CF		 793 TCGReg rd, TCGReg rn, unsigned int m)
794{
795 int max = ext ? 63 : 31;
796 tcg_out_sbfm(s, ext, rd, rn, m & max, max);
797}
798
7763ffa0 799static inline void tcg_out_rotr(TCGContext *s, TCGType ext,
4a136e0a
CF		 800 TCGReg rd, TCGReg rn, unsigned int m)
801{
802 int max = ext ? 63 : 31;
803 tcg_out_extr(s, ext, rd, rn, rn, m & max);
804}
805
7763ffa0 806static inline void tcg_out_rotl(TCGContext *s, TCGType ext,
4a136e0a
CF		 807 TCGReg rd, TCGReg rn, unsigned int m)
808{
b3c56df7
RH		 809 int bits = ext ? 64 : 32;
810 int max = bits - 1;
4a136e0a
CF		 811 tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
812}
813
b3c56df7
RH		 814static inline void tcg_out_dep(TCGContext *s, TCGType ext, TCGReg rd,
815 TCGReg rn, unsigned lsb, unsigned width)
816{
817 unsigned size = ext ? 64 : 32;
818 unsigned a = (size - lsb) & (size - 1);
819 unsigned b = width - 1;
820 tcg_out_bfm(s, ext, rd, rn, a, b);
821}
822
90f1cd91
RH		 823static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a,
824 tcg_target_long b, bool const_b)
4a136e0a 825{
90f1cd91
RH		 826 if (const_b) {
827 /* Using CMP or CMN aliases. */
828 if (b >= 0) {
829 tcg_out_insn(s, 3401, SUBSI, ext, TCG_REG_XZR, a, b);
830 } else {
831 tcg_out_insn(s, 3401, ADDSI, ext, TCG_REG_XZR, a, -b);
832 }
833 } else {
834 /* Using CMP alias SUBS wzr, Wn, Wm */
835 tcg_out_insn(s, 3502, SUBS, ext, TCG_REG_XZR, a, b);
836 }
4a136e0a
CF		 837}
838
017a86f7 839static inline void tcg_out_goto(TCGContext *s, intptr_t target)
4a136e0a 840{
017a86f7 841 intptr_t offset = (target - (intptr_t)s->code_ptr) / 4;
4a136e0a
CF		 842
843 if (offset < -0x02000000 || offset >= 0x02000000) {
844 /* out of 26bit range */
845 tcg_abort();
846 }
847
81d8a5ee 848 tcg_out_insn(s, 3206, B, offset);
4a136e0a
CF		 849}
850
851static inline void tcg_out_goto_noaddr(TCGContext *s)
852{
81d8a5ee
RH		 853 /* We pay attention here to not modify the branch target by reading from
854 the buffer. This ensure that caches and memory are kept coherent during
855 retranslation. Mask away possible garbage in the high bits for the
856 first translation, while keeping the offset bits for retranslation. */
857 uint32_t old = tcg_in32(s);
858 tcg_out_insn(s, 3206, B, old);
4a136e0a
CF		 859}
860
861static inline void tcg_out_goto_cond_noaddr(TCGContext *s, TCGCond c)
862{
81d8a5ee
RH		 863 /* See comments in tcg_out_goto_noaddr. */
864 uint32_t old = tcg_in32(s) >> 5;
865 tcg_out_insn(s, 3202, B_C, c, old);
4a136e0a
CF		 866}
867
017a86f7 868static inline void tcg_out_goto_cond(TCGContext *s, TCGCond c, intptr_t target)
4a136e0a 869{
017a86f7 870 intptr_t offset = (target - (intptr_t)s->code_ptr) / 4;
4a136e0a
CF		 871
872 if (offset < -0x40000 || offset >= 0x40000) {
873 /* out of 19bit range */
874 tcg_abort();
875 }
876
81d8a5ee 877 tcg_out_insn(s, 3202, B_C, c, offset);
4a136e0a
CF		 878}
879
880static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
881{
81d8a5ee 882 tcg_out_insn(s, 3207, BLR, reg);
4a136e0a
CF		 883}
884
017a86f7 885static inline void tcg_out_call(TCGContext *s, intptr_t target)
4a136e0a 886{
017a86f7 887 intptr_t offset = (target - (intptr_t)s->code_ptr) / 4;
4a136e0a
CF		 888
889 if (offset < -0x02000000 || offset >= 0x02000000) { /* out of 26bit rng */
890 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, target);
891 tcg_out_callr(s, TCG_REG_TMP);
892 } else {
81d8a5ee 893 tcg_out_insn(s, 3206, BL, offset);
4a136e0a
CF		 894 }
895}
896
4a136e0a
CF		 897void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
898{
017a86f7
RH		 899 intptr_t target = addr;
900 intptr_t offset = (target - (intptr_t)jmp_addr) / 4;
4a136e0a
CF		 901
902 if (offset < -0x02000000 || offset >= 0x02000000) {
903 /* out of 26bit range */
904 tcg_abort();
905 }
906
907 patch_reloc((uint8_t *)jmp_addr, R_AARCH64_JUMP26, target, 0);
908 flush_icache_range(jmp_addr, jmp_addr + 4);
909}
910
911static inline void tcg_out_goto_label(TCGContext *s, int label_index)
912{
913 TCGLabel *l = &s->labels[label_index];
914
915 if (!l->has_value) {
916 tcg_out_reloc(s, s->code_ptr, R_AARCH64_JUMP26, label_index, 0);
917 tcg_out_goto_noaddr(s);
918 } else {
919 tcg_out_goto(s, l->u.value);
920 }
921}
922
923static inline void tcg_out_goto_label_cond(TCGContext *s,
924 TCGCond c, int label_index)
925{
926 TCGLabel *l = &s->labels[label_index];
927
928 if (!l->has_value) {
929 tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, label_index, 0);
930 tcg_out_goto_cond_noaddr(s, c);
931 } else {
932 tcg_out_goto_cond(s, c, l->u.value);
933 }
934}
935
7763ffa0
RH		 936static inline void tcg_out_rev(TCGContext *s, TCGType ext,
937 TCGReg rd, TCGReg rm)
9c4a059d
CF		 938{
939 /* using REV 0x5ac00800 */
940 unsigned int base = ext ? 0xdac00c00 : 0x5ac00800;
941 tcg_out32(s, base | rm << 5 | rd);
942}
943
7763ffa0
RH		 944static inline void tcg_out_rev16(TCGContext *s, TCGType ext,
945 TCGReg rd, TCGReg rm)
9c4a059d
CF		 946{
947 /* using REV16 0x5ac00400 */
948 unsigned int base = ext ? 0xdac00400 : 0x5ac00400;
949 tcg_out32(s, base | rm << 5 | rd);
950}
951
929f8b55 952static inline void tcg_out_sxt(TCGContext *s, TCGType ext, TCGMemOp s_bits,
31f1275b
CF		 953 TCGReg rd, TCGReg rn)
954{
b3c56df7 955 /* Using ALIASes SXTB, SXTH, SXTW, of SBFM Xd, Xn, #0, #7|15|31 */
929f8b55 956 int bits = (8 << s_bits) - 1;
31f1275b
CF		 957 tcg_out_sbfm(s, ext, rd, rn, 0, bits);
958}
959
929f8b55 960static inline void tcg_out_uxt(TCGContext *s, TCGMemOp s_bits,
31f1275b
CF		 961 TCGReg rd, TCGReg rn)
962{
b3c56df7 963 /* Using ALIASes UXTB, UXTH of UBFM Wd, Wn, #0, #7|15 */
929f8b55 964 int bits = (8 << s_bits) - 1;
31f1275b
CF		 965 tcg_out_ubfm(s, 0, rd, rn, 0, bits);
966}
967
90f1cd91
RH		 968static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
969 TCGReg rn, int64_t aimm)
970{
971 if (aimm >= 0) {
972 tcg_out_insn(s, 3401, ADDI, ext, rd, rn, aimm);
973 } else {
974 tcg_out_insn(s, 3401, SUBI, ext, rd, rn, -aimm);
975 }
976}
977
c6e929e7
RH		 978static inline void tcg_out_addsub2(TCGContext *s, int ext, TCGReg rl,
979 TCGReg rh, TCGReg al, TCGReg ah,
980 tcg_target_long bl, tcg_target_long bh,
981 bool const_bl, bool const_bh, bool sub)
982{
983 TCGReg orig_rl = rl;
984 AArch64Insn insn;
985
986 if (rl == ah || (!const_bh && rl == bh)) {
987 rl = TCG_REG_TMP;
988 }
989
990 if (const_bl) {
991 insn = I3401_ADDSI;
992 if ((bl < 0) ^ sub) {
993 insn = I3401_SUBSI;
994 bl = -bl;
995 }
996 tcg_out_insn_3401(s, insn, ext, rl, al, bl);
997 } else {
998 tcg_out_insn_3502(s, sub ? I3502_SUBS : I3502_ADDS, ext, rl, al, bl);
999 }
1000
1001 insn = I3503_ADC;
1002 if (const_bh) {
1003 /* Note that the only two constants we support are 0 and -1, and
1004 that SBC = rn + ~rm + c, so adc -1 is sbc 0, and vice-versa. */
1005 if ((bh != 0) ^ sub) {
1006 insn = I3503_SBC;
1007 }
1008 bh = TCG_REG_XZR;
1009 } else if (sub) {
1010 insn = I3503_SBC;
1011 }
1012 tcg_out_insn_3503(s, insn, ext, rh, ah, bh);
1013
1014 if (rl != orig_rl) {
1015 tcg_out_movr(s, ext, orig_rl, rl);
1016 }
1017}
1018
4a136e0a 1019#ifdef CONFIG_SOFTMMU
023261ef
RH		 1020/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
1021 * int mmu_idx, uintptr_t ra)
1022 */
4a136e0a 1023static const void * const qemu_ld_helpers[4] = {
023261ef
RH		 1024 helper_ret_ldub_mmu,
1025 helper_ret_lduw_mmu,
1026 helper_ret_ldul_mmu,
1027 helper_ret_ldq_mmu,
4a136e0a
CF		 1028};
1029
023261ef
RH		 1030/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
1031 * uintxx_t val, int mmu_idx, uintptr_t ra)
1032 */
4a136e0a 1033static const void * const qemu_st_helpers[4] = {
023261ef
RH		 1034 helper_ret_stb_mmu,
1035 helper_ret_stw_mmu,
1036 helper_ret_stl_mmu,
1037 helper_ret_stq_mmu,
4a136e0a
CF		 1038};
1039
c6d8ed24
JK		 1040static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1041{
929f8b55
RH		 1042 TCGMemOp opc = lb->opc;
1043 TCGMemOp size = opc & MO_SIZE;
1044
017a86f7
RH		 1045 reloc_pc19(lb->label_ptr[0], (intptr_t)s->code_ptr);
1046
929f8b55
RH		 1047 tcg_out_movr(s, TCG_TYPE_I64, TCG_REG_X0, TCG_AREG0);
1048 tcg_out_movr(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
c6d8ed24 1049 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, lb->mem_index);
8bf56493 1050 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_X3, (intptr_t)lb->raddr);
929f8b55 1051 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)qemu_ld_helpers[size]);
c6d8ed24 1052 tcg_out_callr(s, TCG_REG_TMP);
929f8b55
RH		 1053 if (opc & MO_SIGN) {
1054 tcg_out_sxt(s, TCG_TYPE_I64, size, lb->datalo_reg, TCG_REG_X0);
c6d8ed24 1055 } else {
929f8b55 1056 tcg_out_movr(s, TCG_TYPE_I64, lb->datalo_reg, TCG_REG_X0);
c6d8ed24
JK		 1057 }
1058
017a86f7 1059 tcg_out_goto(s, (intptr_t)lb->raddr);
c6d8ed24
JK		 1060}
1061
1062static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1063{
929f8b55
RH		 1064 TCGMemOp size = lb->opc;
1065
017a86f7 1066 reloc_pc19(lb->label_ptr[0], (intptr_t)s->code_ptr);
c6d8ed24 1067
929f8b55
RH		 1068 tcg_out_movr(s, TCG_TYPE_I64, TCG_REG_X0, TCG_AREG0);
1069 tcg_out_movr(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
1070 tcg_out_movr(s, size == MO_64, TCG_REG_X2, lb->datalo_reg);
c6d8ed24 1071 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, lb->mem_index);
017a86f7 1072 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_X4, (intptr_t)lb->raddr);
929f8b55 1073 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)qemu_st_helpers[size]);
c6d8ed24 1074 tcg_out_callr(s, TCG_REG_TMP);
8bf56493 1075 tcg_out_goto(s, (intptr_t)lb->raddr);
c6d8ed24
JK		 1076}
1077
c6d8ed24
JK		 1078static void add_qemu_ldst_label(TCGContext *s, int is_ld, int opc,
1079 TCGReg data_reg, TCGReg addr_reg,
1080 int mem_index,
1081 uint8_t *raddr, uint8_t *label_ptr)
1082{
9ecefc84 1083 TCGLabelQemuLdst *label = new_ldst_label(s);
c6d8ed24 1084
c6d8ed24
JK		 1085 label->is_ld = is_ld;
1086 label->opc = opc;
1087 label->datalo_reg = data_reg;
1088 label->addrlo_reg = addr_reg;
1089 label->mem_index = mem_index;
1090 label->raddr = raddr;
1091 label->label_ptr[0] = label_ptr;
1092}
1093
1094/* Load and compare a TLB entry, emitting the conditional jump to the
1095 slow path for the failure case, which will be patched later when finalizing
1096 the slow path. Generated code returns the host addend in X1,
1097 clobbers X0,X2,X3,TMP. */
1098static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg,
1099 int s_bits, uint8_t **label_ptr, int mem_index, int is_read)
1100{
1101 TCGReg base = TCG_AREG0;
1102 int tlb_offset = is_read ?
1103 offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
1104 : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
1105 /* Extract the TLB index from the address into X0.
1106 X0<CPU_TLB_BITS:0> =
1107 addr_reg<TARGET_PAGE_BITS+CPU_TLB_BITS:TARGET_PAGE_BITS> */
1108 tcg_out_ubfm(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, addr_reg,
1109 TARGET_PAGE_BITS, TARGET_PAGE_BITS + CPU_TLB_BITS);
1110 /* Store the page mask part of the address and the low s_bits into X3.
1111 Later this allows checking for equality and alignment at the same time.
1112 X3 = addr_reg & (PAGE_MASK | ((1 << s_bits) - 1)) */
e029f293
RH		 1113 tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64, TCG_REG_X3,
1114 addr_reg, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
c6d8ed24 1115 /* Add any "high bits" from the tlb offset to the env address into X2,
096c46c0 1116 to take advantage of the LSL12 form of the ADDI instruction.
c6d8ed24 1117 X2 = env + (tlb_offset & 0xfff000) */
096c46c0
RH		 1118 tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_X2, base,
1119 tlb_offset & 0xfff000);
c6d8ed24
JK		 1120 /* Merge the tlb index contribution into X2.
1121 X2 = X2 + (X0 << CPU_TLB_ENTRY_BITS) */
50573c66
RH		 1122 tcg_out_insn(s, 3502S, ADD_LSL, 1, TCG_REG_X2, TCG_REG_X2,
1123 TCG_REG_X0, CPU_TLB_ENTRY_BITS);
c6d8ed24
JK		 1124 /* Merge "low bits" from tlb offset, load the tlb comparator into X0.
1125 X0 = load [X2 + (tlb_offset & 0x000fff)] */
1126 tcg_out_ldst(s, TARGET_LONG_BITS == 64 ? LDST_64 : LDST_32,
1127 LDST_LD, TCG_REG_X0, TCG_REG_X2,
1128 (tlb_offset & 0xfff));
1129 /* Load the tlb addend. Do that early to avoid stalling.
1130 X1 = load [X2 + (tlb_offset & 0xfff) + offsetof(addend)] */
1131 tcg_out_ldst(s, LDST_64, LDST_LD, TCG_REG_X1, TCG_REG_X2,
1132 (tlb_offset & 0xfff) + (offsetof(CPUTLBEntry, addend)) -
1133 (is_read ? offsetof(CPUTLBEntry, addr_read)
1134 : offsetof(CPUTLBEntry, addr_write)));
1135 /* Perform the address comparison. */
90f1cd91 1136 tcg_out_cmp(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, TCG_REG_X3, 0);
c6d8ed24
JK		 1137 *label_ptr = s->code_ptr;
1138 /* If not equal, we jump to the slow path. */
1139 tcg_out_goto_cond_noaddr(s, TCG_COND_NE);
1140}
1141
1142#endif /* CONFIG_SOFTMMU */
6a91c7c9
JK		 1143
1144static void tcg_out_qemu_ld_direct(TCGContext *s, int opc, TCGReg data_r,
1145 TCGReg addr_r, TCGReg off_r)
1146{
1147 switch (opc) {
1148 case 0:
1149 tcg_out_ldst_r(s, LDST_8, LDST_LD, data_r, addr_r, off_r);
1150 break;
1151 case 0 | 4:
1152 tcg_out_ldst_r(s, LDST_8, LDST_LD_S_X, data_r, addr_r, off_r);
1153 break;
1154 case 1:
1155 tcg_out_ldst_r(s, LDST_16, LDST_LD, data_r, addr_r, off_r);
1156 if (TCG_LDST_BSWAP) {
929f8b55 1157 tcg_out_rev16(s, TCG_TYPE_I32, data_r, data_r);
6a91c7c9
JK		 1158 }
1159 break;
1160 case 1 | 4:
1161 if (TCG_LDST_BSWAP) {
1162 tcg_out_ldst_r(s, LDST_16, LDST_LD, data_r, addr_r, off_r);
929f8b55
RH		 1163 tcg_out_rev16(s, TCG_TYPE_I32, data_r, data_r);
1164 tcg_out_sxt(s, TCG_TYPE_I64, MO_16, data_r, data_r);
6a91c7c9
JK		 1165 } else {
1166 tcg_out_ldst_r(s, LDST_16, LDST_LD_S_X, data_r, addr_r, off_r);
1167 }
1168 break;
1169 case 2:
1170 tcg_out_ldst_r(s, LDST_32, LDST_LD, data_r, addr_r, off_r);
1171 if (TCG_LDST_BSWAP) {
929f8b55 1172 tcg_out_rev(s, TCG_TYPE_I32, data_r, data_r);
6a91c7c9
JK		 1173 }
1174 break;
1175 case 2 | 4:
1176 if (TCG_LDST_BSWAP) {
1177 tcg_out_ldst_r(s, LDST_32, LDST_LD, data_r, addr_r, off_r);
929f8b55
RH		 1178 tcg_out_rev(s, TCG_TYPE_I32, data_r, data_r);
1179 tcg_out_sxt(s, TCG_TYPE_I64, MO_32, data_r, data_r);
6a91c7c9
JK		 1180 } else {
1181 tcg_out_ldst_r(s, LDST_32, LDST_LD_S_X, data_r, addr_r, off_r);
1182 }
1183 break;
1184 case 3:
1185 tcg_out_ldst_r(s, LDST_64, LDST_LD, data_r, addr_r, off_r);
1186 if (TCG_LDST_BSWAP) {
929f8b55 1187 tcg_out_rev(s, TCG_TYPE_I64, data_r, data_r);
6a91c7c9
JK		 1188 }
1189 break;
1190 default:
1191 tcg_abort();
1192 }
1193}
1194
1195static void tcg_out_qemu_st_direct(TCGContext *s, int opc, TCGReg data_r,
1196 TCGReg addr_r, TCGReg off_r)
1197{
1198 switch (opc) {
1199 case 0:
1200 tcg_out_ldst_r(s, LDST_8, LDST_ST, data_r, addr_r, off_r);
1201 break;
1202 case 1:
1203 if (TCG_LDST_BSWAP) {
929f8b55 1204 tcg_out_rev16(s, TCG_TYPE_I32, TCG_REG_TMP, data_r);
6a91c7c9
JK		 1205 tcg_out_ldst_r(s, LDST_16, LDST_ST, TCG_REG_TMP, addr_r, off_r);
1206 } else {
1207 tcg_out_ldst_r(s, LDST_16, LDST_ST, data_r, addr_r, off_r);
1208 }
1209 break;
1210 case 2:
1211 if (TCG_LDST_BSWAP) {
929f8b55 1212 tcg_out_rev(s, TCG_TYPE_I32, TCG_REG_TMP, data_r);
6a91c7c9
JK		 1213 tcg_out_ldst_r(s, LDST_32, LDST_ST, TCG_REG_TMP, addr_r, off_r);
1214 } else {
1215 tcg_out_ldst_r(s, LDST_32, LDST_ST, data_r, addr_r, off_r);
1216 }
1217 break;
1218 case 3:
1219 if (TCG_LDST_BSWAP) {
929f8b55 1220 tcg_out_rev(s, TCG_TYPE_I64, TCG_REG_TMP, data_r);
6a91c7c9
JK		 1221 tcg_out_ldst_r(s, LDST_64, LDST_ST, TCG_REG_TMP, addr_r, off_r);
1222 } else {
1223 tcg_out_ldst_r(s, LDST_64, LDST_ST, data_r, addr_r, off_r);
1224 }
1225 break;
1226 default:
1227 tcg_abort();
1228 }
1229}
4a136e0a
CF		 1230
1231static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
1232{
1233 TCGReg addr_reg, data_reg;
1234#ifdef CONFIG_SOFTMMU
1235 int mem_index, s_bits;
c6d8ed24 1236 uint8_t *label_ptr;
4a136e0a
CF		 1237#endif
1238 data_reg = args[0];
1239 addr_reg = args[1];
1240
1241#ifdef CONFIG_SOFTMMU
1242 mem_index = args[2];
1243 s_bits = opc & 3;
c6d8ed24
JK		 1244 tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 1);
1245 tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, TCG_REG_X1);
1246 add_qemu_ldst_label(s, 1, opc, data_reg, addr_reg,
1247 mem_index, s->code_ptr, label_ptr);
4a136e0a 1248#else /* !CONFIG_SOFTMMU */
6a91c7c9
JK		 1249 tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg,
1250 GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR);
1251#endif /* CONFIG_SOFTMMU */
4a136e0a
CF		 1252}
1253
1254static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
1255{
1256 TCGReg addr_reg, data_reg;
1257#ifdef CONFIG_SOFTMMU
1258 int mem_index, s_bits;
c6d8ed24 1259 uint8_t *label_ptr;
4a136e0a
CF		 1260#endif
1261 data_reg = args[0];
1262 addr_reg = args[1];
1263
1264#ifdef CONFIG_SOFTMMU
1265 mem_index = args[2];
1266 s_bits = opc & 3;
1267
c6d8ed24
JK		 1268 tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 0);
1269 tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, TCG_REG_X1);
1270 add_qemu_ldst_label(s, 0, opc, data_reg, addr_reg,
1271 mem_index, s->code_ptr, label_ptr);
4a136e0a 1272#else /* !CONFIG_SOFTMMU */
6a91c7c9
JK		 1273 tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg,
1274 GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR);
1275#endif /* CONFIG_SOFTMMU */
4a136e0a
CF		 1276}
1277
1278static uint8_t *tb_ret_addr;
1279
1280/* callee stack use example:
1281 stp x29, x30, [sp,#-32]!
1282 mov x29, sp
1283 stp x1, x2, [sp,#16]
1284 ...
1285 ldp x1, x2, [sp,#16]
1286 ldp x29, x30, [sp],#32
1287 ret
1288*/
1289
1290/* push r1 and r2, and alloc stack space for a total of
1291 alloc_n elements (1 element=16 bytes, must be between 1 and 31. */
1292static inline void tcg_out_push_pair(TCGContext *s, TCGReg addr,
1293 TCGReg r1, TCGReg r2, int alloc_n)
1294{
1295 /* using indexed scaled simm7 STP 0x28800000 | (ext) | 0x01000000 (pre-idx)
1296 | alloc_n * (-1) << 16 | r2 << 10 | addr << 5 | r1 */
1297 assert(alloc_n > 0 && alloc_n < 0x20);
1298 alloc_n = (-alloc_n) & 0x3f;
1299 tcg_out32(s, 0xa9800000 | alloc_n << 16 | r2 << 10 | addr << 5 | r1);
1300}
1301
1302/* dealloc stack space for a total of alloc_n elements and pop r1, r2. */
1303static inline void tcg_out_pop_pair(TCGContext *s, TCGReg addr,
1304 TCGReg r1, TCGReg r2, int alloc_n)
1305{
1306 /* using indexed scaled simm7 LDP 0x28c00000 | (ext) | nothing (post-idx)
1307 | alloc_n << 16 | r2 << 10 | addr << 5 | r1 */
1308 assert(alloc_n > 0 && alloc_n < 0x20);
1309 tcg_out32(s, 0xa8c00000 | alloc_n << 16 | r2 << 10 | addr << 5 | r1);
1310}
1311
1312static inline void tcg_out_store_pair(TCGContext *s, TCGReg addr,
1313 TCGReg r1, TCGReg r2, int idx)
1314{
1315 /* using register pair offset simm7 STP 0x29000000 | (ext)
1316 | idx << 16 | r2 << 10 | addr << 5 | r1 */
1317 assert(idx > 0 && idx < 0x20);
1318 tcg_out32(s, 0xa9000000 | idx << 16 | r2 << 10 | addr << 5 | r1);
1319}
1320
1321static inline void tcg_out_load_pair(TCGContext *s, TCGReg addr,
1322 TCGReg r1, TCGReg r2, int idx)
1323{
1324 /* using register pair offset simm7 LDP 0x29400000 | (ext)
1325 | idx << 16 | r2 << 10 | addr << 5 | r1 */
1326 assert(idx > 0 && idx < 0x20);
1327 tcg_out32(s, 0xa9400000 | idx << 16 | r2 << 10 | addr << 5 | r1);
1328}
1329
1330static void tcg_out_op(TCGContext *s, TCGOpcode opc,
8d8db193
RH		 1331 const TCGArg args[TCG_MAX_OP_ARGS],
1332 const int const_args[TCG_MAX_OP_ARGS])
4a136e0a 1333{
f0293414
RH		 1334 /* 99% of the time, we can signal the use of extension registers
1335 by looking to see if the opcode handles 64-bit data. */
1336 TCGType ext = (tcg_op_defs[opc].flags & TCG_OPF_64BIT) != 0;
4a136e0a 1337
8d8db193
RH		 1338 /* Hoist the loads of the most common arguments. */
1339 TCGArg a0 = args[0];
1340 TCGArg a1 = args[1];
1341 TCGArg a2 = args[2];
1342 int c2 = const_args[2];
1343
04ce397b
RH		 1344 /* Some operands are defined with "rZ" constraint, a register or
1345 the zero register. These need not actually test args[I] == 0. */
1346#define REG0(I) (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
1347
4a136e0a
CF		 1348 switch (opc) {
1349 case INDEX_op_exit_tb:
8d8db193 1350 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
017a86f7 1351 tcg_out_goto(s, (intptr_t)tb_ret_addr);
4a136e0a
CF		 1352 break;
1353
1354 case INDEX_op_goto_tb:
1355#ifndef USE_DIRECT_JUMP
1356#error "USE_DIRECT_JUMP required for aarch64"
1357#endif
1358 assert(s->tb_jmp_offset != NULL); /* consistency for USE_DIRECT_JUMP */
8d8db193 1359 s->tb_jmp_offset[a0] = s->code_ptr - s->code_buf;
4a136e0a
CF		 1360 /* actual branch destination will be patched by
1361 aarch64_tb_set_jmp_target later, beware retranslation. */
1362 tcg_out_goto_noaddr(s);
8d8db193 1363 s->tb_next_offset[a0] = s->code_ptr - s->code_buf;
4a136e0a
CF		 1364 break;
1365
1366 case INDEX_op_call:
1367 if (const_args[0]) {
8d8db193 1368 tcg_out_call(s, a0);
4a136e0a 1369 } else {
8d8db193 1370 tcg_out_callr(s, a0);
4a136e0a
CF		 1371 }
1372 break;
1373
1374 case INDEX_op_br:
8d8db193 1375 tcg_out_goto_label(s, a0);
4a136e0a
CF		 1376 break;
1377
1378 case INDEX_op_ld_i32:
1379 case INDEX_op_ld_i64:
1380 case INDEX_op_st_i32:
1381 case INDEX_op_st_i64:
1382 case INDEX_op_ld8u_i32:
1383 case INDEX_op_ld8s_i32:
1384 case INDEX_op_ld16u_i32:
1385 case INDEX_op_ld16s_i32:
1386 case INDEX_op_ld8u_i64:
1387 case INDEX_op_ld8s_i64:
1388 case INDEX_op_ld16u_i64:
1389 case INDEX_op_ld16s_i64:
1390 case INDEX_op_ld32u_i64:
1391 case INDEX_op_ld32s_i64:
1392 case INDEX_op_st8_i32:
1393 case INDEX_op_st8_i64:
1394 case INDEX_op_st16_i32:
1395 case INDEX_op_st16_i64:
1396 case INDEX_op_st32_i64:
1397 tcg_out_ldst(s, aarch64_ldst_get_data(opc), aarch64_ldst_get_type(opc),
8d8db193 1398 a0, a1, a2);
4a136e0a
CF		 1399 break;
1400
4a136e0a 1401 case INDEX_op_add_i32:
90f1cd91
RH		 1402 a2 = (int32_t)a2;
1403 /* FALLTHRU */
1404 case INDEX_op_add_i64:
1405 if (c2) {
1406 tcg_out_addsubi(s, ext, a0, a1, a2);
1407 } else {
1408 tcg_out_insn(s, 3502, ADD, ext, a0, a1, a2);
1409 }
4a136e0a
CF		 1410 break;
1411
4a136e0a 1412 case INDEX_op_sub_i32:
90f1cd91
RH		 1413 a2 = (int32_t)a2;
1414 /* FALLTHRU */
1415 case INDEX_op_sub_i64:
1416 if (c2) {
1417 tcg_out_addsubi(s, ext, a0, a1, -a2);
1418 } else {
1419 tcg_out_insn(s, 3502, SUB, ext, a0, a1, a2);
1420 }
4a136e0a
CF		 1421 break;
1422
14b155dd
RH		 1423 case INDEX_op_neg_i64:
1424 case INDEX_op_neg_i32:
1425 tcg_out_insn(s, 3502, SUB, ext, a0, TCG_REG_XZR, a1);
1426 break;
1427
4a136e0a 1428 case INDEX_op_and_i32:
e029f293
RH		 1429 a2 = (int32_t)a2;
1430 /* FALLTHRU */
1431 case INDEX_op_and_i64:
1432 if (c2) {
1433 tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, a2);
1434 } else {
1435 tcg_out_insn(s, 3510, AND, ext, a0, a1, a2);
1436 }
4a136e0a
CF		 1437 break;
1438
14b155dd
RH		 1439 case INDEX_op_andc_i32:
1440 a2 = (int32_t)a2;
1441 /* FALLTHRU */
1442 case INDEX_op_andc_i64:
1443 if (c2) {
1444 tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, ~a2);
1445 } else {
1446 tcg_out_insn(s, 3510, BIC, ext, a0, a1, a2);
1447 }
1448 break;
1449
4a136e0a 1450 case INDEX_op_or_i32:
e029f293
RH		 1451 a2 = (int32_t)a2;
1452 /* FALLTHRU */
1453 case INDEX_op_or_i64:
1454 if (c2) {
1455 tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, a2);
1456 } else {
1457 tcg_out_insn(s, 3510, ORR, ext, a0, a1, a2);
1458 }
4a136e0a
CF		 1459 break;
1460
14b155dd
RH		 1461 case INDEX_op_orc_i32:
1462 a2 = (int32_t)a2;
1463 /* FALLTHRU */
1464 case INDEX_op_orc_i64:
1465 if (c2) {
1466 tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, ~a2);
1467 } else {
1468 tcg_out_insn(s, 3510, ORN, ext, a0, a1, a2);
1469 }
1470 break;
1471
4a136e0a 1472 case INDEX_op_xor_i32:
e029f293
RH		 1473 a2 = (int32_t)a2;
1474 /* FALLTHRU */
1475 case INDEX_op_xor_i64:
1476 if (c2) {
1477 tcg_out_logicali(s, I3404_EORI, ext, a0, a1, a2);
1478 } else {
1479 tcg_out_insn(s, 3510, EOR, ext, a0, a1, a2);
1480 }
4a136e0a
CF		 1481 break;
1482
14b155dd
RH		 1483 case INDEX_op_eqv_i32:
1484 a2 = (int32_t)a2;
1485 /* FALLTHRU */
1486 case INDEX_op_eqv_i64:
1487 if (c2) {
1488 tcg_out_logicali(s, I3404_EORI, ext, a0, a1, ~a2);
1489 } else {
1490 tcg_out_insn(s, 3510, EON, ext, a0, a1, a2);
1491 }
1492 break;
1493
1494 case INDEX_op_not_i64:
1495 case INDEX_op_not_i32:
1496 tcg_out_insn(s, 3510, ORN, ext, a0, TCG_REG_XZR, a1);
1497 break;
1498
4a136e0a 1499 case INDEX_op_mul_i64:
4a136e0a 1500 case INDEX_op_mul_i32:
8678b71c
RH		 1501 tcg_out_insn(s, 3509, MADD, ext, a0, a1, a2, TCG_REG_XZR);
1502 break;
1503
1504 case INDEX_op_div_i64:
1505 case INDEX_op_div_i32:
1506 tcg_out_insn(s, 3508, SDIV, ext, a0, a1, a2);
1507 break;
1508 case INDEX_op_divu_i64:
1509 case INDEX_op_divu_i32:
1510 tcg_out_insn(s, 3508, UDIV, ext, a0, a1, a2);
1511 break;
1512
1513 case INDEX_op_rem_i64:
1514 case INDEX_op_rem_i32:
1515 tcg_out_insn(s, 3508, SDIV, ext, TCG_REG_TMP, a1, a2);
1516 tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
1517 break;
1518 case INDEX_op_remu_i64:
1519 case INDEX_op_remu_i32:
1520 tcg_out_insn(s, 3508, UDIV, ext, TCG_REG_TMP, a1, a2);
1521 tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
4a136e0a
CF		 1522 break;
1523
1524 case INDEX_op_shl_i64:
4a136e0a 1525 case INDEX_op_shl_i32:
df9351e3 1526 if (c2) {
8d8db193 1527 tcg_out_shl(s, ext, a0, a1, a2);
df9351e3
RH		 1528 } else {
1529 tcg_out_insn(s, 3508, LSLV, ext, a0, a1, a2);
4a136e0a
CF		 1530 }
1531 break;
1532
1533 case INDEX_op_shr_i64:
4a136e0a 1534 case INDEX_op_shr_i32:
df9351e3 1535 if (c2) {
8d8db193 1536 tcg_out_shr(s, ext, a0, a1, a2);
df9351e3
RH		 1537 } else {
1538 tcg_out_insn(s, 3508, LSRV, ext, a0, a1, a2);
4a136e0a
CF		 1539 }
1540 break;
1541
1542 case INDEX_op_sar_i64:
4a136e0a 1543 case INDEX_op_sar_i32:
df9351e3 1544 if (c2) {
8d8db193 1545 tcg_out_sar(s, ext, a0, a1, a2);
df9351e3
RH		 1546 } else {
1547 tcg_out_insn(s, 3508, ASRV, ext, a0, a1, a2);
4a136e0a
CF		 1548 }
1549 break;
1550
1551 case INDEX_op_rotr_i64:
4a136e0a 1552 case INDEX_op_rotr_i32:
df9351e3 1553 if (c2) {
8d8db193 1554 tcg_out_rotr(s, ext, a0, a1, a2);
df9351e3
RH		 1555 } else {
1556 tcg_out_insn(s, 3508, RORV, ext, a0, a1, a2);
4a136e0a
CF		 1557 }
1558 break;
1559
1560 case INDEX_op_rotl_i64:
df9351e3
RH		 1561 case INDEX_op_rotl_i32:
1562 if (c2) {
8d8db193 1563 tcg_out_rotl(s, ext, a0, a1, a2);
4a136e0a 1564 } else {
50573c66 1565 tcg_out_insn(s, 3502, SUB, 0, TCG_REG_TMP, TCG_REG_XZR, a2);
df9351e3 1566 tcg_out_insn(s, 3508, RORV, ext, a0, a1, TCG_REG_TMP);
4a136e0a
CF		 1567 }
1568 break;
1569
8d8db193 1570 case INDEX_op_brcond_i32:
90f1cd91
RH		 1571 a1 = (int32_t)a1;
1572 /* FALLTHRU */
1573 case INDEX_op_brcond_i64:
1574 tcg_out_cmp(s, ext, a0, a1, const_args[1]);
8d8db193 1575 tcg_out_goto_label_cond(s, a2, args[3]);
4a136e0a
CF		 1576 break;
1577
4a136e0a 1578 case INDEX_op_setcond_i32:
90f1cd91
RH		 1579 a2 = (int32_t)a2;
1580 /* FALLTHRU */
1581 case INDEX_op_setcond_i64:
1582 tcg_out_cmp(s, ext, a1, a2, c2);
ed7a0aa8
RH		 1583 /* Use CSET alias of CSINC Wd, WZR, WZR, invert(cond). */
1584 tcg_out_insn(s, 3506, CSINC, TCG_TYPE_I32, a0, TCG_REG_XZR,
1585 TCG_REG_XZR, tcg_invert_cond(args[3]));
4a136e0a
CF		 1586 break;
1587
04ce397b
RH		 1588 case INDEX_op_movcond_i32:
1589 a2 = (int32_t)a2;
1590 /* FALLTHRU */
1591 case INDEX_op_movcond_i64:
1592 tcg_out_cmp(s, ext, a1, a2, c2);
1593 tcg_out_insn(s, 3506, CSEL, ext, a0, REG0(3), REG0(4), args[5]);
1594 break;
1595
4a136e0a
CF		 1596 case INDEX_op_qemu_ld8u:
1597 tcg_out_qemu_ld(s, args, 0 | 0);
1598 break;
1599 case INDEX_op_qemu_ld8s:
1600 tcg_out_qemu_ld(s, args, 4 | 0);
1601 break;
1602 case INDEX_op_qemu_ld16u:
1603 tcg_out_qemu_ld(s, args, 0 | 1);
1604 break;
1605 case INDEX_op_qemu_ld16s:
1606 tcg_out_qemu_ld(s, args, 4 | 1);
1607 break;
1608 case INDEX_op_qemu_ld32u:
1609 tcg_out_qemu_ld(s, args, 0 | 2);
1610 break;
1611 case INDEX_op_qemu_ld32s:
1612 tcg_out_qemu_ld(s, args, 4 | 2);
1613 break;
1614 case INDEX_op_qemu_ld32:
1615 tcg_out_qemu_ld(s, args, 0 | 2);
1616 break;
1617 case INDEX_op_qemu_ld64:
1618 tcg_out_qemu_ld(s, args, 0 | 3);
1619 break;
1620 case INDEX_op_qemu_st8:
1621 tcg_out_qemu_st(s, args, 0);
1622 break;
1623 case INDEX_op_qemu_st16:
1624 tcg_out_qemu_st(s, args, 1);
1625 break;
1626 case INDEX_op_qemu_st32:
1627 tcg_out_qemu_st(s, args, 2);
1628 break;
1629 case INDEX_op_qemu_st64:
1630 tcg_out_qemu_st(s, args, 3);
1631 break;
1632
9c4a059d 1633 case INDEX_op_bswap32_i64:
f0293414
RH		 1634 /* Despite the _i64, this is a 32-bit bswap. */
1635 ext = 0;
1636 /* FALLTHRU */
1637 case INDEX_op_bswap64_i64:
9c4a059d 1638 case INDEX_op_bswap32_i32:
8d8db193 1639 tcg_out_rev(s, ext, a0, a1);
9c4a059d
CF		 1640 break;
1641 case INDEX_op_bswap16_i64:
1642 case INDEX_op_bswap16_i32:
929f8b55 1643 tcg_out_rev16(s, TCG_TYPE_I32, a0, a1);
9c4a059d
CF		 1644 break;
1645
31f1275b 1646 case INDEX_op_ext8s_i64:
31f1275b 1647 case INDEX_op_ext8s_i32:
929f8b55 1648 tcg_out_sxt(s, ext, MO_8, a0, a1);
31f1275b
CF		 1649 break;
1650 case INDEX_op_ext16s_i64:
31f1275b 1651 case INDEX_op_ext16s_i32:
929f8b55 1652 tcg_out_sxt(s, ext, MO_16, a0, a1);
31f1275b
CF		 1653 break;
1654 case INDEX_op_ext32s_i64:
929f8b55 1655 tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
31f1275b
CF		 1656 break;
1657 case INDEX_op_ext8u_i64:
1658 case INDEX_op_ext8u_i32:
929f8b55 1659 tcg_out_uxt(s, MO_8, a0, a1);
31f1275b
CF		 1660 break;
1661 case INDEX_op_ext16u_i64:
1662 case INDEX_op_ext16u_i32:
929f8b55 1663 tcg_out_uxt(s, MO_16, a0, a1);
31f1275b
CF		 1664 break;
1665 case INDEX_op_ext32u_i64:
929f8b55 1666 tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
31f1275b
CF		 1667 break;
1668
b3c56df7
RH		 1669 case INDEX_op_deposit_i64:
1670 case INDEX_op_deposit_i32:
1671 tcg_out_dep(s, ext, a0, REG0(2), args[3], args[4]);
1672 break;
1673
c6e929e7
RH		 1674 case INDEX_op_add2_i32:
1675 tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
1676 (int32_t)args[4], args[5], const_args[4],
1677 const_args[5], false);
1678 break;
1679 case INDEX_op_add2_i64:
1680 tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
1681 args[5], const_args[4], const_args[5], false);
1682 break;
1683 case INDEX_op_sub2_i32:
1684 tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
1685 (int32_t)args[4], args[5], const_args[4],
1686 const_args[5], true);
1687 break;
1688 case INDEX_op_sub2_i64:
1689 tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
1690 args[5], const_args[4], const_args[5], true);
1691 break;
1692
1fcc9ddf
RH		 1693 case INDEX_op_muluh_i64:
1694 tcg_out_insn(s, 3508, UMULH, TCG_TYPE_I64, a0, a1, a2);
1695 break;
1696 case INDEX_op_mulsh_i64:
1697 tcg_out_insn(s, 3508, SMULH, TCG_TYPE_I64, a0, a1, a2);
1698 break;
1699
a51a6b6a
RH		 1700 case INDEX_op_mov_i64:
1701 case INDEX_op_mov_i32:
1702 case INDEX_op_movi_i64:
1703 case INDEX_op_movi_i32:
1704 /* Always implemented with tcg_out_mov/i, never with tcg_out_op. */
4a136e0a 1705 default:
a51a6b6a
RH		 1706 /* Opcode not implemented. */
1707 tcg_abort();
4a136e0a 1708 }
04ce397b
RH		 1709
1710#undef REG0
4a136e0a
CF		 1711}
1712
1713static const TCGTargetOpDef aarch64_op_defs[] = {
1714 { INDEX_op_exit_tb, { } },
1715 { INDEX_op_goto_tb, { } },
1716 { INDEX_op_call, { "ri" } },
1717 { INDEX_op_br, { } },
1718
1719 { INDEX_op_mov_i32, { "r", "r" } },
1720 { INDEX_op_mov_i64, { "r", "r" } },
1721
1722 { INDEX_op_movi_i32, { "r" } },
1723 { INDEX_op_movi_i64, { "r" } },
1724
1725 { INDEX_op_ld8u_i32, { "r", "r" } },
1726 { INDEX_op_ld8s_i32, { "r", "r" } },
1727 { INDEX_op_ld16u_i32, { "r", "r" } },
1728 { INDEX_op_ld16s_i32, { "r", "r" } },
1729 { INDEX_op_ld_i32, { "r", "r" } },
1730 { INDEX_op_ld8u_i64, { "r", "r" } },
1731 { INDEX_op_ld8s_i64, { "r", "r" } },
1732 { INDEX_op_ld16u_i64, { "r", "r" } },
1733 { INDEX_op_ld16s_i64, { "r", "r" } },
1734 { INDEX_op_ld32u_i64, { "r", "r" } },
1735 { INDEX_op_ld32s_i64, { "r", "r" } },
1736 { INDEX_op_ld_i64, { "r", "r" } },
1737
1738 { INDEX_op_st8_i32, { "r", "r" } },
1739 { INDEX_op_st16_i32, { "r", "r" } },
1740 { INDEX_op_st_i32, { "r", "r" } },
1741 { INDEX_op_st8_i64, { "r", "r" } },
1742 { INDEX_op_st16_i64, { "r", "r" } },
1743 { INDEX_op_st32_i64, { "r", "r" } },
1744 { INDEX_op_st_i64, { "r", "r" } },
1745
90f1cd91
RH		 1746 { INDEX_op_add_i32, { "r", "r", "rwA" } },
1747 { INDEX_op_add_i64, { "r", "r", "rA" } },
1748 { INDEX_op_sub_i32, { "r", "r", "rwA" } },
1749 { INDEX_op_sub_i64, { "r", "r", "rA" } },
4a136e0a
CF		 1750 { INDEX_op_mul_i32, { "r", "r", "r" } },
1751 { INDEX_op_mul_i64, { "r", "r", "r" } },
8678b71c
RH		 1752 { INDEX_op_div_i32, { "r", "r", "r" } },
1753 { INDEX_op_div_i64, { "r", "r", "r" } },
1754 { INDEX_op_divu_i32, { "r", "r", "r" } },
1755 { INDEX_op_divu_i64, { "r", "r", "r" } },
1756 { INDEX_op_rem_i32, { "r", "r", "r" } },
1757 { INDEX_op_rem_i64, { "r", "r", "r" } },
1758 { INDEX_op_remu_i32, { "r", "r", "r" } },
1759 { INDEX_op_remu_i64, { "r", "r", "r" } },
e029f293
RH		 1760 { INDEX_op_and_i32, { "r", "r", "rwL" } },
1761 { INDEX_op_and_i64, { "r", "r", "rL" } },
1762 { INDEX_op_or_i32, { "r", "r", "rwL" } },
1763 { INDEX_op_or_i64, { "r", "r", "rL" } },
1764 { INDEX_op_xor_i32, { "r", "r", "rwL" } },
1765 { INDEX_op_xor_i64, { "r", "r", "rL" } },
14b155dd
RH		 1766 { INDEX_op_andc_i32, { "r", "r", "rwL" } },
1767 { INDEX_op_andc_i64, { "r", "r", "rL" } },
1768 { INDEX_op_orc_i32, { "r", "r", "rwL" } },
1769 { INDEX_op_orc_i64, { "r", "r", "rL" } },
1770 { INDEX_op_eqv_i32, { "r", "r", "rwL" } },
1771 { INDEX_op_eqv_i64, { "r", "r", "rL" } },
1772
1773 { INDEX_op_neg_i32, { "r", "r" } },
1774 { INDEX_op_neg_i64, { "r", "r" } },
1775 { INDEX_op_not_i32, { "r", "r" } },
1776 { INDEX_op_not_i64, { "r", "r" } },
4a136e0a
CF		 1777
1778 { INDEX_op_shl_i32, { "r", "r", "ri" } },
1779 { INDEX_op_shr_i32, { "r", "r", "ri" } },
1780 { INDEX_op_sar_i32, { "r", "r", "ri" } },
1781 { INDEX_op_rotl_i32, { "r", "r", "ri" } },
1782 { INDEX_op_rotr_i32, { "r", "r", "ri" } },
1783 { INDEX_op_shl_i64, { "r", "r", "ri" } },
1784 { INDEX_op_shr_i64, { "r", "r", "ri" } },
1785 { INDEX_op_sar_i64, { "r", "r", "ri" } },
1786 { INDEX_op_rotl_i64, { "r", "r", "ri" } },
1787 { INDEX_op_rotr_i64, { "r", "r", "ri" } },
1788
90f1cd91
RH		 1789 { INDEX_op_brcond_i32, { "r", "rwA" } },
1790 { INDEX_op_brcond_i64, { "r", "rA" } },
1791 { INDEX_op_setcond_i32, { "r", "r", "rwA" } },
1792 { INDEX_op_setcond_i64, { "r", "r", "rA" } },
04ce397b
RH		 1793 { INDEX_op_movcond_i32, { "r", "r", "rwA", "rZ", "rZ" } },
1794 { INDEX_op_movcond_i64, { "r", "r", "rA", "rZ", "rZ" } },
4a136e0a
CF		 1795
1796 { INDEX_op_qemu_ld8u, { "r", "l" } },
1797 { INDEX_op_qemu_ld8s, { "r", "l" } },
1798 { INDEX_op_qemu_ld16u, { "r", "l" } },
1799 { INDEX_op_qemu_ld16s, { "r", "l" } },
1800 { INDEX_op_qemu_ld32u, { "r", "l" } },
1801 { INDEX_op_qemu_ld32s, { "r", "l" } },
1802
1803 { INDEX_op_qemu_ld32, { "r", "l" } },
1804 { INDEX_op_qemu_ld64, { "r", "l" } },
1805
1806 { INDEX_op_qemu_st8, { "l", "l" } },
1807 { INDEX_op_qemu_st16, { "l", "l" } },
1808 { INDEX_op_qemu_st32, { "l", "l" } },
1809 { INDEX_op_qemu_st64, { "l", "l" } },
9c4a059d
CF		 1810
1811 { INDEX_op_bswap16_i32, { "r", "r" } },
1812 { INDEX_op_bswap32_i32, { "r", "r" } },
1813 { INDEX_op_bswap16_i64, { "r", "r" } },
1814 { INDEX_op_bswap32_i64, { "r", "r" } },
1815 { INDEX_op_bswap64_i64, { "r", "r" } },
1816
31f1275b
CF		 1817 { INDEX_op_ext8s_i32, { "r", "r" } },
1818 { INDEX_op_ext16s_i32, { "r", "r" } },
1819 { INDEX_op_ext8u_i32, { "r", "r" } },
1820 { INDEX_op_ext16u_i32, { "r", "r" } },
1821
1822 { INDEX_op_ext8s_i64, { "r", "r" } },
1823 { INDEX_op_ext16s_i64, { "r", "r" } },
1824 { INDEX_op_ext32s_i64, { "r", "r" } },
1825 { INDEX_op_ext8u_i64, { "r", "r" } },
1826 { INDEX_op_ext16u_i64, { "r", "r" } },
1827 { INDEX_op_ext32u_i64, { "r", "r" } },
1828
b3c56df7
RH		 1829 { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
1830 { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
1831
c6e929e7
RH		 1832 { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rwA", "rwMZ" } },
1833 { INDEX_op_add2_i64, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
1834 { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rwA", "rwMZ" } },
1835 { INDEX_op_sub2_i64, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
1836
1fcc9ddf
RH		 1837 { INDEX_op_muluh_i64, { "r", "r", "r" } },
1838 { INDEX_op_mulsh_i64, { "r", "r", "r" } },
1839
4a136e0a
CF		 1840 { -1 },
1841};
1842
1843static void tcg_target_init(TCGContext *s)
1844{
4a136e0a
CF		 1845 tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
1846 tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffffffff);
1847
1848 tcg_regset_set32(tcg_target_call_clobber_regs, 0,
1849 (1 << TCG_REG_X0) | (1 << TCG_REG_X1) |
1850 (1 << TCG_REG_X2) | (1 << TCG_REG_X3) |
1851 (1 << TCG_REG_X4) | (1 << TCG_REG_X5) |
1852 (1 << TCG_REG_X6) | (1 << TCG_REG_X7) |
1853 (1 << TCG_REG_X8) | (1 << TCG_REG_X9) |
1854 (1 << TCG_REG_X10) | (1 << TCG_REG_X11) |
1855 (1 << TCG_REG_X12) | (1 << TCG_REG_X13) |
1856 (1 << TCG_REG_X14) | (1 << TCG_REG_X15) |
1857 (1 << TCG_REG_X16) | (1 << TCG_REG_X17) |
1858 (1 << TCG_REG_X18));
1859
1860 tcg_regset_clear(s->reserved_regs);
1861 tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);
1862 tcg_regset_set_reg(s->reserved_regs, TCG_REG_FP);
1863 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP);
1864 tcg_regset_set_reg(s->reserved_regs, TCG_REG_X18); /* platform register */
1865
1866 tcg_add_target_add_op_defs(aarch64_op_defs);
1867}
1868
4a136e0a
CF		 1869static void tcg_target_qemu_prologue(TCGContext *s)
1870{
1871 /* NB: frame sizes are in 16 byte stack units! */
1872 int frame_size_callee_saved, frame_size_tcg_locals;
1873 TCGReg r;
1874
1875 /* save pairs (FP, LR) and (X19, X20) .. (X27, X28) */
1876 frame_size_callee_saved = (1) + (TCG_REG_X28 - TCG_REG_X19) / 2 + 1;
1877
1878 /* frame size requirement for TCG local variables */
1879 frame_size_tcg_locals = TCG_STATIC_CALL_ARGS_SIZE
1880 + CPU_TEMP_BUF_NLONGS * sizeof(long)
1881 + (TCG_TARGET_STACK_ALIGN - 1);
1882 frame_size_tcg_locals &= ~(TCG_TARGET_STACK_ALIGN - 1);
1883 frame_size_tcg_locals /= TCG_TARGET_STACK_ALIGN;
1884
1885 /* push (FP, LR) and update sp */
1886 tcg_out_push_pair(s, TCG_REG_SP,
1887 TCG_REG_FP, TCG_REG_LR, frame_size_callee_saved);
1888
1889 /* FP -> callee_saved */
929f8b55 1890 tcg_out_movr_sp(s, TCG_TYPE_I64, TCG_REG_FP, TCG_REG_SP);
4a136e0a
CF		 1891
1892 /* store callee-preserved regs x19..x28 using FP -> callee_saved */
1893 for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
1894 int idx = (r - TCG_REG_X19) / 2 + 1;
1895 tcg_out_store_pair(s, TCG_REG_FP, r, r + 1, idx);
1896 }
1897
096c46c0
RH		 1898 /* Make stack space for TCG locals. */
1899 tcg_out_insn(s, 3401, SUBI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
4a136e0a 1900 frame_size_tcg_locals * TCG_TARGET_STACK_ALIGN);
096c46c0 1901
4a136e0a
CF		 1902 /* inform TCG about how to find TCG locals with register, offset, size */
1903 tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
1904 CPU_TEMP_BUF_NLONGS * sizeof(long));
1905
6a91c7c9
JK		 1906#if defined(CONFIG_USE_GUEST_BASE)
1907 if (GUEST_BASE) {
1908 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, GUEST_BASE);
1909 tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE);
1910 }
1911#endif
1912
4a136e0a 1913 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
81d8a5ee 1914 tcg_out_insn(s, 3207, BR, tcg_target_call_iarg_regs[1]);
4a136e0a
CF		 1915
1916 tb_ret_addr = s->code_ptr;
1917
096c46c0
RH		 1918 /* Remove TCG locals stack space. */
1919 tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
4a136e0a
CF		 1920 frame_size_tcg_locals * TCG_TARGET_STACK_ALIGN);
1921
1922 /* restore registers x19..x28.
1923 FP must be preserved, so it still points to callee_saved area */
1924 for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
1925 int idx = (r - TCG_REG_X19) / 2 + 1;
1926 tcg_out_load_pair(s, TCG_REG_FP, r, r + 1, idx);
1927 }
1928
1929 /* pop (FP, LR), restore SP to previous frame, return */
1930 tcg_out_pop_pair(s, TCG_REG_SP,
1931 TCG_REG_FP, TCG_REG_LR, frame_size_callee_saved);
81d8a5ee 1932 tcg_out_insn(s, 3207, RET, TCG_REG_LR);
4a136e0a 1933}
QEMU mirror