]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'pull-aspeed-20240201' of https://github.com/legoater/qemu into staging
[mirror_qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
757e725b 25#include "qemu/osdep.h"
cca82982 26
813da627
RH		 27/* Define to jump the ELF file used to communicate with GDB. */
28#undef DEBUG_JIT
29
72fd2efb 30#include "qemu/error-report.h"
f348b6d1 31#include "qemu/cutils.h"
1de7afc9 32#include "qemu/host-utils.h"
d4c51a0a 33#include "qemu/qemu-print.h"
084cfca1 34#include "qemu/cacheflush.h"
ad768e6f 35#include "qemu/cacheinfo.h"
533206f0 36#include "qemu/timer.h"
cac9b0fd 37#include "exec/translation-block.h"
d0a9bb5e 38#include "exec/tlb-common.h"
d7ec12f8 39#include "tcg/startup.h"
ad3d0e4d 40#include "tcg/tcg-op-common.h"
813da627 41
edee2579 42#if UINTPTR_MAX == UINT32_MAX
813da627 43# define ELF_CLASS ELFCLASS32
edee2579
RH		 44#else
45# define ELF_CLASS ELFCLASS64
813da627 46#endif
e03b5686 47#if HOST_BIG_ENDIAN
813da627
RH		 48# define ELF_DATA ELFDATA2MSB
49#else
50# define ELF_DATA ELFDATA2LSB
51#endif
52
c896fe29 53#include "elf.h"
508127e2 54#include "exec/log.h"
d2ba8026 55#include "tcg/tcg-ldst.h"
47f7313d 56#include "tcg/tcg-temp-internal.h"
5ff7258c 57#include "tcg-internal.h"
327b75a4 58#include "tcg/perf.h"
7d478306
RH		 59#ifdef CONFIG_USER_ONLY
60#include "exec/user/guest-base.h"
61#endif
c896fe29 62
139c1837 63/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 64 used here. */
e4d58b41
RH		 65static void tcg_target_init(TCGContext *s);
66static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 67static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 68 intptr_t value, intptr_t addend);
c896fe29 69
497a22eb
RH		 70/* The CIE and FDE header definitions will be common to all hosts. */
71typedef struct {
72 uint32_t len __attribute__((aligned((sizeof(void *)))));
73 uint32_t id;
74 uint8_t version;
75 char augmentation[1];
76 uint8_t code_align;
77 uint8_t data_align;
78 uint8_t return_column;
79} DebugFrameCIE;
80
81typedef struct QEMU_PACKED {
82 uint32_t len __attribute__((aligned((sizeof(void *)))));
83 uint32_t cie_offset;
edee2579
RH		 84 uintptr_t func_start;
85 uintptr_t func_len;
497a22eb
RH		 86} DebugFrameFDEHeader;
87
2c90784a
RH		 88typedef struct QEMU_PACKED {
89 DebugFrameCIE cie;
90 DebugFrameFDEHeader fde;
91} DebugFrameHeader;
92
2528f771
RH		 93typedef struct TCGLabelQemuLdst {
94 bool is_ld; /* qemu_ld: true, qemu_st: false */
95 MemOpIdx oi;
96 TCGType type; /* result type of a load */
97 TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */
98 TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */
99 TCGReg datalo_reg; /* reg index for low word to be loaded or stored */
100 TCGReg datahi_reg; /* reg index for high word to be loaded or stored */
101 const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */
102 tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
103 QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
104} TCGLabelQemuLdst;
105
755bf9e5 106static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 107 const void *debug_frame,
108 size_t debug_frame_size)
813da627
RH		 109 __attribute__((unused));
110
139c1837 111/* Forward declarations for functions declared and used in tcg-target.c.inc. */
9358fbbf 112static void tcg_out_tb_start(TCGContext *s);
2a534aff 113static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 114 intptr_t arg2);
78113e83 115static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 116static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 117 TCGReg ret, tcg_target_long arg);
678155b2 118static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
753e42ea 119static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
d0e66c89 120static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
379afdff 121static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
52bf3398 122static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
9ecf5f61 123static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
9c6aa274 124static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
b9bfe000 125static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
b8b94ac6 126static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
313bdea8 127static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
129f1f9e 128static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2);
b55a8d9d 129static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
cf7d6b8e 130static void tcg_out_goto_tb(TCGContext *s, int which);
5e8892db
MR		 131static void tcg_out_op(TCGContext *s, TCGOpcode opc,
132 const TCGArg args[TCG_MAX_OP_ARGS],
133 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 134#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 135static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
136 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 137static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
138 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 139static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
140 TCGReg dst, int64_t arg);
5e8892db
MR		 141static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
142 unsigned vecl, unsigned vece,
143 const TCGArg args[TCG_MAX_OP_ARGS],
144 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 145#else
e7632cfa
RH		 146static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
147 TCGReg dst, TCGReg src)
148{
149 g_assert_not_reached();
150}
d6ecb4a9
RH		 151static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
152 TCGReg dst, TCGReg base, intptr_t offset)
153{
154 g_assert_not_reached();
155}
4e186175
RH		 156static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
157 TCGReg dst, int64_t arg)
e7632cfa
RH		 158{
159 g_assert_not_reached();
160}
5e8892db
MR		 161static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
162 unsigned vecl, unsigned vece,
163 const TCGArg args[TCG_MAX_OP_ARGS],
164 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 165{
166 g_assert_not_reached();
167}
168#endif
2a534aff 169static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 170 intptr_t arg2);
59d7c14e
RH		 171static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
172 TCGReg base, intptr_t ofs);
7b7d8b2d 173static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
cee44b03 174 const TCGHelperInfo *info);
5e3d0c19 175static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
ebe92db2 176static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece);
659ef5cb 177#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 178static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 179#endif
c896fe29 180
23088ca0
RH		 181#ifndef CONFIG_USER_ONLY
182#define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; })
183#endif
184
8429a1ca
RH		 185typedef struct TCGLdstHelperParam {
186 TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg);
187 unsigned ntmp;
188 int tmp[3];
189} TCGLdstHelperParam;
190
191static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
192 const TCGLdstHelperParam *p)
193 __attribute__((unused));
194static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l,
195 bool load_sign, const TCGLdstHelperParam *p)
196 __attribute__((unused));
197static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
198 const TCGLdstHelperParam *p)
199 __attribute__((unused));
200
de95016d 201static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = {
0cadc1ed
RH		 202 [MO_UB] = helper_ldub_mmu,
203 [MO_SB] = helper_ldsb_mmu,
204 [MO_UW] = helper_lduw_mmu,
205 [MO_SW] = helper_ldsw_mmu,
206 [MO_UL] = helper_ldul_mmu,
207 [MO_UQ] = helper_ldq_mmu,
208#if TCG_TARGET_REG_BITS == 64
209 [MO_SL] = helper_ldsl_mmu,
ebebea53 210 [MO_128] = helper_ld16_mmu,
0cadc1ed
RH		 211#endif
212};
213
de95016d 214static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = {
0cadc1ed
RH		 215 [MO_8] = helper_stb_mmu,
216 [MO_16] = helper_stw_mmu,
217 [MO_32] = helper_stl_mmu,
218 [MO_64] = helper_stq_mmu,
ebebea53
RH		 219#if TCG_TARGET_REG_BITS == 64
220 [MO_128] = helper_st16_mmu,
221#endif
0cadc1ed 222};
0cadc1ed 223
e63b8a29
RH		 224typedef struct {
225 MemOp atom; /* lg2 bits of atomicity required */
226 MemOp align; /* lg2 bits of alignment to use */
227} TCGAtomAlign;
228
229static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
230 MemOp host_atom, bool allow_two_ops)
231 __attribute__((unused));
232
397cabaa
RH		 233#ifdef CONFIG_USER_ONLY
234bool tcg_use_softmmu;
235#endif
236
42eb6dfc
RH		 237TCGContext tcg_init_ctx;
238__thread TCGContext *tcg_ctx;
239
5ff7258c 240TCGContext **tcg_ctxs;
0e2d61cf
RH		 241unsigned int tcg_cur_ctxs;
242unsigned int tcg_max_ctxs;
ad75a51e 243TCGv_env tcg_env;
c8bc1168 244const void *tcg_code_gen_epilogue;
db0c51a3 245uintptr_t tcg_splitwx_diff;
df2cce29 246
b91ccb31
RH		 247#ifndef CONFIG_TCG_INTERPRETER
248tcg_prologue_fn *tcg_qemu_tb_exec;
249#endif
250
d2fd745f 251static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 252static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 253
1813e175 254#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 255static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 256{
257 *s->code_ptr++ = v;
258}
259
4196dca6
PM		 260static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
261 uint8_t v)
5c53bb81 262{
1813e175 263 *p = v;
5c53bb81 264}
1813e175 265#endif
5c53bb81 266
1813e175 267#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 268static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 269{
1813e175
RH		 270 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
271 *s->code_ptr++ = v;
272 } else {
273 tcg_insn_unit *p = s->code_ptr;
274 memcpy(p, &v, sizeof(v));
275 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
276 }
c896fe29
FB		 277}
278
4196dca6
PM		 279static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
280 uint16_t v)
5c53bb81 281{
1813e175
RH		 282 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
283 *p = v;
284 } else {
285 memcpy(p, &v, sizeof(v));
286 }
5c53bb81 287}
1813e175 288#endif
5c53bb81 289
1813e175 290#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 291static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 292{
1813e175
RH		 293 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
294 *s->code_ptr++ = v;
295 } else {
296 tcg_insn_unit *p = s->code_ptr;
297 memcpy(p, &v, sizeof(v));
298 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
299 }
c896fe29
FB		 300}
301
4196dca6
PM		 302static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
303 uint32_t v)
5c53bb81 304{
1813e175
RH		 305 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
306 *p = v;
307 } else {
308 memcpy(p, &v, sizeof(v));
309 }
5c53bb81 310}
1813e175 311#endif
5c53bb81 312
1813e175 313#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 314static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 315{
1813e175
RH		 316 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
317 *s->code_ptr++ = v;
318 } else {
319 tcg_insn_unit *p = s->code_ptr;
320 memcpy(p, &v, sizeof(v));
321 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
322 }
ac26eb69
RH		 323}
324
4196dca6
PM		 325static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
326 uint64_t v)
5c53bb81 327{
1813e175
RH		 328 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
329 *p = v;
330 } else {
331 memcpy(p, &v, sizeof(v));
332 }
5c53bb81 333}
1813e175 334#endif
5c53bb81 335
c896fe29
FB		 336/* label relocation processing */
337
1813e175 338static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 339 TCGLabel *l, intptr_t addend)
c896fe29 340{
7ecd02a0 341 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 342
7ecd02a0
RH		 343 r->type = type;
344 r->ptr = code_ptr;
345 r->addend = addend;
346 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 347}
348
92ab8e7d 349static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 350{
eabb7b91 351 tcg_debug_assert(!l->has_value);
c896fe29 352 l->has_value = 1;
92ab8e7d 353 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 354}
355
42a268c2 356TCGLabel *gen_new_label(void)
c896fe29 357{
b1311c4a 358 TCGContext *s = tcg_ctx;
51e3972c 359 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 360
7ecd02a0
RH		 361 memset(l, 0, sizeof(TCGLabel));
362 l->id = s->nb_labels++;
f85b1fc4 363 QSIMPLEQ_INIT(&l->branches);
7ecd02a0
RH		 364 QSIMPLEQ_INIT(&l->relocs);
365
bef16ab4 366 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 367
368 return l;
c896fe29
FB		 369}
370
7ecd02a0
RH		 371static bool tcg_resolve_relocs(TCGContext *s)
372{
373 TCGLabel *l;
374
375 QSIMPLEQ_FOREACH(l, &s->labels, next) {
376 TCGRelocation *r;
377 uintptr_t value = l->u.value;
378
379 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
380 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
381 return false;
382 }
383 }
384 }
385 return true;
386}
387
9f754620
RH		 388static void set_jmp_reset_offset(TCGContext *s, int which)
389{
f14bed3f
RH		 390 /*
391 * We will check for overflow at the end of the opcode loop in
392 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
393 */
b7e4afbd 394 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 395}
396
b52a2c03
RH		 397static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
398{
399 /*
400 * We will check for overflow at the end of the opcode loop in
401 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
402 */
9da6079b 403 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
b52a2c03
RH		 404}
405
becc452a
RH		 406static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
407{
408 /*
409 * Return the read-execute version of the pointer, for the benefit
410 * of any pc-relative addressing mode.
411 */
9da6079b 412 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
becc452a
RH		 413}
414
397cabaa
RH		 415static int __attribute__((unused))
416tlb_mask_table_ofs(TCGContext *s, int which)
d0a9bb5e 417{
7857ee11
RH		 418 return (offsetof(CPUNegativeOffsetState, tlb.f[which]) -
419 sizeof(CPUNegativeOffsetState));
d0a9bb5e 420}
d0a9bb5e 421
db6b7d0c 422/* Signal overflow, starting over with fewer guest insns. */
8905770b
MAL		 423static G_NORETURN
424void tcg_raise_tb_overflow(TCGContext *s)
db6b7d0c
RH		 425{
426 siglongjmp(s->jmp_trans, -2);
427}
428
8429a1ca
RH		 429/*
430 * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext.
431 * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg.
432 *
433 * However, tcg_out_helper_load_slots reuses this field to hold an
434 * argument slot number (which may designate a argument register or an
435 * argument stack slot), converting to TCGReg once all arguments that
436 * are destined for the stack are processed.
437 */
129f1f9e 438typedef struct TCGMovExtend {
8429a1ca 439 unsigned dst;
129f1f9e
RH		 440 TCGReg src;
441 TCGType dst_type;
442 TCGType src_type;
443 MemOp src_ext;
444} TCGMovExtend;
445
b3dfd5fc
RH		 446/**
447 * tcg_out_movext -- move and extend
448 * @s: tcg context
449 * @dst_type: integral type for destination
450 * @dst: destination register
451 * @src_type: integral type for source
452 * @src_ext: extension to apply to source
453 * @src: source register
454 *
455 * Move or extend @src into @dst, depending on @src_ext and the types.
456 */
129f1f9e
RH		 457static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
458 TCGType src_type, MemOp src_ext, TCGReg src)
b3dfd5fc
RH		 459{
460 switch (src_ext) {
461 case MO_UB:
462 tcg_out_ext8u(s, dst, src);
463 break;
464 case MO_SB:
465 tcg_out_ext8s(s, dst_type, dst, src);
466 break;
467 case MO_UW:
468 tcg_out_ext16u(s, dst, src);
469 break;
470 case MO_SW:
471 tcg_out_ext16s(s, dst_type, dst, src);
472 break;
473 case MO_UL:
474 case MO_SL:
475 if (dst_type == TCG_TYPE_I32) {
476 if (src_type == TCG_TYPE_I32) {
477 tcg_out_mov(s, TCG_TYPE_I32, dst, src);
478 } else {
479 tcg_out_extrl_i64_i32(s, dst, src);
480 }
481 } else if (src_type == TCG_TYPE_I32) {
482 if (src_ext & MO_SIGN) {
483 tcg_out_exts_i32_i64(s, dst, src);
484 } else {
485 tcg_out_extu_i32_i64(s, dst, src);
486 }
487 } else {
488 if (src_ext & MO_SIGN) {
489 tcg_out_ext32s(s, dst, src);
490 } else {
491 tcg_out_ext32u(s, dst, src);
492 }
493 }
494 break;
495 case MO_UQ:
496 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
497 if (dst_type == TCG_TYPE_I32) {
498 tcg_out_extrl_i64_i32(s, dst, src);
499 } else {
500 tcg_out_mov(s, TCG_TYPE_I64, dst, src);
501 }
502 break;
503 default:
504 g_assert_not_reached();
505 }
506}
507
129f1f9e
RH		 508/* Minor variations on a theme, using a structure. */
509static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i,
510 TCGReg src)
511{
512 tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src);
513}
514
515static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i)
516{
517 tcg_out_movext1_new_src(s, i, i->src);
518}
519
520/**
521 * tcg_out_movext2 -- move and extend two pair
522 * @s: tcg context
523 * @i1: first move description
524 * @i2: second move description
525 * @scratch: temporary register, or -1 for none
526 *
527 * As tcg_out_movext, for both @i1 and @i2, caring for overlap
528 * between the sources and destinations.
529 */
530
8429a1ca
RH		 531static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1,
532 const TCGMovExtend *i2, int scratch)
129f1f9e
RH		 533{
534 TCGReg src1 = i1->src;
535 TCGReg src2 = i2->src;
536
537 if (i1->dst != src2) {
538 tcg_out_movext1(s, i1);
539 tcg_out_movext1(s, i2);
540 return;
541 }
542 if (i2->dst == src1) {
543 TCGType src1_type = i1->src_type;
544 TCGType src2_type = i2->src_type;
545
546 if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) {
547 /* The data is now in the correct registers, now extend. */
548 src1 = i2->src;
549 src2 = i1->src;
550 } else {
551 tcg_debug_assert(scratch >= 0);
552 tcg_out_mov(s, src1_type, scratch, src1);
553 src1 = scratch;
554 }
555 }
556 tcg_out_movext1_new_src(s, i2, src2);
557 tcg_out_movext1_new_src(s, i1, src1);
558}
559
2462e30e
RH		 560/**
561 * tcg_out_movext3 -- move and extend three pair
562 * @s: tcg context
563 * @i1: first move description
564 * @i2: second move description
565 * @i3: third move description
566 * @scratch: temporary register, or -1 for none
567 *
568 * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap
569 * between the sources and destinations.
570 */
571
572static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1,
573 const TCGMovExtend *i2, const TCGMovExtend *i3,
574 int scratch)
575{
576 TCGReg src1 = i1->src;
577 TCGReg src2 = i2->src;
578 TCGReg src3 = i3->src;
579
580 if (i1->dst != src2 && i1->dst != src3) {
581 tcg_out_movext1(s, i1);
582 tcg_out_movext2(s, i2, i3, scratch);
583 return;
584 }
585 if (i2->dst != src1 && i2->dst != src3) {
586 tcg_out_movext1(s, i2);
587 tcg_out_movext2(s, i1, i3, scratch);
588 return;
589 }
590 if (i3->dst != src1 && i3->dst != src2) {
591 tcg_out_movext1(s, i3);
592 tcg_out_movext2(s, i1, i2, scratch);
593 return;
594 }
595
596 /*
597 * There is a cycle. Since there are only 3 nodes, the cycle is
598 * either "clockwise" or "anti-clockwise", and can be solved with
599 * a single scratch or two xchg.
600 */
601 if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) {
602 /* "Clockwise" */
603 if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) {
604 tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3);
605 /* The data is now in the correct registers, now extend. */
606 tcg_out_movext1_new_src(s, i1, i1->dst);
607 tcg_out_movext1_new_src(s, i2, i2->dst);
608 tcg_out_movext1_new_src(s, i3, i3->dst);
609 } else {
610 tcg_debug_assert(scratch >= 0);
611 tcg_out_mov(s, i1->src_type, scratch, src1);
612 tcg_out_movext1(s, i3);
613 tcg_out_movext1(s, i2);
614 tcg_out_movext1_new_src(s, i1, scratch);
615 }
616 } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) {
617 /* "Anti-clockwise" */
618 if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) {
619 tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2);
620 /* The data is now in the correct registers, now extend. */
621 tcg_out_movext1_new_src(s, i1, i1->dst);
622 tcg_out_movext1_new_src(s, i2, i2->dst);
623 tcg_out_movext1_new_src(s, i3, i3->dst);
624 } else {
625 tcg_debug_assert(scratch >= 0);
626 tcg_out_mov(s, i1->src_type, scratch, src1);
627 tcg_out_movext1(s, i2);
628 tcg_out_movext1(s, i3);
629 tcg_out_movext1_new_src(s, i1, scratch);
630 }
631 } else {
632 g_assert_not_reached();
633 }
634}
635
4c22e840
RH		 636#define C_PFX1(P, A) P##A
637#define C_PFX2(P, A, B) P##A##_##B
638#define C_PFX3(P, A, B, C) P##A##_##B##_##C
639#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
640#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
641#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
642
643/* Define an enumeration for the various combinations. */
644
645#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
646#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
647#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
648#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
649
650#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
651#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
652#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
653#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
654
655#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
ca5bed07 656#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1),
fa645b48 657#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1),
4c22e840
RH		 658
659#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
660#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
661#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
662#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
22d2e535 663#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4),
4c22e840
RH		 664
665typedef enum {
666#include "tcg-target-con-set.h"
667} TCGConstraintSetIndex;
668
669static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
670
671#undef C_O0_I1
672#undef C_O0_I2
673#undef C_O0_I3
674#undef C_O0_I4
675#undef C_O1_I1
676#undef C_O1_I2
677#undef C_O1_I3
678#undef C_O1_I4
679#undef C_N1_I2
ca5bed07 680#undef C_N1O1_I1
fa645b48 681#undef C_N2_I1
4c22e840
RH		 682#undef C_O2_I1
683#undef C_O2_I2
684#undef C_O2_I3
685#undef C_O2_I4
22d2e535 686#undef C_N1_O1_I4
4c22e840
RH		 687
688/* Put all of the constraint sets into an array, indexed by the enum. */
689
690#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
691#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
692#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
693#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
694
695#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
696#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
697#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
698#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
699
700#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
ca5bed07 701#define C_N1O1_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, #O2, #I1 } },
fa645b48 702#define C_N2_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, "&" #O2, #I1 } },
4c22e840
RH		 703
704#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
705#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
706#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
707#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
22d2e535 708#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { "&" #O1, #O2, #I1, #I2, #I3, #I4 } },
4c22e840
RH		 709
710static const TCGTargetOpDef constraint_sets[] = {
711#include "tcg-target-con-set.h"
712};
713
714
715#undef C_O0_I1
716#undef C_O0_I2
717#undef C_O0_I3
718#undef C_O0_I4
719#undef C_O1_I1
720#undef C_O1_I2
721#undef C_O1_I3
722#undef C_O1_I4
723#undef C_N1_I2
ca5bed07 724#undef C_N1O1_I1
fa645b48 725#undef C_N2_I1
4c22e840
RH		 726#undef C_O2_I1
727#undef C_O2_I2
728#undef C_O2_I3
729#undef C_O2_I4
22d2e535 730#undef C_N1_O1_I4
4c22e840
RH		 731
732/* Expand the enumerator to be returned from tcg_target_op_def(). */
733
734#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
735#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
736#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
737#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
738
739#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
740#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
741#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
742#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
743
744#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
ca5bed07 745#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1)
fa645b48 746#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1)
4c22e840
RH		 747
748#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
749#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
750#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
751#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
22d2e535 752#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4)
4c22e840 753
139c1837 754#include "tcg-target.c.inc"
c896fe29 755
7857ee11
RH		 756#ifndef CONFIG_TCG_INTERPRETER
757/* Validate CPUTLBDescFast placement. */
758QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
759 sizeof(CPUNegativeOffsetState))
760 < MIN_TLB_MASK_TABLE_OFS);
761#endif
762
38b47b19
EC		 763static void alloc_tcg_plugin_context(TCGContext *s)
764{
765#ifdef CONFIG_PLUGIN
766 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
767 s->plugin_tb->insns =
768 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
769#endif
770}
771
3468b59e
EC		 772/*
773 * All TCG threads except the parent (i.e. the one that called tcg_context_init
774 * and registered the target's TCG globals) must register with this function
775 * before initiating translation.
776 *
777 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
778 * of tcg_region_init() for the reasoning behind this.
779 *
7893e42d
PMD		 780 * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
781 * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
3468b59e
EC		 782 * is not used anymore for translation once this function is called.
783 *
7893e42d
PMD		 784 * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
785 * iterates over the array (e.g. tcg_code_size() the same for both system/user
786 * modes.
3468b59e
EC		 787 */
788#ifdef CONFIG_USER_ONLY
789void tcg_register_thread(void)
790{
791 tcg_ctx = &tcg_init_ctx;
792}
793#else
794void tcg_register_thread(void)
795{
796 TCGContext *s = g_malloc(sizeof(*s));
797 unsigned int i, n;
3468b59e
EC		 798
799 *s = tcg_init_ctx;
800
801 /* Relink mem_base. */
802 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
803 if (tcg_init_ctx.temps[i].mem_base) {
804 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
805 tcg_debug_assert(b >= 0 && b < n);
806 s->temps[i].mem_base = &s->temps[b];
807 }
808 }
809
810 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 811 n = qatomic_fetch_inc(&tcg_cur_ctxs);
812 g_assert(n < tcg_max_ctxs);
d73415a3 813 qatomic_set(&tcg_ctxs[n], s);
3468b59e 814
38b47b19
EC		 815 if (n > 0) {
816 alloc_tcg_plugin_context(s);
bf042e8e 817 tcg_region_initial_alloc(s);
38b47b19
EC		 818 }
819
3468b59e 820 tcg_ctx = s;
e8feb96f 821}
3468b59e 822#endif /* !CONFIG_USER_ONLY */
e8feb96f 823
c896fe29
FB		 824/* pool based memory allocation */
825void *tcg_malloc_internal(TCGContext *s, int size)
826{
827 TCGPool *p;
828 int pool_size;
a813e36f 829
c896fe29
FB		 830 if (size > TCG_POOL_CHUNK_SIZE) {
831 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 832 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 833 p->size = size;
4055299e
KB		 834 p->next = s->pool_first_large;
835 s->pool_first_large = p;
836 return p->data;
c896fe29
FB		 837 } else {
838 p = s->pool_current;
839 if (!p) {
840 p = s->pool_first;
841 if (!p)
842 goto new_pool;
843 } else {
844 if (!p->next) {
845 new_pool:
846 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 847 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 848 p->size = pool_size;
849 p->next = NULL;
a813e36f 850 if (s->pool_current) {
c896fe29 851 s->pool_current->next = p;
a813e36f 852 } else {
c896fe29 853 s->pool_first = p;
a813e36f 854 }
c896fe29
FB		 855 } else {
856 p = p->next;
857 }
858 }
859 }
860 s->pool_current = p;
861 s->pool_cur = p->data + size;
862 s->pool_end = p->data + p->size;
863 return p->data;
864}
865
866void tcg_pool_reset(TCGContext *s)
867{
4055299e
KB		 868 TCGPool *p, *t;
869 for (p = s->pool_first_large; p; p = t) {
870 t = p->next;
871 g_free(p);
872 }
873 s->pool_first_large = NULL;
c896fe29
FB		 874 s->pool_cur = s->pool_end = NULL;
875 s->pool_current = NULL;
876}
877
8429a1ca
RH		 878/*
879 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
880 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
881 * We only use these for layout in tcg_out_ld_helper_ret and
882 * tcg_out_st_helper_args, and share them between several of
883 * the helpers, with the end result that it's easier to build manually.
884 */
885
886#if TCG_TARGET_REG_BITS == 32
887# define dh_typecode_ttl dh_typecode_i32
888#else
889# define dh_typecode_ttl dh_typecode_i64
890#endif
891
892static TCGHelperInfo info_helper_ld32_mmu = {
893 .flags = TCG_CALL_NO_WG,
894 .typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */
895 | dh_typemask(env, 1)
24e46e6c 896 | dh_typemask(i64, 2) /* uint64_t addr */
8429a1ca
RH		 897 | dh_typemask(i32, 3) /* unsigned oi */
898 | dh_typemask(ptr, 4) /* uintptr_t ra */
899};
900
901static TCGHelperInfo info_helper_ld64_mmu = {
902 .flags = TCG_CALL_NO_WG,
903 .typemask = dh_typemask(i64, 0) /* return uint64_t */
904 | dh_typemask(env, 1)
24e46e6c 905 | dh_typemask(i64, 2) /* uint64_t addr */
8429a1ca
RH		 906 | dh_typemask(i32, 3) /* unsigned oi */
907 | dh_typemask(ptr, 4) /* uintptr_t ra */
908};
909
ebebea53
RH		 910static TCGHelperInfo info_helper_ld128_mmu = {
911 .flags = TCG_CALL_NO_WG,
912 .typemask = dh_typemask(i128, 0) /* return Int128 */
913 | dh_typemask(env, 1)
24e46e6c 914 | dh_typemask(i64, 2) /* uint64_t addr */
ebebea53
RH		 915 | dh_typemask(i32, 3) /* unsigned oi */
916 | dh_typemask(ptr, 4) /* uintptr_t ra */
917};
918
8429a1ca
RH		 919static TCGHelperInfo info_helper_st32_mmu = {
920 .flags = TCG_CALL_NO_WG,
921 .typemask = dh_typemask(void, 0)
922 | dh_typemask(env, 1)
24e46e6c 923 | dh_typemask(i64, 2) /* uint64_t addr */
8429a1ca
RH		 924 | dh_typemask(i32, 3) /* uint32_t data */
925 | dh_typemask(i32, 4) /* unsigned oi */
926 | dh_typemask(ptr, 5) /* uintptr_t ra */
927};
928
929static TCGHelperInfo info_helper_st64_mmu = {
930 .flags = TCG_CALL_NO_WG,
931 .typemask = dh_typemask(void, 0)
932 | dh_typemask(env, 1)
24e46e6c 933 | dh_typemask(i64, 2) /* uint64_t addr */
8429a1ca
RH		 934 | dh_typemask(i64, 3) /* uint64_t data */
935 | dh_typemask(i32, 4) /* unsigned oi */
936 | dh_typemask(ptr, 5) /* uintptr_t ra */
937};
938
ebebea53
RH		 939static TCGHelperInfo info_helper_st128_mmu = {
940 .flags = TCG_CALL_NO_WG,
941 .typemask = dh_typemask(void, 0)
942 | dh_typemask(env, 1)
24e46e6c 943 | dh_typemask(i64, 2) /* uint64_t addr */
ebebea53
RH		 944 | dh_typemask(i128, 3) /* Int128 data */
945 | dh_typemask(i32, 4) /* unsigned oi */
946 | dh_typemask(ptr, 5) /* uintptr_t ra */
947};
948
22f15579 949#ifdef CONFIG_TCG_INTERPRETER
c6ef8c7b
PMD		 950static ffi_type *typecode_to_ffi(int argmask)
951{
e9709e17
RH		 952 /*
953 * libffi does not support __int128_t, so we have forced Int128
954 * to use the structure definition instead of the builtin type.
955 */
956 static ffi_type *ffi_type_i128_elements[3] = {
957 &ffi_type_uint64,
958 &ffi_type_uint64,
959 NULL
960 };
961 static ffi_type ffi_type_i128 = {
962 .size = 16,
963 .alignment = __alignof__(Int128),
964 .type = FFI_TYPE_STRUCT,
965 .elements = ffi_type_i128_elements,
966 };
967
c6ef8c7b
PMD		 968 switch (argmask) {
969 case dh_typecode_void:
970 return &ffi_type_void;
971 case dh_typecode_i32:
972 return &ffi_type_uint32;
973 case dh_typecode_s32:
974 return &ffi_type_sint32;
975 case dh_typecode_i64:
976 return &ffi_type_uint64;
977 case dh_typecode_s64:
978 return &ffi_type_sint64;
979 case dh_typecode_ptr:
980 return &ffi_type_pointer;
e9709e17
RH		 981 case dh_typecode_i128:
982 return &ffi_type_i128;
c6ef8c7b
PMD		 983 }
984 g_assert_not_reached();
985}
0c22e176 986
d53106c9 987static ffi_cif *init_ffi_layout(TCGHelperInfo *info)
0c22e176 988{
d53106c9
RH		 989 unsigned typemask = info->typemask;
990 struct {
991 ffi_cif cif;
992 ffi_type *args[];
993 } *ca;
994 ffi_status status;
995 int nargs;
996
997 /* Ignoring the return type, find the last non-zero field. */
998 nargs = 32 - clz32(typemask >> 3);
999 nargs = DIV_ROUND_UP(nargs, 3);
1000 assert(nargs <= MAX_CALL_IARGS);
1001
1002 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
1003 ca->cif.rtype = typecode_to_ffi(typemask & 7);
1004 ca->cif.nargs = nargs;
1005
1006 if (nargs != 0) {
1007 ca->cif.arg_types = ca->args;
1008 for (int j = 0; j < nargs; ++j) {
1009 int typecode = extract32(typemask, (j + 1) * 3, 3);
1010 ca->args[j] = typecode_to_ffi(typecode);
0c22e176 1011 }
0c22e176 1012 }
f9c4bb80 1013
d53106c9
RH		 1014 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
1015 ca->cif.rtype, ca->cif.arg_types);
1016 assert(status == FFI_OK);
1017
1018 return &ca->cif;
0c22e176 1019}
d53106c9
RH		 1020
1021#define HELPER_INFO_INIT(I) (&(I)->cif)
1022#define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I)
1023#else
1024#define HELPER_INFO_INIT(I) (&(I)->init)
1025#define HELPER_INFO_INIT_VAL(I) 1
0c22e176 1026#endif /* CONFIG_TCG_INTERPRETER */
22f15579 1027
338b61e9
RH		 1028static inline bool arg_slot_reg_p(unsigned arg_slot)
1029{
1030 /*
1031 * Split the sizeof away from the comparison to avoid Werror from
1032 * "unsigned < 0 is always false", when iarg_regs is empty.
1033 */
1034 unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs);
1035 return arg_slot < nreg;
1036}
1037
d78e4a4f
RH		 1038static inline int arg_slot_stk_ofs(unsigned arg_slot)
1039{
1040 unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1041 unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
1042
1043 tcg_debug_assert(stk_slot < max);
1044 return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long);
1045}
1046
39004a71
RH		 1047typedef struct TCGCumulativeArgs {
1048 int arg_idx; /* tcg_gen_callN args[] */
1049 int info_in_idx; /* TCGHelperInfo in[] */
1050 int arg_slot; /* regs+stack slot */
1051 int ref_slot; /* stack slots for references */
1052} TCGCumulativeArgs;
1053
1054static void layout_arg_even(TCGCumulativeArgs *cum)
1055{
1056 cum->arg_slot += cum->arg_slot & 1;
1057}
1058
1059static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
1060 TCGCallArgumentKind kind)
1061{
1062 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1063
1064 *loc = (TCGCallArgumentLoc){
1065 .kind = kind,
1066 .arg_idx = cum->arg_idx,
1067 .arg_slot = cum->arg_slot,
1068 };
1069 cum->info_in_idx++;
1070 cum->arg_slot++;
1071}
1072
1073static void layout_arg_normal_n(TCGCumulativeArgs *cum,
1074 TCGHelperInfo *info, int n)
1075{
1076 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1077
1078 for (int i = 0; i < n; ++i) {
1079 /* Layout all using the same arg_idx, adjusting the subindex. */
1080 loc[i] = (TCGCallArgumentLoc){
1081 .kind = TCG_CALL_ARG_NORMAL,
1082 .arg_idx = cum->arg_idx,
1083 .tmp_subindex = i,
1084 .arg_slot = cum->arg_slot + i,
1085 };
1086 }
1087 cum->info_in_idx += n;
1088 cum->arg_slot += n;
1089}
1090
313bdea8
RH		 1091static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info)
1092{
1093 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1094 int n = 128 / TCG_TARGET_REG_BITS;
1095
1096 /* The first subindex carries the pointer. */
1097 layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);
1098
1099 /*
1100 * The callee is allowed to clobber memory associated with
1101 * structure pass by-reference. Therefore we must make copies.
1102 * Allocate space from "ref_slot", which will be adjusted to
1103 * follow the parameters on the stack.
1104 */
1105 loc[0].ref_slot = cum->ref_slot;
1106
1107 /*
1108 * Subsequent words also go into the reference slot, but
1109 * do not accumulate into the regular arguments.
1110 */
1111 for (int i = 1; i < n; ++i) {
1112 loc[i] = (TCGCallArgumentLoc){
1113 .kind = TCG_CALL_ARG_BY_REF_N,
1114 .arg_idx = cum->arg_idx,
1115 .tmp_subindex = i,
1116 .ref_slot = cum->ref_slot + i,
1117 };
1118 }
e18ed26c 1119 cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */
313bdea8
RH		 1120 cum->ref_slot += n;
1121}
1122
39004a71
RH		 1123static void init_call_layout(TCGHelperInfo *info)
1124{
1125 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
1126 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1127 unsigned typemask = info->typemask;
1128 unsigned typecode;
1129 TCGCumulativeArgs cum = { };
1130
1131 /*
1132 * Parse and place any function return value.
1133 */
1134 typecode = typemask & 7;
1135 switch (typecode) {
1136 case dh_typecode_void:
1137 info->nr_out = 0;
1138 break;
1139 case dh_typecode_i32:
1140 case dh_typecode_s32:
1141 case dh_typecode_ptr:
1142 info->nr_out = 1;
1143 info->out_kind = TCG_CALL_RET_NORMAL;
1144 break;
1145 case dh_typecode_i64:
1146 case dh_typecode_s64:
1147 info->nr_out = 64 / TCG_TARGET_REG_BITS;
1148 info->out_kind = TCG_CALL_RET_NORMAL;
5e3d0c19
RH		 1149 /* Query the last register now to trigger any assert early. */
1150 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
466d3759
RH		 1151 break;
1152 case dh_typecode_i128:
1153 info->nr_out = 128 / TCG_TARGET_REG_BITS;
5427a9a7
RH		 1154 info->out_kind = TCG_TARGET_CALL_RET_I128;
1155 switch (TCG_TARGET_CALL_RET_I128) {
466d3759 1156 case TCG_CALL_RET_NORMAL:
5e3d0c19
RH		 1157 /* Query the last register now to trigger any assert early. */
1158 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
466d3759 1159 break;
c6556aa0
RH		 1160 case TCG_CALL_RET_BY_VEC:
1161 /* Query the single register now to trigger any assert early. */
1162 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
1163 break;
313bdea8
RH		 1164 case TCG_CALL_RET_BY_REF:
1165 /*
1166 * Allocate the first argument to the output.
1167 * We don't need to store this anywhere, just make it
1168 * unavailable for use in the input loop below.
1169 */
1170 cum.arg_slot = 1;
1171 break;
466d3759
RH		 1172 default:
1173 qemu_build_not_reached();
1174 }
39004a71
RH		 1175 break;
1176 default:
1177 g_assert_not_reached();
1178 }
39004a71
RH		 1179
1180 /*
1181 * Parse and place function arguments.
1182 */
1183 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
1184 TCGCallArgumentKind kind;
1185 TCGType type;
1186
1187 typecode = typemask & 7;
1188 switch (typecode) {
1189 case dh_typecode_i32:
1190 case dh_typecode_s32:
1191 type = TCG_TYPE_I32;
1192 break;
1193 case dh_typecode_i64:
1194 case dh_typecode_s64:
1195 type = TCG_TYPE_I64;
1196 break;
1197 case dh_typecode_ptr:
1198 type = TCG_TYPE_PTR;
1199 break;
466d3759
RH		 1200 case dh_typecode_i128:
1201 type = TCG_TYPE_I128;
1202 break;
39004a71
RH		 1203 default:
1204 g_assert_not_reached();
1205 }
1206
1207 switch (type) {
1208 case TCG_TYPE_I32:
1209 switch (TCG_TARGET_CALL_ARG_I32) {
1210 case TCG_CALL_ARG_EVEN:
1211 layout_arg_even(&cum);
1212 /* fall through */
1213 case TCG_CALL_ARG_NORMAL:
1214 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1215 break;
1216 case TCG_CALL_ARG_EXTEND:
1217 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
1218 layout_arg_1(&cum, info, kind);
1219 break;
1220 default:
1221 qemu_build_not_reached();
1222 }
1223 break;
1224
1225 case TCG_TYPE_I64:
1226 switch (TCG_TARGET_CALL_ARG_I64) {
1227 case TCG_CALL_ARG_EVEN:
1228 layout_arg_even(&cum);
1229 /* fall through */
1230 case TCG_CALL_ARG_NORMAL:
1231 if (TCG_TARGET_REG_BITS == 32) {
1232 layout_arg_normal_n(&cum, info, 2);
1233 } else {
1234 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1235 }
1236 break;
1237 default:
1238 qemu_build_not_reached();
1239 }
1240 break;
1241
466d3759 1242 case TCG_TYPE_I128:
5427a9a7 1243 switch (TCG_TARGET_CALL_ARG_I128) {
466d3759
RH		 1244 case TCG_CALL_ARG_EVEN:
1245 layout_arg_even(&cum);
1246 /* fall through */
1247 case TCG_CALL_ARG_NORMAL:
1248 layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
1249 break;
313bdea8
RH		 1250 case TCG_CALL_ARG_BY_REF:
1251 layout_arg_by_ref(&cum, info);
1252 break;
466d3759
RH		 1253 default:
1254 qemu_build_not_reached();
1255 }
1256 break;
1257
39004a71
RH		 1258 default:
1259 g_assert_not_reached();
1260 }
1261 }
1262 info->nr_in = cum.info_in_idx;
1263
1264 /* Validate that we didn't overrun the input array. */
1265 assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
1266 /* Validate the backend has enough argument space. */
1267 assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
313bdea8
RH		 1268
1269 /*
1270 * Relocate the "ref_slot" area to the end of the parameters.
1271 * Minimizing this stack offset helps code size for x86,
1272 * which has a signed 8-bit offset encoding.
1273 */
1274 if (cum.ref_slot != 0) {
1275 int ref_base = 0;
1276
1277 if (cum.arg_slot > max_reg_slots) {
1278 int align = __alignof(Int128) / sizeof(tcg_target_long);
1279
1280 ref_base = cum.arg_slot - max_reg_slots;
1281 if (align > 1) {
1282 ref_base = ROUND_UP(ref_base, align);
1283 }
1284 }
1285 assert(ref_base + cum.ref_slot <= max_stk_slots);
d78e4a4f 1286 ref_base += max_reg_slots;
313bdea8
RH		 1287
1288 if (ref_base != 0) {
1289 for (int i = cum.info_in_idx - 1; i >= 0; --i) {
1290 TCGCallArgumentLoc *loc = &info->in[i];
1291 switch (loc->kind) {
1292 case TCG_CALL_ARG_BY_REF:
1293 case TCG_CALL_ARG_BY_REF_N:
1294 loc->ref_slot += ref_base;
1295 break;
1296 default:
1297 break;
1298 }
1299 }
1300 }
1301 }
39004a71
RH		 1302}
1303
91478cef 1304static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 1305static void process_op_defs(TCGContext *s);
1c2adb95
RH		 1306static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1307 TCGReg reg, const char *name);
91478cef 1308
43b972b7 1309static void tcg_context_init(unsigned max_cpus)
c896fe29 1310{
a76aabd3 1311 TCGContext *s = &tcg_init_ctx;
100b5e01 1312 int op, total_args, n, i;
c896fe29
FB		 1313 TCGOpDef *def;
1314 TCGArgConstraint *args_ct;
1c2adb95 1315 TCGTemp *ts;
c896fe29
FB		 1316
1317 memset(s, 0, sizeof(*s));
c896fe29 1318 s->nb_globals = 0;
c70fbf0a 1319
c896fe29
FB		 1320 /* Count total number of arguments and allocate the corresponding
1321 space */
1322 total_args = 0;
1323 for(op = 0; op < NB_OPS; op++) {
1324 def = &tcg_op_defs[op];
1325 n = def->nb_iargs + def->nb_oargs;
1326 total_args += n;
1327 }
1328
bc2b17e6 1329 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 1330
1331 for(op = 0; op < NB_OPS; op++) {
1332 def = &tcg_op_defs[op];
1333 def->args_ct = args_ct;
c896fe29 1334 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 1335 args_ct += n;
1336 }
5cd8f621 1337
8429a1ca
RH		 1338 init_call_layout(&info_helper_ld32_mmu);
1339 init_call_layout(&info_helper_ld64_mmu);
ebebea53 1340 init_call_layout(&info_helper_ld128_mmu);
8429a1ca
RH		 1341 init_call_layout(&info_helper_st32_mmu);
1342 init_call_layout(&info_helper_st64_mmu);
ebebea53 1343 init_call_layout(&info_helper_st128_mmu);
8429a1ca 1344
c896fe29 1345 tcg_target_init(s);
f69d277e 1346 process_op_defs(s);
91478cef
RH		 1347
1348 /* Reverse the order of the saved registers, assuming they're all at
1349 the start of tcg_target_reg_alloc_order. */
1350 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
1351 int r = tcg_target_reg_alloc_order[n];
1352 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
1353 break;
1354 }
1355 }
1356 for (i = 0; i < n; ++i) {
1357 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
1358 }
1359 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
1360 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
1361 }
b1311c4a 1362
38b47b19
EC		 1363 alloc_tcg_plugin_context(s);
1364
b1311c4a 1365 tcg_ctx = s;
3468b59e
EC		 1366 /*
1367 * In user-mode we simply share the init context among threads, since we
1368 * use a single region. See the documentation tcg_region_init() for the
1369 * reasoning behind this.
7893e42d 1370 * In system-mode we will have at most max_cpus TCG threads.
3468b59e
EC		 1371 */
1372#ifdef CONFIG_USER_ONLY
df2cce29 1373 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 1374 tcg_cur_ctxs = 1;
1375 tcg_max_ctxs = 1;
3468b59e 1376#else
0e2d61cf
RH		 1377 tcg_max_ctxs = max_cpus;
1378 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 1379#endif
1c2adb95
RH		 1380
1381 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1382 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
ad75a51e 1383 tcg_env = temp_tcgv_ptr(ts);
9002ec79 1384}
b03cce8e 1385
43b972b7 1386void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 1387{
43b972b7
RH		 1388 tcg_context_init(max_cpus);
1389 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 1390}
1391
6e3b2bfd
EC		 1392/*
1393 * Allocate TBs right before their corresponding translated code, making
1394 * sure that TBs and code are on different cache lines.
1395 */
1396TranslationBlock *tcg_tb_alloc(TCGContext *s)
1397{
1398 uintptr_t align = qemu_icache_linesize;
1399 TranslationBlock *tb;
1400 void *next;
1401
e8feb96f 1402 retry:
6e3b2bfd
EC		 1403 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1404 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1405
1406 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 1407 if (tcg_region_alloc(s)) {
1408 return NULL;
1409 }
1410 goto retry;
6e3b2bfd 1411 }
d73415a3 1412 qatomic_set(&s->code_gen_ptr, next);
57a26946 1413 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 1414 return tb;
1415}
1416
935f75ae 1417void tcg_prologue_init(void)
9002ec79 1418{
935f75ae 1419 TCGContext *s = tcg_ctx;
b0a0794a 1420 size_t prologue_size;
8163b749 1421
b0a0794a
RH		 1422 s->code_ptr = s->code_gen_ptr;
1423 s->code_buf = s->code_gen_ptr;
5b38ee31 1424 s->data_gen_ptr = NULL;
b91ccb31
RH		 1425
1426#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 1427 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 1428#endif
8163b749 1429
5b38ee31
RH		 1430#ifdef TCG_TARGET_NEED_POOL_LABELS
1431 s->pool_labels = NULL;
1432#endif
1433
653b87eb 1434 qemu_thread_jit_write();
8163b749 1435 /* Generate the prologue. */
b03cce8e 1436 tcg_target_qemu_prologue(s);
5b38ee31
RH		 1437
1438#ifdef TCG_TARGET_NEED_POOL_LABELS
1439 /* Allow the prologue to put e.g. guest_base into a pool entry. */
1440 {
1768987b
RH		 1441 int result = tcg_out_pool_finalize(s);
1442 tcg_debug_assert(result == 0);
5b38ee31
RH		 1443 }
1444#endif
1445
b0a0794a 1446 prologue_size = tcg_current_code_size(s);
5584e2db 1447 perf_report_prologue(s->code_gen_ptr, prologue_size);
b0a0794a 1448
df5d2b16 1449#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 1450 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1451 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 1452#endif
8163b749 1453
d6b64b2b 1454 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
c60f599b 1455 FILE *logfile = qemu_log_trylock();
78b54858
RH		 1456 if (logfile) {
1457 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
1458 if (s->data_gen_ptr) {
1459 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
1460 size_t data_size = prologue_size - code_size;
1461 size_t i;
1462
1463 disas(logfile, s->code_gen_ptr, code_size);
1464
1465 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1466 if (sizeof(tcg_target_ulong) == 8) {
1467 fprintf(logfile,
1468 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1469 (uintptr_t)s->data_gen_ptr + i,
1470 *(uint64_t *)(s->data_gen_ptr + i));
1471 } else {
1472 fprintf(logfile,
1473 "0x%08" PRIxPTR ": .long 0x%08x\n",
1474 (uintptr_t)s->data_gen_ptr + i,
1475 *(uint32_t *)(s->data_gen_ptr + i));
1476 }
5b38ee31 1477 }
78b54858
RH		 1478 } else {
1479 disas(logfile, s->code_gen_ptr, prologue_size);
5b38ee31 1480 }
78b54858 1481 fprintf(logfile, "\n");
78b54858 1482 qemu_log_unlock(logfile);
5b38ee31 1483 }
d6b64b2b 1484 }
cedbcb01 1485
6eea0434
RH		 1486#ifndef CONFIG_TCG_INTERPRETER
1487 /*
1488 * Assert that goto_ptr is implemented completely, setting an epilogue.
1489 * For tci, we use NULL as the signal to return from the interpreter,
1490 * so skip this check.
1491 */
f4e01e30 1492 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 1493#endif
d1c74ab3
RH		 1494
1495 tcg_region_prologue_set(s);
c896fe29
FB		 1496}
1497
c896fe29
FB		 1498void tcg_func_start(TCGContext *s)
1499{
1500 tcg_pool_reset(s);
1501 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 1502
1503 /* No temps have been previously allocated for size or locality. */
1504 memset(s->free_temps, 0, sizeof(s->free_temps));
1505
c0522136
RH		 1506 /* No constant temps have been previously allocated. */
1507 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1508 if (s->const_table[i]) {
1509 g_hash_table_remove_all(s->const_table[i]);
1510 }
1511 }
1512
abebf925 1513 s->nb_ops = 0;
c896fe29
FB		 1514 s->nb_labels = 0;
1515 s->current_frame_offset = s->frame_start;
1516
0a209d4b
RH		 1517#ifdef CONFIG_DEBUG_TCG
1518 s->goto_tb_issue_mask = 0;
1519#endif
1520
15fa08f8
RH		 1521 QTAILQ_INIT(&s->ops);
1522 QTAILQ_INIT(&s->free_ops);
bef16ab4 1523 QSIMPLEQ_INIT(&s->labels);
4baf3978
RH		 1524
1525 tcg_debug_assert(s->addr_type == TCG_TYPE_I32 ||
1526 s->addr_type == TCG_TYPE_I64);
d0a9bb5e 1527
747bd69d 1528 tcg_debug_assert(s->insn_start_words > 0);
c896fe29
FB		 1529}
1530
ae30e866 1531static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 1532{
1533 int n = s->nb_temps++;
ae30e866
RH		 1534
1535 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 1536 tcg_raise_tb_overflow(s);
ae30e866 1537 }
7ca4b752
RH		 1538 return memset(&s->temps[n], 0, sizeof(TCGTemp));
1539}
1540
ae30e866 1541static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 1542{
fa477d25
RH		 1543 TCGTemp *ts;
1544
7ca4b752 1545 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 1546 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 1547 s->nb_globals++;
fa477d25 1548 ts = tcg_temp_alloc(s);
ee17db83 1549 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 1550
1551 return ts;
c896fe29
FB		 1552}
1553
085272b3
RH		 1554static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1555 TCGReg reg, const char *name)
c896fe29 1556{
c896fe29 1557 TCGTemp *ts;
c896fe29 1558
1a057554 1559 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
7ca4b752
RH		 1560
1561 ts = tcg_global_alloc(s);
c896fe29
FB		 1562 ts->base_type = type;
1563 ts->type = type;
ee17db83 1564 ts->kind = TEMP_FIXED;
c896fe29 1565 ts->reg = reg;
c896fe29 1566 ts->name = name;
c896fe29 1567 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 1568
085272b3 1569 return ts;
a7812ae4
PB		 1570}
1571
b6638662 1572void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 1573{
b3a62939
RH		 1574 s->frame_start = start;
1575 s->frame_end = start + size;
085272b3
RH		 1576 s->frame_temp
1577 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 1578}
1579
4643f3e0
RH		 1580static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset,
1581 const char *name, TCGType type)
c896fe29 1582{
b1311c4a 1583 TCGContext *s = tcg_ctx;
dc41aa7d 1584 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 1585 TCGTemp *ts = tcg_global_alloc(s);
aef85402 1586 int indirect_reg = 0;
c896fe29 1587
c0522136
RH		 1588 switch (base_ts->kind) {
1589 case TEMP_FIXED:
1590 break;
1591 case TEMP_GLOBAL:
5a18407f
RH		 1592 /* We do not support double-indirect registers. */
1593 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 1594 base_ts->indirect_base = 1;
5a18407f
RH		 1595 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1596 ? 2 : 1);
1597 indirect_reg = 1;
c0522136
RH		 1598 break;
1599 default:
1600 g_assert_not_reached();
b3915dbb
RH		 1601 }
1602
7ca4b752
RH		 1603 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1604 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 1605 char buf[64];
7ca4b752
RH		 1606
1607 ts->base_type = TCG_TYPE_I64;
c896fe29 1608 ts->type = TCG_TYPE_I32;
b3915dbb 1609 ts->indirect_reg = indirect_reg;
c896fe29 1610 ts->mem_allocated = 1;
b3a62939 1611 ts->mem_base = base_ts;
aef85402 1612 ts->mem_offset = offset;
c896fe29
FB		 1613 pstrcpy(buf, sizeof(buf), name);
1614 pstrcat(buf, sizeof(buf), "_0");
1615 ts->name = strdup(buf);
c896fe29 1616
7ca4b752
RH		 1617 tcg_debug_assert(ts2 == ts + 1);
1618 ts2->base_type = TCG_TYPE_I64;
1619 ts2->type = TCG_TYPE_I32;
b3915dbb 1620 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 1621 ts2->mem_allocated = 1;
1622 ts2->mem_base = base_ts;
aef85402 1623 ts2->mem_offset = offset + 4;
fac87bd2 1624 ts2->temp_subindex = 1;
c896fe29
FB		 1625 pstrcpy(buf, sizeof(buf), name);
1626 pstrcat(buf, sizeof(buf), "_1");
120c1084 1627 ts2->name = strdup(buf);
7ca4b752 1628 } else {
c896fe29
FB		 1629 ts->base_type = type;
1630 ts->type = type;
b3915dbb 1631 ts->indirect_reg = indirect_reg;
c896fe29 1632 ts->mem_allocated = 1;
b3a62939 1633 ts->mem_base = base_ts;
c896fe29 1634 ts->mem_offset = offset;
c896fe29 1635 ts->name = name;
c896fe29 1636 }
085272b3 1637 return ts;
a7812ae4
PB		 1638}
1639
4643f3e0
RH		 1640TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name)
1641{
1642 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32);
1643 return temp_tcgv_i32(ts);
1644}
1645
1646TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name)
1647{
1648 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64);
1649 return temp_tcgv_i64(ts);
1650}
1651
1652TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name)
1653{
1654 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR);
1655 return temp_tcgv_ptr(ts);
1656}
1657
1658static TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
c896fe29 1659{
b1311c4a 1660 TCGContext *s = tcg_ctx;
c896fe29 1661 TCGTemp *ts;
e1c08b00 1662 int n;
7ca4b752 1663
e1c08b00
RH		 1664 if (kind == TEMP_EBB) {
1665 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);
1666
1667 if (idx < TCG_MAX_TEMPS) {
1668 /* There is already an available temp with the right type. */
1669 clear_bit(idx, s->free_temps[type].l);
1670
1671 ts = &s->temps[idx];
1672 ts->temp_allocated = 1;
1673 tcg_debug_assert(ts->base_type == type);
1674 tcg_debug_assert(ts->kind == kind);
2f2e911d 1675 return ts;
43eef72f 1676 }
e1c08b00
RH		 1677 } else {
1678 tcg_debug_assert(kind == TEMP_TB);
1679 }
7ca4b752 1680
e1c08b00
RH		 1681 switch (type) {
1682 case TCG_TYPE_I32:
1683 case TCG_TYPE_V64:
1684 case TCG_TYPE_V128:
1685 case TCG_TYPE_V256:
1686 n = 1;
1687 break;
1688 case TCG_TYPE_I64:
1689 n = 64 / TCG_TARGET_REG_BITS;
1690 break;
1691 case TCG_TYPE_I128:
1692 n = 128 / TCG_TARGET_REG_BITS;
1693 break;
1694 default:
1695 g_assert_not_reached();
1696 }
43eef72f 1697
e1c08b00
RH		 1698 ts = tcg_temp_alloc(s);
1699 ts->base_type = type;
1700 ts->temp_allocated = 1;
1701 ts->kind = kind;
1702
1703 if (n == 1) {
1704 ts->type = type;
1705 } else {
1706 ts->type = TCG_TYPE_REG;
43eef72f 1707
e1c08b00
RH		 1708 for (int i = 1; i < n; ++i) {
1709 TCGTemp *ts2 = tcg_temp_alloc(s);
43eef72f 1710
e1c08b00
RH		 1711 tcg_debug_assert(ts2 == ts + i);
1712 ts2->base_type = type;
1713 ts2->type = TCG_TYPE_REG;
1714 ts2->temp_allocated = 1;
1715 ts2->temp_subindex = i;
1716 ts2->kind = kind;
e8996ee0 1717 }
c896fe29 1718 }
085272b3 1719 return ts;
c896fe29
FB		 1720}
1721
4643f3e0
RH		 1722TCGv_i32 tcg_temp_new_i32(void)
1723{
1724 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB));
1725}
1726
1727TCGv_i32 tcg_temp_ebb_new_i32(void)
1728{
1729 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB));
1730}
1731
1732TCGv_i64 tcg_temp_new_i64(void)
1733{
1734 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB));
1735}
1736
1737TCGv_i64 tcg_temp_ebb_new_i64(void)
1738{
1739 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB));
1740}
1741
1742TCGv_ptr tcg_temp_new_ptr(void)
1743{
1744 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB));
1745}
1746
1747TCGv_ptr tcg_temp_ebb_new_ptr(void)
1748{
1749 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB));
1750}
1751
1752TCGv_i128 tcg_temp_new_i128(void)
1753{
1754 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB));
1755}
1756
1757TCGv_i128 tcg_temp_ebb_new_i128(void)
1758{
1759 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB));
1760}
1761
d2fd745f
RH		 1762TCGv_vec tcg_temp_new_vec(TCGType type)
1763{
1764 TCGTemp *t;
1765
1766#ifdef CONFIG_DEBUG_TCG
1767 switch (type) {
1768 case TCG_TYPE_V64:
1769 assert(TCG_TARGET_HAS_v64);
1770 break;
1771 case TCG_TYPE_V128:
1772 assert(TCG_TARGET_HAS_v128);
1773 break;
1774 case TCG_TYPE_V256:
1775 assert(TCG_TARGET_HAS_v256);
1776 break;
1777 default:
1778 g_assert_not_reached();
1779 }
1780#endif
1781
bbf989bf 1782 t = tcg_temp_new_internal(type, TEMP_EBB);
d2fd745f
RH		 1783 return temp_tcgv_vec(t);
1784}
1785
1786/* Create a new temp of the same type as an existing temp. */
1787TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1788{
1789 TCGTemp *t = tcgv_vec_temp(match);
1790
1791 tcg_debug_assert(t->temp_allocated != 0);
1792
bbf989bf 1793 t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
d2fd745f
RH		 1794 return temp_tcgv_vec(t);
1795}
1796
5bfa8034 1797void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1798{
b1311c4a 1799 TCGContext *s = tcg_ctx;
c896fe29 1800
c7482438
RH		 1801 switch (ts->kind) {
1802 case TEMP_CONST:
f57c6915 1803 case TEMP_TB:
2f2e911d
RH		 1804 /* Silently ignore free. */
1805 break;
1806 case TEMP_EBB:
1807 tcg_debug_assert(ts->temp_allocated != 0);
1808 ts->temp_allocated = 0;
1809 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
c7482438
RH		 1810 break;
1811 default:
2f2e911d 1812 /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */
c7482438 1813 g_assert_not_reached();
c0522136 1814 }
c896fe29
FB		 1815}
1816
58b79713
RH		 1817void tcg_temp_free_i32(TCGv_i32 arg)
1818{
1819 tcg_temp_free_internal(tcgv_i32_temp(arg));
1820}
1821
1822void tcg_temp_free_i64(TCGv_i64 arg)
1823{
1824 tcg_temp_free_internal(tcgv_i64_temp(arg));
1825}
1826
1827void tcg_temp_free_i128(TCGv_i128 arg)
1828{
1829 tcg_temp_free_internal(tcgv_i128_temp(arg));
1830}
1831
1832void tcg_temp_free_ptr(TCGv_ptr arg)
1833{
1834 tcg_temp_free_internal(tcgv_ptr_temp(arg));
1835}
1836
1837void tcg_temp_free_vec(TCGv_vec arg)
1838{
1839 tcg_temp_free_internal(tcgv_vec_temp(arg));
1840}
1841
c0522136
RH		 1842TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1843{
1844 TCGContext *s = tcg_ctx;
1845 GHashTable *h = s->const_table[type];
1846 TCGTemp *ts;
1847
1848 if (h == NULL) {
1849 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1850 s->const_table[type] = h;
1851 }
1852
1853 ts = g_hash_table_lookup(h, &val);
1854 if (ts == NULL) {
aef85402
RH		 1855 int64_t *val_ptr;
1856
c0522136
RH		 1857 ts = tcg_temp_alloc(s);
1858
1859 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1860 TCGTemp *ts2 = tcg_temp_alloc(s);
1861
aef85402
RH		 1862 tcg_debug_assert(ts2 == ts + 1);
1863
c0522136
RH		 1864 ts->base_type = TCG_TYPE_I64;
1865 ts->type = TCG_TYPE_I32;
1866 ts->kind = TEMP_CONST;
1867 ts->temp_allocated = 1;
c0522136 1868
c0522136
RH		 1869 ts2->base_type = TCG_TYPE_I64;
1870 ts2->type = TCG_TYPE_I32;
1871 ts2->kind = TEMP_CONST;
1872 ts2->temp_allocated = 1;
fac87bd2 1873 ts2->temp_subindex = 1;
aef85402
RH		 1874
1875 /*
1876 * Retain the full value of the 64-bit constant in the low
1877 * part, so that the hash table works. Actual uses will
1878 * truncate the value to the low part.
1879 */
1880 ts[HOST_BIG_ENDIAN].val = val;
1881 ts[!HOST_BIG_ENDIAN].val = val >> 32;
1882 val_ptr = &ts[HOST_BIG_ENDIAN].val;
c0522136
RH		 1883 } else {
1884 ts->base_type = type;
1885 ts->type = type;
1886 ts->kind = TEMP_CONST;
1887 ts->temp_allocated = 1;
1888 ts->val = val;
aef85402 1889 val_ptr = &ts->val;
c0522136 1890 }
aef85402 1891 g_hash_table_insert(h, val_ptr, ts);
c0522136
RH		 1892 }
1893
1894 return ts;
1895}
1896
16edaee7
RH		 1897TCGv_i32 tcg_constant_i32(int32_t val)
1898{
1899 return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
1900}
1901
1902TCGv_i64 tcg_constant_i64(int64_t val)
1903{
1904 return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
1905}
1906
1907TCGv_ptr tcg_constant_ptr_int(intptr_t val)
1908{
1909 return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val));
1910}
1911
c0522136
RH		 1912TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1913{
1914 val = dup_const(vece, val);
1915 return temp_tcgv_vec(tcg_constant_internal(type, val));
1916}
1917
88d4005b
RH		 1918TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1919{
1920 TCGTemp *t = tcgv_vec_temp(match);
1921
1922 tcg_debug_assert(t->temp_allocated != 0);
1923 return tcg_constant_vec(t->base_type, vece, val);
1924}
1925
177f648f
RH		 1926#ifdef CONFIG_DEBUG_TCG
1927size_t temp_idx(TCGTemp *ts)
1928{
1929 ptrdiff_t n = ts - tcg_ctx->temps;
1930 assert(n >= 0 && n < tcg_ctx->nb_temps);
1931 return n;
1932}
1933
1934TCGTemp *tcgv_i32_temp(TCGv_i32 v)
1935{
1936 uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps);
1937
1938 assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps);
1939 assert(o % sizeof(TCGTemp) == 0);
1940
1941 return (void *)tcg_ctx + (uintptr_t)v;
1942}
1943#endif /* CONFIG_DEBUG_TCG */
1944
be0f34b5
RH		 1945/* Return true if OP may appear in the opcode stream.
1946 Test the runtime variable that controls each opcode. */
1947bool tcg_op_supported(TCGOpcode op)
1948{
d2fd745f
RH		 1949 const bool have_vec
1950 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1951
be0f34b5
RH		 1952 switch (op) {
1953 case INDEX_op_discard:
1954 case INDEX_op_set_label:
1955 case INDEX_op_call:
1956 case INDEX_op_br:
1957 case INDEX_op_mb:
1958 case INDEX_op_insn_start:
1959 case INDEX_op_exit_tb:
1960 case INDEX_op_goto_tb:
f4e01e30 1961 case INDEX_op_goto_ptr:
fecccfcc
RH		 1962 case INDEX_op_qemu_ld_a32_i32:
1963 case INDEX_op_qemu_ld_a64_i32:
1964 case INDEX_op_qemu_st_a32_i32:
1965 case INDEX_op_qemu_st_a64_i32:
1966 case INDEX_op_qemu_ld_a32_i64:
1967 case INDEX_op_qemu_ld_a64_i64:
1968 case INDEX_op_qemu_st_a32_i64:
1969 case INDEX_op_qemu_st_a64_i64:
be0f34b5
RH		 1970 return true;
1971
fecccfcc
RH		 1972 case INDEX_op_qemu_st8_a32_i32:
1973 case INDEX_op_qemu_st8_a64_i32:
07ce0b05
RH		 1974 return TCG_TARGET_HAS_qemu_st8_i32;
1975
fecccfcc
RH		 1976 case INDEX_op_qemu_ld_a32_i128:
1977 case INDEX_op_qemu_ld_a64_i128:
1978 case INDEX_op_qemu_st_a32_i128:
1979 case INDEX_op_qemu_st_a64_i128:
12fde9bc
RH		 1980 return TCG_TARGET_HAS_qemu_ldst_i128;
1981
be0f34b5 1982 case INDEX_op_mov_i32:
be0f34b5
RH		 1983 case INDEX_op_setcond_i32:
1984 case INDEX_op_brcond_i32:
3871be75 1985 case INDEX_op_movcond_i32:
be0f34b5
RH		 1986 case INDEX_op_ld8u_i32:
1987 case INDEX_op_ld8s_i32:
1988 case INDEX_op_ld16u_i32:
1989 case INDEX_op_ld16s_i32:
1990 case INDEX_op_ld_i32:
1991 case INDEX_op_st8_i32:
1992 case INDEX_op_st16_i32:
1993 case INDEX_op_st_i32:
1994 case INDEX_op_add_i32:
1995 case INDEX_op_sub_i32:
b701f195 1996 case INDEX_op_neg_i32:
be0f34b5
RH		 1997 case INDEX_op_mul_i32:
1998 case INDEX_op_and_i32:
1999 case INDEX_op_or_i32:
2000 case INDEX_op_xor_i32:
2001 case INDEX_op_shl_i32:
2002 case INDEX_op_shr_i32:
2003 case INDEX_op_sar_i32:
2004 return true;
2005
3635502d
RH		 2006 case INDEX_op_negsetcond_i32:
2007 return TCG_TARGET_HAS_negsetcond_i32;
be0f34b5
RH		 2008 case INDEX_op_div_i32:
2009 case INDEX_op_divu_i32:
2010 return TCG_TARGET_HAS_div_i32;
2011 case INDEX_op_rem_i32:
2012 case INDEX_op_remu_i32:
2013 return TCG_TARGET_HAS_rem_i32;
2014 case INDEX_op_div2_i32:
2015 case INDEX_op_divu2_i32:
2016 return TCG_TARGET_HAS_div2_i32;
2017 case INDEX_op_rotl_i32:
2018 case INDEX_op_rotr_i32:
2019 return TCG_TARGET_HAS_rot_i32;
2020 case INDEX_op_deposit_i32:
2021 return TCG_TARGET_HAS_deposit_i32;
2022 case INDEX_op_extract_i32:
2023 return TCG_TARGET_HAS_extract_i32;
2024 case INDEX_op_sextract_i32:
2025 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 2026 case INDEX_op_extract2_i32:
2027 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 2028 case INDEX_op_add2_i32:
2029 return TCG_TARGET_HAS_add2_i32;
2030 case INDEX_op_sub2_i32:
2031 return TCG_TARGET_HAS_sub2_i32;
2032 case INDEX_op_mulu2_i32:
2033 return TCG_TARGET_HAS_mulu2_i32;
2034 case INDEX_op_muls2_i32:
2035 return TCG_TARGET_HAS_muls2_i32;
2036 case INDEX_op_muluh_i32:
2037 return TCG_TARGET_HAS_muluh_i32;
2038 case INDEX_op_mulsh_i32:
2039 return TCG_TARGET_HAS_mulsh_i32;
2040 case INDEX_op_ext8s_i32:
2041 return TCG_TARGET_HAS_ext8s_i32;
2042 case INDEX_op_ext16s_i32:
2043 return TCG_TARGET_HAS_ext16s_i32;
2044 case INDEX_op_ext8u_i32:
2045 return TCG_TARGET_HAS_ext8u_i32;
2046 case INDEX_op_ext16u_i32:
2047 return TCG_TARGET_HAS_ext16u_i32;
2048 case INDEX_op_bswap16_i32:
2049 return TCG_TARGET_HAS_bswap16_i32;
2050 case INDEX_op_bswap32_i32:
2051 return TCG_TARGET_HAS_bswap32_i32;
2052 case INDEX_op_not_i32:
2053 return TCG_TARGET_HAS_not_i32;
be0f34b5
RH		 2054 case INDEX_op_andc_i32:
2055 return TCG_TARGET_HAS_andc_i32;
2056 case INDEX_op_orc_i32:
2057 return TCG_TARGET_HAS_orc_i32;
2058 case INDEX_op_eqv_i32:
2059 return TCG_TARGET_HAS_eqv_i32;
2060 case INDEX_op_nand_i32:
2061 return TCG_TARGET_HAS_nand_i32;
2062 case INDEX_op_nor_i32:
2063 return TCG_TARGET_HAS_nor_i32;
2064 case INDEX_op_clz_i32:
2065 return TCG_TARGET_HAS_clz_i32;
2066 case INDEX_op_ctz_i32:
2067 return TCG_TARGET_HAS_ctz_i32;
2068 case INDEX_op_ctpop_i32:
2069 return TCG_TARGET_HAS_ctpop_i32;
2070
2071 case INDEX_op_brcond2_i32:
2072 case INDEX_op_setcond2_i32:
2073 return TCG_TARGET_REG_BITS == 32;
2074
2075 case INDEX_op_mov_i64:
be0f34b5
RH		 2076 case INDEX_op_setcond_i64:
2077 case INDEX_op_brcond_i64:
3871be75 2078 case INDEX_op_movcond_i64:
be0f34b5
RH		 2079 case INDEX_op_ld8u_i64:
2080 case INDEX_op_ld8s_i64:
2081 case INDEX_op_ld16u_i64:
2082 case INDEX_op_ld16s_i64:
2083 case INDEX_op_ld32u_i64:
2084 case INDEX_op_ld32s_i64:
2085 case INDEX_op_ld_i64:
2086 case INDEX_op_st8_i64:
2087 case INDEX_op_st16_i64:
2088 case INDEX_op_st32_i64:
2089 case INDEX_op_st_i64:
2090 case INDEX_op_add_i64:
2091 case INDEX_op_sub_i64:
b701f195 2092 case INDEX_op_neg_i64:
be0f34b5
RH		 2093 case INDEX_op_mul_i64:
2094 case INDEX_op_and_i64:
2095 case INDEX_op_or_i64:
2096 case INDEX_op_xor_i64:
2097 case INDEX_op_shl_i64:
2098 case INDEX_op_shr_i64:
2099 case INDEX_op_sar_i64:
2100 case INDEX_op_ext_i32_i64:
2101 case INDEX_op_extu_i32_i64:
2102 return TCG_TARGET_REG_BITS == 64;
2103
3635502d
RH		 2104 case INDEX_op_negsetcond_i64:
2105 return TCG_TARGET_HAS_negsetcond_i64;
be0f34b5
RH		 2106 case INDEX_op_div_i64:
2107 case INDEX_op_divu_i64:
2108 return TCG_TARGET_HAS_div_i64;
2109 case INDEX_op_rem_i64:
2110 case INDEX_op_remu_i64:
2111 return TCG_TARGET_HAS_rem_i64;
2112 case INDEX_op_div2_i64:
2113 case INDEX_op_divu2_i64:
2114 return TCG_TARGET_HAS_div2_i64;
2115 case INDEX_op_rotl_i64:
2116 case INDEX_op_rotr_i64:
2117 return TCG_TARGET_HAS_rot_i64;
2118 case INDEX_op_deposit_i64:
2119 return TCG_TARGET_HAS_deposit_i64;
2120 case INDEX_op_extract_i64:
2121 return TCG_TARGET_HAS_extract_i64;
2122 case INDEX_op_sextract_i64:
2123 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 2124 case INDEX_op_extract2_i64:
2125 return TCG_TARGET_HAS_extract2_i64;
be0f34b5 2126 case INDEX_op_extrl_i64_i32:
be0f34b5 2127 case INDEX_op_extrh_i64_i32:
13d885b0 2128 return TCG_TARGET_HAS_extr_i64_i32;
be0f34b5
RH		 2129 case INDEX_op_ext8s_i64:
2130 return TCG_TARGET_HAS_ext8s_i64;
2131 case INDEX_op_ext16s_i64:
2132 return TCG_TARGET_HAS_ext16s_i64;
2133 case INDEX_op_ext32s_i64:
2134 return TCG_TARGET_HAS_ext32s_i64;
2135 case INDEX_op_ext8u_i64:
2136 return TCG_TARGET_HAS_ext8u_i64;
2137 case INDEX_op_ext16u_i64:
2138 return TCG_TARGET_HAS_ext16u_i64;
2139 case INDEX_op_ext32u_i64:
2140 return TCG_TARGET_HAS_ext32u_i64;
2141 case INDEX_op_bswap16_i64:
2142 return TCG_TARGET_HAS_bswap16_i64;
2143 case INDEX_op_bswap32_i64:
2144 return TCG_TARGET_HAS_bswap32_i64;
2145 case INDEX_op_bswap64_i64:
2146 return TCG_TARGET_HAS_bswap64_i64;
2147 case INDEX_op_not_i64:
2148 return TCG_TARGET_HAS_not_i64;
be0f34b5
RH		 2149 case INDEX_op_andc_i64:
2150 return TCG_TARGET_HAS_andc_i64;
2151 case INDEX_op_orc_i64:
2152 return TCG_TARGET_HAS_orc_i64;
2153 case INDEX_op_eqv_i64:
2154 return TCG_TARGET_HAS_eqv_i64;
2155 case INDEX_op_nand_i64:
2156 return TCG_TARGET_HAS_nand_i64;
2157 case INDEX_op_nor_i64:
2158 return TCG_TARGET_HAS_nor_i64;
2159 case INDEX_op_clz_i64:
2160 return TCG_TARGET_HAS_clz_i64;
2161 case INDEX_op_ctz_i64:
2162 return TCG_TARGET_HAS_ctz_i64;
2163 case INDEX_op_ctpop_i64:
2164 return TCG_TARGET_HAS_ctpop_i64;
2165 case INDEX_op_add2_i64:
2166 return TCG_TARGET_HAS_add2_i64;
2167 case INDEX_op_sub2_i64:
2168 return TCG_TARGET_HAS_sub2_i64;
2169 case INDEX_op_mulu2_i64:
2170 return TCG_TARGET_HAS_mulu2_i64;
2171 case INDEX_op_muls2_i64:
2172 return TCG_TARGET_HAS_muls2_i64;
2173 case INDEX_op_muluh_i64:
2174 return TCG_TARGET_HAS_muluh_i64;
2175 case INDEX_op_mulsh_i64:
2176 return TCG_TARGET_HAS_mulsh_i64;
2177
d2fd745f
RH		 2178 case INDEX_op_mov_vec:
2179 case INDEX_op_dup_vec:
37ee55a0 2180 case INDEX_op_dupm_vec:
d2fd745f
RH		 2181 case INDEX_op_ld_vec:
2182 case INDEX_op_st_vec:
2183 case INDEX_op_add_vec:
2184 case INDEX_op_sub_vec:
2185 case INDEX_op_and_vec:
2186 case INDEX_op_or_vec:
2187 case INDEX_op_xor_vec:
212be173 2188 case INDEX_op_cmp_vec:
d2fd745f
RH		 2189 return have_vec;
2190 case INDEX_op_dup2_vec:
2191 return have_vec && TCG_TARGET_REG_BITS == 32;
2192 case INDEX_op_not_vec:
2193 return have_vec && TCG_TARGET_HAS_not_vec;
2194 case INDEX_op_neg_vec:
2195 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 2196 case INDEX_op_abs_vec:
2197 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 2198 case INDEX_op_andc_vec:
2199 return have_vec && TCG_TARGET_HAS_andc_vec;
2200 case INDEX_op_orc_vec:
2201 return have_vec && TCG_TARGET_HAS_orc_vec;
ed523473
RH		 2202 case INDEX_op_nand_vec:
2203 return have_vec && TCG_TARGET_HAS_nand_vec;
2204 case INDEX_op_nor_vec:
2205 return have_vec && TCG_TARGET_HAS_nor_vec;
2206 case INDEX_op_eqv_vec:
2207 return have_vec && TCG_TARGET_HAS_eqv_vec;
3774030a
RH		 2208 case INDEX_op_mul_vec:
2209 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 2210 case INDEX_op_shli_vec:
2211 case INDEX_op_shri_vec:
2212 case INDEX_op_sari_vec:
2213 return have_vec && TCG_TARGET_HAS_shi_vec;
2214 case INDEX_op_shls_vec:
2215 case INDEX_op_shrs_vec:
2216 case INDEX_op_sars_vec:
2217 return have_vec && TCG_TARGET_HAS_shs_vec;
2218 case INDEX_op_shlv_vec:
2219 case INDEX_op_shrv_vec:
2220 case INDEX_op_sarv_vec:
2221 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 2222 case INDEX_op_rotli_vec:
2223 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 2224 case INDEX_op_rotls_vec:
2225 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 2226 case INDEX_op_rotlv_vec:
2227 case INDEX_op_rotrv_vec:
2228 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 2229 case INDEX_op_ssadd_vec:
2230 case INDEX_op_usadd_vec:
2231 case INDEX_op_sssub_vec:
2232 case INDEX_op_ussub_vec:
2233 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 2234 case INDEX_op_smin_vec:
2235 case INDEX_op_umin_vec:
2236 case INDEX_op_smax_vec:
2237 case INDEX_op_umax_vec:
2238 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 2239 case INDEX_op_bitsel_vec:
2240 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 2241 case INDEX_op_cmpsel_vec:
2242 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 2243
db432672
RH		 2244 default:
2245 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
2246 return true;
be0f34b5 2247 }
be0f34b5
RH		 2248}
2249
39004a71
RH		 2250static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
2251
a3a692b8 2252static void tcg_gen_callN(TCGHelperInfo *info, TCGTemp *ret, TCGTemp **args)
c896fe29 2253{
39004a71
RH		 2254 TCGv_i64 extend_free[MAX_CALL_IARGS];
2255 int n_extend = 0;
75e8b9b7 2256 TCGOp *op;
39004a71 2257 int i, n, pi = 0, total_args;
afb49896 2258
d53106c9
RH		 2259 if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) {
2260 init_call_layout(info);
2261 g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info));
2262 }
2263
39004a71
RH		 2264 total_args = info->nr_out + info->nr_in + 2;
2265 op = tcg_op_alloc(INDEX_op_call, total_args);
2bece2c8 2266
38b47b19 2267#ifdef CONFIG_PLUGIN
17083f6f
EC		 2268 /* Flag helpers that may affect guest state */
2269 if (tcg_ctx->plugin_insn &&
2270 !(info->flags & TCG_CALL_PLUGIN) &&
2271 !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
38b47b19
EC		 2272 tcg_ctx->plugin_insn->calls_helpers = true;
2273 }
2274#endif
2275
39004a71
RH		 2276 TCGOP_CALLO(op) = n = info->nr_out;
2277 switch (n) {
2278 case 0:
2279 tcg_debug_assert(ret == NULL);
2280 break;
2281 case 1:
2282 tcg_debug_assert(ret != NULL);
2283 op->args[pi++] = temp_arg(ret);
2284 break;
2285 case 2:
466d3759 2286 case 4:
39004a71 2287 tcg_debug_assert(ret != NULL);
466d3759 2288 tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
39004a71 2289 tcg_debug_assert(ret->temp_subindex == 0);
466d3759
RH		 2290 for (i = 0; i < n; ++i) {
2291 op->args[pi++] = temp_arg(ret + i);
2292 }
39004a71
RH		 2293 break;
2294 default:
2295 g_assert_not_reached();
2296 }
2297
2298 TCGOP_CALLI(op) = n = info->nr_in;
2299 for (i = 0; i < n; i++) {
2300 const TCGCallArgumentLoc *loc = &info->in[i];
2301 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
2302
2303 switch (loc->kind) {
2304 case TCG_CALL_ARG_NORMAL:
313bdea8
RH		 2305 case TCG_CALL_ARG_BY_REF:
2306 case TCG_CALL_ARG_BY_REF_N:
39004a71
RH		 2307 op->args[pi++] = temp_arg(ts);
2308 break;
eb8b0224 2309
39004a71
RH		 2310 case TCG_CALL_ARG_EXTEND_U:
2311 case TCG_CALL_ARG_EXTEND_S:
2312 {
5dd48602 2313 TCGv_i64 temp = tcg_temp_ebb_new_i64();
39004a71
RH		 2314 TCGv_i32 orig = temp_tcgv_i32(ts);
2315
2316 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
eb8b0224
RH		 2317 tcg_gen_ext_i32_i64(temp, orig);
2318 } else {
2319 tcg_gen_extu_i32_i64(temp, orig);
2320 }
39004a71
RH		 2321 op->args[pi++] = tcgv_i64_arg(temp);
2322 extend_free[n_extend++] = temp;
2bece2c8 2323 }
e2a9dd6b 2324 break;
7b7d8b2d 2325
e2a9dd6b
RH		 2326 default:
2327 g_assert_not_reached();
c896fe29
FB		 2328 }
2329 }
d53106c9 2330 op->args[pi++] = (uintptr_t)info->func;
3e92aa34 2331 op->args[pi++] = (uintptr_t)info;
39004a71 2332 tcg_debug_assert(pi == total_args);
a7812ae4 2333
39004a71 2334 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
7319d83a 2335
39004a71
RH		 2336 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
2337 for (i = 0; i < n_extend; ++i) {
2338 tcg_temp_free_i64(extend_free[i]);
2bece2c8 2339 }
c896fe29 2340}
c896fe29 2341
a3a692b8
RH		 2342void tcg_gen_call0(TCGHelperInfo *info, TCGTemp *ret)
2343{
2344 tcg_gen_callN(info, ret, NULL);
2345}
2346
2347void tcg_gen_call1(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1)
2348{
2349 tcg_gen_callN(info, ret, &t1);
2350}
2351
2352void tcg_gen_call2(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2)
2353{
2354 TCGTemp *args[2] = { t1, t2 };
2355 tcg_gen_callN(info, ret, args);
2356}
2357
2358void tcg_gen_call3(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2359 TCGTemp *t2, TCGTemp *t3)
2360{
2361 TCGTemp *args[3] = { t1, t2, t3 };
2362 tcg_gen_callN(info, ret, args);
2363}
2364
2365void tcg_gen_call4(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2366 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4)
2367{
2368 TCGTemp *args[4] = { t1, t2, t3, t4 };
2369 tcg_gen_callN(info, ret, args);
2370}
2371
2372void tcg_gen_call5(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2373 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5)
2374{
2375 TCGTemp *args[5] = { t1, t2, t3, t4, t5 };
2376 tcg_gen_callN(info, ret, args);
2377}
2378
2379void tcg_gen_call6(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2,
2380 TCGTemp *t3, TCGTemp *t4, TCGTemp *t5, TCGTemp *t6)
2381{
2382 TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 };
2383 tcg_gen_callN(info, ret, args);
2384}
2385
2386void tcg_gen_call7(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2387 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4,
2388 TCGTemp *t5, TCGTemp *t6, TCGTemp *t7)
2389{
2390 TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 };
2391 tcg_gen_callN(info, ret, args);
2392}
2393
8fcd3692 2394static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 2395{
ac3b8891 2396 int i, n;
ac3b8891 2397
ee17db83
RH		 2398 for (i = 0, n = s->nb_temps; i < n; i++) {
2399 TCGTemp *ts = &s->temps[i];
2400 TCGTempVal val = TEMP_VAL_MEM;
2401
2402 switch (ts->kind) {
c0522136
RH		 2403 case TEMP_CONST:
2404 val = TEMP_VAL_CONST;
2405 break;
ee17db83
RH		 2406 case TEMP_FIXED:
2407 val = TEMP_VAL_REG;
2408 break;
2409 case TEMP_GLOBAL:
2410 break;
c7482438 2411 case TEMP_EBB:
ee17db83
RH		 2412 val = TEMP_VAL_DEAD;
2413 /* fall through */
f57c6915 2414 case TEMP_TB:
ee17db83
RH		 2415 ts->mem_allocated = 0;
2416 break;
2417 default:
2418 g_assert_not_reached();
2419 }
2420 ts->val_type = val;
e8996ee0 2421 }
f8b2f202
RH		 2422
2423 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 2424}
2425
f8b2f202
RH		 2426static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
2427 TCGTemp *ts)
c896fe29 2428{
1807f4c4 2429 int idx = temp_idx(ts);
ac56dd48 2430
ee17db83
RH		 2431 switch (ts->kind) {
2432 case TEMP_FIXED:
2433 case TEMP_GLOBAL:
ac56dd48 2434 pstrcpy(buf, buf_size, ts->name);
ee17db83 2435 break;
f57c6915 2436 case TEMP_TB:
f8b2f202 2437 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83 2438 break;
c7482438 2439 case TEMP_EBB:
f8b2f202 2440 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 2441 break;
c0522136
RH		 2442 case TEMP_CONST:
2443 switch (ts->type) {
2444 case TCG_TYPE_I32:
2445 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
2446 break;
2447#if TCG_TARGET_REG_BITS > 32
2448 case TCG_TYPE_I64:
2449 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
2450 break;
2451#endif
2452 case TCG_TYPE_V64:
2453 case TCG_TYPE_V128:
2454 case TCG_TYPE_V256:
2455 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
2456 64 << (ts->type - TCG_TYPE_V64), ts->val);
2457 break;
2458 default:
2459 g_assert_not_reached();
2460 }
2461 break;
c896fe29
FB		 2462 }
2463 return buf;
2464}
2465
43439139
RH		 2466static char *tcg_get_arg_str(TCGContext *s, char *buf,
2467 int buf_size, TCGArg arg)
f8b2f202 2468{
43439139 2469 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 2470}
2471
f48f3ede
BS		 2472static const char * const cond_name[] =
2473{
0aed257f
RH		 2474 [TCG_COND_NEVER] = "never",
2475 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 2476 [TCG_COND_EQ] = "eq",
2477 [TCG_COND_NE] = "ne",
2478 [TCG_COND_LT] = "lt",
2479 [TCG_COND_GE] = "ge",
2480 [TCG_COND_LE] = "le",
2481 [TCG_COND_GT] = "gt",
2482 [TCG_COND_LTU] = "ltu",
2483 [TCG_COND_GEU] = "geu",
2484 [TCG_COND_LEU] = "leu",
2485 [TCG_COND_GTU] = "gtu"
2486};
2487
12fde9bc 2488static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] =
f713d6ad
RH		 2489{
2490 [MO_UB] = "ub",
2491 [MO_SB] = "sb",
2492 [MO_LEUW] = "leuw",
2493 [MO_LESW] = "lesw",
2494 [MO_LEUL] = "leul",
2495 [MO_LESL] = "lesl",
fc313c64 2496 [MO_LEUQ] = "leq",
f713d6ad
RH		 2497 [MO_BEUW] = "beuw",
2498 [MO_BESW] = "besw",
2499 [MO_BEUL] = "beul",
2500 [MO_BESL] = "besl",
fc313c64 2501 [MO_BEUQ] = "beq",
12fde9bc
RH		 2502 [MO_128 + MO_BE] = "beo",
2503 [MO_128 + MO_LE] = "leo",
f713d6ad
RH		 2504};
2505
1f00b27f 2506static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
1f00b27f 2507 [MO_UNALN >> MO_ASHIFT] = "un+",
1f00b27f 2508 [MO_ALIGN >> MO_ASHIFT] = "al+",
1f00b27f
SS		 2509 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
2510 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
2511 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
2512 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
2513 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
2514 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
2515};
2516
37031fef
RH		 2517static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = {
2518 [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "",
2519 [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+",
2520 [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+",
2521 [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+",
2522 [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+",
2523 [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+",
2524};
2525
587195bd
RH		 2526static const char bswap_flag_name[][6] = {
2527 [TCG_BSWAP_IZ] = "iz",
2528 [TCG_BSWAP_OZ] = "oz",
2529 [TCG_BSWAP_OS] = "os",
2530 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
2531 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
2532};
2533
b016486e
RH		 2534static inline bool tcg_regset_single(TCGRegSet d)
2535{
2536 return (d & (d - 1)) == 0;
2537}
2538
2539static inline TCGReg tcg_regset_first(TCGRegSet d)
2540{
2541 if (TCG_TARGET_NB_REGS <= 32) {
2542 return ctz32(d);
2543 } else {
2544 return ctz64(d);
2545 }
2546}
2547
b7a83ff8
RH		 2548/* Return only the number of characters output -- no error return. */
2549#define ne_fprintf(...) \
2550 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
2551
2552static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
c896fe29 2553{
c896fe29 2554 char buf[128];
c45cb8bb 2555 TCGOp *op;
c45cb8bb 2556
15fa08f8 2557 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 2558 int i, k, nb_oargs, nb_iargs, nb_cargs;
2559 const TCGOpDef *def;
c45cb8bb 2560 TCGOpcode c;
bdfb460e 2561 int col = 0;
c896fe29 2562
c45cb8bb 2563 c = op->opc;
c896fe29 2564 def = &tcg_op_defs[c];
c45cb8bb 2565
765b842a 2566 if (c == INDEX_op_insn_start) {
b016486e 2567 nb_oargs = 0;
b7a83ff8 2568 col += ne_fprintf(f, "\n ----");
9aef40ed 2569
747bd69d 2570 for (i = 0, k = s->insn_start_words; i < k; ++i) {
c9ad8d27
RH		 2571 col += ne_fprintf(f, " %016" PRIx64,
2572 tcg_get_insn_start_param(op, i));
eeacee4d 2573 }
7e4597d7 2574 } else if (c == INDEX_op_call) {
3e92aa34 2575 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 2576 void *func = tcg_call_func(op);
3e92aa34 2577
c896fe29 2578 /* variable number of arguments */
cd9090aa
RH		 2579 nb_oargs = TCGOP_CALLO(op);
2580 nb_iargs = TCGOP_CALLI(op);
c896fe29 2581 nb_cargs = def->nb_cargs;
c896fe29 2582
b7a83ff8 2583 col += ne_fprintf(f, " %s ", def->name);
3e92aa34
RH		 2584
2585 /*
2586 * Print the function name from TCGHelperInfo, if available.
2587 * Note that plugins have a template function for the info,
2588 * but the actual function pointer comes from the plugin.
2589 */
3e92aa34 2590 if (func == info->func) {
b7a83ff8 2591 col += ne_fprintf(f, "%s", info->name);
3e92aa34 2592 } else {
b7a83ff8 2593 col += ne_fprintf(f, "plugin(%p)", func);
3e92aa34
RH		 2594 }
2595
b7a83ff8 2596 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
cf066674 2597 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 2598 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2599 op->args[i]));
b03cce8e 2600 }
cf066674 2601 for (i = 0; i < nb_iargs; i++) {
efee3746 2602 TCGArg arg = op->args[nb_oargs + i];
39004a71 2603 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
b7a83ff8 2604 col += ne_fprintf(f, ",%s", t);
e8996ee0 2605 }
b03cce8e 2606 } else {
b7a83ff8 2607 col += ne_fprintf(f, " %s ", def->name);
c45cb8bb
RH		 2608
2609 nb_oargs = def->nb_oargs;
2610 nb_iargs = def->nb_iargs;
2611 nb_cargs = def->nb_cargs;
2612
d2fd745f 2613 if (def->flags & TCG_OPF_VECTOR) {
b7a83ff8
RH		 2614 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
2615 8 << TCGOP_VECE(op));
d2fd745f
RH		 2616 }
2617
b03cce8e 2618 k = 0;
c45cb8bb 2619 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 2620 const char *sep = k ? "," : "";
2621 col += ne_fprintf(f, "%s%s", sep,
2622 tcg_get_arg_str(s, buf, sizeof(buf),
2623 op->args[k++]));
b03cce8e 2624 }
c45cb8bb 2625 for (i = 0; i < nb_iargs; i++) {
b7a83ff8
RH		 2626 const char *sep = k ? "," : "";
2627 col += ne_fprintf(f, "%s%s", sep,
2628 tcg_get_arg_str(s, buf, sizeof(buf),
2629 op->args[k++]));
b03cce8e 2630 }
be210acb
RH		 2631 switch (c) {
2632 case INDEX_op_brcond_i32:
be210acb 2633 case INDEX_op_setcond_i32:
3635502d 2634 case INDEX_op_negsetcond_i32:
ffc5ea09 2635 case INDEX_op_movcond_i32:
ffc5ea09 2636 case INDEX_op_brcond2_i32:
be210acb 2637 case INDEX_op_setcond2_i32:
ffc5ea09 2638 case INDEX_op_brcond_i64:
be210acb 2639 case INDEX_op_setcond_i64:
3635502d 2640 case INDEX_op_negsetcond_i64:
ffc5ea09 2641 case INDEX_op_movcond_i64:
212be173 2642 case INDEX_op_cmp_vec:
f75da298 2643 case INDEX_op_cmpsel_vec:
efee3746
RH		 2644 if (op->args[k] < ARRAY_SIZE(cond_name)
2645 && cond_name[op->args[k]]) {
b7a83ff8 2646 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
eeacee4d 2647 } else {
b7a83ff8 2648 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 2649 }
f48f3ede 2650 i = 1;
be210acb 2651 break;
fecccfcc
RH		 2652 case INDEX_op_qemu_ld_a32_i32:
2653 case INDEX_op_qemu_ld_a64_i32:
2654 case INDEX_op_qemu_st_a32_i32:
2655 case INDEX_op_qemu_st_a64_i32:
2656 case INDEX_op_qemu_st8_a32_i32:
2657 case INDEX_op_qemu_st8_a64_i32:
2658 case INDEX_op_qemu_ld_a32_i64:
2659 case INDEX_op_qemu_ld_a64_i64:
2660 case INDEX_op_qemu_st_a32_i64:
2661 case INDEX_op_qemu_st_a64_i64:
2662 case INDEX_op_qemu_ld_a32_i128:
2663 case INDEX_op_qemu_ld_a64_i128:
2664 case INDEX_op_qemu_st_a32_i128:
2665 case INDEX_op_qemu_st_a64_i128:
59227d5d 2666 {
37031fef 2667 const char *s_al, *s_op, *s_at;
9002ffcb 2668 MemOpIdx oi = op->args[k++];
9a239c6e 2669 MemOp mop = get_memop(oi);
59227d5d
RH		 2670 unsigned ix = get_mmuidx(oi);
2671
9a239c6e
PMD		 2672 s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT];
2673 s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)];
2674 s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
2675 mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK);
37031fef
RH		 2676
2677 /* If all fields are accounted for, print symbolically. */
9a239c6e 2678 if (!mop && s_al && s_op && s_at) {
37031fef
RH		 2679 col += ne_fprintf(f, ",%s%s%s,%u",
2680 s_at, s_al, s_op, ix);
59c4b7e8 2681 } else {
9a239c6e
PMD		 2682 mop = get_memop(oi);
2683 col += ne_fprintf(f, ",$0x%x,%u", mop, ix);
59227d5d
RH		 2684 }
2685 i = 1;
f713d6ad 2686 }
f713d6ad 2687 break;
587195bd
RH		 2688 case INDEX_op_bswap16_i32:
2689 case INDEX_op_bswap16_i64:
2690 case INDEX_op_bswap32_i32:
2691 case INDEX_op_bswap32_i64:
2692 case INDEX_op_bswap64_i64:
2693 {
2694 TCGArg flags = op->args[k];
2695 const char *name = NULL;
2696
2697 if (flags < ARRAY_SIZE(bswap_flag_name)) {
2698 name = bswap_flag_name[flags];
2699 }
2700 if (name) {
b7a83ff8 2701 col += ne_fprintf(f, ",%s", name);
587195bd 2702 } else {
b7a83ff8 2703 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
587195bd
RH		 2704 }
2705 i = k = 1;
2706 }
2707 break;
be210acb 2708 default:
f48f3ede 2709 i = 0;
be210acb
RH		 2710 break;
2711 }
51e3972c
RH		 2712 switch (c) {
2713 case INDEX_op_set_label:
2714 case INDEX_op_br:
2715 case INDEX_op_brcond_i32:
2716 case INDEX_op_brcond_i64:
2717 case INDEX_op_brcond2_i32:
b7a83ff8
RH		 2718 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
2719 arg_label(op->args[k])->id);
51e3972c
RH		 2720 i++, k++;
2721 break;
3470867b
RH		 2722 case INDEX_op_mb:
2723 {
2724 TCGBar membar = op->args[k];
2725 const char *b_op, *m_op;
2726
2727 switch (membar & TCG_BAR_SC) {
2728 case 0:
2729 b_op = "none";
2730 break;
2731 case TCG_BAR_LDAQ:
2732 b_op = "acq";
2733 break;
2734 case TCG_BAR_STRL:
2735 b_op = "rel";
2736 break;
2737 case TCG_BAR_SC:
2738 b_op = "seq";
2739 break;
2740 default:
2741 g_assert_not_reached();
2742 }
2743
2744 switch (membar & TCG_MO_ALL) {
2745 case 0:
2746 m_op = "none";
2747 break;
2748 case TCG_MO_LD_LD:
2749 m_op = "rr";
2750 break;
2751 case TCG_MO_LD_ST:
2752 m_op = "rw";
2753 break;
2754 case TCG_MO_ST_LD:
2755 m_op = "wr";
2756 break;
2757 case TCG_MO_ST_ST:
2758 m_op = "ww";
2759 break;
2760 case TCG_MO_LD_LD | TCG_MO_LD_ST:
2761 m_op = "rr+rw";
2762 break;
2763 case TCG_MO_LD_LD | TCG_MO_ST_LD:
2764 m_op = "rr+wr";
2765 break;
2766 case TCG_MO_LD_LD | TCG_MO_ST_ST:
2767 m_op = "rr+ww";
2768 break;
2769 case TCG_MO_LD_ST | TCG_MO_ST_LD:
2770 m_op = "rw+wr";
2771 break;
2772 case TCG_MO_LD_ST | TCG_MO_ST_ST:
2773 m_op = "rw+ww";
2774 break;
2775 case TCG_MO_ST_LD | TCG_MO_ST_ST:
2776 m_op = "wr+ww";
2777 break;
2778 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD:
2779 m_op = "rr+rw+wr";
2780 break;
2781 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST:
2782 m_op = "rr+rw+ww";
2783 break;
2784 case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST:
2785 m_op = "rr+wr+ww";
2786 break;
2787 case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST:
2788 m_op = "rw+wr+ww";
2789 break;
2790 case TCG_MO_ALL:
2791 m_op = "all";
2792 break;
2793 default:
2794 g_assert_not_reached();
2795 }
2796
2797 col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op);
2798 i++, k++;
2799 }
2800 break;
51e3972c
RH		 2801 default:
2802 break;
2803 }
2804 for (; i < nb_cargs; i++, k++) {
b7a83ff8
RH		 2805 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
2806 op->args[k]);
bdfb460e
RH		 2807 }
2808 }
bdfb460e 2809
1894f69a 2810 if (have_prefs || op->life) {
b7a83ff8
RH		 2811 for (; col < 40; ++col) {
2812 putc(' ', f);
bdfb460e 2813 }
1894f69a
RH		 2814 }
2815
2816 if (op->life) {
2817 unsigned life = op->life;
bdfb460e
RH		 2818
2819 if (life & (SYNC_ARG * 3)) {
b7a83ff8 2820 ne_fprintf(f, " sync:");
bdfb460e
RH		 2821 for (i = 0; i < 2; ++i) {
2822 if (life & (SYNC_ARG << i)) {
b7a83ff8 2823 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2824 }
2825 }
2826 }
2827 life /= DEAD_ARG;
2828 if (life) {
b7a83ff8 2829 ne_fprintf(f, " dead:");
bdfb460e
RH		 2830 for (i = 0; life; ++i, life >>= 1) {
2831 if (life & 1) {
b7a83ff8 2832 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2833 }
2834 }
b03cce8e 2835 }
c896fe29 2836 }
1894f69a
RH		 2837
2838 if (have_prefs) {
2839 for (i = 0; i < nb_oargs; ++i) {
31fd884b 2840 TCGRegSet set = output_pref(op, i);
1894f69a
RH		 2841
2842 if (i == 0) {
b7a83ff8 2843 ne_fprintf(f, " pref=");
1894f69a 2844 } else {
b7a83ff8 2845 ne_fprintf(f, ",");
1894f69a
RH		 2846 }
2847 if (set == 0) {
b7a83ff8 2848 ne_fprintf(f, "none");
1894f69a 2849 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
b7a83ff8 2850 ne_fprintf(f, "all");
1894f69a
RH		 2851#ifdef CONFIG_DEBUG_TCG
2852 } else if (tcg_regset_single(set)) {
2853 TCGReg reg = tcg_regset_first(set);
b7a83ff8 2854 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
1894f69a
RH		 2855#endif
2856 } else if (TCG_TARGET_NB_REGS <= 32) {
b7a83ff8 2857 ne_fprintf(f, "0x%x", (uint32_t)set);
1894f69a 2858 } else {
b7a83ff8 2859 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
1894f69a
RH		 2860 }
2861 }
2862 }
2863
b7a83ff8 2864 putc('\n', f);
c896fe29
FB		 2865 }
2866}
2867
2868/* we give more priority to constraints with less registers */
2869static int get_constraint_priority(const TCGOpDef *def, int k)
2870{
74a11790 2871 const TCGArgConstraint *arg_ct = &def->args_ct[k];
29f5e925 2872 int n = ctpop64(arg_ct->regs);
c896fe29 2873
29f5e925
RH		 2874 /*
2875 * Sort constraints of a single register first, which includes output
2876 * aliases (which must exactly match the input already allocated).
2877 */
2878 if (n == 1 || arg_ct->oalias) {
2879 return INT_MAX;
2880 }
2881
2882 /*
2883 * Sort register pairs next, first then second immediately after.
2884 * Arbitrarily sort multiple pairs by the index of the first reg;
2885 * there shouldn't be many pairs.
2886 */
2887 switch (arg_ct->pair) {
2888 case 1:
2889 case 3:
2890 return (k + 1) * 2;
2891 case 2:
2892 return (arg_ct->pair_index + 1) * 2 - 1;
c896fe29 2893 }
29f5e925
RH		 2894
2895 /* Finally, sort by decreasing register count. */
2896 assert(n > 1);
2897 return -n;
c896fe29
FB		 2898}
2899
2900/* sort from highest priority to lowest */
2901static void sort_constraints(TCGOpDef *def, int start, int n)
2902{
66792f90
RH		 2903 int i, j;
2904 TCGArgConstraint *a = def->args_ct;
c896fe29 2905
66792f90
RH		 2906 for (i = 0; i < n; i++) {
2907 a[start + i].sort_index = start + i;
2908 }
2909 if (n <= 1) {
c896fe29 2910 return;
66792f90
RH		 2911 }
2912 for (i = 0; i < n - 1; i++) {
2913 for (j = i + 1; j < n; j++) {
2914 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2915 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2916 if (p1 < p2) {
66792f90
RH		 2917 int tmp = a[start + i].sort_index;
2918 a[start + i].sort_index = a[start + j].sort_index;
2919 a[start + j].sort_index = tmp;
c896fe29
FB		 2920 }
2921 }
2922 }
2923}
2924
f69d277e 2925static void process_op_defs(TCGContext *s)
c896fe29 2926{
a9751609 2927 TCGOpcode op;
c896fe29 2928
f69d277e
RH		 2929 for (op = 0; op < NB_OPS; op++) {
2930 TCGOpDef *def = &tcg_op_defs[op];
2931 const TCGTargetOpDef *tdefs;
29f5e925
RH		 2932 bool saw_alias_pair = false;
2933 int i, o, i2, o2, nb_args;
f69d277e
RH		 2934
2935 if (def->flags & TCG_OPF_NOT_PRESENT) {
2936 continue;
2937 }
2938
c896fe29 2939 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2940 if (nb_args == 0) {
2941 continue;
2942 }
2943
4c22e840
RH		 2944 /*
2945 * Macro magic should make it impossible, but double-check that
2946 * the array index is in range. Since the signness of an enum
2947 * is implementation defined, force the result to unsigned.
2948 */
2949 unsigned con_set = tcg_target_op_def(op);
2950 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2951 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2952
2953 for (i = 0; i < nb_args; i++) {
2954 const char *ct_str = tdefs->args_ct_str[i];
8940ea0d
PMD		 2955 bool input_p = i >= def->nb_oargs;
2956
f69d277e 2957 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2958 tcg_debug_assert(ct_str != NULL);
f69d277e 2959
8940ea0d
PMD		 2960 switch (*ct_str) {
2961 case '0' ... '9':
2962 o = *ct_str - '0';
2963 tcg_debug_assert(input_p);
2964 tcg_debug_assert(o < def->nb_oargs);
2965 tcg_debug_assert(def->args_ct[o].regs != 0);
2966 tcg_debug_assert(!def->args_ct[o].oalias);
2967 def->args_ct[i] = def->args_ct[o];
2968 /* The output sets oalias. */
2969 def->args_ct[o].oalias = 1;
2970 def->args_ct[o].alias_index = i;
2971 /* The input sets ialias. */
2972 def->args_ct[i].ialias = 1;
2973 def->args_ct[i].alias_index = o;
29f5e925
RH		 2974 if (def->args_ct[i].pair) {
2975 saw_alias_pair = true;
2976 }
8940ea0d
PMD		 2977 tcg_debug_assert(ct_str[1] == '\0');
2978 continue;
2979
2980 case '&':
2981 tcg_debug_assert(!input_p);
2982 def->args_ct[i].newreg = true;
2983 ct_str++;
2984 break;
29f5e925
RH		 2985
2986 case 'p': /* plus */
2987 /* Allocate to the register after the previous. */
2988 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2989 o = i - 1;
2990 tcg_debug_assert(!def->args_ct[o].pair);
2991 tcg_debug_assert(!def->args_ct[o].ct);
2992 def->args_ct[i] = (TCGArgConstraint){
2993 .pair = 2,
2994 .pair_index = o,
2995 .regs = def->args_ct[o].regs << 1,
ca5bed07 2996 .newreg = def->args_ct[o].newreg,
29f5e925
RH		 2997 };
2998 def->args_ct[o].pair = 1;
2999 def->args_ct[o].pair_index = i;
3000 tcg_debug_assert(ct_str[1] == '\0');
3001 continue;
3002
3003 case 'm': /* minus */
3004 /* Allocate to the register before the previous. */
3005 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
3006 o = i - 1;
3007 tcg_debug_assert(!def->args_ct[o].pair);
3008 tcg_debug_assert(!def->args_ct[o].ct);
3009 def->args_ct[i] = (TCGArgConstraint){
3010 .pair = 1,
3011 .pair_index = o,
3012 .regs = def->args_ct[o].regs >> 1,
ca5bed07 3013 .newreg = def->args_ct[o].newreg,
29f5e925
RH		 3014 };
3015 def->args_ct[o].pair = 2;
3016 def->args_ct[o].pair_index = i;
3017 tcg_debug_assert(ct_str[1] == '\0');
3018 continue;
8940ea0d
PMD		 3019 }
3020
3021 do {
3022 switch (*ct_str) {
17280ff4
RH		 3023 case 'i':
3024 def->args_ct[i].ct |= TCG_CT_CONST;
17280ff4 3025 break;
358b4923 3026
358b4923
RH		 3027 /* Include all of the target-specific constraints. */
3028
3029#undef CONST
3030#define CONST(CASE, MASK) \
8940ea0d 3031 case CASE: def->args_ct[i].ct |= MASK; break;
358b4923 3032#define REGS(CASE, MASK) \
8940ea0d 3033 case CASE: def->args_ct[i].regs |= MASK; break;
358b4923
RH		 3034
3035#include "tcg-target-con-str.h"
3036
3037#undef REGS
3038#undef CONST
17280ff4 3039 default:
8940ea0d
PMD		 3040 case '0' ... '9':
3041 case '&':
29f5e925
RH		 3042 case 'p':
3043 case 'm':
17280ff4 3044 /* Typo in TCGTargetOpDef constraint. */
358b4923 3045 g_assert_not_reached();
c896fe29 3046 }
8940ea0d 3047 } while (*++ct_str != '\0');
c896fe29
FB		 3048 }
3049
c68aaa18 3050 /* TCGTargetOpDef entry with too much information? */
eabb7b91 3051 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 3052
29f5e925
RH		 3053 /*
3054 * Fix up output pairs that are aliased with inputs.
3055 * When we created the alias, we copied pair from the output.
3056 * There are three cases:
3057 * (1a) Pairs of inputs alias pairs of outputs.
3058 * (1b) One input aliases the first of a pair of outputs.
3059 * (2) One input aliases the second of a pair of outputs.
3060 *
3061 * Case 1a is handled by making sure that the pair_index'es are
3062 * properly updated so that they appear the same as a pair of inputs.
3063 *
3064 * Case 1b is handled by setting the pair_index of the input to
3065 * itself, simply so it doesn't point to an unrelated argument.
3066 * Since we don't encounter the "second" during the input allocation
3067 * phase, nothing happens with the second half of the input pair.
3068 *
3069 * Case 2 is handled by setting the second input to pair=3, the
3070 * first output to pair=3, and the pair_index'es to match.
3071 */
3072 if (saw_alias_pair) {
3073 for (i = def->nb_oargs; i < nb_args; i++) {
3074 /*
3075 * Since [0-9pm] must be alone in the constraint string,
3076 * the only way they can both be set is if the pair comes
3077 * from the output alias.
3078 */
3079 if (!def->args_ct[i].ialias) {
3080 continue;
3081 }
3082 switch (def->args_ct[i].pair) {
3083 case 0:
3084 break;
3085 case 1:
3086 o = def->args_ct[i].alias_index;
3087 o2 = def->args_ct[o].pair_index;
3088 tcg_debug_assert(def->args_ct[o].pair == 1);
3089 tcg_debug_assert(def->args_ct[o2].pair == 2);
3090 if (def->args_ct[o2].oalias) {
3091 /* Case 1a */
3092 i2 = def->args_ct[o2].alias_index;
3093 tcg_debug_assert(def->args_ct[i2].pair == 2);
3094 def->args_ct[i2].pair_index = i;
3095 def->args_ct[i].pair_index = i2;
3096 } else {
3097 /* Case 1b */
3098 def->args_ct[i].pair_index = i;
3099 }
3100 break;
3101 case 2:
3102 o = def->args_ct[i].alias_index;
3103 o2 = def->args_ct[o].pair_index;
3104 tcg_debug_assert(def->args_ct[o].pair == 2);
3105 tcg_debug_assert(def->args_ct[o2].pair == 1);
3106 if (def->args_ct[o2].oalias) {
3107 /* Case 1a */
3108 i2 = def->args_ct[o2].alias_index;
3109 tcg_debug_assert(def->args_ct[i2].pair == 1);
3110 def->args_ct[i2].pair_index = i;
3111 def->args_ct[i].pair_index = i2;
3112 } else {
3113 /* Case 2 */
3114 def->args_ct[i].pair = 3;
3115 def->args_ct[o2].pair = 3;
3116 def->args_ct[i].pair_index = o2;
3117 def->args_ct[o2].pair_index = i;
3118 }
3119 break;
3120 default:
3121 g_assert_not_reached();
3122 }
3123 }
3124 }
3125
c896fe29
FB		 3126 /* sort the constraints (XXX: this is just an heuristic) */
3127 sort_constraints(def, 0, def->nb_oargs);
3128 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 3129 }
c896fe29
FB		 3130}
3131
f85b1fc4 3132static void remove_label_use(TCGOp *op, int idx)
0c627cdc 3133{
f85b1fc4
RH		 3134 TCGLabel *label = arg_label(op->args[idx]);
3135 TCGLabelUse *use;
d88a117e 3136
f85b1fc4
RH		 3137 QSIMPLEQ_FOREACH(use, &label->branches, next) {
3138 if (use->op == op) {
3139 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next);
3140 return;
3141 }
3142 }
3143 g_assert_not_reached();
3144}
3145
3146void tcg_op_remove(TCGContext *s, TCGOp *op)
3147{
d88a117e
RH		 3148 switch (op->opc) {
3149 case INDEX_op_br:
f85b1fc4 3150 remove_label_use(op, 0);
d88a117e
RH		 3151 break;
3152 case INDEX_op_brcond_i32:
3153 case INDEX_op_brcond_i64:
f85b1fc4 3154 remove_label_use(op, 3);
d88a117e
RH		 3155 break;
3156 case INDEX_op_brcond2_i32:
f85b1fc4 3157 remove_label_use(op, 5);
d88a117e
RH		 3158 break;
3159 default:
3160 break;
3161 }
3162
15fa08f8
RH		 3163 QTAILQ_REMOVE(&s->ops, op, link);
3164 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 3165 s->nb_ops--;
0c627cdc
RH		 3166}
3167
a80cdd31
RH		 3168void tcg_remove_ops_after(TCGOp *op)
3169{
3170 TCGContext *s = tcg_ctx;
3171
3172 while (true) {
3173 TCGOp *last = tcg_last_op();
3174 if (last == op) {
3175 return;
3176 }
3177 tcg_op_remove(s, last);
3178 }
3179}
3180
d4478943 3181static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
5a18407f 3182{
15fa08f8 3183 TCGContext *s = tcg_ctx;
cb10bc63
RH		 3184 TCGOp *op = NULL;
3185
3186 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
3187 QTAILQ_FOREACH(op, &s->free_ops, link) {
3188 if (nargs <= op->nargs) {
3189 QTAILQ_REMOVE(&s->free_ops, op, link);
3190 nargs = op->nargs;
3191 goto found;
3192 }
3193 }
15fa08f8 3194 }
cb10bc63
RH		 3195
3196 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
3197 nargs = MAX(4, nargs);
3198 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
3199
3200 found:
15fa08f8
RH		 3201 memset(op, 0, offsetof(TCGOp, link));
3202 op->opc = opc;
cb10bc63
RH		 3203 op->nargs = nargs;
3204
3205 /* Check for bitfield overflow. */
3206 tcg_debug_assert(op->nargs == nargs);
5a18407f 3207
cb10bc63 3208 s->nb_ops++;
15fa08f8
RH		 3209 return op;
3210}
3211
d4478943 3212TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
15fa08f8 3213{
d4478943 3214 TCGOp *op = tcg_op_alloc(opc, nargs);
15fa08f8
RH		 3215 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
3216 return op;
3217}
5a18407f 3218
d4478943
PMD		 3219TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
3220 TCGOpcode opc, unsigned nargs)
15fa08f8 3221{
d4478943 3222 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 3223 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 3224 return new_op;
3225}
3226
d4478943
PMD		 3227TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
3228 TCGOpcode opc, unsigned nargs)
5a18407f 3229{
d4478943 3230 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 3231 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 3232 return new_op;
3233}
3234
968f305e
RH		 3235static void move_label_uses(TCGLabel *to, TCGLabel *from)
3236{
3237 TCGLabelUse *u;
3238
3239 QSIMPLEQ_FOREACH(u, &from->branches, next) {
3240 TCGOp *op = u->op;
3241 switch (op->opc) {
3242 case INDEX_op_br:
3243 op->args[0] = label_arg(to);
3244 break;
3245 case INDEX_op_brcond_i32:
3246 case INDEX_op_brcond_i64:
3247 op->args[3] = label_arg(to);
3248 break;
3249 case INDEX_op_brcond2_i32:
3250 op->args[5] = label_arg(to);
3251 break;
3252 default:
3253 g_assert_not_reached();
3254 }
3255 }
3256
3257 QSIMPLEQ_CONCAT(&to->branches, &from->branches);
3258}
3259
b4fc67c7 3260/* Reachable analysis : remove unreachable code. */
9bbee4c0
RH		 3261static void __attribute__((noinline))
3262reachable_code_pass(TCGContext *s)
b4fc67c7 3263{
4d89d0bb 3264 TCGOp *op, *op_next, *op_prev;
b4fc67c7
RH		 3265 bool dead = false;
3266
3267 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3268 bool remove = dead;
3269 TCGLabel *label;
b4fc67c7
RH		 3270
3271 switch (op->opc) {
3272 case INDEX_op_set_label:
3273 label = arg_label(op->args[0]);
4d89d0bb 3274
968f305e
RH		 3275 /*
3276 * Note that the first op in the TB is always a load,
3277 * so there is always something before a label.
3278 */
3279 op_prev = QTAILQ_PREV(op, link);
3280
3281 /*
3282 * If we find two sequential labels, move all branches to
3283 * reference the second label and remove the first label.
3284 * Do this before branch to next optimization, so that the
3285 * middle label is out of the way.
3286 */
3287 if (op_prev->opc == INDEX_op_set_label) {
3288 move_label_uses(label, arg_label(op_prev->args[0]));
3289 tcg_op_remove(s, op_prev);
3290 op_prev = QTAILQ_PREV(op, link);
3291 }
3292
4d89d0bb
RH		 3293 /*
3294 * Optimization can fold conditional branches to unconditional.
3295 * If we find a label which is preceded by an unconditional
3296 * branch to next, remove the branch. We couldn't do this when
3297 * processing the branch because any dead code between the branch
3298 * and label had not yet been removed.
3299 */
4d89d0bb
RH		 3300 if (op_prev->opc == INDEX_op_br &&
3301 label == arg_label(op_prev->args[0])) {
3302 tcg_op_remove(s, op_prev);
3303 /* Fall through means insns become live again. */
3304 dead = false;
3305 }
3306
f85b1fc4 3307 if (QSIMPLEQ_EMPTY(&label->branches)) {
b4fc67c7
RH		 3308 /*
3309 * While there is an occasional backward branch, virtually
3310 * all branches generated by the translators are forward.
3311 * Which means that generally we will have already removed
3312 * all references to the label that will be, and there is
3313 * little to be gained by iterating.
3314 */
3315 remove = true;
3316 } else {
3317 /* Once we see a label, insns become live again. */
3318 dead = false;
3319 remove = false;
b4fc67c7
RH		 3320 }
3321 break;
3322
3323 case INDEX_op_br:
3324 case INDEX_op_exit_tb:
3325 case INDEX_op_goto_ptr:
3326 /* Unconditional branches; everything following is dead. */
3327 dead = true;
3328 break;
3329
3330 case INDEX_op_call:
3331 /* Notice noreturn helper calls, raising exceptions. */
90163900 3332 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 3333 dead = true;
3334 }
3335 break;
3336
3337 case INDEX_op_insn_start:
3338 /* Never remove -- we need to keep these for unwind. */
3339 remove = false;
3340 break;
3341
3342 default:
3343 break;
3344 }
3345
3346 if (remove) {
3347 tcg_op_remove(s, op);
3348 }
3349 }
3350}
3351
c70fbf0a
RH		 3352#define TS_DEAD 1
3353#define TS_MEM 2
3354
5a18407f
RH		 3355#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
3356#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
3357
25f49c5f
RH		 3358/* For liveness_pass_1, the register preferences for a given temp. */
3359static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
3360{
3361 return ts->state_ptr;
3362}
3363
3364/* For liveness_pass_1, reset the preferences for a given temp to the
3365 * maximal regset for its type.
3366 */
3367static inline void la_reset_pref(TCGTemp *ts)
3368{
3369 *la_temp_pref(ts)
3370 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
3371}
3372
9c43b68d
AJ		 3373/* liveness analysis: end of function: all temps are dead, and globals
3374 should be in memory. */
2616c808 3375static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 3376{
b83eabea
RH		 3377 int i;
3378
3379 for (i = 0; i < ng; ++i) {
3380 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 3381 la_reset_pref(&s->temps[i]);
b83eabea
RH		 3382 }
3383 for (i = ng; i < nt; ++i) {
3384 s->temps[i].state = TS_DEAD;
25f49c5f 3385 la_reset_pref(&s->temps[i]);
b83eabea 3386 }
c896fe29
FB		 3387}
3388
9c43b68d
AJ		 3389/* liveness analysis: end of basic block: all temps are dead, globals
3390 and local temps should be in memory. */
2616c808 3391static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 3392{
b83eabea 3393 int i;
641d5fbe 3394
ee17db83
RH		 3395 for (i = 0; i < nt; ++i) {
3396 TCGTemp *ts = &s->temps[i];
3397 int state;
3398
3399 switch (ts->kind) {
3400 case TEMP_FIXED:
3401 case TEMP_GLOBAL:
f57c6915 3402 case TEMP_TB:
ee17db83
RH		 3403 state = TS_DEAD | TS_MEM;
3404 break;
c7482438 3405 case TEMP_EBB:
c0522136 3406 case TEMP_CONST:
ee17db83
RH		 3407 state = TS_DEAD;
3408 break;
3409 default:
3410 g_assert_not_reached();
3411 }
3412 ts->state = state;
3413 la_reset_pref(ts);
641d5fbe
FB		 3414 }
3415}
3416
f65a061c
RH		 3417/* liveness analysis: sync globals back to memory. */
3418static void la_global_sync(TCGContext *s, int ng)
3419{
3420 int i;
3421
3422 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 3423 int state = s->temps[i].state;
3424 s->temps[i].state = state | TS_MEM;
3425 if (state == TS_DEAD) {
3426 /* If the global was previously dead, reset prefs. */
3427 la_reset_pref(&s->temps[i]);
3428 }
f65a061c
RH		 3429 }
3430}
3431
b4cb76e6 3432/*
c7482438
RH		 3433 * liveness analysis: conditional branch: all temps are dead unless
3434 * explicitly live-across-conditional-branch, globals and local temps
3435 * should be synced.
b4cb76e6
RH		 3436 */
3437static void la_bb_sync(TCGContext *s, int ng, int nt)
3438{
3439 la_global_sync(s, ng);
3440
3441 for (int i = ng; i < nt; ++i) {
c0522136
RH		 3442 TCGTemp *ts = &s->temps[i];
3443 int state;
3444
3445 switch (ts->kind) {
f57c6915 3446 case TEMP_TB:
c0522136
RH		 3447 state = ts->state;
3448 ts->state = state | TS_MEM;
b4cb76e6
RH		 3449 if (state != TS_DEAD) {
3450 continue;
3451 }
c0522136 3452 break;
c7482438 3453 case TEMP_EBB:
c0522136
RH		 3454 case TEMP_CONST:
3455 continue;
3456 default:
3457 g_assert_not_reached();
b4cb76e6
RH		 3458 }
3459 la_reset_pref(&s->temps[i]);
3460 }
3461}
3462
f65a061c
RH		 3463/* liveness analysis: sync globals back to memory and kill. */
3464static void la_global_kill(TCGContext *s, int ng)
3465{
3466 int i;
3467
3468 for (i = 0; i < ng; i++) {
3469 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 3470 la_reset_pref(&s->temps[i]);
3471 }
3472}
3473
3474/* liveness analysis: note live globals crossing calls. */
3475static void la_cross_call(TCGContext *s, int nt)
3476{
3477 TCGRegSet mask = ~tcg_target_call_clobber_regs;
3478 int i;
3479
3480 for (i = 0; i < nt; i++) {
3481 TCGTemp *ts = &s->temps[i];
3482 if (!(ts->state & TS_DEAD)) {
3483 TCGRegSet *pset = la_temp_pref(ts);
3484 TCGRegSet set = *pset;
3485
3486 set &= mask;
3487 /* If the combination is not possible, restart. */
3488 if (set == 0) {
3489 set = tcg_target_available_regs[ts->type] & mask;
3490 }
3491 *pset = set;
3492 }
f65a061c
RH		 3493 }
3494}
3495
874b8574
RH		 3496/*
3497 * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce
3498 * to TEMP_EBB, if possible.
3499 */
3500static void __attribute__((noinline))
3501liveness_pass_0(TCGContext *s)
3502{
3503 void * const multiple_ebb = (void *)(uintptr_t)-1;
3504 int nb_temps = s->nb_temps;
3505 TCGOp *op, *ebb;
3506
3507 for (int i = s->nb_globals; i < nb_temps; ++i) {
3508 s->temps[i].state_ptr = NULL;
3509 }
3510
3511 /*
3512 * Represent each EBB by the op at which it begins. In the case of
3513 * the first EBB, this is the first op, otherwise it is a label.
3514 * Collect the uses of each TEMP_TB: NULL for unused, EBB for use
3515 * within a single EBB, else MULTIPLE_EBB.
3516 */
3517 ebb = QTAILQ_FIRST(&s->ops);
3518 QTAILQ_FOREACH(op, &s->ops, link) {
3519 const TCGOpDef *def;
3520 int nb_oargs, nb_iargs;
3521
3522 switch (op->opc) {
3523 case INDEX_op_set_label:
3524 ebb = op;
3525 continue;
3526 case INDEX_op_discard:
3527 continue;
3528 case INDEX_op_call:
3529 nb_oargs = TCGOP_CALLO(op);
3530 nb_iargs = TCGOP_CALLI(op);
3531 break;
3532 default:
3533 def = &tcg_op_defs[op->opc];
3534 nb_oargs = def->nb_oargs;
3535 nb_iargs = def->nb_iargs;
3536 break;
3537 }
3538
3539 for (int i = 0; i < nb_oargs + nb_iargs; ++i) {
3540 TCGTemp *ts = arg_temp(op->args[i]);
3541
3542 if (ts->kind != TEMP_TB) {
3543 continue;
3544 }
3545 if (ts->state_ptr == NULL) {
3546 ts->state_ptr = ebb;
3547 } else if (ts->state_ptr != ebb) {
3548 ts->state_ptr = multiple_ebb;
3549 }
3550 }
3551 }
3552
3553 /*
3554 * For TEMP_TB that turned out not to be used beyond one EBB,
3555 * reduce the liveness to TEMP_EBB.
3556 */
3557 for (int i = s->nb_globals; i < nb_temps; ++i) {
3558 TCGTemp *ts = &s->temps[i];
3559 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
3560 ts->kind = TEMP_EBB;
3561 }
3562 }
3563}
3564
a1b3c48d 3565/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 3566 given input arguments is dead. Instructions updating dead
3567 temporaries are removed. */
9bbee4c0
RH		 3568static void __attribute__((noinline))
3569liveness_pass_1(TCGContext *s)
c896fe29 3570{
c70fbf0a 3571 int nb_globals = s->nb_globals;
2616c808 3572 int nb_temps = s->nb_temps;
15fa08f8 3573 TCGOp *op, *op_prev;
25f49c5f
RH		 3574 TCGRegSet *prefs;
3575 int i;
3576
3577 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
3578 for (i = 0; i < nb_temps; ++i) {
3579 s->temps[i].state_ptr = prefs + i;
3580 }
a1b3c48d 3581
ae36a246 3582 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 3583 la_func_end(s, nb_globals, nb_temps);
c896fe29 3584
eae3eb3e 3585 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 3586 int nb_iargs, nb_oargs;
c45cb8bb
RH		 3587 TCGOpcode opc_new, opc_new2;
3588 bool have_opc_new2;
a1b3c48d 3589 TCGLifeData arg_life = 0;
25f49c5f 3590 TCGTemp *ts;
c45cb8bb
RH		 3591 TCGOpcode opc = op->opc;
3592 const TCGOpDef *def = &tcg_op_defs[opc];
3593
c45cb8bb 3594 switch (opc) {
c896fe29 3595 case INDEX_op_call:
c6e113f5 3596 {
39004a71
RH		 3597 const TCGHelperInfo *info = tcg_call_info(op);
3598 int call_flags = tcg_call_flags(op);
c896fe29 3599
cd9090aa
RH		 3600 nb_oargs = TCGOP_CALLO(op);
3601 nb_iargs = TCGOP_CALLI(op);
c6e113f5 3602
c45cb8bb 3603 /* pure functions can be removed if their result is unused */
78505279 3604 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 3605 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 3606 ts = arg_temp(op->args[i]);
3607 if (ts->state != TS_DEAD) {
c6e113f5 3608 goto do_not_remove_call;
9c43b68d 3609 }
c6e113f5 3610 }
c45cb8bb 3611 goto do_remove;
152c35aa
RH		 3612 }
3613 do_not_remove_call:
c896fe29 3614
25f49c5f 3615 /* Output args are dead. */
152c35aa 3616 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 3617 ts = arg_temp(op->args[i]);
3618 if (ts->state & TS_DEAD) {
152c35aa
RH		 3619 arg_life |= DEAD_ARG << i;
3620 }
25f49c5f 3621 if (ts->state & TS_MEM) {
152c35aa 3622 arg_life |= SYNC_ARG << i;
c6e113f5 3623 }
25f49c5f
RH		 3624 ts->state = TS_DEAD;
3625 la_reset_pref(ts);
152c35aa 3626 }
78505279 3627
31fd884b
RH		 3628 /* Not used -- it will be tcg_target_call_oarg_reg(). */
3629 memset(op->output_pref, 0, sizeof(op->output_pref));
3630
152c35aa
RH		 3631 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
3632 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 3633 la_global_kill(s, nb_globals);
152c35aa 3634 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 3635 la_global_sync(s, nb_globals);
152c35aa 3636 }
b9c18f56 3637
25f49c5f 3638 /* Record arguments that die in this helper. */
152c35aa 3639 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f 3640 ts = arg_temp(op->args[i]);
39004a71 3641 if (ts->state & TS_DEAD) {
152c35aa 3642 arg_life |= DEAD_ARG << i;
c6e113f5 3643 }
152c35aa 3644 }
25f49c5f
RH		 3645
3646 /* For all live registers, remove call-clobbered prefs. */
3647 la_cross_call(s, nb_temps);
3648
39004a71
RH		 3649 /*
3650 * Input arguments are live for preceding opcodes.
3651 *
3652 * For those arguments that die, and will be allocated in
3653 * registers, clear the register set for that arg, to be
3654 * filled in below. For args that will be on the stack,
3655 * reset to any available reg. Process arguments in reverse
3656 * order so that if a temp is used more than once, the stack
3657 * reset to max happens before the register reset to 0.
3658 */
3659 for (i = nb_iargs - 1; i >= 0; i--) {
3660 const TCGCallArgumentLoc *loc = &info->in[i];
3661 ts = arg_temp(op->args[nb_oargs + i]);
25f49c5f 3662
39004a71
RH		 3663 if (ts->state & TS_DEAD) {
3664 switch (loc->kind) {
3665 case TCG_CALL_ARG_NORMAL:
3666 case TCG_CALL_ARG_EXTEND_U:
3667 case TCG_CALL_ARG_EXTEND_S:
338b61e9 3668 if (arg_slot_reg_p(loc->arg_slot)) {
39004a71
RH		 3669 *la_temp_pref(ts) = 0;
3670 break;
3671 }
3672 /* fall through */
3673 default:
3674 *la_temp_pref(ts) =
3675 tcg_target_available_regs[ts->type];
3676 break;
3677 }
25f49c5f
RH		 3678 ts->state &= ~TS_DEAD;
3679 }
3680 }
3681
39004a71
RH		 3682 /*
3683 * For each input argument, add its input register to prefs.
3684 * If a temp is used once, this produces a single set bit;
3685 * if a temp is used multiple times, this produces a set.
3686 */
3687 for (i = 0; i < nb_iargs; i++) {
3688 const TCGCallArgumentLoc *loc = &info->in[i];
3689 ts = arg_temp(op->args[nb_oargs + i]);
3690
3691 switch (loc->kind) {
3692 case TCG_CALL_ARG_NORMAL:
3693 case TCG_CALL_ARG_EXTEND_U:
3694 case TCG_CALL_ARG_EXTEND_S:
338b61e9 3695 if (arg_slot_reg_p(loc->arg_slot)) {
39004a71
RH		 3696 tcg_regset_set_reg(*la_temp_pref(ts),
3697 tcg_target_call_iarg_regs[loc->arg_slot]);
3698 }
3699 break;
3700 default:
3701 break;
c19f47bf 3702 }
c896fe29 3703 }
c896fe29 3704 }
c896fe29 3705 break;
765b842a 3706 case INDEX_op_insn_start:
c896fe29 3707 break;
5ff9d6a4 3708 case INDEX_op_discard:
5ff9d6a4 3709 /* mark the temporary as dead */
25f49c5f
RH		 3710 ts = arg_temp(op->args[0]);
3711 ts->state = TS_DEAD;
3712 la_reset_pref(ts);
5ff9d6a4 3713 break;
1305c451
RH		 3714
3715 case INDEX_op_add2_i32:
c45cb8bb 3716 opc_new = INDEX_op_add_i32;
f1fae40c 3717 goto do_addsub2;
1305c451 3718 case INDEX_op_sub2_i32:
c45cb8bb 3719 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 3720 goto do_addsub2;
3721 case INDEX_op_add2_i64:
c45cb8bb 3722 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 3723 goto do_addsub2;
3724 case INDEX_op_sub2_i64:
c45cb8bb 3725 opc_new = INDEX_op_sub_i64;
f1fae40c 3726 do_addsub2:
1305c451
RH		 3727 nb_iargs = 4;
3728 nb_oargs = 2;
3729 /* Test if the high part of the operation is dead, but not
3730 the low part. The result can be optimized to a simple
3731 add or sub. This happens often for x86_64 guest when the
3732 cpu mode is set to 32 bit. */
b83eabea
RH		 3733 if (arg_temp(op->args[1])->state == TS_DEAD) {
3734 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 3735 goto do_remove;
3736 }
c45cb8bb
RH		 3737 /* Replace the opcode and adjust the args in place,
3738 leaving 3 unused args at the end. */
3739 op->opc = opc = opc_new;
efee3746
RH		 3740 op->args[1] = op->args[2];
3741 op->args[2] = op->args[4];
1305c451
RH		 3742 /* Fall through and mark the single-word operation live. */
3743 nb_iargs = 2;
3744 nb_oargs = 1;
3745 }
3746 goto do_not_remove;
3747
1414968a 3748 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 3749 opc_new = INDEX_op_mul_i32;
3750 opc_new2 = INDEX_op_muluh_i32;
3751 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 3752 goto do_mul2;
f1fae40c 3753 case INDEX_op_muls2_i32:
c45cb8bb
RH		 3754 opc_new = INDEX_op_mul_i32;
3755 opc_new2 = INDEX_op_mulsh_i32;
3756 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 3757 goto do_mul2;
3758 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 3759 opc_new = INDEX_op_mul_i64;
3760 opc_new2 = INDEX_op_muluh_i64;
3761 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 3762 goto do_mul2;
f1fae40c 3763 case INDEX_op_muls2_i64:
c45cb8bb
RH		 3764 opc_new = INDEX_op_mul_i64;
3765 opc_new2 = INDEX_op_mulsh_i64;
3766 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 3767 goto do_mul2;
f1fae40c 3768 do_mul2:
1414968a
RH		 3769 nb_iargs = 2;
3770 nb_oargs = 2;
b83eabea
RH		 3771 if (arg_temp(op->args[1])->state == TS_DEAD) {
3772 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 3773 /* Both parts of the operation are dead. */
1414968a
RH		 3774 goto do_remove;
3775 }
03271524 3776 /* The high part of the operation is dead; generate the low. */
c45cb8bb 3777 op->opc = opc = opc_new;
efee3746
RH		 3778 op->args[1] = op->args[2];
3779 op->args[2] = op->args[3];
b83eabea 3780 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 3781 /* The low part of the operation is dead; generate the high. */
3782 op->opc = opc = opc_new2;
efee3746
RH		 3783 op->args[0] = op->args[1];
3784 op->args[1] = op->args[2];
3785 op->args[2] = op->args[3];
03271524
RH		 3786 } else {
3787 goto do_not_remove;
1414968a 3788 }
03271524
RH		 3789 /* Mark the single-word operation live. */
3790 nb_oargs = 1;
1414968a
RH		 3791 goto do_not_remove;
3792
c896fe29 3793 default:
1305c451 3794 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 3795 nb_iargs = def->nb_iargs;
3796 nb_oargs = def->nb_oargs;
c896fe29 3797
49516bc0
AJ		 3798 /* Test if the operation can be removed because all
3799 its outputs are dead. We assume that nb_oargs == 0
3800 implies side effects */
3801 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 3802 for (i = 0; i < nb_oargs; i++) {
b83eabea 3803 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 3804 goto do_not_remove;
9c43b68d 3805 }
49516bc0 3806 }
152c35aa
RH		 3807 goto do_remove;
3808 }
3809 goto do_not_remove;
49516bc0 3810
152c35aa
RH		 3811 do_remove:
3812 tcg_op_remove(s, op);
3813 break;
3814
3815 do_not_remove:
152c35aa 3816 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 3817 ts = arg_temp(op->args[i]);
3818
3819 /* Remember the preference of the uses that followed. */
31fd884b
RH		 3820 if (i < ARRAY_SIZE(op->output_pref)) {
3821 op->output_pref[i] = *la_temp_pref(ts);
3822 }
25f49c5f
RH		 3823
3824 /* Output args are dead. */
3825 if (ts->state & TS_DEAD) {
152c35aa 3826 arg_life |= DEAD_ARG << i;
49516bc0 3827 }
25f49c5f 3828 if (ts->state & TS_MEM) {
152c35aa
RH		 3829 arg_life |= SYNC_ARG << i;
3830 }
25f49c5f
RH		 3831 ts->state = TS_DEAD;
3832 la_reset_pref(ts);
152c35aa 3833 }
49516bc0 3834
25f49c5f 3835 /* If end of basic block, update. */
ae36a246
RH		 3836 if (def->flags & TCG_OPF_BB_EXIT) {
3837 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 3838 } else if (def->flags & TCG_OPF_COND_BRANCH) {
3839 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 3840 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 3841 la_bb_end(s, nb_globals, nb_temps);
152c35aa 3842 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 3843 la_global_sync(s, nb_globals);
25f49c5f
RH		 3844 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3845 la_cross_call(s, nb_temps);
3846 }
152c35aa
RH		 3847 }
3848
25f49c5f 3849 /* Record arguments that die in this opcode. */
152c35aa 3850 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 3851 ts = arg_temp(op->args[i]);
3852 if (ts->state & TS_DEAD) {
152c35aa 3853 arg_life |= DEAD_ARG << i;
c896fe29 3854 }
c896fe29 3855 }
25f49c5f
RH		 3856
3857 /* Input arguments are live for preceding opcodes. */
152c35aa 3858 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 3859 ts = arg_temp(op->args[i]);
3860 if (ts->state & TS_DEAD) {
3861 /* For operands that were dead, initially allow
3862 all regs for the type. */
3863 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
3864 ts->state &= ~TS_DEAD;
3865 }
3866 }
3867
3868 /* Incorporate constraints for this operand. */
3869 switch (opc) {
3870 case INDEX_op_mov_i32:
3871 case INDEX_op_mov_i64:
3872 /* Note that these are TCG_OPF_NOT_PRESENT and do not
3873 have proper constraints. That said, special case
3874 moves to propagate preferences backward. */
3875 if (IS_DEAD_ARG(1)) {
3876 *la_temp_pref(arg_temp(op->args[0]))
3877 = *la_temp_pref(arg_temp(op->args[1]));
3878 }
3879 break;
3880
3881 default:
3882 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3883 const TCGArgConstraint *ct = &def->args_ct[i];
3884 TCGRegSet set, *pset;
3885
3886 ts = arg_temp(op->args[i]);
3887 pset = la_temp_pref(ts);
3888 set = *pset;
3889
9be0d080 3890 set &= ct->regs;
bc2b17e6 3891 if (ct->ialias) {
31fd884b 3892 set &= output_pref(op, ct->alias_index);
25f49c5f
RH		 3893 }
3894 /* If the combination is not possible, restart. */
3895 if (set == 0) {
9be0d080 3896 set = ct->regs;
25f49c5f
RH		 3897 }
3898 *pset = set;
3899 }
3900 break;
152c35aa 3901 }
c896fe29
FB		 3902 break;
3903 }
bee158cb 3904 op->life = arg_life;
1ff0a2c5 3905 }
c896fe29 3906}
c896fe29 3907
5a18407f 3908/* Liveness analysis: Convert indirect regs to direct temporaries. */
9bbee4c0
RH		 3909static bool __attribute__((noinline))
3910liveness_pass_2(TCGContext *s)
5a18407f
RH		 3911{
3912 int nb_globals = s->nb_globals;
15fa08f8 3913 int nb_temps, i;
5a18407f 3914 bool changes = false;
15fa08f8 3915 TCGOp *op, *op_next;
5a18407f 3916
5a18407f
RH		 3917 /* Create a temporary for each indirect global. */
3918 for (i = 0; i < nb_globals; ++i) {
3919 TCGTemp *its = &s->temps[i];
3920 if (its->indirect_reg) {
3921 TCGTemp *dts = tcg_temp_alloc(s);
3922 dts->type = its->type;
3923 dts->base_type = its->base_type;
e1e64652 3924 dts->temp_subindex = its->temp_subindex;
c7482438 3925 dts->kind = TEMP_EBB;
b83eabea
RH		 3926 its->state_ptr = dts;
3927 } else {
3928 its->state_ptr = NULL;
5a18407f 3929 }
b83eabea
RH		 3930 /* All globals begin dead. */
3931 its->state = TS_DEAD;
3932 }
3933 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3934 TCGTemp *its = &s->temps[i];
3935 its->state_ptr = NULL;
3936 its->state = TS_DEAD;
5a18407f 3937 }
5a18407f 3938
15fa08f8 3939 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 3940 TCGOpcode opc = op->opc;
3941 const TCGOpDef *def = &tcg_op_defs[opc];
3942 TCGLifeData arg_life = op->life;
3943 int nb_iargs, nb_oargs, call_flags;
b83eabea 3944 TCGTemp *arg_ts, *dir_ts;
5a18407f 3945
5a18407f 3946 if (opc == INDEX_op_call) {
cd9090aa
RH		 3947 nb_oargs = TCGOP_CALLO(op);
3948 nb_iargs = TCGOP_CALLI(op);
90163900 3949 call_flags = tcg_call_flags(op);
5a18407f
RH		 3950 } else {
3951 nb_iargs = def->nb_iargs;
3952 nb_oargs = def->nb_oargs;
3953
3954 /* Set flags similar to how calls require. */
b4cb76e6
RH		 3955 if (def->flags & TCG_OPF_COND_BRANCH) {
3956 /* Like reading globals: sync_globals */
3957 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3958 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 3959 /* Like writing globals: save_globals */
3960 call_flags = 0;
3961 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3962 /* Like reading globals: sync_globals */
3963 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3964 } else {
3965 /* No effect on globals. */
3966 call_flags = (TCG_CALL_NO_READ_GLOBALS |
3967 TCG_CALL_NO_WRITE_GLOBALS);
3968 }
3969 }
3970
3971 /* Make sure that input arguments are available. */
3972 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3973 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3974 dir_ts = arg_ts->state_ptr;
3975 if (dir_ts && arg_ts->state == TS_DEAD) {
3976 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
3977 ? INDEX_op_ld_i32
3978 : INDEX_op_ld_i64);
3979 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
3980
3981 lop->args[0] = temp_arg(dir_ts);
3982 lop->args[1] = temp_arg(arg_ts->mem_base);
3983 lop->args[2] = arg_ts->mem_offset;
3984
3985 /* Loaded, but synced with memory. */
3986 arg_ts->state = TS_MEM;
5a18407f
RH		 3987 }
3988 }
3989
3990 /* Perform input replacement, and mark inputs that became dead.
3991 No action is required except keeping temp_state up to date
3992 so that we reload when needed. */
3993 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3994 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3995 dir_ts = arg_ts->state_ptr;
3996 if (dir_ts) {
3997 op->args[i] = temp_arg(dir_ts);
3998 changes = true;
3999 if (IS_DEAD_ARG(i)) {
4000 arg_ts->state = TS_DEAD;
5a18407f
RH		 4001 }
4002 }
4003 }
4004
4005 /* Liveness analysis should ensure that the following are
4006 all correct, for call sites and basic block end points. */
4007 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
4008 /* Nothing to do */
4009 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
4010 for (i = 0; i < nb_globals; ++i) {
4011 /* Liveness should see that globals are synced back,
4012 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 4013 arg_ts = &s->temps[i];
4014 tcg_debug_assert(arg_ts->state_ptr == 0
4015 || arg_ts->state != 0);
5a18407f
RH		 4016 }
4017 } else {
4018 for (i = 0; i < nb_globals; ++i) {
4019 /* Liveness should see that globals are saved back,
4020 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 4021 arg_ts = &s->temps[i];
4022 tcg_debug_assert(arg_ts->state_ptr == 0
4023 || arg_ts->state == TS_DEAD);
5a18407f
RH		 4024 }
4025 }
4026
4027 /* Outputs become available. */
61f15c48
RH		 4028 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
4029 arg_ts = arg_temp(op->args[0]);
b83eabea 4030 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 4031 if (dir_ts) {
4032 op->args[0] = temp_arg(dir_ts);
4033 changes = true;
4034
4035 /* The output is now live and modified. */
4036 arg_ts->state = 0;
4037
4038 if (NEED_SYNC_ARG(0)) {
4039 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
4040 ? INDEX_op_st_i32
4041 : INDEX_op_st_i64);
d4478943 4042 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
61f15c48
RH		 4043 TCGTemp *out_ts = dir_ts;
4044
4045 if (IS_DEAD_ARG(0)) {
4046 out_ts = arg_temp(op->args[1]);
4047 arg_ts->state = TS_DEAD;
4048 tcg_op_remove(s, op);
4049 } else {
4050 arg_ts->state = TS_MEM;
4051 }
4052
4053 sop->args[0] = temp_arg(out_ts);
4054 sop->args[1] = temp_arg(arg_ts->mem_base);
4055 sop->args[2] = arg_ts->mem_offset;
4056 } else {
4057 tcg_debug_assert(!IS_DEAD_ARG(0));
4058 }
5a18407f 4059 }
61f15c48
RH		 4060 } else {
4061 for (i = 0; i < nb_oargs; i++) {
4062 arg_ts = arg_temp(op->args[i]);
4063 dir_ts = arg_ts->state_ptr;
4064 if (!dir_ts) {
4065 continue;
4066 }
4067 op->args[i] = temp_arg(dir_ts);
4068 changes = true;
5a18407f 4069
61f15c48
RH		 4070 /* The output is now live and modified. */
4071 arg_ts->state = 0;
5a18407f 4072
61f15c48
RH		 4073 /* Sync outputs upon their last write. */
4074 if (NEED_SYNC_ARG(i)) {
4075 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
4076 ? INDEX_op_st_i32
4077 : INDEX_op_st_i64);
d4478943 4078 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
5a18407f 4079
61f15c48
RH		 4080 sop->args[0] = temp_arg(dir_ts);
4081 sop->args[1] = temp_arg(arg_ts->mem_base);
4082 sop->args[2] = arg_ts->mem_offset;
5a18407f 4083
61f15c48
RH		 4084 arg_ts->state = TS_MEM;
4085 }
4086 /* Drop outputs that are dead. */
4087 if (IS_DEAD_ARG(i)) {
4088 arg_ts->state = TS_DEAD;
4089 }
5a18407f
RH		 4090 }
4091 }
4092 }
4093
4094 return changes;
4095}
4096
2272e4a7 4097static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 4098{
31c96417 4099 intptr_t off;
273eb50c 4100 int size, align;
c1c09194 4101
273eb50c
RH		 4102 /* When allocating an object, look at the full type. */
4103 size = tcg_type_size(ts->base_type);
4104 switch (ts->base_type) {
c1c09194 4105 case TCG_TYPE_I32:
31c96417 4106 align = 4;
c1c09194
RH		 4107 break;
4108 case TCG_TYPE_I64:
4109 case TCG_TYPE_V64:
31c96417 4110 align = 8;
c1c09194 4111 break;
43eef72f 4112 case TCG_TYPE_I128:
c1c09194 4113 case TCG_TYPE_V128:
c1c09194 4114 case TCG_TYPE_V256:
43eef72f
RH		 4115 /*
4116 * Note that we do not require aligned storage for V256,
4117 * and that we provide alignment for I128 to match V128,
4118 * even if that's above what the host ABI requires.
4119 */
31c96417 4120 align = 16;
c1c09194
RH		 4121 break;
4122 default:
4123 g_assert_not_reached();
b591dc59 4124 }
c1c09194 4125
b9537d59
RH		 4126 /*
4127 * Assume the stack is sufficiently aligned.
4128 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
4129 * and do not require 16 byte vector alignment. This seems slightly
4130 * easier than fully parameterizing the above switch statement.
4131 */
4132 align = MIN(TCG_TARGET_STACK_ALIGN, align);
c1c09194 4133 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 4134
4135 /* If we've exhausted the stack frame, restart with a smaller TB. */
4136 if (off + size > s->frame_end) {
4137 tcg_raise_tb_overflow(s);
4138 }
c1c09194 4139 s->current_frame_offset = off + size;
9defd1bd 4140#if defined(__sparc__)
273eb50c 4141 off += TCG_TARGET_STACK_BIAS;
9defd1bd 4142#endif
273eb50c
RH		 4143
4144 /* If the object was subdivided, assign memory to all the parts. */
4145 if (ts->base_type != ts->type) {
4146 int part_size = tcg_type_size(ts->type);
4147 int part_count = size / part_size;
4148
4149 /*
4150 * Each part is allocated sequentially in tcg_temp_new_internal.
4151 * Jump back to the first part by subtracting the current index.
4152 */
4153 ts -= ts->temp_subindex;
4154 for (int i = 0; i < part_count; ++i) {
4155 ts[i].mem_offset = off + i * part_size;
4156 ts[i].mem_base = s->frame_temp;
4157 ts[i].mem_allocated = 1;
4158 }
4159 } else {
4160 ts->mem_offset = off;
4161 ts->mem_base = s->frame_temp;
4162 ts->mem_allocated = 1;
4163 }
c896fe29
FB		 4164}
4165
098859f1
RH		 4166/* Assign @reg to @ts, and update reg_to_temp[]. */
4167static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
4168{
4169 if (ts->val_type == TEMP_VAL_REG) {
4170 TCGReg old = ts->reg;
4171 tcg_debug_assert(s->reg_to_temp[old] == ts);
4172 if (old == reg) {
4173 return;
4174 }
4175 s->reg_to_temp[old] = NULL;
4176 }
4177 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
4178 s->reg_to_temp[reg] = ts;
4179 ts->val_type = TEMP_VAL_REG;
4180 ts->reg = reg;
4181}
4182
4183/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
4184static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
4185{
4186 tcg_debug_assert(type != TEMP_VAL_REG);
4187 if (ts->val_type == TEMP_VAL_REG) {
4188 TCGReg reg = ts->reg;
4189 tcg_debug_assert(s->reg_to_temp[reg] == ts);
4190 s->reg_to_temp[reg] = NULL;
4191 }
4192 ts->val_type = type;
4193}
4194
b722452a 4195static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 4196
59d7c14e
RH		 4197/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
4198 mark it free; otherwise mark it dead. */
4199static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 4200{
c0522136
RH		 4201 TCGTempVal new_type;
4202
4203 switch (ts->kind) {
4204 case TEMP_FIXED:
59d7c14e 4205 return;
c0522136 4206 case TEMP_GLOBAL:
f57c6915 4207 case TEMP_TB:
c0522136
RH		 4208 new_type = TEMP_VAL_MEM;
4209 break;
c7482438 4210 case TEMP_EBB:
c0522136
RH		 4211 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
4212 break;
4213 case TEMP_CONST:
4214 new_type = TEMP_VAL_CONST;
4215 break;
4216 default:
4217 g_assert_not_reached();
59d7c14e 4218 }
098859f1 4219 set_temp_val_nonreg(s, ts, new_type);
59d7c14e 4220}
7f6ceedf 4221
59d7c14e
RH		 4222/* Mark a temporary as dead. */
4223static inline void temp_dead(TCGContext *s, TCGTemp *ts)
4224{
4225 temp_free_or_dead(s, ts, 1);
4226}
4227
4228/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
4229 registers needs to be allocated to store a constant. If 'free_or_dead'
4230 is non-zero, subsequently release the temporary; if it is positive, the
4231 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 4232static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
4233 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 4234{
c0522136 4235 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 4236 if (!ts->mem_allocated) {
2272e4a7 4237 temp_allocate_frame(s, ts);
59d7c14e 4238 }
59d7c14e
RH		 4239 switch (ts->val_type) {
4240 case TEMP_VAL_CONST:
4241 /* If we're going to free the temp immediately, then we won't
4242 require it later in a register, so attempt to store the
4243 constant to memory directly. */
4244 if (free_or_dead
4245 && tcg_out_sti(s, ts->type, ts->val,
4246 ts->mem_base->reg, ts->mem_offset)) {
4247 break;
4248 }
4249 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 4250 allocated_regs, preferred_regs);
59d7c14e
RH		 4251 /* fallthrough */
4252
4253 case TEMP_VAL_REG:
4254 tcg_out_st(s, ts->type, ts->reg,
4255 ts->mem_base->reg, ts->mem_offset);
4256 break;
4257
4258 case TEMP_VAL_MEM:
4259 break;
4260
4261 case TEMP_VAL_DEAD:
4262 default:
732e89f4 4263 g_assert_not_reached();
59d7c14e
RH		 4264 }
4265 ts->mem_coherent = 1;
4266 }
4267 if (free_or_dead) {
4268 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 4269 }
7f6ceedf
AJ		 4270}
4271
c896fe29 4272/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 4273static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 4274{
f8b2f202 4275 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 4276 if (ts != NULL) {
98b4e186 4277 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 4278 }
4279}
4280
b016486e
RH		 4281/**
4282 * tcg_reg_alloc:
4283 * @required_regs: Set of registers in which we must allocate.
4284 * @allocated_regs: Set of registers which must be avoided.
4285 * @preferred_regs: Set of registers we should prefer.
4286 * @rev: True if we search the registers in "indirect" order.
4287 *
4288 * The allocated register must be in @required_regs & ~@allocated_regs,
4289 * but if we can put it in @preferred_regs we may save a move later.
4290 */
4291static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
4292 TCGRegSet allocated_regs,
4293 TCGRegSet preferred_regs, bool rev)
c896fe29 4294{
b016486e
RH		 4295 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4296 TCGRegSet reg_ct[2];
91478cef 4297 const int *order;
c896fe29 4298
b016486e
RH		 4299 reg_ct[1] = required_regs & ~allocated_regs;
4300 tcg_debug_assert(reg_ct[1] != 0);
4301 reg_ct[0] = reg_ct[1] & preferred_regs;
4302
4303 /* Skip the preferred_regs option if it cannot be satisfied,
4304 or if the preference made no difference. */
4305 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
4306
91478cef 4307 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 4308
b016486e
RH		 4309 /* Try free registers, preferences first. */
4310 for (j = f; j < 2; j++) {
4311 TCGRegSet set = reg_ct[j];
4312
4313 if (tcg_regset_single(set)) {
4314 /* One register in the set. */
4315 TCGReg reg = tcg_regset_first(set);
4316 if (s->reg_to_temp[reg] == NULL) {
4317 return reg;
4318 }
4319 } else {
4320 for (i = 0; i < n; i++) {
4321 TCGReg reg = order[i];
4322 if (s->reg_to_temp[reg] == NULL &&
4323 tcg_regset_test_reg(set, reg)) {
4324 return reg;
4325 }
4326 }
4327 }
c896fe29
FB		 4328 }
4329
b016486e
RH		 4330 /* We must spill something. */
4331 for (j = f; j < 2; j++) {
4332 TCGRegSet set = reg_ct[j];
4333
4334 if (tcg_regset_single(set)) {
4335 /* One register in the set. */
4336 TCGReg reg = tcg_regset_first(set);
b3915dbb 4337 tcg_reg_free(s, reg, allocated_regs);
c896fe29 4338 return reg;
b016486e
RH		 4339 } else {
4340 for (i = 0; i < n; i++) {
4341 TCGReg reg = order[i];
4342 if (tcg_regset_test_reg(set, reg)) {
4343 tcg_reg_free(s, reg, allocated_regs);
4344 return reg;
4345 }
4346 }
c896fe29
FB		 4347 }
4348 }
4349
732e89f4 4350 g_assert_not_reached();
c896fe29
FB		 4351}
4352
29f5e925
RH		 4353static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
4354 TCGRegSet allocated_regs,
4355 TCGRegSet preferred_regs, bool rev)
4356{
4357 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4358 TCGRegSet reg_ct[2];
4359 const int *order;
4360
4361 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
4362 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
4363 tcg_debug_assert(reg_ct[1] != 0);
4364 reg_ct[0] = reg_ct[1] & preferred_regs;
4365
4366 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
4367
4368 /*
4369 * Skip the preferred_regs option if it cannot be satisfied,
4370 * or if the preference made no difference.
4371 */
4372 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
4373
4374 /*
4375 * Minimize the number of flushes by looking for 2 free registers first,
4376 * then a single flush, then two flushes.
4377 */
4378 for (fmin = 2; fmin >= 0; fmin--) {
4379 for (j = k; j < 2; j++) {
4380 TCGRegSet set = reg_ct[j];
4381
4382 for (i = 0; i < n; i++) {
4383 TCGReg reg = order[i];
4384
4385 if (tcg_regset_test_reg(set, reg)) {
4386 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
4387 if (f >= fmin) {
4388 tcg_reg_free(s, reg, allocated_regs);
4389 tcg_reg_free(s, reg + 1, allocated_regs);
4390 return reg;
4391 }
4392 }
4393 }
4394 }
4395 }
732e89f4 4396 g_assert_not_reached();
29f5e925
RH		 4397}
4398
40ae5c62
RH		 4399/* Make sure the temporary is in a register. If needed, allocate the register
4400 from DESIRED while avoiding ALLOCATED. */
4401static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 4402 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 4403{
4404 TCGReg reg;
4405
4406 switch (ts->val_type) {
4407 case TEMP_VAL_REG:
4408 return;
4409 case TEMP_VAL_CONST:
b016486e 4410 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 4411 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 4412 if (ts->type <= TCG_TYPE_I64) {
4413 tcg_out_movi(s, ts->type, reg, ts->val);
4414 } else {
4e186175
RH		 4415 uint64_t val = ts->val;
4416 MemOp vece = MO_64;
4417
4418 /*
4419 * Find the minimal vector element that matches the constant.
4420 * The targets will, in general, have to do this search anyway,
4421 * do this generically.
4422 */
4e186175
RH		 4423 if (val == dup_const(MO_8, val)) {
4424 vece = MO_8;
4425 } else if (val == dup_const(MO_16, val)) {
4426 vece = MO_16;
0b4286dd 4427 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 4428 vece = MO_32;
4429 }
4430
4431 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 4432 }
40ae5c62
RH		 4433 ts->mem_coherent = 0;
4434 break;
4435 case TEMP_VAL_MEM:
b016486e 4436 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 4437 preferred_regs, ts->indirect_base);
40ae5c62
RH		 4438 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
4439 ts->mem_coherent = 1;
4440 break;
4441 case TEMP_VAL_DEAD:
4442 default:
732e89f4 4443 g_assert_not_reached();
40ae5c62 4444 }
098859f1 4445 set_temp_val_reg(s, ts, reg);
40ae5c62
RH		 4446}
4447
59d7c14e
RH		 4448/* Save a temporary to memory. 'allocated_regs' is used in case a
4449 temporary registers needs to be allocated to store a constant. */
4450static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 4451{
5a18407f
RH		 4452 /* The liveness analysis already ensures that globals are back
4453 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 4454 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 4455}
4456
9814dd27 4457/* save globals to their canonical location and assume they can be
e8996ee0
FB		 4458 modified be the following code. 'allocated_regs' is used in case a
4459 temporary registers needs to be allocated to store a constant. */
4460static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 4461{
ac3b8891 4462 int i, n;
c896fe29 4463
ac3b8891 4464 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 4465 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 4466 }
e5097dc8
FB		 4467}
4468
3d5c5f87
AJ		 4469/* sync globals to their canonical location and assume they can be
4470 read by the following code. 'allocated_regs' is used in case a
4471 temporary registers needs to be allocated to store a constant. */
4472static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
4473{
ac3b8891 4474 int i, n;
3d5c5f87 4475
ac3b8891 4476 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 4477 TCGTemp *ts = &s->temps[i];
5a18407f 4478 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 4479 || ts->kind == TEMP_FIXED
5a18407f 4480 || ts->mem_coherent);
3d5c5f87
AJ		 4481 }
4482}
4483
e5097dc8 4484/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 4485 all globals are stored at their canonical location. */
4486static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 4487{
e5097dc8
FB		 4488 int i;
4489
b13eb728
RH		 4490 for (i = s->nb_globals; i < s->nb_temps; i++) {
4491 TCGTemp *ts = &s->temps[i];
c0522136
RH		 4492
4493 switch (ts->kind) {
f57c6915 4494 case TEMP_TB:
b13eb728 4495 temp_save(s, ts, allocated_regs);
c0522136 4496 break;
c7482438 4497 case TEMP_EBB:
5a18407f
RH		 4498 /* The liveness analysis already ensures that temps are dead.
4499 Keep an tcg_debug_assert for safety. */
4500 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 4501 break;
4502 case TEMP_CONST:
4503 /* Similarly, we should have freed any allocated register. */
4504 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
4505 break;
4506 default:
4507 g_assert_not_reached();
c896fe29
FB		 4508 }
4509 }
e8996ee0
FB		 4510
4511 save_globals(s, allocated_regs);
c896fe29
FB		 4512}
4513
b4cb76e6 4514/*
c7482438
RH		 4515 * At a conditional branch, we assume all temporaries are dead unless
4516 * explicitly live-across-conditional-branch; all globals and local
4517 * temps are synced to their location.
b4cb76e6
RH		 4518 */
4519static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
4520{
4521 sync_globals(s, allocated_regs);
4522
4523 for (int i = s->nb_globals; i < s->nb_temps; i++) {
4524 TCGTemp *ts = &s->temps[i];
4525 /*
4526 * The liveness analysis already ensures that temps are dead.
4527 * Keep tcg_debug_asserts for safety.
4528 */
c0522136 4529 switch (ts->kind) {
f57c6915 4530 case TEMP_TB:
b4cb76e6 4531 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136 4532 break;
c7482438 4533 case TEMP_EBB:
c0522136
RH		 4534 case TEMP_CONST:
4535 break;
4536 default:
4537 g_assert_not_reached();
b4cb76e6
RH		 4538 }
4539 }
4540}
4541
bab1671f 4542/*
c58f4c97 4543 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 4544 */
0fe4fca4 4545static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 4546 tcg_target_ulong val, TCGLifeData arg_life,
4547 TCGRegSet preferred_regs)
e8996ee0 4548{
d63e3b6e 4549 /* ENV should not be modified. */
e01fa97d 4550 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 4551
4552 /* The movi is not explicitly generated here. */
098859f1 4553 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
59d7c14e
RH		 4554 ots->val = val;
4555 ots->mem_coherent = 0;
4556 if (NEED_SYNC_ARG(0)) {
ba87719c 4557 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 4558 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 4559 temp_dead(s, ots);
4c4e1ab2 4560 }
e8996ee0
FB		 4561}
4562
bab1671f
RH		 4563/*
4564 * Specialized code generation for INDEX_op_mov_*.
4565 */
dd186292 4566static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 4567{
dd186292 4568 const TCGLifeData arg_life = op->life;
69e3706d 4569 TCGRegSet allocated_regs, preferred_regs;
c896fe29 4570 TCGTemp *ts, *ots;
450445d5 4571 TCGType otype, itype;
098859f1 4572 TCGReg oreg, ireg;
c896fe29 4573
d21369f5 4574 allocated_regs = s->reserved_regs;
31fd884b 4575 preferred_regs = output_pref(op, 0);
43439139
RH		 4576 ots = arg_temp(op->args[0]);
4577 ts = arg_temp(op->args[1]);
450445d5 4578
d63e3b6e 4579 /* ENV should not be modified. */
e01fa97d 4580 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 4581
450445d5
RH		 4582 /* Note that otype != itype for no-op truncation. */
4583 otype = ots->type;
4584 itype = ts->type;
c29c1d7e 4585
0fe4fca4
PB		 4586 if (ts->val_type == TEMP_VAL_CONST) {
4587 /* propagate constant or generate sti */
4588 tcg_target_ulong val = ts->val;
4589 if (IS_DEAD_ARG(1)) {
4590 temp_dead(s, ts);
4591 }
69e3706d 4592 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 4593 return;
4594 }
4595
4596 /* If the source value is in memory we're going to be forced
4597 to have it in a register in order to perform the copy. Copy
4598 the SOURCE value into its own register first, that way we
4599 don't have to reload SOURCE the next time it is used. */
4600 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 4601 temp_load(s, ts, tcg_target_available_regs[itype],
4602 allocated_regs, preferred_regs);
c29c1d7e 4603 }
0fe4fca4 4604 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
098859f1
RH		 4605 ireg = ts->reg;
4606
d63e3b6e 4607 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 4608 /* mov to a non-saved dead register makes no sense (even with
4609 liveness analysis disabled). */
eabb7b91 4610 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 4611 if (!ots->mem_allocated) {
2272e4a7 4612 temp_allocate_frame(s, ots);
c29c1d7e 4613 }
098859f1 4614 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 4615 if (IS_DEAD_ARG(1)) {
f8bf00f1 4616 temp_dead(s, ts);
c29c1d7e 4617 }
f8bf00f1 4618 temp_dead(s, ots);
098859f1
RH		 4619 return;
4620 }
4621
4622 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
4623 /*
4624 * The mov can be suppressed. Kill input first, so that it
4625 * is unlinked from reg_to_temp, then set the output to the
4626 * reg that we saved from the input.
4627 */
4628 temp_dead(s, ts);
4629 oreg = ireg;
c29c1d7e 4630 } else {
098859f1
RH		 4631 if (ots->val_type == TEMP_VAL_REG) {
4632 oreg = ots->reg;
c896fe29 4633 } else {
098859f1
RH		 4634 /* Make sure to not spill the input register during allocation. */
4635 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
4636 allocated_regs | ((TCGRegSet)1 << ireg),
4637 preferred_regs, ots->indirect_base);
c896fe29 4638 }
098859f1
RH		 4639 if (!tcg_out_mov(s, otype, oreg, ireg)) {
4640 /*
4641 * Cross register class move not supported.
4642 * Store the source register into the destination slot
4643 * and leave the destination temp as TEMP_VAL_MEM.
4644 */
4645 assert(!temp_readonly(ots));
4646 if (!ts->mem_allocated) {
4647 temp_allocate_frame(s, ots);
4648 }
4649 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
4650 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
4651 ots->mem_coherent = 1;
4652 return;
c896fe29 4653 }
ec7a869d 4654 }
098859f1
RH		 4655 set_temp_val_reg(s, ots, oreg);
4656 ots->mem_coherent = 0;
4657
4658 if (NEED_SYNC_ARG(0)) {
4659 temp_sync(s, ots, allocated_regs, 0, 0);
4660 }
c896fe29
FB		 4661}
4662
bab1671f
RH		 4663/*
4664 * Specialized code generation for INDEX_op_dup_vec.
4665 */
4666static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
4667{
4668 const TCGLifeData arg_life = op->life;
4669 TCGRegSet dup_out_regs, dup_in_regs;
4670 TCGTemp *its, *ots;
4671 TCGType itype, vtype;
4672 unsigned vece;
31c96417 4673 int lowpart_ofs;
bab1671f
RH		 4674 bool ok;
4675
4676 ots = arg_temp(op->args[0]);
4677 its = arg_temp(op->args[1]);
4678
4679 /* ENV should not be modified. */
e01fa97d 4680 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 4681
4682 itype = its->type;
4683 vece = TCGOP_VECE(op);
4684 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4685
4686 if (its->val_type == TEMP_VAL_CONST) {
4687 /* Propagate constant via movi -> dupi. */
4688 tcg_target_ulong val = its->val;
4689 if (IS_DEAD_ARG(1)) {
4690 temp_dead(s, its);
4691 }
31fd884b 4692 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
bab1671f
RH		 4693 return;
4694 }
4695
9be0d080
RH		 4696 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
4697 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 4698
4699 /* Allocate the output register now. */
4700 if (ots->val_type != TEMP_VAL_REG) {
4701 TCGRegSet allocated_regs = s->reserved_regs;
098859f1 4702 TCGReg oreg;
bab1671f
RH		 4703
4704 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
4705 /* Make sure to not spill the input register. */
4706 tcg_regset_set_reg(allocated_regs, its->reg);
4707 }
098859f1 4708 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 4709 output_pref(op, 0), ots->indirect_base);
098859f1 4710 set_temp_val_reg(s, ots, oreg);
bab1671f
RH		 4711 }
4712
4713 switch (its->val_type) {
4714 case TEMP_VAL_REG:
4715 /*
4716 * The dup constriaints must be broad, covering all possible VECE.
4717 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
4718 * to fail, indicating that extra moves are required for that case.
4719 */
4720 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
4721 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
4722 goto done;
4723 }
4724 /* Try again from memory or a vector input register. */
4725 }
4726 if (!its->mem_coherent) {
4727 /*
4728 * The input register is not synced, and so an extra store
4729 * would be required to use memory. Attempt an integer-vector
4730 * register move first. We do not have a TCGRegSet for this.
4731 */
4732 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
4733 break;
4734 }
4735 /* Sync the temp back to its slot and load from there. */
4736 temp_sync(s, its, s->reserved_regs, 0, 0);
4737 }
4738 /* fall through */
4739
4740 case TEMP_VAL_MEM:
31c96417
RH		 4741 lowpart_ofs = 0;
4742 if (HOST_BIG_ENDIAN) {
4743 lowpart_ofs = tcg_type_size(itype) - (1 << vece);
4744 }
d6ecb4a9 4745 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
31c96417 4746 its->mem_offset + lowpart_ofs)) {
d6ecb4a9
RH		 4747 goto done;
4748 }
098859f1 4749 /* Load the input into the destination vector register. */
bab1671f
RH		 4750 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
4751 break;
4752
4753 default:
4754 g_assert_not_reached();
4755 }
4756
4757 /* We now have a vector input register, so dup must succeed. */
4758 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
4759 tcg_debug_assert(ok);
4760
4761 done:
36f5539c 4762 ots->mem_coherent = 0;
bab1671f
RH		 4763 if (IS_DEAD_ARG(1)) {
4764 temp_dead(s, its);
4765 }
4766 if (NEED_SYNC_ARG(0)) {
4767 temp_sync(s, ots, s->reserved_regs, 0, 0);
4768 }
4769 if (IS_DEAD_ARG(0)) {
4770 temp_dead(s, ots);
4771 }
4772}
4773
dd186292 4774static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 4775{
dd186292
RH		 4776 const TCGLifeData arg_life = op->life;
4777 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 4778 TCGRegSet i_allocated_regs;
4779 TCGRegSet o_allocated_regs;
b6638662
RH		 4780 int i, k, nb_iargs, nb_oargs;
4781 TCGReg reg;
c896fe29
FB		 4782 TCGArg arg;
4783 const TCGArgConstraint *arg_ct;
4784 TCGTemp *ts;
4785 TCGArg new_args[TCG_MAX_OP_ARGS];
4786 int const_args[TCG_MAX_OP_ARGS];
4787
4788 nb_oargs = def->nb_oargs;
4789 nb_iargs = def->nb_iargs;
4790
4791 /* copy constants */
a813e36f 4792 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 4793 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 4794 sizeof(TCGArg) * def->nb_cargs);
4795
d21369f5
RH		 4796 i_allocated_regs = s->reserved_regs;
4797 o_allocated_regs = s->reserved_regs;
82790a87 4798
a813e36f 4799 /* satisfy input constraints */
dd186292 4800 for (k = 0; k < nb_iargs; k++) {
29f5e925
RH		 4801 TCGRegSet i_preferred_regs, i_required_regs;
4802 bool allocate_new_reg, copyto_new_reg;
4803 TCGTemp *ts2;
4804 int i1, i2;
d62816f2 4805
66792f90 4806 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 4807 arg = op->args[i];
c896fe29 4808 arg_ct = &def->args_ct[i];
43439139 4809 ts = arg_temp(arg);
40ae5c62
RH		 4810
4811 if (ts->val_type == TEMP_VAL_CONST
ebe92db2 4812 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct, TCGOP_VECE(op))) {
40ae5c62
RH		 4813 /* constant is OK for instruction */
4814 const_args[i] = 1;
4815 new_args[i] = ts->val;
d62816f2 4816 continue;
c896fe29 4817 }
40ae5c62 4818
1c1824dc
RH		 4819 reg = ts->reg;
4820 i_preferred_regs = 0;
29f5e925 4821 i_required_regs = arg_ct->regs;
1c1824dc 4822 allocate_new_reg = false;
29f5e925
RH		 4823 copyto_new_reg = false;
4824
4825 switch (arg_ct->pair) {
4826 case 0: /* not paired */
4827 if (arg_ct->ialias) {
31fd884b 4828 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 4829
4830 /*
4831 * If the input is readonly, then it cannot also be an
4832 * output and aliased to itself. If the input is not
4833 * dead after the instruction, we must allocate a new
4834 * register and move it.
4835 */
22d2e535
IL		 4836 if (temp_readonly(ts) || !IS_DEAD_ARG(i)
4837 || def->args_ct[arg_ct->alias_index].newreg) {
29f5e925
RH		 4838 allocate_new_reg = true;
4839 } else if (ts->val_type == TEMP_VAL_REG) {
4840 /*
4841 * Check if the current register has already been
4842 * allocated for another input.
4843 */
4844 allocate_new_reg =
4845 tcg_regset_test_reg(i_allocated_regs, reg);
4846 }
4847 }
4848 if (!allocate_new_reg) {
4849 temp_load(s, ts, i_required_regs, i_allocated_regs,
4850 i_preferred_regs);
4851 reg = ts->reg;
4852 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
4853 }
4854 if (allocate_new_reg) {
4855 /*
4856 * Allocate a new register matching the constraint
4857 * and move the temporary register into it.
4858 */
4859 temp_load(s, ts, tcg_target_available_regs[ts->type],
4860 i_allocated_regs, 0);
4861 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
4862 i_preferred_regs, ts->indirect_base);
4863 copyto_new_reg = true;
4864 }
4865 break;
4866
4867 case 1:
4868 /* First of an input pair; if i1 == i2, the second is an output. */
4869 i1 = i;
4870 i2 = arg_ct->pair_index;
4871 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
4872
4873 /*
4874 * It is easier to default to allocating a new pair
4875 * and to identify a few cases where it's not required.
4876 */
4877 if (arg_ct->ialias) {
31fd884b 4878 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 4879 if (IS_DEAD_ARG(i1) &&
4880 IS_DEAD_ARG(i2) &&
4881 !temp_readonly(ts) &&
4882 ts->val_type == TEMP_VAL_REG &&
4883 ts->reg < TCG_TARGET_NB_REGS - 1 &&
4884 tcg_regset_test_reg(i_required_regs, reg) &&
4885 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4886 !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
4887 (ts2
4888 ? ts2->val_type == TEMP_VAL_REG &&
4889 ts2->reg == reg + 1 &&
4890 !temp_readonly(ts2)
4891 : s->reg_to_temp[reg + 1] == NULL)) {
4892 break;
4893 }
4894 } else {
4895 /* Without aliasing, the pair must also be an input. */
4896 tcg_debug_assert(ts2);
4897 if (ts->val_type == TEMP_VAL_REG &&
4898 ts2->val_type == TEMP_VAL_REG &&
4899 ts2->reg == reg + 1 &&
4900 tcg_regset_test_reg(i_required_regs, reg)) {
4901 break;
4902 }
4903 }
4904 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
4905 0, ts->indirect_base);
4906 goto do_pair;
4907
4908 case 2: /* pair second */
4909 reg = new_args[arg_ct->pair_index] + 1;
4910 goto do_pair;
1c1824dc 4911
29f5e925
RH		 4912 case 3: /* ialias with second output, no first input */
4913 tcg_debug_assert(arg_ct->ialias);
31fd884b 4914 i_preferred_regs = output_pref(op, arg_ct->alias_index);
d62816f2 4915
29f5e925
RH		 4916 if (IS_DEAD_ARG(i) &&
4917 !temp_readonly(ts) &&
4918 ts->val_type == TEMP_VAL_REG &&
4919 reg > 0 &&
4920 s->reg_to_temp[reg - 1] == NULL &&
4921 tcg_regset_test_reg(i_required_regs, reg) &&
4922 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4923 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
4924 tcg_regset_set_reg(i_allocated_regs, reg - 1);
4925 break;
4926 }
4927 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
4928 i_allocated_regs, 0,
4929 ts->indirect_base);
4930 tcg_regset_set_reg(i_allocated_regs, reg);
4931 reg += 1;
4932 goto do_pair;
4933
4934 do_pair:
c0522136 4935 /*
29f5e925
RH		 4936 * If an aliased input is not dead after the instruction,
4937 * we must allocate a new register and move it.
c0522136 4938 */
29f5e925
RH		 4939 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
4940 TCGRegSet t_allocated_regs = i_allocated_regs;
4941
1c1824dc 4942 /*
29f5e925
RH		 4943 * Because of the alias, and the continued life, make sure
4944 * that the temp is somewhere *other* than the reg pair,
4945 * and we get a copy in reg.
1c1824dc 4946 */
29f5e925
RH		 4947 tcg_regset_set_reg(t_allocated_regs, reg);
4948 tcg_regset_set_reg(t_allocated_regs, reg + 1);
4949 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
4950 /* If ts was already in reg, copy it somewhere else. */
4951 TCGReg nr;
4952 bool ok;
4953
4954 tcg_debug_assert(ts->kind != TEMP_FIXED);
4955 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
4956 t_allocated_regs, 0, ts->indirect_base);
4957 ok = tcg_out_mov(s, ts->type, nr, reg);
4958 tcg_debug_assert(ok);
4959
4960 set_temp_val_reg(s, ts, nr);
4961 } else {
4962 temp_load(s, ts, tcg_target_available_regs[ts->type],
4963 t_allocated_regs, 0);
4964 copyto_new_reg = true;
4965 }
4966 } else {
4967 /* Preferably allocate to reg, otherwise copy. */
4968 i_required_regs = (TCGRegSet)1 << reg;
4969 temp_load(s, ts, i_required_regs, i_allocated_regs,
4970 i_preferred_regs);
4971 copyto_new_reg = ts->reg != reg;
5ff9d6a4 4972 }
29f5e925 4973 break;
d62816f2 4974
29f5e925
RH		 4975 default:
4976 g_assert_not_reached();
1c1824dc 4977 }
d62816f2 4978
29f5e925 4979 if (copyto_new_reg) {
78113e83 4980 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 4981 /*
4982 * Cross register class move not supported. Sync the
4983 * temp back to its slot and load from there.
4984 */
4985 temp_sync(s, ts, i_allocated_regs, 0, 0);
4986 tcg_out_ld(s, ts->type, reg,
4987 ts->mem_base->reg, ts->mem_offset);
78113e83 4988 }
c896fe29 4989 }
c896fe29
FB		 4990 new_args[i] = reg;
4991 const_args[i] = 0;
82790a87 4992 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29 4993 }
a813e36f 4994
a52ad07e
AJ		 4995 /* mark dead temporaries and free the associated registers */
4996 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4997 if (IS_DEAD_ARG(i)) {
43439139 4998 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 4999 }
5000 }
5001
b4cb76e6
RH		 5002 if (def->flags & TCG_OPF_COND_BRANCH) {
5003 tcg_reg_alloc_cbranch(s, i_allocated_regs);
5004 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 5005 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 5006 } else {
e8996ee0 5007 if (def->flags & TCG_OPF_CALL_CLOBBER) {
a813e36f 5008 /* XXX: permit generic clobber register list ? */
c8074023
RH		 5009 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
5010 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 5011 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 5012 }
c896fe29 5013 }
3d5c5f87
AJ		 5014 }
5015 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
5016 /* sync globals if the op has side effects and might trigger
5017 an exception. */
82790a87 5018 sync_globals(s, i_allocated_regs);
c896fe29 5019 }
a813e36f 5020
e8996ee0 5021 /* satisfy the output constraints */
e8996ee0 5022 for(k = 0; k < nb_oargs; k++) {
66792f90 5023 i = def->args_ct[k].sort_index;
dd186292 5024 arg = op->args[i];
e8996ee0 5025 arg_ct = &def->args_ct[i];
43439139 5026 ts = arg_temp(arg);
d63e3b6e
RH		 5027
5028 /* ENV should not be modified. */
e01fa97d 5029 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 5030
29f5e925
RH		 5031 switch (arg_ct->pair) {
5032 case 0: /* not paired */
5033 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
5034 reg = new_args[arg_ct->alias_index];
5035 } else if (arg_ct->newreg) {
5036 reg = tcg_reg_alloc(s, arg_ct->regs,
5037 i_allocated_regs | o_allocated_regs,
31fd884b 5038 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 5039 } else {
5040 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
31fd884b 5041 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 5042 }
5043 break;
5044
5045 case 1: /* first of pair */
29f5e925
RH		 5046 if (arg_ct->oalias) {
5047 reg = new_args[arg_ct->alias_index];
ca5bed07
RH		 5048 } else if (arg_ct->newreg) {
5049 reg = tcg_reg_alloc_pair(s, arg_ct->regs,
5050 i_allocated_regs | o_allocated_regs,
5051 output_pref(op, k),
5052 ts->indirect_base);
5053 } else {
5054 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
5055 output_pref(op, k),
5056 ts->indirect_base);
29f5e925 5057 }
29f5e925
RH		 5058 break;
5059
5060 case 2: /* second of pair */
29f5e925
RH		 5061 if (arg_ct->oalias) {
5062 reg = new_args[arg_ct->alias_index];
5063 } else {
5064 reg = new_args[arg_ct->pair_index] + 1;
5065 }
5066 break;
5067
5068 case 3: /* first of pair, aliasing with a second input */
5069 tcg_debug_assert(!arg_ct->newreg);
5070 reg = new_args[arg_ct->pair_index] - 1;
5071 break;
5072
5073 default:
5074 g_assert_not_reached();
c896fe29 5075 }
82790a87 5076 tcg_regset_set_reg(o_allocated_regs, reg);
098859f1 5077 set_temp_val_reg(s, ts, reg);
d63e3b6e 5078 ts->mem_coherent = 0;
e8996ee0 5079 new_args[i] = reg;
c896fe29 5080 }
c896fe29
FB		 5081 }
5082
c896fe29 5083 /* emit instruction */
678155b2
RH		 5084 switch (op->opc) {
5085 case INDEX_op_ext8s_i32:
5086 tcg_out_ext8s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
5087 break;
5088 case INDEX_op_ext8s_i64:
5089 tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
5090 break;
d0e66c89
RH		 5091 case INDEX_op_ext8u_i32:
5092 case INDEX_op_ext8u_i64:
5093 tcg_out_ext8u(s, new_args[0], new_args[1]);
5094 break;
753e42ea
RH		 5095 case INDEX_op_ext16s_i32:
5096 tcg_out_ext16s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
5097 break;
5098 case INDEX_op_ext16s_i64:
5099 tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
5100 break;
379afdff
RH		 5101 case INDEX_op_ext16u_i32:
5102 case INDEX_op_ext16u_i64:
5103 tcg_out_ext16u(s, new_args[0], new_args[1]);
5104 break;
52bf3398
RH		 5105 case INDEX_op_ext32s_i64:
5106 tcg_out_ext32s(s, new_args[0], new_args[1]);
5107 break;
9ecf5f61
RH		 5108 case INDEX_op_ext32u_i64:
5109 tcg_out_ext32u(s, new_args[0], new_args[1]);
5110 break;
9c6aa274
RH		 5111 case INDEX_op_ext_i32_i64:
5112 tcg_out_exts_i32_i64(s, new_args[0], new_args[1]);
5113 break;
b9bfe000
RH		 5114 case INDEX_op_extu_i32_i64:
5115 tcg_out_extu_i32_i64(s, new_args[0], new_args[1]);
5116 break;
b8b94ac6
RH		 5117 case INDEX_op_extrl_i64_i32:
5118 tcg_out_extrl_i64_i32(s, new_args[0], new_args[1]);
5119 break;
678155b2
RH		 5120 default:
5121 if (def->flags & TCG_OPF_VECTOR) {
5122 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
5123 new_args, const_args);
5124 } else {
5125 tcg_out_op(s, op->opc, new_args, const_args);
5126 }
5127 break;
d2fd745f
RH		 5128 }
5129
c896fe29
FB		 5130 /* move the outputs in the correct register if needed */
5131 for(i = 0; i < nb_oargs; i++) {
43439139 5132 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 5133
5134 /* ENV should not be modified. */
e01fa97d 5135 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 5136
ec7a869d 5137 if (NEED_SYNC_ARG(i)) {
98b4e186 5138 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 5139 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 5140 temp_dead(s, ts);
ec7a869d 5141 }
c896fe29
FB		 5142 }
5143}
5144
efe86b21
RH		 5145static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
5146{
5147 const TCGLifeData arg_life = op->life;
5148 TCGTemp *ots, *itsl, *itsh;
5149 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
5150
5151 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
5152 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
5153 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
5154
5155 ots = arg_temp(op->args[0]);
5156 itsl = arg_temp(op->args[1]);
5157 itsh = arg_temp(op->args[2]);
5158
5159 /* ENV should not be modified. */
5160 tcg_debug_assert(!temp_readonly(ots));
5161
5162 /* Allocate the output register now. */
5163 if (ots->val_type != TEMP_VAL_REG) {
5164 TCGRegSet allocated_regs = s->reserved_regs;
5165 TCGRegSet dup_out_regs =
5166 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
098859f1 5167 TCGReg oreg;
efe86b21
RH		 5168
5169 /* Make sure to not spill the input registers. */
5170 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
5171 tcg_regset_set_reg(allocated_regs, itsl->reg);
5172 }
5173 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
5174 tcg_regset_set_reg(allocated_regs, itsh->reg);
5175 }
5176
098859f1 5177 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 5178 output_pref(op, 0), ots->indirect_base);
098859f1 5179 set_temp_val_reg(s, ots, oreg);
efe86b21
RH		 5180 }
5181
5182 /* Promote dup2 of immediates to dupi_vec. */
5183 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
5184 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
5185 MemOp vece = MO_64;
5186
5187 if (val == dup_const(MO_8, val)) {
5188 vece = MO_8;
5189 } else if (val == dup_const(MO_16, val)) {
5190 vece = MO_16;
5191 } else if (val == dup_const(MO_32, val)) {
5192 vece = MO_32;
5193 }
5194
5195 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
5196 goto done;
5197 }
5198
5199 /* If the two inputs form one 64-bit value, try dupm_vec. */
aef85402
RH		 5200 if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
5201 itsh->temp_subindex == !HOST_BIG_ENDIAN &&
5202 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
5203 TCGTemp *its = itsl - HOST_BIG_ENDIAN;
5204
5205 temp_sync(s, its + 0, s->reserved_regs, 0, 0);
5206 temp_sync(s, its + 1, s->reserved_regs, 0, 0);
5207
efe86b21
RH		 5208 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
5209 its->mem_base->reg, its->mem_offset)) {
5210 goto done;
5211 }
5212 }
5213
5214 /* Fall back to generic expansion. */
5215 return false;
5216
5217 done:
36f5539c 5218 ots->mem_coherent = 0;
efe86b21
RH		 5219 if (IS_DEAD_ARG(1)) {
5220 temp_dead(s, itsl);
5221 }
5222 if (IS_DEAD_ARG(2)) {
5223 temp_dead(s, itsh);
5224 }
5225 if (NEED_SYNC_ARG(0)) {
5226 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
5227 } else if (IS_DEAD_ARG(0)) {
5228 temp_dead(s, ots);
5229 }
5230 return true;
5231}
5232
39004a71
RH		 5233static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
5234 TCGRegSet allocated_regs)
c896fe29 5235{
39004a71
RH		 5236 if (ts->val_type == TEMP_VAL_REG) {
5237 if (ts->reg != reg) {
5238 tcg_reg_free(s, reg, allocated_regs);
5239 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
5240 /*
5241 * Cross register class move not supported. Sync the
5242 * temp back to its slot and load from there.
5243 */
5244 temp_sync(s, ts, allocated_regs, 0, 0);
5245 tcg_out_ld(s, ts->type, reg,
5246 ts->mem_base->reg, ts->mem_offset);
5247 }
5248 }
5249 } else {
5250 TCGRegSet arg_set = 0;
c896fe29 5251
39004a71
RH		 5252 tcg_reg_free(s, reg, allocated_regs);
5253 tcg_regset_set_reg(arg_set, reg);
5254 temp_load(s, ts, arg_set, allocated_regs, 0);
b03cce8e 5255 }
39004a71 5256}
39cf05d3 5257
d78e4a4f 5258static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts,
39004a71
RH		 5259 TCGRegSet allocated_regs)
5260{
5261 /*
5262 * When the destination is on the stack, load up the temp and store.
5263 * If there are many call-saved registers, the temp might live to
5264 * see another use; otherwise it'll be discarded.
5265 */
5266 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
5267 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
d78e4a4f 5268 arg_slot_stk_ofs(arg_slot));
39004a71 5269}
a813e36f 5270
39004a71
RH		 5271static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
5272 TCGTemp *ts, TCGRegSet *allocated_regs)
5273{
338b61e9 5274 if (arg_slot_reg_p(l->arg_slot)) {
39004a71
RH		 5275 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
5276 load_arg_reg(s, reg, ts, *allocated_regs);
5277 tcg_regset_set_reg(*allocated_regs, reg);
5278 } else {
d78e4a4f 5279 load_arg_stk(s, l->arg_slot, ts, *allocated_regs);
39004a71
RH		 5280 }
5281}
40ae5c62 5282
d78e4a4f 5283static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base,
313bdea8
RH		 5284 intptr_t ref_off, TCGRegSet *allocated_regs)
5285{
5286 TCGReg reg;
313bdea8 5287
d78e4a4f 5288 if (arg_slot_reg_p(arg_slot)) {
313bdea8
RH		 5289 reg = tcg_target_call_iarg_regs[arg_slot];
5290 tcg_reg_free(s, reg, *allocated_regs);
5291 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
5292 tcg_regset_set_reg(*allocated_regs, reg);
5293 } else {
5294 reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR],
5295 *allocated_regs, 0, false);
5296 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
5297 tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK,
d78e4a4f 5298 arg_slot_stk_ofs(arg_slot));
313bdea8
RH		 5299 }
5300}
5301
39004a71
RH		 5302static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
5303{
5304 const int nb_oargs = TCGOP_CALLO(op);
5305 const int nb_iargs = TCGOP_CALLI(op);
5306 const TCGLifeData arg_life = op->life;
5307 const TCGHelperInfo *info = tcg_call_info(op);
5308 TCGRegSet allocated_regs = s->reserved_regs;
5309 int i;
40ae5c62 5310
39004a71
RH		 5311 /*
5312 * Move inputs into place in reverse order,
5313 * so that we place stacked arguments first.
5314 */
5315 for (i = nb_iargs - 1; i >= 0; --i) {
5316 const TCGCallArgumentLoc *loc = &info->in[i];
5317 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
40ae5c62 5318
39004a71
RH		 5319 switch (loc->kind) {
5320 case TCG_CALL_ARG_NORMAL:
5321 case TCG_CALL_ARG_EXTEND_U:
5322 case TCG_CALL_ARG_EXTEND_S:
5323 load_arg_normal(s, loc, ts, &allocated_regs);
5324 break;
313bdea8
RH		 5325 case TCG_CALL_ARG_BY_REF:
5326 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
5327 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK,
d78e4a4f 5328 arg_slot_stk_ofs(loc->ref_slot),
313bdea8
RH		 5329 &allocated_regs);
5330 break;
5331 case TCG_CALL_ARG_BY_REF_N:
5332 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
5333 break;
39004a71
RH		 5334 default:
5335 g_assert_not_reached();
c896fe29 5336 }
c896fe29 5337 }
a813e36f 5338
39004a71 5339 /* Mark dead temporaries and free the associated registers. */
dd186292 5340 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 5341 if (IS_DEAD_ARG(i)) {
43439139 5342 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 5343 }
5344 }
a813e36f 5345
39004a71 5346 /* Clobber call registers. */
c8074023
RH		 5347 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
5348 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 5349 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 5350 }
5351 }
78505279 5352
39004a71
RH		 5353 /*
5354 * Save globals if they might be written by the helper,
5355 * sync them if they might be read.
5356 */
5357 if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
78505279 5358 /* Nothing to do */
39004a71 5359 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
78505279
AJ		 5360 sync_globals(s, allocated_regs);
5361 } else {
b9c18f56
AJ		 5362 save_globals(s, allocated_regs);
5363 }
c896fe29 5364
313bdea8
RH		 5365 /*
5366 * If the ABI passes a pointer to the returned struct as the first
5367 * argument, load that now. Pass a pointer to the output home slot.
5368 */
5369 if (info->out_kind == TCG_CALL_RET_BY_REF) {
5370 TCGTemp *ts = arg_temp(op->args[0]);
5371
5372 if (!ts->mem_allocated) {
5373 temp_allocate_frame(s, ts);
5374 }
5375 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs);
5376 }
5377
cee44b03 5378 tcg_out_call(s, tcg_call_func(op), info);
c896fe29 5379
39004a71
RH		 5380 /* Assign output registers and emit moves if needed. */
5381 switch (info->out_kind) {
5382 case TCG_CALL_RET_NORMAL:
5383 for (i = 0; i < nb_oargs; i++) {
5384 TCGTemp *ts = arg_temp(op->args[i]);
5e3d0c19 5385 TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i);
d63e3b6e 5386
39004a71
RH		 5387 /* ENV should not be modified. */
5388 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 5389
39004a71
RH		 5390 set_temp_val_reg(s, ts, reg);
5391 ts->mem_coherent = 0;
5392 }
5393 break;
313bdea8 5394
c6556aa0
RH		 5395 case TCG_CALL_RET_BY_VEC:
5396 {
5397 TCGTemp *ts = arg_temp(op->args[0]);
5398
5399 tcg_debug_assert(ts->base_type == TCG_TYPE_I128);
5400 tcg_debug_assert(ts->temp_subindex == 0);
5401 if (!ts->mem_allocated) {
5402 temp_allocate_frame(s, ts);
5403 }
5404 tcg_out_st(s, TCG_TYPE_V128,
5405 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
5406 ts->mem_base->reg, ts->mem_offset);
5407 }
5408 /* fall through to mark all parts in memory */
5409
313bdea8
RH		 5410 case TCG_CALL_RET_BY_REF:
5411 /* The callee has performed a write through the reference. */
5412 for (i = 0; i < nb_oargs; i++) {
5413 TCGTemp *ts = arg_temp(op->args[i]);
5414 ts->val_type = TEMP_VAL_MEM;
5415 }
5416 break;
5417
39004a71
RH		 5418 default:
5419 g_assert_not_reached();
5420 }
5421
5422 /* Flush or discard output registers as needed. */
5423 for (i = 0; i < nb_oargs; i++) {
5424 TCGTemp *ts = arg_temp(op->args[i]);
d63e3b6e 5425 if (NEED_SYNC_ARG(i)) {
39004a71 5426 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
d63e3b6e
RH		 5427 } else if (IS_DEAD_ARG(i)) {
5428 temp_dead(s, ts);
c896fe29
FB		 5429 }
5430 }
c896fe29
FB		 5431}
5432
e63b8a29
RH		 5433/**
5434 * atom_and_align_for_opc:
5435 * @s: tcg context
5436 * @opc: memory operation code
5437 * @host_atom: MO_ATOM_{IFALIGN,WITHIN16,SUBALIGN} for host operations
5438 * @allow_two_ops: true if we are prepared to issue two operations
5439 *
5440 * Return the alignment and atomicity to use for the inline fast path
5441 * for the given memory operation. The alignment may be larger than
5442 * that specified in @opc, and the correct alignment will be diagnosed
5443 * by the slow path helper.
5444 *
5445 * If @allow_two_ops, the host is prepared to test for 2x alignment,
5446 * and issue two loads or stores for subalignment.
5447 */
5448static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
5449 MemOp host_atom, bool allow_two_ops)
5450{
5451 MemOp align = get_alignment_bits(opc);
5452 MemOp size = opc & MO_SIZE;
5453 MemOp half = size ? size - 1 : 0;
cbb14556 5454 MemOp atom = opc & MO_ATOM_MASK;
e63b8a29 5455 MemOp atmax;
e63b8a29
RH		 5456
5457 switch (atom) {
5458 case MO_ATOM_NONE:
5459 /* The operation requires no specific atomicity. */
5460 atmax = MO_8;
5461 break;
5462
5463 case MO_ATOM_IFALIGN:
5464 atmax = size;
5465 break;
5466
5467 case MO_ATOM_IFALIGN_PAIR:
5468 atmax = half;
5469 break;
5470
5471 case MO_ATOM_WITHIN16:
5472 atmax = size;
5473 if (size == MO_128) {
5474 /* Misalignment implies !within16, and therefore no atomicity. */
5475 } else if (host_atom != MO_ATOM_WITHIN16) {
5476 /* The host does not implement within16, so require alignment. */
5477 align = MAX(align, size);
5478 }
5479 break;
5480
5481 case MO_ATOM_WITHIN16_PAIR:
5482 atmax = size;
5483 /*
5484 * Misalignment implies !within16, and therefore half atomicity.
5485 * Any host prepared for two operations can implement this with
5486 * half alignment.
5487 */
5488 if (host_atom != MO_ATOM_WITHIN16 && allow_two_ops) {
5489 align = MAX(align, half);
5490 }
5491 break;
5492
5493 case MO_ATOM_SUBALIGN:
5494 atmax = size;
5495 if (host_atom != MO_ATOM_SUBALIGN) {
5496 /* If unaligned but not odd, there are subobjects up to half. */
5497 if (allow_two_ops) {
5498 align = MAX(align, half);
5499 } else {
5500 align = MAX(align, size);
5501 }
5502 }
5503 break;
5504
5505 default:
5506 g_assert_not_reached();
5507 }
5508
5509 return (TCGAtomAlign){ .atom = atmax, .align = align };
5510}
5511
8429a1ca
RH		 5512/*
5513 * Similarly for qemu_ld/st slow path helpers.
5514 * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously,
5515 * using only the provided backend tcg_out_* functions.
5516 */
5517
5518static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot)
5519{
5520 int ofs = arg_slot_stk_ofs(slot);
5521
5522 /*
5523 * Each stack slot is TCG_TARGET_LONG_BITS. If the host does not
5524 * require extension to uint64_t, adjust the address for uint32_t.
5525 */
5526 if (HOST_BIG_ENDIAN &&
5527 TCG_TARGET_REG_BITS == 64 &&
5528 type == TCG_TYPE_I32) {
5529 ofs += 4;
5530 }
5531 return ofs;
5532}
5533
8d314041
RH		 5534static void tcg_out_helper_load_slots(TCGContext *s,
5535 unsigned nmov, TCGMovExtend *mov,
5536 const TCGLdstHelperParam *parm)
8429a1ca 5537{
8d314041 5538 unsigned i;
2462e30e
RH		 5539 TCGReg dst3;
5540
8d314041
RH		 5541 /*
5542 * Start from the end, storing to the stack first.
5543 * This frees those registers, so we need not consider overlap.
5544 */
5545 for (i = nmov; i-- > 0; ) {
5546 unsigned slot = mov[i].dst;
5547
5548 if (arg_slot_reg_p(slot)) {
5549 goto found_reg;
5550 }
5551
5552 TCGReg src = mov[i].src;
5553 TCGType dst_type = mov[i].dst_type;
5554 MemOp dst_mo = dst_type == TCG_TYPE_I32 ? MO_32 : MO_64;
5555
5556 /* The argument is going onto the stack; extend into scratch. */
5557 if ((mov[i].src_ext & MO_SIZE) != dst_mo) {
5558 tcg_debug_assert(parm->ntmp != 0);
5559 mov[i].dst = src = parm->tmp[0];
5560 tcg_out_movext1(s, &mov[i]);
5561 }
5562
5563 tcg_out_st(s, dst_type, src, TCG_REG_CALL_STACK,
5564 tcg_out_helper_stk_ofs(dst_type, slot));
5565 }
5566 return;
5567
5568 found_reg:
5569 /*
5570 * The remaining arguments are in registers.
5571 * Convert slot numbers to argument registers.
5572 */
5573 nmov = i + 1;
5574 for (i = 0; i < nmov; ++i) {
5575 mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst];
5576 }
5577
8429a1ca 5578 switch (nmov) {
2462e30e 5579 case 4:
8429a1ca 5580 /* The backend must have provided enough temps for the worst case. */
2462e30e 5581 tcg_debug_assert(parm->ntmp >= 2);
8429a1ca 5582
2462e30e
RH		 5583 dst3 = mov[3].dst;
5584 for (unsigned j = 0; j < 3; ++j) {
5585 if (dst3 == mov[j].src) {
5586 /*
5587 * Conflict. Copy the source to a temporary, perform the
5588 * remaining moves, then the extension from our scratch
5589 * on the way out.
5590 */
5591 TCGReg scratch = parm->tmp[1];
8429a1ca 5592
2462e30e
RH		 5593 tcg_out_mov(s, mov[3].src_type, scratch, mov[3].src);
5594 tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]);
5595 tcg_out_movext1_new_src(s, &mov[3], scratch);
5596 break;
8429a1ca 5597 }
8429a1ca 5598 }
8429a1ca 5599
2462e30e
RH		 5600 /* No conflicts: perform this move and continue. */
5601 tcg_out_movext1(s, &mov[3]);
5602 /* fall through */
5603
5604 case 3:
5605 tcg_out_movext3(s, mov, mov + 1, mov + 2,
5606 parm->ntmp ? parm->tmp[0] : -1);
5607 break;
8429a1ca 5608 case 2:
2462e30e
RH		 5609 tcg_out_movext2(s, mov, mov + 1,
5610 parm->ntmp ? parm->tmp[0] : -1);
5611 break;
8429a1ca
RH		 5612 case 1:
5613 tcg_out_movext1(s, mov);
2462e30e
RH		 5614 break;
5615 default:
8429a1ca
RH		 5616 g_assert_not_reached();
5617 }
5618}
5619
8429a1ca
RH		 5620static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot,
5621 TCGType type, tcg_target_long imm,
5622 const TCGLdstHelperParam *parm)
5623{
5624 if (arg_slot_reg_p(slot)) {
5625 tcg_out_movi(s, type, tcg_target_call_iarg_regs[slot], imm);
5626 } else {
5627 int ofs = tcg_out_helper_stk_ofs(type, slot);
5628 if (!tcg_out_sti(s, type, imm, TCG_REG_CALL_STACK, ofs)) {
5629 tcg_debug_assert(parm->ntmp != 0);
5630 tcg_out_movi(s, type, parm->tmp[0], imm);
5631 tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs);
5632 }
5633 }
5634}
5635
5636static void tcg_out_helper_load_common_args(TCGContext *s,
5637 const TCGLabelQemuLdst *ldst,
5638 const TCGLdstHelperParam *parm,
5639 const TCGHelperInfo *info,
5640 unsigned next_arg)
5641{
5642 TCGMovExtend ptr_mov = {
5643 .dst_type = TCG_TYPE_PTR,
5644 .src_type = TCG_TYPE_PTR,
5645 .src_ext = sizeof(void *) == 4 ? MO_32 : MO_64
5646 };
5647 const TCGCallArgumentLoc *loc = &info->in[0];
5648 TCGType type;
5649 unsigned slot;
5650 tcg_target_ulong imm;
5651
5652 /*
5653 * Handle env, which is always first.
5654 */
5655 ptr_mov.dst = loc->arg_slot;
5656 ptr_mov.src = TCG_AREG0;
5657 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
5658
5659 /*
5660 * Handle oi.
5661 */
5662 imm = ldst->oi;
5663 loc = &info->in[next_arg];
5664 type = TCG_TYPE_I32;
5665 switch (loc->kind) {
5666 case TCG_CALL_ARG_NORMAL:
5667 break;
5668 case TCG_CALL_ARG_EXTEND_U:
5669 case TCG_CALL_ARG_EXTEND_S:
5670 /* No extension required for MemOpIdx. */
5671 tcg_debug_assert(imm <= INT32_MAX);
5672 type = TCG_TYPE_REG;
5673 break;
5674 default:
5675 g_assert_not_reached();
5676 }
5677 tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm);
5678 next_arg++;
5679
5680 /*
5681 * Handle ra.
5682 */
5683 loc = &info->in[next_arg];
5684 slot = loc->arg_slot;
5685 if (parm->ra_gen) {
5686 int arg_reg = -1;
5687 TCGReg ra_reg;
5688
5689 if (arg_slot_reg_p(slot)) {
5690 arg_reg = tcg_target_call_iarg_regs[slot];
5691 }
5692 ra_reg = parm->ra_gen(s, ldst, arg_reg);
5693
5694 ptr_mov.dst = slot;
5695 ptr_mov.src = ra_reg;
5696 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
5697 } else {
5698 imm = (uintptr_t)ldst->raddr;
5699 tcg_out_helper_load_imm(s, slot, TCG_TYPE_PTR, imm, parm);
5700 }
5701}
5702
5703static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov,
5704 const TCGCallArgumentLoc *loc,
5705 TCGType dst_type, TCGType src_type,
5706 TCGReg lo, TCGReg hi)
5707{
ebebea53
RH		 5708 MemOp reg_mo;
5709
8429a1ca
RH		 5710 if (dst_type <= TCG_TYPE_REG) {
5711 MemOp src_ext;
5712
5713 switch (loc->kind) {
5714 case TCG_CALL_ARG_NORMAL:
5715 src_ext = src_type == TCG_TYPE_I32 ? MO_32 : MO_64;
5716 break;
5717 case TCG_CALL_ARG_EXTEND_U:
5718 dst_type = TCG_TYPE_REG;
5719 src_ext = MO_UL;
5720 break;
5721 case TCG_CALL_ARG_EXTEND_S:
5722 dst_type = TCG_TYPE_REG;
5723 src_ext = MO_SL;
5724 break;
5725 default:
5726 g_assert_not_reached();
5727 }
5728
5729 mov[0].dst = loc->arg_slot;
5730 mov[0].dst_type = dst_type;
5731 mov[0].src = lo;
5732 mov[0].src_type = src_type;
5733 mov[0].src_ext = src_ext;
5734 return 1;
5735 }
5736
ebebea53
RH		 5737 if (TCG_TARGET_REG_BITS == 32) {
5738 assert(dst_type == TCG_TYPE_I64);
5739 reg_mo = MO_32;
5740 } else {
5741 assert(dst_type == TCG_TYPE_I128);
5742 reg_mo = MO_64;
5743 }
8429a1ca
RH		 5744
5745 mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot;
5746 mov[0].src = lo;
ebebea53
RH		 5747 mov[0].dst_type = TCG_TYPE_REG;
5748 mov[0].src_type = TCG_TYPE_REG;
5749 mov[0].src_ext = reg_mo;
8429a1ca
RH		 5750
5751 mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot;
5752 mov[1].src = hi;
ebebea53
RH		 5753 mov[1].dst_type = TCG_TYPE_REG;
5754 mov[1].src_type = TCG_TYPE_REG;
5755 mov[1].src_ext = reg_mo;
8429a1ca
RH		 5756
5757 return 2;
5758}
5759
5760static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
5761 const TCGLdstHelperParam *parm)
5762{
5763 const TCGHelperInfo *info;
5764 const TCGCallArgumentLoc *loc;
5765 TCGMovExtend mov[2];
5766 unsigned next_arg, nmov;
5767 MemOp mop = get_memop(ldst->oi);
5768
5769 switch (mop & MO_SIZE) {
5770 case MO_8:
5771 case MO_16:
5772 case MO_32:
5773 info = &info_helper_ld32_mmu;
5774 break;
5775 case MO_64:
5776 info = &info_helper_ld64_mmu;
5777 break;
ebebea53
RH		 5778 case MO_128:
5779 info = &info_helper_ld128_mmu;
5780 break;
8429a1ca
RH		 5781 default:
5782 g_assert_not_reached();
5783 }
5784
5785 /* Defer env argument. */
5786 next_arg = 1;
5787
5788 loc = &info->in[next_arg];
c31e5fa4 5789 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
24e46e6c
RH		 5790 /*
5791 * 32-bit host with 32-bit guest: zero-extend the guest address
5792 * to 64-bits for the helper by storing the low part, then
5793 * load a zero for the high part.
5794 */
5795 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN,
5796 TCG_TYPE_I32, TCG_TYPE_I32,
5797 ldst->addrlo_reg, -1);
5798 tcg_out_helper_load_slots(s, 1, mov, parm);
8429a1ca 5799
24e46e6c
RH		 5800 tcg_out_helper_load_imm(s, loc[!HOST_BIG_ENDIAN].arg_slot,
5801 TCG_TYPE_I32, 0, parm);
5802 next_arg += 2;
c31e5fa4
RH		 5803 } else {
5804 nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
5805 ldst->addrlo_reg, ldst->addrhi_reg);
5806 tcg_out_helper_load_slots(s, nmov, mov, parm);
5807 next_arg += nmov;
24e46e6c 5808 }
8429a1ca 5809
ebebea53
RH		 5810 switch (info->out_kind) {
5811 case TCG_CALL_RET_NORMAL:
5812 case TCG_CALL_RET_BY_VEC:
5813 break;
5814 case TCG_CALL_RET_BY_REF:
5815 /*
5816 * The return reference is in the first argument slot.
5817 * We need memory in which to return: re-use the top of stack.
5818 */
5819 {
5820 int ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
5821
5822 if (arg_slot_reg_p(0)) {
5823 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[0],
5824 TCG_REG_CALL_STACK, ofs_slot0);
5825 } else {
5826 tcg_debug_assert(parm->ntmp != 0);
5827 tcg_out_addi_ptr(s, parm->tmp[0],
5828 TCG_REG_CALL_STACK, ofs_slot0);
5829 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
5830 TCG_REG_CALL_STACK, ofs_slot0);
5831 }
5832 }
5833 break;
5834 default:
5835 g_assert_not_reached();
5836 }
8429a1ca
RH		 5837
5838 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
5839}
5840
5841static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst,
5842 bool load_sign,
5843 const TCGLdstHelperParam *parm)
5844{
ebebea53 5845 MemOp mop = get_memop(ldst->oi);
8429a1ca 5846 TCGMovExtend mov[2];
ebebea53 5847 int ofs_slot0;
8429a1ca 5848
ebebea53
RH		 5849 switch (ldst->type) {
5850 case TCG_TYPE_I64:
5851 if (TCG_TARGET_REG_BITS == 32) {
5852 break;
5853 }
5854 /* fall through */
8429a1ca 5855
ebebea53 5856 case TCG_TYPE_I32:
8429a1ca
RH		 5857 mov[0].dst = ldst->datalo_reg;
5858 mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0);
5859 mov[0].dst_type = ldst->type;
5860 mov[0].src_type = TCG_TYPE_REG;
5861
5862 /*
5863 * If load_sign, then we allowed the helper to perform the
5864 * appropriate sign extension to tcg_target_ulong, and all
5865 * we need now is a plain move.
5866 *
5867 * If they do not, then we expect the relevant extension
5868 * instruction to be no more expensive than a move, and
5869 * we thus save the icache etc by only using one of two
5870 * helper functions.
5871 */
5872 if (load_sign || !(mop & MO_SIGN)) {
5873 if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) {
5874 mov[0].src_ext = MO_32;
5875 } else {
5876 mov[0].src_ext = MO_64;
5877 }
5878 } else {
5879 mov[0].src_ext = mop & MO_SSIZE;
5880 }
5881 tcg_out_movext1(s, mov);
ebebea53 5882 return;
8429a1ca 5883
ebebea53
RH		 5884 case TCG_TYPE_I128:
5885 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
5886 ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
5887 switch (TCG_TARGET_CALL_RET_I128) {
5888 case TCG_CALL_RET_NORMAL:
5889 break;
5890 case TCG_CALL_RET_BY_VEC:
5891 tcg_out_st(s, TCG_TYPE_V128,
5892 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
5893 TCG_REG_CALL_STACK, ofs_slot0);
5894 /* fall through */
5895 case TCG_CALL_RET_BY_REF:
5896 tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg,
5897 TCG_REG_CALL_STACK, ofs_slot0 + 8 * HOST_BIG_ENDIAN);
5898 tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg,
5899 TCG_REG_CALL_STACK, ofs_slot0 + 8 * !HOST_BIG_ENDIAN);
5900 return;
5901 default:
5902 g_assert_not_reached();
5903 }
5904 break;
8429a1ca 5905
ebebea53
RH		 5906 default:
5907 g_assert_not_reached();
8429a1ca 5908 }
ebebea53
RH		 5909
5910 mov[0].dst = ldst->datalo_reg;
5911 mov[0].src =
5912 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN);
723d3a27
RH		 5913 mov[0].dst_type = TCG_TYPE_REG;
5914 mov[0].src_type = TCG_TYPE_REG;
ebebea53
RH		 5915 mov[0].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64;
5916
5917 mov[1].dst = ldst->datahi_reg;
5918 mov[1].src =
5919 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN);
5920 mov[1].dst_type = TCG_TYPE_REG;
5921 mov[1].src_type = TCG_TYPE_REG;
5922 mov[1].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64;
5923
5924 tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1);
8429a1ca
RH		 5925}
5926
5927static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
5928 const TCGLdstHelperParam *parm)
5929{
5930 const TCGHelperInfo *info;
5931 const TCGCallArgumentLoc *loc;
5932 TCGMovExtend mov[4];
5933 TCGType data_type;
5934 unsigned next_arg, nmov, n;
5935 MemOp mop = get_memop(ldst->oi);
5936
5937 switch (mop & MO_SIZE) {
5938 case MO_8:
5939 case MO_16:
5940 case MO_32:
5941 info = &info_helper_st32_mmu;
5942 data_type = TCG_TYPE_I32;
5943 break;
5944 case MO_64:
5945 info = &info_helper_st64_mmu;
5946 data_type = TCG_TYPE_I64;
5947 break;
ebebea53
RH		 5948 case MO_128:
5949 info = &info_helper_st128_mmu;
5950 data_type = TCG_TYPE_I128;
5951 break;
8429a1ca
RH		 5952 default:
5953 g_assert_not_reached();
5954 }
5955
5956 /* Defer env argument. */
5957 next_arg = 1;
5958 nmov = 0;
5959
5960 /* Handle addr argument. */
5961 loc = &info->in[next_arg];
c31e5fa4 5962 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
24e46e6c
RH		 5963 /*
5964 * 32-bit host with 32-bit guest: zero-extend the guest address
5965 * to 64-bits for the helper by storing the low part. Later,
5966 * after we have processed the register inputs, we will load a
5967 * zero for the high part.
5968 */
5969 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN,
5970 TCG_TYPE_I32, TCG_TYPE_I32,
5971 ldst->addrlo_reg, -1);
5972 next_arg += 2;
5973 nmov += 1;
c31e5fa4
RH		 5974 } else {
5975 n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
5976 ldst->addrlo_reg, ldst->addrhi_reg);
5977 next_arg += n;
5978 nmov += n;
24e46e6c 5979 }
8429a1ca
RH		 5980
5981 /* Handle data argument. */
5982 loc = &info->in[next_arg];
ebebea53
RH		 5983 switch (loc->kind) {
5984 case TCG_CALL_ARG_NORMAL:
5985 case TCG_CALL_ARG_EXTEND_U:
5986 case TCG_CALL_ARG_EXTEND_S:
5987 n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type,
5988 ldst->datalo_reg, ldst->datahi_reg);
5989 next_arg += n;
5990 nmov += n;
5991 tcg_out_helper_load_slots(s, nmov, mov, parm);
5992 break;
5993
5994 case TCG_CALL_ARG_BY_REF:
5995 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
5996 tcg_debug_assert(data_type == TCG_TYPE_I128);
5997 tcg_out_st(s, TCG_TYPE_I64,
5998 HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg,
5999 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[0].ref_slot));
6000 tcg_out_st(s, TCG_TYPE_I64,
6001 HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg,
6002 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[1].ref_slot));
6003
6004 tcg_out_helper_load_slots(s, nmov, mov, parm);
6005
6006 if (arg_slot_reg_p(loc->arg_slot)) {
6007 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot],
6008 TCG_REG_CALL_STACK,
6009 arg_slot_stk_ofs(loc->ref_slot));
6010 } else {
6011 tcg_debug_assert(parm->ntmp != 0);
6012 tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK,
6013 arg_slot_stk_ofs(loc->ref_slot));
6014 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
6015 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot));
6016 }
6017 next_arg += 2;
6018 break;
6019
6020 default:
6021 g_assert_not_reached();
6022 }
8429a1ca 6023
c31e5fa4
RH		 6024 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
6025 /* Zero extend the address by loading a zero for the high part. */
24e46e6c
RH		 6026 loc = &info->in[1 + !HOST_BIG_ENDIAN];
6027 tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm);
6028 }
6029
8429a1ca
RH		 6030 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
6031}
6032
76cef4b2 6033int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
c896fe29 6034{
747bd69d 6035 int i, start_words, num_insns;
15fa08f8 6036 TCGOp *op;
c896fe29 6037
d977e1c2 6038 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
fbf59aad 6039 && qemu_log_in_addr_range(pc_start))) {
c60f599b 6040 FILE *logfile = qemu_log_trylock();
78b54858
RH		 6041 if (logfile) {
6042 fprintf(logfile, "OP:\n");
b7a83ff8 6043 tcg_dump_ops(s, logfile, false);
78b54858
RH		 6044 fprintf(logfile, "\n");
6045 qemu_log_unlock(logfile);
6046 }
c896fe29 6047 }
c896fe29 6048
bef16ab4
RH		 6049#ifdef CONFIG_DEBUG_TCG
6050 /* Ensure all labels referenced have been emitted. */
6051 {
6052 TCGLabel *l;
6053 bool error = false;
6054
6055 QSIMPLEQ_FOREACH(l, &s->labels, next) {
f85b1fc4 6056 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) {
bef16ab4
RH		 6057 qemu_log_mask(CPU_LOG_TB_OP,
6058 "$L%d referenced but not present.\n", l->id);
6059 error = true;
6060 }
6061 }
6062 assert(!error);
6063 }
6064#endif
6065
c45cb8bb 6066 tcg_optimize(s);
8f2e8c07 6067
b4fc67c7 6068 reachable_code_pass(s);
874b8574 6069 liveness_pass_0(s);
b83eabea 6070 liveness_pass_1(s);
5a18407f 6071
b83eabea 6072 if (s->nb_indirects > 0) {
b83eabea 6073 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
fbf59aad 6074 && qemu_log_in_addr_range(pc_start))) {
c60f599b 6075 FILE *logfile = qemu_log_trylock();
78b54858
RH		 6076 if (logfile) {
6077 fprintf(logfile, "OP before indirect lowering:\n");
b7a83ff8 6078 tcg_dump_ops(s, logfile, false);
78b54858
RH		 6079 fprintf(logfile, "\n");
6080 qemu_log_unlock(logfile);
6081 }
b83eabea 6082 }
645e3a81 6083
b83eabea
RH		 6084 /* Replace indirect temps with direct temps. */
6085 if (liveness_pass_2(s)) {
6086 /* If changes were made, re-run liveness. */
6087 liveness_pass_1(s);
5a18407f
RH		 6088 }
6089 }
c5cc28ff 6090
d977e1c2 6091 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
fbf59aad 6092 && qemu_log_in_addr_range(pc_start))) {
c60f599b 6093 FILE *logfile = qemu_log_trylock();
78b54858
RH		 6094 if (logfile) {
6095 fprintf(logfile, "OP after optimization and liveness analysis:\n");
b7a83ff8 6096 tcg_dump_ops(s, logfile, true);
78b54858
RH		 6097 fprintf(logfile, "\n");
6098 qemu_log_unlock(logfile);
6099 }
c896fe29 6100 }
c896fe29 6101
35abb009 6102 /* Initialize goto_tb jump offsets. */
3a50f424
RH		 6103 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
6104 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
9da6079b
RH		 6105 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
6106 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
35abb009 6107
c896fe29
FB		 6108 tcg_reg_alloc_start(s);
6109
db0c51a3
RH		 6110 /*
6111 * Reset the buffer pointers when restarting after overflow.
6112 * TODO: Move this into translate-all.c with the rest of the
6113 * buffer management. Having only this done here is confusing.
6114 */
6115 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
6116 s->code_ptr = s->code_buf;
c896fe29 6117
659ef5cb 6118#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 6119 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 6120#endif
57a26946
RH		 6121#ifdef TCG_TARGET_NEED_POOL_LABELS
6122 s->pool_labels = NULL;
6123#endif
9ecefc84 6124
747bd69d
RH		 6125 start_words = s->insn_start_words;
6126 s->gen_insn_data =
6127 tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * start_words);
6128
9358fbbf
RH		 6129 tcg_out_tb_start(s);
6130
fca8a500 6131 num_insns = -1;
15fa08f8 6132 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 6133 TCGOpcode opc = op->opc;
b3db8758 6134
c45cb8bb 6135 switch (opc) {
c896fe29 6136 case INDEX_op_mov_i32:
c896fe29 6137 case INDEX_op_mov_i64:
d2fd745f 6138 case INDEX_op_mov_vec:
dd186292 6139 tcg_reg_alloc_mov(s, op);
c896fe29 6140 break;
bab1671f
RH		 6141 case INDEX_op_dup_vec:
6142 tcg_reg_alloc_dup(s, op);
6143 break;
765b842a 6144 case INDEX_op_insn_start:
fca8a500 6145 if (num_insns >= 0) {
9f754620
RH		 6146 size_t off = tcg_current_code_size(s);
6147 s->gen_insn_end_off[num_insns] = off;
6148 /* Assert that we do not overflow our stored offset. */
6149 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 6150 }
6151 num_insns++;
747bd69d
RH		 6152 for (i = 0; i < start_words; ++i) {
6153 s->gen_insn_data[num_insns * start_words + i] =
c9ad8d27 6154 tcg_get_insn_start_param(op, i);
bad729e2 6155 }
c896fe29 6156 break;
5ff9d6a4 6157 case INDEX_op_discard:
43439139 6158 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 6159 break;
c896fe29 6160 case INDEX_op_set_label:
e8996ee0 6161 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 6162 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 6163 break;
6164 case INDEX_op_call:
dd186292 6165 tcg_reg_alloc_call(s, op);
c45cb8bb 6166 break;
b55a8d9d
RH		 6167 case INDEX_op_exit_tb:
6168 tcg_out_exit_tb(s, op->args[0]);
6169 break;
cf7d6b8e
RH		 6170 case INDEX_op_goto_tb:
6171 tcg_out_goto_tb(s, op->args[0]);
6172 break;
efe86b21
RH		 6173 case INDEX_op_dup2_vec:
6174 if (tcg_reg_alloc_dup2(s, op)) {
6175 break;
6176 }
6177 /* fall through */
c896fe29 6178 default:
25c4d9cc 6179 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 6180 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 6181 /* Note: in order to speed up the code, it would be much
6182 faster to have specialized register allocator functions for
6183 some common argument patterns */
dd186292 6184 tcg_reg_alloc_op(s, op);
c896fe29
FB		 6185 break;
6186 }
b125f9dc
RH		 6187 /* Test for (pending) buffer overflow. The assumption is that any
6188 one operation beginning below the high water mark cannot overrun
6189 the buffer completely. Thus we can test for overflow after
6190 generating code without having to check during generation. */
644da9b3 6191 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 6192 return -1;
6193 }
6e6c4efe
RH		 6194 /* Test for TB overflow, as seen by gen_insn_end_off. */
6195 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
6196 return -2;
6197 }
c896fe29 6198 }
747bd69d 6199 tcg_debug_assert(num_insns + 1 == s->gen_tb->icount);
fca8a500 6200 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 6201
b76f0d8c 6202 /* Generate TB finalization at the end of block */
659ef5cb 6203#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 6204 i = tcg_out_ldst_finalize(s);
6205 if (i < 0) {
6206 return i;
23dceda6 6207 }
659ef5cb 6208#endif
57a26946 6209#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 6210 i = tcg_out_pool_finalize(s);
6211 if (i < 0) {
6212 return i;
57a26946
RH		 6213 }
6214#endif
7ecd02a0
RH		 6215 if (!tcg_resolve_relocs(s)) {
6216 return -2;
6217 }
c896fe29 6218
df5d2b16 6219#ifndef CONFIG_TCG_INTERPRETER
c896fe29 6220 /* flush instruction cache */
db0c51a3
RH		 6221 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
6222 (uintptr_t)s->code_buf,
1da8de39 6223 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 6224#endif
2aeabc08 6225
1813e175 6226 return tcg_current_code_size(s);
c896fe29
FB		 6227}
6228
813da627 6229#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 6230/* In order to use this feature, the backend needs to do three things:
6231
6232 (1) Define ELF_HOST_MACHINE to indicate both what value to
6233 put into the ELF image and to indicate support for the feature.
6234
6235 (2) Define tcg_register_jit. This should create a buffer containing
6236 the contents of a .debug_frame section that describes the post-
6237 prologue unwind info for the tcg machine.
6238
6239 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
6240*/
813da627
RH		 6241
6242/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
6243typedef enum {
6244 JIT_NOACTION = 0,
6245 JIT_REGISTER_FN,
6246 JIT_UNREGISTER_FN
6247} jit_actions_t;
6248
6249struct jit_code_entry {
6250 struct jit_code_entry *next_entry;
6251 struct jit_code_entry *prev_entry;
6252 const void *symfile_addr;
6253 uint64_t symfile_size;
6254};
6255
6256struct jit_descriptor {
6257 uint32_t version;
6258 uint32_t action_flag;
6259 struct jit_code_entry *relevant_entry;
6260 struct jit_code_entry *first_entry;
6261};
6262
6263void __jit_debug_register_code(void) __attribute__((noinline));
6264void __jit_debug_register_code(void)
6265{
6266 asm("");
6267}
6268
6269/* Must statically initialize the version, because GDB may check
6270 the version before we can set it. */
6271struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
6272
6273/* End GDB interface. */
6274
6275static int find_string(const char *strtab, const char *str)
6276{
6277 const char *p = strtab + 1;
6278
6279 while (1) {
6280 if (strcmp(p, str) == 0) {
6281 return p - strtab;
6282 }
6283 p += strlen(p) + 1;
6284 }
6285}
6286
755bf9e5 6287static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 6288 const void *debug_frame,
6289 size_t debug_frame_size)
813da627 6290{
5872bbf2
RH		 6291 struct __attribute__((packed)) DebugInfo {
6292 uint32_t len;
6293 uint16_t version;
6294 uint32_t abbrev;
6295 uint8_t ptr_size;
6296 uint8_t cu_die;
6297 uint16_t cu_lang;
6298 uintptr_t cu_low_pc;
6299 uintptr_t cu_high_pc;
6300 uint8_t fn_die;
6301 char fn_name[16];
6302 uintptr_t fn_low_pc;
6303 uintptr_t fn_high_pc;
6304 uint8_t cu_eoc;
6305 };
813da627
RH		 6306
6307 struct ElfImage {
6308 ElfW(Ehdr) ehdr;
6309 ElfW(Phdr) phdr;
5872bbf2
RH		 6310 ElfW(Shdr) shdr[7];
6311 ElfW(Sym) sym[2];
6312 struct DebugInfo di;
6313 uint8_t da[24];
6314 char str[80];
6315 };
6316
6317 struct ElfImage *img;
6318
6319 static const struct ElfImage img_template = {
6320 .ehdr = {
6321 .e_ident[EI_MAG0] = ELFMAG0,
6322 .e_ident[EI_MAG1] = ELFMAG1,
6323 .e_ident[EI_MAG2] = ELFMAG2,
6324 .e_ident[EI_MAG3] = ELFMAG3,
6325 .e_ident[EI_CLASS] = ELF_CLASS,
6326 .e_ident[EI_DATA] = ELF_DATA,
6327 .e_ident[EI_VERSION] = EV_CURRENT,
6328 .e_type = ET_EXEC,
6329 .e_machine = ELF_HOST_MACHINE,
6330 .e_version = EV_CURRENT,
6331 .e_phoff = offsetof(struct ElfImage, phdr),
6332 .e_shoff = offsetof(struct ElfImage, shdr),
6333 .e_ehsize = sizeof(ElfW(Shdr)),
6334 .e_phentsize = sizeof(ElfW(Phdr)),
6335 .e_phnum = 1,
6336 .e_shentsize = sizeof(ElfW(Shdr)),
6337 .e_shnum = ARRAY_SIZE(img->shdr),
6338 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 6339#ifdef ELF_HOST_FLAGS
6340 .e_flags = ELF_HOST_FLAGS,
6341#endif
6342#ifdef ELF_OSABI
6343 .e_ident[EI_OSABI] = ELF_OSABI,
6344#endif
5872bbf2
RH		 6345 },
6346 .phdr = {
6347 .p_type = PT_LOAD,
6348 .p_flags = PF_X,
6349 },
6350 .shdr = {
6351 [0] = { .sh_type = SHT_NULL },
6352 /* Trick: The contents of code_gen_buffer are not present in
6353 this fake ELF file; that got allocated elsewhere. Therefore
6354 we mark .text as SHT_NOBITS (similar to .bss) so that readers
6355 will not look for contents. We can record any address. */
6356 [1] = { /* .text */
6357 .sh_type = SHT_NOBITS,
6358 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
6359 },
6360 [2] = { /* .debug_info */
6361 .sh_type = SHT_PROGBITS,
6362 .sh_offset = offsetof(struct ElfImage, di),
6363 .sh_size = sizeof(struct DebugInfo),
6364 },
6365 [3] = { /* .debug_abbrev */
6366 .sh_type = SHT_PROGBITS,
6367 .sh_offset = offsetof(struct ElfImage, da),
6368 .sh_size = sizeof(img->da),
6369 },
6370 [4] = { /* .debug_frame */
6371 .sh_type = SHT_PROGBITS,
6372 .sh_offset = sizeof(struct ElfImage),
6373 },
6374 [5] = { /* .symtab */
6375 .sh_type = SHT_SYMTAB,
6376 .sh_offset = offsetof(struct ElfImage, sym),
6377 .sh_size = sizeof(img->sym),
6378 .sh_info = 1,
6379 .sh_link = ARRAY_SIZE(img->shdr) - 1,
6380 .sh_entsize = sizeof(ElfW(Sym)),
6381 },
6382 [6] = { /* .strtab */
6383 .sh_type = SHT_STRTAB,
6384 .sh_offset = offsetof(struct ElfImage, str),
6385 .sh_size = sizeof(img->str),
6386 }
6387 },
6388 .sym = {
6389 [1] = { /* code_gen_buffer */
6390 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
6391 .st_shndx = 1,
6392 }
6393 },
6394 .di = {
6395 .len = sizeof(struct DebugInfo) - 4,
6396 .version = 2,
6397 .ptr_size = sizeof(void *),
6398 .cu_die = 1,
6399 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
6400 .fn_die = 2,
6401 .fn_name = "code_gen_buffer"
6402 },
6403 .da = {
6404 1, /* abbrev number (the cu) */
6405 0x11, 1, /* DW_TAG_compile_unit, has children */
6406 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
6407 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
6408 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
6409 0, 0, /* end of abbrev */
6410 2, /* abbrev number (the fn) */
6411 0x2e, 0, /* DW_TAG_subprogram, no children */
6412 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
6413 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
6414 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
6415 0, 0, /* end of abbrev */
6416 0 /* no more abbrev */
6417 },
6418 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
6419 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 6420 };
6421
6422 /* We only need a single jit entry; statically allocate it. */
6423 static struct jit_code_entry one_entry;
6424
5872bbf2 6425 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 6426 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 6427 DebugFrameHeader *dfh;
813da627 6428
5872bbf2
RH		 6429 img = g_malloc(img_size);
6430 *img = img_template;
813da627 6431
5872bbf2
RH		 6432 img->phdr.p_vaddr = buf;
6433 img->phdr.p_paddr = buf;
6434 img->phdr.p_memsz = buf_size;
813da627 6435
813da627 6436 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 6437 img->shdr[1].sh_addr = buf;
813da627
RH		 6438 img->shdr[1].sh_size = buf_size;
6439
5872bbf2
RH		 6440 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
6441 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
6442
6443 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
6444 img->shdr[4].sh_size = debug_frame_size;
6445
6446 img->shdr[5].sh_name = find_string(img->str, ".symtab");
6447 img->shdr[6].sh_name = find_string(img->str, ".strtab");
6448
6449 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
6450 img->sym[1].st_value = buf;
6451 img->sym[1].st_size = buf_size;
813da627 6452
5872bbf2 6453 img->di.cu_low_pc = buf;
45aba097 6454 img->di.cu_high_pc = buf + buf_size;
5872bbf2 6455 img->di.fn_low_pc = buf;
45aba097 6456 img->di.fn_high_pc = buf + buf_size;
813da627 6457
2c90784a
RH		 6458 dfh = (DebugFrameHeader *)(img + 1);
6459 memcpy(dfh, debug_frame, debug_frame_size);
6460 dfh->fde.func_start = buf;
6461 dfh->fde.func_len = buf_size;
6462
813da627
RH		 6463#ifdef DEBUG_JIT
6464 /* Enable this block to be able to debug the ELF image file creation.
6465 One can use readelf, objdump, or other inspection utilities. */
6466 {
eb6b2edf
BM		 6467 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
6468 FILE *f = fopen(jit, "w+b");
813da627 6469 if (f) {
5872bbf2 6470 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 6471 /* Avoid stupid unused return value warning for fwrite. */
6472 }
6473 fclose(f);
6474 }
6475 }
6476#endif
6477
6478 one_entry.symfile_addr = img;
6479 one_entry.symfile_size = img_size;
6480
6481 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
6482 __jit_debug_descriptor.relevant_entry = &one_entry;
6483 __jit_debug_descriptor.first_entry = &one_entry;
6484 __jit_debug_register_code();
6485}
6486#else
5872bbf2
RH		 6487/* No support for the feature. Provide the entry point expected by exec.c,
6488 and implement the internal function we declared earlier. */
813da627 6489
755bf9e5 6490static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 6491 const void *debug_frame,
6492 size_t debug_frame_size)
813da627
RH		 6493{
6494}
6495
755bf9e5 6496void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 6497{
6498}
6499#endif /* ELF_HOST_MACHINE */
db432672
RH		 6500
6501#if !TCG_TARGET_MAYBE_vec
6502void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
6503{
6504 g_assert_not_reached();
6505}
6506#endif
QEMU mirror