]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg: Move TCGLabelQemuLdst to tcg.c
[mirror_qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
084cfca1 37#include "qemu/cacheflush.h"
ad768e6f 38#include "qemu/cacheinfo.h"
533206f0 39#include "qemu/timer.h"
c896fe29 40
c5d3c498 41/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 42 CPU definitions. Currently they are used for qemu_ld/st
43 instructions */
44#define NO_CPU_IO_DEFS
c896fe29 45
63c91552 46#include "exec/exec-all.h"
dcb32f1d 47#include "tcg/tcg-op.h"
813da627 48
edee2579 49#if UINTPTR_MAX == UINT32_MAX
813da627 50# define ELF_CLASS ELFCLASS32
edee2579
RH		 51#else
52# define ELF_CLASS ELFCLASS64
813da627 53#endif
e03b5686 54#if HOST_BIG_ENDIAN
813da627
RH		 55# define ELF_DATA ELFDATA2MSB
56#else
57# define ELF_DATA ELFDATA2LSB
58#endif
59
c896fe29 60#include "elf.h"
508127e2 61#include "exec/log.h"
d2ba8026 62#include "tcg/tcg-ldst.h"
47f7313d 63#include "tcg/tcg-temp-internal.h"
5ff7258c 64#include "tcg-internal.h"
5584e2db 65#include "accel/tcg/perf.h"
c896fe29 66
139c1837 67/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 68 used here. */
e4d58b41
RH		 69static void tcg_target_init(TCGContext *s);
70static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 71static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 72 intptr_t value, intptr_t addend);
c896fe29 73
497a22eb
RH		 74/* The CIE and FDE header definitions will be common to all hosts. */
75typedef struct {
76 uint32_t len __attribute__((aligned((sizeof(void *)))));
77 uint32_t id;
78 uint8_t version;
79 char augmentation[1];
80 uint8_t code_align;
81 uint8_t data_align;
82 uint8_t return_column;
83} DebugFrameCIE;
84
85typedef struct QEMU_PACKED {
86 uint32_t len __attribute__((aligned((sizeof(void *)))));
87 uint32_t cie_offset;
edee2579
RH		 88 uintptr_t func_start;
89 uintptr_t func_len;
497a22eb
RH		 90} DebugFrameFDEHeader;
91
2c90784a
RH		 92typedef struct QEMU_PACKED {
93 DebugFrameCIE cie;
94 DebugFrameFDEHeader fde;
95} DebugFrameHeader;
96
2528f771
RH		 97typedef struct TCGLabelQemuLdst {
98 bool is_ld; /* qemu_ld: true, qemu_st: false */
99 MemOpIdx oi;
100 TCGType type; /* result type of a load */
101 TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */
102 TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */
103 TCGReg datalo_reg; /* reg index for low word to be loaded or stored */
104 TCGReg datahi_reg; /* reg index for high word to be loaded or stored */
105 const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */
106 tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
107 QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
108} TCGLabelQemuLdst;
109
755bf9e5 110static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 111 const void *debug_frame,
112 size_t debug_frame_size)
813da627
RH		 113 __attribute__((unused));
114
139c1837 115/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 116static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 117 intptr_t arg2);
78113e83 118static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 119static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 120 TCGReg ret, tcg_target_long arg);
678155b2 121static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
753e42ea 122static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
d0e66c89 123static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
379afdff 124static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
52bf3398 125static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
9ecf5f61 126static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
9c6aa274 127static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
b9bfe000 128static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
b8b94ac6 129static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
313bdea8 130static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
129f1f9e 131static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2);
b55a8d9d 132static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
cf7d6b8e 133static void tcg_out_goto_tb(TCGContext *s, int which);
5e8892db
MR		 134static void tcg_out_op(TCGContext *s, TCGOpcode opc,
135 const TCGArg args[TCG_MAX_OP_ARGS],
136 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 137#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 138static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
139 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 140static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
141 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 142static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
143 TCGReg dst, int64_t arg);
5e8892db
MR		 144static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
145 unsigned vecl, unsigned vece,
146 const TCGArg args[TCG_MAX_OP_ARGS],
147 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 148#else
e7632cfa
RH		 149static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
150 TCGReg dst, TCGReg src)
151{
152 g_assert_not_reached();
153}
d6ecb4a9
RH		 154static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
155 TCGReg dst, TCGReg base, intptr_t offset)
156{
157 g_assert_not_reached();
158}
4e186175
RH		 159static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
160 TCGReg dst, int64_t arg)
e7632cfa
RH		 161{
162 g_assert_not_reached();
163}
5e8892db
MR		 164static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
165 unsigned vecl, unsigned vece,
166 const TCGArg args[TCG_MAX_OP_ARGS],
167 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 168{
169 g_assert_not_reached();
170}
171#endif
2a534aff 172static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 173 intptr_t arg2);
59d7c14e
RH		 174static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
175 TCGReg base, intptr_t ofs);
7b7d8b2d 176static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
cee44b03 177 const TCGHelperInfo *info);
5e3d0c19 178static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
a4fbbd77 179static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 180#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 181static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 182#endif
c896fe29 183
42eb6dfc
RH		 184TCGContext tcg_init_ctx;
185__thread TCGContext *tcg_ctx;
186
5ff7258c 187TCGContext **tcg_ctxs;
0e2d61cf
RH		 188unsigned int tcg_cur_ctxs;
189unsigned int tcg_max_ctxs;
1c2adb95 190TCGv_env cpu_env = 0;
c8bc1168 191const void *tcg_code_gen_epilogue;
db0c51a3 192uintptr_t tcg_splitwx_diff;
df2cce29 193
b91ccb31
RH		 194#ifndef CONFIG_TCG_INTERPRETER
195tcg_prologue_fn *tcg_qemu_tb_exec;
196#endif
197
d2fd745f 198static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 199static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 200
1813e175 201#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 202static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 203{
204 *s->code_ptr++ = v;
205}
206
4196dca6
PM		 207static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
208 uint8_t v)
5c53bb81 209{
1813e175 210 *p = v;
5c53bb81 211}
1813e175 212#endif
5c53bb81 213
1813e175 214#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 215static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 216{
1813e175
RH		 217 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
218 *s->code_ptr++ = v;
219 } else {
220 tcg_insn_unit *p = s->code_ptr;
221 memcpy(p, &v, sizeof(v));
222 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
223 }
c896fe29
FB		 224}
225
4196dca6
PM		 226static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
227 uint16_t v)
5c53bb81 228{
1813e175
RH		 229 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
230 *p = v;
231 } else {
232 memcpy(p, &v, sizeof(v));
233 }
5c53bb81 234}
1813e175 235#endif
5c53bb81 236
1813e175 237#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 238static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 239{
1813e175
RH		 240 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
241 *s->code_ptr++ = v;
242 } else {
243 tcg_insn_unit *p = s->code_ptr;
244 memcpy(p, &v, sizeof(v));
245 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
246 }
c896fe29
FB		 247}
248
4196dca6
PM		 249static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
250 uint32_t v)
5c53bb81 251{
1813e175
RH		 252 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
253 *p = v;
254 } else {
255 memcpy(p, &v, sizeof(v));
256 }
5c53bb81 257}
1813e175 258#endif
5c53bb81 259
1813e175 260#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 261static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 262{
1813e175
RH		 263 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
264 *s->code_ptr++ = v;
265 } else {
266 tcg_insn_unit *p = s->code_ptr;
267 memcpy(p, &v, sizeof(v));
268 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
269 }
ac26eb69
RH		 270}
271
4196dca6
PM		 272static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
273 uint64_t v)
5c53bb81 274{
1813e175
RH		 275 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
276 *p = v;
277 } else {
278 memcpy(p, &v, sizeof(v));
279 }
5c53bb81 280}
1813e175 281#endif
5c53bb81 282
c896fe29
FB		 283/* label relocation processing */
284
1813e175 285static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 286 TCGLabel *l, intptr_t addend)
c896fe29 287{
7ecd02a0 288 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 289
7ecd02a0
RH		 290 r->type = type;
291 r->ptr = code_ptr;
292 r->addend = addend;
293 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 294}
295
92ab8e7d 296static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 297{
eabb7b91 298 tcg_debug_assert(!l->has_value);
c896fe29 299 l->has_value = 1;
92ab8e7d 300 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 301}
302
42a268c2 303TCGLabel *gen_new_label(void)
c896fe29 304{
b1311c4a 305 TCGContext *s = tcg_ctx;
51e3972c 306 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 307
7ecd02a0
RH		 308 memset(l, 0, sizeof(TCGLabel));
309 l->id = s->nb_labels++;
f85b1fc4 310 QSIMPLEQ_INIT(&l->branches);
7ecd02a0
RH		 311 QSIMPLEQ_INIT(&l->relocs);
312
bef16ab4 313 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 314
315 return l;
c896fe29
FB		 316}
317
7ecd02a0
RH		 318static bool tcg_resolve_relocs(TCGContext *s)
319{
320 TCGLabel *l;
321
322 QSIMPLEQ_FOREACH(l, &s->labels, next) {
323 TCGRelocation *r;
324 uintptr_t value = l->u.value;
325
326 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
327 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
328 return false;
329 }
330 }
331 }
332 return true;
333}
334
9f754620
RH		 335static void set_jmp_reset_offset(TCGContext *s, int which)
336{
f14bed3f
RH		 337 /*
338 * We will check for overflow at the end of the opcode loop in
339 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
340 */
b7e4afbd 341 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 342}
343
b52a2c03
RH		 344static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
345{
346 /*
347 * We will check for overflow at the end of the opcode loop in
348 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
349 */
9da6079b 350 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
b52a2c03
RH		 351}
352
becc452a
RH		 353static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
354{
355 /*
356 * Return the read-execute version of the pointer, for the benefit
357 * of any pc-relative addressing mode.
358 */
9da6079b 359 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
becc452a
RH		 360}
361
db6b7d0c 362/* Signal overflow, starting over with fewer guest insns. */
8905770b
MAL		 363static G_NORETURN
364void tcg_raise_tb_overflow(TCGContext *s)
db6b7d0c
RH		 365{
366 siglongjmp(s->jmp_trans, -2);
367}
368
129f1f9e
RH		 369typedef struct TCGMovExtend {
370 TCGReg dst;
371 TCGReg src;
372 TCGType dst_type;
373 TCGType src_type;
374 MemOp src_ext;
375} TCGMovExtend;
376
b3dfd5fc
RH		 377/**
378 * tcg_out_movext -- move and extend
379 * @s: tcg context
380 * @dst_type: integral type for destination
381 * @dst: destination register
382 * @src_type: integral type for source
383 * @src_ext: extension to apply to source
384 * @src: source register
385 *
386 * Move or extend @src into @dst, depending on @src_ext and the types.
387 */
129f1f9e
RH		 388static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
389 TCGType src_type, MemOp src_ext, TCGReg src)
b3dfd5fc
RH		 390{
391 switch (src_ext) {
392 case MO_UB:
393 tcg_out_ext8u(s, dst, src);
394 break;
395 case MO_SB:
396 tcg_out_ext8s(s, dst_type, dst, src);
397 break;
398 case MO_UW:
399 tcg_out_ext16u(s, dst, src);
400 break;
401 case MO_SW:
402 tcg_out_ext16s(s, dst_type, dst, src);
403 break;
404 case MO_UL:
405 case MO_SL:
406 if (dst_type == TCG_TYPE_I32) {
407 if (src_type == TCG_TYPE_I32) {
408 tcg_out_mov(s, TCG_TYPE_I32, dst, src);
409 } else {
410 tcg_out_extrl_i64_i32(s, dst, src);
411 }
412 } else if (src_type == TCG_TYPE_I32) {
413 if (src_ext & MO_SIGN) {
414 tcg_out_exts_i32_i64(s, dst, src);
415 } else {
416 tcg_out_extu_i32_i64(s, dst, src);
417 }
418 } else {
419 if (src_ext & MO_SIGN) {
420 tcg_out_ext32s(s, dst, src);
421 } else {
422 tcg_out_ext32u(s, dst, src);
423 }
424 }
425 break;
426 case MO_UQ:
427 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
428 if (dst_type == TCG_TYPE_I32) {
429 tcg_out_extrl_i64_i32(s, dst, src);
430 } else {
431 tcg_out_mov(s, TCG_TYPE_I64, dst, src);
432 }
433 break;
434 default:
435 g_assert_not_reached();
436 }
437}
438
129f1f9e
RH		 439/* Minor variations on a theme, using a structure. */
440static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i,
441 TCGReg src)
442{
443 tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src);
444}
445
446static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i)
447{
448 tcg_out_movext1_new_src(s, i, i->src);
449}
450
451/**
452 * tcg_out_movext2 -- move and extend two pair
453 * @s: tcg context
454 * @i1: first move description
455 * @i2: second move description
456 * @scratch: temporary register, or -1 for none
457 *
458 * As tcg_out_movext, for both @i1 and @i2, caring for overlap
459 * between the sources and destinations.
460 */
461
462static void __attribute__((unused))
463tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1,
464 const TCGMovExtend *i2, int scratch)
465{
466 TCGReg src1 = i1->src;
467 TCGReg src2 = i2->src;
468
469 if (i1->dst != src2) {
470 tcg_out_movext1(s, i1);
471 tcg_out_movext1(s, i2);
472 return;
473 }
474 if (i2->dst == src1) {
475 TCGType src1_type = i1->src_type;
476 TCGType src2_type = i2->src_type;
477
478 if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) {
479 /* The data is now in the correct registers, now extend. */
480 src1 = i2->src;
481 src2 = i1->src;
482 } else {
483 tcg_debug_assert(scratch >= 0);
484 tcg_out_mov(s, src1_type, scratch, src1);
485 src1 = scratch;
486 }
487 }
488 tcg_out_movext1_new_src(s, i2, src2);
489 tcg_out_movext1_new_src(s, i1, src1);
490}
491
4c22e840
RH		 492#define C_PFX1(P, A) P##A
493#define C_PFX2(P, A, B) P##A##_##B
494#define C_PFX3(P, A, B, C) P##A##_##B##_##C
495#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
496#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
497#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
498
499/* Define an enumeration for the various combinations. */
500
501#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
502#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
503#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
504#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
505
506#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
507#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
508#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
509#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
510
511#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
512
513#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
514#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
515#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
516#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
517
518typedef enum {
519#include "tcg-target-con-set.h"
520} TCGConstraintSetIndex;
521
522static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
523
524#undef C_O0_I1
525#undef C_O0_I2
526#undef C_O0_I3
527#undef C_O0_I4
528#undef C_O1_I1
529#undef C_O1_I2
530#undef C_O1_I3
531#undef C_O1_I4
532#undef C_N1_I2
533#undef C_O2_I1
534#undef C_O2_I2
535#undef C_O2_I3
536#undef C_O2_I4
537
538/* Put all of the constraint sets into an array, indexed by the enum. */
539
540#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
541#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
542#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
543#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
544
545#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
546#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
547#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
548#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
549
550#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
551
552#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
553#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
554#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
555#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
556
557static const TCGTargetOpDef constraint_sets[] = {
558#include "tcg-target-con-set.h"
559};
560
561
562#undef C_O0_I1
563#undef C_O0_I2
564#undef C_O0_I3
565#undef C_O0_I4
566#undef C_O1_I1
567#undef C_O1_I2
568#undef C_O1_I3
569#undef C_O1_I4
570#undef C_N1_I2
571#undef C_O2_I1
572#undef C_O2_I2
573#undef C_O2_I3
574#undef C_O2_I4
575
576/* Expand the enumerator to be returned from tcg_target_op_def(). */
577
578#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
579#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
580#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
581#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
582
583#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
584#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
585#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
586#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
587
588#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
589
590#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
591#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
592#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
593#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
594
139c1837 595#include "tcg-target.c.inc"
c896fe29 596
38b47b19
EC		 597static void alloc_tcg_plugin_context(TCGContext *s)
598{
599#ifdef CONFIG_PLUGIN
600 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
601 s->plugin_tb->insns =
602 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
603#endif
604}
605
3468b59e
EC		 606/*
607 * All TCG threads except the parent (i.e. the one that called tcg_context_init
608 * and registered the target's TCG globals) must register with this function
609 * before initiating translation.
610 *
611 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
612 * of tcg_region_init() for the reasoning behind this.
613 *
614 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
615 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
616 * is not used anymore for translation once this function is called.
617 *
618 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
619 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
620 */
621#ifdef CONFIG_USER_ONLY
622void tcg_register_thread(void)
623{
624 tcg_ctx = &tcg_init_ctx;
625}
626#else
627void tcg_register_thread(void)
628{
629 TCGContext *s = g_malloc(sizeof(*s));
630 unsigned int i, n;
3468b59e
EC		 631
632 *s = tcg_init_ctx;
633
634 /* Relink mem_base. */
635 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
636 if (tcg_init_ctx.temps[i].mem_base) {
637 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
638 tcg_debug_assert(b >= 0 && b < n);
639 s->temps[i].mem_base = &s->temps[b];
640 }
641 }
642
643 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 644 n = qatomic_fetch_inc(&tcg_cur_ctxs);
645 g_assert(n < tcg_max_ctxs);
d73415a3 646 qatomic_set(&tcg_ctxs[n], s);
3468b59e 647
38b47b19
EC		 648 if (n > 0) {
649 alloc_tcg_plugin_context(s);
bf042e8e 650 tcg_region_initial_alloc(s);
38b47b19
EC		 651 }
652
3468b59e 653 tcg_ctx = s;
e8feb96f 654}
3468b59e 655#endif /* !CONFIG_USER_ONLY */
e8feb96f 656
c896fe29
FB		 657/* pool based memory allocation */
658void *tcg_malloc_internal(TCGContext *s, int size)
659{
660 TCGPool *p;
661 int pool_size;
a813e36f 662
c896fe29
FB		 663 if (size > TCG_POOL_CHUNK_SIZE) {
664 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 665 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 666 p->size = size;
4055299e
KB		 667 p->next = s->pool_first_large;
668 s->pool_first_large = p;
669 return p->data;
c896fe29
FB		 670 } else {
671 p = s->pool_current;
672 if (!p) {
673 p = s->pool_first;
674 if (!p)
675 goto new_pool;
676 } else {
677 if (!p->next) {
678 new_pool:
679 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 680 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 681 p->size = pool_size;
682 p->next = NULL;
a813e36f 683 if (s->pool_current) {
c896fe29 684 s->pool_current->next = p;
a813e36f 685 } else {
c896fe29 686 s->pool_first = p;
a813e36f 687 }
c896fe29
FB		 688 } else {
689 p = p->next;
690 }
691 }
692 }
693 s->pool_current = p;
694 s->pool_cur = p->data + size;
695 s->pool_end = p->data + p->size;
696 return p->data;
697}
698
699void tcg_pool_reset(TCGContext *s)
700{
4055299e
KB		 701 TCGPool *p, *t;
702 for (p = s->pool_first_large; p; p = t) {
703 t = p->next;
704 g_free(p);
705 }
706 s->pool_first_large = NULL;
c896fe29
FB		 707 s->pool_cur = s->pool_end = NULL;
708 s->pool_current = NULL;
709}
710
2ef6175a
RH		 711#include "exec/helper-proto.h"
712
39004a71 713static TCGHelperInfo all_helpers[] = {
2ef6175a 714#include "exec/helper-tcg.h"
100b5e01 715};
619205fd 716static GHashTable *helper_table;
100b5e01 717
22f15579 718#ifdef CONFIG_TCG_INTERPRETER
c6ef8c7b
PMD		 719static ffi_type *typecode_to_ffi(int argmask)
720{
e9709e17
RH		 721 /*
722 * libffi does not support __int128_t, so we have forced Int128
723 * to use the structure definition instead of the builtin type.
724 */
725 static ffi_type *ffi_type_i128_elements[3] = {
726 &ffi_type_uint64,
727 &ffi_type_uint64,
728 NULL
729 };
730 static ffi_type ffi_type_i128 = {
731 .size = 16,
732 .alignment = __alignof__(Int128),
733 .type = FFI_TYPE_STRUCT,
734 .elements = ffi_type_i128_elements,
735 };
736
c6ef8c7b
PMD		 737 switch (argmask) {
738 case dh_typecode_void:
739 return &ffi_type_void;
740 case dh_typecode_i32:
741 return &ffi_type_uint32;
742 case dh_typecode_s32:
743 return &ffi_type_sint32;
744 case dh_typecode_i64:
745 return &ffi_type_uint64;
746 case dh_typecode_s64:
747 return &ffi_type_sint64;
748 case dh_typecode_ptr:
749 return &ffi_type_pointer;
e9709e17
RH		 750 case dh_typecode_i128:
751 return &ffi_type_i128;
c6ef8c7b
PMD		 752 }
753 g_assert_not_reached();
754}
0c22e176
PMD		 755
756static void init_ffi_layouts(void)
757{
758 /* g_direct_hash/equal for direct comparisons on uint32_t. */
f9c4bb80
RH		 759 GHashTable *ffi_table = g_hash_table_new(NULL, NULL);
760
0c22e176 761 for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
f9c4bb80
RH		 762 TCGHelperInfo *info = &all_helpers[i];
763 unsigned typemask = info->typemask;
0c22e176
PMD		 764 gpointer hash = (gpointer)(uintptr_t)typemask;
765 struct {
766 ffi_cif cif;
767 ffi_type *args[];
768 } *ca;
769 ffi_status status;
770 int nargs;
f9c4bb80 771 ffi_cif *cif;
0c22e176 772
f9c4bb80
RH		 773 cif = g_hash_table_lookup(ffi_table, hash);
774 if (cif) {
775 info->cif = cif;
0c22e176
PMD		 776 continue;
777 }
778
779 /* Ignoring the return type, find the last non-zero field. */
780 nargs = 32 - clz32(typemask >> 3);
781 nargs = DIV_ROUND_UP(nargs, 3);
e9709e17 782 assert(nargs <= MAX_CALL_IARGS);
0c22e176
PMD		 783
784 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
785 ca->cif.rtype = typecode_to_ffi(typemask & 7);
786 ca->cif.nargs = nargs;
787
788 if (nargs != 0) {
789 ca->cif.arg_types = ca->args;
790 for (int j = 0; j < nargs; ++j) {
791 int typecode = extract32(typemask, (j + 1) * 3, 3);
792 ca->args[j] = typecode_to_ffi(typecode);
793 }
794 }
795
796 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
797 ca->cif.rtype, ca->cif.arg_types);
798 assert(status == FFI_OK);
799
f9c4bb80
RH		 800 cif = &ca->cif;
801 info->cif = cif;
802 g_hash_table_insert(ffi_table, hash, (gpointer)cif);
0c22e176 803 }
f9c4bb80
RH		 804
805 g_hash_table_destroy(ffi_table);
0c22e176
PMD		 806}
807#endif /* CONFIG_TCG_INTERPRETER */
22f15579 808
39004a71
RH		 809typedef struct TCGCumulativeArgs {
810 int arg_idx; /* tcg_gen_callN args[] */
811 int info_in_idx; /* TCGHelperInfo in[] */
812 int arg_slot; /* regs+stack slot */
813 int ref_slot; /* stack slots for references */
814} TCGCumulativeArgs;
815
816static void layout_arg_even(TCGCumulativeArgs *cum)
817{
818 cum->arg_slot += cum->arg_slot & 1;
819}
820
821static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
822 TCGCallArgumentKind kind)
823{
824 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
825
826 *loc = (TCGCallArgumentLoc){
827 .kind = kind,
828 .arg_idx = cum->arg_idx,
829 .arg_slot = cum->arg_slot,
830 };
831 cum->info_in_idx++;
832 cum->arg_slot++;
833}
834
835static void layout_arg_normal_n(TCGCumulativeArgs *cum,
836 TCGHelperInfo *info, int n)
837{
838 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
839
840 for (int i = 0; i < n; ++i) {
841 /* Layout all using the same arg_idx, adjusting the subindex. */
842 loc[i] = (TCGCallArgumentLoc){
843 .kind = TCG_CALL_ARG_NORMAL,
844 .arg_idx = cum->arg_idx,
845 .tmp_subindex = i,
846 .arg_slot = cum->arg_slot + i,
847 };
848 }
849 cum->info_in_idx += n;
850 cum->arg_slot += n;
851}
852
313bdea8
RH		 853static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info)
854{
855 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
856 int n = 128 / TCG_TARGET_REG_BITS;
857
858 /* The first subindex carries the pointer. */
859 layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);
860
861 /*
862 * The callee is allowed to clobber memory associated with
863 * structure pass by-reference. Therefore we must make copies.
864 * Allocate space from "ref_slot", which will be adjusted to
865 * follow the parameters on the stack.
866 */
867 loc[0].ref_slot = cum->ref_slot;
868
869 /*
870 * Subsequent words also go into the reference slot, but
871 * do not accumulate into the regular arguments.
872 */
873 for (int i = 1; i < n; ++i) {
874 loc[i] = (TCGCallArgumentLoc){
875 .kind = TCG_CALL_ARG_BY_REF_N,
876 .arg_idx = cum->arg_idx,
877 .tmp_subindex = i,
878 .ref_slot = cum->ref_slot + i,
879 };
880 }
881 cum->info_in_idx += n;
882 cum->ref_slot += n;
883}
884
39004a71
RH		 885static void init_call_layout(TCGHelperInfo *info)
886{
887 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
888 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
889 unsigned typemask = info->typemask;
890 unsigned typecode;
891 TCGCumulativeArgs cum = { };
892
893 /*
894 * Parse and place any function return value.
895 */
896 typecode = typemask & 7;
897 switch (typecode) {
898 case dh_typecode_void:
899 info->nr_out = 0;
900 break;
901 case dh_typecode_i32:
902 case dh_typecode_s32:
903 case dh_typecode_ptr:
904 info->nr_out = 1;
905 info->out_kind = TCG_CALL_RET_NORMAL;
906 break;
907 case dh_typecode_i64:
908 case dh_typecode_s64:
909 info->nr_out = 64 / TCG_TARGET_REG_BITS;
910 info->out_kind = TCG_CALL_RET_NORMAL;
5e3d0c19
RH		 911 /* Query the last register now to trigger any assert early. */
912 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
466d3759
RH		 913 break;
914 case dh_typecode_i128:
915 info->nr_out = 128 / TCG_TARGET_REG_BITS;
5427a9a7
RH		 916 info->out_kind = TCG_TARGET_CALL_RET_I128;
917 switch (TCG_TARGET_CALL_RET_I128) {
466d3759 918 case TCG_CALL_RET_NORMAL:
5e3d0c19
RH		 919 /* Query the last register now to trigger any assert early. */
920 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
466d3759 921 break;
c6556aa0
RH		 922 case TCG_CALL_RET_BY_VEC:
923 /* Query the single register now to trigger any assert early. */
924 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
925 break;
313bdea8
RH		 926 case TCG_CALL_RET_BY_REF:
927 /*
928 * Allocate the first argument to the output.
929 * We don't need to store this anywhere, just make it
930 * unavailable for use in the input loop below.
931 */
932 cum.arg_slot = 1;
933 break;
466d3759
RH		 934 default:
935 qemu_build_not_reached();
936 }
39004a71
RH		 937 break;
938 default:
939 g_assert_not_reached();
940 }
39004a71
RH		 941
942 /*
943 * Parse and place function arguments.
944 */
945 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
946 TCGCallArgumentKind kind;
947 TCGType type;
948
949 typecode = typemask & 7;
950 switch (typecode) {
951 case dh_typecode_i32:
952 case dh_typecode_s32:
953 type = TCG_TYPE_I32;
954 break;
955 case dh_typecode_i64:
956 case dh_typecode_s64:
957 type = TCG_TYPE_I64;
958 break;
959 case dh_typecode_ptr:
960 type = TCG_TYPE_PTR;
961 break;
466d3759
RH		 962 case dh_typecode_i128:
963 type = TCG_TYPE_I128;
964 break;
39004a71
RH		 965 default:
966 g_assert_not_reached();
967 }
968
969 switch (type) {
970 case TCG_TYPE_I32:
971 switch (TCG_TARGET_CALL_ARG_I32) {
972 case TCG_CALL_ARG_EVEN:
973 layout_arg_even(&cum);
974 /* fall through */
975 case TCG_CALL_ARG_NORMAL:
976 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
977 break;
978 case TCG_CALL_ARG_EXTEND:
979 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
980 layout_arg_1(&cum, info, kind);
981 break;
982 default:
983 qemu_build_not_reached();
984 }
985 break;
986
987 case TCG_TYPE_I64:
988 switch (TCG_TARGET_CALL_ARG_I64) {
989 case TCG_CALL_ARG_EVEN:
990 layout_arg_even(&cum);
991 /* fall through */
992 case TCG_CALL_ARG_NORMAL:
993 if (TCG_TARGET_REG_BITS == 32) {
994 layout_arg_normal_n(&cum, info, 2);
995 } else {
996 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
997 }
998 break;
999 default:
1000 qemu_build_not_reached();
1001 }
1002 break;
1003
466d3759 1004 case TCG_TYPE_I128:
5427a9a7 1005 switch (TCG_TARGET_CALL_ARG_I128) {
466d3759
RH		 1006 case TCG_CALL_ARG_EVEN:
1007 layout_arg_even(&cum);
1008 /* fall through */
1009 case TCG_CALL_ARG_NORMAL:
1010 layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
1011 break;
313bdea8
RH		 1012 case TCG_CALL_ARG_BY_REF:
1013 layout_arg_by_ref(&cum, info);
1014 break;
466d3759
RH		 1015 default:
1016 qemu_build_not_reached();
1017 }
1018 break;
1019
39004a71
RH		 1020 default:
1021 g_assert_not_reached();
1022 }
1023 }
1024 info->nr_in = cum.info_in_idx;
1025
1026 /* Validate that we didn't overrun the input array. */
1027 assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
1028 /* Validate the backend has enough argument space. */
1029 assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
313bdea8
RH		 1030
1031 /*
1032 * Relocate the "ref_slot" area to the end of the parameters.
1033 * Minimizing this stack offset helps code size for x86,
1034 * which has a signed 8-bit offset encoding.
1035 */
1036 if (cum.ref_slot != 0) {
1037 int ref_base = 0;
1038
1039 if (cum.arg_slot > max_reg_slots) {
1040 int align = __alignof(Int128) / sizeof(tcg_target_long);
1041
1042 ref_base = cum.arg_slot - max_reg_slots;
1043 if (align > 1) {
1044 ref_base = ROUND_UP(ref_base, align);
1045 }
1046 }
1047 assert(ref_base + cum.ref_slot <= max_stk_slots);
1048
1049 if (ref_base != 0) {
1050 for (int i = cum.info_in_idx - 1; i >= 0; --i) {
1051 TCGCallArgumentLoc *loc = &info->in[i];
1052 switch (loc->kind) {
1053 case TCG_CALL_ARG_BY_REF:
1054 case TCG_CALL_ARG_BY_REF_N:
1055 loc->ref_slot += ref_base;
1056 break;
1057 default:
1058 break;
1059 }
1060 }
1061 }
1062 }
39004a71
RH		 1063}
1064
91478cef 1065static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 1066static void process_op_defs(TCGContext *s);
1c2adb95
RH		 1067static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1068 TCGReg reg, const char *name);
91478cef 1069
43b972b7 1070static void tcg_context_init(unsigned max_cpus)
c896fe29 1071{
a76aabd3 1072 TCGContext *s = &tcg_init_ctx;
100b5e01 1073 int op, total_args, n, i;
c896fe29
FB		 1074 TCGOpDef *def;
1075 TCGArgConstraint *args_ct;
1c2adb95 1076 TCGTemp *ts;
c896fe29
FB		 1077
1078 memset(s, 0, sizeof(*s));
c896fe29 1079 s->nb_globals = 0;
c70fbf0a 1080
c896fe29
FB		 1081 /* Count total number of arguments and allocate the corresponding
1082 space */
1083 total_args = 0;
1084 for(op = 0; op < NB_OPS; op++) {
1085 def = &tcg_op_defs[op];
1086 n = def->nb_iargs + def->nb_oargs;
1087 total_args += n;
1088 }
1089
bc2b17e6 1090 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 1091
1092 for(op = 0; op < NB_OPS; op++) {
1093 def = &tcg_op_defs[op];
1094 def->args_ct = args_ct;
c896fe29 1095 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 1096 args_ct += n;
1097 }
5cd8f621
RH		 1098
1099 /* Register helpers. */
84fd9dd3 1100 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 1101 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 1102
100b5e01 1103 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
39004a71 1104 init_call_layout(&all_helpers[i]);
84fd9dd3 1105 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 1106 (gpointer)&all_helpers[i]);
100b5e01 1107 }
5cd8f621 1108
22f15579 1109#ifdef CONFIG_TCG_INTERPRETER
0c22e176 1110 init_ffi_layouts();
22f15579
RH		 1111#endif
1112
c896fe29 1113 tcg_target_init(s);
f69d277e 1114 process_op_defs(s);
91478cef
RH		 1115
1116 /* Reverse the order of the saved registers, assuming they're all at
1117 the start of tcg_target_reg_alloc_order. */
1118 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
1119 int r = tcg_target_reg_alloc_order[n];
1120 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
1121 break;
1122 }
1123 }
1124 for (i = 0; i < n; ++i) {
1125 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
1126 }
1127 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
1128 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
1129 }
b1311c4a 1130
38b47b19
EC		 1131 alloc_tcg_plugin_context(s);
1132
b1311c4a 1133 tcg_ctx = s;
3468b59e
EC		 1134 /*
1135 * In user-mode we simply share the init context among threads, since we
1136 * use a single region. See the documentation tcg_region_init() for the
1137 * reasoning behind this.
1138 * In softmmu we will have at most max_cpus TCG threads.
1139 */
1140#ifdef CONFIG_USER_ONLY
df2cce29 1141 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 1142 tcg_cur_ctxs = 1;
1143 tcg_max_ctxs = 1;
3468b59e 1144#else
0e2d61cf
RH		 1145 tcg_max_ctxs = max_cpus;
1146 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 1147#endif
1c2adb95
RH		 1148
1149 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1150 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
1151 cpu_env = temp_tcgv_ptr(ts);
9002ec79 1152}
b03cce8e 1153
43b972b7 1154void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 1155{
43b972b7
RH		 1156 tcg_context_init(max_cpus);
1157 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 1158}
1159
6e3b2bfd
EC		 1160/*
1161 * Allocate TBs right before their corresponding translated code, making
1162 * sure that TBs and code are on different cache lines.
1163 */
1164TranslationBlock *tcg_tb_alloc(TCGContext *s)
1165{
1166 uintptr_t align = qemu_icache_linesize;
1167 TranslationBlock *tb;
1168 void *next;
1169
e8feb96f 1170 retry:
6e3b2bfd
EC		 1171 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1172 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1173
1174 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 1175 if (tcg_region_alloc(s)) {
1176 return NULL;
1177 }
1178 goto retry;
6e3b2bfd 1179 }
d73415a3 1180 qatomic_set(&s->code_gen_ptr, next);
57a26946 1181 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 1182 return tb;
1183}
1184
9002ec79
RH		 1185void tcg_prologue_init(TCGContext *s)
1186{
b0a0794a 1187 size_t prologue_size;
8163b749 1188
b0a0794a
RH		 1189 s->code_ptr = s->code_gen_ptr;
1190 s->code_buf = s->code_gen_ptr;
5b38ee31 1191 s->data_gen_ptr = NULL;
b91ccb31
RH		 1192
1193#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 1194 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 1195#endif
8163b749 1196
5b38ee31
RH		 1197#ifdef TCG_TARGET_NEED_POOL_LABELS
1198 s->pool_labels = NULL;
1199#endif
1200
653b87eb 1201 qemu_thread_jit_write();
8163b749 1202 /* Generate the prologue. */
b03cce8e 1203 tcg_target_qemu_prologue(s);
5b38ee31
RH		 1204
1205#ifdef TCG_TARGET_NEED_POOL_LABELS
1206 /* Allow the prologue to put e.g. guest_base into a pool entry. */
1207 {
1768987b
RH		 1208 int result = tcg_out_pool_finalize(s);
1209 tcg_debug_assert(result == 0);
5b38ee31
RH		 1210 }
1211#endif
1212
b0a0794a 1213 prologue_size = tcg_current_code_size(s);
5584e2db 1214 perf_report_prologue(s->code_gen_ptr, prologue_size);
b0a0794a 1215
df5d2b16 1216#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 1217 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1218 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 1219#endif
8163b749 1220
d6b64b2b
RH		 1221#ifdef DEBUG_DISAS
1222 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
c60f599b 1223 FILE *logfile = qemu_log_trylock();
78b54858
RH		 1224 if (logfile) {
1225 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
1226 if (s->data_gen_ptr) {
1227 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
1228 size_t data_size = prologue_size - code_size;
1229 size_t i;
1230
1231 disas(logfile, s->code_gen_ptr, code_size);
1232
1233 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1234 if (sizeof(tcg_target_ulong) == 8) {
1235 fprintf(logfile,
1236 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1237 (uintptr_t)s->data_gen_ptr + i,
1238 *(uint64_t *)(s->data_gen_ptr + i));
1239 } else {
1240 fprintf(logfile,
1241 "0x%08" PRIxPTR ": .long 0x%08x\n",
1242 (uintptr_t)s->data_gen_ptr + i,
1243 *(uint32_t *)(s->data_gen_ptr + i));
1244 }
5b38ee31 1245 }
78b54858
RH		 1246 } else {
1247 disas(logfile, s->code_gen_ptr, prologue_size);
5b38ee31 1248 }
78b54858 1249 fprintf(logfile, "\n");
78b54858 1250 qemu_log_unlock(logfile);
5b38ee31 1251 }
d6b64b2b
RH		 1252 }
1253#endif
cedbcb01 1254
6eea0434
RH		 1255#ifndef CONFIG_TCG_INTERPRETER
1256 /*
1257 * Assert that goto_ptr is implemented completely, setting an epilogue.
1258 * For tci, we use NULL as the signal to return from the interpreter,
1259 * so skip this check.
1260 */
f4e01e30 1261 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 1262#endif
d1c74ab3
RH		 1263
1264 tcg_region_prologue_set(s);
c896fe29
FB		 1265}
1266
c896fe29
FB		 1267void tcg_func_start(TCGContext *s)
1268{
1269 tcg_pool_reset(s);
1270 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 1271
1272 /* No temps have been previously allocated for size or locality. */
1273 memset(s->free_temps, 0, sizeof(s->free_temps));
1274
c0522136
RH		 1275 /* No constant temps have been previously allocated. */
1276 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1277 if (s->const_table[i]) {
1278 g_hash_table_remove_all(s->const_table[i]);
1279 }
1280 }
1281
abebf925 1282 s->nb_ops = 0;
c896fe29
FB		 1283 s->nb_labels = 0;
1284 s->current_frame_offset = s->frame_start;
1285
0a209d4b
RH		 1286#ifdef CONFIG_DEBUG_TCG
1287 s->goto_tb_issue_mask = 0;
1288#endif
1289
15fa08f8
RH		 1290 QTAILQ_INIT(&s->ops);
1291 QTAILQ_INIT(&s->free_ops);
bef16ab4 1292 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 1293}
1294
ae30e866 1295static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 1296{
1297 int n = s->nb_temps++;
ae30e866
RH		 1298
1299 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 1300 tcg_raise_tb_overflow(s);
ae30e866 1301 }
7ca4b752
RH		 1302 return memset(&s->temps[n], 0, sizeof(TCGTemp));
1303}
1304
ae30e866 1305static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 1306{
fa477d25
RH		 1307 TCGTemp *ts;
1308
7ca4b752 1309 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 1310 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 1311 s->nb_globals++;
fa477d25 1312 ts = tcg_temp_alloc(s);
ee17db83 1313 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 1314
1315 return ts;
c896fe29
FB		 1316}
1317
085272b3
RH		 1318static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1319 TCGReg reg, const char *name)
c896fe29 1320{
c896fe29 1321 TCGTemp *ts;
c896fe29 1322
1a057554 1323 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
7ca4b752
RH		 1324
1325 ts = tcg_global_alloc(s);
c896fe29
FB		 1326 ts->base_type = type;
1327 ts->type = type;
ee17db83 1328 ts->kind = TEMP_FIXED;
c896fe29 1329 ts->reg = reg;
c896fe29 1330 ts->name = name;
c896fe29 1331 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 1332
085272b3 1333 return ts;
a7812ae4
PB		 1334}
1335
b6638662 1336void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 1337{
b3a62939
RH		 1338 s->frame_start = start;
1339 s->frame_end = start + size;
085272b3
RH		 1340 s->frame_temp
1341 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 1342}
1343
085272b3
RH		 1344TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
1345 intptr_t offset, const char *name)
c896fe29 1346{
b1311c4a 1347 TCGContext *s = tcg_ctx;
dc41aa7d 1348 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 1349 TCGTemp *ts = tcg_global_alloc(s);
aef85402 1350 int indirect_reg = 0;
c896fe29 1351
c0522136
RH		 1352 switch (base_ts->kind) {
1353 case TEMP_FIXED:
1354 break;
1355 case TEMP_GLOBAL:
5a18407f
RH		 1356 /* We do not support double-indirect registers. */
1357 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 1358 base_ts->indirect_base = 1;
5a18407f
RH		 1359 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1360 ? 2 : 1);
1361 indirect_reg = 1;
c0522136
RH		 1362 break;
1363 default:
1364 g_assert_not_reached();
b3915dbb
RH		 1365 }
1366
7ca4b752
RH		 1367 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1368 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 1369 char buf[64];
7ca4b752
RH		 1370
1371 ts->base_type = TCG_TYPE_I64;
c896fe29 1372 ts->type = TCG_TYPE_I32;
b3915dbb 1373 ts->indirect_reg = indirect_reg;
c896fe29 1374 ts->mem_allocated = 1;
b3a62939 1375 ts->mem_base = base_ts;
aef85402 1376 ts->mem_offset = offset;
c896fe29
FB		 1377 pstrcpy(buf, sizeof(buf), name);
1378 pstrcat(buf, sizeof(buf), "_0");
1379 ts->name = strdup(buf);
c896fe29 1380
7ca4b752
RH		 1381 tcg_debug_assert(ts2 == ts + 1);
1382 ts2->base_type = TCG_TYPE_I64;
1383 ts2->type = TCG_TYPE_I32;
b3915dbb 1384 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 1385 ts2->mem_allocated = 1;
1386 ts2->mem_base = base_ts;
aef85402 1387 ts2->mem_offset = offset + 4;
fac87bd2 1388 ts2->temp_subindex = 1;
c896fe29
FB		 1389 pstrcpy(buf, sizeof(buf), name);
1390 pstrcat(buf, sizeof(buf), "_1");
120c1084 1391 ts2->name = strdup(buf);
7ca4b752 1392 } else {
c896fe29
FB		 1393 ts->base_type = type;
1394 ts->type = type;
b3915dbb 1395 ts->indirect_reg = indirect_reg;
c896fe29 1396 ts->mem_allocated = 1;
b3a62939 1397 ts->mem_base = base_ts;
c896fe29 1398 ts->mem_offset = offset;
c896fe29 1399 ts->name = name;
c896fe29 1400 }
085272b3 1401 return ts;
a7812ae4
PB		 1402}
1403
bbf989bf 1404TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
c896fe29 1405{
b1311c4a 1406 TCGContext *s = tcg_ctx;
c896fe29 1407 TCGTemp *ts;
e1c08b00 1408 int n;
7ca4b752 1409
e1c08b00
RH		 1410 if (kind == TEMP_EBB) {
1411 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);
1412
1413 if (idx < TCG_MAX_TEMPS) {
1414 /* There is already an available temp with the right type. */
1415 clear_bit(idx, s->free_temps[type].l);
1416
1417 ts = &s->temps[idx];
1418 ts->temp_allocated = 1;
1419 tcg_debug_assert(ts->base_type == type);
1420 tcg_debug_assert(ts->kind == kind);
2f2e911d 1421 return ts;
43eef72f 1422 }
e1c08b00
RH		 1423 } else {
1424 tcg_debug_assert(kind == TEMP_TB);
1425 }
7ca4b752 1426
e1c08b00
RH		 1427 switch (type) {
1428 case TCG_TYPE_I32:
1429 case TCG_TYPE_V64:
1430 case TCG_TYPE_V128:
1431 case TCG_TYPE_V256:
1432 n = 1;
1433 break;
1434 case TCG_TYPE_I64:
1435 n = 64 / TCG_TARGET_REG_BITS;
1436 break;
1437 case TCG_TYPE_I128:
1438 n = 128 / TCG_TARGET_REG_BITS;
1439 break;
1440 default:
1441 g_assert_not_reached();
1442 }
43eef72f 1443
e1c08b00
RH		 1444 ts = tcg_temp_alloc(s);
1445 ts->base_type = type;
1446 ts->temp_allocated = 1;
1447 ts->kind = kind;
1448
1449 if (n == 1) {
1450 ts->type = type;
1451 } else {
1452 ts->type = TCG_TYPE_REG;
43eef72f 1453
e1c08b00
RH		 1454 for (int i = 1; i < n; ++i) {
1455 TCGTemp *ts2 = tcg_temp_alloc(s);
43eef72f 1456
e1c08b00
RH		 1457 tcg_debug_assert(ts2 == ts + i);
1458 ts2->base_type = type;
1459 ts2->type = TCG_TYPE_REG;
1460 ts2->temp_allocated = 1;
1461 ts2->temp_subindex = i;
1462 ts2->kind = kind;
e8996ee0 1463 }
c896fe29 1464 }
085272b3 1465 return ts;
c896fe29
FB		 1466}
1467
d2fd745f
RH		 1468TCGv_vec tcg_temp_new_vec(TCGType type)
1469{
1470 TCGTemp *t;
1471
1472#ifdef CONFIG_DEBUG_TCG
1473 switch (type) {
1474 case TCG_TYPE_V64:
1475 assert(TCG_TARGET_HAS_v64);
1476 break;
1477 case TCG_TYPE_V128:
1478 assert(TCG_TARGET_HAS_v128);
1479 break;
1480 case TCG_TYPE_V256:
1481 assert(TCG_TARGET_HAS_v256);
1482 break;
1483 default:
1484 g_assert_not_reached();
1485 }
1486#endif
1487
bbf989bf 1488 t = tcg_temp_new_internal(type, TEMP_EBB);
d2fd745f
RH		 1489 return temp_tcgv_vec(t);
1490}
1491
1492/* Create a new temp of the same type as an existing temp. */
1493TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1494{
1495 TCGTemp *t = tcgv_vec_temp(match);
1496
1497 tcg_debug_assert(t->temp_allocated != 0);
1498
bbf989bf 1499 t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
d2fd745f
RH		 1500 return temp_tcgv_vec(t);
1501}
1502
5bfa8034 1503void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1504{
b1311c4a 1505 TCGContext *s = tcg_ctx;
c896fe29 1506
c7482438
RH		 1507 switch (ts->kind) {
1508 case TEMP_CONST:
f57c6915 1509 case TEMP_TB:
2f2e911d
RH		 1510 /* Silently ignore free. */
1511 break;
1512 case TEMP_EBB:
1513 tcg_debug_assert(ts->temp_allocated != 0);
1514 ts->temp_allocated = 0;
1515 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
c7482438
RH		 1516 break;
1517 default:
2f2e911d 1518 /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */
c7482438 1519 g_assert_not_reached();
c0522136 1520 }
c896fe29
FB		 1521}
1522
c0522136
RH		 1523TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1524{
1525 TCGContext *s = tcg_ctx;
1526 GHashTable *h = s->const_table[type];
1527 TCGTemp *ts;
1528
1529 if (h == NULL) {
1530 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1531 s->const_table[type] = h;
1532 }
1533
1534 ts = g_hash_table_lookup(h, &val);
1535 if (ts == NULL) {
aef85402
RH		 1536 int64_t *val_ptr;
1537
c0522136
RH		 1538 ts = tcg_temp_alloc(s);
1539
1540 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1541 TCGTemp *ts2 = tcg_temp_alloc(s);
1542
aef85402
RH		 1543 tcg_debug_assert(ts2 == ts + 1);
1544
c0522136
RH		 1545 ts->base_type = TCG_TYPE_I64;
1546 ts->type = TCG_TYPE_I32;
1547 ts->kind = TEMP_CONST;
1548 ts->temp_allocated = 1;
c0522136 1549
c0522136
RH		 1550 ts2->base_type = TCG_TYPE_I64;
1551 ts2->type = TCG_TYPE_I32;
1552 ts2->kind = TEMP_CONST;
1553 ts2->temp_allocated = 1;
fac87bd2 1554 ts2->temp_subindex = 1;
aef85402
RH		 1555
1556 /*
1557 * Retain the full value of the 64-bit constant in the low
1558 * part, so that the hash table works. Actual uses will
1559 * truncate the value to the low part.
1560 */
1561 ts[HOST_BIG_ENDIAN].val = val;
1562 ts[!HOST_BIG_ENDIAN].val = val >> 32;
1563 val_ptr = &ts[HOST_BIG_ENDIAN].val;
c0522136
RH		 1564 } else {
1565 ts->base_type = type;
1566 ts->type = type;
1567 ts->kind = TEMP_CONST;
1568 ts->temp_allocated = 1;
1569 ts->val = val;
aef85402 1570 val_ptr = &ts->val;
c0522136 1571 }
aef85402 1572 g_hash_table_insert(h, val_ptr, ts);
c0522136
RH		 1573 }
1574
1575 return ts;
1576}
1577
1578TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1579{
1580 val = dup_const(vece, val);
1581 return temp_tcgv_vec(tcg_constant_internal(type, val));
1582}
1583
88d4005b
RH		 1584TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1585{
1586 TCGTemp *t = tcgv_vec_temp(match);
1587
1588 tcg_debug_assert(t->temp_allocated != 0);
1589 return tcg_constant_vec(t->base_type, vece, val);
1590}
1591
be0f34b5
RH		 1592/* Return true if OP may appear in the opcode stream.
1593 Test the runtime variable that controls each opcode. */
1594bool tcg_op_supported(TCGOpcode op)
1595{
d2fd745f
RH		 1596 const bool have_vec
1597 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1598
be0f34b5
RH		 1599 switch (op) {
1600 case INDEX_op_discard:
1601 case INDEX_op_set_label:
1602 case INDEX_op_call:
1603 case INDEX_op_br:
1604 case INDEX_op_mb:
1605 case INDEX_op_insn_start:
1606 case INDEX_op_exit_tb:
1607 case INDEX_op_goto_tb:
f4e01e30 1608 case INDEX_op_goto_ptr:
be0f34b5
RH		 1609 case INDEX_op_qemu_ld_i32:
1610 case INDEX_op_qemu_st_i32:
1611 case INDEX_op_qemu_ld_i64:
1612 case INDEX_op_qemu_st_i64:
1613 return true;
1614
07ce0b05
RH		 1615 case INDEX_op_qemu_st8_i32:
1616 return TCG_TARGET_HAS_qemu_st8_i32;
1617
be0f34b5 1618 case INDEX_op_mov_i32:
be0f34b5
RH		 1619 case INDEX_op_setcond_i32:
1620 case INDEX_op_brcond_i32:
1621 case INDEX_op_ld8u_i32:
1622 case INDEX_op_ld8s_i32:
1623 case INDEX_op_ld16u_i32:
1624 case INDEX_op_ld16s_i32:
1625 case INDEX_op_ld_i32:
1626 case INDEX_op_st8_i32:
1627 case INDEX_op_st16_i32:
1628 case INDEX_op_st_i32:
1629 case INDEX_op_add_i32:
1630 case INDEX_op_sub_i32:
1631 case INDEX_op_mul_i32:
1632 case INDEX_op_and_i32:
1633 case INDEX_op_or_i32:
1634 case INDEX_op_xor_i32:
1635 case INDEX_op_shl_i32:
1636 case INDEX_op_shr_i32:
1637 case INDEX_op_sar_i32:
1638 return true;
1639
1640 case INDEX_op_movcond_i32:
1641 return TCG_TARGET_HAS_movcond_i32;
1642 case INDEX_op_div_i32:
1643 case INDEX_op_divu_i32:
1644 return TCG_TARGET_HAS_div_i32;
1645 case INDEX_op_rem_i32:
1646 case INDEX_op_remu_i32:
1647 return TCG_TARGET_HAS_rem_i32;
1648 case INDEX_op_div2_i32:
1649 case INDEX_op_divu2_i32:
1650 return TCG_TARGET_HAS_div2_i32;
1651 case INDEX_op_rotl_i32:
1652 case INDEX_op_rotr_i32:
1653 return TCG_TARGET_HAS_rot_i32;
1654 case INDEX_op_deposit_i32:
1655 return TCG_TARGET_HAS_deposit_i32;
1656 case INDEX_op_extract_i32:
1657 return TCG_TARGET_HAS_extract_i32;
1658 case INDEX_op_sextract_i32:
1659 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1660 case INDEX_op_extract2_i32:
1661 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1662 case INDEX_op_add2_i32:
1663 return TCG_TARGET_HAS_add2_i32;
1664 case INDEX_op_sub2_i32:
1665 return TCG_TARGET_HAS_sub2_i32;
1666 case INDEX_op_mulu2_i32:
1667 return TCG_TARGET_HAS_mulu2_i32;
1668 case INDEX_op_muls2_i32:
1669 return TCG_TARGET_HAS_muls2_i32;
1670 case INDEX_op_muluh_i32:
1671 return TCG_TARGET_HAS_muluh_i32;
1672 case INDEX_op_mulsh_i32:
1673 return TCG_TARGET_HAS_mulsh_i32;
1674 case INDEX_op_ext8s_i32:
1675 return TCG_TARGET_HAS_ext8s_i32;
1676 case INDEX_op_ext16s_i32:
1677 return TCG_TARGET_HAS_ext16s_i32;
1678 case INDEX_op_ext8u_i32:
1679 return TCG_TARGET_HAS_ext8u_i32;
1680 case INDEX_op_ext16u_i32:
1681 return TCG_TARGET_HAS_ext16u_i32;
1682 case INDEX_op_bswap16_i32:
1683 return TCG_TARGET_HAS_bswap16_i32;
1684 case INDEX_op_bswap32_i32:
1685 return TCG_TARGET_HAS_bswap32_i32;
1686 case INDEX_op_not_i32:
1687 return TCG_TARGET_HAS_not_i32;
1688 case INDEX_op_neg_i32:
1689 return TCG_TARGET_HAS_neg_i32;
1690 case INDEX_op_andc_i32:
1691 return TCG_TARGET_HAS_andc_i32;
1692 case INDEX_op_orc_i32:
1693 return TCG_TARGET_HAS_orc_i32;
1694 case INDEX_op_eqv_i32:
1695 return TCG_TARGET_HAS_eqv_i32;
1696 case INDEX_op_nand_i32:
1697 return TCG_TARGET_HAS_nand_i32;
1698 case INDEX_op_nor_i32:
1699 return TCG_TARGET_HAS_nor_i32;
1700 case INDEX_op_clz_i32:
1701 return TCG_TARGET_HAS_clz_i32;
1702 case INDEX_op_ctz_i32:
1703 return TCG_TARGET_HAS_ctz_i32;
1704 case INDEX_op_ctpop_i32:
1705 return TCG_TARGET_HAS_ctpop_i32;
1706
1707 case INDEX_op_brcond2_i32:
1708 case INDEX_op_setcond2_i32:
1709 return TCG_TARGET_REG_BITS == 32;
1710
1711 case INDEX_op_mov_i64:
be0f34b5
RH		 1712 case INDEX_op_setcond_i64:
1713 case INDEX_op_brcond_i64:
1714 case INDEX_op_ld8u_i64:
1715 case INDEX_op_ld8s_i64:
1716 case INDEX_op_ld16u_i64:
1717 case INDEX_op_ld16s_i64:
1718 case INDEX_op_ld32u_i64:
1719 case INDEX_op_ld32s_i64:
1720 case INDEX_op_ld_i64:
1721 case INDEX_op_st8_i64:
1722 case INDEX_op_st16_i64:
1723 case INDEX_op_st32_i64:
1724 case INDEX_op_st_i64:
1725 case INDEX_op_add_i64:
1726 case INDEX_op_sub_i64:
1727 case INDEX_op_mul_i64:
1728 case INDEX_op_and_i64:
1729 case INDEX_op_or_i64:
1730 case INDEX_op_xor_i64:
1731 case INDEX_op_shl_i64:
1732 case INDEX_op_shr_i64:
1733 case INDEX_op_sar_i64:
1734 case INDEX_op_ext_i32_i64:
1735 case INDEX_op_extu_i32_i64:
1736 return TCG_TARGET_REG_BITS == 64;
1737
1738 case INDEX_op_movcond_i64:
1739 return TCG_TARGET_HAS_movcond_i64;
1740 case INDEX_op_div_i64:
1741 case INDEX_op_divu_i64:
1742 return TCG_TARGET_HAS_div_i64;
1743 case INDEX_op_rem_i64:
1744 case INDEX_op_remu_i64:
1745 return TCG_TARGET_HAS_rem_i64;
1746 case INDEX_op_div2_i64:
1747 case INDEX_op_divu2_i64:
1748 return TCG_TARGET_HAS_div2_i64;
1749 case INDEX_op_rotl_i64:
1750 case INDEX_op_rotr_i64:
1751 return TCG_TARGET_HAS_rot_i64;
1752 case INDEX_op_deposit_i64:
1753 return TCG_TARGET_HAS_deposit_i64;
1754 case INDEX_op_extract_i64:
1755 return TCG_TARGET_HAS_extract_i64;
1756 case INDEX_op_sextract_i64:
1757 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1758 case INDEX_op_extract2_i64:
1759 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1760 case INDEX_op_extrl_i64_i32:
1761 return TCG_TARGET_HAS_extrl_i64_i32;
1762 case INDEX_op_extrh_i64_i32:
1763 return TCG_TARGET_HAS_extrh_i64_i32;
1764 case INDEX_op_ext8s_i64:
1765 return TCG_TARGET_HAS_ext8s_i64;
1766 case INDEX_op_ext16s_i64:
1767 return TCG_TARGET_HAS_ext16s_i64;
1768 case INDEX_op_ext32s_i64:
1769 return TCG_TARGET_HAS_ext32s_i64;
1770 case INDEX_op_ext8u_i64:
1771 return TCG_TARGET_HAS_ext8u_i64;
1772 case INDEX_op_ext16u_i64:
1773 return TCG_TARGET_HAS_ext16u_i64;
1774 case INDEX_op_ext32u_i64:
1775 return TCG_TARGET_HAS_ext32u_i64;
1776 case INDEX_op_bswap16_i64:
1777 return TCG_TARGET_HAS_bswap16_i64;
1778 case INDEX_op_bswap32_i64:
1779 return TCG_TARGET_HAS_bswap32_i64;
1780 case INDEX_op_bswap64_i64:
1781 return TCG_TARGET_HAS_bswap64_i64;
1782 case INDEX_op_not_i64:
1783 return TCG_TARGET_HAS_not_i64;
1784 case INDEX_op_neg_i64:
1785 return TCG_TARGET_HAS_neg_i64;
1786 case INDEX_op_andc_i64:
1787 return TCG_TARGET_HAS_andc_i64;
1788 case INDEX_op_orc_i64:
1789 return TCG_TARGET_HAS_orc_i64;
1790 case INDEX_op_eqv_i64:
1791 return TCG_TARGET_HAS_eqv_i64;
1792 case INDEX_op_nand_i64:
1793 return TCG_TARGET_HAS_nand_i64;
1794 case INDEX_op_nor_i64:
1795 return TCG_TARGET_HAS_nor_i64;
1796 case INDEX_op_clz_i64:
1797 return TCG_TARGET_HAS_clz_i64;
1798 case INDEX_op_ctz_i64:
1799 return TCG_TARGET_HAS_ctz_i64;
1800 case INDEX_op_ctpop_i64:
1801 return TCG_TARGET_HAS_ctpop_i64;
1802 case INDEX_op_add2_i64:
1803 return TCG_TARGET_HAS_add2_i64;
1804 case INDEX_op_sub2_i64:
1805 return TCG_TARGET_HAS_sub2_i64;
1806 case INDEX_op_mulu2_i64:
1807 return TCG_TARGET_HAS_mulu2_i64;
1808 case INDEX_op_muls2_i64:
1809 return TCG_TARGET_HAS_muls2_i64;
1810 case INDEX_op_muluh_i64:
1811 return TCG_TARGET_HAS_muluh_i64;
1812 case INDEX_op_mulsh_i64:
1813 return TCG_TARGET_HAS_mulsh_i64;
1814
d2fd745f
RH		 1815 case INDEX_op_mov_vec:
1816 case INDEX_op_dup_vec:
37ee55a0 1817 case INDEX_op_dupm_vec:
d2fd745f
RH		 1818 case INDEX_op_ld_vec:
1819 case INDEX_op_st_vec:
1820 case INDEX_op_add_vec:
1821 case INDEX_op_sub_vec:
1822 case INDEX_op_and_vec:
1823 case INDEX_op_or_vec:
1824 case INDEX_op_xor_vec:
212be173 1825 case INDEX_op_cmp_vec:
d2fd745f
RH		 1826 return have_vec;
1827 case INDEX_op_dup2_vec:
1828 return have_vec && TCG_TARGET_REG_BITS == 32;
1829 case INDEX_op_not_vec:
1830 return have_vec && TCG_TARGET_HAS_not_vec;
1831 case INDEX_op_neg_vec:
1832 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1833 case INDEX_op_abs_vec:
1834 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1835 case INDEX_op_andc_vec:
1836 return have_vec && TCG_TARGET_HAS_andc_vec;
1837 case INDEX_op_orc_vec:
1838 return have_vec && TCG_TARGET_HAS_orc_vec;
ed523473
RH		 1839 case INDEX_op_nand_vec:
1840 return have_vec && TCG_TARGET_HAS_nand_vec;
1841 case INDEX_op_nor_vec:
1842 return have_vec && TCG_TARGET_HAS_nor_vec;
1843 case INDEX_op_eqv_vec:
1844 return have_vec && TCG_TARGET_HAS_eqv_vec;
3774030a
RH		 1845 case INDEX_op_mul_vec:
1846 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1847 case INDEX_op_shli_vec:
1848 case INDEX_op_shri_vec:
1849 case INDEX_op_sari_vec:
1850 return have_vec && TCG_TARGET_HAS_shi_vec;
1851 case INDEX_op_shls_vec:
1852 case INDEX_op_shrs_vec:
1853 case INDEX_op_sars_vec:
1854 return have_vec && TCG_TARGET_HAS_shs_vec;
1855 case INDEX_op_shlv_vec:
1856 case INDEX_op_shrv_vec:
1857 case INDEX_op_sarv_vec:
1858 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1859 case INDEX_op_rotli_vec:
1860 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1861 case INDEX_op_rotls_vec:
1862 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1863 case INDEX_op_rotlv_vec:
1864 case INDEX_op_rotrv_vec:
1865 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1866 case INDEX_op_ssadd_vec:
1867 case INDEX_op_usadd_vec:
1868 case INDEX_op_sssub_vec:
1869 case INDEX_op_ussub_vec:
1870 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1871 case INDEX_op_smin_vec:
1872 case INDEX_op_umin_vec:
1873 case INDEX_op_smax_vec:
1874 case INDEX_op_umax_vec:
1875 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1876 case INDEX_op_bitsel_vec:
1877 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1878 case INDEX_op_cmpsel_vec:
1879 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1880
db432672
RH		 1881 default:
1882 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1883 return true;
be0f34b5 1884 }
be0f34b5
RH		 1885}
1886
39004a71
RH		 1887static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
1888
ae8b75dc 1889void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1890{
3e92aa34 1891 const TCGHelperInfo *info;
39004a71
RH		 1892 TCGv_i64 extend_free[MAX_CALL_IARGS];
1893 int n_extend = 0;
75e8b9b7 1894 TCGOp *op;
39004a71 1895 int i, n, pi = 0, total_args;
afb49896 1896
619205fd 1897 info = g_hash_table_lookup(helper_table, (gpointer)func);
39004a71
RH		 1898 total_args = info->nr_out + info->nr_in + 2;
1899 op = tcg_op_alloc(INDEX_op_call, total_args);
2bece2c8 1900
38b47b19 1901#ifdef CONFIG_PLUGIN
17083f6f
EC		 1902 /* Flag helpers that may affect guest state */
1903 if (tcg_ctx->plugin_insn &&
1904 !(info->flags & TCG_CALL_PLUGIN) &&
1905 !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
38b47b19
EC		 1906 tcg_ctx->plugin_insn->calls_helpers = true;
1907 }
1908#endif
1909
39004a71
RH		 1910 TCGOP_CALLO(op) = n = info->nr_out;
1911 switch (n) {
1912 case 0:
1913 tcg_debug_assert(ret == NULL);
1914 break;
1915 case 1:
1916 tcg_debug_assert(ret != NULL);
1917 op->args[pi++] = temp_arg(ret);
1918 break;
1919 case 2:
466d3759 1920 case 4:
39004a71 1921 tcg_debug_assert(ret != NULL);
466d3759 1922 tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
39004a71 1923 tcg_debug_assert(ret->temp_subindex == 0);
466d3759
RH		 1924 for (i = 0; i < n; ++i) {
1925 op->args[pi++] = temp_arg(ret + i);
1926 }
39004a71
RH		 1927 break;
1928 default:
1929 g_assert_not_reached();
1930 }
1931
1932 TCGOP_CALLI(op) = n = info->nr_in;
1933 for (i = 0; i < n; i++) {
1934 const TCGCallArgumentLoc *loc = &info->in[i];
1935 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
1936
1937 switch (loc->kind) {
1938 case TCG_CALL_ARG_NORMAL:
313bdea8
RH		 1939 case TCG_CALL_ARG_BY_REF:
1940 case TCG_CALL_ARG_BY_REF_N:
39004a71
RH		 1941 op->args[pi++] = temp_arg(ts);
1942 break;
eb8b0224 1943
39004a71
RH		 1944 case TCG_CALL_ARG_EXTEND_U:
1945 case TCG_CALL_ARG_EXTEND_S:
1946 {
5dd48602 1947 TCGv_i64 temp = tcg_temp_ebb_new_i64();
39004a71
RH		 1948 TCGv_i32 orig = temp_tcgv_i32(ts);
1949
1950 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
eb8b0224
RH		 1951 tcg_gen_ext_i32_i64(temp, orig);
1952 } else {
1953 tcg_gen_extu_i32_i64(temp, orig);
1954 }
39004a71
RH		 1955 op->args[pi++] = tcgv_i64_arg(temp);
1956 extend_free[n_extend++] = temp;
2bece2c8 1957 }
e2a9dd6b 1958 break;
7b7d8b2d 1959
e2a9dd6b
RH		 1960 default:
1961 g_assert_not_reached();
c896fe29
FB		 1962 }
1963 }
75e8b9b7 1964 op->args[pi++] = (uintptr_t)func;
3e92aa34 1965 op->args[pi++] = (uintptr_t)info;
39004a71 1966 tcg_debug_assert(pi == total_args);
a7812ae4 1967
39004a71 1968 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
7319d83a 1969
39004a71
RH		 1970 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
1971 for (i = 0; i < n_extend; ++i) {
1972 tcg_temp_free_i64(extend_free[i]);
2bece2c8 1973 }
c896fe29 1974}
c896fe29 1975
8fcd3692 1976static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1977{
ac3b8891 1978 int i, n;
ac3b8891 1979
ee17db83
RH		 1980 for (i = 0, n = s->nb_temps; i < n; i++) {
1981 TCGTemp *ts = &s->temps[i];
1982 TCGTempVal val = TEMP_VAL_MEM;
1983
1984 switch (ts->kind) {
c0522136
RH		 1985 case TEMP_CONST:
1986 val = TEMP_VAL_CONST;
1987 break;
ee17db83
RH		 1988 case TEMP_FIXED:
1989 val = TEMP_VAL_REG;
1990 break;
1991 case TEMP_GLOBAL:
1992 break;
c7482438 1993 case TEMP_EBB:
ee17db83
RH		 1994 val = TEMP_VAL_DEAD;
1995 /* fall through */
f57c6915 1996 case TEMP_TB:
ee17db83
RH		 1997 ts->mem_allocated = 0;
1998 break;
1999 default:
2000 g_assert_not_reached();
2001 }
2002 ts->val_type = val;
e8996ee0 2003 }
f8b2f202
RH		 2004
2005 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 2006}
2007
f8b2f202
RH		 2008static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
2009 TCGTemp *ts)
c896fe29 2010{
1807f4c4 2011 int idx = temp_idx(ts);
ac56dd48 2012
ee17db83
RH		 2013 switch (ts->kind) {
2014 case TEMP_FIXED:
2015 case TEMP_GLOBAL:
ac56dd48 2016 pstrcpy(buf, buf_size, ts->name);
ee17db83 2017 break;
f57c6915 2018 case TEMP_TB:
f8b2f202 2019 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83 2020 break;
c7482438 2021 case TEMP_EBB:
f8b2f202 2022 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 2023 break;
c0522136
RH		 2024 case TEMP_CONST:
2025 switch (ts->type) {
2026 case TCG_TYPE_I32:
2027 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
2028 break;
2029#if TCG_TARGET_REG_BITS > 32
2030 case TCG_TYPE_I64:
2031 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
2032 break;
2033#endif
2034 case TCG_TYPE_V64:
2035 case TCG_TYPE_V128:
2036 case TCG_TYPE_V256:
2037 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
2038 64 << (ts->type - TCG_TYPE_V64), ts->val);
2039 break;
2040 default:
2041 g_assert_not_reached();
2042 }
2043 break;
c896fe29
FB		 2044 }
2045 return buf;
2046}
2047
43439139
RH		 2048static char *tcg_get_arg_str(TCGContext *s, char *buf,
2049 int buf_size, TCGArg arg)
f8b2f202 2050{
43439139 2051 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 2052}
2053
f48f3ede
BS		 2054static const char * const cond_name[] =
2055{
0aed257f
RH		 2056 [TCG_COND_NEVER] = "never",
2057 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 2058 [TCG_COND_EQ] = "eq",
2059 [TCG_COND_NE] = "ne",
2060 [TCG_COND_LT] = "lt",
2061 [TCG_COND_GE] = "ge",
2062 [TCG_COND_LE] = "le",
2063 [TCG_COND_GT] = "gt",
2064 [TCG_COND_LTU] = "ltu",
2065 [TCG_COND_GEU] = "geu",
2066 [TCG_COND_LEU] = "leu",
2067 [TCG_COND_GTU] = "gtu"
2068};
2069
f713d6ad
RH		 2070static const char * const ldst_name[] =
2071{
2072 [MO_UB] = "ub",
2073 [MO_SB] = "sb",
2074 [MO_LEUW] = "leuw",
2075 [MO_LESW] = "lesw",
2076 [MO_LEUL] = "leul",
2077 [MO_LESL] = "lesl",
fc313c64 2078 [MO_LEUQ] = "leq",
f713d6ad
RH		 2079 [MO_BEUW] = "beuw",
2080 [MO_BESW] = "besw",
2081 [MO_BEUL] = "beul",
2082 [MO_BESL] = "besl",
fc313c64 2083 [MO_BEUQ] = "beq",
f713d6ad
RH		 2084};
2085
1f00b27f 2086static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 2087#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 2088 [MO_UNALN >> MO_ASHIFT] = "un+",
2089 [MO_ALIGN >> MO_ASHIFT] = "",
2090#else
2091 [MO_UNALN >> MO_ASHIFT] = "",
2092 [MO_ALIGN >> MO_ASHIFT] = "al+",
2093#endif
2094 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
2095 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
2096 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
2097 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
2098 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
2099 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
2100};
2101
587195bd
RH		 2102static const char bswap_flag_name[][6] = {
2103 [TCG_BSWAP_IZ] = "iz",
2104 [TCG_BSWAP_OZ] = "oz",
2105 [TCG_BSWAP_OS] = "os",
2106 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
2107 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
2108};
2109
b016486e
RH		 2110static inline bool tcg_regset_single(TCGRegSet d)
2111{
2112 return (d & (d - 1)) == 0;
2113}
2114
2115static inline TCGReg tcg_regset_first(TCGRegSet d)
2116{
2117 if (TCG_TARGET_NB_REGS <= 32) {
2118 return ctz32(d);
2119 } else {
2120 return ctz64(d);
2121 }
2122}
2123
b7a83ff8
RH		 2124/* Return only the number of characters output -- no error return. */
2125#define ne_fprintf(...) \
2126 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
2127
2128static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
c896fe29 2129{
c896fe29 2130 char buf[128];
c45cb8bb 2131 TCGOp *op;
c45cb8bb 2132
15fa08f8 2133 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 2134 int i, k, nb_oargs, nb_iargs, nb_cargs;
2135 const TCGOpDef *def;
c45cb8bb 2136 TCGOpcode c;
bdfb460e 2137 int col = 0;
c896fe29 2138
c45cb8bb 2139 c = op->opc;
c896fe29 2140 def = &tcg_op_defs[c];
c45cb8bb 2141
765b842a 2142 if (c == INDEX_op_insn_start) {
b016486e 2143 nb_oargs = 0;
b7a83ff8 2144 col += ne_fprintf(f, "\n ----");
9aef40ed
RH		 2145
2146 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
2147 target_ulong a;
7e4597d7 2148#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 2149 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 2150#else
efee3746 2151 a = op->args[i];
7e4597d7 2152#endif
b7a83ff8 2153 col += ne_fprintf(f, " " TARGET_FMT_lx, a);
eeacee4d 2154 }
7e4597d7 2155 } else if (c == INDEX_op_call) {
3e92aa34 2156 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 2157 void *func = tcg_call_func(op);
3e92aa34 2158
c896fe29 2159 /* variable number of arguments */
cd9090aa
RH		 2160 nb_oargs = TCGOP_CALLO(op);
2161 nb_iargs = TCGOP_CALLI(op);
c896fe29 2162 nb_cargs = def->nb_cargs;
c896fe29 2163
b7a83ff8 2164 col += ne_fprintf(f, " %s ", def->name);
3e92aa34
RH		 2165
2166 /*
2167 * Print the function name from TCGHelperInfo, if available.
2168 * Note that plugins have a template function for the info,
2169 * but the actual function pointer comes from the plugin.
2170 */
3e92aa34 2171 if (func == info->func) {
b7a83ff8 2172 col += ne_fprintf(f, "%s", info->name);
3e92aa34 2173 } else {
b7a83ff8 2174 col += ne_fprintf(f, "plugin(%p)", func);
3e92aa34
RH		 2175 }
2176
b7a83ff8 2177 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
cf066674 2178 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 2179 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2180 op->args[i]));
b03cce8e 2181 }
cf066674 2182 for (i = 0; i < nb_iargs; i++) {
efee3746 2183 TCGArg arg = op->args[nb_oargs + i];
39004a71 2184 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
b7a83ff8 2185 col += ne_fprintf(f, ",%s", t);
e8996ee0 2186 }
b03cce8e 2187 } else {
b7a83ff8 2188 col += ne_fprintf(f, " %s ", def->name);
c45cb8bb
RH		 2189
2190 nb_oargs = def->nb_oargs;
2191 nb_iargs = def->nb_iargs;
2192 nb_cargs = def->nb_cargs;
2193
d2fd745f 2194 if (def->flags & TCG_OPF_VECTOR) {
b7a83ff8
RH		 2195 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
2196 8 << TCGOP_VECE(op));
d2fd745f
RH		 2197 }
2198
b03cce8e 2199 k = 0;
c45cb8bb 2200 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 2201 const char *sep = k ? "," : "";
2202 col += ne_fprintf(f, "%s%s", sep,
2203 tcg_get_arg_str(s, buf, sizeof(buf),
2204 op->args[k++]));
b03cce8e 2205 }
c45cb8bb 2206 for (i = 0; i < nb_iargs; i++) {
b7a83ff8
RH		 2207 const char *sep = k ? "," : "";
2208 col += ne_fprintf(f, "%s%s", sep,
2209 tcg_get_arg_str(s, buf, sizeof(buf),
2210 op->args[k++]));
b03cce8e 2211 }
be210acb
RH		 2212 switch (c) {
2213 case INDEX_op_brcond_i32:
be210acb 2214 case INDEX_op_setcond_i32:
ffc5ea09 2215 case INDEX_op_movcond_i32:
ffc5ea09 2216 case INDEX_op_brcond2_i32:
be210acb 2217 case INDEX_op_setcond2_i32:
ffc5ea09 2218 case INDEX_op_brcond_i64:
be210acb 2219 case INDEX_op_setcond_i64:
ffc5ea09 2220 case INDEX_op_movcond_i64:
212be173 2221 case INDEX_op_cmp_vec:
f75da298 2222 case INDEX_op_cmpsel_vec:
efee3746
RH		 2223 if (op->args[k] < ARRAY_SIZE(cond_name)
2224 && cond_name[op->args[k]]) {
b7a83ff8 2225 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
eeacee4d 2226 } else {
b7a83ff8 2227 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 2228 }
f48f3ede 2229 i = 1;
be210acb 2230 break;
f713d6ad
RH		 2231 case INDEX_op_qemu_ld_i32:
2232 case INDEX_op_qemu_st_i32:
07ce0b05 2233 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 2234 case INDEX_op_qemu_ld_i64:
2235 case INDEX_op_qemu_st_i64:
59227d5d 2236 {
9002ffcb 2237 MemOpIdx oi = op->args[k++];
14776ab5 2238 MemOp op = get_memop(oi);
59227d5d
RH		 2239 unsigned ix = get_mmuidx(oi);
2240
59c4b7e8 2241 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
b7a83ff8 2242 col += ne_fprintf(f, ",$0x%x,%u", op, ix);
59c4b7e8 2243 } else {
1f00b27f
SS		 2244 const char *s_al, *s_op;
2245 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 2246 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
b7a83ff8 2247 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 2248 }
2249 i = 1;
f713d6ad 2250 }
f713d6ad 2251 break;
587195bd
RH		 2252 case INDEX_op_bswap16_i32:
2253 case INDEX_op_bswap16_i64:
2254 case INDEX_op_bswap32_i32:
2255 case INDEX_op_bswap32_i64:
2256 case INDEX_op_bswap64_i64:
2257 {
2258 TCGArg flags = op->args[k];
2259 const char *name = NULL;
2260
2261 if (flags < ARRAY_SIZE(bswap_flag_name)) {
2262 name = bswap_flag_name[flags];
2263 }
2264 if (name) {
b7a83ff8 2265 col += ne_fprintf(f, ",%s", name);
587195bd 2266 } else {
b7a83ff8 2267 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
587195bd
RH		 2268 }
2269 i = k = 1;
2270 }
2271 break;
be210acb 2272 default:
f48f3ede 2273 i = 0;
be210acb
RH		 2274 break;
2275 }
51e3972c
RH		 2276 switch (c) {
2277 case INDEX_op_set_label:
2278 case INDEX_op_br:
2279 case INDEX_op_brcond_i32:
2280 case INDEX_op_brcond_i64:
2281 case INDEX_op_brcond2_i32:
b7a83ff8
RH		 2282 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
2283 arg_label(op->args[k])->id);
51e3972c
RH		 2284 i++, k++;
2285 break;
3470867b
RH		 2286 case INDEX_op_mb:
2287 {
2288 TCGBar membar = op->args[k];
2289 const char *b_op, *m_op;
2290
2291 switch (membar & TCG_BAR_SC) {
2292 case 0:
2293 b_op = "none";
2294 break;
2295 case TCG_BAR_LDAQ:
2296 b_op = "acq";
2297 break;
2298 case TCG_BAR_STRL:
2299 b_op = "rel";
2300 break;
2301 case TCG_BAR_SC:
2302 b_op = "seq";
2303 break;
2304 default:
2305 g_assert_not_reached();
2306 }
2307
2308 switch (membar & TCG_MO_ALL) {
2309 case 0:
2310 m_op = "none";
2311 break;
2312 case TCG_MO_LD_LD:
2313 m_op = "rr";
2314 break;
2315 case TCG_MO_LD_ST:
2316 m_op = "rw";
2317 break;
2318 case TCG_MO_ST_LD:
2319 m_op = "wr";
2320 break;
2321 case TCG_MO_ST_ST:
2322 m_op = "ww";
2323 break;
2324 case TCG_MO_LD_LD | TCG_MO_LD_ST:
2325 m_op = "rr+rw";
2326 break;
2327 case TCG_MO_LD_LD | TCG_MO_ST_LD:
2328 m_op = "rr+wr";
2329 break;
2330 case TCG_MO_LD_LD | TCG_MO_ST_ST:
2331 m_op = "rr+ww";
2332 break;
2333 case TCG_MO_LD_ST | TCG_MO_ST_LD:
2334 m_op = "rw+wr";
2335 break;
2336 case TCG_MO_LD_ST | TCG_MO_ST_ST:
2337 m_op = "rw+ww";
2338 break;
2339 case TCG_MO_ST_LD | TCG_MO_ST_ST:
2340 m_op = "wr+ww";
2341 break;
2342 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD:
2343 m_op = "rr+rw+wr";
2344 break;
2345 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST:
2346 m_op = "rr+rw+ww";
2347 break;
2348 case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST:
2349 m_op = "rr+wr+ww";
2350 break;
2351 case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST:
2352 m_op = "rw+wr+ww";
2353 break;
2354 case TCG_MO_ALL:
2355 m_op = "all";
2356 break;
2357 default:
2358 g_assert_not_reached();
2359 }
2360
2361 col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op);
2362 i++, k++;
2363 }
2364 break;
51e3972c
RH		 2365 default:
2366 break;
2367 }
2368 for (; i < nb_cargs; i++, k++) {
b7a83ff8
RH		 2369 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
2370 op->args[k]);
bdfb460e
RH		 2371 }
2372 }
bdfb460e 2373
1894f69a 2374 if (have_prefs || op->life) {
b7a83ff8
RH		 2375 for (; col < 40; ++col) {
2376 putc(' ', f);
bdfb460e 2377 }
1894f69a
RH		 2378 }
2379
2380 if (op->life) {
2381 unsigned life = op->life;
bdfb460e
RH		 2382
2383 if (life & (SYNC_ARG * 3)) {
b7a83ff8 2384 ne_fprintf(f, " sync:");
bdfb460e
RH		 2385 for (i = 0; i < 2; ++i) {
2386 if (life & (SYNC_ARG << i)) {
b7a83ff8 2387 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2388 }
2389 }
2390 }
2391 life /= DEAD_ARG;
2392 if (life) {
b7a83ff8 2393 ne_fprintf(f, " dead:");
bdfb460e
RH		 2394 for (i = 0; life; ++i, life >>= 1) {
2395 if (life & 1) {
b7a83ff8 2396 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2397 }
2398 }
b03cce8e 2399 }
c896fe29 2400 }
1894f69a
RH		 2401
2402 if (have_prefs) {
2403 for (i = 0; i < nb_oargs; ++i) {
31fd884b 2404 TCGRegSet set = output_pref(op, i);
1894f69a
RH		 2405
2406 if (i == 0) {
b7a83ff8 2407 ne_fprintf(f, " pref=");
1894f69a 2408 } else {
b7a83ff8 2409 ne_fprintf(f, ",");
1894f69a
RH		 2410 }
2411 if (set == 0) {
b7a83ff8 2412 ne_fprintf(f, "none");
1894f69a 2413 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
b7a83ff8 2414 ne_fprintf(f, "all");
1894f69a
RH		 2415#ifdef CONFIG_DEBUG_TCG
2416 } else if (tcg_regset_single(set)) {
2417 TCGReg reg = tcg_regset_first(set);
b7a83ff8 2418 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
1894f69a
RH		 2419#endif
2420 } else if (TCG_TARGET_NB_REGS <= 32) {
b7a83ff8 2421 ne_fprintf(f, "0x%x", (uint32_t)set);
1894f69a 2422 } else {
b7a83ff8 2423 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
1894f69a
RH		 2424 }
2425 }
2426 }
2427
b7a83ff8 2428 putc('\n', f);
c896fe29
FB		 2429 }
2430}
2431
2432/* we give more priority to constraints with less registers */
2433static int get_constraint_priority(const TCGOpDef *def, int k)
2434{
74a11790 2435 const TCGArgConstraint *arg_ct = &def->args_ct[k];
29f5e925 2436 int n = ctpop64(arg_ct->regs);
c896fe29 2437
29f5e925
RH		 2438 /*
2439 * Sort constraints of a single register first, which includes output
2440 * aliases (which must exactly match the input already allocated).
2441 */
2442 if (n == 1 || arg_ct->oalias) {
2443 return INT_MAX;
2444 }
2445
2446 /*
2447 * Sort register pairs next, first then second immediately after.
2448 * Arbitrarily sort multiple pairs by the index of the first reg;
2449 * there shouldn't be many pairs.
2450 */
2451 switch (arg_ct->pair) {
2452 case 1:
2453 case 3:
2454 return (k + 1) * 2;
2455 case 2:
2456 return (arg_ct->pair_index + 1) * 2 - 1;
c896fe29 2457 }
29f5e925
RH		 2458
2459 /* Finally, sort by decreasing register count. */
2460 assert(n > 1);
2461 return -n;
c896fe29
FB		 2462}
2463
2464/* sort from highest priority to lowest */
2465static void sort_constraints(TCGOpDef *def, int start, int n)
2466{
66792f90
RH		 2467 int i, j;
2468 TCGArgConstraint *a = def->args_ct;
c896fe29 2469
66792f90
RH		 2470 for (i = 0; i < n; i++) {
2471 a[start + i].sort_index = start + i;
2472 }
2473 if (n <= 1) {
c896fe29 2474 return;
66792f90
RH		 2475 }
2476 for (i = 0; i < n - 1; i++) {
2477 for (j = i + 1; j < n; j++) {
2478 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2479 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2480 if (p1 < p2) {
66792f90
RH		 2481 int tmp = a[start + i].sort_index;
2482 a[start + i].sort_index = a[start + j].sort_index;
2483 a[start + j].sort_index = tmp;
c896fe29
FB		 2484 }
2485 }
2486 }
2487}
2488
f69d277e 2489static void process_op_defs(TCGContext *s)
c896fe29 2490{
a9751609 2491 TCGOpcode op;
c896fe29 2492
f69d277e
RH		 2493 for (op = 0; op < NB_OPS; op++) {
2494 TCGOpDef *def = &tcg_op_defs[op];
2495 const TCGTargetOpDef *tdefs;
29f5e925
RH		 2496 bool saw_alias_pair = false;
2497 int i, o, i2, o2, nb_args;
f69d277e
RH		 2498
2499 if (def->flags & TCG_OPF_NOT_PRESENT) {
2500 continue;
2501 }
2502
c896fe29 2503 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2504 if (nb_args == 0) {
2505 continue;
2506 }
2507
4c22e840
RH		 2508 /*
2509 * Macro magic should make it impossible, but double-check that
2510 * the array index is in range. Since the signness of an enum
2511 * is implementation defined, force the result to unsigned.
2512 */
2513 unsigned con_set = tcg_target_op_def(op);
2514 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2515 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2516
2517 for (i = 0; i < nb_args; i++) {
2518 const char *ct_str = tdefs->args_ct_str[i];
8940ea0d
PMD		 2519 bool input_p = i >= def->nb_oargs;
2520
f69d277e 2521 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2522 tcg_debug_assert(ct_str != NULL);
f69d277e 2523
8940ea0d
PMD		 2524 switch (*ct_str) {
2525 case '0' ... '9':
2526 o = *ct_str - '0';
2527 tcg_debug_assert(input_p);
2528 tcg_debug_assert(o < def->nb_oargs);
2529 tcg_debug_assert(def->args_ct[o].regs != 0);
2530 tcg_debug_assert(!def->args_ct[o].oalias);
2531 def->args_ct[i] = def->args_ct[o];
2532 /* The output sets oalias. */
2533 def->args_ct[o].oalias = 1;
2534 def->args_ct[o].alias_index = i;
2535 /* The input sets ialias. */
2536 def->args_ct[i].ialias = 1;
2537 def->args_ct[i].alias_index = o;
29f5e925
RH		 2538 if (def->args_ct[i].pair) {
2539 saw_alias_pair = true;
2540 }
8940ea0d
PMD		 2541 tcg_debug_assert(ct_str[1] == '\0');
2542 continue;
2543
2544 case '&':
2545 tcg_debug_assert(!input_p);
2546 def->args_ct[i].newreg = true;
2547 ct_str++;
2548 break;
29f5e925
RH		 2549
2550 case 'p': /* plus */
2551 /* Allocate to the register after the previous. */
2552 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2553 o = i - 1;
2554 tcg_debug_assert(!def->args_ct[o].pair);
2555 tcg_debug_assert(!def->args_ct[o].ct);
2556 def->args_ct[i] = (TCGArgConstraint){
2557 .pair = 2,
2558 .pair_index = o,
2559 .regs = def->args_ct[o].regs << 1,
2560 };
2561 def->args_ct[o].pair = 1;
2562 def->args_ct[o].pair_index = i;
2563 tcg_debug_assert(ct_str[1] == '\0');
2564 continue;
2565
2566 case 'm': /* minus */
2567 /* Allocate to the register before the previous. */
2568 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2569 o = i - 1;
2570 tcg_debug_assert(!def->args_ct[o].pair);
2571 tcg_debug_assert(!def->args_ct[o].ct);
2572 def->args_ct[i] = (TCGArgConstraint){
2573 .pair = 1,
2574 .pair_index = o,
2575 .regs = def->args_ct[o].regs >> 1,
2576 };
2577 def->args_ct[o].pair = 2;
2578 def->args_ct[o].pair_index = i;
2579 tcg_debug_assert(ct_str[1] == '\0');
2580 continue;
8940ea0d
PMD		 2581 }
2582
2583 do {
2584 switch (*ct_str) {
17280ff4
RH		 2585 case 'i':
2586 def->args_ct[i].ct |= TCG_CT_CONST;
17280ff4 2587 break;
358b4923 2588
358b4923
RH		 2589 /* Include all of the target-specific constraints. */
2590
2591#undef CONST
2592#define CONST(CASE, MASK) \
8940ea0d 2593 case CASE: def->args_ct[i].ct |= MASK; break;
358b4923 2594#define REGS(CASE, MASK) \
8940ea0d 2595 case CASE: def->args_ct[i].regs |= MASK; break;
358b4923
RH		 2596
2597#include "tcg-target-con-str.h"
2598
2599#undef REGS
2600#undef CONST
17280ff4 2601 default:
8940ea0d
PMD		 2602 case '0' ... '9':
2603 case '&':
29f5e925
RH		 2604 case 'p':
2605 case 'm':
17280ff4 2606 /* Typo in TCGTargetOpDef constraint. */
358b4923 2607 g_assert_not_reached();
c896fe29 2608 }
8940ea0d 2609 } while (*++ct_str != '\0');
c896fe29
FB		 2610 }
2611
c68aaa18 2612 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2613 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2614
29f5e925
RH		 2615 /*
2616 * Fix up output pairs that are aliased with inputs.
2617 * When we created the alias, we copied pair from the output.
2618 * There are three cases:
2619 * (1a) Pairs of inputs alias pairs of outputs.
2620 * (1b) One input aliases the first of a pair of outputs.
2621 * (2) One input aliases the second of a pair of outputs.
2622 *
2623 * Case 1a is handled by making sure that the pair_index'es are
2624 * properly updated so that they appear the same as a pair of inputs.
2625 *
2626 * Case 1b is handled by setting the pair_index of the input to
2627 * itself, simply so it doesn't point to an unrelated argument.
2628 * Since we don't encounter the "second" during the input allocation
2629 * phase, nothing happens with the second half of the input pair.
2630 *
2631 * Case 2 is handled by setting the second input to pair=3, the
2632 * first output to pair=3, and the pair_index'es to match.
2633 */
2634 if (saw_alias_pair) {
2635 for (i = def->nb_oargs; i < nb_args; i++) {
2636 /*
2637 * Since [0-9pm] must be alone in the constraint string,
2638 * the only way they can both be set is if the pair comes
2639 * from the output alias.
2640 */
2641 if (!def->args_ct[i].ialias) {
2642 continue;
2643 }
2644 switch (def->args_ct[i].pair) {
2645 case 0:
2646 break;
2647 case 1:
2648 o = def->args_ct[i].alias_index;
2649 o2 = def->args_ct[o].pair_index;
2650 tcg_debug_assert(def->args_ct[o].pair == 1);
2651 tcg_debug_assert(def->args_ct[o2].pair == 2);
2652 if (def->args_ct[o2].oalias) {
2653 /* Case 1a */
2654 i2 = def->args_ct[o2].alias_index;
2655 tcg_debug_assert(def->args_ct[i2].pair == 2);
2656 def->args_ct[i2].pair_index = i;
2657 def->args_ct[i].pair_index = i2;
2658 } else {
2659 /* Case 1b */
2660 def->args_ct[i].pair_index = i;
2661 }
2662 break;
2663 case 2:
2664 o = def->args_ct[i].alias_index;
2665 o2 = def->args_ct[o].pair_index;
2666 tcg_debug_assert(def->args_ct[o].pair == 2);
2667 tcg_debug_assert(def->args_ct[o2].pair == 1);
2668 if (def->args_ct[o2].oalias) {
2669 /* Case 1a */
2670 i2 = def->args_ct[o2].alias_index;
2671 tcg_debug_assert(def->args_ct[i2].pair == 1);
2672 def->args_ct[i2].pair_index = i;
2673 def->args_ct[i].pair_index = i2;
2674 } else {
2675 /* Case 2 */
2676 def->args_ct[i].pair = 3;
2677 def->args_ct[o2].pair = 3;
2678 def->args_ct[i].pair_index = o2;
2679 def->args_ct[o2].pair_index = i;
2680 }
2681 break;
2682 default:
2683 g_assert_not_reached();
2684 }
2685 }
2686 }
2687
c896fe29
FB		 2688 /* sort the constraints (XXX: this is just an heuristic) */
2689 sort_constraints(def, 0, def->nb_oargs);
2690 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2691 }
c896fe29
FB		 2692}
2693
f85b1fc4 2694static void remove_label_use(TCGOp *op, int idx)
0c627cdc 2695{
f85b1fc4
RH		 2696 TCGLabel *label = arg_label(op->args[idx]);
2697 TCGLabelUse *use;
d88a117e 2698
f85b1fc4
RH		 2699 QSIMPLEQ_FOREACH(use, &label->branches, next) {
2700 if (use->op == op) {
2701 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next);
2702 return;
2703 }
2704 }
2705 g_assert_not_reached();
2706}
2707
2708void tcg_op_remove(TCGContext *s, TCGOp *op)
2709{
d88a117e
RH		 2710 switch (op->opc) {
2711 case INDEX_op_br:
f85b1fc4 2712 remove_label_use(op, 0);
d88a117e
RH		 2713 break;
2714 case INDEX_op_brcond_i32:
2715 case INDEX_op_brcond_i64:
f85b1fc4 2716 remove_label_use(op, 3);
d88a117e
RH		 2717 break;
2718 case INDEX_op_brcond2_i32:
f85b1fc4 2719 remove_label_use(op, 5);
d88a117e
RH		 2720 break;
2721 default:
2722 break;
2723 }
2724
15fa08f8
RH		 2725 QTAILQ_REMOVE(&s->ops, op, link);
2726 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2727 s->nb_ops--;
0c627cdc
RH		 2728
2729#ifdef CONFIG_PROFILER
d73415a3 2730 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2731#endif
2732}
2733
a80cdd31
RH		 2734void tcg_remove_ops_after(TCGOp *op)
2735{
2736 TCGContext *s = tcg_ctx;
2737
2738 while (true) {
2739 TCGOp *last = tcg_last_op();
2740 if (last == op) {
2741 return;
2742 }
2743 tcg_op_remove(s, last);
2744 }
2745}
2746
d4478943 2747static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
5a18407f 2748{
15fa08f8 2749 TCGContext *s = tcg_ctx;
cb10bc63
RH		 2750 TCGOp *op = NULL;
2751
2752 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
2753 QTAILQ_FOREACH(op, &s->free_ops, link) {
2754 if (nargs <= op->nargs) {
2755 QTAILQ_REMOVE(&s->free_ops, op, link);
2756 nargs = op->nargs;
2757 goto found;
2758 }
2759 }
15fa08f8 2760 }
cb10bc63
RH		 2761
2762 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
2763 nargs = MAX(4, nargs);
2764 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
2765
2766 found:
15fa08f8
RH		 2767 memset(op, 0, offsetof(TCGOp, link));
2768 op->opc = opc;
cb10bc63
RH		 2769 op->nargs = nargs;
2770
2771 /* Check for bitfield overflow. */
2772 tcg_debug_assert(op->nargs == nargs);
5a18407f 2773
cb10bc63 2774 s->nb_ops++;
15fa08f8
RH		 2775 return op;
2776}
2777
d4478943 2778TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
15fa08f8 2779{
d4478943 2780 TCGOp *op = tcg_op_alloc(opc, nargs);
15fa08f8
RH		 2781 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2782 return op;
2783}
5a18407f 2784
d4478943
PMD		 2785TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
2786 TCGOpcode opc, unsigned nargs)
15fa08f8 2787{
d4478943 2788 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 2789 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2790 return new_op;
2791}
2792
d4478943
PMD		 2793TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
2794 TCGOpcode opc, unsigned nargs)
5a18407f 2795{
d4478943 2796 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 2797 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2798 return new_op;
2799}
2800
968f305e
RH		 2801static void move_label_uses(TCGLabel *to, TCGLabel *from)
2802{
2803 TCGLabelUse *u;
2804
2805 QSIMPLEQ_FOREACH(u, &from->branches, next) {
2806 TCGOp *op = u->op;
2807 switch (op->opc) {
2808 case INDEX_op_br:
2809 op->args[0] = label_arg(to);
2810 break;
2811 case INDEX_op_brcond_i32:
2812 case INDEX_op_brcond_i64:
2813 op->args[3] = label_arg(to);
2814 break;
2815 case INDEX_op_brcond2_i32:
2816 op->args[5] = label_arg(to);
2817 break;
2818 default:
2819 g_assert_not_reached();
2820 }
2821 }
2822
2823 QSIMPLEQ_CONCAT(&to->branches, &from->branches);
2824}
2825
b4fc67c7 2826/* Reachable analysis : remove unreachable code. */
9bbee4c0
RH		 2827static void __attribute__((noinline))
2828reachable_code_pass(TCGContext *s)
b4fc67c7 2829{
4d89d0bb 2830 TCGOp *op, *op_next, *op_prev;
b4fc67c7
RH		 2831 bool dead = false;
2832
2833 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2834 bool remove = dead;
2835 TCGLabel *label;
b4fc67c7
RH		 2836
2837 switch (op->opc) {
2838 case INDEX_op_set_label:
2839 label = arg_label(op->args[0]);
4d89d0bb 2840
968f305e
RH		 2841 /*
2842 * Note that the first op in the TB is always a load,
2843 * so there is always something before a label.
2844 */
2845 op_prev = QTAILQ_PREV(op, link);
2846
2847 /*
2848 * If we find two sequential labels, move all branches to
2849 * reference the second label and remove the first label.
2850 * Do this before branch to next optimization, so that the
2851 * middle label is out of the way.
2852 */
2853 if (op_prev->opc == INDEX_op_set_label) {
2854 move_label_uses(label, arg_label(op_prev->args[0]));
2855 tcg_op_remove(s, op_prev);
2856 op_prev = QTAILQ_PREV(op, link);
2857 }
2858
4d89d0bb
RH		 2859 /*
2860 * Optimization can fold conditional branches to unconditional.
2861 * If we find a label which is preceded by an unconditional
2862 * branch to next, remove the branch. We couldn't do this when
2863 * processing the branch because any dead code between the branch
2864 * and label had not yet been removed.
2865 */
4d89d0bb
RH		 2866 if (op_prev->opc == INDEX_op_br &&
2867 label == arg_label(op_prev->args[0])) {
2868 tcg_op_remove(s, op_prev);
2869 /* Fall through means insns become live again. */
2870 dead = false;
2871 }
2872
f85b1fc4 2873 if (QSIMPLEQ_EMPTY(&label->branches)) {
b4fc67c7
RH		 2874 /*
2875 * While there is an occasional backward branch, virtually
2876 * all branches generated by the translators are forward.
2877 * Which means that generally we will have already removed
2878 * all references to the label that will be, and there is
2879 * little to be gained by iterating.
2880 */
2881 remove = true;
2882 } else {
2883 /* Once we see a label, insns become live again. */
2884 dead = false;
2885 remove = false;
b4fc67c7
RH		 2886 }
2887 break;
2888
2889 case INDEX_op_br:
2890 case INDEX_op_exit_tb:
2891 case INDEX_op_goto_ptr:
2892 /* Unconditional branches; everything following is dead. */
2893 dead = true;
2894 break;
2895
2896 case INDEX_op_call:
2897 /* Notice noreturn helper calls, raising exceptions. */
90163900 2898 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2899 dead = true;
2900 }
2901 break;
2902
2903 case INDEX_op_insn_start:
2904 /* Never remove -- we need to keep these for unwind. */
2905 remove = false;
2906 break;
2907
2908 default:
2909 break;
2910 }
2911
2912 if (remove) {
2913 tcg_op_remove(s, op);
2914 }
2915 }
2916}
2917
c70fbf0a
RH		 2918#define TS_DEAD 1
2919#define TS_MEM 2
2920
5a18407f
RH		 2921#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2922#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2923
25f49c5f
RH		 2924/* For liveness_pass_1, the register preferences for a given temp. */
2925static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2926{
2927 return ts->state_ptr;
2928}
2929
2930/* For liveness_pass_1, reset the preferences for a given temp to the
2931 * maximal regset for its type.
2932 */
2933static inline void la_reset_pref(TCGTemp *ts)
2934{
2935 *la_temp_pref(ts)
2936 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2937}
2938
9c43b68d
AJ		 2939/* liveness analysis: end of function: all temps are dead, and globals
2940 should be in memory. */
2616c808 2941static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2942{
b83eabea
RH		 2943 int i;
2944
2945 for (i = 0; i < ng; ++i) {
2946 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2947 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2948 }
2949 for (i = ng; i < nt; ++i) {
2950 s->temps[i].state = TS_DEAD;
25f49c5f 2951 la_reset_pref(&s->temps[i]);
b83eabea 2952 }
c896fe29
FB		 2953}
2954
9c43b68d
AJ		 2955/* liveness analysis: end of basic block: all temps are dead, globals
2956 and local temps should be in memory. */
2616c808 2957static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2958{
b83eabea 2959 int i;
641d5fbe 2960
ee17db83
RH		 2961 for (i = 0; i < nt; ++i) {
2962 TCGTemp *ts = &s->temps[i];
2963 int state;
2964
2965 switch (ts->kind) {
2966 case TEMP_FIXED:
2967 case TEMP_GLOBAL:
f57c6915 2968 case TEMP_TB:
ee17db83
RH		 2969 state = TS_DEAD | TS_MEM;
2970 break;
c7482438 2971 case TEMP_EBB:
c0522136 2972 case TEMP_CONST:
ee17db83
RH		 2973 state = TS_DEAD;
2974 break;
2975 default:
2976 g_assert_not_reached();
2977 }
2978 ts->state = state;
2979 la_reset_pref(ts);
641d5fbe
FB		 2980 }
2981}
2982
f65a061c
RH		 2983/* liveness analysis: sync globals back to memory. */
2984static void la_global_sync(TCGContext *s, int ng)
2985{
2986 int i;
2987
2988 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2989 int state = s->temps[i].state;
2990 s->temps[i].state = state | TS_MEM;
2991 if (state == TS_DEAD) {
2992 /* If the global was previously dead, reset prefs. */
2993 la_reset_pref(&s->temps[i]);
2994 }
f65a061c
RH		 2995 }
2996}
2997
b4cb76e6 2998/*
c7482438
RH		 2999 * liveness analysis: conditional branch: all temps are dead unless
3000 * explicitly live-across-conditional-branch, globals and local temps
3001 * should be synced.
b4cb76e6
RH		 3002 */
3003static void la_bb_sync(TCGContext *s, int ng, int nt)
3004{
3005 la_global_sync(s, ng);
3006
3007 for (int i = ng; i < nt; ++i) {
c0522136
RH		 3008 TCGTemp *ts = &s->temps[i];
3009 int state;
3010
3011 switch (ts->kind) {
f57c6915 3012 case TEMP_TB:
c0522136
RH		 3013 state = ts->state;
3014 ts->state = state | TS_MEM;
b4cb76e6
RH		 3015 if (state != TS_DEAD) {
3016 continue;
3017 }
c0522136 3018 break;
c7482438 3019 case TEMP_EBB:
c0522136
RH		 3020 case TEMP_CONST:
3021 continue;
3022 default:
3023 g_assert_not_reached();
b4cb76e6
RH		 3024 }
3025 la_reset_pref(&s->temps[i]);
3026 }
3027}
3028
f65a061c
RH		 3029/* liveness analysis: sync globals back to memory and kill. */
3030static void la_global_kill(TCGContext *s, int ng)
3031{
3032 int i;
3033
3034 for (i = 0; i < ng; i++) {
3035 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 3036 la_reset_pref(&s->temps[i]);
3037 }
3038}
3039
3040/* liveness analysis: note live globals crossing calls. */
3041static void la_cross_call(TCGContext *s, int nt)
3042{
3043 TCGRegSet mask = ~tcg_target_call_clobber_regs;
3044 int i;
3045
3046 for (i = 0; i < nt; i++) {
3047 TCGTemp *ts = &s->temps[i];
3048 if (!(ts->state & TS_DEAD)) {
3049 TCGRegSet *pset = la_temp_pref(ts);
3050 TCGRegSet set = *pset;
3051
3052 set &= mask;
3053 /* If the combination is not possible, restart. */
3054 if (set == 0) {
3055 set = tcg_target_available_regs[ts->type] & mask;
3056 }
3057 *pset = set;
3058 }
f65a061c
RH		 3059 }
3060}
3061
874b8574
RH		 3062/*
3063 * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce
3064 * to TEMP_EBB, if possible.
3065 */
3066static void __attribute__((noinline))
3067liveness_pass_0(TCGContext *s)
3068{
3069 void * const multiple_ebb = (void *)(uintptr_t)-1;
3070 int nb_temps = s->nb_temps;
3071 TCGOp *op, *ebb;
3072
3073 for (int i = s->nb_globals; i < nb_temps; ++i) {
3074 s->temps[i].state_ptr = NULL;
3075 }
3076
3077 /*
3078 * Represent each EBB by the op at which it begins. In the case of
3079 * the first EBB, this is the first op, otherwise it is a label.
3080 * Collect the uses of each TEMP_TB: NULL for unused, EBB for use
3081 * within a single EBB, else MULTIPLE_EBB.
3082 */
3083 ebb = QTAILQ_FIRST(&s->ops);
3084 QTAILQ_FOREACH(op, &s->ops, link) {
3085 const TCGOpDef *def;
3086 int nb_oargs, nb_iargs;
3087
3088 switch (op->opc) {
3089 case INDEX_op_set_label:
3090 ebb = op;
3091 continue;
3092 case INDEX_op_discard:
3093 continue;
3094 case INDEX_op_call:
3095 nb_oargs = TCGOP_CALLO(op);
3096 nb_iargs = TCGOP_CALLI(op);
3097 break;
3098 default:
3099 def = &tcg_op_defs[op->opc];
3100 nb_oargs = def->nb_oargs;
3101 nb_iargs = def->nb_iargs;
3102 break;
3103 }
3104
3105 for (int i = 0; i < nb_oargs + nb_iargs; ++i) {
3106 TCGTemp *ts = arg_temp(op->args[i]);
3107
3108 if (ts->kind != TEMP_TB) {
3109 continue;
3110 }
3111 if (ts->state_ptr == NULL) {
3112 ts->state_ptr = ebb;
3113 } else if (ts->state_ptr != ebb) {
3114 ts->state_ptr = multiple_ebb;
3115 }
3116 }
3117 }
3118
3119 /*
3120 * For TEMP_TB that turned out not to be used beyond one EBB,
3121 * reduce the liveness to TEMP_EBB.
3122 */
3123 for (int i = s->nb_globals; i < nb_temps; ++i) {
3124 TCGTemp *ts = &s->temps[i];
3125 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
3126 ts->kind = TEMP_EBB;
3127 }
3128 }
3129}
3130
a1b3c48d 3131/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 3132 given input arguments is dead. Instructions updating dead
3133 temporaries are removed. */
9bbee4c0
RH		 3134static void __attribute__((noinline))
3135liveness_pass_1(TCGContext *s)
c896fe29 3136{
c70fbf0a 3137 int nb_globals = s->nb_globals;
2616c808 3138 int nb_temps = s->nb_temps;
15fa08f8 3139 TCGOp *op, *op_prev;
25f49c5f
RH		 3140 TCGRegSet *prefs;
3141 int i;
3142
3143 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
3144 for (i = 0; i < nb_temps; ++i) {
3145 s->temps[i].state_ptr = prefs + i;
3146 }
a1b3c48d 3147
ae36a246 3148 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 3149 la_func_end(s, nb_globals, nb_temps);
c896fe29 3150
eae3eb3e 3151 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 3152 int nb_iargs, nb_oargs;
c45cb8bb
RH		 3153 TCGOpcode opc_new, opc_new2;
3154 bool have_opc_new2;
a1b3c48d 3155 TCGLifeData arg_life = 0;
25f49c5f 3156 TCGTemp *ts;
c45cb8bb
RH		 3157 TCGOpcode opc = op->opc;
3158 const TCGOpDef *def = &tcg_op_defs[opc];
3159
c45cb8bb 3160 switch (opc) {
c896fe29 3161 case INDEX_op_call:
c6e113f5 3162 {
39004a71
RH		 3163 const TCGHelperInfo *info = tcg_call_info(op);
3164 int call_flags = tcg_call_flags(op);
c896fe29 3165
cd9090aa
RH		 3166 nb_oargs = TCGOP_CALLO(op);
3167 nb_iargs = TCGOP_CALLI(op);
c6e113f5 3168
c45cb8bb 3169 /* pure functions can be removed if their result is unused */
78505279 3170 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 3171 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 3172 ts = arg_temp(op->args[i]);
3173 if (ts->state != TS_DEAD) {
c6e113f5 3174 goto do_not_remove_call;
9c43b68d 3175 }
c6e113f5 3176 }
c45cb8bb 3177 goto do_remove;
152c35aa
RH		 3178 }
3179 do_not_remove_call:
c896fe29 3180
25f49c5f 3181 /* Output args are dead. */
152c35aa 3182 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 3183 ts = arg_temp(op->args[i]);
3184 if (ts->state & TS_DEAD) {
152c35aa
RH		 3185 arg_life |= DEAD_ARG << i;
3186 }
25f49c5f 3187 if (ts->state & TS_MEM) {
152c35aa 3188 arg_life |= SYNC_ARG << i;
c6e113f5 3189 }
25f49c5f
RH		 3190 ts->state = TS_DEAD;
3191 la_reset_pref(ts);
152c35aa 3192 }
78505279 3193
31fd884b
RH		 3194 /* Not used -- it will be tcg_target_call_oarg_reg(). */
3195 memset(op->output_pref, 0, sizeof(op->output_pref));
3196
152c35aa
RH		 3197 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
3198 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 3199 la_global_kill(s, nb_globals);
152c35aa 3200 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 3201 la_global_sync(s, nb_globals);
152c35aa 3202 }
b9c18f56 3203
25f49c5f 3204 /* Record arguments that die in this helper. */
152c35aa 3205 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f 3206 ts = arg_temp(op->args[i]);
39004a71 3207 if (ts->state & TS_DEAD) {
152c35aa 3208 arg_life |= DEAD_ARG << i;
c6e113f5 3209 }
152c35aa 3210 }
25f49c5f
RH		 3211
3212 /* For all live registers, remove call-clobbered prefs. */
3213 la_cross_call(s, nb_temps);
3214
39004a71
RH		 3215 /*
3216 * Input arguments are live for preceding opcodes.
3217 *
3218 * For those arguments that die, and will be allocated in
3219 * registers, clear the register set for that arg, to be
3220 * filled in below. For args that will be on the stack,
3221 * reset to any available reg. Process arguments in reverse
3222 * order so that if a temp is used more than once, the stack
3223 * reset to max happens before the register reset to 0.
3224 */
3225 for (i = nb_iargs - 1; i >= 0; i--) {
3226 const TCGCallArgumentLoc *loc = &info->in[i];
3227 ts = arg_temp(op->args[nb_oargs + i]);
25f49c5f 3228
39004a71
RH		 3229 if (ts->state & TS_DEAD) {
3230 switch (loc->kind) {
3231 case TCG_CALL_ARG_NORMAL:
3232 case TCG_CALL_ARG_EXTEND_U:
3233 case TCG_CALL_ARG_EXTEND_S:
3234 if (REG_P(loc)) {
3235 *la_temp_pref(ts) = 0;
3236 break;
3237 }
3238 /* fall through */
3239 default:
3240 *la_temp_pref(ts) =
3241 tcg_target_available_regs[ts->type];
3242 break;
3243 }
25f49c5f
RH		 3244 ts->state &= ~TS_DEAD;
3245 }
3246 }
3247
39004a71
RH		 3248 /*
3249 * For each input argument, add its input register to prefs.
3250 * If a temp is used once, this produces a single set bit;
3251 * if a temp is used multiple times, this produces a set.
3252 */
3253 for (i = 0; i < nb_iargs; i++) {
3254 const TCGCallArgumentLoc *loc = &info->in[i];
3255 ts = arg_temp(op->args[nb_oargs + i]);
3256
3257 switch (loc->kind) {
3258 case TCG_CALL_ARG_NORMAL:
3259 case TCG_CALL_ARG_EXTEND_U:
3260 case TCG_CALL_ARG_EXTEND_S:
3261 if (REG_P(loc)) {
3262 tcg_regset_set_reg(*la_temp_pref(ts),
3263 tcg_target_call_iarg_regs[loc->arg_slot]);
3264 }
3265 break;
3266 default:
3267 break;
c19f47bf 3268 }
c896fe29 3269 }
c896fe29 3270 }
c896fe29 3271 break;
765b842a 3272 case INDEX_op_insn_start:
c896fe29 3273 break;
5ff9d6a4 3274 case INDEX_op_discard:
5ff9d6a4 3275 /* mark the temporary as dead */
25f49c5f
RH		 3276 ts = arg_temp(op->args[0]);
3277 ts->state = TS_DEAD;
3278 la_reset_pref(ts);
5ff9d6a4 3279 break;
1305c451
RH		 3280
3281 case INDEX_op_add2_i32:
c45cb8bb 3282 opc_new = INDEX_op_add_i32;
f1fae40c 3283 goto do_addsub2;
1305c451 3284 case INDEX_op_sub2_i32:
c45cb8bb 3285 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 3286 goto do_addsub2;
3287 case INDEX_op_add2_i64:
c45cb8bb 3288 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 3289 goto do_addsub2;
3290 case INDEX_op_sub2_i64:
c45cb8bb 3291 opc_new = INDEX_op_sub_i64;
f1fae40c 3292 do_addsub2:
1305c451
RH		 3293 nb_iargs = 4;
3294 nb_oargs = 2;
3295 /* Test if the high part of the operation is dead, but not
3296 the low part. The result can be optimized to a simple
3297 add or sub. This happens often for x86_64 guest when the
3298 cpu mode is set to 32 bit. */
b83eabea
RH		 3299 if (arg_temp(op->args[1])->state == TS_DEAD) {
3300 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 3301 goto do_remove;
3302 }
c45cb8bb
RH		 3303 /* Replace the opcode and adjust the args in place,
3304 leaving 3 unused args at the end. */
3305 op->opc = opc = opc_new;
efee3746
RH		 3306 op->args[1] = op->args[2];
3307 op->args[2] = op->args[4];
1305c451
RH		 3308 /* Fall through and mark the single-word operation live. */
3309 nb_iargs = 2;
3310 nb_oargs = 1;
3311 }
3312 goto do_not_remove;
3313
1414968a 3314 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 3315 opc_new = INDEX_op_mul_i32;
3316 opc_new2 = INDEX_op_muluh_i32;
3317 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 3318 goto do_mul2;
f1fae40c 3319 case INDEX_op_muls2_i32:
c45cb8bb
RH		 3320 opc_new = INDEX_op_mul_i32;
3321 opc_new2 = INDEX_op_mulsh_i32;
3322 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 3323 goto do_mul2;
3324 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 3325 opc_new = INDEX_op_mul_i64;
3326 opc_new2 = INDEX_op_muluh_i64;
3327 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 3328 goto do_mul2;
f1fae40c 3329 case INDEX_op_muls2_i64:
c45cb8bb
RH		 3330 opc_new = INDEX_op_mul_i64;
3331 opc_new2 = INDEX_op_mulsh_i64;
3332 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 3333 goto do_mul2;
f1fae40c 3334 do_mul2:
1414968a
RH		 3335 nb_iargs = 2;
3336 nb_oargs = 2;
b83eabea
RH		 3337 if (arg_temp(op->args[1])->state == TS_DEAD) {
3338 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 3339 /* Both parts of the operation are dead. */
1414968a
RH		 3340 goto do_remove;
3341 }
03271524 3342 /* The high part of the operation is dead; generate the low. */
c45cb8bb 3343 op->opc = opc = opc_new;
efee3746
RH		 3344 op->args[1] = op->args[2];
3345 op->args[2] = op->args[3];
b83eabea 3346 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 3347 /* The low part of the operation is dead; generate the high. */
3348 op->opc = opc = opc_new2;
efee3746
RH		 3349 op->args[0] = op->args[1];
3350 op->args[1] = op->args[2];
3351 op->args[2] = op->args[3];
03271524
RH		 3352 } else {
3353 goto do_not_remove;
1414968a 3354 }
03271524
RH		 3355 /* Mark the single-word operation live. */
3356 nb_oargs = 1;
1414968a
RH		 3357 goto do_not_remove;
3358
c896fe29 3359 default:
1305c451 3360 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 3361 nb_iargs = def->nb_iargs;
3362 nb_oargs = def->nb_oargs;
c896fe29 3363
49516bc0
AJ		 3364 /* Test if the operation can be removed because all
3365 its outputs are dead. We assume that nb_oargs == 0
3366 implies side effects */
3367 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 3368 for (i = 0; i < nb_oargs; i++) {
b83eabea 3369 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 3370 goto do_not_remove;
9c43b68d 3371 }
49516bc0 3372 }
152c35aa
RH		 3373 goto do_remove;
3374 }
3375 goto do_not_remove;
49516bc0 3376
152c35aa
RH		 3377 do_remove:
3378 tcg_op_remove(s, op);
3379 break;
3380
3381 do_not_remove:
152c35aa 3382 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 3383 ts = arg_temp(op->args[i]);
3384
3385 /* Remember the preference of the uses that followed. */
31fd884b
RH		 3386 if (i < ARRAY_SIZE(op->output_pref)) {
3387 op->output_pref[i] = *la_temp_pref(ts);
3388 }
25f49c5f
RH		 3389
3390 /* Output args are dead. */
3391 if (ts->state & TS_DEAD) {
152c35aa 3392 arg_life |= DEAD_ARG << i;
49516bc0 3393 }
25f49c5f 3394 if (ts->state & TS_MEM) {
152c35aa
RH		 3395 arg_life |= SYNC_ARG << i;
3396 }
25f49c5f
RH		 3397 ts->state = TS_DEAD;
3398 la_reset_pref(ts);
152c35aa 3399 }
49516bc0 3400
25f49c5f 3401 /* If end of basic block, update. */
ae36a246
RH		 3402 if (def->flags & TCG_OPF_BB_EXIT) {
3403 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 3404 } else if (def->flags & TCG_OPF_COND_BRANCH) {
3405 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 3406 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 3407 la_bb_end(s, nb_globals, nb_temps);
152c35aa 3408 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 3409 la_global_sync(s, nb_globals);
25f49c5f
RH		 3410 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3411 la_cross_call(s, nb_temps);
3412 }
152c35aa
RH		 3413 }
3414
25f49c5f 3415 /* Record arguments that die in this opcode. */
152c35aa 3416 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 3417 ts = arg_temp(op->args[i]);
3418 if (ts->state & TS_DEAD) {
152c35aa 3419 arg_life |= DEAD_ARG << i;
c896fe29 3420 }
c896fe29 3421 }
25f49c5f
RH		 3422
3423 /* Input arguments are live for preceding opcodes. */
152c35aa 3424 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 3425 ts = arg_temp(op->args[i]);
3426 if (ts->state & TS_DEAD) {
3427 /* For operands that were dead, initially allow
3428 all regs for the type. */
3429 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
3430 ts->state &= ~TS_DEAD;
3431 }
3432 }
3433
3434 /* Incorporate constraints for this operand. */
3435 switch (opc) {
3436 case INDEX_op_mov_i32:
3437 case INDEX_op_mov_i64:
3438 /* Note that these are TCG_OPF_NOT_PRESENT and do not
3439 have proper constraints. That said, special case
3440 moves to propagate preferences backward. */
3441 if (IS_DEAD_ARG(1)) {
3442 *la_temp_pref(arg_temp(op->args[0]))
3443 = *la_temp_pref(arg_temp(op->args[1]));
3444 }
3445 break;
3446
3447 default:
3448 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3449 const TCGArgConstraint *ct = &def->args_ct[i];
3450 TCGRegSet set, *pset;
3451
3452 ts = arg_temp(op->args[i]);
3453 pset = la_temp_pref(ts);
3454 set = *pset;
3455
9be0d080 3456 set &= ct->regs;
bc2b17e6 3457 if (ct->ialias) {
31fd884b 3458 set &= output_pref(op, ct->alias_index);
25f49c5f
RH		 3459 }
3460 /* If the combination is not possible, restart. */
3461 if (set == 0) {
9be0d080 3462 set = ct->regs;
25f49c5f
RH		 3463 }
3464 *pset = set;
3465 }
3466 break;
152c35aa 3467 }
c896fe29
FB		 3468 break;
3469 }
bee158cb 3470 op->life = arg_life;
1ff0a2c5 3471 }
c896fe29 3472}
c896fe29 3473
5a18407f 3474/* Liveness analysis: Convert indirect regs to direct temporaries. */
9bbee4c0
RH		 3475static bool __attribute__((noinline))
3476liveness_pass_2(TCGContext *s)
5a18407f
RH		 3477{
3478 int nb_globals = s->nb_globals;
15fa08f8 3479 int nb_temps, i;
5a18407f 3480 bool changes = false;
15fa08f8 3481 TCGOp *op, *op_next;
5a18407f 3482
5a18407f
RH		 3483 /* Create a temporary for each indirect global. */
3484 for (i = 0; i < nb_globals; ++i) {
3485 TCGTemp *its = &s->temps[i];
3486 if (its->indirect_reg) {
3487 TCGTemp *dts = tcg_temp_alloc(s);
3488 dts->type = its->type;
3489 dts->base_type = its->base_type;
e1e64652 3490 dts->temp_subindex = its->temp_subindex;
c7482438 3491 dts->kind = TEMP_EBB;
b83eabea
RH		 3492 its->state_ptr = dts;
3493 } else {
3494 its->state_ptr = NULL;
5a18407f 3495 }
b83eabea
RH		 3496 /* All globals begin dead. */
3497 its->state = TS_DEAD;
3498 }
3499 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3500 TCGTemp *its = &s->temps[i];
3501 its->state_ptr = NULL;
3502 its->state = TS_DEAD;
5a18407f 3503 }
5a18407f 3504
15fa08f8 3505 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 3506 TCGOpcode opc = op->opc;
3507 const TCGOpDef *def = &tcg_op_defs[opc];
3508 TCGLifeData arg_life = op->life;
3509 int nb_iargs, nb_oargs, call_flags;
b83eabea 3510 TCGTemp *arg_ts, *dir_ts;
5a18407f 3511
5a18407f 3512 if (opc == INDEX_op_call) {
cd9090aa
RH		 3513 nb_oargs = TCGOP_CALLO(op);
3514 nb_iargs = TCGOP_CALLI(op);
90163900 3515 call_flags = tcg_call_flags(op);
5a18407f
RH		 3516 } else {
3517 nb_iargs = def->nb_iargs;
3518 nb_oargs = def->nb_oargs;
3519
3520 /* Set flags similar to how calls require. */
b4cb76e6
RH		 3521 if (def->flags & TCG_OPF_COND_BRANCH) {
3522 /* Like reading globals: sync_globals */
3523 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3524 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 3525 /* Like writing globals: save_globals */
3526 call_flags = 0;
3527 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3528 /* Like reading globals: sync_globals */
3529 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3530 } else {
3531 /* No effect on globals. */
3532 call_flags = (TCG_CALL_NO_READ_GLOBALS |
3533 TCG_CALL_NO_WRITE_GLOBALS);
3534 }
3535 }
3536
3537 /* Make sure that input arguments are available. */
3538 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3539 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3540 dir_ts = arg_ts->state_ptr;
3541 if (dir_ts && arg_ts->state == TS_DEAD) {
3542 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
3543 ? INDEX_op_ld_i32
3544 : INDEX_op_ld_i64);
3545 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
3546
3547 lop->args[0] = temp_arg(dir_ts);
3548 lop->args[1] = temp_arg(arg_ts->mem_base);
3549 lop->args[2] = arg_ts->mem_offset;
3550
3551 /* Loaded, but synced with memory. */
3552 arg_ts->state = TS_MEM;
5a18407f
RH		 3553 }
3554 }
3555
3556 /* Perform input replacement, and mark inputs that became dead.
3557 No action is required except keeping temp_state up to date
3558 so that we reload when needed. */
3559 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3560 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3561 dir_ts = arg_ts->state_ptr;
3562 if (dir_ts) {
3563 op->args[i] = temp_arg(dir_ts);
3564 changes = true;
3565 if (IS_DEAD_ARG(i)) {
3566 arg_ts->state = TS_DEAD;
5a18407f
RH		 3567 }
3568 }
3569 }
3570
3571 /* Liveness analysis should ensure that the following are
3572 all correct, for call sites and basic block end points. */
3573 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
3574 /* Nothing to do */
3575 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
3576 for (i = 0; i < nb_globals; ++i) {
3577 /* Liveness should see that globals are synced back,
3578 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 3579 arg_ts = &s->temps[i];
3580 tcg_debug_assert(arg_ts->state_ptr == 0
3581 || arg_ts->state != 0);
5a18407f
RH		 3582 }
3583 } else {
3584 for (i = 0; i < nb_globals; ++i) {
3585 /* Liveness should see that globals are saved back,
3586 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 3587 arg_ts = &s->temps[i];
3588 tcg_debug_assert(arg_ts->state_ptr == 0
3589 || arg_ts->state == TS_DEAD);
5a18407f
RH		 3590 }
3591 }
3592
3593 /* Outputs become available. */
61f15c48
RH		 3594 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
3595 arg_ts = arg_temp(op->args[0]);
b83eabea 3596 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 3597 if (dir_ts) {
3598 op->args[0] = temp_arg(dir_ts);
3599 changes = true;
3600
3601 /* The output is now live and modified. */
3602 arg_ts->state = 0;
3603
3604 if (NEED_SYNC_ARG(0)) {
3605 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3606 ? INDEX_op_st_i32
3607 : INDEX_op_st_i64);
d4478943 3608 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
61f15c48
RH		 3609 TCGTemp *out_ts = dir_ts;
3610
3611 if (IS_DEAD_ARG(0)) {
3612 out_ts = arg_temp(op->args[1]);
3613 arg_ts->state = TS_DEAD;
3614 tcg_op_remove(s, op);
3615 } else {
3616 arg_ts->state = TS_MEM;
3617 }
3618
3619 sop->args[0] = temp_arg(out_ts);
3620 sop->args[1] = temp_arg(arg_ts->mem_base);
3621 sop->args[2] = arg_ts->mem_offset;
3622 } else {
3623 tcg_debug_assert(!IS_DEAD_ARG(0));
3624 }
5a18407f 3625 }
61f15c48
RH		 3626 } else {
3627 for (i = 0; i < nb_oargs; i++) {
3628 arg_ts = arg_temp(op->args[i]);
3629 dir_ts = arg_ts->state_ptr;
3630 if (!dir_ts) {
3631 continue;
3632 }
3633 op->args[i] = temp_arg(dir_ts);
3634 changes = true;
5a18407f 3635
61f15c48
RH		 3636 /* The output is now live and modified. */
3637 arg_ts->state = 0;
5a18407f 3638
61f15c48
RH		 3639 /* Sync outputs upon their last write. */
3640 if (NEED_SYNC_ARG(i)) {
3641 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3642 ? INDEX_op_st_i32
3643 : INDEX_op_st_i64);
d4478943 3644 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
5a18407f 3645
61f15c48
RH		 3646 sop->args[0] = temp_arg(dir_ts);
3647 sop->args[1] = temp_arg(arg_ts->mem_base);
3648 sop->args[2] = arg_ts->mem_offset;
5a18407f 3649
61f15c48
RH		 3650 arg_ts->state = TS_MEM;
3651 }
3652 /* Drop outputs that are dead. */
3653 if (IS_DEAD_ARG(i)) {
3654 arg_ts->state = TS_DEAD;
3655 }
5a18407f
RH		 3656 }
3657 }
3658 }
3659
3660 return changes;
3661}
3662
2272e4a7 3663static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 3664{
31c96417 3665 intptr_t off;
273eb50c 3666 int size, align;
c1c09194 3667
273eb50c
RH		 3668 /* When allocating an object, look at the full type. */
3669 size = tcg_type_size(ts->base_type);
3670 switch (ts->base_type) {
c1c09194 3671 case TCG_TYPE_I32:
31c96417 3672 align = 4;
c1c09194
RH		 3673 break;
3674 case TCG_TYPE_I64:
3675 case TCG_TYPE_V64:
31c96417 3676 align = 8;
c1c09194 3677 break;
43eef72f 3678 case TCG_TYPE_I128:
c1c09194 3679 case TCG_TYPE_V128:
c1c09194 3680 case TCG_TYPE_V256:
43eef72f
RH		 3681 /*
3682 * Note that we do not require aligned storage for V256,
3683 * and that we provide alignment for I128 to match V128,
3684 * even if that's above what the host ABI requires.
3685 */
31c96417 3686 align = 16;
c1c09194
RH		 3687 break;
3688 default:
3689 g_assert_not_reached();
b591dc59 3690 }
c1c09194 3691
b9537d59
RH		 3692 /*
3693 * Assume the stack is sufficiently aligned.
3694 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3695 * and do not require 16 byte vector alignment. This seems slightly
3696 * easier than fully parameterizing the above switch statement.
3697 */
3698 align = MIN(TCG_TARGET_STACK_ALIGN, align);
c1c09194 3699 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 3700
3701 /* If we've exhausted the stack frame, restart with a smaller TB. */
3702 if (off + size > s->frame_end) {
3703 tcg_raise_tb_overflow(s);
3704 }
c1c09194 3705 s->current_frame_offset = off + size;
9defd1bd 3706#if defined(__sparc__)
273eb50c 3707 off += TCG_TARGET_STACK_BIAS;
9defd1bd 3708#endif
273eb50c
RH		 3709
3710 /* If the object was subdivided, assign memory to all the parts. */
3711 if (ts->base_type != ts->type) {
3712 int part_size = tcg_type_size(ts->type);
3713 int part_count = size / part_size;
3714
3715 /*
3716 * Each part is allocated sequentially in tcg_temp_new_internal.
3717 * Jump back to the first part by subtracting the current index.
3718 */
3719 ts -= ts->temp_subindex;
3720 for (int i = 0; i < part_count; ++i) {
3721 ts[i].mem_offset = off + i * part_size;
3722 ts[i].mem_base = s->frame_temp;
3723 ts[i].mem_allocated = 1;
3724 }
3725 } else {
3726 ts->mem_offset = off;
3727 ts->mem_base = s->frame_temp;
3728 ts->mem_allocated = 1;
3729 }
c896fe29
FB		 3730}
3731
098859f1
RH		 3732/* Assign @reg to @ts, and update reg_to_temp[]. */
3733static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
3734{
3735 if (ts->val_type == TEMP_VAL_REG) {
3736 TCGReg old = ts->reg;
3737 tcg_debug_assert(s->reg_to_temp[old] == ts);
3738 if (old == reg) {
3739 return;
3740 }
3741 s->reg_to_temp[old] = NULL;
3742 }
3743 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
3744 s->reg_to_temp[reg] = ts;
3745 ts->val_type = TEMP_VAL_REG;
3746 ts->reg = reg;
3747}
3748
3749/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
3750static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
3751{
3752 tcg_debug_assert(type != TEMP_VAL_REG);
3753 if (ts->val_type == TEMP_VAL_REG) {
3754 TCGReg reg = ts->reg;
3755 tcg_debug_assert(s->reg_to_temp[reg] == ts);
3756 s->reg_to_temp[reg] = NULL;
3757 }
3758 ts->val_type = type;
3759}
3760
b722452a 3761static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3762
59d7c14e
RH		 3763/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3764 mark it free; otherwise mark it dead. */
3765static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3766{
c0522136
RH		 3767 TCGTempVal new_type;
3768
3769 switch (ts->kind) {
3770 case TEMP_FIXED:
59d7c14e 3771 return;
c0522136 3772 case TEMP_GLOBAL:
f57c6915 3773 case TEMP_TB:
c0522136
RH		 3774 new_type = TEMP_VAL_MEM;
3775 break;
c7482438 3776 case TEMP_EBB:
c0522136
RH		 3777 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3778 break;
3779 case TEMP_CONST:
3780 new_type = TEMP_VAL_CONST;
3781 break;
3782 default:
3783 g_assert_not_reached();
59d7c14e 3784 }
098859f1 3785 set_temp_val_nonreg(s, ts, new_type);
59d7c14e 3786}
7f6ceedf 3787
59d7c14e
RH		 3788/* Mark a temporary as dead. */
3789static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3790{
3791 temp_free_or_dead(s, ts, 1);
3792}
3793
3794/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3795 registers needs to be allocated to store a constant. If 'free_or_dead'
3796 is non-zero, subsequently release the temporary; if it is positive, the
3797 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3798static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3799 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3800{
c0522136 3801 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3802 if (!ts->mem_allocated) {
2272e4a7 3803 temp_allocate_frame(s, ts);
59d7c14e 3804 }
59d7c14e
RH		 3805 switch (ts->val_type) {
3806 case TEMP_VAL_CONST:
3807 /* If we're going to free the temp immediately, then we won't
3808 require it later in a register, so attempt to store the
3809 constant to memory directly. */
3810 if (free_or_dead
3811 && tcg_out_sti(s, ts->type, ts->val,
3812 ts->mem_base->reg, ts->mem_offset)) {
3813 break;
3814 }
3815 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3816 allocated_regs, preferred_regs);
59d7c14e
RH		 3817 /* fallthrough */
3818
3819 case TEMP_VAL_REG:
3820 tcg_out_st(s, ts->type, ts->reg,
3821 ts->mem_base->reg, ts->mem_offset);
3822 break;
3823
3824 case TEMP_VAL_MEM:
3825 break;
3826
3827 case TEMP_VAL_DEAD:
3828 default:
732e89f4 3829 g_assert_not_reached();
59d7c14e
RH		 3830 }
3831 ts->mem_coherent = 1;
3832 }
3833 if (free_or_dead) {
3834 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3835 }
7f6ceedf
AJ		 3836}
3837
c896fe29 3838/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3839static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3840{
f8b2f202 3841 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3842 if (ts != NULL) {
98b4e186 3843 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3844 }
3845}
3846
b016486e
RH		 3847/**
3848 * tcg_reg_alloc:
3849 * @required_regs: Set of registers in which we must allocate.
3850 * @allocated_regs: Set of registers which must be avoided.
3851 * @preferred_regs: Set of registers we should prefer.
3852 * @rev: True if we search the registers in "indirect" order.
3853 *
3854 * The allocated register must be in @required_regs & ~@allocated_regs,
3855 * but if we can put it in @preferred_regs we may save a move later.
3856 */
3857static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3858 TCGRegSet allocated_regs,
3859 TCGRegSet preferred_regs, bool rev)
c896fe29 3860{
b016486e
RH		 3861 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3862 TCGRegSet reg_ct[2];
91478cef 3863 const int *order;
c896fe29 3864
b016486e
RH		 3865 reg_ct[1] = required_regs & ~allocated_regs;
3866 tcg_debug_assert(reg_ct[1] != 0);
3867 reg_ct[0] = reg_ct[1] & preferred_regs;
3868
3869 /* Skip the preferred_regs option if it cannot be satisfied,
3870 or if the preference made no difference. */
3871 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3872
91478cef 3873 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3874
b016486e
RH		 3875 /* Try free registers, preferences first. */
3876 for (j = f; j < 2; j++) {
3877 TCGRegSet set = reg_ct[j];
3878
3879 if (tcg_regset_single(set)) {
3880 /* One register in the set. */
3881 TCGReg reg = tcg_regset_first(set);
3882 if (s->reg_to_temp[reg] == NULL) {
3883 return reg;
3884 }
3885 } else {
3886 for (i = 0; i < n; i++) {
3887 TCGReg reg = order[i];
3888 if (s->reg_to_temp[reg] == NULL &&
3889 tcg_regset_test_reg(set, reg)) {
3890 return reg;
3891 }
3892 }
3893 }
c896fe29
FB		 3894 }
3895
b016486e
RH		 3896 /* We must spill something. */
3897 for (j = f; j < 2; j++) {
3898 TCGRegSet set = reg_ct[j];
3899
3900 if (tcg_regset_single(set)) {
3901 /* One register in the set. */
3902 TCGReg reg = tcg_regset_first(set);
b3915dbb 3903 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3904 return reg;
b016486e
RH		 3905 } else {
3906 for (i = 0; i < n; i++) {
3907 TCGReg reg = order[i];
3908 if (tcg_regset_test_reg(set, reg)) {
3909 tcg_reg_free(s, reg, allocated_regs);
3910 return reg;
3911 }
3912 }
c896fe29
FB		 3913 }
3914 }
3915
732e89f4 3916 g_assert_not_reached();
c896fe29
FB		 3917}
3918
29f5e925
RH		 3919static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
3920 TCGRegSet allocated_regs,
3921 TCGRegSet preferred_regs, bool rev)
3922{
3923 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3924 TCGRegSet reg_ct[2];
3925 const int *order;
3926
3927 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
3928 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
3929 tcg_debug_assert(reg_ct[1] != 0);
3930 reg_ct[0] = reg_ct[1] & preferred_regs;
3931
3932 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3933
3934 /*
3935 * Skip the preferred_regs option if it cannot be satisfied,
3936 * or if the preference made no difference.
3937 */
3938 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3939
3940 /*
3941 * Minimize the number of flushes by looking for 2 free registers first,
3942 * then a single flush, then two flushes.
3943 */
3944 for (fmin = 2; fmin >= 0; fmin--) {
3945 for (j = k; j < 2; j++) {
3946 TCGRegSet set = reg_ct[j];
3947
3948 for (i = 0; i < n; i++) {
3949 TCGReg reg = order[i];
3950
3951 if (tcg_regset_test_reg(set, reg)) {
3952 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
3953 if (f >= fmin) {
3954 tcg_reg_free(s, reg, allocated_regs);
3955 tcg_reg_free(s, reg + 1, allocated_regs);
3956 return reg;
3957 }
3958 }
3959 }
3960 }
3961 }
732e89f4 3962 g_assert_not_reached();
29f5e925
RH		 3963}
3964
40ae5c62
RH		 3965/* Make sure the temporary is in a register. If needed, allocate the register
3966 from DESIRED while avoiding ALLOCATED. */
3967static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3968 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3969{
3970 TCGReg reg;
3971
3972 switch (ts->val_type) {
3973 case TEMP_VAL_REG:
3974 return;
3975 case TEMP_VAL_CONST:
b016486e 3976 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3977 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3978 if (ts->type <= TCG_TYPE_I64) {
3979 tcg_out_movi(s, ts->type, reg, ts->val);
3980 } else {
4e186175
RH		 3981 uint64_t val = ts->val;
3982 MemOp vece = MO_64;
3983
3984 /*
3985 * Find the minimal vector element that matches the constant.
3986 * The targets will, in general, have to do this search anyway,
3987 * do this generically.
3988 */
4e186175
RH		 3989 if (val == dup_const(MO_8, val)) {
3990 vece = MO_8;
3991 } else if (val == dup_const(MO_16, val)) {
3992 vece = MO_16;
0b4286dd 3993 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3994 vece = MO_32;
3995 }
3996
3997 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3998 }
40ae5c62
RH		 3999 ts->mem_coherent = 0;
4000 break;
4001 case TEMP_VAL_MEM:
b016486e 4002 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 4003 preferred_regs, ts->indirect_base);
40ae5c62
RH		 4004 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
4005 ts->mem_coherent = 1;
4006 break;
4007 case TEMP_VAL_DEAD:
4008 default:
732e89f4 4009 g_assert_not_reached();
40ae5c62 4010 }
098859f1 4011 set_temp_val_reg(s, ts, reg);
40ae5c62
RH		 4012}
4013
59d7c14e
RH		 4014/* Save a temporary to memory. 'allocated_regs' is used in case a
4015 temporary registers needs to be allocated to store a constant. */
4016static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 4017{
5a18407f
RH		 4018 /* The liveness analysis already ensures that globals are back
4019 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 4020 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 4021}
4022
9814dd27 4023/* save globals to their canonical location and assume they can be
e8996ee0
FB		 4024 modified be the following code. 'allocated_regs' is used in case a
4025 temporary registers needs to be allocated to store a constant. */
4026static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 4027{
ac3b8891 4028 int i, n;
c896fe29 4029
ac3b8891 4030 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 4031 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 4032 }
e5097dc8
FB		 4033}
4034
3d5c5f87
AJ		 4035/* sync globals to their canonical location and assume they can be
4036 read by the following code. 'allocated_regs' is used in case a
4037 temporary registers needs to be allocated to store a constant. */
4038static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
4039{
ac3b8891 4040 int i, n;
3d5c5f87 4041
ac3b8891 4042 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 4043 TCGTemp *ts = &s->temps[i];
5a18407f 4044 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 4045 || ts->kind == TEMP_FIXED
5a18407f 4046 || ts->mem_coherent);
3d5c5f87
AJ		 4047 }
4048}
4049
e5097dc8 4050/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 4051 all globals are stored at their canonical location. */
4052static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 4053{
e5097dc8
FB		 4054 int i;
4055
b13eb728
RH		 4056 for (i = s->nb_globals; i < s->nb_temps; i++) {
4057 TCGTemp *ts = &s->temps[i];
c0522136
RH		 4058
4059 switch (ts->kind) {
f57c6915 4060 case TEMP_TB:
b13eb728 4061 temp_save(s, ts, allocated_regs);
c0522136 4062 break;
c7482438 4063 case TEMP_EBB:
5a18407f
RH		 4064 /* The liveness analysis already ensures that temps are dead.
4065 Keep an tcg_debug_assert for safety. */
4066 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 4067 break;
4068 case TEMP_CONST:
4069 /* Similarly, we should have freed any allocated register. */
4070 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
4071 break;
4072 default:
4073 g_assert_not_reached();
c896fe29
FB		 4074 }
4075 }
e8996ee0
FB		 4076
4077 save_globals(s, allocated_regs);
c896fe29
FB		 4078}
4079
b4cb76e6 4080/*
c7482438
RH		 4081 * At a conditional branch, we assume all temporaries are dead unless
4082 * explicitly live-across-conditional-branch; all globals and local
4083 * temps are synced to their location.
b4cb76e6
RH		 4084 */
4085static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
4086{
4087 sync_globals(s, allocated_regs);
4088
4089 for (int i = s->nb_globals; i < s->nb_temps; i++) {
4090 TCGTemp *ts = &s->temps[i];
4091 /*
4092 * The liveness analysis already ensures that temps are dead.
4093 * Keep tcg_debug_asserts for safety.
4094 */
c0522136 4095 switch (ts->kind) {
f57c6915 4096 case TEMP_TB:
b4cb76e6 4097 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136 4098 break;
c7482438 4099 case TEMP_EBB:
c0522136
RH		 4100 case TEMP_CONST:
4101 break;
4102 default:
4103 g_assert_not_reached();
b4cb76e6
RH		 4104 }
4105 }
4106}
4107
bab1671f 4108/*
c58f4c97 4109 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 4110 */
0fe4fca4 4111static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 4112 tcg_target_ulong val, TCGLifeData arg_life,
4113 TCGRegSet preferred_regs)
e8996ee0 4114{
d63e3b6e 4115 /* ENV should not be modified. */
e01fa97d 4116 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 4117
4118 /* The movi is not explicitly generated here. */
098859f1 4119 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
59d7c14e
RH		 4120 ots->val = val;
4121 ots->mem_coherent = 0;
4122 if (NEED_SYNC_ARG(0)) {
ba87719c 4123 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 4124 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 4125 temp_dead(s, ots);
4c4e1ab2 4126 }
e8996ee0
FB		 4127}
4128
bab1671f
RH		 4129/*
4130 * Specialized code generation for INDEX_op_mov_*.
4131 */
dd186292 4132static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 4133{
dd186292 4134 const TCGLifeData arg_life = op->life;
69e3706d 4135 TCGRegSet allocated_regs, preferred_regs;
c896fe29 4136 TCGTemp *ts, *ots;
450445d5 4137 TCGType otype, itype;
098859f1 4138 TCGReg oreg, ireg;
c896fe29 4139
d21369f5 4140 allocated_regs = s->reserved_regs;
31fd884b 4141 preferred_regs = output_pref(op, 0);
43439139
RH		 4142 ots = arg_temp(op->args[0]);
4143 ts = arg_temp(op->args[1]);
450445d5 4144
d63e3b6e 4145 /* ENV should not be modified. */
e01fa97d 4146 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 4147
450445d5
RH		 4148 /* Note that otype != itype for no-op truncation. */
4149 otype = ots->type;
4150 itype = ts->type;
c29c1d7e 4151
0fe4fca4
PB		 4152 if (ts->val_type == TEMP_VAL_CONST) {
4153 /* propagate constant or generate sti */
4154 tcg_target_ulong val = ts->val;
4155 if (IS_DEAD_ARG(1)) {
4156 temp_dead(s, ts);
4157 }
69e3706d 4158 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 4159 return;
4160 }
4161
4162 /* If the source value is in memory we're going to be forced
4163 to have it in a register in order to perform the copy. Copy
4164 the SOURCE value into its own register first, that way we
4165 don't have to reload SOURCE the next time it is used. */
4166 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 4167 temp_load(s, ts, tcg_target_available_regs[itype],
4168 allocated_regs, preferred_regs);
c29c1d7e 4169 }
0fe4fca4 4170 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
098859f1
RH		 4171 ireg = ts->reg;
4172
d63e3b6e 4173 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 4174 /* mov to a non-saved dead register makes no sense (even with
4175 liveness analysis disabled). */
eabb7b91 4176 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 4177 if (!ots->mem_allocated) {
2272e4a7 4178 temp_allocate_frame(s, ots);
c29c1d7e 4179 }
098859f1 4180 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 4181 if (IS_DEAD_ARG(1)) {
f8bf00f1 4182 temp_dead(s, ts);
c29c1d7e 4183 }
f8bf00f1 4184 temp_dead(s, ots);
098859f1
RH		 4185 return;
4186 }
4187
4188 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
4189 /*
4190 * The mov can be suppressed. Kill input first, so that it
4191 * is unlinked from reg_to_temp, then set the output to the
4192 * reg that we saved from the input.
4193 */
4194 temp_dead(s, ts);
4195 oreg = ireg;
c29c1d7e 4196 } else {
098859f1
RH		 4197 if (ots->val_type == TEMP_VAL_REG) {
4198 oreg = ots->reg;
c896fe29 4199 } else {
098859f1
RH		 4200 /* Make sure to not spill the input register during allocation. */
4201 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
4202 allocated_regs | ((TCGRegSet)1 << ireg),
4203 preferred_regs, ots->indirect_base);
c896fe29 4204 }
098859f1
RH		 4205 if (!tcg_out_mov(s, otype, oreg, ireg)) {
4206 /*
4207 * Cross register class move not supported.
4208 * Store the source register into the destination slot
4209 * and leave the destination temp as TEMP_VAL_MEM.
4210 */
4211 assert(!temp_readonly(ots));
4212 if (!ts->mem_allocated) {
4213 temp_allocate_frame(s, ots);
4214 }
4215 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
4216 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
4217 ots->mem_coherent = 1;
4218 return;
c896fe29 4219 }
ec7a869d 4220 }
098859f1
RH		 4221 set_temp_val_reg(s, ots, oreg);
4222 ots->mem_coherent = 0;
4223
4224 if (NEED_SYNC_ARG(0)) {
4225 temp_sync(s, ots, allocated_regs, 0, 0);
4226 }
c896fe29
FB		 4227}
4228
bab1671f
RH		 4229/*
4230 * Specialized code generation for INDEX_op_dup_vec.
4231 */
4232static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
4233{
4234 const TCGLifeData arg_life = op->life;
4235 TCGRegSet dup_out_regs, dup_in_regs;
4236 TCGTemp *its, *ots;
4237 TCGType itype, vtype;
4238 unsigned vece;
31c96417 4239 int lowpart_ofs;
bab1671f
RH		 4240 bool ok;
4241
4242 ots = arg_temp(op->args[0]);
4243 its = arg_temp(op->args[1]);
4244
4245 /* ENV should not be modified. */
e01fa97d 4246 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 4247
4248 itype = its->type;
4249 vece = TCGOP_VECE(op);
4250 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4251
4252 if (its->val_type == TEMP_VAL_CONST) {
4253 /* Propagate constant via movi -> dupi. */
4254 tcg_target_ulong val = its->val;
4255 if (IS_DEAD_ARG(1)) {
4256 temp_dead(s, its);
4257 }
31fd884b 4258 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
bab1671f
RH		 4259 return;
4260 }
4261
9be0d080
RH		 4262 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
4263 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 4264
4265 /* Allocate the output register now. */
4266 if (ots->val_type != TEMP_VAL_REG) {
4267 TCGRegSet allocated_regs = s->reserved_regs;
098859f1 4268 TCGReg oreg;
bab1671f
RH		 4269
4270 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
4271 /* Make sure to not spill the input register. */
4272 tcg_regset_set_reg(allocated_regs, its->reg);
4273 }
098859f1 4274 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 4275 output_pref(op, 0), ots->indirect_base);
098859f1 4276 set_temp_val_reg(s, ots, oreg);
bab1671f
RH		 4277 }
4278
4279 switch (its->val_type) {
4280 case TEMP_VAL_REG:
4281 /*
4282 * The dup constriaints must be broad, covering all possible VECE.
4283 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
4284 * to fail, indicating that extra moves are required for that case.
4285 */
4286 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
4287 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
4288 goto done;
4289 }
4290 /* Try again from memory or a vector input register. */
4291 }
4292 if (!its->mem_coherent) {
4293 /*
4294 * The input register is not synced, and so an extra store
4295 * would be required to use memory. Attempt an integer-vector
4296 * register move first. We do not have a TCGRegSet for this.
4297 */
4298 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
4299 break;
4300 }
4301 /* Sync the temp back to its slot and load from there. */
4302 temp_sync(s, its, s->reserved_regs, 0, 0);
4303 }
4304 /* fall through */
4305
4306 case TEMP_VAL_MEM:
31c96417
RH		 4307 lowpart_ofs = 0;
4308 if (HOST_BIG_ENDIAN) {
4309 lowpart_ofs = tcg_type_size(itype) - (1 << vece);
4310 }
d6ecb4a9 4311 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
31c96417 4312 its->mem_offset + lowpart_ofs)) {
d6ecb4a9
RH		 4313 goto done;
4314 }
098859f1 4315 /* Load the input into the destination vector register. */
bab1671f
RH		 4316 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
4317 break;
4318
4319 default:
4320 g_assert_not_reached();
4321 }
4322
4323 /* We now have a vector input register, so dup must succeed. */
4324 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
4325 tcg_debug_assert(ok);
4326
4327 done:
36f5539c 4328 ots->mem_coherent = 0;
bab1671f
RH		 4329 if (IS_DEAD_ARG(1)) {
4330 temp_dead(s, its);
4331 }
4332 if (NEED_SYNC_ARG(0)) {
4333 temp_sync(s, ots, s->reserved_regs, 0, 0);
4334 }
4335 if (IS_DEAD_ARG(0)) {
4336 temp_dead(s, ots);
4337 }
4338}
4339
dd186292 4340static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 4341{
dd186292
RH		 4342 const TCGLifeData arg_life = op->life;
4343 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 4344 TCGRegSet i_allocated_regs;
4345 TCGRegSet o_allocated_regs;
b6638662
RH		 4346 int i, k, nb_iargs, nb_oargs;
4347 TCGReg reg;
c896fe29
FB		 4348 TCGArg arg;
4349 const TCGArgConstraint *arg_ct;
4350 TCGTemp *ts;
4351 TCGArg new_args[TCG_MAX_OP_ARGS];
4352 int const_args[TCG_MAX_OP_ARGS];
4353
4354 nb_oargs = def->nb_oargs;
4355 nb_iargs = def->nb_iargs;
4356
4357 /* copy constants */
a813e36f 4358 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 4359 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 4360 sizeof(TCGArg) * def->nb_cargs);
4361
d21369f5
RH		 4362 i_allocated_regs = s->reserved_regs;
4363 o_allocated_regs = s->reserved_regs;
82790a87 4364
a813e36f 4365 /* satisfy input constraints */
dd186292 4366 for (k = 0; k < nb_iargs; k++) {
29f5e925
RH		 4367 TCGRegSet i_preferred_regs, i_required_regs;
4368 bool allocate_new_reg, copyto_new_reg;
4369 TCGTemp *ts2;
4370 int i1, i2;
d62816f2 4371
66792f90 4372 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 4373 arg = op->args[i];
c896fe29 4374 arg_ct = &def->args_ct[i];
43439139 4375 ts = arg_temp(arg);
40ae5c62
RH		 4376
4377 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 4378 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 4379 /* constant is OK for instruction */
4380 const_args[i] = 1;
4381 new_args[i] = ts->val;
d62816f2 4382 continue;
c896fe29 4383 }
40ae5c62 4384
1c1824dc
RH		 4385 reg = ts->reg;
4386 i_preferred_regs = 0;
29f5e925 4387 i_required_regs = arg_ct->regs;
1c1824dc 4388 allocate_new_reg = false;
29f5e925
RH		 4389 copyto_new_reg = false;
4390
4391 switch (arg_ct->pair) {
4392 case 0: /* not paired */
4393 if (arg_ct->ialias) {
31fd884b 4394 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 4395
4396 /*
4397 * If the input is readonly, then it cannot also be an
4398 * output and aliased to itself. If the input is not
4399 * dead after the instruction, we must allocate a new
4400 * register and move it.
4401 */
4402 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
4403 allocate_new_reg = true;
4404 } else if (ts->val_type == TEMP_VAL_REG) {
4405 /*
4406 * Check if the current register has already been
4407 * allocated for another input.
4408 */
4409 allocate_new_reg =
4410 tcg_regset_test_reg(i_allocated_regs, reg);
4411 }
4412 }
4413 if (!allocate_new_reg) {
4414 temp_load(s, ts, i_required_regs, i_allocated_regs,
4415 i_preferred_regs);
4416 reg = ts->reg;
4417 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
4418 }
4419 if (allocate_new_reg) {
4420 /*
4421 * Allocate a new register matching the constraint
4422 * and move the temporary register into it.
4423 */
4424 temp_load(s, ts, tcg_target_available_regs[ts->type],
4425 i_allocated_regs, 0);
4426 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
4427 i_preferred_regs, ts->indirect_base);
4428 copyto_new_reg = true;
4429 }
4430 break;
4431
4432 case 1:
4433 /* First of an input pair; if i1 == i2, the second is an output. */
4434 i1 = i;
4435 i2 = arg_ct->pair_index;
4436 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
4437
4438 /*
4439 * It is easier to default to allocating a new pair
4440 * and to identify a few cases where it's not required.
4441 */
4442 if (arg_ct->ialias) {
31fd884b 4443 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 4444 if (IS_DEAD_ARG(i1) &&
4445 IS_DEAD_ARG(i2) &&
4446 !temp_readonly(ts) &&
4447 ts->val_type == TEMP_VAL_REG &&
4448 ts->reg < TCG_TARGET_NB_REGS - 1 &&
4449 tcg_regset_test_reg(i_required_regs, reg) &&
4450 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4451 !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
4452 (ts2
4453 ? ts2->val_type == TEMP_VAL_REG &&
4454 ts2->reg == reg + 1 &&
4455 !temp_readonly(ts2)
4456 : s->reg_to_temp[reg + 1] == NULL)) {
4457 break;
4458 }
4459 } else {
4460 /* Without aliasing, the pair must also be an input. */
4461 tcg_debug_assert(ts2);
4462 if (ts->val_type == TEMP_VAL_REG &&
4463 ts2->val_type == TEMP_VAL_REG &&
4464 ts2->reg == reg + 1 &&
4465 tcg_regset_test_reg(i_required_regs, reg)) {
4466 break;
4467 }
4468 }
4469 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
4470 0, ts->indirect_base);
4471 goto do_pair;
4472
4473 case 2: /* pair second */
4474 reg = new_args[arg_ct->pair_index] + 1;
4475 goto do_pair;
1c1824dc 4476
29f5e925
RH		 4477 case 3: /* ialias with second output, no first input */
4478 tcg_debug_assert(arg_ct->ialias);
31fd884b 4479 i_preferred_regs = output_pref(op, arg_ct->alias_index);
d62816f2 4480
29f5e925
RH		 4481 if (IS_DEAD_ARG(i) &&
4482 !temp_readonly(ts) &&
4483 ts->val_type == TEMP_VAL_REG &&
4484 reg > 0 &&
4485 s->reg_to_temp[reg - 1] == NULL &&
4486 tcg_regset_test_reg(i_required_regs, reg) &&
4487 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4488 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
4489 tcg_regset_set_reg(i_allocated_regs, reg - 1);
4490 break;
4491 }
4492 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
4493 i_allocated_regs, 0,
4494 ts->indirect_base);
4495 tcg_regset_set_reg(i_allocated_regs, reg);
4496 reg += 1;
4497 goto do_pair;
4498
4499 do_pair:
c0522136 4500 /*
29f5e925
RH		 4501 * If an aliased input is not dead after the instruction,
4502 * we must allocate a new register and move it.
c0522136 4503 */
29f5e925
RH		 4504 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
4505 TCGRegSet t_allocated_regs = i_allocated_regs;
4506
1c1824dc 4507 /*
29f5e925
RH		 4508 * Because of the alias, and the continued life, make sure
4509 * that the temp is somewhere *other* than the reg pair,
4510 * and we get a copy in reg.
1c1824dc 4511 */
29f5e925
RH		 4512 tcg_regset_set_reg(t_allocated_regs, reg);
4513 tcg_regset_set_reg(t_allocated_regs, reg + 1);
4514 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
4515 /* If ts was already in reg, copy it somewhere else. */
4516 TCGReg nr;
4517 bool ok;
4518
4519 tcg_debug_assert(ts->kind != TEMP_FIXED);
4520 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
4521 t_allocated_regs, 0, ts->indirect_base);
4522 ok = tcg_out_mov(s, ts->type, nr, reg);
4523 tcg_debug_assert(ok);
4524
4525 set_temp_val_reg(s, ts, nr);
4526 } else {
4527 temp_load(s, ts, tcg_target_available_regs[ts->type],
4528 t_allocated_regs, 0);
4529 copyto_new_reg = true;
4530 }
4531 } else {
4532 /* Preferably allocate to reg, otherwise copy. */
4533 i_required_regs = (TCGRegSet)1 << reg;
4534 temp_load(s, ts, i_required_regs, i_allocated_regs,
4535 i_preferred_regs);
4536 copyto_new_reg = ts->reg != reg;
5ff9d6a4 4537 }
29f5e925 4538 break;
d62816f2 4539
29f5e925
RH		 4540 default:
4541 g_assert_not_reached();
1c1824dc 4542 }
d62816f2 4543
29f5e925 4544 if (copyto_new_reg) {
78113e83 4545 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 4546 /*
4547 * Cross register class move not supported. Sync the
4548 * temp back to its slot and load from there.
4549 */
4550 temp_sync(s, ts, i_allocated_regs, 0, 0);
4551 tcg_out_ld(s, ts->type, reg,
4552 ts->mem_base->reg, ts->mem_offset);
78113e83 4553 }
c896fe29 4554 }
c896fe29
FB		 4555 new_args[i] = reg;
4556 const_args[i] = 0;
82790a87 4557 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29 4558 }
a813e36f 4559
a52ad07e
AJ		 4560 /* mark dead temporaries and free the associated registers */
4561 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4562 if (IS_DEAD_ARG(i)) {
43439139 4563 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 4564 }
4565 }
4566
b4cb76e6
RH		 4567 if (def->flags & TCG_OPF_COND_BRANCH) {
4568 tcg_reg_alloc_cbranch(s, i_allocated_regs);
4569 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 4570 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 4571 } else {
e8996ee0 4572 if (def->flags & TCG_OPF_CALL_CLOBBER) {
a813e36f 4573 /* XXX: permit generic clobber register list ? */
c8074023
RH		 4574 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4575 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 4576 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 4577 }
c896fe29 4578 }
3d5c5f87
AJ		 4579 }
4580 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4581 /* sync globals if the op has side effects and might trigger
4582 an exception. */
82790a87 4583 sync_globals(s, i_allocated_regs);
c896fe29 4584 }
a813e36f 4585
e8996ee0 4586 /* satisfy the output constraints */
e8996ee0 4587 for(k = 0; k < nb_oargs; k++) {
66792f90 4588 i = def->args_ct[k].sort_index;
dd186292 4589 arg = op->args[i];
e8996ee0 4590 arg_ct = &def->args_ct[i];
43439139 4591 ts = arg_temp(arg);
d63e3b6e
RH		 4592
4593 /* ENV should not be modified. */
e01fa97d 4594 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4595
29f5e925
RH		 4596 switch (arg_ct->pair) {
4597 case 0: /* not paired */
4598 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
4599 reg = new_args[arg_ct->alias_index];
4600 } else if (arg_ct->newreg) {
4601 reg = tcg_reg_alloc(s, arg_ct->regs,
4602 i_allocated_regs | o_allocated_regs,
31fd884b 4603 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4604 } else {
4605 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
31fd884b 4606 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4607 }
4608 break;
4609
4610 case 1: /* first of pair */
4611 tcg_debug_assert(!arg_ct->newreg);
4612 if (arg_ct->oalias) {
4613 reg = new_args[arg_ct->alias_index];
4614 break;
4615 }
4616 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
31fd884b 4617 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4618 break;
4619
4620 case 2: /* second of pair */
4621 tcg_debug_assert(!arg_ct->newreg);
4622 if (arg_ct->oalias) {
4623 reg = new_args[arg_ct->alias_index];
4624 } else {
4625 reg = new_args[arg_ct->pair_index] + 1;
4626 }
4627 break;
4628
4629 case 3: /* first of pair, aliasing with a second input */
4630 tcg_debug_assert(!arg_ct->newreg);
4631 reg = new_args[arg_ct->pair_index] - 1;
4632 break;
4633
4634 default:
4635 g_assert_not_reached();
c896fe29 4636 }
82790a87 4637 tcg_regset_set_reg(o_allocated_regs, reg);
098859f1 4638 set_temp_val_reg(s, ts, reg);
d63e3b6e 4639 ts->mem_coherent = 0;
e8996ee0 4640 new_args[i] = reg;
c896fe29 4641 }
c896fe29
FB		 4642 }
4643
c896fe29 4644 /* emit instruction */
678155b2
RH		 4645 switch (op->opc) {
4646 case INDEX_op_ext8s_i32:
4647 tcg_out_ext8s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
4648 break;
4649 case INDEX_op_ext8s_i64:
4650 tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
4651 break;
d0e66c89
RH		 4652 case INDEX_op_ext8u_i32:
4653 case INDEX_op_ext8u_i64:
4654 tcg_out_ext8u(s, new_args[0], new_args[1]);
4655 break;
753e42ea
RH		 4656 case INDEX_op_ext16s_i32:
4657 tcg_out_ext16s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
4658 break;
4659 case INDEX_op_ext16s_i64:
4660 tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
4661 break;
379afdff
RH		 4662 case INDEX_op_ext16u_i32:
4663 case INDEX_op_ext16u_i64:
4664 tcg_out_ext16u(s, new_args[0], new_args[1]);
4665 break;
52bf3398
RH		 4666 case INDEX_op_ext32s_i64:
4667 tcg_out_ext32s(s, new_args[0], new_args[1]);
4668 break;
9ecf5f61
RH		 4669 case INDEX_op_ext32u_i64:
4670 tcg_out_ext32u(s, new_args[0], new_args[1]);
4671 break;
9c6aa274
RH		 4672 case INDEX_op_ext_i32_i64:
4673 tcg_out_exts_i32_i64(s, new_args[0], new_args[1]);
4674 break;
b9bfe000
RH		 4675 case INDEX_op_extu_i32_i64:
4676 tcg_out_extu_i32_i64(s, new_args[0], new_args[1]);
4677 break;
b8b94ac6
RH		 4678 case INDEX_op_extrl_i64_i32:
4679 tcg_out_extrl_i64_i32(s, new_args[0], new_args[1]);
4680 break;
678155b2
RH		 4681 default:
4682 if (def->flags & TCG_OPF_VECTOR) {
4683 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
4684 new_args, const_args);
4685 } else {
4686 tcg_out_op(s, op->opc, new_args, const_args);
4687 }
4688 break;
d2fd745f
RH		 4689 }
4690
c896fe29
FB		 4691 /* move the outputs in the correct register if needed */
4692 for(i = 0; i < nb_oargs; i++) {
43439139 4693 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 4694
4695 /* ENV should not be modified. */
e01fa97d 4696 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4697
ec7a869d 4698 if (NEED_SYNC_ARG(i)) {
98b4e186 4699 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 4700 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 4701 temp_dead(s, ts);
ec7a869d 4702 }
c896fe29
FB		 4703 }
4704}
4705
efe86b21
RH		 4706static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
4707{
4708 const TCGLifeData arg_life = op->life;
4709 TCGTemp *ots, *itsl, *itsh;
4710 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4711
4712 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
4713 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
4714 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
4715
4716 ots = arg_temp(op->args[0]);
4717 itsl = arg_temp(op->args[1]);
4718 itsh = arg_temp(op->args[2]);
4719
4720 /* ENV should not be modified. */
4721 tcg_debug_assert(!temp_readonly(ots));
4722
4723 /* Allocate the output register now. */
4724 if (ots->val_type != TEMP_VAL_REG) {
4725 TCGRegSet allocated_regs = s->reserved_regs;
4726 TCGRegSet dup_out_regs =
4727 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
098859f1 4728 TCGReg oreg;
efe86b21
RH		 4729
4730 /* Make sure to not spill the input registers. */
4731 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
4732 tcg_regset_set_reg(allocated_regs, itsl->reg);
4733 }
4734 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
4735 tcg_regset_set_reg(allocated_regs, itsh->reg);
4736 }
4737
098859f1 4738 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 4739 output_pref(op, 0), ots->indirect_base);
098859f1 4740 set_temp_val_reg(s, ots, oreg);
efe86b21
RH		 4741 }
4742
4743 /* Promote dup2 of immediates to dupi_vec. */
4744 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
4745 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
4746 MemOp vece = MO_64;
4747
4748 if (val == dup_const(MO_8, val)) {
4749 vece = MO_8;
4750 } else if (val == dup_const(MO_16, val)) {
4751 vece = MO_16;
4752 } else if (val == dup_const(MO_32, val)) {
4753 vece = MO_32;
4754 }
4755
4756 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
4757 goto done;
4758 }
4759
4760 /* If the two inputs form one 64-bit value, try dupm_vec. */
aef85402
RH		 4761 if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
4762 itsh->temp_subindex == !HOST_BIG_ENDIAN &&
4763 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
4764 TCGTemp *its = itsl - HOST_BIG_ENDIAN;
4765
4766 temp_sync(s, its + 0, s->reserved_regs, 0, 0);
4767 temp_sync(s, its + 1, s->reserved_regs, 0, 0);
4768
efe86b21
RH		 4769 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
4770 its->mem_base->reg, its->mem_offset)) {
4771 goto done;
4772 }
4773 }
4774
4775 /* Fall back to generic expansion. */
4776 return false;
4777
4778 done:
36f5539c 4779 ots->mem_coherent = 0;
efe86b21
RH		 4780 if (IS_DEAD_ARG(1)) {
4781 temp_dead(s, itsl);
4782 }
4783 if (IS_DEAD_ARG(2)) {
4784 temp_dead(s, itsh);
4785 }
4786 if (NEED_SYNC_ARG(0)) {
4787 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
4788 } else if (IS_DEAD_ARG(0)) {
4789 temp_dead(s, ots);
4790 }
4791 return true;
4792}
4793
39004a71
RH		 4794static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
4795 TCGRegSet allocated_regs)
c896fe29 4796{
39004a71
RH		 4797 if (ts->val_type == TEMP_VAL_REG) {
4798 if (ts->reg != reg) {
4799 tcg_reg_free(s, reg, allocated_regs);
4800 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4801 /*
4802 * Cross register class move not supported. Sync the
4803 * temp back to its slot and load from there.
4804 */
4805 temp_sync(s, ts, allocated_regs, 0, 0);
4806 tcg_out_ld(s, ts->type, reg,
4807 ts->mem_base->reg, ts->mem_offset);
4808 }
4809 }
4810 } else {
4811 TCGRegSet arg_set = 0;
c896fe29 4812
39004a71
RH		 4813 tcg_reg_free(s, reg, allocated_regs);
4814 tcg_regset_set_reg(arg_set, reg);
4815 temp_load(s, ts, arg_set, allocated_regs, 0);
b03cce8e 4816 }
39004a71 4817}
39cf05d3 4818
39004a71
RH		 4819static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts,
4820 TCGRegSet allocated_regs)
4821{
4822 /*
4823 * When the destination is on the stack, load up the temp and store.
4824 * If there are many call-saved registers, the temp might live to
4825 * see another use; otherwise it'll be discarded.
4826 */
4827 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
4828 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
4829 TCG_TARGET_CALL_STACK_OFFSET +
4830 stk_slot * sizeof(tcg_target_long));
4831}
a813e36f 4832
39004a71
RH		 4833static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
4834 TCGTemp *ts, TCGRegSet *allocated_regs)
4835{
4836 if (REG_P(l)) {
4837 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
4838 load_arg_reg(s, reg, ts, *allocated_regs);
4839 tcg_regset_set_reg(*allocated_regs, reg);
4840 } else {
4841 load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs),
4842 ts, *allocated_regs);
4843 }
4844}
40ae5c62 4845
313bdea8
RH		 4846static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base,
4847 intptr_t ref_off, TCGRegSet *allocated_regs)
4848{
4849 TCGReg reg;
4850 int stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
4851
4852 if (stk_slot < 0) {
4853 reg = tcg_target_call_iarg_regs[arg_slot];
4854 tcg_reg_free(s, reg, *allocated_regs);
4855 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
4856 tcg_regset_set_reg(*allocated_regs, reg);
4857 } else {
4858 reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR],
4859 *allocated_regs, 0, false);
4860 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
4861 tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK,
4862 TCG_TARGET_CALL_STACK_OFFSET
4863 + stk_slot * sizeof(tcg_target_long));
4864 }
4865}
4866
39004a71
RH		 4867static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
4868{
4869 const int nb_oargs = TCGOP_CALLO(op);
4870 const int nb_iargs = TCGOP_CALLI(op);
4871 const TCGLifeData arg_life = op->life;
4872 const TCGHelperInfo *info = tcg_call_info(op);
4873 TCGRegSet allocated_regs = s->reserved_regs;
4874 int i;
40ae5c62 4875
39004a71
RH		 4876 /*
4877 * Move inputs into place in reverse order,
4878 * so that we place stacked arguments first.
4879 */
4880 for (i = nb_iargs - 1; i >= 0; --i) {
4881 const TCGCallArgumentLoc *loc = &info->in[i];
4882 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
40ae5c62 4883
39004a71
RH		 4884 switch (loc->kind) {
4885 case TCG_CALL_ARG_NORMAL:
4886 case TCG_CALL_ARG_EXTEND_U:
4887 case TCG_CALL_ARG_EXTEND_S:
4888 load_arg_normal(s, loc, ts, &allocated_regs);
4889 break;
313bdea8
RH		 4890 case TCG_CALL_ARG_BY_REF:
4891 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
4892 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK,
4893 TCG_TARGET_CALL_STACK_OFFSET
4894 + loc->ref_slot * sizeof(tcg_target_long),
4895 &allocated_regs);
4896 break;
4897 case TCG_CALL_ARG_BY_REF_N:
4898 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
4899 break;
39004a71
RH		 4900 default:
4901 g_assert_not_reached();
c896fe29 4902 }
c896fe29 4903 }
a813e36f 4904
39004a71 4905 /* Mark dead temporaries and free the associated registers. */
dd186292 4906 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 4907 if (IS_DEAD_ARG(i)) {
43439139 4908 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 4909 }
4910 }
a813e36f 4911
39004a71 4912 /* Clobber call registers. */
c8074023
RH		 4913 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4914 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 4915 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 4916 }
4917 }
78505279 4918
39004a71
RH		 4919 /*
4920 * Save globals if they might be written by the helper,
4921 * sync them if they might be read.
4922 */
4923 if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
78505279 4924 /* Nothing to do */
39004a71 4925 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
78505279
AJ		 4926 sync_globals(s, allocated_regs);
4927 } else {
b9c18f56
AJ		 4928 save_globals(s, allocated_regs);
4929 }
c896fe29 4930
313bdea8
RH		 4931 /*
4932 * If the ABI passes a pointer to the returned struct as the first
4933 * argument, load that now. Pass a pointer to the output home slot.
4934 */
4935 if (info->out_kind == TCG_CALL_RET_BY_REF) {
4936 TCGTemp *ts = arg_temp(op->args[0]);
4937
4938 if (!ts->mem_allocated) {
4939 temp_allocate_frame(s, ts);
4940 }
4941 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs);
4942 }
4943
cee44b03 4944 tcg_out_call(s, tcg_call_func(op), info);
c896fe29 4945
39004a71
RH		 4946 /* Assign output registers and emit moves if needed. */
4947 switch (info->out_kind) {
4948 case TCG_CALL_RET_NORMAL:
4949 for (i = 0; i < nb_oargs; i++) {
4950 TCGTemp *ts = arg_temp(op->args[i]);
5e3d0c19 4951 TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i);
d63e3b6e 4952
39004a71
RH		 4953 /* ENV should not be modified. */
4954 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4955
39004a71
RH		 4956 set_temp_val_reg(s, ts, reg);
4957 ts->mem_coherent = 0;
4958 }
4959 break;
313bdea8 4960
c6556aa0
RH		 4961 case TCG_CALL_RET_BY_VEC:
4962 {
4963 TCGTemp *ts = arg_temp(op->args[0]);
4964
4965 tcg_debug_assert(ts->base_type == TCG_TYPE_I128);
4966 tcg_debug_assert(ts->temp_subindex == 0);
4967 if (!ts->mem_allocated) {
4968 temp_allocate_frame(s, ts);
4969 }
4970 tcg_out_st(s, TCG_TYPE_V128,
4971 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
4972 ts->mem_base->reg, ts->mem_offset);
4973 }
4974 /* fall through to mark all parts in memory */
4975
313bdea8
RH		 4976 case TCG_CALL_RET_BY_REF:
4977 /* The callee has performed a write through the reference. */
4978 for (i = 0; i < nb_oargs; i++) {
4979 TCGTemp *ts = arg_temp(op->args[i]);
4980 ts->val_type = TEMP_VAL_MEM;
4981 }
4982 break;
4983
39004a71
RH		 4984 default:
4985 g_assert_not_reached();
4986 }
4987
4988 /* Flush or discard output registers as needed. */
4989 for (i = 0; i < nb_oargs; i++) {
4990 TCGTemp *ts = arg_temp(op->args[i]);
d63e3b6e 4991 if (NEED_SYNC_ARG(i)) {
39004a71 4992 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
d63e3b6e
RH		 4993 } else if (IS_DEAD_ARG(i)) {
4994 temp_dead(s, ts);
c896fe29
FB		 4995 }
4996 }
c896fe29
FB		 4997}
4998
4999#ifdef CONFIG_PROFILER
5000
c3fac113
EC		 5001/* avoid copy/paste errors */
5002#define PROF_ADD(to, from, field) \
5003 do { \
d73415a3 5004 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 5005 } while (0)
5006
5007#define PROF_MAX(to, from, field) \
5008 do { \
d73415a3 5009 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 5010 if (val__ > (to)->field) { \
5011 (to)->field = val__; \
5012 } \
5013 } while (0)
5014
5015/* Pass in a zero'ed @prof */
5016static inline
5017void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
5018{
0e2d61cf 5019 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 5020 unsigned int i;
5021
3468b59e 5022 for (i = 0; i < n_ctxs; i++) {
d73415a3 5023 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 5024 const TCGProfile *orig = &s->prof;
c3fac113
EC		 5025
5026 if (counters) {
72fd2efb 5027 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 5028 PROF_ADD(prof, orig, tb_count1);
5029 PROF_ADD(prof, orig, tb_count);
5030 PROF_ADD(prof, orig, op_count);
5031 PROF_MAX(prof, orig, op_count_max);
5032 PROF_ADD(prof, orig, temp_count);
5033 PROF_MAX(prof, orig, temp_count_max);
5034 PROF_ADD(prof, orig, del_op_count);
5035 PROF_ADD(prof, orig, code_in_len);
5036 PROF_ADD(prof, orig, code_out_len);
5037 PROF_ADD(prof, orig, search_out_len);
5038 PROF_ADD(prof, orig, interm_time);
5039 PROF_ADD(prof, orig, code_time);
5040 PROF_ADD(prof, orig, la_time);
5041 PROF_ADD(prof, orig, opt_time);
5042 PROF_ADD(prof, orig, restore_count);
5043 PROF_ADD(prof, orig, restore_time);
5044 }
5045 if (table) {
5046 int i;
5047
5048 for (i = 0; i < NB_OPS; i++) {
5049 PROF_ADD(prof, orig, table_op_count[i]);
5050 }
5051 }
5052 }
5053}
5054
5055#undef PROF_ADD
5056#undef PROF_MAX
5057
5058static void tcg_profile_snapshot_counters(TCGProfile *prof)
5059{
5060 tcg_profile_snapshot(prof, true, false);
5061}
5062
5063static void tcg_profile_snapshot_table(TCGProfile *prof)
5064{
5065 tcg_profile_snapshot(prof, false, true);
5066}
c896fe29 5067
b6a7f3e0 5068void tcg_dump_op_count(GString *buf)
c896fe29 5069{
c3fac113 5070 TCGProfile prof = {};
c896fe29 5071 int i;
d70724ce 5072
c3fac113 5073 tcg_profile_snapshot_table(&prof);
15fc7daa 5074 for (i = 0; i < NB_OPS; i++) {
b6a7f3e0
DB		 5075 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
5076 prof.table_op_count[i]);
c896fe29 5077 }
c896fe29 5078}
72fd2efb
EC		 5079
5080int64_t tcg_cpu_exec_time(void)
5081{
0e2d61cf 5082 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 5083 unsigned int i;
5084 int64_t ret = 0;
5085
5086 for (i = 0; i < n_ctxs; i++) {
d73415a3 5087 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 5088 const TCGProfile *prof = &s->prof;
5089
d73415a3 5090 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 5091 }
5092 return ret;
5093}
246ae24d 5094#else
b6a7f3e0 5095void tcg_dump_op_count(GString *buf)
246ae24d 5096{
b6a7f3e0 5097 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
246ae24d 5098}
72fd2efb
EC		 5099
5100int64_t tcg_cpu_exec_time(void)
5101{
5102 error_report("%s: TCG profiler not compiled", __func__);
5103 exit(EXIT_FAILURE);
5104}
c896fe29
FB		 5105#endif
5106
5107
fbf59aad 5108int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
c896fe29 5109{
c3fac113
EC		 5110#ifdef CONFIG_PROFILER
5111 TCGProfile *prof = &s->prof;
5112#endif
15fa08f8
RH		 5113 int i, num_insns;
5114 TCGOp *op;
c896fe29 5115
04fe6400
RH		 5116#ifdef CONFIG_PROFILER
5117 {
c1f543b7 5118 int n = 0;
04fe6400 5119
15fa08f8
RH		 5120 QTAILQ_FOREACH(op, &s->ops, link) {
5121 n++;
5122 }
d73415a3 5123 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 5124 if (n > prof->op_count_max) {
d73415a3 5125 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 5126 }
5127
5128 n = s->nb_temps;
d73415a3 5129 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 5130 if (n > prof->temp_count_max) {
d73415a3 5131 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 5132 }
5133 }
5134#endif
5135
c896fe29 5136#ifdef DEBUG_DISAS
d977e1c2 5137 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
fbf59aad 5138 && qemu_log_in_addr_range(pc_start))) {
c60f599b 5139 FILE *logfile = qemu_log_trylock();
78b54858
RH		 5140 if (logfile) {
5141 fprintf(logfile, "OP:\n");
b7a83ff8 5142 tcg_dump_ops(s, logfile, false);
78b54858
RH		 5143 fprintf(logfile, "\n");
5144 qemu_log_unlock(logfile);
5145 }
c896fe29
FB		 5146 }
5147#endif
5148
bef16ab4
RH		 5149#ifdef CONFIG_DEBUG_TCG
5150 /* Ensure all labels referenced have been emitted. */
5151 {
5152 TCGLabel *l;
5153 bool error = false;
5154
5155 QSIMPLEQ_FOREACH(l, &s->labels, next) {
f85b1fc4 5156 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) {
bef16ab4
RH		 5157 qemu_log_mask(CPU_LOG_TB_OP,
5158 "$L%d referenced but not present.\n", l->id);
5159 error = true;
5160 }
5161 }
5162 assert(!error);
5163 }
5164#endif
5165
c5cc28ff 5166#ifdef CONFIG_PROFILER
d73415a3 5167 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 5168#endif
5169
8f2e8c07 5170#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 5171 tcg_optimize(s);
8f2e8c07
KB		 5172#endif
5173
a23a9ec6 5174#ifdef CONFIG_PROFILER
d73415a3
SH		 5175 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
5176 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 5177#endif
c5cc28ff 5178
b4fc67c7 5179 reachable_code_pass(s);
874b8574 5180 liveness_pass_0(s);
b83eabea 5181 liveness_pass_1(s);
5a18407f 5182
b83eabea 5183 if (s->nb_indirects > 0) {
5a18407f 5184#ifdef DEBUG_DISAS
b83eabea 5185 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
fbf59aad 5186 && qemu_log_in_addr_range(pc_start))) {
c60f599b 5187 FILE *logfile = qemu_log_trylock();
78b54858
RH		 5188 if (logfile) {
5189 fprintf(logfile, "OP before indirect lowering:\n");
b7a83ff8 5190 tcg_dump_ops(s, logfile, false);
78b54858
RH		 5191 fprintf(logfile, "\n");
5192 qemu_log_unlock(logfile);
5193 }
b83eabea 5194 }
5a18407f 5195#endif
b83eabea
RH		 5196 /* Replace indirect temps with direct temps. */
5197 if (liveness_pass_2(s)) {
5198 /* If changes were made, re-run liveness. */
5199 liveness_pass_1(s);
5a18407f
RH		 5200 }
5201 }
c5cc28ff 5202
a23a9ec6 5203#ifdef CONFIG_PROFILER
d73415a3 5204 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 5205#endif
c896fe29
FB		 5206
5207#ifdef DEBUG_DISAS
d977e1c2 5208 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
fbf59aad 5209 && qemu_log_in_addr_range(pc_start))) {
c60f599b 5210 FILE *logfile = qemu_log_trylock();
78b54858
RH		 5211 if (logfile) {
5212 fprintf(logfile, "OP after optimization and liveness analysis:\n");
b7a83ff8 5213 tcg_dump_ops(s, logfile, true);
78b54858
RH		 5214 fprintf(logfile, "\n");
5215 qemu_log_unlock(logfile);
5216 }
c896fe29
FB		 5217 }
5218#endif
5219
35abb009 5220 /* Initialize goto_tb jump offsets. */
3a50f424
RH		 5221 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
5222 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
9da6079b
RH		 5223 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
5224 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
35abb009 5225
c896fe29
FB		 5226 tcg_reg_alloc_start(s);
5227
db0c51a3
RH		 5228 /*
5229 * Reset the buffer pointers when restarting after overflow.
5230 * TODO: Move this into translate-all.c with the rest of the
5231 * buffer management. Having only this done here is confusing.
5232 */
5233 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
5234 s->code_ptr = s->code_buf;
c896fe29 5235
659ef5cb 5236#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 5237 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 5238#endif
57a26946
RH		 5239#ifdef TCG_TARGET_NEED_POOL_LABELS
5240 s->pool_labels = NULL;
5241#endif
9ecefc84 5242
fca8a500 5243 num_insns = -1;
15fa08f8 5244 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 5245 TCGOpcode opc = op->opc;
b3db8758 5246
c896fe29 5247#ifdef CONFIG_PROFILER
d73415a3 5248 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 5249#endif
c45cb8bb
RH		 5250
5251 switch (opc) {
c896fe29 5252 case INDEX_op_mov_i32:
c896fe29 5253 case INDEX_op_mov_i64:
d2fd745f 5254 case INDEX_op_mov_vec:
dd186292 5255 tcg_reg_alloc_mov(s, op);
c896fe29 5256 break;
bab1671f
RH		 5257 case INDEX_op_dup_vec:
5258 tcg_reg_alloc_dup(s, op);
5259 break;
765b842a 5260 case INDEX_op_insn_start:
fca8a500 5261 if (num_insns >= 0) {
9f754620
RH		 5262 size_t off = tcg_current_code_size(s);
5263 s->gen_insn_end_off[num_insns] = off;
5264 /* Assert that we do not overflow our stored offset. */
5265 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 5266 }
5267 num_insns++;
bad729e2
RH		 5268 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
5269 target_ulong a;
5270#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 5271 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 5272#else
efee3746 5273 a = op->args[i];
bad729e2 5274#endif
fca8a500 5275 s->gen_insn_data[num_insns][i] = a;
bad729e2 5276 }
c896fe29 5277 break;
5ff9d6a4 5278 case INDEX_op_discard:
43439139 5279 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 5280 break;
c896fe29 5281 case INDEX_op_set_label:
e8996ee0 5282 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 5283 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 5284 break;
5285 case INDEX_op_call:
dd186292 5286 tcg_reg_alloc_call(s, op);
c45cb8bb 5287 break;
b55a8d9d
RH		 5288 case INDEX_op_exit_tb:
5289 tcg_out_exit_tb(s, op->args[0]);
5290 break;
cf7d6b8e
RH		 5291 case INDEX_op_goto_tb:
5292 tcg_out_goto_tb(s, op->args[0]);
5293 break;
efe86b21
RH		 5294 case INDEX_op_dup2_vec:
5295 if (tcg_reg_alloc_dup2(s, op)) {
5296 break;
5297 }
5298 /* fall through */
c896fe29 5299 default:
25c4d9cc 5300 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 5301 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 5302 /* Note: in order to speed up the code, it would be much
5303 faster to have specialized register allocator functions for
5304 some common argument patterns */
dd186292 5305 tcg_reg_alloc_op(s, op);
c896fe29
FB		 5306 break;
5307 }
b125f9dc
RH		 5308 /* Test for (pending) buffer overflow. The assumption is that any
5309 one operation beginning below the high water mark cannot overrun
5310 the buffer completely. Thus we can test for overflow after
5311 generating code without having to check during generation. */
644da9b3 5312 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 5313 return -1;
5314 }
6e6c4efe
RH		 5315 /* Test for TB overflow, as seen by gen_insn_end_off. */
5316 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
5317 return -2;
5318 }
c896fe29 5319 }
fca8a500
RH		 5320 tcg_debug_assert(num_insns >= 0);
5321 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 5322
b76f0d8c 5323 /* Generate TB finalization at the end of block */
659ef5cb 5324#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 5325 i = tcg_out_ldst_finalize(s);
5326 if (i < 0) {
5327 return i;
23dceda6 5328 }
659ef5cb 5329#endif
57a26946 5330#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 5331 i = tcg_out_pool_finalize(s);
5332 if (i < 0) {
5333 return i;
57a26946
RH		 5334 }
5335#endif
7ecd02a0
RH		 5336 if (!tcg_resolve_relocs(s)) {
5337 return -2;
5338 }
c896fe29 5339
df5d2b16 5340#ifndef CONFIG_TCG_INTERPRETER
c896fe29 5341 /* flush instruction cache */
db0c51a3
RH		 5342 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
5343 (uintptr_t)s->code_buf,
1da8de39 5344 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 5345#endif
2aeabc08 5346
1813e175 5347 return tcg_current_code_size(s);
c896fe29
FB		 5348}
5349
a23a9ec6 5350#ifdef CONFIG_PROFILER
3a841ab5 5351void tcg_dump_info(GString *buf)
a23a9ec6 5352{
c3fac113
EC		 5353 TCGProfile prof = {};
5354 const TCGProfile *s;
5355 int64_t tb_count;
5356 int64_t tb_div_count;
5357 int64_t tot;
5358
5359 tcg_profile_snapshot_counters(&prof);
5360 s = &prof;
5361 tb_count = s->tb_count;
5362 tb_div_count = tb_count ? tb_count : 1;
5363 tot = s->interm_time + s->code_time;
a23a9ec6 5364
3a841ab5
DB		 5365 g_string_append_printf(buf, "JIT cycles %" PRId64
5366 " (%0.3f s at 2.4 GHz)\n",
5367 tot, tot / 2.4e9);
5368 g_string_append_printf(buf, "translated TBs %" PRId64
5369 " (aborted=%" PRId64 " %0.1f%%)\n",
5370 tb_count, s->tb_count1 - tb_count,
5371 (double)(s->tb_count1 - s->tb_count)
5372 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
5373 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
5374 (double)s->op_count / tb_div_count, s->op_count_max);
5375 g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
5376 (double)s->del_op_count / tb_div_count);
5377 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
5378 (double)s->temp_count / tb_div_count,
5379 s->temp_count_max);
5380 g_string_append_printf(buf, "avg host code/TB %0.1f\n",
5381 (double)s->code_out_len / tb_div_count);
5382 g_string_append_printf(buf, "avg search data/TB %0.1f\n",
5383 (double)s->search_out_len / tb_div_count);
a813e36f 5384
3a841ab5
DB		 5385 g_string_append_printf(buf, "cycles/op %0.1f\n",
5386 s->op_count ? (double)tot / s->op_count : 0);
5387 g_string_append_printf(buf, "cycles/in byte %0.1f\n",
5388 s->code_in_len ? (double)tot / s->code_in_len : 0);
5389 g_string_append_printf(buf, "cycles/out byte %0.1f\n",
5390 s->code_out_len ? (double)tot / s->code_out_len : 0);
5391 g_string_append_printf(buf, "cycles/search byte %0.1f\n",
5392 s->search_out_len ?
5393 (double)tot / s->search_out_len : 0);
fca8a500 5394 if (tot == 0) {
a23a9ec6 5395 tot = 1;
fca8a500 5396 }
3a841ab5
DB		 5397 g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
5398 (double)s->interm_time / tot * 100.0);
5399 g_string_append_printf(buf, " gen_code time %0.1f%%\n",
5400 (double)s->code_time / tot * 100.0);
5401 g_string_append_printf(buf, "optim./code time %0.1f%%\n",
5402 (double)s->opt_time / (s->code_time ?
5403 s->code_time : 1)
5404 * 100.0);
5405 g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
5406 (double)s->la_time / (s->code_time ?
5407 s->code_time : 1) * 100.0);
5408 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
5409 s->restore_count);
5410 g_string_append_printf(buf, " avg cycles %0.1f\n",
5411 s->restore_count ?
5412 (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 5413}
5414#else
3a841ab5 5415void tcg_dump_info(GString *buf)
a23a9ec6 5416{
3a841ab5 5417 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
a23a9ec6
FB		 5418}
5419#endif
813da627
RH		 5420
5421#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 5422/* In order to use this feature, the backend needs to do three things:
5423
5424 (1) Define ELF_HOST_MACHINE to indicate both what value to
5425 put into the ELF image and to indicate support for the feature.
5426
5427 (2) Define tcg_register_jit. This should create a buffer containing
5428 the contents of a .debug_frame section that describes the post-
5429 prologue unwind info for the tcg machine.
5430
5431 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
5432*/
813da627
RH		 5433
5434/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
5435typedef enum {
5436 JIT_NOACTION = 0,
5437 JIT_REGISTER_FN,
5438 JIT_UNREGISTER_FN
5439} jit_actions_t;
5440
5441struct jit_code_entry {
5442 struct jit_code_entry *next_entry;
5443 struct jit_code_entry *prev_entry;
5444 const void *symfile_addr;
5445 uint64_t symfile_size;
5446};
5447
5448struct jit_descriptor {
5449 uint32_t version;
5450 uint32_t action_flag;
5451 struct jit_code_entry *relevant_entry;
5452 struct jit_code_entry *first_entry;
5453};
5454
5455void __jit_debug_register_code(void) __attribute__((noinline));
5456void __jit_debug_register_code(void)
5457{
5458 asm("");
5459}
5460
5461/* Must statically initialize the version, because GDB may check
5462 the version before we can set it. */
5463struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
5464
5465/* End GDB interface. */
5466
5467static int find_string(const char *strtab, const char *str)
5468{
5469 const char *p = strtab + 1;
5470
5471 while (1) {
5472 if (strcmp(p, str) == 0) {
5473 return p - strtab;
5474 }
5475 p += strlen(p) + 1;
5476 }
5477}
5478
755bf9e5 5479static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 5480 const void *debug_frame,
5481 size_t debug_frame_size)
813da627 5482{
5872bbf2
RH		 5483 struct __attribute__((packed)) DebugInfo {
5484 uint32_t len;
5485 uint16_t version;
5486 uint32_t abbrev;
5487 uint8_t ptr_size;
5488 uint8_t cu_die;
5489 uint16_t cu_lang;
5490 uintptr_t cu_low_pc;
5491 uintptr_t cu_high_pc;
5492 uint8_t fn_die;
5493 char fn_name[16];
5494 uintptr_t fn_low_pc;
5495 uintptr_t fn_high_pc;
5496 uint8_t cu_eoc;
5497 };
813da627
RH		 5498
5499 struct ElfImage {
5500 ElfW(Ehdr) ehdr;
5501 ElfW(Phdr) phdr;
5872bbf2
RH		 5502 ElfW(Shdr) shdr[7];
5503 ElfW(Sym) sym[2];
5504 struct DebugInfo di;
5505 uint8_t da[24];
5506 char str[80];
5507 };
5508
5509 struct ElfImage *img;
5510
5511 static const struct ElfImage img_template = {
5512 .ehdr = {
5513 .e_ident[EI_MAG0] = ELFMAG0,
5514 .e_ident[EI_MAG1] = ELFMAG1,
5515 .e_ident[EI_MAG2] = ELFMAG2,
5516 .e_ident[EI_MAG3] = ELFMAG3,
5517 .e_ident[EI_CLASS] = ELF_CLASS,
5518 .e_ident[EI_DATA] = ELF_DATA,
5519 .e_ident[EI_VERSION] = EV_CURRENT,
5520 .e_type = ET_EXEC,
5521 .e_machine = ELF_HOST_MACHINE,
5522 .e_version = EV_CURRENT,
5523 .e_phoff = offsetof(struct ElfImage, phdr),
5524 .e_shoff = offsetof(struct ElfImage, shdr),
5525 .e_ehsize = sizeof(ElfW(Shdr)),
5526 .e_phentsize = sizeof(ElfW(Phdr)),
5527 .e_phnum = 1,
5528 .e_shentsize = sizeof(ElfW(Shdr)),
5529 .e_shnum = ARRAY_SIZE(img->shdr),
5530 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 5531#ifdef ELF_HOST_FLAGS
5532 .e_flags = ELF_HOST_FLAGS,
5533#endif
5534#ifdef ELF_OSABI
5535 .e_ident[EI_OSABI] = ELF_OSABI,
5536#endif
5872bbf2
RH		 5537 },
5538 .phdr = {
5539 .p_type = PT_LOAD,
5540 .p_flags = PF_X,
5541 },
5542 .shdr = {
5543 [0] = { .sh_type = SHT_NULL },
5544 /* Trick: The contents of code_gen_buffer are not present in
5545 this fake ELF file; that got allocated elsewhere. Therefore
5546 we mark .text as SHT_NOBITS (similar to .bss) so that readers
5547 will not look for contents. We can record any address. */
5548 [1] = { /* .text */
5549 .sh_type = SHT_NOBITS,
5550 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
5551 },
5552 [2] = { /* .debug_info */
5553 .sh_type = SHT_PROGBITS,
5554 .sh_offset = offsetof(struct ElfImage, di),
5555 .sh_size = sizeof(struct DebugInfo),
5556 },
5557 [3] = { /* .debug_abbrev */
5558 .sh_type = SHT_PROGBITS,
5559 .sh_offset = offsetof(struct ElfImage, da),
5560 .sh_size = sizeof(img->da),
5561 },
5562 [4] = { /* .debug_frame */
5563 .sh_type = SHT_PROGBITS,
5564 .sh_offset = sizeof(struct ElfImage),
5565 },
5566 [5] = { /* .symtab */
5567 .sh_type = SHT_SYMTAB,
5568 .sh_offset = offsetof(struct ElfImage, sym),
5569 .sh_size = sizeof(img->sym),
5570 .sh_info = 1,
5571 .sh_link = ARRAY_SIZE(img->shdr) - 1,
5572 .sh_entsize = sizeof(ElfW(Sym)),
5573 },
5574 [6] = { /* .strtab */
5575 .sh_type = SHT_STRTAB,
5576 .sh_offset = offsetof(struct ElfImage, str),
5577 .sh_size = sizeof(img->str),
5578 }
5579 },
5580 .sym = {
5581 [1] = { /* code_gen_buffer */
5582 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
5583 .st_shndx = 1,
5584 }
5585 },
5586 .di = {
5587 .len = sizeof(struct DebugInfo) - 4,
5588 .version = 2,
5589 .ptr_size = sizeof(void *),
5590 .cu_die = 1,
5591 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
5592 .fn_die = 2,
5593 .fn_name = "code_gen_buffer"
5594 },
5595 .da = {
5596 1, /* abbrev number (the cu) */
5597 0x11, 1, /* DW_TAG_compile_unit, has children */
5598 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
5599 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5600 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5601 0, 0, /* end of abbrev */
5602 2, /* abbrev number (the fn) */
5603 0x2e, 0, /* DW_TAG_subprogram, no children */
5604 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
5605 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5606 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5607 0, 0, /* end of abbrev */
5608 0 /* no more abbrev */
5609 },
5610 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
5611 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 5612 };
5613
5614 /* We only need a single jit entry; statically allocate it. */
5615 static struct jit_code_entry one_entry;
5616
5872bbf2 5617 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 5618 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 5619 DebugFrameHeader *dfh;
813da627 5620
5872bbf2
RH		 5621 img = g_malloc(img_size);
5622 *img = img_template;
813da627 5623
5872bbf2
RH		 5624 img->phdr.p_vaddr = buf;
5625 img->phdr.p_paddr = buf;
5626 img->phdr.p_memsz = buf_size;
813da627 5627
813da627 5628 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 5629 img->shdr[1].sh_addr = buf;
813da627
RH		 5630 img->shdr[1].sh_size = buf_size;
5631
5872bbf2
RH		 5632 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
5633 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
5634
5635 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
5636 img->shdr[4].sh_size = debug_frame_size;
5637
5638 img->shdr[5].sh_name = find_string(img->str, ".symtab");
5639 img->shdr[6].sh_name = find_string(img->str, ".strtab");
5640
5641 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
5642 img->sym[1].st_value = buf;
5643 img->sym[1].st_size = buf_size;
813da627 5644
5872bbf2 5645 img->di.cu_low_pc = buf;
45aba097 5646 img->di.cu_high_pc = buf + buf_size;
5872bbf2 5647 img->di.fn_low_pc = buf;
45aba097 5648 img->di.fn_high_pc = buf + buf_size;
813da627 5649
2c90784a
RH		 5650 dfh = (DebugFrameHeader *)(img + 1);
5651 memcpy(dfh, debug_frame, debug_frame_size);
5652 dfh->fde.func_start = buf;
5653 dfh->fde.func_len = buf_size;
5654
813da627
RH		 5655#ifdef DEBUG_JIT
5656 /* Enable this block to be able to debug the ELF image file creation.
5657 One can use readelf, objdump, or other inspection utilities. */
5658 {
eb6b2edf
BM		 5659 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
5660 FILE *f = fopen(jit, "w+b");
813da627 5661 if (f) {
5872bbf2 5662 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 5663 /* Avoid stupid unused return value warning for fwrite. */
5664 }
5665 fclose(f);
5666 }
5667 }
5668#endif
5669
5670 one_entry.symfile_addr = img;
5671 one_entry.symfile_size = img_size;
5672
5673 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
5674 __jit_debug_descriptor.relevant_entry = &one_entry;
5675 __jit_debug_descriptor.first_entry = &one_entry;
5676 __jit_debug_register_code();
5677}
5678#else
5872bbf2
RH		 5679/* No support for the feature. Provide the entry point expected by exec.c,
5680 and implement the internal function we declared earlier. */
813da627 5681
755bf9e5 5682static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 5683 const void *debug_frame,
5684 size_t debug_frame_size)
813da627
RH		 5685{
5686}
5687
755bf9e5 5688void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 5689{
5690}
5691#endif /* ELF_HOST_MACHINE */
db432672
RH		 5692
5693#if !TCG_TARGET_MAYBE_vec
5694void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
5695{
5696 g_assert_not_reached();
5697}
5698#endif
QEMU mirror