]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
accel/tcg/plugin: Tidy plugin_gen_disable_mem_helpers
[mirror_qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
084cfca1 37#include "qemu/cacheflush.h"
ad768e6f 38#include "qemu/cacheinfo.h"
c896fe29 39
c5d3c498 40/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 41 CPU definitions. Currently they are used for qemu_ld/st
42 instructions */
43#define NO_CPU_IO_DEFS
c896fe29 44
63c91552 45#include "exec/exec-all.h"
dcb32f1d 46#include "tcg/tcg-op.h"
813da627 47
edee2579 48#if UINTPTR_MAX == UINT32_MAX
813da627 49# define ELF_CLASS ELFCLASS32
edee2579
RH		 50#else
51# define ELF_CLASS ELFCLASS64
813da627 52#endif
e03b5686 53#if HOST_BIG_ENDIAN
813da627
RH		 54# define ELF_DATA ELFDATA2MSB
55#else
56# define ELF_DATA ELFDATA2LSB
57#endif
58
c896fe29 59#include "elf.h"
508127e2 60#include "exec/log.h"
d2ba8026 61#include "tcg/tcg-ldst.h"
5ff7258c 62#include "tcg-internal.h"
5584e2db 63#include "accel/tcg/perf.h"
c896fe29 64
139c1837 65/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 66 used here. */
e4d58b41
RH		 67static void tcg_target_init(TCGContext *s);
68static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 69static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 70 intptr_t value, intptr_t addend);
c896fe29 71
497a22eb
RH		 72/* The CIE and FDE header definitions will be common to all hosts. */
73typedef struct {
74 uint32_t len __attribute__((aligned((sizeof(void *)))));
75 uint32_t id;
76 uint8_t version;
77 char augmentation[1];
78 uint8_t code_align;
79 uint8_t data_align;
80 uint8_t return_column;
81} DebugFrameCIE;
82
83typedef struct QEMU_PACKED {
84 uint32_t len __attribute__((aligned((sizeof(void *)))));
85 uint32_t cie_offset;
edee2579
RH		 86 uintptr_t func_start;
87 uintptr_t func_len;
497a22eb
RH		 88} DebugFrameFDEHeader;
89
2c90784a
RH		 90typedef struct QEMU_PACKED {
91 DebugFrameCIE cie;
92 DebugFrameFDEHeader fde;
93} DebugFrameHeader;
94
755bf9e5 95static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 96 const void *debug_frame,
97 size_t debug_frame_size)
813da627
RH		 98 __attribute__((unused));
99
139c1837 100/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 101static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 102 intptr_t arg2);
78113e83 103static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 104static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 105 TCGReg ret, tcg_target_long arg);
313bdea8 106static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
b55a8d9d 107static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
cf7d6b8e 108static void tcg_out_goto_tb(TCGContext *s, int which);
5e8892db
MR		 109static void tcg_out_op(TCGContext *s, TCGOpcode opc,
110 const TCGArg args[TCG_MAX_OP_ARGS],
111 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 112#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 113static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
114 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 115static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
116 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 117static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
118 TCGReg dst, int64_t arg);
5e8892db
MR		 119static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
120 unsigned vecl, unsigned vece,
121 const TCGArg args[TCG_MAX_OP_ARGS],
122 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 123#else
e7632cfa
RH		 124static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
125 TCGReg dst, TCGReg src)
126{
127 g_assert_not_reached();
128}
d6ecb4a9
RH		 129static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
130 TCGReg dst, TCGReg base, intptr_t offset)
131{
132 g_assert_not_reached();
133}
4e186175
RH		 134static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
135 TCGReg dst, int64_t arg)
e7632cfa
RH		 136{
137 g_assert_not_reached();
138}
5e8892db
MR		 139static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
140 unsigned vecl, unsigned vece,
141 const TCGArg args[TCG_MAX_OP_ARGS],
142 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 143{
144 g_assert_not_reached();
145}
146#endif
2a534aff 147static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 148 intptr_t arg2);
59d7c14e
RH		 149static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
150 TCGReg base, intptr_t ofs);
7b7d8b2d 151static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
cee44b03 152 const TCGHelperInfo *info);
5e3d0c19 153static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
a4fbbd77 154static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 155#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 156static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 157#endif
c896fe29 158
42eb6dfc
RH		 159TCGContext tcg_init_ctx;
160__thread TCGContext *tcg_ctx;
161
5ff7258c 162TCGContext **tcg_ctxs;
0e2d61cf
RH		 163unsigned int tcg_cur_ctxs;
164unsigned int tcg_max_ctxs;
1c2adb95 165TCGv_env cpu_env = 0;
c8bc1168 166const void *tcg_code_gen_epilogue;
db0c51a3 167uintptr_t tcg_splitwx_diff;
df2cce29 168
b91ccb31
RH		 169#ifndef CONFIG_TCG_INTERPRETER
170tcg_prologue_fn *tcg_qemu_tb_exec;
171#endif
172
d2fd745f 173static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 174static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 175
1813e175 176#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 177static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 178{
179 *s->code_ptr++ = v;
180}
181
4196dca6
PM		 182static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
183 uint8_t v)
5c53bb81 184{
1813e175 185 *p = v;
5c53bb81 186}
1813e175 187#endif
5c53bb81 188
1813e175 189#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 190static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 191{
1813e175
RH		 192 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
193 *s->code_ptr++ = v;
194 } else {
195 tcg_insn_unit *p = s->code_ptr;
196 memcpy(p, &v, sizeof(v));
197 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
198 }
c896fe29
FB		 199}
200
4196dca6
PM		 201static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
202 uint16_t v)
5c53bb81 203{
1813e175
RH		 204 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
205 *p = v;
206 } else {
207 memcpy(p, &v, sizeof(v));
208 }
5c53bb81 209}
1813e175 210#endif
5c53bb81 211
1813e175 212#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 213static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 214{
1813e175
RH		 215 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
216 *s->code_ptr++ = v;
217 } else {
218 tcg_insn_unit *p = s->code_ptr;
219 memcpy(p, &v, sizeof(v));
220 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
221 }
c896fe29
FB		 222}
223
4196dca6
PM		 224static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
225 uint32_t v)
5c53bb81 226{
1813e175
RH		 227 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
228 *p = v;
229 } else {
230 memcpy(p, &v, sizeof(v));
231 }
5c53bb81 232}
1813e175 233#endif
5c53bb81 234
1813e175 235#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 236static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 237{
1813e175
RH		 238 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
239 *s->code_ptr++ = v;
240 } else {
241 tcg_insn_unit *p = s->code_ptr;
242 memcpy(p, &v, sizeof(v));
243 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
244 }
ac26eb69
RH		 245}
246
4196dca6
PM		 247static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
248 uint64_t v)
5c53bb81 249{
1813e175
RH		 250 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
251 *p = v;
252 } else {
253 memcpy(p, &v, sizeof(v));
254 }
5c53bb81 255}
1813e175 256#endif
5c53bb81 257
c896fe29
FB		 258/* label relocation processing */
259
1813e175 260static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 261 TCGLabel *l, intptr_t addend)
c896fe29 262{
7ecd02a0 263 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 264
7ecd02a0
RH		 265 r->type = type;
266 r->ptr = code_ptr;
267 r->addend = addend;
268 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 269}
270
92ab8e7d 271static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 272{
eabb7b91 273 tcg_debug_assert(!l->has_value);
c896fe29 274 l->has_value = 1;
92ab8e7d 275 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 276}
277
42a268c2 278TCGLabel *gen_new_label(void)
c896fe29 279{
b1311c4a 280 TCGContext *s = tcg_ctx;
51e3972c 281 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 282
7ecd02a0
RH		 283 memset(l, 0, sizeof(TCGLabel));
284 l->id = s->nb_labels++;
285 QSIMPLEQ_INIT(&l->relocs);
286
bef16ab4 287 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 288
289 return l;
c896fe29
FB		 290}
291
7ecd02a0
RH		 292static bool tcg_resolve_relocs(TCGContext *s)
293{
294 TCGLabel *l;
295
296 QSIMPLEQ_FOREACH(l, &s->labels, next) {
297 TCGRelocation *r;
298 uintptr_t value = l->u.value;
299
300 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
301 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
302 return false;
303 }
304 }
305 }
306 return true;
307}
308
9f754620
RH		 309static void set_jmp_reset_offset(TCGContext *s, int which)
310{
f14bed3f
RH		 311 /*
312 * We will check for overflow at the end of the opcode loop in
313 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
314 */
b7e4afbd 315 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 316}
317
b52a2c03
RH		 318static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
319{
320 /*
321 * We will check for overflow at the end of the opcode loop in
322 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
323 */
9da6079b 324 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
b52a2c03
RH		 325}
326
becc452a
RH		 327static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
328{
329 /*
330 * Return the read-execute version of the pointer, for the benefit
331 * of any pc-relative addressing mode.
332 */
9da6079b 333 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
becc452a
RH		 334}
335
db6b7d0c 336/* Signal overflow, starting over with fewer guest insns. */
8905770b
MAL		 337static G_NORETURN
338void tcg_raise_tb_overflow(TCGContext *s)
db6b7d0c
RH		 339{
340 siglongjmp(s->jmp_trans, -2);
341}
342
4c22e840
RH		 343#define C_PFX1(P, A) P##A
344#define C_PFX2(P, A, B) P##A##_##B
345#define C_PFX3(P, A, B, C) P##A##_##B##_##C
346#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
347#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
348#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
349
350/* Define an enumeration for the various combinations. */
351
352#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
353#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
354#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
355#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
356
357#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
358#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
359#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
360#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
361
362#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
363
364#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
365#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
366#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
367#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
368
369typedef enum {
370#include "tcg-target-con-set.h"
371} TCGConstraintSetIndex;
372
373static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
374
375#undef C_O0_I1
376#undef C_O0_I2
377#undef C_O0_I3
378#undef C_O0_I4
379#undef C_O1_I1
380#undef C_O1_I2
381#undef C_O1_I3
382#undef C_O1_I4
383#undef C_N1_I2
384#undef C_O2_I1
385#undef C_O2_I2
386#undef C_O2_I3
387#undef C_O2_I4
388
389/* Put all of the constraint sets into an array, indexed by the enum. */
390
391#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
392#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
393#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
394#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
395
396#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
397#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
398#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
399#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
400
401#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
402
403#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
404#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
405#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
406#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
407
408static const TCGTargetOpDef constraint_sets[] = {
409#include "tcg-target-con-set.h"
410};
411
412
413#undef C_O0_I1
414#undef C_O0_I2
415#undef C_O0_I3
416#undef C_O0_I4
417#undef C_O1_I1
418#undef C_O1_I2
419#undef C_O1_I3
420#undef C_O1_I4
421#undef C_N1_I2
422#undef C_O2_I1
423#undef C_O2_I2
424#undef C_O2_I3
425#undef C_O2_I4
426
427/* Expand the enumerator to be returned from tcg_target_op_def(). */
428
429#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
430#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
431#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
432#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
433
434#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
435#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
436#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
437#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
438
439#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
440
441#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
442#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
443#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
444#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
445
139c1837 446#include "tcg-target.c.inc"
c896fe29 447
38b47b19
EC		 448static void alloc_tcg_plugin_context(TCGContext *s)
449{
450#ifdef CONFIG_PLUGIN
451 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
452 s->plugin_tb->insns =
453 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
454#endif
455}
456
3468b59e
EC		 457/*
458 * All TCG threads except the parent (i.e. the one that called tcg_context_init
459 * and registered the target's TCG globals) must register with this function
460 * before initiating translation.
461 *
462 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
463 * of tcg_region_init() for the reasoning behind this.
464 *
465 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
466 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
467 * is not used anymore for translation once this function is called.
468 *
469 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
470 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
471 */
472#ifdef CONFIG_USER_ONLY
473void tcg_register_thread(void)
474{
475 tcg_ctx = &tcg_init_ctx;
476}
477#else
478void tcg_register_thread(void)
479{
480 TCGContext *s = g_malloc(sizeof(*s));
481 unsigned int i, n;
3468b59e
EC		 482
483 *s = tcg_init_ctx;
484
485 /* Relink mem_base. */
486 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
487 if (tcg_init_ctx.temps[i].mem_base) {
488 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
489 tcg_debug_assert(b >= 0 && b < n);
490 s->temps[i].mem_base = &s->temps[b];
491 }
492 }
493
494 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 495 n = qatomic_fetch_inc(&tcg_cur_ctxs);
496 g_assert(n < tcg_max_ctxs);
d73415a3 497 qatomic_set(&tcg_ctxs[n], s);
3468b59e 498
38b47b19
EC		 499 if (n > 0) {
500 alloc_tcg_plugin_context(s);
bf042e8e 501 tcg_region_initial_alloc(s);
38b47b19
EC		 502 }
503
3468b59e 504 tcg_ctx = s;
e8feb96f 505}
3468b59e 506#endif /* !CONFIG_USER_ONLY */
e8feb96f 507
c896fe29
FB		 508/* pool based memory allocation */
509void *tcg_malloc_internal(TCGContext *s, int size)
510{
511 TCGPool *p;
512 int pool_size;
a813e36f 513
c896fe29
FB		 514 if (size > TCG_POOL_CHUNK_SIZE) {
515 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 516 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 517 p->size = size;
4055299e
KB		 518 p->next = s->pool_first_large;
519 s->pool_first_large = p;
520 return p->data;
c896fe29
FB		 521 } else {
522 p = s->pool_current;
523 if (!p) {
524 p = s->pool_first;
525 if (!p)
526 goto new_pool;
527 } else {
528 if (!p->next) {
529 new_pool:
530 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 531 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 532 p->size = pool_size;
533 p->next = NULL;
a813e36f 534 if (s->pool_current) {
c896fe29 535 s->pool_current->next = p;
a813e36f 536 } else {
c896fe29 537 s->pool_first = p;
a813e36f 538 }
c896fe29
FB		 539 } else {
540 p = p->next;
541 }
542 }
543 }
544 s->pool_current = p;
545 s->pool_cur = p->data + size;
546 s->pool_end = p->data + p->size;
547 return p->data;
548}
549
550void tcg_pool_reset(TCGContext *s)
551{
4055299e
KB		 552 TCGPool *p, *t;
553 for (p = s->pool_first_large; p; p = t) {
554 t = p->next;
555 g_free(p);
556 }
557 s->pool_first_large = NULL;
c896fe29
FB		 558 s->pool_cur = s->pool_end = NULL;
559 s->pool_current = NULL;
560}
561
2ef6175a
RH		 562#include "exec/helper-proto.h"
563
39004a71 564static TCGHelperInfo all_helpers[] = {
2ef6175a 565#include "exec/helper-tcg.h"
100b5e01 566};
619205fd 567static GHashTable *helper_table;
100b5e01 568
22f15579 569#ifdef CONFIG_TCG_INTERPRETER
c6ef8c7b
PMD		 570static ffi_type *typecode_to_ffi(int argmask)
571{
e9709e17
RH		 572 /*
573 * libffi does not support __int128_t, so we have forced Int128
574 * to use the structure definition instead of the builtin type.
575 */
576 static ffi_type *ffi_type_i128_elements[3] = {
577 &ffi_type_uint64,
578 &ffi_type_uint64,
579 NULL
580 };
581 static ffi_type ffi_type_i128 = {
582 .size = 16,
583 .alignment = __alignof__(Int128),
584 .type = FFI_TYPE_STRUCT,
585 .elements = ffi_type_i128_elements,
586 };
587
c6ef8c7b
PMD		 588 switch (argmask) {
589 case dh_typecode_void:
590 return &ffi_type_void;
591 case dh_typecode_i32:
592 return &ffi_type_uint32;
593 case dh_typecode_s32:
594 return &ffi_type_sint32;
595 case dh_typecode_i64:
596 return &ffi_type_uint64;
597 case dh_typecode_s64:
598 return &ffi_type_sint64;
599 case dh_typecode_ptr:
600 return &ffi_type_pointer;
e9709e17
RH		 601 case dh_typecode_i128:
602 return &ffi_type_i128;
c6ef8c7b
PMD		 603 }
604 g_assert_not_reached();
605}
0c22e176
PMD		 606
607static void init_ffi_layouts(void)
608{
609 /* g_direct_hash/equal for direct comparisons on uint32_t. */
f9c4bb80
RH		 610 GHashTable *ffi_table = g_hash_table_new(NULL, NULL);
611
0c22e176 612 for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
f9c4bb80
RH		 613 TCGHelperInfo *info = &all_helpers[i];
614 unsigned typemask = info->typemask;
0c22e176
PMD		 615 gpointer hash = (gpointer)(uintptr_t)typemask;
616 struct {
617 ffi_cif cif;
618 ffi_type *args[];
619 } *ca;
620 ffi_status status;
621 int nargs;
f9c4bb80 622 ffi_cif *cif;
0c22e176 623
f9c4bb80
RH		 624 cif = g_hash_table_lookup(ffi_table, hash);
625 if (cif) {
626 info->cif = cif;
0c22e176
PMD		 627 continue;
628 }
629
630 /* Ignoring the return type, find the last non-zero field. */
631 nargs = 32 - clz32(typemask >> 3);
632 nargs = DIV_ROUND_UP(nargs, 3);
e9709e17 633 assert(nargs <= MAX_CALL_IARGS);
0c22e176
PMD		 634
635 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
636 ca->cif.rtype = typecode_to_ffi(typemask & 7);
637 ca->cif.nargs = nargs;
638
639 if (nargs != 0) {
640 ca->cif.arg_types = ca->args;
641 for (int j = 0; j < nargs; ++j) {
642 int typecode = extract32(typemask, (j + 1) * 3, 3);
643 ca->args[j] = typecode_to_ffi(typecode);
644 }
645 }
646
647 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
648 ca->cif.rtype, ca->cif.arg_types);
649 assert(status == FFI_OK);
650
f9c4bb80
RH		 651 cif = &ca->cif;
652 info->cif = cif;
653 g_hash_table_insert(ffi_table, hash, (gpointer)cif);
0c22e176 654 }
f9c4bb80
RH		 655
656 g_hash_table_destroy(ffi_table);
0c22e176
PMD		 657}
658#endif /* CONFIG_TCG_INTERPRETER */
22f15579 659
39004a71
RH		 660typedef struct TCGCumulativeArgs {
661 int arg_idx; /* tcg_gen_callN args[] */
662 int info_in_idx; /* TCGHelperInfo in[] */
663 int arg_slot; /* regs+stack slot */
664 int ref_slot; /* stack slots for references */
665} TCGCumulativeArgs;
666
667static void layout_arg_even(TCGCumulativeArgs *cum)
668{
669 cum->arg_slot += cum->arg_slot & 1;
670}
671
672static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
673 TCGCallArgumentKind kind)
674{
675 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
676
677 *loc = (TCGCallArgumentLoc){
678 .kind = kind,
679 .arg_idx = cum->arg_idx,
680 .arg_slot = cum->arg_slot,
681 };
682 cum->info_in_idx++;
683 cum->arg_slot++;
684}
685
686static void layout_arg_normal_n(TCGCumulativeArgs *cum,
687 TCGHelperInfo *info, int n)
688{
689 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
690
691 for (int i = 0; i < n; ++i) {
692 /* Layout all using the same arg_idx, adjusting the subindex. */
693 loc[i] = (TCGCallArgumentLoc){
694 .kind = TCG_CALL_ARG_NORMAL,
695 .arg_idx = cum->arg_idx,
696 .tmp_subindex = i,
697 .arg_slot = cum->arg_slot + i,
698 };
699 }
700 cum->info_in_idx += n;
701 cum->arg_slot += n;
702}
703
313bdea8
RH		 704static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info)
705{
706 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
707 int n = 128 / TCG_TARGET_REG_BITS;
708
709 /* The first subindex carries the pointer. */
710 layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);
711
712 /*
713 * The callee is allowed to clobber memory associated with
714 * structure pass by-reference. Therefore we must make copies.
715 * Allocate space from "ref_slot", which will be adjusted to
716 * follow the parameters on the stack.
717 */
718 loc[0].ref_slot = cum->ref_slot;
719
720 /*
721 * Subsequent words also go into the reference slot, but
722 * do not accumulate into the regular arguments.
723 */
724 for (int i = 1; i < n; ++i) {
725 loc[i] = (TCGCallArgumentLoc){
726 .kind = TCG_CALL_ARG_BY_REF_N,
727 .arg_idx = cum->arg_idx,
728 .tmp_subindex = i,
729 .ref_slot = cum->ref_slot + i,
730 };
731 }
732 cum->info_in_idx += n;
733 cum->ref_slot += n;
734}
735
39004a71
RH		 736static void init_call_layout(TCGHelperInfo *info)
737{
738 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
739 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
740 unsigned typemask = info->typemask;
741 unsigned typecode;
742 TCGCumulativeArgs cum = { };
743
744 /*
745 * Parse and place any function return value.
746 */
747 typecode = typemask & 7;
748 switch (typecode) {
749 case dh_typecode_void:
750 info->nr_out = 0;
751 break;
752 case dh_typecode_i32:
753 case dh_typecode_s32:
754 case dh_typecode_ptr:
755 info->nr_out = 1;
756 info->out_kind = TCG_CALL_RET_NORMAL;
757 break;
758 case dh_typecode_i64:
759 case dh_typecode_s64:
760 info->nr_out = 64 / TCG_TARGET_REG_BITS;
761 info->out_kind = TCG_CALL_RET_NORMAL;
5e3d0c19
RH		 762 /* Query the last register now to trigger any assert early. */
763 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
466d3759
RH		 764 break;
765 case dh_typecode_i128:
766 info->nr_out = 128 / TCG_TARGET_REG_BITS;
5427a9a7
RH		 767 info->out_kind = TCG_TARGET_CALL_RET_I128;
768 switch (TCG_TARGET_CALL_RET_I128) {
466d3759 769 case TCG_CALL_RET_NORMAL:
5e3d0c19
RH		 770 /* Query the last register now to trigger any assert early. */
771 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
466d3759 772 break;
c6556aa0
RH		 773 case TCG_CALL_RET_BY_VEC:
774 /* Query the single register now to trigger any assert early. */
775 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
776 break;
313bdea8
RH		 777 case TCG_CALL_RET_BY_REF:
778 /*
779 * Allocate the first argument to the output.
780 * We don't need to store this anywhere, just make it
781 * unavailable for use in the input loop below.
782 */
783 cum.arg_slot = 1;
784 break;
466d3759
RH		 785 default:
786 qemu_build_not_reached();
787 }
39004a71
RH		 788 break;
789 default:
790 g_assert_not_reached();
791 }
39004a71
RH		 792
793 /*
794 * Parse and place function arguments.
795 */
796 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
797 TCGCallArgumentKind kind;
798 TCGType type;
799
800 typecode = typemask & 7;
801 switch (typecode) {
802 case dh_typecode_i32:
803 case dh_typecode_s32:
804 type = TCG_TYPE_I32;
805 break;
806 case dh_typecode_i64:
807 case dh_typecode_s64:
808 type = TCG_TYPE_I64;
809 break;
810 case dh_typecode_ptr:
811 type = TCG_TYPE_PTR;
812 break;
466d3759
RH		 813 case dh_typecode_i128:
814 type = TCG_TYPE_I128;
815 break;
39004a71
RH		 816 default:
817 g_assert_not_reached();
818 }
819
820 switch (type) {
821 case TCG_TYPE_I32:
822 switch (TCG_TARGET_CALL_ARG_I32) {
823 case TCG_CALL_ARG_EVEN:
824 layout_arg_even(&cum);
825 /* fall through */
826 case TCG_CALL_ARG_NORMAL:
827 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
828 break;
829 case TCG_CALL_ARG_EXTEND:
830 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
831 layout_arg_1(&cum, info, kind);
832 break;
833 default:
834 qemu_build_not_reached();
835 }
836 break;
837
838 case TCG_TYPE_I64:
839 switch (TCG_TARGET_CALL_ARG_I64) {
840 case TCG_CALL_ARG_EVEN:
841 layout_arg_even(&cum);
842 /* fall through */
843 case TCG_CALL_ARG_NORMAL:
844 if (TCG_TARGET_REG_BITS == 32) {
845 layout_arg_normal_n(&cum, info, 2);
846 } else {
847 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
848 }
849 break;
850 default:
851 qemu_build_not_reached();
852 }
853 break;
854
466d3759 855 case TCG_TYPE_I128:
5427a9a7 856 switch (TCG_TARGET_CALL_ARG_I128) {
466d3759
RH		 857 case TCG_CALL_ARG_EVEN:
858 layout_arg_even(&cum);
859 /* fall through */
860 case TCG_CALL_ARG_NORMAL:
861 layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
862 break;
313bdea8
RH		 863 case TCG_CALL_ARG_BY_REF:
864 layout_arg_by_ref(&cum, info);
865 break;
466d3759
RH		 866 default:
867 qemu_build_not_reached();
868 }
869 break;
870
39004a71
RH		 871 default:
872 g_assert_not_reached();
873 }
874 }
875 info->nr_in = cum.info_in_idx;
876
877 /* Validate that we didn't overrun the input array. */
878 assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
879 /* Validate the backend has enough argument space. */
880 assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
313bdea8
RH		 881
882 /*
883 * Relocate the "ref_slot" area to the end of the parameters.
884 * Minimizing this stack offset helps code size for x86,
885 * which has a signed 8-bit offset encoding.
886 */
887 if (cum.ref_slot != 0) {
888 int ref_base = 0;
889
890 if (cum.arg_slot > max_reg_slots) {
891 int align = __alignof(Int128) / sizeof(tcg_target_long);
892
893 ref_base = cum.arg_slot - max_reg_slots;
894 if (align > 1) {
895 ref_base = ROUND_UP(ref_base, align);
896 }
897 }
898 assert(ref_base + cum.ref_slot <= max_stk_slots);
899
900 if (ref_base != 0) {
901 for (int i = cum.info_in_idx - 1; i >= 0; --i) {
902 TCGCallArgumentLoc *loc = &info->in[i];
903 switch (loc->kind) {
904 case TCG_CALL_ARG_BY_REF:
905 case TCG_CALL_ARG_BY_REF_N:
906 loc->ref_slot += ref_base;
907 break;
908 default:
909 break;
910 }
911 }
912 }
913 }
39004a71
RH		 914}
915
91478cef 916static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 917static void process_op_defs(TCGContext *s);
1c2adb95
RH		 918static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
919 TCGReg reg, const char *name);
91478cef 920
43b972b7 921static void tcg_context_init(unsigned max_cpus)
c896fe29 922{
a76aabd3 923 TCGContext *s = &tcg_init_ctx;
100b5e01 924 int op, total_args, n, i;
c896fe29
FB		 925 TCGOpDef *def;
926 TCGArgConstraint *args_ct;
1c2adb95 927 TCGTemp *ts;
c896fe29
FB		 928
929 memset(s, 0, sizeof(*s));
c896fe29 930 s->nb_globals = 0;
c70fbf0a 931
c896fe29
FB		 932 /* Count total number of arguments and allocate the corresponding
933 space */
934 total_args = 0;
935 for(op = 0; op < NB_OPS; op++) {
936 def = &tcg_op_defs[op];
937 n = def->nb_iargs + def->nb_oargs;
938 total_args += n;
939 }
940
bc2b17e6 941 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 942
943 for(op = 0; op < NB_OPS; op++) {
944 def = &tcg_op_defs[op];
945 def->args_ct = args_ct;
c896fe29 946 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 947 args_ct += n;
948 }
5cd8f621
RH		 949
950 /* Register helpers. */
84fd9dd3 951 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 952 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 953
100b5e01 954 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
39004a71 955 init_call_layout(&all_helpers[i]);
84fd9dd3 956 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 957 (gpointer)&all_helpers[i]);
100b5e01 958 }
5cd8f621 959
22f15579 960#ifdef CONFIG_TCG_INTERPRETER
0c22e176 961 init_ffi_layouts();
22f15579
RH		 962#endif
963
c896fe29 964 tcg_target_init(s);
f69d277e 965 process_op_defs(s);
91478cef
RH		 966
967 /* Reverse the order of the saved registers, assuming they're all at
968 the start of tcg_target_reg_alloc_order. */
969 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
970 int r = tcg_target_reg_alloc_order[n];
971 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
972 break;
973 }
974 }
975 for (i = 0; i < n; ++i) {
976 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
977 }
978 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
979 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
980 }
b1311c4a 981
38b47b19
EC		 982 alloc_tcg_plugin_context(s);
983
b1311c4a 984 tcg_ctx = s;
3468b59e
EC		 985 /*
986 * In user-mode we simply share the init context among threads, since we
987 * use a single region. See the documentation tcg_region_init() for the
988 * reasoning behind this.
989 * In softmmu we will have at most max_cpus TCG threads.
990 */
991#ifdef CONFIG_USER_ONLY
df2cce29 992 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 993 tcg_cur_ctxs = 1;
994 tcg_max_ctxs = 1;
3468b59e 995#else
0e2d61cf
RH		 996 tcg_max_ctxs = max_cpus;
997 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 998#endif
1c2adb95
RH		 999
1000 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1001 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
1002 cpu_env = temp_tcgv_ptr(ts);
9002ec79 1003}
b03cce8e 1004
43b972b7 1005void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 1006{
43b972b7
RH		 1007 tcg_context_init(max_cpus);
1008 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 1009}
1010
6e3b2bfd
EC		 1011/*
1012 * Allocate TBs right before their corresponding translated code, making
1013 * sure that TBs and code are on different cache lines.
1014 */
1015TranslationBlock *tcg_tb_alloc(TCGContext *s)
1016{
1017 uintptr_t align = qemu_icache_linesize;
1018 TranslationBlock *tb;
1019 void *next;
1020
e8feb96f 1021 retry:
6e3b2bfd
EC		 1022 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1023 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1024
1025 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 1026 if (tcg_region_alloc(s)) {
1027 return NULL;
1028 }
1029 goto retry;
6e3b2bfd 1030 }
d73415a3 1031 qatomic_set(&s->code_gen_ptr, next);
57a26946 1032 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 1033 return tb;
1034}
1035
9002ec79
RH		 1036void tcg_prologue_init(TCGContext *s)
1037{
b0a0794a 1038 size_t prologue_size;
8163b749 1039
b0a0794a
RH		 1040 s->code_ptr = s->code_gen_ptr;
1041 s->code_buf = s->code_gen_ptr;
5b38ee31 1042 s->data_gen_ptr = NULL;
b91ccb31
RH		 1043
1044#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 1045 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 1046#endif
8163b749 1047
5b38ee31
RH		 1048#ifdef TCG_TARGET_NEED_POOL_LABELS
1049 s->pool_labels = NULL;
1050#endif
1051
653b87eb 1052 qemu_thread_jit_write();
8163b749 1053 /* Generate the prologue. */
b03cce8e 1054 tcg_target_qemu_prologue(s);
5b38ee31
RH		 1055
1056#ifdef TCG_TARGET_NEED_POOL_LABELS
1057 /* Allow the prologue to put e.g. guest_base into a pool entry. */
1058 {
1768987b
RH		 1059 int result = tcg_out_pool_finalize(s);
1060 tcg_debug_assert(result == 0);
5b38ee31
RH		 1061 }
1062#endif
1063
b0a0794a 1064 prologue_size = tcg_current_code_size(s);
5584e2db 1065 perf_report_prologue(s->code_gen_ptr, prologue_size);
b0a0794a 1066
df5d2b16 1067#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 1068 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1069 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 1070#endif
8163b749 1071
d6b64b2b
RH		 1072#ifdef DEBUG_DISAS
1073 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
c60f599b 1074 FILE *logfile = qemu_log_trylock();
78b54858
RH		 1075 if (logfile) {
1076 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
1077 if (s->data_gen_ptr) {
1078 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
1079 size_t data_size = prologue_size - code_size;
1080 size_t i;
1081
1082 disas(logfile, s->code_gen_ptr, code_size);
1083
1084 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1085 if (sizeof(tcg_target_ulong) == 8) {
1086 fprintf(logfile,
1087 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1088 (uintptr_t)s->data_gen_ptr + i,
1089 *(uint64_t *)(s->data_gen_ptr + i));
1090 } else {
1091 fprintf(logfile,
1092 "0x%08" PRIxPTR ": .long 0x%08x\n",
1093 (uintptr_t)s->data_gen_ptr + i,
1094 *(uint32_t *)(s->data_gen_ptr + i));
1095 }
5b38ee31 1096 }
78b54858
RH		 1097 } else {
1098 disas(logfile, s->code_gen_ptr, prologue_size);
5b38ee31 1099 }
78b54858 1100 fprintf(logfile, "\n");
78b54858 1101 qemu_log_unlock(logfile);
5b38ee31 1102 }
d6b64b2b
RH		 1103 }
1104#endif
cedbcb01 1105
6eea0434
RH		 1106#ifndef CONFIG_TCG_INTERPRETER
1107 /*
1108 * Assert that goto_ptr is implemented completely, setting an epilogue.
1109 * For tci, we use NULL as the signal to return from the interpreter,
1110 * so skip this check.
1111 */
f4e01e30 1112 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 1113#endif
d1c74ab3
RH		 1114
1115 tcg_region_prologue_set(s);
c896fe29
FB		 1116}
1117
c896fe29
FB		 1118void tcg_func_start(TCGContext *s)
1119{
1120 tcg_pool_reset(s);
1121 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 1122
1123 /* No temps have been previously allocated for size or locality. */
1124 memset(s->free_temps, 0, sizeof(s->free_temps));
1125
c0522136
RH		 1126 /* No constant temps have been previously allocated. */
1127 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1128 if (s->const_table[i]) {
1129 g_hash_table_remove_all(s->const_table[i]);
1130 }
1131 }
1132
abebf925 1133 s->nb_ops = 0;
c896fe29
FB		 1134 s->nb_labels = 0;
1135 s->current_frame_offset = s->frame_start;
1136
0a209d4b
RH		 1137#ifdef CONFIG_DEBUG_TCG
1138 s->goto_tb_issue_mask = 0;
1139#endif
1140
15fa08f8
RH		 1141 QTAILQ_INIT(&s->ops);
1142 QTAILQ_INIT(&s->free_ops);
bef16ab4 1143 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 1144}
1145
ae30e866 1146static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 1147{
1148 int n = s->nb_temps++;
ae30e866
RH		 1149
1150 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 1151 tcg_raise_tb_overflow(s);
ae30e866 1152 }
7ca4b752
RH		 1153 return memset(&s->temps[n], 0, sizeof(TCGTemp));
1154}
1155
ae30e866 1156static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 1157{
fa477d25
RH		 1158 TCGTemp *ts;
1159
7ca4b752 1160 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 1161 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 1162 s->nb_globals++;
fa477d25 1163 ts = tcg_temp_alloc(s);
ee17db83 1164 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 1165
1166 return ts;
c896fe29
FB		 1167}
1168
085272b3
RH		 1169static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1170 TCGReg reg, const char *name)
c896fe29 1171{
c896fe29 1172 TCGTemp *ts;
c896fe29 1173
b3a62939 1174 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 1175 tcg_abort();
b3a62939 1176 }
7ca4b752
RH		 1177
1178 ts = tcg_global_alloc(s);
c896fe29
FB		 1179 ts->base_type = type;
1180 ts->type = type;
ee17db83 1181 ts->kind = TEMP_FIXED;
c896fe29 1182 ts->reg = reg;
c896fe29 1183 ts->name = name;
c896fe29 1184 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 1185
085272b3 1186 return ts;
a7812ae4
PB		 1187}
1188
b6638662 1189void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 1190{
b3a62939
RH		 1191 s->frame_start = start;
1192 s->frame_end = start + size;
085272b3
RH		 1193 s->frame_temp
1194 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 1195}
1196
085272b3
RH		 1197TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
1198 intptr_t offset, const char *name)
c896fe29 1199{
b1311c4a 1200 TCGContext *s = tcg_ctx;
dc41aa7d 1201 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 1202 TCGTemp *ts = tcg_global_alloc(s);
aef85402 1203 int indirect_reg = 0;
c896fe29 1204
c0522136
RH		 1205 switch (base_ts->kind) {
1206 case TEMP_FIXED:
1207 break;
1208 case TEMP_GLOBAL:
5a18407f
RH		 1209 /* We do not support double-indirect registers. */
1210 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 1211 base_ts->indirect_base = 1;
5a18407f
RH		 1212 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1213 ? 2 : 1);
1214 indirect_reg = 1;
c0522136
RH		 1215 break;
1216 default:
1217 g_assert_not_reached();
b3915dbb
RH		 1218 }
1219
7ca4b752
RH		 1220 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1221 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 1222 char buf[64];
7ca4b752
RH		 1223
1224 ts->base_type = TCG_TYPE_I64;
c896fe29 1225 ts->type = TCG_TYPE_I32;
b3915dbb 1226 ts->indirect_reg = indirect_reg;
c896fe29 1227 ts->mem_allocated = 1;
b3a62939 1228 ts->mem_base = base_ts;
aef85402 1229 ts->mem_offset = offset;
c896fe29
FB		 1230 pstrcpy(buf, sizeof(buf), name);
1231 pstrcat(buf, sizeof(buf), "_0");
1232 ts->name = strdup(buf);
c896fe29 1233
7ca4b752
RH		 1234 tcg_debug_assert(ts2 == ts + 1);
1235 ts2->base_type = TCG_TYPE_I64;
1236 ts2->type = TCG_TYPE_I32;
b3915dbb 1237 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 1238 ts2->mem_allocated = 1;
1239 ts2->mem_base = base_ts;
aef85402 1240 ts2->mem_offset = offset + 4;
fac87bd2 1241 ts2->temp_subindex = 1;
c896fe29
FB		 1242 pstrcpy(buf, sizeof(buf), name);
1243 pstrcat(buf, sizeof(buf), "_1");
120c1084 1244 ts2->name = strdup(buf);
7ca4b752 1245 } else {
c896fe29
FB		 1246 ts->base_type = type;
1247 ts->type = type;
b3915dbb 1248 ts->indirect_reg = indirect_reg;
c896fe29 1249 ts->mem_allocated = 1;
b3a62939 1250 ts->mem_base = base_ts;
c896fe29 1251 ts->mem_offset = offset;
c896fe29 1252 ts->name = name;
c896fe29 1253 }
085272b3 1254 return ts;
a7812ae4
PB		 1255}
1256
bbf989bf 1257TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
c896fe29 1258{
b1311c4a 1259 TCGContext *s = tcg_ctx;
bbf989bf 1260 bool temp_local = kind == TEMP_TB;
c896fe29 1261 TCGTemp *ts;
641d5fbe 1262 int idx, k;
c896fe29 1263
0ec9eabc
RH		 1264 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
1265 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
1266 if (idx < TCG_MAX_TEMPS) {
1267 /* There is already an available temp with the right type. */
1268 clear_bit(idx, s->free_temps[k].l);
1269
e8996ee0 1270 ts = &s->temps[idx];
e8996ee0 1271 ts->temp_allocated = 1;
7ca4b752 1272 tcg_debug_assert(ts->base_type == type);
ee17db83 1273 tcg_debug_assert(ts->kind == kind);
e8996ee0 1274 } else {
43eef72f 1275 int i, n;
7ca4b752 1276
43eef72f
RH		 1277 switch (type) {
1278 case TCG_TYPE_I32:
1279 case TCG_TYPE_V64:
1280 case TCG_TYPE_V128:
1281 case TCG_TYPE_V256:
1282 n = 1;
1283 break;
1284 case TCG_TYPE_I64:
1285 n = 64 / TCG_TARGET_REG_BITS;
1286 break;
1287 case TCG_TYPE_I128:
1288 n = 128 / TCG_TARGET_REG_BITS;
1289 break;
1290 default:
1291 g_assert_not_reached();
1292 }
7ca4b752 1293
43eef72f
RH		 1294 ts = tcg_temp_alloc(s);
1295 ts->base_type = type;
1296 ts->temp_allocated = 1;
1297 ts->kind = kind;
1298
1299 if (n == 1) {
e8996ee0 1300 ts->type = type;
43eef72f
RH		 1301 } else {
1302 ts->type = TCG_TYPE_REG;
1303
1304 for (i = 1; i < n; ++i) {
1305 TCGTemp *ts2 = tcg_temp_alloc(s);
1306
1307 tcg_debug_assert(ts2 == ts + i);
1308 ts2->base_type = type;
1309 ts2->type = TCG_TYPE_REG;
1310 ts2->temp_allocated = 1;
1311 ts2->temp_subindex = i;
1312 ts2->kind = kind;
1313 }
e8996ee0 1314 }
c896fe29 1315 }
27bfd83c
PM		 1316
1317#if defined(CONFIG_DEBUG_TCG)
1318 s->temps_in_use++;
1319#endif
085272b3 1320 return ts;
c896fe29
FB		 1321}
1322
d2fd745f
RH		 1323TCGv_vec tcg_temp_new_vec(TCGType type)
1324{
1325 TCGTemp *t;
1326
1327#ifdef CONFIG_DEBUG_TCG
1328 switch (type) {
1329 case TCG_TYPE_V64:
1330 assert(TCG_TARGET_HAS_v64);
1331 break;
1332 case TCG_TYPE_V128:
1333 assert(TCG_TARGET_HAS_v128);
1334 break;
1335 case TCG_TYPE_V256:
1336 assert(TCG_TARGET_HAS_v256);
1337 break;
1338 default:
1339 g_assert_not_reached();
1340 }
1341#endif
1342
bbf989bf 1343 t = tcg_temp_new_internal(type, TEMP_EBB);
d2fd745f
RH		 1344 return temp_tcgv_vec(t);
1345}
1346
1347/* Create a new temp of the same type as an existing temp. */
1348TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1349{
1350 TCGTemp *t = tcgv_vec_temp(match);
1351
1352 tcg_debug_assert(t->temp_allocated != 0);
1353
bbf989bf 1354 t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
d2fd745f
RH		 1355 return temp_tcgv_vec(t);
1356}
1357
5bfa8034 1358void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1359{
b1311c4a 1360 TCGContext *s = tcg_ctx;
085272b3 1361 int k, idx;
c896fe29 1362
c7482438
RH		 1363 switch (ts->kind) {
1364 case TEMP_CONST:
1365 /*
1366 * In order to simplify users of tcg_constant_*,
1367 * silently ignore free.
1368 */
c0522136 1369 return;
0218e040 1370 case TEMP_EBB:
f57c6915 1371 case TEMP_TB:
c7482438
RH		 1372 break;
1373 default:
1374 g_assert_not_reached();
c0522136
RH		 1375 }
1376
0ef3d704
RH		 1377 tcg_debug_assert(ts->temp_allocated != 0);
1378 ts->temp_allocated = 0;
1379
27bfd83c 1380#if defined(CONFIG_DEBUG_TCG)
0ef3d704 1381 assert(s->temps_in_use > 0);
27bfd83c 1382 s->temps_in_use--;
27bfd83c
PM		 1383#endif
1384
085272b3 1385 idx = temp_idx(ts);
0218e040 1386 k = ts->base_type + (ts->kind == TEMP_EBB ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1387 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1388}
1389
c0522136
RH		 1390TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1391{
1392 TCGContext *s = tcg_ctx;
1393 GHashTable *h = s->const_table[type];
1394 TCGTemp *ts;
1395
1396 if (h == NULL) {
1397 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1398 s->const_table[type] = h;
1399 }
1400
1401 ts = g_hash_table_lookup(h, &val);
1402 if (ts == NULL) {
aef85402
RH		 1403 int64_t *val_ptr;
1404
c0522136
RH		 1405 ts = tcg_temp_alloc(s);
1406
1407 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1408 TCGTemp *ts2 = tcg_temp_alloc(s);
1409
aef85402
RH		 1410 tcg_debug_assert(ts2 == ts + 1);
1411
c0522136
RH		 1412 ts->base_type = TCG_TYPE_I64;
1413 ts->type = TCG_TYPE_I32;
1414 ts->kind = TEMP_CONST;
1415 ts->temp_allocated = 1;
c0522136 1416
c0522136
RH		 1417 ts2->base_type = TCG_TYPE_I64;
1418 ts2->type = TCG_TYPE_I32;
1419 ts2->kind = TEMP_CONST;
1420 ts2->temp_allocated = 1;
fac87bd2 1421 ts2->temp_subindex = 1;
aef85402
RH		 1422
1423 /*
1424 * Retain the full value of the 64-bit constant in the low
1425 * part, so that the hash table works. Actual uses will
1426 * truncate the value to the low part.
1427 */
1428 ts[HOST_BIG_ENDIAN].val = val;
1429 ts[!HOST_BIG_ENDIAN].val = val >> 32;
1430 val_ptr = &ts[HOST_BIG_ENDIAN].val;
c0522136
RH		 1431 } else {
1432 ts->base_type = type;
1433 ts->type = type;
1434 ts->kind = TEMP_CONST;
1435 ts->temp_allocated = 1;
1436 ts->val = val;
aef85402 1437 val_ptr = &ts->val;
c0522136 1438 }
aef85402 1439 g_hash_table_insert(h, val_ptr, ts);
c0522136
RH		 1440 }
1441
1442 return ts;
1443}
1444
1445TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1446{
1447 val = dup_const(vece, val);
1448 return temp_tcgv_vec(tcg_constant_internal(type, val));
1449}
1450
88d4005b
RH		 1451TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1452{
1453 TCGTemp *t = tcgv_vec_temp(match);
1454
1455 tcg_debug_assert(t->temp_allocated != 0);
1456 return tcg_constant_vec(t->base_type, vece, val);
1457}
1458
a7812ae4 1459TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1460{
a7812ae4
PB		 1461 TCGv_i32 t0;
1462 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1463 tcg_gen_movi_i32(t0, val);
1464 return t0;
1465}
c896fe29 1466
a7812ae4 1467TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1468{
a7812ae4
PB		 1469 TCGv_i64 t0;
1470 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1471 tcg_gen_movi_i64(t0, val);
1472 return t0;
c896fe29
FB		 1473}
1474
a7812ae4 1475TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1476{
a7812ae4
PB		 1477 TCGv_i32 t0;
1478 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1479 tcg_gen_movi_i32(t0, val);
1480 return t0;
1481}
1482
a7812ae4 1483TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1484{
a7812ae4
PB		 1485 TCGv_i64 t0;
1486 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1487 tcg_gen_movi_i64(t0, val);
1488 return t0;
1489}
1490
27bfd83c
PM		 1491#if defined(CONFIG_DEBUG_TCG)
1492void tcg_clear_temp_count(void)
1493{
b1311c4a 1494 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1495 s->temps_in_use = 0;
1496}
1497
1498int tcg_check_temp_count(void)
1499{
b1311c4a 1500 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1501 if (s->temps_in_use) {
1502 /* Clear the count so that we don't give another
1503 * warning immediately next time around.
1504 */
1505 s->temps_in_use = 0;
1506 return 1;
1507 }
1508 return 0;
1509}
1510#endif
1511
be0f34b5
RH		 1512/* Return true if OP may appear in the opcode stream.
1513 Test the runtime variable that controls each opcode. */
1514bool tcg_op_supported(TCGOpcode op)
1515{
d2fd745f
RH		 1516 const bool have_vec
1517 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1518
be0f34b5
RH		 1519 switch (op) {
1520 case INDEX_op_discard:
1521 case INDEX_op_set_label:
1522 case INDEX_op_call:
1523 case INDEX_op_br:
1524 case INDEX_op_mb:
1525 case INDEX_op_insn_start:
1526 case INDEX_op_exit_tb:
1527 case INDEX_op_goto_tb:
f4e01e30 1528 case INDEX_op_goto_ptr:
be0f34b5
RH		 1529 case INDEX_op_qemu_ld_i32:
1530 case INDEX_op_qemu_st_i32:
1531 case INDEX_op_qemu_ld_i64:
1532 case INDEX_op_qemu_st_i64:
1533 return true;
1534
07ce0b05
RH		 1535 case INDEX_op_qemu_st8_i32:
1536 return TCG_TARGET_HAS_qemu_st8_i32;
1537
be0f34b5 1538 case INDEX_op_mov_i32:
be0f34b5
RH		 1539 case INDEX_op_setcond_i32:
1540 case INDEX_op_brcond_i32:
1541 case INDEX_op_ld8u_i32:
1542 case INDEX_op_ld8s_i32:
1543 case INDEX_op_ld16u_i32:
1544 case INDEX_op_ld16s_i32:
1545 case INDEX_op_ld_i32:
1546 case INDEX_op_st8_i32:
1547 case INDEX_op_st16_i32:
1548 case INDEX_op_st_i32:
1549 case INDEX_op_add_i32:
1550 case INDEX_op_sub_i32:
1551 case INDEX_op_mul_i32:
1552 case INDEX_op_and_i32:
1553 case INDEX_op_or_i32:
1554 case INDEX_op_xor_i32:
1555 case INDEX_op_shl_i32:
1556 case INDEX_op_shr_i32:
1557 case INDEX_op_sar_i32:
1558 return true;
1559
1560 case INDEX_op_movcond_i32:
1561 return TCG_TARGET_HAS_movcond_i32;
1562 case INDEX_op_div_i32:
1563 case INDEX_op_divu_i32:
1564 return TCG_TARGET_HAS_div_i32;
1565 case INDEX_op_rem_i32:
1566 case INDEX_op_remu_i32:
1567 return TCG_TARGET_HAS_rem_i32;
1568 case INDEX_op_div2_i32:
1569 case INDEX_op_divu2_i32:
1570 return TCG_TARGET_HAS_div2_i32;
1571 case INDEX_op_rotl_i32:
1572 case INDEX_op_rotr_i32:
1573 return TCG_TARGET_HAS_rot_i32;
1574 case INDEX_op_deposit_i32:
1575 return TCG_TARGET_HAS_deposit_i32;
1576 case INDEX_op_extract_i32:
1577 return TCG_TARGET_HAS_extract_i32;
1578 case INDEX_op_sextract_i32:
1579 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1580 case INDEX_op_extract2_i32:
1581 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1582 case INDEX_op_add2_i32:
1583 return TCG_TARGET_HAS_add2_i32;
1584 case INDEX_op_sub2_i32:
1585 return TCG_TARGET_HAS_sub2_i32;
1586 case INDEX_op_mulu2_i32:
1587 return TCG_TARGET_HAS_mulu2_i32;
1588 case INDEX_op_muls2_i32:
1589 return TCG_TARGET_HAS_muls2_i32;
1590 case INDEX_op_muluh_i32:
1591 return TCG_TARGET_HAS_muluh_i32;
1592 case INDEX_op_mulsh_i32:
1593 return TCG_TARGET_HAS_mulsh_i32;
1594 case INDEX_op_ext8s_i32:
1595 return TCG_TARGET_HAS_ext8s_i32;
1596 case INDEX_op_ext16s_i32:
1597 return TCG_TARGET_HAS_ext16s_i32;
1598 case INDEX_op_ext8u_i32:
1599 return TCG_TARGET_HAS_ext8u_i32;
1600 case INDEX_op_ext16u_i32:
1601 return TCG_TARGET_HAS_ext16u_i32;
1602 case INDEX_op_bswap16_i32:
1603 return TCG_TARGET_HAS_bswap16_i32;
1604 case INDEX_op_bswap32_i32:
1605 return TCG_TARGET_HAS_bswap32_i32;
1606 case INDEX_op_not_i32:
1607 return TCG_TARGET_HAS_not_i32;
1608 case INDEX_op_neg_i32:
1609 return TCG_TARGET_HAS_neg_i32;
1610 case INDEX_op_andc_i32:
1611 return TCG_TARGET_HAS_andc_i32;
1612 case INDEX_op_orc_i32:
1613 return TCG_TARGET_HAS_orc_i32;
1614 case INDEX_op_eqv_i32:
1615 return TCG_TARGET_HAS_eqv_i32;
1616 case INDEX_op_nand_i32:
1617 return TCG_TARGET_HAS_nand_i32;
1618 case INDEX_op_nor_i32:
1619 return TCG_TARGET_HAS_nor_i32;
1620 case INDEX_op_clz_i32:
1621 return TCG_TARGET_HAS_clz_i32;
1622 case INDEX_op_ctz_i32:
1623 return TCG_TARGET_HAS_ctz_i32;
1624 case INDEX_op_ctpop_i32:
1625 return TCG_TARGET_HAS_ctpop_i32;
1626
1627 case INDEX_op_brcond2_i32:
1628 case INDEX_op_setcond2_i32:
1629 return TCG_TARGET_REG_BITS == 32;
1630
1631 case INDEX_op_mov_i64:
be0f34b5
RH		 1632 case INDEX_op_setcond_i64:
1633 case INDEX_op_brcond_i64:
1634 case INDEX_op_ld8u_i64:
1635 case INDEX_op_ld8s_i64:
1636 case INDEX_op_ld16u_i64:
1637 case INDEX_op_ld16s_i64:
1638 case INDEX_op_ld32u_i64:
1639 case INDEX_op_ld32s_i64:
1640 case INDEX_op_ld_i64:
1641 case INDEX_op_st8_i64:
1642 case INDEX_op_st16_i64:
1643 case INDEX_op_st32_i64:
1644 case INDEX_op_st_i64:
1645 case INDEX_op_add_i64:
1646 case INDEX_op_sub_i64:
1647 case INDEX_op_mul_i64:
1648 case INDEX_op_and_i64:
1649 case INDEX_op_or_i64:
1650 case INDEX_op_xor_i64:
1651 case INDEX_op_shl_i64:
1652 case INDEX_op_shr_i64:
1653 case INDEX_op_sar_i64:
1654 case INDEX_op_ext_i32_i64:
1655 case INDEX_op_extu_i32_i64:
1656 return TCG_TARGET_REG_BITS == 64;
1657
1658 case INDEX_op_movcond_i64:
1659 return TCG_TARGET_HAS_movcond_i64;
1660 case INDEX_op_div_i64:
1661 case INDEX_op_divu_i64:
1662 return TCG_TARGET_HAS_div_i64;
1663 case INDEX_op_rem_i64:
1664 case INDEX_op_remu_i64:
1665 return TCG_TARGET_HAS_rem_i64;
1666 case INDEX_op_div2_i64:
1667 case INDEX_op_divu2_i64:
1668 return TCG_TARGET_HAS_div2_i64;
1669 case INDEX_op_rotl_i64:
1670 case INDEX_op_rotr_i64:
1671 return TCG_TARGET_HAS_rot_i64;
1672 case INDEX_op_deposit_i64:
1673 return TCG_TARGET_HAS_deposit_i64;
1674 case INDEX_op_extract_i64:
1675 return TCG_TARGET_HAS_extract_i64;
1676 case INDEX_op_sextract_i64:
1677 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1678 case INDEX_op_extract2_i64:
1679 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1680 case INDEX_op_extrl_i64_i32:
1681 return TCG_TARGET_HAS_extrl_i64_i32;
1682 case INDEX_op_extrh_i64_i32:
1683 return TCG_TARGET_HAS_extrh_i64_i32;
1684 case INDEX_op_ext8s_i64:
1685 return TCG_TARGET_HAS_ext8s_i64;
1686 case INDEX_op_ext16s_i64:
1687 return TCG_TARGET_HAS_ext16s_i64;
1688 case INDEX_op_ext32s_i64:
1689 return TCG_TARGET_HAS_ext32s_i64;
1690 case INDEX_op_ext8u_i64:
1691 return TCG_TARGET_HAS_ext8u_i64;
1692 case INDEX_op_ext16u_i64:
1693 return TCG_TARGET_HAS_ext16u_i64;
1694 case INDEX_op_ext32u_i64:
1695 return TCG_TARGET_HAS_ext32u_i64;
1696 case INDEX_op_bswap16_i64:
1697 return TCG_TARGET_HAS_bswap16_i64;
1698 case INDEX_op_bswap32_i64:
1699 return TCG_TARGET_HAS_bswap32_i64;
1700 case INDEX_op_bswap64_i64:
1701 return TCG_TARGET_HAS_bswap64_i64;
1702 case INDEX_op_not_i64:
1703 return TCG_TARGET_HAS_not_i64;
1704 case INDEX_op_neg_i64:
1705 return TCG_TARGET_HAS_neg_i64;
1706 case INDEX_op_andc_i64:
1707 return TCG_TARGET_HAS_andc_i64;
1708 case INDEX_op_orc_i64:
1709 return TCG_TARGET_HAS_orc_i64;
1710 case INDEX_op_eqv_i64:
1711 return TCG_TARGET_HAS_eqv_i64;
1712 case INDEX_op_nand_i64:
1713 return TCG_TARGET_HAS_nand_i64;
1714 case INDEX_op_nor_i64:
1715 return TCG_TARGET_HAS_nor_i64;
1716 case INDEX_op_clz_i64:
1717 return TCG_TARGET_HAS_clz_i64;
1718 case INDEX_op_ctz_i64:
1719 return TCG_TARGET_HAS_ctz_i64;
1720 case INDEX_op_ctpop_i64:
1721 return TCG_TARGET_HAS_ctpop_i64;
1722 case INDEX_op_add2_i64:
1723 return TCG_TARGET_HAS_add2_i64;
1724 case INDEX_op_sub2_i64:
1725 return TCG_TARGET_HAS_sub2_i64;
1726 case INDEX_op_mulu2_i64:
1727 return TCG_TARGET_HAS_mulu2_i64;
1728 case INDEX_op_muls2_i64:
1729 return TCG_TARGET_HAS_muls2_i64;
1730 case INDEX_op_muluh_i64:
1731 return TCG_TARGET_HAS_muluh_i64;
1732 case INDEX_op_mulsh_i64:
1733 return TCG_TARGET_HAS_mulsh_i64;
1734
d2fd745f
RH		 1735 case INDEX_op_mov_vec:
1736 case INDEX_op_dup_vec:
37ee55a0 1737 case INDEX_op_dupm_vec:
d2fd745f
RH		 1738 case INDEX_op_ld_vec:
1739 case INDEX_op_st_vec:
1740 case INDEX_op_add_vec:
1741 case INDEX_op_sub_vec:
1742 case INDEX_op_and_vec:
1743 case INDEX_op_or_vec:
1744 case INDEX_op_xor_vec:
212be173 1745 case INDEX_op_cmp_vec:
d2fd745f
RH		 1746 return have_vec;
1747 case INDEX_op_dup2_vec:
1748 return have_vec && TCG_TARGET_REG_BITS == 32;
1749 case INDEX_op_not_vec:
1750 return have_vec && TCG_TARGET_HAS_not_vec;
1751 case INDEX_op_neg_vec:
1752 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1753 case INDEX_op_abs_vec:
1754 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1755 case INDEX_op_andc_vec:
1756 return have_vec && TCG_TARGET_HAS_andc_vec;
1757 case INDEX_op_orc_vec:
1758 return have_vec && TCG_TARGET_HAS_orc_vec;
ed523473
RH		 1759 case INDEX_op_nand_vec:
1760 return have_vec && TCG_TARGET_HAS_nand_vec;
1761 case INDEX_op_nor_vec:
1762 return have_vec && TCG_TARGET_HAS_nor_vec;
1763 case INDEX_op_eqv_vec:
1764 return have_vec && TCG_TARGET_HAS_eqv_vec;
3774030a
RH		 1765 case INDEX_op_mul_vec:
1766 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1767 case INDEX_op_shli_vec:
1768 case INDEX_op_shri_vec:
1769 case INDEX_op_sari_vec:
1770 return have_vec && TCG_TARGET_HAS_shi_vec;
1771 case INDEX_op_shls_vec:
1772 case INDEX_op_shrs_vec:
1773 case INDEX_op_sars_vec:
1774 return have_vec && TCG_TARGET_HAS_shs_vec;
1775 case INDEX_op_shlv_vec:
1776 case INDEX_op_shrv_vec:
1777 case INDEX_op_sarv_vec:
1778 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1779 case INDEX_op_rotli_vec:
1780 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1781 case INDEX_op_rotls_vec:
1782 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1783 case INDEX_op_rotlv_vec:
1784 case INDEX_op_rotrv_vec:
1785 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1786 case INDEX_op_ssadd_vec:
1787 case INDEX_op_usadd_vec:
1788 case INDEX_op_sssub_vec:
1789 case INDEX_op_ussub_vec:
1790 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1791 case INDEX_op_smin_vec:
1792 case INDEX_op_umin_vec:
1793 case INDEX_op_smax_vec:
1794 case INDEX_op_umax_vec:
1795 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1796 case INDEX_op_bitsel_vec:
1797 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1798 case INDEX_op_cmpsel_vec:
1799 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1800
db432672
RH		 1801 default:
1802 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1803 return true;
be0f34b5 1804 }
be0f34b5
RH		 1805}
1806
39004a71
RH		 1807static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
1808
ae8b75dc 1809void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1810{
3e92aa34 1811 const TCGHelperInfo *info;
39004a71
RH		 1812 TCGv_i64 extend_free[MAX_CALL_IARGS];
1813 int n_extend = 0;
75e8b9b7 1814 TCGOp *op;
39004a71 1815 int i, n, pi = 0, total_args;
afb49896 1816
619205fd 1817 info = g_hash_table_lookup(helper_table, (gpointer)func);
39004a71
RH		 1818 total_args = info->nr_out + info->nr_in + 2;
1819 op = tcg_op_alloc(INDEX_op_call, total_args);
2bece2c8 1820
38b47b19 1821#ifdef CONFIG_PLUGIN
17083f6f
EC		 1822 /* Flag helpers that may affect guest state */
1823 if (tcg_ctx->plugin_insn &&
1824 !(info->flags & TCG_CALL_PLUGIN) &&
1825 !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
38b47b19
EC		 1826 tcg_ctx->plugin_insn->calls_helpers = true;
1827 }
1828#endif
1829
39004a71
RH		 1830 TCGOP_CALLO(op) = n = info->nr_out;
1831 switch (n) {
1832 case 0:
1833 tcg_debug_assert(ret == NULL);
1834 break;
1835 case 1:
1836 tcg_debug_assert(ret != NULL);
1837 op->args[pi++] = temp_arg(ret);
1838 break;
1839 case 2:
466d3759 1840 case 4:
39004a71 1841 tcg_debug_assert(ret != NULL);
466d3759 1842 tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
39004a71 1843 tcg_debug_assert(ret->temp_subindex == 0);
466d3759
RH		 1844 for (i = 0; i < n; ++i) {
1845 op->args[pi++] = temp_arg(ret + i);
1846 }
39004a71
RH		 1847 break;
1848 default:
1849 g_assert_not_reached();
1850 }
1851
1852 TCGOP_CALLI(op) = n = info->nr_in;
1853 for (i = 0; i < n; i++) {
1854 const TCGCallArgumentLoc *loc = &info->in[i];
1855 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
1856
1857 switch (loc->kind) {
1858 case TCG_CALL_ARG_NORMAL:
313bdea8
RH		 1859 case TCG_CALL_ARG_BY_REF:
1860 case TCG_CALL_ARG_BY_REF_N:
39004a71
RH		 1861 op->args[pi++] = temp_arg(ts);
1862 break;
eb8b0224 1863
39004a71
RH		 1864 case TCG_CALL_ARG_EXTEND_U:
1865 case TCG_CALL_ARG_EXTEND_S:
1866 {
5dd48602 1867 TCGv_i64 temp = tcg_temp_ebb_new_i64();
39004a71
RH		 1868 TCGv_i32 orig = temp_tcgv_i32(ts);
1869
1870 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
eb8b0224
RH		 1871 tcg_gen_ext_i32_i64(temp, orig);
1872 } else {
1873 tcg_gen_extu_i32_i64(temp, orig);
1874 }
39004a71
RH		 1875 op->args[pi++] = tcgv_i64_arg(temp);
1876 extend_free[n_extend++] = temp;
2bece2c8 1877 }
e2a9dd6b 1878 break;
7b7d8b2d 1879
e2a9dd6b
RH		 1880 default:
1881 g_assert_not_reached();
c896fe29
FB		 1882 }
1883 }
75e8b9b7 1884 op->args[pi++] = (uintptr_t)func;
3e92aa34 1885 op->args[pi++] = (uintptr_t)info;
39004a71 1886 tcg_debug_assert(pi == total_args);
a7812ae4 1887
39004a71 1888 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
7319d83a 1889
39004a71
RH		 1890 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
1891 for (i = 0; i < n_extend; ++i) {
1892 tcg_temp_free_i64(extend_free[i]);
2bece2c8 1893 }
c896fe29 1894}
c896fe29 1895
8fcd3692 1896static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1897{
ac3b8891 1898 int i, n;
ac3b8891 1899
ee17db83
RH		 1900 for (i = 0, n = s->nb_temps; i < n; i++) {
1901 TCGTemp *ts = &s->temps[i];
1902 TCGTempVal val = TEMP_VAL_MEM;
1903
1904 switch (ts->kind) {
c0522136
RH		 1905 case TEMP_CONST:
1906 val = TEMP_VAL_CONST;
1907 break;
ee17db83
RH		 1908 case TEMP_FIXED:
1909 val = TEMP_VAL_REG;
1910 break;
1911 case TEMP_GLOBAL:
1912 break;
c7482438 1913 case TEMP_EBB:
ee17db83
RH		 1914 val = TEMP_VAL_DEAD;
1915 /* fall through */
f57c6915 1916 case TEMP_TB:
ee17db83
RH		 1917 ts->mem_allocated = 0;
1918 break;
1919 default:
1920 g_assert_not_reached();
1921 }
1922 ts->val_type = val;
e8996ee0 1923 }
f8b2f202
RH		 1924
1925 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1926}
1927
f8b2f202
RH		 1928static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1929 TCGTemp *ts)
c896fe29 1930{
1807f4c4 1931 int idx = temp_idx(ts);
ac56dd48 1932
ee17db83
RH		 1933 switch (ts->kind) {
1934 case TEMP_FIXED:
1935 case TEMP_GLOBAL:
ac56dd48 1936 pstrcpy(buf, buf_size, ts->name);
ee17db83 1937 break;
f57c6915 1938 case TEMP_TB:
f8b2f202 1939 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83 1940 break;
c7482438 1941 case TEMP_EBB:
f8b2f202 1942 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1943 break;
c0522136
RH		 1944 case TEMP_CONST:
1945 switch (ts->type) {
1946 case TCG_TYPE_I32:
1947 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1948 break;
1949#if TCG_TARGET_REG_BITS > 32
1950 case TCG_TYPE_I64:
1951 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1952 break;
1953#endif
1954 case TCG_TYPE_V64:
1955 case TCG_TYPE_V128:
1956 case TCG_TYPE_V256:
1957 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1958 64 << (ts->type - TCG_TYPE_V64), ts->val);
1959 break;
1960 default:
1961 g_assert_not_reached();
1962 }
1963 break;
c896fe29
FB		 1964 }
1965 return buf;
1966}
1967
43439139
RH		 1968static char *tcg_get_arg_str(TCGContext *s, char *buf,
1969 int buf_size, TCGArg arg)
f8b2f202 1970{
43439139 1971 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1972}
1973
f48f3ede
BS		 1974static const char * const cond_name[] =
1975{
0aed257f
RH		 1976 [TCG_COND_NEVER] = "never",
1977 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1978 [TCG_COND_EQ] = "eq",
1979 [TCG_COND_NE] = "ne",
1980 [TCG_COND_LT] = "lt",
1981 [TCG_COND_GE] = "ge",
1982 [TCG_COND_LE] = "le",
1983 [TCG_COND_GT] = "gt",
1984 [TCG_COND_LTU] = "ltu",
1985 [TCG_COND_GEU] = "geu",
1986 [TCG_COND_LEU] = "leu",
1987 [TCG_COND_GTU] = "gtu"
1988};
1989
f713d6ad
RH		 1990static const char * const ldst_name[] =
1991{
1992 [MO_UB] = "ub",
1993 [MO_SB] = "sb",
1994 [MO_LEUW] = "leuw",
1995 [MO_LESW] = "lesw",
1996 [MO_LEUL] = "leul",
1997 [MO_LESL] = "lesl",
fc313c64 1998 [MO_LEUQ] = "leq",
f713d6ad
RH		 1999 [MO_BEUW] = "beuw",
2000 [MO_BESW] = "besw",
2001 [MO_BEUL] = "beul",
2002 [MO_BESL] = "besl",
fc313c64 2003 [MO_BEUQ] = "beq",
f713d6ad
RH		 2004};
2005
1f00b27f 2006static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 2007#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 2008 [MO_UNALN >> MO_ASHIFT] = "un+",
2009 [MO_ALIGN >> MO_ASHIFT] = "",
2010#else
2011 [MO_UNALN >> MO_ASHIFT] = "",
2012 [MO_ALIGN >> MO_ASHIFT] = "al+",
2013#endif
2014 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
2015 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
2016 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
2017 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
2018 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
2019 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
2020};
2021
587195bd
RH		 2022static const char bswap_flag_name[][6] = {
2023 [TCG_BSWAP_IZ] = "iz",
2024 [TCG_BSWAP_OZ] = "oz",
2025 [TCG_BSWAP_OS] = "os",
2026 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
2027 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
2028};
2029
b016486e
RH		 2030static inline bool tcg_regset_single(TCGRegSet d)
2031{
2032 return (d & (d - 1)) == 0;
2033}
2034
2035static inline TCGReg tcg_regset_first(TCGRegSet d)
2036{
2037 if (TCG_TARGET_NB_REGS <= 32) {
2038 return ctz32(d);
2039 } else {
2040 return ctz64(d);
2041 }
2042}
2043
b7a83ff8
RH		 2044/* Return only the number of characters output -- no error return. */
2045#define ne_fprintf(...) \
2046 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
2047
2048static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
c896fe29 2049{
c896fe29 2050 char buf[128];
c45cb8bb 2051 TCGOp *op;
c45cb8bb 2052
15fa08f8 2053 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 2054 int i, k, nb_oargs, nb_iargs, nb_cargs;
2055 const TCGOpDef *def;
c45cb8bb 2056 TCGOpcode c;
bdfb460e 2057 int col = 0;
c896fe29 2058
c45cb8bb 2059 c = op->opc;
c896fe29 2060 def = &tcg_op_defs[c];
c45cb8bb 2061
765b842a 2062 if (c == INDEX_op_insn_start) {
b016486e 2063 nb_oargs = 0;
b7a83ff8 2064 col += ne_fprintf(f, "\n ----");
9aef40ed
RH		 2065
2066 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
2067 target_ulong a;
7e4597d7 2068#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 2069 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 2070#else
efee3746 2071 a = op->args[i];
7e4597d7 2072#endif
b7a83ff8 2073 col += ne_fprintf(f, " " TARGET_FMT_lx, a);
eeacee4d 2074 }
7e4597d7 2075 } else if (c == INDEX_op_call) {
3e92aa34 2076 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 2077 void *func = tcg_call_func(op);
3e92aa34 2078
c896fe29 2079 /* variable number of arguments */
cd9090aa
RH		 2080 nb_oargs = TCGOP_CALLO(op);
2081 nb_iargs = TCGOP_CALLI(op);
c896fe29 2082 nb_cargs = def->nb_cargs;
c896fe29 2083
b7a83ff8 2084 col += ne_fprintf(f, " %s ", def->name);
3e92aa34
RH		 2085
2086 /*
2087 * Print the function name from TCGHelperInfo, if available.
2088 * Note that plugins have a template function for the info,
2089 * but the actual function pointer comes from the plugin.
2090 */
3e92aa34 2091 if (func == info->func) {
b7a83ff8 2092 col += ne_fprintf(f, "%s", info->name);
3e92aa34 2093 } else {
b7a83ff8 2094 col += ne_fprintf(f, "plugin(%p)", func);
3e92aa34
RH		 2095 }
2096
b7a83ff8 2097 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
cf066674 2098 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 2099 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2100 op->args[i]));
b03cce8e 2101 }
cf066674 2102 for (i = 0; i < nb_iargs; i++) {
efee3746 2103 TCGArg arg = op->args[nb_oargs + i];
39004a71 2104 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
b7a83ff8 2105 col += ne_fprintf(f, ",%s", t);
e8996ee0 2106 }
b03cce8e 2107 } else {
b7a83ff8 2108 col += ne_fprintf(f, " %s ", def->name);
c45cb8bb
RH		 2109
2110 nb_oargs = def->nb_oargs;
2111 nb_iargs = def->nb_iargs;
2112 nb_cargs = def->nb_cargs;
2113
d2fd745f 2114 if (def->flags & TCG_OPF_VECTOR) {
b7a83ff8
RH		 2115 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
2116 8 << TCGOP_VECE(op));
d2fd745f
RH		 2117 }
2118
b03cce8e 2119 k = 0;
c45cb8bb 2120 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 2121 const char *sep = k ? "," : "";
2122 col += ne_fprintf(f, "%s%s", sep,
2123 tcg_get_arg_str(s, buf, sizeof(buf),
2124 op->args[k++]));
b03cce8e 2125 }
c45cb8bb 2126 for (i = 0; i < nb_iargs; i++) {
b7a83ff8
RH		 2127 const char *sep = k ? "," : "";
2128 col += ne_fprintf(f, "%s%s", sep,
2129 tcg_get_arg_str(s, buf, sizeof(buf),
2130 op->args[k++]));
b03cce8e 2131 }
be210acb
RH		 2132 switch (c) {
2133 case INDEX_op_brcond_i32:
be210acb 2134 case INDEX_op_setcond_i32:
ffc5ea09 2135 case INDEX_op_movcond_i32:
ffc5ea09 2136 case INDEX_op_brcond2_i32:
be210acb 2137 case INDEX_op_setcond2_i32:
ffc5ea09 2138 case INDEX_op_brcond_i64:
be210acb 2139 case INDEX_op_setcond_i64:
ffc5ea09 2140 case INDEX_op_movcond_i64:
212be173 2141 case INDEX_op_cmp_vec:
f75da298 2142 case INDEX_op_cmpsel_vec:
efee3746
RH		 2143 if (op->args[k] < ARRAY_SIZE(cond_name)
2144 && cond_name[op->args[k]]) {
b7a83ff8 2145 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
eeacee4d 2146 } else {
b7a83ff8 2147 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 2148 }
f48f3ede 2149 i = 1;
be210acb 2150 break;
f713d6ad
RH		 2151 case INDEX_op_qemu_ld_i32:
2152 case INDEX_op_qemu_st_i32:
07ce0b05 2153 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 2154 case INDEX_op_qemu_ld_i64:
2155 case INDEX_op_qemu_st_i64:
59227d5d 2156 {
9002ffcb 2157 MemOpIdx oi = op->args[k++];
14776ab5 2158 MemOp op = get_memop(oi);
59227d5d
RH		 2159 unsigned ix = get_mmuidx(oi);
2160
59c4b7e8 2161 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
b7a83ff8 2162 col += ne_fprintf(f, ",$0x%x,%u", op, ix);
59c4b7e8 2163 } else {
1f00b27f
SS		 2164 const char *s_al, *s_op;
2165 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 2166 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
b7a83ff8 2167 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 2168 }
2169 i = 1;
f713d6ad 2170 }
f713d6ad 2171 break;
587195bd
RH		 2172 case INDEX_op_bswap16_i32:
2173 case INDEX_op_bswap16_i64:
2174 case INDEX_op_bswap32_i32:
2175 case INDEX_op_bswap32_i64:
2176 case INDEX_op_bswap64_i64:
2177 {
2178 TCGArg flags = op->args[k];
2179 const char *name = NULL;
2180
2181 if (flags < ARRAY_SIZE(bswap_flag_name)) {
2182 name = bswap_flag_name[flags];
2183 }
2184 if (name) {
b7a83ff8 2185 col += ne_fprintf(f, ",%s", name);
587195bd 2186 } else {
b7a83ff8 2187 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
587195bd
RH		 2188 }
2189 i = k = 1;
2190 }
2191 break;
be210acb 2192 default:
f48f3ede 2193 i = 0;
be210acb
RH		 2194 break;
2195 }
51e3972c
RH		 2196 switch (c) {
2197 case INDEX_op_set_label:
2198 case INDEX_op_br:
2199 case INDEX_op_brcond_i32:
2200 case INDEX_op_brcond_i64:
2201 case INDEX_op_brcond2_i32:
b7a83ff8
RH		 2202 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
2203 arg_label(op->args[k])->id);
51e3972c
RH		 2204 i++, k++;
2205 break;
2206 default:
2207 break;
2208 }
2209 for (; i < nb_cargs; i++, k++) {
b7a83ff8
RH		 2210 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
2211 op->args[k]);
bdfb460e
RH		 2212 }
2213 }
bdfb460e 2214
1894f69a 2215 if (have_prefs || op->life) {
b7a83ff8
RH		 2216 for (; col < 40; ++col) {
2217 putc(' ', f);
bdfb460e 2218 }
1894f69a
RH		 2219 }
2220
2221 if (op->life) {
2222 unsigned life = op->life;
bdfb460e
RH		 2223
2224 if (life & (SYNC_ARG * 3)) {
b7a83ff8 2225 ne_fprintf(f, " sync:");
bdfb460e
RH		 2226 for (i = 0; i < 2; ++i) {
2227 if (life & (SYNC_ARG << i)) {
b7a83ff8 2228 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2229 }
2230 }
2231 }
2232 life /= DEAD_ARG;
2233 if (life) {
b7a83ff8 2234 ne_fprintf(f, " dead:");
bdfb460e
RH		 2235 for (i = 0; life; ++i, life >>= 1) {
2236 if (life & 1) {
b7a83ff8 2237 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2238 }
2239 }
b03cce8e 2240 }
c896fe29 2241 }
1894f69a
RH		 2242
2243 if (have_prefs) {
2244 for (i = 0; i < nb_oargs; ++i) {
31fd884b 2245 TCGRegSet set = output_pref(op, i);
1894f69a
RH		 2246
2247 if (i == 0) {
b7a83ff8 2248 ne_fprintf(f, " pref=");
1894f69a 2249 } else {
b7a83ff8 2250 ne_fprintf(f, ",");
1894f69a
RH		 2251 }
2252 if (set == 0) {
b7a83ff8 2253 ne_fprintf(f, "none");
1894f69a 2254 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
b7a83ff8 2255 ne_fprintf(f, "all");
1894f69a
RH		 2256#ifdef CONFIG_DEBUG_TCG
2257 } else if (tcg_regset_single(set)) {
2258 TCGReg reg = tcg_regset_first(set);
b7a83ff8 2259 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
1894f69a
RH		 2260#endif
2261 } else if (TCG_TARGET_NB_REGS <= 32) {
b7a83ff8 2262 ne_fprintf(f, "0x%x", (uint32_t)set);
1894f69a 2263 } else {
b7a83ff8 2264 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
1894f69a
RH		 2265 }
2266 }
2267 }
2268
b7a83ff8 2269 putc('\n', f);
c896fe29
FB		 2270 }
2271}
2272
2273/* we give more priority to constraints with less registers */
2274static int get_constraint_priority(const TCGOpDef *def, int k)
2275{
74a11790 2276 const TCGArgConstraint *arg_ct = &def->args_ct[k];
29f5e925 2277 int n = ctpop64(arg_ct->regs);
c896fe29 2278
29f5e925
RH		 2279 /*
2280 * Sort constraints of a single register first, which includes output
2281 * aliases (which must exactly match the input already allocated).
2282 */
2283 if (n == 1 || arg_ct->oalias) {
2284 return INT_MAX;
2285 }
2286
2287 /*
2288 * Sort register pairs next, first then second immediately after.
2289 * Arbitrarily sort multiple pairs by the index of the first reg;
2290 * there shouldn't be many pairs.
2291 */
2292 switch (arg_ct->pair) {
2293 case 1:
2294 case 3:
2295 return (k + 1) * 2;
2296 case 2:
2297 return (arg_ct->pair_index + 1) * 2 - 1;
c896fe29 2298 }
29f5e925
RH		 2299
2300 /* Finally, sort by decreasing register count. */
2301 assert(n > 1);
2302 return -n;
c896fe29
FB		 2303}
2304
2305/* sort from highest priority to lowest */
2306static void sort_constraints(TCGOpDef *def, int start, int n)
2307{
66792f90
RH		 2308 int i, j;
2309 TCGArgConstraint *a = def->args_ct;
c896fe29 2310
66792f90
RH		 2311 for (i = 0; i < n; i++) {
2312 a[start + i].sort_index = start + i;
2313 }
2314 if (n <= 1) {
c896fe29 2315 return;
66792f90
RH		 2316 }
2317 for (i = 0; i < n - 1; i++) {
2318 for (j = i + 1; j < n; j++) {
2319 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2320 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2321 if (p1 < p2) {
66792f90
RH		 2322 int tmp = a[start + i].sort_index;
2323 a[start + i].sort_index = a[start + j].sort_index;
2324 a[start + j].sort_index = tmp;
c896fe29
FB		 2325 }
2326 }
2327 }
2328}
2329
f69d277e 2330static void process_op_defs(TCGContext *s)
c896fe29 2331{
a9751609 2332 TCGOpcode op;
c896fe29 2333
f69d277e
RH		 2334 for (op = 0; op < NB_OPS; op++) {
2335 TCGOpDef *def = &tcg_op_defs[op];
2336 const TCGTargetOpDef *tdefs;
29f5e925
RH		 2337 bool saw_alias_pair = false;
2338 int i, o, i2, o2, nb_args;
f69d277e
RH		 2339
2340 if (def->flags & TCG_OPF_NOT_PRESENT) {
2341 continue;
2342 }
2343
c896fe29 2344 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2345 if (nb_args == 0) {
2346 continue;
2347 }
2348
4c22e840
RH		 2349 /*
2350 * Macro magic should make it impossible, but double-check that
2351 * the array index is in range. Since the signness of an enum
2352 * is implementation defined, force the result to unsigned.
2353 */
2354 unsigned con_set = tcg_target_op_def(op);
2355 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2356 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2357
2358 for (i = 0; i < nb_args; i++) {
2359 const char *ct_str = tdefs->args_ct_str[i];
8940ea0d
PMD		 2360 bool input_p = i >= def->nb_oargs;
2361
f69d277e 2362 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2363 tcg_debug_assert(ct_str != NULL);
f69d277e 2364
8940ea0d
PMD		 2365 switch (*ct_str) {
2366 case '0' ... '9':
2367 o = *ct_str - '0';
2368 tcg_debug_assert(input_p);
2369 tcg_debug_assert(o < def->nb_oargs);
2370 tcg_debug_assert(def->args_ct[o].regs != 0);
2371 tcg_debug_assert(!def->args_ct[o].oalias);
2372 def->args_ct[i] = def->args_ct[o];
2373 /* The output sets oalias. */
2374 def->args_ct[o].oalias = 1;
2375 def->args_ct[o].alias_index = i;
2376 /* The input sets ialias. */
2377 def->args_ct[i].ialias = 1;
2378 def->args_ct[i].alias_index = o;
29f5e925
RH		 2379 if (def->args_ct[i].pair) {
2380 saw_alias_pair = true;
2381 }
8940ea0d
PMD		 2382 tcg_debug_assert(ct_str[1] == '\0');
2383 continue;
2384
2385 case '&':
2386 tcg_debug_assert(!input_p);
2387 def->args_ct[i].newreg = true;
2388 ct_str++;
2389 break;
29f5e925
RH		 2390
2391 case 'p': /* plus */
2392 /* Allocate to the register after the previous. */
2393 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2394 o = i - 1;
2395 tcg_debug_assert(!def->args_ct[o].pair);
2396 tcg_debug_assert(!def->args_ct[o].ct);
2397 def->args_ct[i] = (TCGArgConstraint){
2398 .pair = 2,
2399 .pair_index = o,
2400 .regs = def->args_ct[o].regs << 1,
2401 };
2402 def->args_ct[o].pair = 1;
2403 def->args_ct[o].pair_index = i;
2404 tcg_debug_assert(ct_str[1] == '\0');
2405 continue;
2406
2407 case 'm': /* minus */
2408 /* Allocate to the register before the previous. */
2409 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2410 o = i - 1;
2411 tcg_debug_assert(!def->args_ct[o].pair);
2412 tcg_debug_assert(!def->args_ct[o].ct);
2413 def->args_ct[i] = (TCGArgConstraint){
2414 .pair = 1,
2415 .pair_index = o,
2416 .regs = def->args_ct[o].regs >> 1,
2417 };
2418 def->args_ct[o].pair = 2;
2419 def->args_ct[o].pair_index = i;
2420 tcg_debug_assert(ct_str[1] == '\0');
2421 continue;
8940ea0d
PMD		 2422 }
2423
2424 do {
2425 switch (*ct_str) {
17280ff4
RH		 2426 case 'i':
2427 def->args_ct[i].ct |= TCG_CT_CONST;
17280ff4 2428 break;
358b4923 2429
358b4923
RH		 2430 /* Include all of the target-specific constraints. */
2431
2432#undef CONST
2433#define CONST(CASE, MASK) \
8940ea0d 2434 case CASE: def->args_ct[i].ct |= MASK; break;
358b4923 2435#define REGS(CASE, MASK) \
8940ea0d 2436 case CASE: def->args_ct[i].regs |= MASK; break;
358b4923
RH		 2437
2438#include "tcg-target-con-str.h"
2439
2440#undef REGS
2441#undef CONST
17280ff4 2442 default:
8940ea0d
PMD		 2443 case '0' ... '9':
2444 case '&':
29f5e925
RH		 2445 case 'p':
2446 case 'm':
17280ff4 2447 /* Typo in TCGTargetOpDef constraint. */
358b4923 2448 g_assert_not_reached();
c896fe29 2449 }
8940ea0d 2450 } while (*++ct_str != '\0');
c896fe29
FB		 2451 }
2452
c68aaa18 2453 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2454 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2455
29f5e925
RH		 2456 /*
2457 * Fix up output pairs that are aliased with inputs.
2458 * When we created the alias, we copied pair from the output.
2459 * There are three cases:
2460 * (1a) Pairs of inputs alias pairs of outputs.
2461 * (1b) One input aliases the first of a pair of outputs.
2462 * (2) One input aliases the second of a pair of outputs.
2463 *
2464 * Case 1a is handled by making sure that the pair_index'es are
2465 * properly updated so that they appear the same as a pair of inputs.
2466 *
2467 * Case 1b is handled by setting the pair_index of the input to
2468 * itself, simply so it doesn't point to an unrelated argument.
2469 * Since we don't encounter the "second" during the input allocation
2470 * phase, nothing happens with the second half of the input pair.
2471 *
2472 * Case 2 is handled by setting the second input to pair=3, the
2473 * first output to pair=3, and the pair_index'es to match.
2474 */
2475 if (saw_alias_pair) {
2476 for (i = def->nb_oargs; i < nb_args; i++) {
2477 /*
2478 * Since [0-9pm] must be alone in the constraint string,
2479 * the only way they can both be set is if the pair comes
2480 * from the output alias.
2481 */
2482 if (!def->args_ct[i].ialias) {
2483 continue;
2484 }
2485 switch (def->args_ct[i].pair) {
2486 case 0:
2487 break;
2488 case 1:
2489 o = def->args_ct[i].alias_index;
2490 o2 = def->args_ct[o].pair_index;
2491 tcg_debug_assert(def->args_ct[o].pair == 1);
2492 tcg_debug_assert(def->args_ct[o2].pair == 2);
2493 if (def->args_ct[o2].oalias) {
2494 /* Case 1a */
2495 i2 = def->args_ct[o2].alias_index;
2496 tcg_debug_assert(def->args_ct[i2].pair == 2);
2497 def->args_ct[i2].pair_index = i;
2498 def->args_ct[i].pair_index = i2;
2499 } else {
2500 /* Case 1b */
2501 def->args_ct[i].pair_index = i;
2502 }
2503 break;
2504 case 2:
2505 o = def->args_ct[i].alias_index;
2506 o2 = def->args_ct[o].pair_index;
2507 tcg_debug_assert(def->args_ct[o].pair == 2);
2508 tcg_debug_assert(def->args_ct[o2].pair == 1);
2509 if (def->args_ct[o2].oalias) {
2510 /* Case 1a */
2511 i2 = def->args_ct[o2].alias_index;
2512 tcg_debug_assert(def->args_ct[i2].pair == 1);
2513 def->args_ct[i2].pair_index = i;
2514 def->args_ct[i].pair_index = i2;
2515 } else {
2516 /* Case 2 */
2517 def->args_ct[i].pair = 3;
2518 def->args_ct[o2].pair = 3;
2519 def->args_ct[i].pair_index = o2;
2520 def->args_ct[o2].pair_index = i;
2521 }
2522 break;
2523 default:
2524 g_assert_not_reached();
2525 }
2526 }
2527 }
2528
c896fe29
FB		 2529 /* sort the constraints (XXX: this is just an heuristic) */
2530 sort_constraints(def, 0, def->nb_oargs);
2531 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2532 }
c896fe29
FB		 2533}
2534
0c627cdc
RH		 2535void tcg_op_remove(TCGContext *s, TCGOp *op)
2536{
d88a117e
RH		 2537 TCGLabel *label;
2538
2539 switch (op->opc) {
2540 case INDEX_op_br:
2541 label = arg_label(op->args[0]);
2542 label->refs--;
2543 break;
2544 case INDEX_op_brcond_i32:
2545 case INDEX_op_brcond_i64:
2546 label = arg_label(op->args[3]);
2547 label->refs--;
2548 break;
2549 case INDEX_op_brcond2_i32:
2550 label = arg_label(op->args[5]);
2551 label->refs--;
2552 break;
2553 default:
2554 break;
2555 }
2556
15fa08f8
RH		 2557 QTAILQ_REMOVE(&s->ops, op, link);
2558 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2559 s->nb_ops--;
0c627cdc
RH		 2560
2561#ifdef CONFIG_PROFILER
d73415a3 2562 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2563#endif
2564}
2565
a80cdd31
RH		 2566void tcg_remove_ops_after(TCGOp *op)
2567{
2568 TCGContext *s = tcg_ctx;
2569
2570 while (true) {
2571 TCGOp *last = tcg_last_op();
2572 if (last == op) {
2573 return;
2574 }
2575 tcg_op_remove(s, last);
2576 }
2577}
2578
d4478943 2579static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
5a18407f 2580{
15fa08f8 2581 TCGContext *s = tcg_ctx;
cb10bc63
RH		 2582 TCGOp *op = NULL;
2583
2584 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
2585 QTAILQ_FOREACH(op, &s->free_ops, link) {
2586 if (nargs <= op->nargs) {
2587 QTAILQ_REMOVE(&s->free_ops, op, link);
2588 nargs = op->nargs;
2589 goto found;
2590 }
2591 }
15fa08f8 2592 }
cb10bc63
RH		 2593
2594 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
2595 nargs = MAX(4, nargs);
2596 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
2597
2598 found:
15fa08f8
RH		 2599 memset(op, 0, offsetof(TCGOp, link));
2600 op->opc = opc;
cb10bc63
RH		 2601 op->nargs = nargs;
2602
2603 /* Check for bitfield overflow. */
2604 tcg_debug_assert(op->nargs == nargs);
5a18407f 2605
cb10bc63 2606 s->nb_ops++;
15fa08f8
RH		 2607 return op;
2608}
2609
d4478943 2610TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
15fa08f8 2611{
d4478943 2612 TCGOp *op = tcg_op_alloc(opc, nargs);
15fa08f8
RH		 2613 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2614 return op;
2615}
5a18407f 2616
d4478943
PMD		 2617TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
2618 TCGOpcode opc, unsigned nargs)
15fa08f8 2619{
d4478943 2620 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 2621 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2622 return new_op;
2623}
2624
d4478943
PMD		 2625TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
2626 TCGOpcode opc, unsigned nargs)
5a18407f 2627{
d4478943 2628 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 2629 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2630 return new_op;
2631}
2632
b4fc67c7 2633/* Reachable analysis : remove unreachable code. */
9bbee4c0
RH		 2634static void __attribute__((noinline))
2635reachable_code_pass(TCGContext *s)
b4fc67c7 2636{
4d89d0bb 2637 TCGOp *op, *op_next, *op_prev;
b4fc67c7
RH		 2638 bool dead = false;
2639
2640 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2641 bool remove = dead;
2642 TCGLabel *label;
b4fc67c7
RH		 2643
2644 switch (op->opc) {
2645 case INDEX_op_set_label:
2646 label = arg_label(op->args[0]);
4d89d0bb
RH		 2647
2648 /*
2649 * Optimization can fold conditional branches to unconditional.
2650 * If we find a label which is preceded by an unconditional
2651 * branch to next, remove the branch. We couldn't do this when
2652 * processing the branch because any dead code between the branch
2653 * and label had not yet been removed.
2654 */
2655 op_prev = QTAILQ_PREV(op, link);
2656 if (op_prev->opc == INDEX_op_br &&
2657 label == arg_label(op_prev->args[0])) {
2658 tcg_op_remove(s, op_prev);
2659 /* Fall through means insns become live again. */
2660 dead = false;
2661 }
2662
b4fc67c7
RH		 2663 if (label->refs == 0) {
2664 /*
2665 * While there is an occasional backward branch, virtually
2666 * all branches generated by the translators are forward.
2667 * Which means that generally we will have already removed
2668 * all references to the label that will be, and there is
2669 * little to be gained by iterating.
2670 */
2671 remove = true;
2672 } else {
2673 /* Once we see a label, insns become live again. */
2674 dead = false;
2675 remove = false;
b4fc67c7
RH		 2676 }
2677 break;
2678
2679 case INDEX_op_br:
2680 case INDEX_op_exit_tb:
2681 case INDEX_op_goto_ptr:
2682 /* Unconditional branches; everything following is dead. */
2683 dead = true;
2684 break;
2685
2686 case INDEX_op_call:
2687 /* Notice noreturn helper calls, raising exceptions. */
90163900 2688 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2689 dead = true;
2690 }
2691 break;
2692
2693 case INDEX_op_insn_start:
2694 /* Never remove -- we need to keep these for unwind. */
2695 remove = false;
2696 break;
2697
2698 default:
2699 break;
2700 }
2701
2702 if (remove) {
2703 tcg_op_remove(s, op);
2704 }
2705 }
2706}
2707
c70fbf0a
RH		 2708#define TS_DEAD 1
2709#define TS_MEM 2
2710
5a18407f
RH		 2711#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2712#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2713
25f49c5f
RH		 2714/* For liveness_pass_1, the register preferences for a given temp. */
2715static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2716{
2717 return ts->state_ptr;
2718}
2719
2720/* For liveness_pass_1, reset the preferences for a given temp to the
2721 * maximal regset for its type.
2722 */
2723static inline void la_reset_pref(TCGTemp *ts)
2724{
2725 *la_temp_pref(ts)
2726 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2727}
2728
9c43b68d
AJ		 2729/* liveness analysis: end of function: all temps are dead, and globals
2730 should be in memory. */
2616c808 2731static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2732{
b83eabea
RH		 2733 int i;
2734
2735 for (i = 0; i < ng; ++i) {
2736 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2737 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2738 }
2739 for (i = ng; i < nt; ++i) {
2740 s->temps[i].state = TS_DEAD;
25f49c5f 2741 la_reset_pref(&s->temps[i]);
b83eabea 2742 }
c896fe29
FB		 2743}
2744
9c43b68d
AJ		 2745/* liveness analysis: end of basic block: all temps are dead, globals
2746 and local temps should be in memory. */
2616c808 2747static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2748{
b83eabea 2749 int i;
641d5fbe 2750
ee17db83
RH		 2751 for (i = 0; i < nt; ++i) {
2752 TCGTemp *ts = &s->temps[i];
2753 int state;
2754
2755 switch (ts->kind) {
2756 case TEMP_FIXED:
2757 case TEMP_GLOBAL:
f57c6915 2758 case TEMP_TB:
ee17db83
RH		 2759 state = TS_DEAD | TS_MEM;
2760 break;
c7482438 2761 case TEMP_EBB:
c0522136 2762 case TEMP_CONST:
ee17db83
RH		 2763 state = TS_DEAD;
2764 break;
2765 default:
2766 g_assert_not_reached();
2767 }
2768 ts->state = state;
2769 la_reset_pref(ts);
641d5fbe
FB		 2770 }
2771}
2772
f65a061c
RH		 2773/* liveness analysis: sync globals back to memory. */
2774static void la_global_sync(TCGContext *s, int ng)
2775{
2776 int i;
2777
2778 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2779 int state = s->temps[i].state;
2780 s->temps[i].state = state | TS_MEM;
2781 if (state == TS_DEAD) {
2782 /* If the global was previously dead, reset prefs. */
2783 la_reset_pref(&s->temps[i]);
2784 }
f65a061c
RH		 2785 }
2786}
2787
b4cb76e6 2788/*
c7482438
RH		 2789 * liveness analysis: conditional branch: all temps are dead unless
2790 * explicitly live-across-conditional-branch, globals and local temps
2791 * should be synced.
b4cb76e6
RH		 2792 */
2793static void la_bb_sync(TCGContext *s, int ng, int nt)
2794{
2795 la_global_sync(s, ng);
2796
2797 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2798 TCGTemp *ts = &s->temps[i];
2799 int state;
2800
2801 switch (ts->kind) {
f57c6915 2802 case TEMP_TB:
c0522136
RH		 2803 state = ts->state;
2804 ts->state = state | TS_MEM;
b4cb76e6
RH		 2805 if (state != TS_DEAD) {
2806 continue;
2807 }
c0522136 2808 break;
c7482438 2809 case TEMP_EBB:
c0522136
RH		 2810 case TEMP_CONST:
2811 continue;
2812 default:
2813 g_assert_not_reached();
b4cb76e6
RH		 2814 }
2815 la_reset_pref(&s->temps[i]);
2816 }
2817}
2818
f65a061c
RH		 2819/* liveness analysis: sync globals back to memory and kill. */
2820static void la_global_kill(TCGContext *s, int ng)
2821{
2822 int i;
2823
2824 for (i = 0; i < ng; i++) {
2825 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2826 la_reset_pref(&s->temps[i]);
2827 }
2828}
2829
2830/* liveness analysis: note live globals crossing calls. */
2831static void la_cross_call(TCGContext *s, int nt)
2832{
2833 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2834 int i;
2835
2836 for (i = 0; i < nt; i++) {
2837 TCGTemp *ts = &s->temps[i];
2838 if (!(ts->state & TS_DEAD)) {
2839 TCGRegSet *pset = la_temp_pref(ts);
2840 TCGRegSet set = *pset;
2841
2842 set &= mask;
2843 /* If the combination is not possible, restart. */
2844 if (set == 0) {
2845 set = tcg_target_available_regs[ts->type] & mask;
2846 }
2847 *pset = set;
2848 }
f65a061c
RH		 2849 }
2850}
2851
874b8574
RH		 2852/*
2853 * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce
2854 * to TEMP_EBB, if possible.
2855 */
2856static void __attribute__((noinline))
2857liveness_pass_0(TCGContext *s)
2858{
2859 void * const multiple_ebb = (void *)(uintptr_t)-1;
2860 int nb_temps = s->nb_temps;
2861 TCGOp *op, *ebb;
2862
2863 for (int i = s->nb_globals; i < nb_temps; ++i) {
2864 s->temps[i].state_ptr = NULL;
2865 }
2866
2867 /*
2868 * Represent each EBB by the op at which it begins. In the case of
2869 * the first EBB, this is the first op, otherwise it is a label.
2870 * Collect the uses of each TEMP_TB: NULL for unused, EBB for use
2871 * within a single EBB, else MULTIPLE_EBB.
2872 */
2873 ebb = QTAILQ_FIRST(&s->ops);
2874 QTAILQ_FOREACH(op, &s->ops, link) {
2875 const TCGOpDef *def;
2876 int nb_oargs, nb_iargs;
2877
2878 switch (op->opc) {
2879 case INDEX_op_set_label:
2880 ebb = op;
2881 continue;
2882 case INDEX_op_discard:
2883 continue;
2884 case INDEX_op_call:
2885 nb_oargs = TCGOP_CALLO(op);
2886 nb_iargs = TCGOP_CALLI(op);
2887 break;
2888 default:
2889 def = &tcg_op_defs[op->opc];
2890 nb_oargs = def->nb_oargs;
2891 nb_iargs = def->nb_iargs;
2892 break;
2893 }
2894
2895 for (int i = 0; i < nb_oargs + nb_iargs; ++i) {
2896 TCGTemp *ts = arg_temp(op->args[i]);
2897
2898 if (ts->kind != TEMP_TB) {
2899 continue;
2900 }
2901 if (ts->state_ptr == NULL) {
2902 ts->state_ptr = ebb;
2903 } else if (ts->state_ptr != ebb) {
2904 ts->state_ptr = multiple_ebb;
2905 }
2906 }
2907 }
2908
2909 /*
2910 * For TEMP_TB that turned out not to be used beyond one EBB,
2911 * reduce the liveness to TEMP_EBB.
2912 */
2913 for (int i = s->nb_globals; i < nb_temps; ++i) {
2914 TCGTemp *ts = &s->temps[i];
2915 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
2916 ts->kind = TEMP_EBB;
2917 }
2918 }
2919}
2920
a1b3c48d 2921/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2922 given input arguments is dead. Instructions updating dead
2923 temporaries are removed. */
9bbee4c0
RH		 2924static void __attribute__((noinline))
2925liveness_pass_1(TCGContext *s)
c896fe29 2926{
c70fbf0a 2927 int nb_globals = s->nb_globals;
2616c808 2928 int nb_temps = s->nb_temps;
15fa08f8 2929 TCGOp *op, *op_prev;
25f49c5f
RH		 2930 TCGRegSet *prefs;
2931 int i;
2932
2933 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2934 for (i = 0; i < nb_temps; ++i) {
2935 s->temps[i].state_ptr = prefs + i;
2936 }
a1b3c48d 2937
ae36a246 2938 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2939 la_func_end(s, nb_globals, nb_temps);
c896fe29 2940
eae3eb3e 2941 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2942 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2943 TCGOpcode opc_new, opc_new2;
2944 bool have_opc_new2;
a1b3c48d 2945 TCGLifeData arg_life = 0;
25f49c5f 2946 TCGTemp *ts;
c45cb8bb
RH		 2947 TCGOpcode opc = op->opc;
2948 const TCGOpDef *def = &tcg_op_defs[opc];
2949
c45cb8bb 2950 switch (opc) {
c896fe29 2951 case INDEX_op_call:
c6e113f5 2952 {
39004a71
RH		 2953 const TCGHelperInfo *info = tcg_call_info(op);
2954 int call_flags = tcg_call_flags(op);
c896fe29 2955
cd9090aa
RH		 2956 nb_oargs = TCGOP_CALLO(op);
2957 nb_iargs = TCGOP_CALLI(op);
c6e113f5 2958
c45cb8bb 2959 /* pure functions can be removed if their result is unused */
78505279 2960 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2961 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2962 ts = arg_temp(op->args[i]);
2963 if (ts->state != TS_DEAD) {
c6e113f5 2964 goto do_not_remove_call;
9c43b68d 2965 }
c6e113f5 2966 }
c45cb8bb 2967 goto do_remove;
152c35aa
RH		 2968 }
2969 do_not_remove_call:
c896fe29 2970
25f49c5f 2971 /* Output args are dead. */
152c35aa 2972 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2973 ts = arg_temp(op->args[i]);
2974 if (ts->state & TS_DEAD) {
152c35aa
RH		 2975 arg_life |= DEAD_ARG << i;
2976 }
25f49c5f 2977 if (ts->state & TS_MEM) {
152c35aa 2978 arg_life |= SYNC_ARG << i;
c6e113f5 2979 }
25f49c5f
RH		 2980 ts->state = TS_DEAD;
2981 la_reset_pref(ts);
152c35aa 2982 }
78505279 2983
31fd884b
RH		 2984 /* Not used -- it will be tcg_target_call_oarg_reg(). */
2985 memset(op->output_pref, 0, sizeof(op->output_pref));
2986
152c35aa
RH		 2987 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2988 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2989 la_global_kill(s, nb_globals);
152c35aa 2990 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2991 la_global_sync(s, nb_globals);
152c35aa 2992 }
b9c18f56 2993
25f49c5f 2994 /* Record arguments that die in this helper. */
152c35aa 2995 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f 2996 ts = arg_temp(op->args[i]);
39004a71 2997 if (ts->state & TS_DEAD) {
152c35aa 2998 arg_life |= DEAD_ARG << i;
c6e113f5 2999 }
152c35aa 3000 }
25f49c5f
RH		 3001
3002 /* For all live registers, remove call-clobbered prefs. */
3003 la_cross_call(s, nb_temps);
3004
39004a71
RH		 3005 /*
3006 * Input arguments are live for preceding opcodes.
3007 *
3008 * For those arguments that die, and will be allocated in
3009 * registers, clear the register set for that arg, to be
3010 * filled in below. For args that will be on the stack,
3011 * reset to any available reg. Process arguments in reverse
3012 * order so that if a temp is used more than once, the stack
3013 * reset to max happens before the register reset to 0.
3014 */
3015 for (i = nb_iargs - 1; i >= 0; i--) {
3016 const TCGCallArgumentLoc *loc = &info->in[i];
3017 ts = arg_temp(op->args[nb_oargs + i]);
25f49c5f 3018
39004a71
RH		 3019 if (ts->state & TS_DEAD) {
3020 switch (loc->kind) {
3021 case TCG_CALL_ARG_NORMAL:
3022 case TCG_CALL_ARG_EXTEND_U:
3023 case TCG_CALL_ARG_EXTEND_S:
3024 if (REG_P(loc)) {
3025 *la_temp_pref(ts) = 0;
3026 break;
3027 }
3028 /* fall through */
3029 default:
3030 *la_temp_pref(ts) =
3031 tcg_target_available_regs[ts->type];
3032 break;
3033 }
25f49c5f
RH		 3034 ts->state &= ~TS_DEAD;
3035 }
3036 }
3037
39004a71
RH		 3038 /*
3039 * For each input argument, add its input register to prefs.
3040 * If a temp is used once, this produces a single set bit;
3041 * if a temp is used multiple times, this produces a set.
3042 */
3043 for (i = 0; i < nb_iargs; i++) {
3044 const TCGCallArgumentLoc *loc = &info->in[i];
3045 ts = arg_temp(op->args[nb_oargs + i]);
3046
3047 switch (loc->kind) {
3048 case TCG_CALL_ARG_NORMAL:
3049 case TCG_CALL_ARG_EXTEND_U:
3050 case TCG_CALL_ARG_EXTEND_S:
3051 if (REG_P(loc)) {
3052 tcg_regset_set_reg(*la_temp_pref(ts),
3053 tcg_target_call_iarg_regs[loc->arg_slot]);
3054 }
3055 break;
3056 default:
3057 break;
c19f47bf 3058 }
c896fe29 3059 }
c896fe29 3060 }
c896fe29 3061 break;
765b842a 3062 case INDEX_op_insn_start:
c896fe29 3063 break;
5ff9d6a4 3064 case INDEX_op_discard:
5ff9d6a4 3065 /* mark the temporary as dead */
25f49c5f
RH		 3066 ts = arg_temp(op->args[0]);
3067 ts->state = TS_DEAD;
3068 la_reset_pref(ts);
5ff9d6a4 3069 break;
1305c451
RH		 3070
3071 case INDEX_op_add2_i32:
c45cb8bb 3072 opc_new = INDEX_op_add_i32;
f1fae40c 3073 goto do_addsub2;
1305c451 3074 case INDEX_op_sub2_i32:
c45cb8bb 3075 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 3076 goto do_addsub2;
3077 case INDEX_op_add2_i64:
c45cb8bb 3078 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 3079 goto do_addsub2;
3080 case INDEX_op_sub2_i64:
c45cb8bb 3081 opc_new = INDEX_op_sub_i64;
f1fae40c 3082 do_addsub2:
1305c451
RH		 3083 nb_iargs = 4;
3084 nb_oargs = 2;
3085 /* Test if the high part of the operation is dead, but not
3086 the low part. The result can be optimized to a simple
3087 add or sub. This happens often for x86_64 guest when the
3088 cpu mode is set to 32 bit. */
b83eabea
RH		 3089 if (arg_temp(op->args[1])->state == TS_DEAD) {
3090 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 3091 goto do_remove;
3092 }
c45cb8bb
RH		 3093 /* Replace the opcode and adjust the args in place,
3094 leaving 3 unused args at the end. */
3095 op->opc = opc = opc_new;
efee3746
RH		 3096 op->args[1] = op->args[2];
3097 op->args[2] = op->args[4];
1305c451
RH		 3098 /* Fall through and mark the single-word operation live. */
3099 nb_iargs = 2;
3100 nb_oargs = 1;
3101 }
3102 goto do_not_remove;
3103
1414968a 3104 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 3105 opc_new = INDEX_op_mul_i32;
3106 opc_new2 = INDEX_op_muluh_i32;
3107 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 3108 goto do_mul2;
f1fae40c 3109 case INDEX_op_muls2_i32:
c45cb8bb
RH		 3110 opc_new = INDEX_op_mul_i32;
3111 opc_new2 = INDEX_op_mulsh_i32;
3112 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 3113 goto do_mul2;
3114 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 3115 opc_new = INDEX_op_mul_i64;
3116 opc_new2 = INDEX_op_muluh_i64;
3117 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 3118 goto do_mul2;
f1fae40c 3119 case INDEX_op_muls2_i64:
c45cb8bb
RH		 3120 opc_new = INDEX_op_mul_i64;
3121 opc_new2 = INDEX_op_mulsh_i64;
3122 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 3123 goto do_mul2;
f1fae40c 3124 do_mul2:
1414968a
RH		 3125 nb_iargs = 2;
3126 nb_oargs = 2;
b83eabea
RH		 3127 if (arg_temp(op->args[1])->state == TS_DEAD) {
3128 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 3129 /* Both parts of the operation are dead. */
1414968a
RH		 3130 goto do_remove;
3131 }
03271524 3132 /* The high part of the operation is dead; generate the low. */
c45cb8bb 3133 op->opc = opc = opc_new;
efee3746
RH		 3134 op->args[1] = op->args[2];
3135 op->args[2] = op->args[3];
b83eabea 3136 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 3137 /* The low part of the operation is dead; generate the high. */
3138 op->opc = opc = opc_new2;
efee3746
RH		 3139 op->args[0] = op->args[1];
3140 op->args[1] = op->args[2];
3141 op->args[2] = op->args[3];
03271524
RH		 3142 } else {
3143 goto do_not_remove;
1414968a 3144 }
03271524
RH		 3145 /* Mark the single-word operation live. */
3146 nb_oargs = 1;
1414968a
RH		 3147 goto do_not_remove;
3148
c896fe29 3149 default:
1305c451 3150 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 3151 nb_iargs = def->nb_iargs;
3152 nb_oargs = def->nb_oargs;
c896fe29 3153
49516bc0
AJ		 3154 /* Test if the operation can be removed because all
3155 its outputs are dead. We assume that nb_oargs == 0
3156 implies side effects */
3157 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 3158 for (i = 0; i < nb_oargs; i++) {
b83eabea 3159 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 3160 goto do_not_remove;
9c43b68d 3161 }
49516bc0 3162 }
152c35aa
RH		 3163 goto do_remove;
3164 }
3165 goto do_not_remove;
49516bc0 3166
152c35aa
RH		 3167 do_remove:
3168 tcg_op_remove(s, op);
3169 break;
3170
3171 do_not_remove:
152c35aa 3172 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 3173 ts = arg_temp(op->args[i]);
3174
3175 /* Remember the preference of the uses that followed. */
31fd884b
RH		 3176 if (i < ARRAY_SIZE(op->output_pref)) {
3177 op->output_pref[i] = *la_temp_pref(ts);
3178 }
25f49c5f
RH		 3179
3180 /* Output args are dead. */
3181 if (ts->state & TS_DEAD) {
152c35aa 3182 arg_life |= DEAD_ARG << i;
49516bc0 3183 }
25f49c5f 3184 if (ts->state & TS_MEM) {
152c35aa
RH		 3185 arg_life |= SYNC_ARG << i;
3186 }
25f49c5f
RH		 3187 ts->state = TS_DEAD;
3188 la_reset_pref(ts);
152c35aa 3189 }
49516bc0 3190
25f49c5f 3191 /* If end of basic block, update. */
ae36a246
RH		 3192 if (def->flags & TCG_OPF_BB_EXIT) {
3193 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 3194 } else if (def->flags & TCG_OPF_COND_BRANCH) {
3195 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 3196 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 3197 la_bb_end(s, nb_globals, nb_temps);
152c35aa 3198 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 3199 la_global_sync(s, nb_globals);
25f49c5f
RH		 3200 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3201 la_cross_call(s, nb_temps);
3202 }
152c35aa
RH		 3203 }
3204
25f49c5f 3205 /* Record arguments that die in this opcode. */
152c35aa 3206 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 3207 ts = arg_temp(op->args[i]);
3208 if (ts->state & TS_DEAD) {
152c35aa 3209 arg_life |= DEAD_ARG << i;
c896fe29 3210 }
c896fe29 3211 }
25f49c5f
RH		 3212
3213 /* Input arguments are live for preceding opcodes. */
152c35aa 3214 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 3215 ts = arg_temp(op->args[i]);
3216 if (ts->state & TS_DEAD) {
3217 /* For operands that were dead, initially allow
3218 all regs for the type. */
3219 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
3220 ts->state &= ~TS_DEAD;
3221 }
3222 }
3223
3224 /* Incorporate constraints for this operand. */
3225 switch (opc) {
3226 case INDEX_op_mov_i32:
3227 case INDEX_op_mov_i64:
3228 /* Note that these are TCG_OPF_NOT_PRESENT and do not
3229 have proper constraints. That said, special case
3230 moves to propagate preferences backward. */
3231 if (IS_DEAD_ARG(1)) {
3232 *la_temp_pref(arg_temp(op->args[0]))
3233 = *la_temp_pref(arg_temp(op->args[1]));
3234 }
3235 break;
3236
3237 default:
3238 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3239 const TCGArgConstraint *ct = &def->args_ct[i];
3240 TCGRegSet set, *pset;
3241
3242 ts = arg_temp(op->args[i]);
3243 pset = la_temp_pref(ts);
3244 set = *pset;
3245
9be0d080 3246 set &= ct->regs;
bc2b17e6 3247 if (ct->ialias) {
31fd884b 3248 set &= output_pref(op, ct->alias_index);
25f49c5f
RH		 3249 }
3250 /* If the combination is not possible, restart. */
3251 if (set == 0) {
9be0d080 3252 set = ct->regs;
25f49c5f
RH		 3253 }
3254 *pset = set;
3255 }
3256 break;
152c35aa 3257 }
c896fe29
FB		 3258 break;
3259 }
bee158cb 3260 op->life = arg_life;
1ff0a2c5 3261 }
c896fe29 3262}
c896fe29 3263
5a18407f 3264/* Liveness analysis: Convert indirect regs to direct temporaries. */
9bbee4c0
RH		 3265static bool __attribute__((noinline))
3266liveness_pass_2(TCGContext *s)
5a18407f
RH		 3267{
3268 int nb_globals = s->nb_globals;
15fa08f8 3269 int nb_temps, i;
5a18407f 3270 bool changes = false;
15fa08f8 3271 TCGOp *op, *op_next;
5a18407f 3272
5a18407f
RH		 3273 /* Create a temporary for each indirect global. */
3274 for (i = 0; i < nb_globals; ++i) {
3275 TCGTemp *its = &s->temps[i];
3276 if (its->indirect_reg) {
3277 TCGTemp *dts = tcg_temp_alloc(s);
3278 dts->type = its->type;
3279 dts->base_type = its->base_type;
e1e64652 3280 dts->temp_subindex = its->temp_subindex;
c7482438 3281 dts->kind = TEMP_EBB;
b83eabea
RH		 3282 its->state_ptr = dts;
3283 } else {
3284 its->state_ptr = NULL;
5a18407f 3285 }
b83eabea
RH		 3286 /* All globals begin dead. */
3287 its->state = TS_DEAD;
3288 }
3289 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3290 TCGTemp *its = &s->temps[i];
3291 its->state_ptr = NULL;
3292 its->state = TS_DEAD;
5a18407f 3293 }
5a18407f 3294
15fa08f8 3295 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 3296 TCGOpcode opc = op->opc;
3297 const TCGOpDef *def = &tcg_op_defs[opc];
3298 TCGLifeData arg_life = op->life;
3299 int nb_iargs, nb_oargs, call_flags;
b83eabea 3300 TCGTemp *arg_ts, *dir_ts;
5a18407f 3301
5a18407f 3302 if (opc == INDEX_op_call) {
cd9090aa
RH		 3303 nb_oargs = TCGOP_CALLO(op);
3304 nb_iargs = TCGOP_CALLI(op);
90163900 3305 call_flags = tcg_call_flags(op);
5a18407f
RH		 3306 } else {
3307 nb_iargs = def->nb_iargs;
3308 nb_oargs = def->nb_oargs;
3309
3310 /* Set flags similar to how calls require. */
b4cb76e6
RH		 3311 if (def->flags & TCG_OPF_COND_BRANCH) {
3312 /* Like reading globals: sync_globals */
3313 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3314 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 3315 /* Like writing globals: save_globals */
3316 call_flags = 0;
3317 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3318 /* Like reading globals: sync_globals */
3319 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3320 } else {
3321 /* No effect on globals. */
3322 call_flags = (TCG_CALL_NO_READ_GLOBALS |
3323 TCG_CALL_NO_WRITE_GLOBALS);
3324 }
3325 }
3326
3327 /* Make sure that input arguments are available. */
3328 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3329 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3330 dir_ts = arg_ts->state_ptr;
3331 if (dir_ts && arg_ts->state == TS_DEAD) {
3332 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
3333 ? INDEX_op_ld_i32
3334 : INDEX_op_ld_i64);
3335 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
3336
3337 lop->args[0] = temp_arg(dir_ts);
3338 lop->args[1] = temp_arg(arg_ts->mem_base);
3339 lop->args[2] = arg_ts->mem_offset;
3340
3341 /* Loaded, but synced with memory. */
3342 arg_ts->state = TS_MEM;
5a18407f
RH		 3343 }
3344 }
3345
3346 /* Perform input replacement, and mark inputs that became dead.
3347 No action is required except keeping temp_state up to date
3348 so that we reload when needed. */
3349 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3350 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3351 dir_ts = arg_ts->state_ptr;
3352 if (dir_ts) {
3353 op->args[i] = temp_arg(dir_ts);
3354 changes = true;
3355 if (IS_DEAD_ARG(i)) {
3356 arg_ts->state = TS_DEAD;
5a18407f
RH		 3357 }
3358 }
3359 }
3360
3361 /* Liveness analysis should ensure that the following are
3362 all correct, for call sites and basic block end points. */
3363 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
3364 /* Nothing to do */
3365 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
3366 for (i = 0; i < nb_globals; ++i) {
3367 /* Liveness should see that globals are synced back,
3368 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 3369 arg_ts = &s->temps[i];
3370 tcg_debug_assert(arg_ts->state_ptr == 0
3371 || arg_ts->state != 0);
5a18407f
RH		 3372 }
3373 } else {
3374 for (i = 0; i < nb_globals; ++i) {
3375 /* Liveness should see that globals are saved back,
3376 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 3377 arg_ts = &s->temps[i];
3378 tcg_debug_assert(arg_ts->state_ptr == 0
3379 || arg_ts->state == TS_DEAD);
5a18407f
RH		 3380 }
3381 }
3382
3383 /* Outputs become available. */
61f15c48
RH		 3384 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
3385 arg_ts = arg_temp(op->args[0]);
b83eabea 3386 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 3387 if (dir_ts) {
3388 op->args[0] = temp_arg(dir_ts);
3389 changes = true;
3390
3391 /* The output is now live and modified. */
3392 arg_ts->state = 0;
3393
3394 if (NEED_SYNC_ARG(0)) {
3395 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3396 ? INDEX_op_st_i32
3397 : INDEX_op_st_i64);
d4478943 3398 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
61f15c48
RH		 3399 TCGTemp *out_ts = dir_ts;
3400
3401 if (IS_DEAD_ARG(0)) {
3402 out_ts = arg_temp(op->args[1]);
3403 arg_ts->state = TS_DEAD;
3404 tcg_op_remove(s, op);
3405 } else {
3406 arg_ts->state = TS_MEM;
3407 }
3408
3409 sop->args[0] = temp_arg(out_ts);
3410 sop->args[1] = temp_arg(arg_ts->mem_base);
3411 sop->args[2] = arg_ts->mem_offset;
3412 } else {
3413 tcg_debug_assert(!IS_DEAD_ARG(0));
3414 }
5a18407f 3415 }
61f15c48
RH		 3416 } else {
3417 for (i = 0; i < nb_oargs; i++) {
3418 arg_ts = arg_temp(op->args[i]);
3419 dir_ts = arg_ts->state_ptr;
3420 if (!dir_ts) {
3421 continue;
3422 }
3423 op->args[i] = temp_arg(dir_ts);
3424 changes = true;
5a18407f 3425
61f15c48
RH		 3426 /* The output is now live and modified. */
3427 arg_ts->state = 0;
5a18407f 3428
61f15c48
RH		 3429 /* Sync outputs upon their last write. */
3430 if (NEED_SYNC_ARG(i)) {
3431 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3432 ? INDEX_op_st_i32
3433 : INDEX_op_st_i64);
d4478943 3434 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
5a18407f 3435
61f15c48
RH		 3436 sop->args[0] = temp_arg(dir_ts);
3437 sop->args[1] = temp_arg(arg_ts->mem_base);
3438 sop->args[2] = arg_ts->mem_offset;
5a18407f 3439
61f15c48
RH		 3440 arg_ts->state = TS_MEM;
3441 }
3442 /* Drop outputs that are dead. */
3443 if (IS_DEAD_ARG(i)) {
3444 arg_ts->state = TS_DEAD;
3445 }
5a18407f
RH		 3446 }
3447 }
3448 }
3449
3450 return changes;
3451}
3452
2272e4a7 3453static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 3454{
31c96417 3455 intptr_t off;
273eb50c 3456 int size, align;
c1c09194 3457
273eb50c
RH		 3458 /* When allocating an object, look at the full type. */
3459 size = tcg_type_size(ts->base_type);
3460 switch (ts->base_type) {
c1c09194 3461 case TCG_TYPE_I32:
31c96417 3462 align = 4;
c1c09194
RH		 3463 break;
3464 case TCG_TYPE_I64:
3465 case TCG_TYPE_V64:
31c96417 3466 align = 8;
c1c09194 3467 break;
43eef72f 3468 case TCG_TYPE_I128:
c1c09194 3469 case TCG_TYPE_V128:
c1c09194 3470 case TCG_TYPE_V256:
43eef72f
RH		 3471 /*
3472 * Note that we do not require aligned storage for V256,
3473 * and that we provide alignment for I128 to match V128,
3474 * even if that's above what the host ABI requires.
3475 */
31c96417 3476 align = 16;
c1c09194
RH		 3477 break;
3478 default:
3479 g_assert_not_reached();
b591dc59 3480 }
c1c09194 3481
b9537d59
RH		 3482 /*
3483 * Assume the stack is sufficiently aligned.
3484 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3485 * and do not require 16 byte vector alignment. This seems slightly
3486 * easier than fully parameterizing the above switch statement.
3487 */
3488 align = MIN(TCG_TARGET_STACK_ALIGN, align);
c1c09194 3489 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 3490
3491 /* If we've exhausted the stack frame, restart with a smaller TB. */
3492 if (off + size > s->frame_end) {
3493 tcg_raise_tb_overflow(s);
3494 }
c1c09194 3495 s->current_frame_offset = off + size;
9defd1bd 3496#if defined(__sparc__)
273eb50c 3497 off += TCG_TARGET_STACK_BIAS;
9defd1bd 3498#endif
273eb50c
RH		 3499
3500 /* If the object was subdivided, assign memory to all the parts. */
3501 if (ts->base_type != ts->type) {
3502 int part_size = tcg_type_size(ts->type);
3503 int part_count = size / part_size;
3504
3505 /*
3506 * Each part is allocated sequentially in tcg_temp_new_internal.
3507 * Jump back to the first part by subtracting the current index.
3508 */
3509 ts -= ts->temp_subindex;
3510 for (int i = 0; i < part_count; ++i) {
3511 ts[i].mem_offset = off + i * part_size;
3512 ts[i].mem_base = s->frame_temp;
3513 ts[i].mem_allocated = 1;
3514 }
3515 } else {
3516 ts->mem_offset = off;
3517 ts->mem_base = s->frame_temp;
3518 ts->mem_allocated = 1;
3519 }
c896fe29
FB		 3520}
3521
098859f1
RH		 3522/* Assign @reg to @ts, and update reg_to_temp[]. */
3523static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
3524{
3525 if (ts->val_type == TEMP_VAL_REG) {
3526 TCGReg old = ts->reg;
3527 tcg_debug_assert(s->reg_to_temp[old] == ts);
3528 if (old == reg) {
3529 return;
3530 }
3531 s->reg_to_temp[old] = NULL;
3532 }
3533 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
3534 s->reg_to_temp[reg] = ts;
3535 ts->val_type = TEMP_VAL_REG;
3536 ts->reg = reg;
3537}
3538
3539/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
3540static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
3541{
3542 tcg_debug_assert(type != TEMP_VAL_REG);
3543 if (ts->val_type == TEMP_VAL_REG) {
3544 TCGReg reg = ts->reg;
3545 tcg_debug_assert(s->reg_to_temp[reg] == ts);
3546 s->reg_to_temp[reg] = NULL;
3547 }
3548 ts->val_type = type;
3549}
3550
b722452a 3551static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3552
59d7c14e
RH		 3553/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3554 mark it free; otherwise mark it dead. */
3555static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3556{
c0522136
RH		 3557 TCGTempVal new_type;
3558
3559 switch (ts->kind) {
3560 case TEMP_FIXED:
59d7c14e 3561 return;
c0522136 3562 case TEMP_GLOBAL:
f57c6915 3563 case TEMP_TB:
c0522136
RH		 3564 new_type = TEMP_VAL_MEM;
3565 break;
c7482438 3566 case TEMP_EBB:
c0522136
RH		 3567 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3568 break;
3569 case TEMP_CONST:
3570 new_type = TEMP_VAL_CONST;
3571 break;
3572 default:
3573 g_assert_not_reached();
59d7c14e 3574 }
098859f1 3575 set_temp_val_nonreg(s, ts, new_type);
59d7c14e 3576}
7f6ceedf 3577
59d7c14e
RH		 3578/* Mark a temporary as dead. */
3579static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3580{
3581 temp_free_or_dead(s, ts, 1);
3582}
3583
3584/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3585 registers needs to be allocated to store a constant. If 'free_or_dead'
3586 is non-zero, subsequently release the temporary; if it is positive, the
3587 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3588static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3589 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3590{
c0522136 3591 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3592 if (!ts->mem_allocated) {
2272e4a7 3593 temp_allocate_frame(s, ts);
59d7c14e 3594 }
59d7c14e
RH		 3595 switch (ts->val_type) {
3596 case TEMP_VAL_CONST:
3597 /* If we're going to free the temp immediately, then we won't
3598 require it later in a register, so attempt to store the
3599 constant to memory directly. */
3600 if (free_or_dead
3601 && tcg_out_sti(s, ts->type, ts->val,
3602 ts->mem_base->reg, ts->mem_offset)) {
3603 break;
3604 }
3605 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3606 allocated_regs, preferred_regs);
59d7c14e
RH		 3607 /* fallthrough */
3608
3609 case TEMP_VAL_REG:
3610 tcg_out_st(s, ts->type, ts->reg,
3611 ts->mem_base->reg, ts->mem_offset);
3612 break;
3613
3614 case TEMP_VAL_MEM:
3615 break;
3616
3617 case TEMP_VAL_DEAD:
3618 default:
3619 tcg_abort();
3620 }
3621 ts->mem_coherent = 1;
3622 }
3623 if (free_or_dead) {
3624 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3625 }
7f6ceedf
AJ		 3626}
3627
c896fe29 3628/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3629static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3630{
f8b2f202 3631 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3632 if (ts != NULL) {
98b4e186 3633 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3634 }
3635}
3636
b016486e
RH		 3637/**
3638 * tcg_reg_alloc:
3639 * @required_regs: Set of registers in which we must allocate.
3640 * @allocated_regs: Set of registers which must be avoided.
3641 * @preferred_regs: Set of registers we should prefer.
3642 * @rev: True if we search the registers in "indirect" order.
3643 *
3644 * The allocated register must be in @required_regs & ~@allocated_regs,
3645 * but if we can put it in @preferred_regs we may save a move later.
3646 */
3647static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3648 TCGRegSet allocated_regs,
3649 TCGRegSet preferred_regs, bool rev)
c896fe29 3650{
b016486e
RH		 3651 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3652 TCGRegSet reg_ct[2];
91478cef 3653 const int *order;
c896fe29 3654
b016486e
RH		 3655 reg_ct[1] = required_regs & ~allocated_regs;
3656 tcg_debug_assert(reg_ct[1] != 0);
3657 reg_ct[0] = reg_ct[1] & preferred_regs;
3658
3659 /* Skip the preferred_regs option if it cannot be satisfied,
3660 or if the preference made no difference. */
3661 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3662
91478cef 3663 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3664
b016486e
RH		 3665 /* Try free registers, preferences first. */
3666 for (j = f; j < 2; j++) {
3667 TCGRegSet set = reg_ct[j];
3668
3669 if (tcg_regset_single(set)) {
3670 /* One register in the set. */
3671 TCGReg reg = tcg_regset_first(set);
3672 if (s->reg_to_temp[reg] == NULL) {
3673 return reg;
3674 }
3675 } else {
3676 for (i = 0; i < n; i++) {
3677 TCGReg reg = order[i];
3678 if (s->reg_to_temp[reg] == NULL &&
3679 tcg_regset_test_reg(set, reg)) {
3680 return reg;
3681 }
3682 }
3683 }
c896fe29
FB		 3684 }
3685
b016486e
RH		 3686 /* We must spill something. */
3687 for (j = f; j < 2; j++) {
3688 TCGRegSet set = reg_ct[j];
3689
3690 if (tcg_regset_single(set)) {
3691 /* One register in the set. */
3692 TCGReg reg = tcg_regset_first(set);
b3915dbb 3693 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3694 return reg;
b016486e
RH		 3695 } else {
3696 for (i = 0; i < n; i++) {
3697 TCGReg reg = order[i];
3698 if (tcg_regset_test_reg(set, reg)) {
3699 tcg_reg_free(s, reg, allocated_regs);
3700 return reg;
3701 }
3702 }
c896fe29
FB		 3703 }
3704 }
3705
3706 tcg_abort();
3707}
3708
29f5e925
RH		 3709static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
3710 TCGRegSet allocated_regs,
3711 TCGRegSet preferred_regs, bool rev)
3712{
3713 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3714 TCGRegSet reg_ct[2];
3715 const int *order;
3716
3717 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
3718 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
3719 tcg_debug_assert(reg_ct[1] != 0);
3720 reg_ct[0] = reg_ct[1] & preferred_regs;
3721
3722 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3723
3724 /*
3725 * Skip the preferred_regs option if it cannot be satisfied,
3726 * or if the preference made no difference.
3727 */
3728 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3729
3730 /*
3731 * Minimize the number of flushes by looking for 2 free registers first,
3732 * then a single flush, then two flushes.
3733 */
3734 for (fmin = 2; fmin >= 0; fmin--) {
3735 for (j = k; j < 2; j++) {
3736 TCGRegSet set = reg_ct[j];
3737
3738 for (i = 0; i < n; i++) {
3739 TCGReg reg = order[i];
3740
3741 if (tcg_regset_test_reg(set, reg)) {
3742 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
3743 if (f >= fmin) {
3744 tcg_reg_free(s, reg, allocated_regs);
3745 tcg_reg_free(s, reg + 1, allocated_regs);
3746 return reg;
3747 }
3748 }
3749 }
3750 }
3751 }
3752 tcg_abort();
3753}
3754
40ae5c62
RH		 3755/* Make sure the temporary is in a register. If needed, allocate the register
3756 from DESIRED while avoiding ALLOCATED. */
3757static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3758 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3759{
3760 TCGReg reg;
3761
3762 switch (ts->val_type) {
3763 case TEMP_VAL_REG:
3764 return;
3765 case TEMP_VAL_CONST:
b016486e 3766 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3767 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3768 if (ts->type <= TCG_TYPE_I64) {
3769 tcg_out_movi(s, ts->type, reg, ts->val);
3770 } else {
4e186175
RH		 3771 uint64_t val = ts->val;
3772 MemOp vece = MO_64;
3773
3774 /*
3775 * Find the minimal vector element that matches the constant.
3776 * The targets will, in general, have to do this search anyway,
3777 * do this generically.
3778 */
4e186175
RH		 3779 if (val == dup_const(MO_8, val)) {
3780 vece = MO_8;
3781 } else if (val == dup_const(MO_16, val)) {
3782 vece = MO_16;
0b4286dd 3783 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3784 vece = MO_32;
3785 }
3786
3787 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3788 }
40ae5c62
RH		 3789 ts->mem_coherent = 0;
3790 break;
3791 case TEMP_VAL_MEM:
b016486e 3792 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3793 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3794 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3795 ts->mem_coherent = 1;
3796 break;
3797 case TEMP_VAL_DEAD:
3798 default:
3799 tcg_abort();
3800 }
098859f1 3801 set_temp_val_reg(s, ts, reg);
40ae5c62
RH		 3802}
3803
59d7c14e
RH		 3804/* Save a temporary to memory. 'allocated_regs' is used in case a
3805 temporary registers needs to be allocated to store a constant. */
3806static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3807{
5a18407f
RH		 3808 /* The liveness analysis already ensures that globals are back
3809 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3810 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3811}
3812
9814dd27 3813/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3814 modified be the following code. 'allocated_regs' is used in case a
3815 temporary registers needs to be allocated to store a constant. */
3816static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3817{
ac3b8891 3818 int i, n;
c896fe29 3819
ac3b8891 3820 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3821 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3822 }
e5097dc8
FB		 3823}
3824
3d5c5f87
AJ		 3825/* sync globals to their canonical location and assume they can be
3826 read by the following code. 'allocated_regs' is used in case a
3827 temporary registers needs to be allocated to store a constant. */
3828static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3829{
ac3b8891 3830 int i, n;
3d5c5f87 3831
ac3b8891 3832 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3833 TCGTemp *ts = &s->temps[i];
5a18407f 3834 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3835 || ts->kind == TEMP_FIXED
5a18407f 3836 || ts->mem_coherent);
3d5c5f87
AJ		 3837 }
3838}
3839
e5097dc8 3840/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3841 all globals are stored at their canonical location. */
3842static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3843{
e5097dc8
FB		 3844 int i;
3845
b13eb728
RH		 3846 for (i = s->nb_globals; i < s->nb_temps; i++) {
3847 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3848
3849 switch (ts->kind) {
f57c6915 3850 case TEMP_TB:
b13eb728 3851 temp_save(s, ts, allocated_regs);
c0522136 3852 break;
c7482438 3853 case TEMP_EBB:
5a18407f
RH		 3854 /* The liveness analysis already ensures that temps are dead.
3855 Keep an tcg_debug_assert for safety. */
3856 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3857 break;
3858 case TEMP_CONST:
3859 /* Similarly, we should have freed any allocated register. */
3860 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3861 break;
3862 default:
3863 g_assert_not_reached();
c896fe29
FB		 3864 }
3865 }
e8996ee0
FB		 3866
3867 save_globals(s, allocated_regs);
c896fe29
FB		 3868}
3869
b4cb76e6 3870/*
c7482438
RH		 3871 * At a conditional branch, we assume all temporaries are dead unless
3872 * explicitly live-across-conditional-branch; all globals and local
3873 * temps are synced to their location.
b4cb76e6
RH		 3874 */
3875static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3876{
3877 sync_globals(s, allocated_regs);
3878
3879 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3880 TCGTemp *ts = &s->temps[i];
3881 /*
3882 * The liveness analysis already ensures that temps are dead.
3883 * Keep tcg_debug_asserts for safety.
3884 */
c0522136 3885 switch (ts->kind) {
f57c6915 3886 case TEMP_TB:
b4cb76e6 3887 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136 3888 break;
c7482438 3889 case TEMP_EBB:
c0522136
RH		 3890 case TEMP_CONST:
3891 break;
3892 default:
3893 g_assert_not_reached();
b4cb76e6
RH		 3894 }
3895 }
3896}
3897
bab1671f 3898/*
c58f4c97 3899 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3900 */
0fe4fca4 3901static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3902 tcg_target_ulong val, TCGLifeData arg_life,
3903 TCGRegSet preferred_regs)
e8996ee0 3904{
d63e3b6e 3905 /* ENV should not be modified. */
e01fa97d 3906 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3907
3908 /* The movi is not explicitly generated here. */
098859f1 3909 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
59d7c14e
RH		 3910 ots->val = val;
3911 ots->mem_coherent = 0;
3912 if (NEED_SYNC_ARG(0)) {
ba87719c 3913 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3914 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3915 temp_dead(s, ots);
4c4e1ab2 3916 }
e8996ee0
FB		 3917}
3918
bab1671f
RH		 3919/*
3920 * Specialized code generation for INDEX_op_mov_*.
3921 */
dd186292 3922static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3923{
dd186292 3924 const TCGLifeData arg_life = op->life;
69e3706d 3925 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3926 TCGTemp *ts, *ots;
450445d5 3927 TCGType otype, itype;
098859f1 3928 TCGReg oreg, ireg;
c896fe29 3929
d21369f5 3930 allocated_regs = s->reserved_regs;
31fd884b 3931 preferred_regs = output_pref(op, 0);
43439139
RH		 3932 ots = arg_temp(op->args[0]);
3933 ts = arg_temp(op->args[1]);
450445d5 3934
d63e3b6e 3935 /* ENV should not be modified. */
e01fa97d 3936 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3937
450445d5
RH		 3938 /* Note that otype != itype for no-op truncation. */
3939 otype = ots->type;
3940 itype = ts->type;
c29c1d7e 3941
0fe4fca4
PB		 3942 if (ts->val_type == TEMP_VAL_CONST) {
3943 /* propagate constant or generate sti */
3944 tcg_target_ulong val = ts->val;
3945 if (IS_DEAD_ARG(1)) {
3946 temp_dead(s, ts);
3947 }
69e3706d 3948 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3949 return;
3950 }
3951
3952 /* If the source value is in memory we're going to be forced
3953 to have it in a register in order to perform the copy. Copy
3954 the SOURCE value into its own register first, that way we
3955 don't have to reload SOURCE the next time it is used. */
3956 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3957 temp_load(s, ts, tcg_target_available_regs[itype],
3958 allocated_regs, preferred_regs);
c29c1d7e 3959 }
0fe4fca4 3960 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
098859f1
RH		 3961 ireg = ts->reg;
3962
d63e3b6e 3963 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3964 /* mov to a non-saved dead register makes no sense (even with
3965 liveness analysis disabled). */
eabb7b91 3966 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3967 if (!ots->mem_allocated) {
2272e4a7 3968 temp_allocate_frame(s, ots);
c29c1d7e 3969 }
098859f1 3970 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3971 if (IS_DEAD_ARG(1)) {
f8bf00f1 3972 temp_dead(s, ts);
c29c1d7e 3973 }
f8bf00f1 3974 temp_dead(s, ots);
098859f1
RH		 3975 return;
3976 }
3977
3978 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
3979 /*
3980 * The mov can be suppressed. Kill input first, so that it
3981 * is unlinked from reg_to_temp, then set the output to the
3982 * reg that we saved from the input.
3983 */
3984 temp_dead(s, ts);
3985 oreg = ireg;
c29c1d7e 3986 } else {
098859f1
RH		 3987 if (ots->val_type == TEMP_VAL_REG) {
3988 oreg = ots->reg;
c896fe29 3989 } else {
098859f1
RH		 3990 /* Make sure to not spill the input register during allocation. */
3991 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
3992 allocated_regs | ((TCGRegSet)1 << ireg),
3993 preferred_regs, ots->indirect_base);
c896fe29 3994 }
098859f1
RH		 3995 if (!tcg_out_mov(s, otype, oreg, ireg)) {
3996 /*
3997 * Cross register class move not supported.
3998 * Store the source register into the destination slot
3999 * and leave the destination temp as TEMP_VAL_MEM.
4000 */
4001 assert(!temp_readonly(ots));
4002 if (!ts->mem_allocated) {
4003 temp_allocate_frame(s, ots);
4004 }
4005 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
4006 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
4007 ots->mem_coherent = 1;
4008 return;
c896fe29 4009 }
ec7a869d 4010 }
098859f1
RH		 4011 set_temp_val_reg(s, ots, oreg);
4012 ots->mem_coherent = 0;
4013
4014 if (NEED_SYNC_ARG(0)) {
4015 temp_sync(s, ots, allocated_regs, 0, 0);
4016 }
c896fe29
FB		 4017}
4018
bab1671f
RH		 4019/*
4020 * Specialized code generation for INDEX_op_dup_vec.
4021 */
4022static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
4023{
4024 const TCGLifeData arg_life = op->life;
4025 TCGRegSet dup_out_regs, dup_in_regs;
4026 TCGTemp *its, *ots;
4027 TCGType itype, vtype;
4028 unsigned vece;
31c96417 4029 int lowpart_ofs;
bab1671f
RH		 4030 bool ok;
4031
4032 ots = arg_temp(op->args[0]);
4033 its = arg_temp(op->args[1]);
4034
4035 /* ENV should not be modified. */
e01fa97d 4036 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 4037
4038 itype = its->type;
4039 vece = TCGOP_VECE(op);
4040 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4041
4042 if (its->val_type == TEMP_VAL_CONST) {
4043 /* Propagate constant via movi -> dupi. */
4044 tcg_target_ulong val = its->val;
4045 if (IS_DEAD_ARG(1)) {
4046 temp_dead(s, its);
4047 }
31fd884b 4048 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
bab1671f
RH		 4049 return;
4050 }
4051
9be0d080
RH		 4052 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
4053 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 4054
4055 /* Allocate the output register now. */
4056 if (ots->val_type != TEMP_VAL_REG) {
4057 TCGRegSet allocated_regs = s->reserved_regs;
098859f1 4058 TCGReg oreg;
bab1671f
RH		 4059
4060 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
4061 /* Make sure to not spill the input register. */
4062 tcg_regset_set_reg(allocated_regs, its->reg);
4063 }
098859f1 4064 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 4065 output_pref(op, 0), ots->indirect_base);
098859f1 4066 set_temp_val_reg(s, ots, oreg);
bab1671f
RH		 4067 }
4068
4069 switch (its->val_type) {
4070 case TEMP_VAL_REG:
4071 /*
4072 * The dup constriaints must be broad, covering all possible VECE.
4073 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
4074 * to fail, indicating that extra moves are required for that case.
4075 */
4076 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
4077 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
4078 goto done;
4079 }
4080 /* Try again from memory or a vector input register. */
4081 }
4082 if (!its->mem_coherent) {
4083 /*
4084 * The input register is not synced, and so an extra store
4085 * would be required to use memory. Attempt an integer-vector
4086 * register move first. We do not have a TCGRegSet for this.
4087 */
4088 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
4089 break;
4090 }
4091 /* Sync the temp back to its slot and load from there. */
4092 temp_sync(s, its, s->reserved_regs, 0, 0);
4093 }
4094 /* fall through */
4095
4096 case TEMP_VAL_MEM:
31c96417
RH		 4097 lowpart_ofs = 0;
4098 if (HOST_BIG_ENDIAN) {
4099 lowpart_ofs = tcg_type_size(itype) - (1 << vece);
4100 }
d6ecb4a9 4101 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
31c96417 4102 its->mem_offset + lowpart_ofs)) {
d6ecb4a9
RH		 4103 goto done;
4104 }
098859f1 4105 /* Load the input into the destination vector register. */
bab1671f
RH		 4106 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
4107 break;
4108
4109 default:
4110 g_assert_not_reached();
4111 }
4112
4113 /* We now have a vector input register, so dup must succeed. */
4114 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
4115 tcg_debug_assert(ok);
4116
4117 done:
36f5539c 4118 ots->mem_coherent = 0;
bab1671f
RH		 4119 if (IS_DEAD_ARG(1)) {
4120 temp_dead(s, its);
4121 }
4122 if (NEED_SYNC_ARG(0)) {
4123 temp_sync(s, ots, s->reserved_regs, 0, 0);
4124 }
4125 if (IS_DEAD_ARG(0)) {
4126 temp_dead(s, ots);
4127 }
4128}
4129
dd186292 4130static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 4131{
dd186292
RH		 4132 const TCGLifeData arg_life = op->life;
4133 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 4134 TCGRegSet i_allocated_regs;
4135 TCGRegSet o_allocated_regs;
b6638662
RH		 4136 int i, k, nb_iargs, nb_oargs;
4137 TCGReg reg;
c896fe29
FB		 4138 TCGArg arg;
4139 const TCGArgConstraint *arg_ct;
4140 TCGTemp *ts;
4141 TCGArg new_args[TCG_MAX_OP_ARGS];
4142 int const_args[TCG_MAX_OP_ARGS];
4143
4144 nb_oargs = def->nb_oargs;
4145 nb_iargs = def->nb_iargs;
4146
4147 /* copy constants */
a813e36f 4148 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 4149 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 4150 sizeof(TCGArg) * def->nb_cargs);
4151
d21369f5
RH		 4152 i_allocated_regs = s->reserved_regs;
4153 o_allocated_regs = s->reserved_regs;
82790a87 4154
a813e36f 4155 /* satisfy input constraints */
dd186292 4156 for (k = 0; k < nb_iargs; k++) {
29f5e925
RH		 4157 TCGRegSet i_preferred_regs, i_required_regs;
4158 bool allocate_new_reg, copyto_new_reg;
4159 TCGTemp *ts2;
4160 int i1, i2;
d62816f2 4161
66792f90 4162 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 4163 arg = op->args[i];
c896fe29 4164 arg_ct = &def->args_ct[i];
43439139 4165 ts = arg_temp(arg);
40ae5c62
RH		 4166
4167 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 4168 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 4169 /* constant is OK for instruction */
4170 const_args[i] = 1;
4171 new_args[i] = ts->val;
d62816f2 4172 continue;
c896fe29 4173 }
40ae5c62 4174
1c1824dc
RH		 4175 reg = ts->reg;
4176 i_preferred_regs = 0;
29f5e925 4177 i_required_regs = arg_ct->regs;
1c1824dc 4178 allocate_new_reg = false;
29f5e925
RH		 4179 copyto_new_reg = false;
4180
4181 switch (arg_ct->pair) {
4182 case 0: /* not paired */
4183 if (arg_ct->ialias) {
31fd884b 4184 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 4185
4186 /*
4187 * If the input is readonly, then it cannot also be an
4188 * output and aliased to itself. If the input is not
4189 * dead after the instruction, we must allocate a new
4190 * register and move it.
4191 */
4192 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
4193 allocate_new_reg = true;
4194 } else if (ts->val_type == TEMP_VAL_REG) {
4195 /*
4196 * Check if the current register has already been
4197 * allocated for another input.
4198 */
4199 allocate_new_reg =
4200 tcg_regset_test_reg(i_allocated_regs, reg);
4201 }
4202 }
4203 if (!allocate_new_reg) {
4204 temp_load(s, ts, i_required_regs, i_allocated_regs,
4205 i_preferred_regs);
4206 reg = ts->reg;
4207 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
4208 }
4209 if (allocate_new_reg) {
4210 /*
4211 * Allocate a new register matching the constraint
4212 * and move the temporary register into it.
4213 */
4214 temp_load(s, ts, tcg_target_available_regs[ts->type],
4215 i_allocated_regs, 0);
4216 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
4217 i_preferred_regs, ts->indirect_base);
4218 copyto_new_reg = true;
4219 }
4220 break;
4221
4222 case 1:
4223 /* First of an input pair; if i1 == i2, the second is an output. */
4224 i1 = i;
4225 i2 = arg_ct->pair_index;
4226 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
4227
4228 /*
4229 * It is easier to default to allocating a new pair
4230 * and to identify a few cases where it's not required.
4231 */
4232 if (arg_ct->ialias) {
31fd884b 4233 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 4234 if (IS_DEAD_ARG(i1) &&
4235 IS_DEAD_ARG(i2) &&
4236 !temp_readonly(ts) &&
4237 ts->val_type == TEMP_VAL_REG &&
4238 ts->reg < TCG_TARGET_NB_REGS - 1 &&
4239 tcg_regset_test_reg(i_required_regs, reg) &&
4240 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4241 !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
4242 (ts2
4243 ? ts2->val_type == TEMP_VAL_REG &&
4244 ts2->reg == reg + 1 &&
4245 !temp_readonly(ts2)
4246 : s->reg_to_temp[reg + 1] == NULL)) {
4247 break;
4248 }
4249 } else {
4250 /* Without aliasing, the pair must also be an input. */
4251 tcg_debug_assert(ts2);
4252 if (ts->val_type == TEMP_VAL_REG &&
4253 ts2->val_type == TEMP_VAL_REG &&
4254 ts2->reg == reg + 1 &&
4255 tcg_regset_test_reg(i_required_regs, reg)) {
4256 break;
4257 }
4258 }
4259 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
4260 0, ts->indirect_base);
4261 goto do_pair;
4262
4263 case 2: /* pair second */
4264 reg = new_args[arg_ct->pair_index] + 1;
4265 goto do_pair;
1c1824dc 4266
29f5e925
RH		 4267 case 3: /* ialias with second output, no first input */
4268 tcg_debug_assert(arg_ct->ialias);
31fd884b 4269 i_preferred_regs = output_pref(op, arg_ct->alias_index);
d62816f2 4270
29f5e925
RH		 4271 if (IS_DEAD_ARG(i) &&
4272 !temp_readonly(ts) &&
4273 ts->val_type == TEMP_VAL_REG &&
4274 reg > 0 &&
4275 s->reg_to_temp[reg - 1] == NULL &&
4276 tcg_regset_test_reg(i_required_regs, reg) &&
4277 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4278 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
4279 tcg_regset_set_reg(i_allocated_regs, reg - 1);
4280 break;
4281 }
4282 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
4283 i_allocated_regs, 0,
4284 ts->indirect_base);
4285 tcg_regset_set_reg(i_allocated_regs, reg);
4286 reg += 1;
4287 goto do_pair;
4288
4289 do_pair:
c0522136 4290 /*
29f5e925
RH		 4291 * If an aliased input is not dead after the instruction,
4292 * we must allocate a new register and move it.
c0522136 4293 */
29f5e925
RH		 4294 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
4295 TCGRegSet t_allocated_regs = i_allocated_regs;
4296
1c1824dc 4297 /*
29f5e925
RH		 4298 * Because of the alias, and the continued life, make sure
4299 * that the temp is somewhere *other* than the reg pair,
4300 * and we get a copy in reg.
1c1824dc 4301 */
29f5e925
RH		 4302 tcg_regset_set_reg(t_allocated_regs, reg);
4303 tcg_regset_set_reg(t_allocated_regs, reg + 1);
4304 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
4305 /* If ts was already in reg, copy it somewhere else. */
4306 TCGReg nr;
4307 bool ok;
4308
4309 tcg_debug_assert(ts->kind != TEMP_FIXED);
4310 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
4311 t_allocated_regs, 0, ts->indirect_base);
4312 ok = tcg_out_mov(s, ts->type, nr, reg);
4313 tcg_debug_assert(ok);
4314
4315 set_temp_val_reg(s, ts, nr);
4316 } else {
4317 temp_load(s, ts, tcg_target_available_regs[ts->type],
4318 t_allocated_regs, 0);
4319 copyto_new_reg = true;
4320 }
4321 } else {
4322 /* Preferably allocate to reg, otherwise copy. */
4323 i_required_regs = (TCGRegSet)1 << reg;
4324 temp_load(s, ts, i_required_regs, i_allocated_regs,
4325 i_preferred_regs);
4326 copyto_new_reg = ts->reg != reg;
5ff9d6a4 4327 }
29f5e925 4328 break;
d62816f2 4329
29f5e925
RH		 4330 default:
4331 g_assert_not_reached();
1c1824dc 4332 }
d62816f2 4333
29f5e925 4334 if (copyto_new_reg) {
78113e83 4335 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 4336 /*
4337 * Cross register class move not supported. Sync the
4338 * temp back to its slot and load from there.
4339 */
4340 temp_sync(s, ts, i_allocated_regs, 0, 0);
4341 tcg_out_ld(s, ts->type, reg,
4342 ts->mem_base->reg, ts->mem_offset);
78113e83 4343 }
c896fe29 4344 }
c896fe29
FB		 4345 new_args[i] = reg;
4346 const_args[i] = 0;
82790a87 4347 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29 4348 }
a813e36f 4349
a52ad07e
AJ		 4350 /* mark dead temporaries and free the associated registers */
4351 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4352 if (IS_DEAD_ARG(i)) {
43439139 4353 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 4354 }
4355 }
4356
b4cb76e6
RH		 4357 if (def->flags & TCG_OPF_COND_BRANCH) {
4358 tcg_reg_alloc_cbranch(s, i_allocated_regs);
4359 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 4360 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 4361 } else {
e8996ee0 4362 if (def->flags & TCG_OPF_CALL_CLOBBER) {
a813e36f 4363 /* XXX: permit generic clobber register list ? */
c8074023
RH		 4364 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4365 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 4366 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 4367 }
c896fe29 4368 }
3d5c5f87
AJ		 4369 }
4370 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4371 /* sync globals if the op has side effects and might trigger
4372 an exception. */
82790a87 4373 sync_globals(s, i_allocated_regs);
c896fe29 4374 }
a813e36f 4375
e8996ee0 4376 /* satisfy the output constraints */
e8996ee0 4377 for(k = 0; k < nb_oargs; k++) {
66792f90 4378 i = def->args_ct[k].sort_index;
dd186292 4379 arg = op->args[i];
e8996ee0 4380 arg_ct = &def->args_ct[i];
43439139 4381 ts = arg_temp(arg);
d63e3b6e
RH		 4382
4383 /* ENV should not be modified. */
e01fa97d 4384 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4385
29f5e925
RH		 4386 switch (arg_ct->pair) {
4387 case 0: /* not paired */
4388 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
4389 reg = new_args[arg_ct->alias_index];
4390 } else if (arg_ct->newreg) {
4391 reg = tcg_reg_alloc(s, arg_ct->regs,
4392 i_allocated_regs | o_allocated_regs,
31fd884b 4393 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4394 } else {
4395 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
31fd884b 4396 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4397 }
4398 break;
4399
4400 case 1: /* first of pair */
4401 tcg_debug_assert(!arg_ct->newreg);
4402 if (arg_ct->oalias) {
4403 reg = new_args[arg_ct->alias_index];
4404 break;
4405 }
4406 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
31fd884b 4407 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4408 break;
4409
4410 case 2: /* second of pair */
4411 tcg_debug_assert(!arg_ct->newreg);
4412 if (arg_ct->oalias) {
4413 reg = new_args[arg_ct->alias_index];
4414 } else {
4415 reg = new_args[arg_ct->pair_index] + 1;
4416 }
4417 break;
4418
4419 case 3: /* first of pair, aliasing with a second input */
4420 tcg_debug_assert(!arg_ct->newreg);
4421 reg = new_args[arg_ct->pair_index] - 1;
4422 break;
4423
4424 default:
4425 g_assert_not_reached();
c896fe29 4426 }
82790a87 4427 tcg_regset_set_reg(o_allocated_regs, reg);
098859f1 4428 set_temp_val_reg(s, ts, reg);
d63e3b6e 4429 ts->mem_coherent = 0;
e8996ee0 4430 new_args[i] = reg;
c896fe29 4431 }
c896fe29
FB		 4432 }
4433
c896fe29 4434 /* emit instruction */
d2fd745f
RH		 4435 if (def->flags & TCG_OPF_VECTOR) {
4436 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
4437 new_args, const_args);
4438 } else {
4439 tcg_out_op(s, op->opc, new_args, const_args);
4440 }
4441
c896fe29
FB		 4442 /* move the outputs in the correct register if needed */
4443 for(i = 0; i < nb_oargs; i++) {
43439139 4444 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 4445
4446 /* ENV should not be modified. */
e01fa97d 4447 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4448
ec7a869d 4449 if (NEED_SYNC_ARG(i)) {
98b4e186 4450 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 4451 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 4452 temp_dead(s, ts);
ec7a869d 4453 }
c896fe29
FB		 4454 }
4455}
4456
efe86b21
RH		 4457static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
4458{
4459 const TCGLifeData arg_life = op->life;
4460 TCGTemp *ots, *itsl, *itsh;
4461 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4462
4463 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
4464 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
4465 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
4466
4467 ots = arg_temp(op->args[0]);
4468 itsl = arg_temp(op->args[1]);
4469 itsh = arg_temp(op->args[2]);
4470
4471 /* ENV should not be modified. */
4472 tcg_debug_assert(!temp_readonly(ots));
4473
4474 /* Allocate the output register now. */
4475 if (ots->val_type != TEMP_VAL_REG) {
4476 TCGRegSet allocated_regs = s->reserved_regs;
4477 TCGRegSet dup_out_regs =
4478 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
098859f1 4479 TCGReg oreg;
efe86b21
RH		 4480
4481 /* Make sure to not spill the input registers. */
4482 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
4483 tcg_regset_set_reg(allocated_regs, itsl->reg);
4484 }
4485 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
4486 tcg_regset_set_reg(allocated_regs, itsh->reg);
4487 }
4488
098859f1 4489 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 4490 output_pref(op, 0), ots->indirect_base);
098859f1 4491 set_temp_val_reg(s, ots, oreg);
efe86b21
RH		 4492 }
4493
4494 /* Promote dup2 of immediates to dupi_vec. */
4495 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
4496 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
4497 MemOp vece = MO_64;
4498
4499 if (val == dup_const(MO_8, val)) {
4500 vece = MO_8;
4501 } else if (val == dup_const(MO_16, val)) {
4502 vece = MO_16;
4503 } else if (val == dup_const(MO_32, val)) {
4504 vece = MO_32;
4505 }
4506
4507 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
4508 goto done;
4509 }
4510
4511 /* If the two inputs form one 64-bit value, try dupm_vec. */
aef85402
RH		 4512 if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
4513 itsh->temp_subindex == !HOST_BIG_ENDIAN &&
4514 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
4515 TCGTemp *its = itsl - HOST_BIG_ENDIAN;
4516
4517 temp_sync(s, its + 0, s->reserved_regs, 0, 0);
4518 temp_sync(s, its + 1, s->reserved_regs, 0, 0);
4519
efe86b21
RH		 4520 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
4521 its->mem_base->reg, its->mem_offset)) {
4522 goto done;
4523 }
4524 }
4525
4526 /* Fall back to generic expansion. */
4527 return false;
4528
4529 done:
36f5539c 4530 ots->mem_coherent = 0;
efe86b21
RH		 4531 if (IS_DEAD_ARG(1)) {
4532 temp_dead(s, itsl);
4533 }
4534 if (IS_DEAD_ARG(2)) {
4535 temp_dead(s, itsh);
4536 }
4537 if (NEED_SYNC_ARG(0)) {
4538 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
4539 } else if (IS_DEAD_ARG(0)) {
4540 temp_dead(s, ots);
4541 }
4542 return true;
4543}
4544
39004a71
RH		 4545static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
4546 TCGRegSet allocated_regs)
c896fe29 4547{
39004a71
RH		 4548 if (ts->val_type == TEMP_VAL_REG) {
4549 if (ts->reg != reg) {
4550 tcg_reg_free(s, reg, allocated_regs);
4551 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4552 /*
4553 * Cross register class move not supported. Sync the
4554 * temp back to its slot and load from there.
4555 */
4556 temp_sync(s, ts, allocated_regs, 0, 0);
4557 tcg_out_ld(s, ts->type, reg,
4558 ts->mem_base->reg, ts->mem_offset);
4559 }
4560 }
4561 } else {
4562 TCGRegSet arg_set = 0;
c896fe29 4563
39004a71
RH		 4564 tcg_reg_free(s, reg, allocated_regs);
4565 tcg_regset_set_reg(arg_set, reg);
4566 temp_load(s, ts, arg_set, allocated_regs, 0);
b03cce8e 4567 }
39004a71 4568}
39cf05d3 4569
39004a71
RH		 4570static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts,
4571 TCGRegSet allocated_regs)
4572{
4573 /*
4574 * When the destination is on the stack, load up the temp and store.
4575 * If there are many call-saved registers, the temp might live to
4576 * see another use; otherwise it'll be discarded.
4577 */
4578 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
4579 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
4580 TCG_TARGET_CALL_STACK_OFFSET +
4581 stk_slot * sizeof(tcg_target_long));
4582}
a813e36f 4583
39004a71
RH		 4584static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
4585 TCGTemp *ts, TCGRegSet *allocated_regs)
4586{
4587 if (REG_P(l)) {
4588 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
4589 load_arg_reg(s, reg, ts, *allocated_regs);
4590 tcg_regset_set_reg(*allocated_regs, reg);
4591 } else {
4592 load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs),
4593 ts, *allocated_regs);
4594 }
4595}
40ae5c62 4596
313bdea8
RH		 4597static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base,
4598 intptr_t ref_off, TCGRegSet *allocated_regs)
4599{
4600 TCGReg reg;
4601 int stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
4602
4603 if (stk_slot < 0) {
4604 reg = tcg_target_call_iarg_regs[arg_slot];
4605 tcg_reg_free(s, reg, *allocated_regs);
4606 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
4607 tcg_regset_set_reg(*allocated_regs, reg);
4608 } else {
4609 reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR],
4610 *allocated_regs, 0, false);
4611 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
4612 tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK,
4613 TCG_TARGET_CALL_STACK_OFFSET
4614 + stk_slot * sizeof(tcg_target_long));
4615 }
4616}
4617
39004a71
RH		 4618static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
4619{
4620 const int nb_oargs = TCGOP_CALLO(op);
4621 const int nb_iargs = TCGOP_CALLI(op);
4622 const TCGLifeData arg_life = op->life;
4623 const TCGHelperInfo *info = tcg_call_info(op);
4624 TCGRegSet allocated_regs = s->reserved_regs;
4625 int i;
40ae5c62 4626
39004a71
RH		 4627 /*
4628 * Move inputs into place in reverse order,
4629 * so that we place stacked arguments first.
4630 */
4631 for (i = nb_iargs - 1; i >= 0; --i) {
4632 const TCGCallArgumentLoc *loc = &info->in[i];
4633 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
40ae5c62 4634
39004a71
RH		 4635 switch (loc->kind) {
4636 case TCG_CALL_ARG_NORMAL:
4637 case TCG_CALL_ARG_EXTEND_U:
4638 case TCG_CALL_ARG_EXTEND_S:
4639 load_arg_normal(s, loc, ts, &allocated_regs);
4640 break;
313bdea8
RH		 4641 case TCG_CALL_ARG_BY_REF:
4642 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
4643 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK,
4644 TCG_TARGET_CALL_STACK_OFFSET
4645 + loc->ref_slot * sizeof(tcg_target_long),
4646 &allocated_regs);
4647 break;
4648 case TCG_CALL_ARG_BY_REF_N:
4649 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
4650 break;
39004a71
RH		 4651 default:
4652 g_assert_not_reached();
c896fe29 4653 }
c896fe29 4654 }
a813e36f 4655
39004a71 4656 /* Mark dead temporaries and free the associated registers. */
dd186292 4657 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 4658 if (IS_DEAD_ARG(i)) {
43439139 4659 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 4660 }
4661 }
a813e36f 4662
39004a71 4663 /* Clobber call registers. */
c8074023
RH		 4664 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4665 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 4666 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 4667 }
4668 }
78505279 4669
39004a71
RH		 4670 /*
4671 * Save globals if they might be written by the helper,
4672 * sync them if they might be read.
4673 */
4674 if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
78505279 4675 /* Nothing to do */
39004a71 4676 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
78505279
AJ		 4677 sync_globals(s, allocated_regs);
4678 } else {
b9c18f56
AJ		 4679 save_globals(s, allocated_regs);
4680 }
c896fe29 4681
313bdea8
RH		 4682 /*
4683 * If the ABI passes a pointer to the returned struct as the first
4684 * argument, load that now. Pass a pointer to the output home slot.
4685 */
4686 if (info->out_kind == TCG_CALL_RET_BY_REF) {
4687 TCGTemp *ts = arg_temp(op->args[0]);
4688
4689 if (!ts->mem_allocated) {
4690 temp_allocate_frame(s, ts);
4691 }
4692 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs);
4693 }
4694
cee44b03 4695 tcg_out_call(s, tcg_call_func(op), info);
c896fe29 4696
39004a71
RH		 4697 /* Assign output registers and emit moves if needed. */
4698 switch (info->out_kind) {
4699 case TCG_CALL_RET_NORMAL:
4700 for (i = 0; i < nb_oargs; i++) {
4701 TCGTemp *ts = arg_temp(op->args[i]);
5e3d0c19 4702 TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i);
d63e3b6e 4703
39004a71
RH		 4704 /* ENV should not be modified. */
4705 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4706
39004a71
RH		 4707 set_temp_val_reg(s, ts, reg);
4708 ts->mem_coherent = 0;
4709 }
4710 break;
313bdea8 4711
c6556aa0
RH		 4712 case TCG_CALL_RET_BY_VEC:
4713 {
4714 TCGTemp *ts = arg_temp(op->args[0]);
4715
4716 tcg_debug_assert(ts->base_type == TCG_TYPE_I128);
4717 tcg_debug_assert(ts->temp_subindex == 0);
4718 if (!ts->mem_allocated) {
4719 temp_allocate_frame(s, ts);
4720 }
4721 tcg_out_st(s, TCG_TYPE_V128,
4722 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
4723 ts->mem_base->reg, ts->mem_offset);
4724 }
4725 /* fall through to mark all parts in memory */
4726
313bdea8
RH		 4727 case TCG_CALL_RET_BY_REF:
4728 /* The callee has performed a write through the reference. */
4729 for (i = 0; i < nb_oargs; i++) {
4730 TCGTemp *ts = arg_temp(op->args[i]);
4731 ts->val_type = TEMP_VAL_MEM;
4732 }
4733 break;
4734
39004a71
RH		 4735 default:
4736 g_assert_not_reached();
4737 }
4738
4739 /* Flush or discard output registers as needed. */
4740 for (i = 0; i < nb_oargs; i++) {
4741 TCGTemp *ts = arg_temp(op->args[i]);
d63e3b6e 4742 if (NEED_SYNC_ARG(i)) {
39004a71 4743 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
d63e3b6e
RH		 4744 } else if (IS_DEAD_ARG(i)) {
4745 temp_dead(s, ts);
c896fe29
FB		 4746 }
4747 }
c896fe29
FB		 4748}
4749
4750#ifdef CONFIG_PROFILER
4751
c3fac113
EC		 4752/* avoid copy/paste errors */
4753#define PROF_ADD(to, from, field) \
4754 do { \
d73415a3 4755 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 4756 } while (0)
4757
4758#define PROF_MAX(to, from, field) \
4759 do { \
d73415a3 4760 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 4761 if (val__ > (to)->field) { \
4762 (to)->field = val__; \
4763 } \
4764 } while (0)
4765
4766/* Pass in a zero'ed @prof */
4767static inline
4768void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4769{
0e2d61cf 4770 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 4771 unsigned int i;
4772
3468b59e 4773 for (i = 0; i < n_ctxs; i++) {
d73415a3 4774 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 4775 const TCGProfile *orig = &s->prof;
c3fac113
EC		 4776
4777 if (counters) {
72fd2efb 4778 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 4779 PROF_ADD(prof, orig, tb_count1);
4780 PROF_ADD(prof, orig, tb_count);
4781 PROF_ADD(prof, orig, op_count);
4782 PROF_MAX(prof, orig, op_count_max);
4783 PROF_ADD(prof, orig, temp_count);
4784 PROF_MAX(prof, orig, temp_count_max);
4785 PROF_ADD(prof, orig, del_op_count);
4786 PROF_ADD(prof, orig, code_in_len);
4787 PROF_ADD(prof, orig, code_out_len);
4788 PROF_ADD(prof, orig, search_out_len);
4789 PROF_ADD(prof, orig, interm_time);
4790 PROF_ADD(prof, orig, code_time);
4791 PROF_ADD(prof, orig, la_time);
4792 PROF_ADD(prof, orig, opt_time);
4793 PROF_ADD(prof, orig, restore_count);
4794 PROF_ADD(prof, orig, restore_time);
4795 }
4796 if (table) {
4797 int i;
4798
4799 for (i = 0; i < NB_OPS; i++) {
4800 PROF_ADD(prof, orig, table_op_count[i]);
4801 }
4802 }
4803 }
4804}
4805
4806#undef PROF_ADD
4807#undef PROF_MAX
4808
4809static void tcg_profile_snapshot_counters(TCGProfile *prof)
4810{
4811 tcg_profile_snapshot(prof, true, false);
4812}
4813
4814static void tcg_profile_snapshot_table(TCGProfile *prof)
4815{
4816 tcg_profile_snapshot(prof, false, true);
4817}
c896fe29 4818
b6a7f3e0 4819void tcg_dump_op_count(GString *buf)
c896fe29 4820{
c3fac113 4821 TCGProfile prof = {};
c896fe29 4822 int i;
d70724ce 4823
c3fac113 4824 tcg_profile_snapshot_table(&prof);
15fc7daa 4825 for (i = 0; i < NB_OPS; i++) {
b6a7f3e0
DB		 4826 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
4827 prof.table_op_count[i]);
c896fe29 4828 }
c896fe29 4829}
72fd2efb
EC		 4830
4831int64_t tcg_cpu_exec_time(void)
4832{
0e2d61cf 4833 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4834 unsigned int i;
4835 int64_t ret = 0;
4836
4837 for (i = 0; i < n_ctxs; i++) {
d73415a3 4838 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4839 const TCGProfile *prof = &s->prof;
4840
d73415a3 4841 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4842 }
4843 return ret;
4844}
246ae24d 4845#else
b6a7f3e0 4846void tcg_dump_op_count(GString *buf)
246ae24d 4847{
b6a7f3e0 4848 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
246ae24d 4849}
72fd2efb
EC		 4850
4851int64_t tcg_cpu_exec_time(void)
4852{
4853 error_report("%s: TCG profiler not compiled", __func__);
4854 exit(EXIT_FAILURE);
4855}
c896fe29
FB		 4856#endif
4857
4858
fbf59aad 4859int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
c896fe29 4860{
c3fac113
EC		 4861#ifdef CONFIG_PROFILER
4862 TCGProfile *prof = &s->prof;
4863#endif
15fa08f8
RH		 4864 int i, num_insns;
4865 TCGOp *op;
c896fe29 4866
04fe6400
RH		 4867#ifdef CONFIG_PROFILER
4868 {
c1f543b7 4869 int n = 0;
04fe6400 4870
15fa08f8
RH		 4871 QTAILQ_FOREACH(op, &s->ops, link) {
4872 n++;
4873 }
d73415a3 4874 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4875 if (n > prof->op_count_max) {
d73415a3 4876 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4877 }
4878
4879 n = s->nb_temps;
d73415a3 4880 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4881 if (n > prof->temp_count_max) {
d73415a3 4882 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4883 }
4884 }
4885#endif
4886
c896fe29 4887#ifdef DEBUG_DISAS
d977e1c2 4888 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
fbf59aad 4889 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4890 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4891 if (logfile) {
4892 fprintf(logfile, "OP:\n");
b7a83ff8 4893 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4894 fprintf(logfile, "\n");
4895 qemu_log_unlock(logfile);
4896 }
c896fe29
FB		 4897 }
4898#endif
4899
bef16ab4
RH		 4900#ifdef CONFIG_DEBUG_TCG
4901 /* Ensure all labels referenced have been emitted. */
4902 {
4903 TCGLabel *l;
4904 bool error = false;
4905
4906 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4907 if (unlikely(!l->present) && l->refs) {
4908 qemu_log_mask(CPU_LOG_TB_OP,
4909 "$L%d referenced but not present.\n", l->id);
4910 error = true;
4911 }
4912 }
4913 assert(!error);
4914 }
4915#endif
4916
c5cc28ff 4917#ifdef CONFIG_PROFILER
d73415a3 4918 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4919#endif
4920
8f2e8c07 4921#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4922 tcg_optimize(s);
8f2e8c07
KB		 4923#endif
4924
a23a9ec6 4925#ifdef CONFIG_PROFILER
d73415a3
SH		 4926 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4927 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4928#endif
c5cc28ff 4929
b4fc67c7 4930 reachable_code_pass(s);
874b8574 4931 liveness_pass_0(s);
b83eabea 4932 liveness_pass_1(s);
5a18407f 4933
b83eabea 4934 if (s->nb_indirects > 0) {
5a18407f 4935#ifdef DEBUG_DISAS
b83eabea 4936 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
fbf59aad 4937 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4938 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4939 if (logfile) {
4940 fprintf(logfile, "OP before indirect lowering:\n");
b7a83ff8 4941 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4942 fprintf(logfile, "\n");
4943 qemu_log_unlock(logfile);
4944 }
b83eabea 4945 }
5a18407f 4946#endif
b83eabea
RH		 4947 /* Replace indirect temps with direct temps. */
4948 if (liveness_pass_2(s)) {
4949 /* If changes were made, re-run liveness. */
4950 liveness_pass_1(s);
5a18407f
RH		 4951 }
4952 }
c5cc28ff 4953
a23a9ec6 4954#ifdef CONFIG_PROFILER
d73415a3 4955 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4956#endif
c896fe29
FB		 4957
4958#ifdef DEBUG_DISAS
d977e1c2 4959 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
fbf59aad 4960 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4961 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4962 if (logfile) {
4963 fprintf(logfile, "OP after optimization and liveness analysis:\n");
b7a83ff8 4964 tcg_dump_ops(s, logfile, true);
78b54858
RH		 4965 fprintf(logfile, "\n");
4966 qemu_log_unlock(logfile);
4967 }
c896fe29
FB		 4968 }
4969#endif
4970
35abb009 4971 /* Initialize goto_tb jump offsets. */
3a50f424
RH		 4972 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
4973 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
9da6079b
RH		 4974 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
4975 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
35abb009 4976
c896fe29
FB		 4977 tcg_reg_alloc_start(s);
4978
db0c51a3
RH		 4979 /*
4980 * Reset the buffer pointers when restarting after overflow.
4981 * TODO: Move this into translate-all.c with the rest of the
4982 * buffer management. Having only this done here is confusing.
4983 */
4984 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4985 s->code_ptr = s->code_buf;
c896fe29 4986
659ef5cb 4987#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4988 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4989#endif
57a26946
RH		 4990#ifdef TCG_TARGET_NEED_POOL_LABELS
4991 s->pool_labels = NULL;
4992#endif
9ecefc84 4993
fca8a500 4994 num_insns = -1;
15fa08f8 4995 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4996 TCGOpcode opc = op->opc;
b3db8758 4997
c896fe29 4998#ifdef CONFIG_PROFILER
d73415a3 4999 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 5000#endif
c45cb8bb
RH		 5001
5002 switch (opc) {
c896fe29 5003 case INDEX_op_mov_i32:
c896fe29 5004 case INDEX_op_mov_i64:
d2fd745f 5005 case INDEX_op_mov_vec:
dd186292 5006 tcg_reg_alloc_mov(s, op);
c896fe29 5007 break;
bab1671f
RH		 5008 case INDEX_op_dup_vec:
5009 tcg_reg_alloc_dup(s, op);
5010 break;
765b842a 5011 case INDEX_op_insn_start:
fca8a500 5012 if (num_insns >= 0) {
9f754620
RH		 5013 size_t off = tcg_current_code_size(s);
5014 s->gen_insn_end_off[num_insns] = off;
5015 /* Assert that we do not overflow our stored offset. */
5016 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 5017 }
5018 num_insns++;
bad729e2
RH		 5019 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
5020 target_ulong a;
5021#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 5022 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 5023#else
efee3746 5024 a = op->args[i];
bad729e2 5025#endif
fca8a500 5026 s->gen_insn_data[num_insns][i] = a;
bad729e2 5027 }
c896fe29 5028 break;
5ff9d6a4 5029 case INDEX_op_discard:
43439139 5030 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 5031 break;
c896fe29 5032 case INDEX_op_set_label:
e8996ee0 5033 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 5034 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 5035 break;
5036 case INDEX_op_call:
dd186292 5037 tcg_reg_alloc_call(s, op);
c45cb8bb 5038 break;
b55a8d9d
RH		 5039 case INDEX_op_exit_tb:
5040 tcg_out_exit_tb(s, op->args[0]);
5041 break;
cf7d6b8e
RH		 5042 case INDEX_op_goto_tb:
5043 tcg_out_goto_tb(s, op->args[0]);
5044 break;
efe86b21
RH		 5045 case INDEX_op_dup2_vec:
5046 if (tcg_reg_alloc_dup2(s, op)) {
5047 break;
5048 }
5049 /* fall through */
c896fe29 5050 default:
25c4d9cc 5051 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 5052 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 5053 /* Note: in order to speed up the code, it would be much
5054 faster to have specialized register allocator functions for
5055 some common argument patterns */
dd186292 5056 tcg_reg_alloc_op(s, op);
c896fe29
FB		 5057 break;
5058 }
b125f9dc
RH		 5059 /* Test for (pending) buffer overflow. The assumption is that any
5060 one operation beginning below the high water mark cannot overrun
5061 the buffer completely. Thus we can test for overflow after
5062 generating code without having to check during generation. */
644da9b3 5063 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 5064 return -1;
5065 }
6e6c4efe
RH		 5066 /* Test for TB overflow, as seen by gen_insn_end_off. */
5067 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
5068 return -2;
5069 }
c896fe29 5070 }
fca8a500
RH		 5071 tcg_debug_assert(num_insns >= 0);
5072 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 5073
b76f0d8c 5074 /* Generate TB finalization at the end of block */
659ef5cb 5075#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 5076 i = tcg_out_ldst_finalize(s);
5077 if (i < 0) {
5078 return i;
23dceda6 5079 }
659ef5cb 5080#endif
57a26946 5081#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 5082 i = tcg_out_pool_finalize(s);
5083 if (i < 0) {
5084 return i;
57a26946
RH		 5085 }
5086#endif
7ecd02a0
RH		 5087 if (!tcg_resolve_relocs(s)) {
5088 return -2;
5089 }
c896fe29 5090
df5d2b16 5091#ifndef CONFIG_TCG_INTERPRETER
c896fe29 5092 /* flush instruction cache */
db0c51a3
RH		 5093 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
5094 (uintptr_t)s->code_buf,
1da8de39 5095 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 5096#endif
2aeabc08 5097
1813e175 5098 return tcg_current_code_size(s);
c896fe29
FB		 5099}
5100
a23a9ec6 5101#ifdef CONFIG_PROFILER
3a841ab5 5102void tcg_dump_info(GString *buf)
a23a9ec6 5103{
c3fac113
EC		 5104 TCGProfile prof = {};
5105 const TCGProfile *s;
5106 int64_t tb_count;
5107 int64_t tb_div_count;
5108 int64_t tot;
5109
5110 tcg_profile_snapshot_counters(&prof);
5111 s = &prof;
5112 tb_count = s->tb_count;
5113 tb_div_count = tb_count ? tb_count : 1;
5114 tot = s->interm_time + s->code_time;
a23a9ec6 5115
3a841ab5
DB		 5116 g_string_append_printf(buf, "JIT cycles %" PRId64
5117 " (%0.3f s at 2.4 GHz)\n",
5118 tot, tot / 2.4e9);
5119 g_string_append_printf(buf, "translated TBs %" PRId64
5120 " (aborted=%" PRId64 " %0.1f%%)\n",
5121 tb_count, s->tb_count1 - tb_count,
5122 (double)(s->tb_count1 - s->tb_count)
5123 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
5124 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
5125 (double)s->op_count / tb_div_count, s->op_count_max);
5126 g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
5127 (double)s->del_op_count / tb_div_count);
5128 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
5129 (double)s->temp_count / tb_div_count,
5130 s->temp_count_max);
5131 g_string_append_printf(buf, "avg host code/TB %0.1f\n",
5132 (double)s->code_out_len / tb_div_count);
5133 g_string_append_printf(buf, "avg search data/TB %0.1f\n",
5134 (double)s->search_out_len / tb_div_count);
a813e36f 5135
3a841ab5
DB		 5136 g_string_append_printf(buf, "cycles/op %0.1f\n",
5137 s->op_count ? (double)tot / s->op_count : 0);
5138 g_string_append_printf(buf, "cycles/in byte %0.1f\n",
5139 s->code_in_len ? (double)tot / s->code_in_len : 0);
5140 g_string_append_printf(buf, "cycles/out byte %0.1f\n",
5141 s->code_out_len ? (double)tot / s->code_out_len : 0);
5142 g_string_append_printf(buf, "cycles/search byte %0.1f\n",
5143 s->search_out_len ?
5144 (double)tot / s->search_out_len : 0);
fca8a500 5145 if (tot == 0) {
a23a9ec6 5146 tot = 1;
fca8a500 5147 }
3a841ab5
DB		 5148 g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
5149 (double)s->interm_time / tot * 100.0);
5150 g_string_append_printf(buf, " gen_code time %0.1f%%\n",
5151 (double)s->code_time / tot * 100.0);
5152 g_string_append_printf(buf, "optim./code time %0.1f%%\n",
5153 (double)s->opt_time / (s->code_time ?
5154 s->code_time : 1)
5155 * 100.0);
5156 g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
5157 (double)s->la_time / (s->code_time ?
5158 s->code_time : 1) * 100.0);
5159 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
5160 s->restore_count);
5161 g_string_append_printf(buf, " avg cycles %0.1f\n",
5162 s->restore_count ?
5163 (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 5164}
5165#else
3a841ab5 5166void tcg_dump_info(GString *buf)
a23a9ec6 5167{
3a841ab5 5168 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
a23a9ec6
FB		 5169}
5170#endif
813da627
RH		 5171
5172#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 5173/* In order to use this feature, the backend needs to do three things:
5174
5175 (1) Define ELF_HOST_MACHINE to indicate both what value to
5176 put into the ELF image and to indicate support for the feature.
5177
5178 (2) Define tcg_register_jit. This should create a buffer containing
5179 the contents of a .debug_frame section that describes the post-
5180 prologue unwind info for the tcg machine.
5181
5182 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
5183*/
813da627
RH		 5184
5185/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
5186typedef enum {
5187 JIT_NOACTION = 0,
5188 JIT_REGISTER_FN,
5189 JIT_UNREGISTER_FN
5190} jit_actions_t;
5191
5192struct jit_code_entry {
5193 struct jit_code_entry *next_entry;
5194 struct jit_code_entry *prev_entry;
5195 const void *symfile_addr;
5196 uint64_t symfile_size;
5197};
5198
5199struct jit_descriptor {
5200 uint32_t version;
5201 uint32_t action_flag;
5202 struct jit_code_entry *relevant_entry;
5203 struct jit_code_entry *first_entry;
5204};
5205
5206void __jit_debug_register_code(void) __attribute__((noinline));
5207void __jit_debug_register_code(void)
5208{
5209 asm("");
5210}
5211
5212/* Must statically initialize the version, because GDB may check
5213 the version before we can set it. */
5214struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
5215
5216/* End GDB interface. */
5217
5218static int find_string(const char *strtab, const char *str)
5219{
5220 const char *p = strtab + 1;
5221
5222 while (1) {
5223 if (strcmp(p, str) == 0) {
5224 return p - strtab;
5225 }
5226 p += strlen(p) + 1;
5227 }
5228}
5229
755bf9e5 5230static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 5231 const void *debug_frame,
5232 size_t debug_frame_size)
813da627 5233{
5872bbf2
RH		 5234 struct __attribute__((packed)) DebugInfo {
5235 uint32_t len;
5236 uint16_t version;
5237 uint32_t abbrev;
5238 uint8_t ptr_size;
5239 uint8_t cu_die;
5240 uint16_t cu_lang;
5241 uintptr_t cu_low_pc;
5242 uintptr_t cu_high_pc;
5243 uint8_t fn_die;
5244 char fn_name[16];
5245 uintptr_t fn_low_pc;
5246 uintptr_t fn_high_pc;
5247 uint8_t cu_eoc;
5248 };
813da627
RH		 5249
5250 struct ElfImage {
5251 ElfW(Ehdr) ehdr;
5252 ElfW(Phdr) phdr;
5872bbf2
RH		 5253 ElfW(Shdr) shdr[7];
5254 ElfW(Sym) sym[2];
5255 struct DebugInfo di;
5256 uint8_t da[24];
5257 char str[80];
5258 };
5259
5260 struct ElfImage *img;
5261
5262 static const struct ElfImage img_template = {
5263 .ehdr = {
5264 .e_ident[EI_MAG0] = ELFMAG0,
5265 .e_ident[EI_MAG1] = ELFMAG1,
5266 .e_ident[EI_MAG2] = ELFMAG2,
5267 .e_ident[EI_MAG3] = ELFMAG3,
5268 .e_ident[EI_CLASS] = ELF_CLASS,
5269 .e_ident[EI_DATA] = ELF_DATA,
5270 .e_ident[EI_VERSION] = EV_CURRENT,
5271 .e_type = ET_EXEC,
5272 .e_machine = ELF_HOST_MACHINE,
5273 .e_version = EV_CURRENT,
5274 .e_phoff = offsetof(struct ElfImage, phdr),
5275 .e_shoff = offsetof(struct ElfImage, shdr),
5276 .e_ehsize = sizeof(ElfW(Shdr)),
5277 .e_phentsize = sizeof(ElfW(Phdr)),
5278 .e_phnum = 1,
5279 .e_shentsize = sizeof(ElfW(Shdr)),
5280 .e_shnum = ARRAY_SIZE(img->shdr),
5281 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 5282#ifdef ELF_HOST_FLAGS
5283 .e_flags = ELF_HOST_FLAGS,
5284#endif
5285#ifdef ELF_OSABI
5286 .e_ident[EI_OSABI] = ELF_OSABI,
5287#endif
5872bbf2
RH		 5288 },
5289 .phdr = {
5290 .p_type = PT_LOAD,
5291 .p_flags = PF_X,
5292 },
5293 .shdr = {
5294 [0] = { .sh_type = SHT_NULL },
5295 /* Trick: The contents of code_gen_buffer are not present in
5296 this fake ELF file; that got allocated elsewhere. Therefore
5297 we mark .text as SHT_NOBITS (similar to .bss) so that readers
5298 will not look for contents. We can record any address. */
5299 [1] = { /* .text */
5300 .sh_type = SHT_NOBITS,
5301 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
5302 },
5303 [2] = { /* .debug_info */
5304 .sh_type = SHT_PROGBITS,
5305 .sh_offset = offsetof(struct ElfImage, di),
5306 .sh_size = sizeof(struct DebugInfo),
5307 },
5308 [3] = { /* .debug_abbrev */
5309 .sh_type = SHT_PROGBITS,
5310 .sh_offset = offsetof(struct ElfImage, da),
5311 .sh_size = sizeof(img->da),
5312 },
5313 [4] = { /* .debug_frame */
5314 .sh_type = SHT_PROGBITS,
5315 .sh_offset = sizeof(struct ElfImage),
5316 },
5317 [5] = { /* .symtab */
5318 .sh_type = SHT_SYMTAB,
5319 .sh_offset = offsetof(struct ElfImage, sym),
5320 .sh_size = sizeof(img->sym),
5321 .sh_info = 1,
5322 .sh_link = ARRAY_SIZE(img->shdr) - 1,
5323 .sh_entsize = sizeof(ElfW(Sym)),
5324 },
5325 [6] = { /* .strtab */
5326 .sh_type = SHT_STRTAB,
5327 .sh_offset = offsetof(struct ElfImage, str),
5328 .sh_size = sizeof(img->str),
5329 }
5330 },
5331 .sym = {
5332 [1] = { /* code_gen_buffer */
5333 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
5334 .st_shndx = 1,
5335 }
5336 },
5337 .di = {
5338 .len = sizeof(struct DebugInfo) - 4,
5339 .version = 2,
5340 .ptr_size = sizeof(void *),
5341 .cu_die = 1,
5342 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
5343 .fn_die = 2,
5344 .fn_name = "code_gen_buffer"
5345 },
5346 .da = {
5347 1, /* abbrev number (the cu) */
5348 0x11, 1, /* DW_TAG_compile_unit, has children */
5349 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
5350 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5351 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5352 0, 0, /* end of abbrev */
5353 2, /* abbrev number (the fn) */
5354 0x2e, 0, /* DW_TAG_subprogram, no children */
5355 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
5356 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5357 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5358 0, 0, /* end of abbrev */
5359 0 /* no more abbrev */
5360 },
5361 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
5362 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 5363 };
5364
5365 /* We only need a single jit entry; statically allocate it. */
5366 static struct jit_code_entry one_entry;
5367
5872bbf2 5368 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 5369 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 5370 DebugFrameHeader *dfh;
813da627 5371
5872bbf2
RH		 5372 img = g_malloc(img_size);
5373 *img = img_template;
813da627 5374
5872bbf2
RH		 5375 img->phdr.p_vaddr = buf;
5376 img->phdr.p_paddr = buf;
5377 img->phdr.p_memsz = buf_size;
813da627 5378
813da627 5379 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 5380 img->shdr[1].sh_addr = buf;
813da627
RH		 5381 img->shdr[1].sh_size = buf_size;
5382
5872bbf2
RH		 5383 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
5384 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
5385
5386 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
5387 img->shdr[4].sh_size = debug_frame_size;
5388
5389 img->shdr[5].sh_name = find_string(img->str, ".symtab");
5390 img->shdr[6].sh_name = find_string(img->str, ".strtab");
5391
5392 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
5393 img->sym[1].st_value = buf;
5394 img->sym[1].st_size = buf_size;
813da627 5395
5872bbf2 5396 img->di.cu_low_pc = buf;
45aba097 5397 img->di.cu_high_pc = buf + buf_size;
5872bbf2 5398 img->di.fn_low_pc = buf;
45aba097 5399 img->di.fn_high_pc = buf + buf_size;
813da627 5400
2c90784a
RH		 5401 dfh = (DebugFrameHeader *)(img + 1);
5402 memcpy(dfh, debug_frame, debug_frame_size);
5403 dfh->fde.func_start = buf;
5404 dfh->fde.func_len = buf_size;
5405
813da627
RH		 5406#ifdef DEBUG_JIT
5407 /* Enable this block to be able to debug the ELF image file creation.
5408 One can use readelf, objdump, or other inspection utilities. */
5409 {
eb6b2edf
BM		 5410 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
5411 FILE *f = fopen(jit, "w+b");
813da627 5412 if (f) {
5872bbf2 5413 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 5414 /* Avoid stupid unused return value warning for fwrite. */
5415 }
5416 fclose(f);
5417 }
5418 }
5419#endif
5420
5421 one_entry.symfile_addr = img;
5422 one_entry.symfile_size = img_size;
5423
5424 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
5425 __jit_debug_descriptor.relevant_entry = &one_entry;
5426 __jit_debug_descriptor.first_entry = &one_entry;
5427 __jit_debug_register_code();
5428}
5429#else
5872bbf2
RH		 5430/* No support for the feature. Provide the entry point expected by exec.c,
5431 and implement the internal function we declared earlier. */
813da627 5432
755bf9e5 5433static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 5434 const void *debug_frame,
5435 size_t debug_frame_size)
813da627
RH		 5436{
5437}
5438
755bf9e5 5439void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 5440{
5441}
5442#endif /* ELF_HOST_MACHINE */
db432672
RH		 5443
5444#if !TCG_TARGET_MAYBE_vec
5445void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
5446{
5447 g_assert_not_reached();
5448}
5449#endif
QEMU mirror