]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg: Split out tcg_out_goto_tb
[mirror_qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
1de7afc9 37#include "qemu/timer.h"
084cfca1 38#include "qemu/cacheflush.h"
ad768e6f 39#include "qemu/cacheinfo.h"
c896fe29 40
c5d3c498 41/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 42 CPU definitions. Currently they are used for qemu_ld/st
43 instructions */
44#define NO_CPU_IO_DEFS
c896fe29 45
63c91552 46#include "exec/exec-all.h"
dcb32f1d 47#include "tcg/tcg-op.h"
813da627 48
edee2579 49#if UINTPTR_MAX == UINT32_MAX
813da627 50# define ELF_CLASS ELFCLASS32
edee2579
RH		 51#else
52# define ELF_CLASS ELFCLASS64
813da627 53#endif
e03b5686 54#if HOST_BIG_ENDIAN
813da627
RH		 55# define ELF_DATA ELFDATA2MSB
56#else
57# define ELF_DATA ELFDATA2LSB
58#endif
59
c896fe29 60#include "elf.h"
508127e2 61#include "exec/log.h"
d2ba8026 62#include "tcg/tcg-ldst.h"
5ff7258c 63#include "tcg-internal.h"
5584e2db 64#include "accel/tcg/perf.h"
c896fe29 65
139c1837 66/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 67 used here. */
e4d58b41
RH		 68static void tcg_target_init(TCGContext *s);
69static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 70static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 71 intptr_t value, intptr_t addend);
c896fe29 72
497a22eb
RH		 73/* The CIE and FDE header definitions will be common to all hosts. */
74typedef struct {
75 uint32_t len __attribute__((aligned((sizeof(void *)))));
76 uint32_t id;
77 uint8_t version;
78 char augmentation[1];
79 uint8_t code_align;
80 uint8_t data_align;
81 uint8_t return_column;
82} DebugFrameCIE;
83
84typedef struct QEMU_PACKED {
85 uint32_t len __attribute__((aligned((sizeof(void *)))));
86 uint32_t cie_offset;
edee2579
RH		 87 uintptr_t func_start;
88 uintptr_t func_len;
497a22eb
RH		 89} DebugFrameFDEHeader;
90
2c90784a
RH		 91typedef struct QEMU_PACKED {
92 DebugFrameCIE cie;
93 DebugFrameFDEHeader fde;
94} DebugFrameHeader;
95
755bf9e5 96static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 97 const void *debug_frame,
98 size_t debug_frame_size)
813da627
RH		 99 __attribute__((unused));
100
139c1837 101/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 102static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 103 intptr_t arg2);
78113e83 104static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 105static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 106 TCGReg ret, tcg_target_long arg);
b55a8d9d 107static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
cf7d6b8e 108static void tcg_out_goto_tb(TCGContext *s, int which);
5e8892db
MR		 109static void tcg_out_op(TCGContext *s, TCGOpcode opc,
110 const TCGArg args[TCG_MAX_OP_ARGS],
111 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 112#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 113static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
114 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 115static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
116 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 117static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
118 TCGReg dst, int64_t arg);
5e8892db
MR		 119static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
120 unsigned vecl, unsigned vece,
121 const TCGArg args[TCG_MAX_OP_ARGS],
122 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 123#else
e7632cfa
RH		 124static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
125 TCGReg dst, TCGReg src)
126{
127 g_assert_not_reached();
128}
d6ecb4a9
RH		 129static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
130 TCGReg dst, TCGReg base, intptr_t offset)
131{
132 g_assert_not_reached();
133}
4e186175
RH		 134static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
135 TCGReg dst, int64_t arg)
e7632cfa
RH		 136{
137 g_assert_not_reached();
138}
5e8892db
MR		 139static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
140 unsigned vecl, unsigned vece,
141 const TCGArg args[TCG_MAX_OP_ARGS],
142 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 143{
144 g_assert_not_reached();
145}
146#endif
2a534aff 147static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 148 intptr_t arg2);
59d7c14e
RH		 149static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
150 TCGReg base, intptr_t ofs);
7b7d8b2d 151static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
cee44b03 152 const TCGHelperInfo *info);
a4fbbd77 153static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 154#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 155static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 156#endif
c896fe29 157
42eb6dfc
RH		 158TCGContext tcg_init_ctx;
159__thread TCGContext *tcg_ctx;
160
5ff7258c 161TCGContext **tcg_ctxs;
0e2d61cf
RH		 162unsigned int tcg_cur_ctxs;
163unsigned int tcg_max_ctxs;
1c2adb95 164TCGv_env cpu_env = 0;
c8bc1168 165const void *tcg_code_gen_epilogue;
db0c51a3 166uintptr_t tcg_splitwx_diff;
df2cce29 167
b91ccb31
RH		 168#ifndef CONFIG_TCG_INTERPRETER
169tcg_prologue_fn *tcg_qemu_tb_exec;
170#endif
171
d2fd745f 172static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 173static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 174
1813e175 175#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 176static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 177{
178 *s->code_ptr++ = v;
179}
180
4196dca6
PM		 181static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
182 uint8_t v)
5c53bb81 183{
1813e175 184 *p = v;
5c53bb81 185}
1813e175 186#endif
5c53bb81 187
1813e175 188#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 189static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 190{
1813e175
RH		 191 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
192 *s->code_ptr++ = v;
193 } else {
194 tcg_insn_unit *p = s->code_ptr;
195 memcpy(p, &v, sizeof(v));
196 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
197 }
c896fe29
FB		 198}
199
4196dca6
PM		 200static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
201 uint16_t v)
5c53bb81 202{
1813e175
RH		 203 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
204 *p = v;
205 } else {
206 memcpy(p, &v, sizeof(v));
207 }
5c53bb81 208}
1813e175 209#endif
5c53bb81 210
1813e175 211#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 212static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 213{
1813e175
RH		 214 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
215 *s->code_ptr++ = v;
216 } else {
217 tcg_insn_unit *p = s->code_ptr;
218 memcpy(p, &v, sizeof(v));
219 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
220 }
c896fe29
FB		 221}
222
4196dca6
PM		 223static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
224 uint32_t v)
5c53bb81 225{
1813e175
RH		 226 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
227 *p = v;
228 } else {
229 memcpy(p, &v, sizeof(v));
230 }
5c53bb81 231}
1813e175 232#endif
5c53bb81 233
1813e175 234#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 235static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 236{
1813e175
RH		 237 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
238 *s->code_ptr++ = v;
239 } else {
240 tcg_insn_unit *p = s->code_ptr;
241 memcpy(p, &v, sizeof(v));
242 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
243 }
ac26eb69
RH		 244}
245
4196dca6
PM		 246static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
247 uint64_t v)
5c53bb81 248{
1813e175
RH		 249 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
250 *p = v;
251 } else {
252 memcpy(p, &v, sizeof(v));
253 }
5c53bb81 254}
1813e175 255#endif
5c53bb81 256
c896fe29
FB		 257/* label relocation processing */
258
1813e175 259static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 260 TCGLabel *l, intptr_t addend)
c896fe29 261{
7ecd02a0 262 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 263
7ecd02a0
RH		 264 r->type = type;
265 r->ptr = code_ptr;
266 r->addend = addend;
267 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 268}
269
92ab8e7d 270static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 271{
eabb7b91 272 tcg_debug_assert(!l->has_value);
c896fe29 273 l->has_value = 1;
92ab8e7d 274 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 275}
276
42a268c2 277TCGLabel *gen_new_label(void)
c896fe29 278{
b1311c4a 279 TCGContext *s = tcg_ctx;
51e3972c 280 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 281
7ecd02a0
RH		 282 memset(l, 0, sizeof(TCGLabel));
283 l->id = s->nb_labels++;
284 QSIMPLEQ_INIT(&l->relocs);
285
bef16ab4 286 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 287
288 return l;
c896fe29
FB		 289}
290
7ecd02a0
RH		 291static bool tcg_resolve_relocs(TCGContext *s)
292{
293 TCGLabel *l;
294
295 QSIMPLEQ_FOREACH(l, &s->labels, next) {
296 TCGRelocation *r;
297 uintptr_t value = l->u.value;
298
299 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
300 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
301 return false;
302 }
303 }
304 }
305 return true;
306}
307
9f754620
RH		 308static void set_jmp_reset_offset(TCGContext *s, int which)
309{
f14bed3f
RH		 310 /*
311 * We will check for overflow at the end of the opcode loop in
312 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
313 */
314 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 315}
316
b52a2c03
RH		 317static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
318{
319 /*
320 * We will check for overflow at the end of the opcode loop in
321 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
322 */
323 tcg_debug_assert(TCG_TARGET_HAS_direct_jump);
324 s->tb_jmp_insn_offset[which] = tcg_current_code_size(s);
325}
326
becc452a
RH		 327static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
328{
329 /*
330 * Return the read-execute version of the pointer, for the benefit
331 * of any pc-relative addressing mode.
332 */
333 return (uintptr_t)tcg_splitwx_to_rx(&s->tb_jmp_target_addr[which]);
334}
335
db6b7d0c 336/* Signal overflow, starting over with fewer guest insns. */
8905770b
MAL		 337static G_NORETURN
338void tcg_raise_tb_overflow(TCGContext *s)
db6b7d0c
RH		 339{
340 siglongjmp(s->jmp_trans, -2);
341}
342
4c22e840
RH		 343#define C_PFX1(P, A) P##A
344#define C_PFX2(P, A, B) P##A##_##B
345#define C_PFX3(P, A, B, C) P##A##_##B##_##C
346#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
347#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
348#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
349
350/* Define an enumeration for the various combinations. */
351
352#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
353#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
354#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
355#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
356
357#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
358#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
359#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
360#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
361
362#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
363
364#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
365#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
366#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
367#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
368
369typedef enum {
370#include "tcg-target-con-set.h"
371} TCGConstraintSetIndex;
372
373static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
374
375#undef C_O0_I1
376#undef C_O0_I2
377#undef C_O0_I3
378#undef C_O0_I4
379#undef C_O1_I1
380#undef C_O1_I2
381#undef C_O1_I3
382#undef C_O1_I4
383#undef C_N1_I2
384#undef C_O2_I1
385#undef C_O2_I2
386#undef C_O2_I3
387#undef C_O2_I4
388
389/* Put all of the constraint sets into an array, indexed by the enum. */
390
391#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
392#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
393#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
394#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
395
396#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
397#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
398#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
399#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
400
401#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
402
403#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
404#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
405#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
406#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
407
408static const TCGTargetOpDef constraint_sets[] = {
409#include "tcg-target-con-set.h"
410};
411
412
413#undef C_O0_I1
414#undef C_O0_I2
415#undef C_O0_I3
416#undef C_O0_I4
417#undef C_O1_I1
418#undef C_O1_I2
419#undef C_O1_I3
420#undef C_O1_I4
421#undef C_N1_I2
422#undef C_O2_I1
423#undef C_O2_I2
424#undef C_O2_I3
425#undef C_O2_I4
426
427/* Expand the enumerator to be returned from tcg_target_op_def(). */
428
429#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
430#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
431#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
432#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
433
434#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
435#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
436#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
437#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
438
439#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
440
441#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
442#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
443#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
444#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
445
139c1837 446#include "tcg-target.c.inc"
c896fe29 447
38b47b19
EC		 448static void alloc_tcg_plugin_context(TCGContext *s)
449{
450#ifdef CONFIG_PLUGIN
451 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
452 s->plugin_tb->insns =
453 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
454#endif
455}
456
3468b59e
EC		 457/*
458 * All TCG threads except the parent (i.e. the one that called tcg_context_init
459 * and registered the target's TCG globals) must register with this function
460 * before initiating translation.
461 *
462 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
463 * of tcg_region_init() for the reasoning behind this.
464 *
465 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
466 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
467 * is not used anymore for translation once this function is called.
468 *
469 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
470 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
471 */
472#ifdef CONFIG_USER_ONLY
473void tcg_register_thread(void)
474{
475 tcg_ctx = &tcg_init_ctx;
476}
477#else
478void tcg_register_thread(void)
479{
480 TCGContext *s = g_malloc(sizeof(*s));
481 unsigned int i, n;
3468b59e
EC		 482
483 *s = tcg_init_ctx;
484
485 /* Relink mem_base. */
486 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
487 if (tcg_init_ctx.temps[i].mem_base) {
488 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
489 tcg_debug_assert(b >= 0 && b < n);
490 s->temps[i].mem_base = &s->temps[b];
491 }
492 }
493
494 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 495 n = qatomic_fetch_inc(&tcg_cur_ctxs);
496 g_assert(n < tcg_max_ctxs);
d73415a3 497 qatomic_set(&tcg_ctxs[n], s);
3468b59e 498
38b47b19
EC		 499 if (n > 0) {
500 alloc_tcg_plugin_context(s);
bf042e8e 501 tcg_region_initial_alloc(s);
38b47b19
EC		 502 }
503
3468b59e 504 tcg_ctx = s;
e8feb96f 505}
3468b59e 506#endif /* !CONFIG_USER_ONLY */
e8feb96f 507
c896fe29
FB		 508/* pool based memory allocation */
509void *tcg_malloc_internal(TCGContext *s, int size)
510{
511 TCGPool *p;
512 int pool_size;
a813e36f 513
c896fe29
FB		 514 if (size > TCG_POOL_CHUNK_SIZE) {
515 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 516 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 517 p->size = size;
4055299e
KB		 518 p->next = s->pool_first_large;
519 s->pool_first_large = p;
520 return p->data;
c896fe29
FB		 521 } else {
522 p = s->pool_current;
523 if (!p) {
524 p = s->pool_first;
525 if (!p)
526 goto new_pool;
527 } else {
528 if (!p->next) {
529 new_pool:
530 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 531 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 532 p->size = pool_size;
533 p->next = NULL;
a813e36f 534 if (s->pool_current) {
c896fe29 535 s->pool_current->next = p;
a813e36f 536 } else {
c896fe29 537 s->pool_first = p;
a813e36f 538 }
c896fe29
FB		 539 } else {
540 p = p->next;
541 }
542 }
543 }
544 s->pool_current = p;
545 s->pool_cur = p->data + size;
546 s->pool_end = p->data + p->size;
547 return p->data;
548}
549
550void tcg_pool_reset(TCGContext *s)
551{
4055299e
KB		 552 TCGPool *p, *t;
553 for (p = s->pool_first_large; p; p = t) {
554 t = p->next;
555 g_free(p);
556 }
557 s->pool_first_large = NULL;
c896fe29
FB		 558 s->pool_cur = s->pool_end = NULL;
559 s->pool_current = NULL;
560}
561
2ef6175a
RH		 562#include "exec/helper-proto.h"
563
39004a71 564static TCGHelperInfo all_helpers[] = {
2ef6175a 565#include "exec/helper-tcg.h"
100b5e01 566};
619205fd 567static GHashTable *helper_table;
100b5e01 568
22f15579 569#ifdef CONFIG_TCG_INTERPRETER
c6ef8c7b
PMD		 570static ffi_type *typecode_to_ffi(int argmask)
571{
572 switch (argmask) {
573 case dh_typecode_void:
574 return &ffi_type_void;
575 case dh_typecode_i32:
576 return &ffi_type_uint32;
577 case dh_typecode_s32:
578 return &ffi_type_sint32;
579 case dh_typecode_i64:
580 return &ffi_type_uint64;
581 case dh_typecode_s64:
582 return &ffi_type_sint64;
583 case dh_typecode_ptr:
584 return &ffi_type_pointer;
585 }
586 g_assert_not_reached();
587}
0c22e176
PMD		 588
589static void init_ffi_layouts(void)
590{
591 /* g_direct_hash/equal for direct comparisons on uint32_t. */
f9c4bb80
RH		 592 GHashTable *ffi_table = g_hash_table_new(NULL, NULL);
593
0c22e176 594 for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
f9c4bb80
RH		 595 TCGHelperInfo *info = &all_helpers[i];
596 unsigned typemask = info->typemask;
0c22e176
PMD		 597 gpointer hash = (gpointer)(uintptr_t)typemask;
598 struct {
599 ffi_cif cif;
600 ffi_type *args[];
601 } *ca;
602 ffi_status status;
603 int nargs;
f9c4bb80 604 ffi_cif *cif;
0c22e176 605
f9c4bb80
RH		 606 cif = g_hash_table_lookup(ffi_table, hash);
607 if (cif) {
608 info->cif = cif;
0c22e176
PMD		 609 continue;
610 }
611
612 /* Ignoring the return type, find the last non-zero field. */
613 nargs = 32 - clz32(typemask >> 3);
614 nargs = DIV_ROUND_UP(nargs, 3);
615
616 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
617 ca->cif.rtype = typecode_to_ffi(typemask & 7);
618 ca->cif.nargs = nargs;
619
620 if (nargs != 0) {
621 ca->cif.arg_types = ca->args;
622 for (int j = 0; j < nargs; ++j) {
623 int typecode = extract32(typemask, (j + 1) * 3, 3);
624 ca->args[j] = typecode_to_ffi(typecode);
625 }
626 }
627
628 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
629 ca->cif.rtype, ca->cif.arg_types);
630 assert(status == FFI_OK);
631
f9c4bb80
RH		 632 cif = &ca->cif;
633 info->cif = cif;
634 g_hash_table_insert(ffi_table, hash, (gpointer)cif);
0c22e176 635 }
f9c4bb80
RH		 636
637 g_hash_table_destroy(ffi_table);
0c22e176
PMD		 638}
639#endif /* CONFIG_TCG_INTERPRETER */
22f15579 640
39004a71
RH		 641typedef struct TCGCumulativeArgs {
642 int arg_idx; /* tcg_gen_callN args[] */
643 int info_in_idx; /* TCGHelperInfo in[] */
644 int arg_slot; /* regs+stack slot */
645 int ref_slot; /* stack slots for references */
646} TCGCumulativeArgs;
647
648static void layout_arg_even(TCGCumulativeArgs *cum)
649{
650 cum->arg_slot += cum->arg_slot & 1;
651}
652
653static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
654 TCGCallArgumentKind kind)
655{
656 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
657
658 *loc = (TCGCallArgumentLoc){
659 .kind = kind,
660 .arg_idx = cum->arg_idx,
661 .arg_slot = cum->arg_slot,
662 };
663 cum->info_in_idx++;
664 cum->arg_slot++;
665}
666
667static void layout_arg_normal_n(TCGCumulativeArgs *cum,
668 TCGHelperInfo *info, int n)
669{
670 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
671
672 for (int i = 0; i < n; ++i) {
673 /* Layout all using the same arg_idx, adjusting the subindex. */
674 loc[i] = (TCGCallArgumentLoc){
675 .kind = TCG_CALL_ARG_NORMAL,
676 .arg_idx = cum->arg_idx,
677 .tmp_subindex = i,
678 .arg_slot = cum->arg_slot + i,
679 };
680 }
681 cum->info_in_idx += n;
682 cum->arg_slot += n;
683}
684
685static void init_call_layout(TCGHelperInfo *info)
686{
687 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
688 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
689 unsigned typemask = info->typemask;
690 unsigned typecode;
691 TCGCumulativeArgs cum = { };
692
693 /*
694 * Parse and place any function return value.
695 */
696 typecode = typemask & 7;
697 switch (typecode) {
698 case dh_typecode_void:
699 info->nr_out = 0;
700 break;
701 case dh_typecode_i32:
702 case dh_typecode_s32:
703 case dh_typecode_ptr:
704 info->nr_out = 1;
705 info->out_kind = TCG_CALL_RET_NORMAL;
706 break;
707 case dh_typecode_i64:
708 case dh_typecode_s64:
709 info->nr_out = 64 / TCG_TARGET_REG_BITS;
710 info->out_kind = TCG_CALL_RET_NORMAL;
711 break;
712 default:
713 g_assert_not_reached();
714 }
715 assert(info->nr_out <= ARRAY_SIZE(tcg_target_call_oarg_regs));
716
717 /*
718 * Parse and place function arguments.
719 */
720 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
721 TCGCallArgumentKind kind;
722 TCGType type;
723
724 typecode = typemask & 7;
725 switch (typecode) {
726 case dh_typecode_i32:
727 case dh_typecode_s32:
728 type = TCG_TYPE_I32;
729 break;
730 case dh_typecode_i64:
731 case dh_typecode_s64:
732 type = TCG_TYPE_I64;
733 break;
734 case dh_typecode_ptr:
735 type = TCG_TYPE_PTR;
736 break;
737 default:
738 g_assert_not_reached();
739 }
740
741 switch (type) {
742 case TCG_TYPE_I32:
743 switch (TCG_TARGET_CALL_ARG_I32) {
744 case TCG_CALL_ARG_EVEN:
745 layout_arg_even(&cum);
746 /* fall through */
747 case TCG_CALL_ARG_NORMAL:
748 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
749 break;
750 case TCG_CALL_ARG_EXTEND:
751 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
752 layout_arg_1(&cum, info, kind);
753 break;
754 default:
755 qemu_build_not_reached();
756 }
757 break;
758
759 case TCG_TYPE_I64:
760 switch (TCG_TARGET_CALL_ARG_I64) {
761 case TCG_CALL_ARG_EVEN:
762 layout_arg_even(&cum);
763 /* fall through */
764 case TCG_CALL_ARG_NORMAL:
765 if (TCG_TARGET_REG_BITS == 32) {
766 layout_arg_normal_n(&cum, info, 2);
767 } else {
768 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
769 }
770 break;
771 default:
772 qemu_build_not_reached();
773 }
774 break;
775
776 default:
777 g_assert_not_reached();
778 }
779 }
780 info->nr_in = cum.info_in_idx;
781
782 /* Validate that we didn't overrun the input array. */
783 assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
784 /* Validate the backend has enough argument space. */
785 assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
786 assert(cum.ref_slot <= max_stk_slots);
787}
788
91478cef 789static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 790static void process_op_defs(TCGContext *s);
1c2adb95
RH		 791static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
792 TCGReg reg, const char *name);
91478cef 793
43b972b7 794static void tcg_context_init(unsigned max_cpus)
c896fe29 795{
a76aabd3 796 TCGContext *s = &tcg_init_ctx;
100b5e01 797 int op, total_args, n, i;
c896fe29
FB		 798 TCGOpDef *def;
799 TCGArgConstraint *args_ct;
1c2adb95 800 TCGTemp *ts;
c896fe29
FB		 801
802 memset(s, 0, sizeof(*s));
c896fe29 803 s->nb_globals = 0;
c70fbf0a 804
c896fe29
FB		 805 /* Count total number of arguments and allocate the corresponding
806 space */
807 total_args = 0;
808 for(op = 0; op < NB_OPS; op++) {
809 def = &tcg_op_defs[op];
810 n = def->nb_iargs + def->nb_oargs;
811 total_args += n;
812 }
813
bc2b17e6 814 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 815
816 for(op = 0; op < NB_OPS; op++) {
817 def = &tcg_op_defs[op];
818 def->args_ct = args_ct;
c896fe29 819 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 820 args_ct += n;
821 }
5cd8f621
RH		 822
823 /* Register helpers. */
84fd9dd3 824 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 825 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 826
100b5e01 827 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
39004a71 828 init_call_layout(&all_helpers[i]);
84fd9dd3 829 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 830 (gpointer)&all_helpers[i]);
100b5e01 831 }
5cd8f621 832
22f15579 833#ifdef CONFIG_TCG_INTERPRETER
0c22e176 834 init_ffi_layouts();
22f15579
RH		 835#endif
836
c896fe29 837 tcg_target_init(s);
f69d277e 838 process_op_defs(s);
91478cef
RH		 839
840 /* Reverse the order of the saved registers, assuming they're all at
841 the start of tcg_target_reg_alloc_order. */
842 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
843 int r = tcg_target_reg_alloc_order[n];
844 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
845 break;
846 }
847 }
848 for (i = 0; i < n; ++i) {
849 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
850 }
851 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
852 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
853 }
b1311c4a 854
38b47b19
EC		 855 alloc_tcg_plugin_context(s);
856
b1311c4a 857 tcg_ctx = s;
3468b59e
EC		 858 /*
859 * In user-mode we simply share the init context among threads, since we
860 * use a single region. See the documentation tcg_region_init() for the
861 * reasoning behind this.
862 * In softmmu we will have at most max_cpus TCG threads.
863 */
864#ifdef CONFIG_USER_ONLY
df2cce29 865 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 866 tcg_cur_ctxs = 1;
867 tcg_max_ctxs = 1;
3468b59e 868#else
0e2d61cf
RH		 869 tcg_max_ctxs = max_cpus;
870 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 871#endif
1c2adb95
RH		 872
873 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
874 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
875 cpu_env = temp_tcgv_ptr(ts);
9002ec79 876}
b03cce8e 877
43b972b7 878void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 879{
43b972b7
RH		 880 tcg_context_init(max_cpus);
881 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 882}
883
6e3b2bfd
EC		 884/*
885 * Allocate TBs right before their corresponding translated code, making
886 * sure that TBs and code are on different cache lines.
887 */
888TranslationBlock *tcg_tb_alloc(TCGContext *s)
889{
890 uintptr_t align = qemu_icache_linesize;
891 TranslationBlock *tb;
892 void *next;
893
e8feb96f 894 retry:
6e3b2bfd
EC		 895 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
896 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
897
898 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 899 if (tcg_region_alloc(s)) {
900 return NULL;
901 }
902 goto retry;
6e3b2bfd 903 }
d73415a3 904 qatomic_set(&s->code_gen_ptr, next);
57a26946 905 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 906 return tb;
907}
908
9002ec79
RH		 909void tcg_prologue_init(TCGContext *s)
910{
b0a0794a 911 size_t prologue_size;
8163b749 912
b0a0794a
RH		 913 s->code_ptr = s->code_gen_ptr;
914 s->code_buf = s->code_gen_ptr;
5b38ee31 915 s->data_gen_ptr = NULL;
b91ccb31
RH		 916
917#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 918 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 919#endif
8163b749 920
5b38ee31
RH		 921#ifdef TCG_TARGET_NEED_POOL_LABELS
922 s->pool_labels = NULL;
923#endif
924
653b87eb 925 qemu_thread_jit_write();
8163b749 926 /* Generate the prologue. */
b03cce8e 927 tcg_target_qemu_prologue(s);
5b38ee31
RH		 928
929#ifdef TCG_TARGET_NEED_POOL_LABELS
930 /* Allow the prologue to put e.g. guest_base into a pool entry. */
931 {
1768987b
RH		 932 int result = tcg_out_pool_finalize(s);
933 tcg_debug_assert(result == 0);
5b38ee31
RH		 934 }
935#endif
936
b0a0794a 937 prologue_size = tcg_current_code_size(s);
5584e2db 938 perf_report_prologue(s->code_gen_ptr, prologue_size);
b0a0794a 939
df5d2b16 940#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 941 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
942 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 943#endif
8163b749 944
d6b64b2b
RH		 945#ifdef DEBUG_DISAS
946 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
c60f599b 947 FILE *logfile = qemu_log_trylock();
78b54858
RH		 948 if (logfile) {
949 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
950 if (s->data_gen_ptr) {
951 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
952 size_t data_size = prologue_size - code_size;
953 size_t i;
954
955 disas(logfile, s->code_gen_ptr, code_size);
956
957 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
958 if (sizeof(tcg_target_ulong) == 8) {
959 fprintf(logfile,
960 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
961 (uintptr_t)s->data_gen_ptr + i,
962 *(uint64_t *)(s->data_gen_ptr + i));
963 } else {
964 fprintf(logfile,
965 "0x%08" PRIxPTR ": .long 0x%08x\n",
966 (uintptr_t)s->data_gen_ptr + i,
967 *(uint32_t *)(s->data_gen_ptr + i));
968 }
5b38ee31 969 }
78b54858
RH		 970 } else {
971 disas(logfile, s->code_gen_ptr, prologue_size);
5b38ee31 972 }
78b54858 973 fprintf(logfile, "\n");
78b54858 974 qemu_log_unlock(logfile);
5b38ee31 975 }
d6b64b2b
RH		 976 }
977#endif
cedbcb01 978
6eea0434
RH		 979#ifndef CONFIG_TCG_INTERPRETER
980 /*
981 * Assert that goto_ptr is implemented completely, setting an epilogue.
982 * For tci, we use NULL as the signal to return from the interpreter,
983 * so skip this check.
984 */
f4e01e30 985 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 986#endif
d1c74ab3
RH		 987
988 tcg_region_prologue_set(s);
c896fe29
FB		 989}
990
c896fe29
FB		 991void tcg_func_start(TCGContext *s)
992{
993 tcg_pool_reset(s);
994 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 995
996 /* No temps have been previously allocated for size or locality. */
997 memset(s->free_temps, 0, sizeof(s->free_temps));
998
c0522136
RH		 999 /* No constant temps have been previously allocated. */
1000 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1001 if (s->const_table[i]) {
1002 g_hash_table_remove_all(s->const_table[i]);
1003 }
1004 }
1005
abebf925 1006 s->nb_ops = 0;
c896fe29
FB		 1007 s->nb_labels = 0;
1008 s->current_frame_offset = s->frame_start;
1009
0a209d4b
RH		 1010#ifdef CONFIG_DEBUG_TCG
1011 s->goto_tb_issue_mask = 0;
1012#endif
1013
15fa08f8
RH		 1014 QTAILQ_INIT(&s->ops);
1015 QTAILQ_INIT(&s->free_ops);
bef16ab4 1016 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 1017}
1018
ae30e866 1019static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 1020{
1021 int n = s->nb_temps++;
ae30e866
RH		 1022
1023 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 1024 tcg_raise_tb_overflow(s);
ae30e866 1025 }
7ca4b752
RH		 1026 return memset(&s->temps[n], 0, sizeof(TCGTemp));
1027}
1028
ae30e866 1029static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 1030{
fa477d25
RH		 1031 TCGTemp *ts;
1032
7ca4b752 1033 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 1034 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 1035 s->nb_globals++;
fa477d25 1036 ts = tcg_temp_alloc(s);
ee17db83 1037 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 1038
1039 return ts;
c896fe29
FB		 1040}
1041
085272b3
RH		 1042static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1043 TCGReg reg, const char *name)
c896fe29 1044{
c896fe29 1045 TCGTemp *ts;
c896fe29 1046
b3a62939 1047 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 1048 tcg_abort();
b3a62939 1049 }
7ca4b752
RH		 1050
1051 ts = tcg_global_alloc(s);
c896fe29
FB		 1052 ts->base_type = type;
1053 ts->type = type;
ee17db83 1054 ts->kind = TEMP_FIXED;
c896fe29 1055 ts->reg = reg;
c896fe29 1056 ts->name = name;
c896fe29 1057 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 1058
085272b3 1059 return ts;
a7812ae4
PB		 1060}
1061
b6638662 1062void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 1063{
b3a62939
RH		 1064 s->frame_start = start;
1065 s->frame_end = start + size;
085272b3
RH		 1066 s->frame_temp
1067 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 1068}
1069
085272b3
RH		 1070TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
1071 intptr_t offset, const char *name)
c896fe29 1072{
b1311c4a 1073 TCGContext *s = tcg_ctx;
dc41aa7d 1074 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 1075 TCGTemp *ts = tcg_global_alloc(s);
aef85402 1076 int indirect_reg = 0;
c896fe29 1077
c0522136
RH		 1078 switch (base_ts->kind) {
1079 case TEMP_FIXED:
1080 break;
1081 case TEMP_GLOBAL:
5a18407f
RH		 1082 /* We do not support double-indirect registers. */
1083 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 1084 base_ts->indirect_base = 1;
5a18407f
RH		 1085 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1086 ? 2 : 1);
1087 indirect_reg = 1;
c0522136
RH		 1088 break;
1089 default:
1090 g_assert_not_reached();
b3915dbb
RH		 1091 }
1092
7ca4b752
RH		 1093 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1094 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 1095 char buf[64];
7ca4b752
RH		 1096
1097 ts->base_type = TCG_TYPE_I64;
c896fe29 1098 ts->type = TCG_TYPE_I32;
b3915dbb 1099 ts->indirect_reg = indirect_reg;
c896fe29 1100 ts->mem_allocated = 1;
b3a62939 1101 ts->mem_base = base_ts;
aef85402 1102 ts->mem_offset = offset;
c896fe29
FB		 1103 pstrcpy(buf, sizeof(buf), name);
1104 pstrcat(buf, sizeof(buf), "_0");
1105 ts->name = strdup(buf);
c896fe29 1106
7ca4b752
RH		 1107 tcg_debug_assert(ts2 == ts + 1);
1108 ts2->base_type = TCG_TYPE_I64;
1109 ts2->type = TCG_TYPE_I32;
b3915dbb 1110 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 1111 ts2->mem_allocated = 1;
1112 ts2->mem_base = base_ts;
aef85402 1113 ts2->mem_offset = offset + 4;
fac87bd2 1114 ts2->temp_subindex = 1;
c896fe29
FB		 1115 pstrcpy(buf, sizeof(buf), name);
1116 pstrcat(buf, sizeof(buf), "_1");
120c1084 1117 ts2->name = strdup(buf);
7ca4b752 1118 } else {
c896fe29
FB		 1119 ts->base_type = type;
1120 ts->type = type;
b3915dbb 1121 ts->indirect_reg = indirect_reg;
c896fe29 1122 ts->mem_allocated = 1;
b3a62939 1123 ts->mem_base = base_ts;
c896fe29 1124 ts->mem_offset = offset;
c896fe29 1125 ts->name = name;
c896fe29 1126 }
085272b3 1127 return ts;
a7812ae4
PB		 1128}
1129
5bfa8034 1130TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
c896fe29 1131{
b1311c4a 1132 TCGContext *s = tcg_ctx;
ee17db83 1133 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
c896fe29 1134 TCGTemp *ts;
641d5fbe 1135 int idx, k;
c896fe29 1136
0ec9eabc
RH		 1137 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
1138 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
1139 if (idx < TCG_MAX_TEMPS) {
1140 /* There is already an available temp with the right type. */
1141 clear_bit(idx, s->free_temps[k].l);
1142
e8996ee0 1143 ts = &s->temps[idx];
e8996ee0 1144 ts->temp_allocated = 1;
7ca4b752 1145 tcg_debug_assert(ts->base_type == type);
ee17db83 1146 tcg_debug_assert(ts->kind == kind);
e8996ee0 1147 } else {
7ca4b752
RH		 1148 ts = tcg_temp_alloc(s);
1149 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1150 TCGTemp *ts2 = tcg_temp_alloc(s);
1151
f6aa2f7d 1152 ts->base_type = type;
e8996ee0
FB		 1153 ts->type = TCG_TYPE_I32;
1154 ts->temp_allocated = 1;
ee17db83 1155 ts->kind = kind;
7ca4b752
RH		 1156
1157 tcg_debug_assert(ts2 == ts + 1);
1158 ts2->base_type = TCG_TYPE_I64;
1159 ts2->type = TCG_TYPE_I32;
1160 ts2->temp_allocated = 1;
fac87bd2 1161 ts2->temp_subindex = 1;
ee17db83 1162 ts2->kind = kind;
7ca4b752 1163 } else {
e8996ee0
FB		 1164 ts->base_type = type;
1165 ts->type = type;
1166 ts->temp_allocated = 1;
ee17db83 1167 ts->kind = kind;
e8996ee0 1168 }
c896fe29 1169 }
27bfd83c
PM		 1170
1171#if defined(CONFIG_DEBUG_TCG)
1172 s->temps_in_use++;
1173#endif
085272b3 1174 return ts;
c896fe29
FB		 1175}
1176
d2fd745f
RH		 1177TCGv_vec tcg_temp_new_vec(TCGType type)
1178{
1179 TCGTemp *t;
1180
1181#ifdef CONFIG_DEBUG_TCG
1182 switch (type) {
1183 case TCG_TYPE_V64:
1184 assert(TCG_TARGET_HAS_v64);
1185 break;
1186 case TCG_TYPE_V128:
1187 assert(TCG_TARGET_HAS_v128);
1188 break;
1189 case TCG_TYPE_V256:
1190 assert(TCG_TARGET_HAS_v256);
1191 break;
1192 default:
1193 g_assert_not_reached();
1194 }
1195#endif
1196
1197 t = tcg_temp_new_internal(type, 0);
1198 return temp_tcgv_vec(t);
1199}
1200
1201/* Create a new temp of the same type as an existing temp. */
1202TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1203{
1204 TCGTemp *t = tcgv_vec_temp(match);
1205
1206 tcg_debug_assert(t->temp_allocated != 0);
1207
1208 t = tcg_temp_new_internal(t->base_type, 0);
1209 return temp_tcgv_vec(t);
1210}
1211
5bfa8034 1212void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1213{
b1311c4a 1214 TCGContext *s = tcg_ctx;
085272b3 1215 int k, idx;
c896fe29 1216
c7482438
RH		 1217 switch (ts->kind) {
1218 case TEMP_CONST:
1219 /*
1220 * In order to simplify users of tcg_constant_*,
1221 * silently ignore free.
1222 */
c0522136 1223 return;
c7482438
RH		 1224 case TEMP_NORMAL:
1225 case TEMP_LOCAL:
1226 break;
1227 default:
1228 g_assert_not_reached();
c0522136
RH		 1229 }
1230
27bfd83c
PM		 1231#if defined(CONFIG_DEBUG_TCG)
1232 s->temps_in_use--;
1233 if (s->temps_in_use < 0) {
1234 fprintf(stderr, "More temporaries freed than allocated!\n");
1235 }
1236#endif
1237
eabb7b91 1238 tcg_debug_assert(ts->temp_allocated != 0);
e8996ee0 1239 ts->temp_allocated = 0;
0ec9eabc 1240
085272b3 1241 idx = temp_idx(ts);
ee17db83 1242 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1243 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1244}
1245
c0522136
RH		 1246TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1247{
1248 TCGContext *s = tcg_ctx;
1249 GHashTable *h = s->const_table[type];
1250 TCGTemp *ts;
1251
1252 if (h == NULL) {
1253 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1254 s->const_table[type] = h;
1255 }
1256
1257 ts = g_hash_table_lookup(h, &val);
1258 if (ts == NULL) {
aef85402
RH		 1259 int64_t *val_ptr;
1260
c0522136
RH		 1261 ts = tcg_temp_alloc(s);
1262
1263 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1264 TCGTemp *ts2 = tcg_temp_alloc(s);
1265
aef85402
RH		 1266 tcg_debug_assert(ts2 == ts + 1);
1267
c0522136
RH		 1268 ts->base_type = TCG_TYPE_I64;
1269 ts->type = TCG_TYPE_I32;
1270 ts->kind = TEMP_CONST;
1271 ts->temp_allocated = 1;
c0522136 1272
c0522136
RH		 1273 ts2->base_type = TCG_TYPE_I64;
1274 ts2->type = TCG_TYPE_I32;
1275 ts2->kind = TEMP_CONST;
1276 ts2->temp_allocated = 1;
fac87bd2 1277 ts2->temp_subindex = 1;
aef85402
RH		 1278
1279 /*
1280 * Retain the full value of the 64-bit constant in the low
1281 * part, so that the hash table works. Actual uses will
1282 * truncate the value to the low part.
1283 */
1284 ts[HOST_BIG_ENDIAN].val = val;
1285 ts[!HOST_BIG_ENDIAN].val = val >> 32;
1286 val_ptr = &ts[HOST_BIG_ENDIAN].val;
c0522136
RH		 1287 } else {
1288 ts->base_type = type;
1289 ts->type = type;
1290 ts->kind = TEMP_CONST;
1291 ts->temp_allocated = 1;
1292 ts->val = val;
aef85402 1293 val_ptr = &ts->val;
c0522136 1294 }
aef85402 1295 g_hash_table_insert(h, val_ptr, ts);
c0522136
RH		 1296 }
1297
1298 return ts;
1299}
1300
1301TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1302{
1303 val = dup_const(vece, val);
1304 return temp_tcgv_vec(tcg_constant_internal(type, val));
1305}
1306
88d4005b
RH		 1307TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1308{
1309 TCGTemp *t = tcgv_vec_temp(match);
1310
1311 tcg_debug_assert(t->temp_allocated != 0);
1312 return tcg_constant_vec(t->base_type, vece, val);
1313}
1314
a7812ae4 1315TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1316{
a7812ae4
PB		 1317 TCGv_i32 t0;
1318 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1319 tcg_gen_movi_i32(t0, val);
1320 return t0;
1321}
c896fe29 1322
a7812ae4 1323TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1324{
a7812ae4
PB		 1325 TCGv_i64 t0;
1326 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1327 tcg_gen_movi_i64(t0, val);
1328 return t0;
c896fe29
FB		 1329}
1330
a7812ae4 1331TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1332{
a7812ae4
PB		 1333 TCGv_i32 t0;
1334 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1335 tcg_gen_movi_i32(t0, val);
1336 return t0;
1337}
1338
a7812ae4 1339TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1340{
a7812ae4
PB		 1341 TCGv_i64 t0;
1342 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1343 tcg_gen_movi_i64(t0, val);
1344 return t0;
1345}
1346
27bfd83c
PM		 1347#if defined(CONFIG_DEBUG_TCG)
1348void tcg_clear_temp_count(void)
1349{
b1311c4a 1350 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1351 s->temps_in_use = 0;
1352}
1353
1354int tcg_check_temp_count(void)
1355{
b1311c4a 1356 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1357 if (s->temps_in_use) {
1358 /* Clear the count so that we don't give another
1359 * warning immediately next time around.
1360 */
1361 s->temps_in_use = 0;
1362 return 1;
1363 }
1364 return 0;
1365}
1366#endif
1367
be0f34b5
RH		 1368/* Return true if OP may appear in the opcode stream.
1369 Test the runtime variable that controls each opcode. */
1370bool tcg_op_supported(TCGOpcode op)
1371{
d2fd745f
RH		 1372 const bool have_vec
1373 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1374
be0f34b5
RH		 1375 switch (op) {
1376 case INDEX_op_discard:
1377 case INDEX_op_set_label:
1378 case INDEX_op_call:
1379 case INDEX_op_br:
1380 case INDEX_op_mb:
1381 case INDEX_op_insn_start:
1382 case INDEX_op_exit_tb:
1383 case INDEX_op_goto_tb:
f4e01e30 1384 case INDEX_op_goto_ptr:
be0f34b5
RH		 1385 case INDEX_op_qemu_ld_i32:
1386 case INDEX_op_qemu_st_i32:
1387 case INDEX_op_qemu_ld_i64:
1388 case INDEX_op_qemu_st_i64:
1389 return true;
1390
07ce0b05
RH		 1391 case INDEX_op_qemu_st8_i32:
1392 return TCG_TARGET_HAS_qemu_st8_i32;
1393
be0f34b5 1394 case INDEX_op_mov_i32:
be0f34b5
RH		 1395 case INDEX_op_setcond_i32:
1396 case INDEX_op_brcond_i32:
1397 case INDEX_op_ld8u_i32:
1398 case INDEX_op_ld8s_i32:
1399 case INDEX_op_ld16u_i32:
1400 case INDEX_op_ld16s_i32:
1401 case INDEX_op_ld_i32:
1402 case INDEX_op_st8_i32:
1403 case INDEX_op_st16_i32:
1404 case INDEX_op_st_i32:
1405 case INDEX_op_add_i32:
1406 case INDEX_op_sub_i32:
1407 case INDEX_op_mul_i32:
1408 case INDEX_op_and_i32:
1409 case INDEX_op_or_i32:
1410 case INDEX_op_xor_i32:
1411 case INDEX_op_shl_i32:
1412 case INDEX_op_shr_i32:
1413 case INDEX_op_sar_i32:
1414 return true;
1415
1416 case INDEX_op_movcond_i32:
1417 return TCG_TARGET_HAS_movcond_i32;
1418 case INDEX_op_div_i32:
1419 case INDEX_op_divu_i32:
1420 return TCG_TARGET_HAS_div_i32;
1421 case INDEX_op_rem_i32:
1422 case INDEX_op_remu_i32:
1423 return TCG_TARGET_HAS_rem_i32;
1424 case INDEX_op_div2_i32:
1425 case INDEX_op_divu2_i32:
1426 return TCG_TARGET_HAS_div2_i32;
1427 case INDEX_op_rotl_i32:
1428 case INDEX_op_rotr_i32:
1429 return TCG_TARGET_HAS_rot_i32;
1430 case INDEX_op_deposit_i32:
1431 return TCG_TARGET_HAS_deposit_i32;
1432 case INDEX_op_extract_i32:
1433 return TCG_TARGET_HAS_extract_i32;
1434 case INDEX_op_sextract_i32:
1435 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1436 case INDEX_op_extract2_i32:
1437 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1438 case INDEX_op_add2_i32:
1439 return TCG_TARGET_HAS_add2_i32;
1440 case INDEX_op_sub2_i32:
1441 return TCG_TARGET_HAS_sub2_i32;
1442 case INDEX_op_mulu2_i32:
1443 return TCG_TARGET_HAS_mulu2_i32;
1444 case INDEX_op_muls2_i32:
1445 return TCG_TARGET_HAS_muls2_i32;
1446 case INDEX_op_muluh_i32:
1447 return TCG_TARGET_HAS_muluh_i32;
1448 case INDEX_op_mulsh_i32:
1449 return TCG_TARGET_HAS_mulsh_i32;
1450 case INDEX_op_ext8s_i32:
1451 return TCG_TARGET_HAS_ext8s_i32;
1452 case INDEX_op_ext16s_i32:
1453 return TCG_TARGET_HAS_ext16s_i32;
1454 case INDEX_op_ext8u_i32:
1455 return TCG_TARGET_HAS_ext8u_i32;
1456 case INDEX_op_ext16u_i32:
1457 return TCG_TARGET_HAS_ext16u_i32;
1458 case INDEX_op_bswap16_i32:
1459 return TCG_TARGET_HAS_bswap16_i32;
1460 case INDEX_op_bswap32_i32:
1461 return TCG_TARGET_HAS_bswap32_i32;
1462 case INDEX_op_not_i32:
1463 return TCG_TARGET_HAS_not_i32;
1464 case INDEX_op_neg_i32:
1465 return TCG_TARGET_HAS_neg_i32;
1466 case INDEX_op_andc_i32:
1467 return TCG_TARGET_HAS_andc_i32;
1468 case INDEX_op_orc_i32:
1469 return TCG_TARGET_HAS_orc_i32;
1470 case INDEX_op_eqv_i32:
1471 return TCG_TARGET_HAS_eqv_i32;
1472 case INDEX_op_nand_i32:
1473 return TCG_TARGET_HAS_nand_i32;
1474 case INDEX_op_nor_i32:
1475 return TCG_TARGET_HAS_nor_i32;
1476 case INDEX_op_clz_i32:
1477 return TCG_TARGET_HAS_clz_i32;
1478 case INDEX_op_ctz_i32:
1479 return TCG_TARGET_HAS_ctz_i32;
1480 case INDEX_op_ctpop_i32:
1481 return TCG_TARGET_HAS_ctpop_i32;
1482
1483 case INDEX_op_brcond2_i32:
1484 case INDEX_op_setcond2_i32:
1485 return TCG_TARGET_REG_BITS == 32;
1486
1487 case INDEX_op_mov_i64:
be0f34b5
RH		 1488 case INDEX_op_setcond_i64:
1489 case INDEX_op_brcond_i64:
1490 case INDEX_op_ld8u_i64:
1491 case INDEX_op_ld8s_i64:
1492 case INDEX_op_ld16u_i64:
1493 case INDEX_op_ld16s_i64:
1494 case INDEX_op_ld32u_i64:
1495 case INDEX_op_ld32s_i64:
1496 case INDEX_op_ld_i64:
1497 case INDEX_op_st8_i64:
1498 case INDEX_op_st16_i64:
1499 case INDEX_op_st32_i64:
1500 case INDEX_op_st_i64:
1501 case INDEX_op_add_i64:
1502 case INDEX_op_sub_i64:
1503 case INDEX_op_mul_i64:
1504 case INDEX_op_and_i64:
1505 case INDEX_op_or_i64:
1506 case INDEX_op_xor_i64:
1507 case INDEX_op_shl_i64:
1508 case INDEX_op_shr_i64:
1509 case INDEX_op_sar_i64:
1510 case INDEX_op_ext_i32_i64:
1511 case INDEX_op_extu_i32_i64:
1512 return TCG_TARGET_REG_BITS == 64;
1513
1514 case INDEX_op_movcond_i64:
1515 return TCG_TARGET_HAS_movcond_i64;
1516 case INDEX_op_div_i64:
1517 case INDEX_op_divu_i64:
1518 return TCG_TARGET_HAS_div_i64;
1519 case INDEX_op_rem_i64:
1520 case INDEX_op_remu_i64:
1521 return TCG_TARGET_HAS_rem_i64;
1522 case INDEX_op_div2_i64:
1523 case INDEX_op_divu2_i64:
1524 return TCG_TARGET_HAS_div2_i64;
1525 case INDEX_op_rotl_i64:
1526 case INDEX_op_rotr_i64:
1527 return TCG_TARGET_HAS_rot_i64;
1528 case INDEX_op_deposit_i64:
1529 return TCG_TARGET_HAS_deposit_i64;
1530 case INDEX_op_extract_i64:
1531 return TCG_TARGET_HAS_extract_i64;
1532 case INDEX_op_sextract_i64:
1533 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1534 case INDEX_op_extract2_i64:
1535 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1536 case INDEX_op_extrl_i64_i32:
1537 return TCG_TARGET_HAS_extrl_i64_i32;
1538 case INDEX_op_extrh_i64_i32:
1539 return TCG_TARGET_HAS_extrh_i64_i32;
1540 case INDEX_op_ext8s_i64:
1541 return TCG_TARGET_HAS_ext8s_i64;
1542 case INDEX_op_ext16s_i64:
1543 return TCG_TARGET_HAS_ext16s_i64;
1544 case INDEX_op_ext32s_i64:
1545 return TCG_TARGET_HAS_ext32s_i64;
1546 case INDEX_op_ext8u_i64:
1547 return TCG_TARGET_HAS_ext8u_i64;
1548 case INDEX_op_ext16u_i64:
1549 return TCG_TARGET_HAS_ext16u_i64;
1550 case INDEX_op_ext32u_i64:
1551 return TCG_TARGET_HAS_ext32u_i64;
1552 case INDEX_op_bswap16_i64:
1553 return TCG_TARGET_HAS_bswap16_i64;
1554 case INDEX_op_bswap32_i64:
1555 return TCG_TARGET_HAS_bswap32_i64;
1556 case INDEX_op_bswap64_i64:
1557 return TCG_TARGET_HAS_bswap64_i64;
1558 case INDEX_op_not_i64:
1559 return TCG_TARGET_HAS_not_i64;
1560 case INDEX_op_neg_i64:
1561 return TCG_TARGET_HAS_neg_i64;
1562 case INDEX_op_andc_i64:
1563 return TCG_TARGET_HAS_andc_i64;
1564 case INDEX_op_orc_i64:
1565 return TCG_TARGET_HAS_orc_i64;
1566 case INDEX_op_eqv_i64:
1567 return TCG_TARGET_HAS_eqv_i64;
1568 case INDEX_op_nand_i64:
1569 return TCG_TARGET_HAS_nand_i64;
1570 case INDEX_op_nor_i64:
1571 return TCG_TARGET_HAS_nor_i64;
1572 case INDEX_op_clz_i64:
1573 return TCG_TARGET_HAS_clz_i64;
1574 case INDEX_op_ctz_i64:
1575 return TCG_TARGET_HAS_ctz_i64;
1576 case INDEX_op_ctpop_i64:
1577 return TCG_TARGET_HAS_ctpop_i64;
1578 case INDEX_op_add2_i64:
1579 return TCG_TARGET_HAS_add2_i64;
1580 case INDEX_op_sub2_i64:
1581 return TCG_TARGET_HAS_sub2_i64;
1582 case INDEX_op_mulu2_i64:
1583 return TCG_TARGET_HAS_mulu2_i64;
1584 case INDEX_op_muls2_i64:
1585 return TCG_TARGET_HAS_muls2_i64;
1586 case INDEX_op_muluh_i64:
1587 return TCG_TARGET_HAS_muluh_i64;
1588 case INDEX_op_mulsh_i64:
1589 return TCG_TARGET_HAS_mulsh_i64;
1590
d2fd745f
RH		 1591 case INDEX_op_mov_vec:
1592 case INDEX_op_dup_vec:
37ee55a0 1593 case INDEX_op_dupm_vec:
d2fd745f
RH		 1594 case INDEX_op_ld_vec:
1595 case INDEX_op_st_vec:
1596 case INDEX_op_add_vec:
1597 case INDEX_op_sub_vec:
1598 case INDEX_op_and_vec:
1599 case INDEX_op_or_vec:
1600 case INDEX_op_xor_vec:
212be173 1601 case INDEX_op_cmp_vec:
d2fd745f
RH		 1602 return have_vec;
1603 case INDEX_op_dup2_vec:
1604 return have_vec && TCG_TARGET_REG_BITS == 32;
1605 case INDEX_op_not_vec:
1606 return have_vec && TCG_TARGET_HAS_not_vec;
1607 case INDEX_op_neg_vec:
1608 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1609 case INDEX_op_abs_vec:
1610 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1611 case INDEX_op_andc_vec:
1612 return have_vec && TCG_TARGET_HAS_andc_vec;
1613 case INDEX_op_orc_vec:
1614 return have_vec && TCG_TARGET_HAS_orc_vec;
ed523473
RH		 1615 case INDEX_op_nand_vec:
1616 return have_vec && TCG_TARGET_HAS_nand_vec;
1617 case INDEX_op_nor_vec:
1618 return have_vec && TCG_TARGET_HAS_nor_vec;
1619 case INDEX_op_eqv_vec:
1620 return have_vec && TCG_TARGET_HAS_eqv_vec;
3774030a
RH		 1621 case INDEX_op_mul_vec:
1622 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1623 case INDEX_op_shli_vec:
1624 case INDEX_op_shri_vec:
1625 case INDEX_op_sari_vec:
1626 return have_vec && TCG_TARGET_HAS_shi_vec;
1627 case INDEX_op_shls_vec:
1628 case INDEX_op_shrs_vec:
1629 case INDEX_op_sars_vec:
1630 return have_vec && TCG_TARGET_HAS_shs_vec;
1631 case INDEX_op_shlv_vec:
1632 case INDEX_op_shrv_vec:
1633 case INDEX_op_sarv_vec:
1634 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1635 case INDEX_op_rotli_vec:
1636 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1637 case INDEX_op_rotls_vec:
1638 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1639 case INDEX_op_rotlv_vec:
1640 case INDEX_op_rotrv_vec:
1641 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1642 case INDEX_op_ssadd_vec:
1643 case INDEX_op_usadd_vec:
1644 case INDEX_op_sssub_vec:
1645 case INDEX_op_ussub_vec:
1646 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1647 case INDEX_op_smin_vec:
1648 case INDEX_op_umin_vec:
1649 case INDEX_op_smax_vec:
1650 case INDEX_op_umax_vec:
1651 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1652 case INDEX_op_bitsel_vec:
1653 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1654 case INDEX_op_cmpsel_vec:
1655 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1656
db432672
RH		 1657 default:
1658 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1659 return true;
be0f34b5 1660 }
be0f34b5
RH		 1661}
1662
39004a71
RH		 1663static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
1664
ae8b75dc 1665void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1666{
3e92aa34 1667 const TCGHelperInfo *info;
39004a71
RH		 1668 TCGv_i64 extend_free[MAX_CALL_IARGS];
1669 int n_extend = 0;
75e8b9b7 1670 TCGOp *op;
39004a71 1671 int i, n, pi = 0, total_args;
afb49896 1672
619205fd 1673 info = g_hash_table_lookup(helper_table, (gpointer)func);
39004a71
RH		 1674 total_args = info->nr_out + info->nr_in + 2;
1675 op = tcg_op_alloc(INDEX_op_call, total_args);
2bece2c8 1676
38b47b19
EC		 1677#ifdef CONFIG_PLUGIN
1678 /* detect non-plugin helpers */
1679 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1680 tcg_ctx->plugin_insn->calls_helpers = true;
1681 }
1682#endif
1683
39004a71
RH		 1684 TCGOP_CALLO(op) = n = info->nr_out;
1685 switch (n) {
1686 case 0:
1687 tcg_debug_assert(ret == NULL);
1688 break;
1689 case 1:
1690 tcg_debug_assert(ret != NULL);
1691 op->args[pi++] = temp_arg(ret);
1692 break;
1693 case 2:
1694 tcg_debug_assert(ret != NULL);
1695 tcg_debug_assert(ret->base_type == ret->type + 1);
1696 tcg_debug_assert(ret->temp_subindex == 0);
1697 op->args[pi++] = temp_arg(ret);
1698 op->args[pi++] = temp_arg(ret + 1);
1699 break;
1700 default:
1701 g_assert_not_reached();
1702 }
1703
1704 TCGOP_CALLI(op) = n = info->nr_in;
1705 for (i = 0; i < n; i++) {
1706 const TCGCallArgumentLoc *loc = &info->in[i];
1707 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
1708
1709 switch (loc->kind) {
1710 case TCG_CALL_ARG_NORMAL:
1711 op->args[pi++] = temp_arg(ts);
1712 break;
eb8b0224 1713
39004a71
RH		 1714 case TCG_CALL_ARG_EXTEND_U:
1715 case TCG_CALL_ARG_EXTEND_S:
1716 {
eb8b0224 1717 TCGv_i64 temp = tcg_temp_new_i64();
39004a71
RH		 1718 TCGv_i32 orig = temp_tcgv_i32(ts);
1719
1720 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
eb8b0224
RH		 1721 tcg_gen_ext_i32_i64(temp, orig);
1722 } else {
1723 tcg_gen_extu_i32_i64(temp, orig);
1724 }
39004a71
RH		 1725 op->args[pi++] = tcgv_i64_arg(temp);
1726 extend_free[n_extend++] = temp;
2bece2c8 1727 }
e2a9dd6b 1728 break;
7b7d8b2d 1729
e2a9dd6b
RH		 1730 default:
1731 g_assert_not_reached();
c896fe29
FB		 1732 }
1733 }
75e8b9b7 1734 op->args[pi++] = (uintptr_t)func;
3e92aa34 1735 op->args[pi++] = (uintptr_t)info;
39004a71 1736 tcg_debug_assert(pi == total_args);
a7812ae4 1737
39004a71 1738 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
7319d83a 1739
39004a71
RH		 1740 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
1741 for (i = 0; i < n_extend; ++i) {
1742 tcg_temp_free_i64(extend_free[i]);
2bece2c8 1743 }
c896fe29 1744}
c896fe29 1745
8fcd3692 1746static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1747{
ac3b8891 1748 int i, n;
ac3b8891 1749
ee17db83
RH		 1750 for (i = 0, n = s->nb_temps; i < n; i++) {
1751 TCGTemp *ts = &s->temps[i];
1752 TCGTempVal val = TEMP_VAL_MEM;
1753
1754 switch (ts->kind) {
c0522136
RH		 1755 case TEMP_CONST:
1756 val = TEMP_VAL_CONST;
1757 break;
ee17db83
RH		 1758 case TEMP_FIXED:
1759 val = TEMP_VAL_REG;
1760 break;
1761 case TEMP_GLOBAL:
1762 break;
1763 case TEMP_NORMAL:
c7482438 1764 case TEMP_EBB:
ee17db83
RH		 1765 val = TEMP_VAL_DEAD;
1766 /* fall through */
1767 case TEMP_LOCAL:
1768 ts->mem_allocated = 0;
1769 break;
1770 default:
1771 g_assert_not_reached();
1772 }
1773 ts->val_type = val;
e8996ee0 1774 }
f8b2f202
RH		 1775
1776 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1777}
1778
f8b2f202
RH		 1779static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1780 TCGTemp *ts)
c896fe29 1781{
1807f4c4 1782 int idx = temp_idx(ts);
ac56dd48 1783
ee17db83
RH		 1784 switch (ts->kind) {
1785 case TEMP_FIXED:
1786 case TEMP_GLOBAL:
ac56dd48 1787 pstrcpy(buf, buf_size, ts->name);
ee17db83
RH		 1788 break;
1789 case TEMP_LOCAL:
f8b2f202 1790 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83 1791 break;
c7482438
RH		 1792 case TEMP_EBB:
1793 snprintf(buf, buf_size, "ebb%d", idx - s->nb_globals);
1794 break;
ee17db83 1795 case TEMP_NORMAL:
f8b2f202 1796 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1797 break;
c0522136
RH		 1798 case TEMP_CONST:
1799 switch (ts->type) {
1800 case TCG_TYPE_I32:
1801 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1802 break;
1803#if TCG_TARGET_REG_BITS > 32
1804 case TCG_TYPE_I64:
1805 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1806 break;
1807#endif
1808 case TCG_TYPE_V64:
1809 case TCG_TYPE_V128:
1810 case TCG_TYPE_V256:
1811 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1812 64 << (ts->type - TCG_TYPE_V64), ts->val);
1813 break;
1814 default:
1815 g_assert_not_reached();
1816 }
1817 break;
c896fe29
FB		 1818 }
1819 return buf;
1820}
1821
43439139
RH		 1822static char *tcg_get_arg_str(TCGContext *s, char *buf,
1823 int buf_size, TCGArg arg)
f8b2f202 1824{
43439139 1825 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1826}
1827
f48f3ede
BS		 1828static const char * const cond_name[] =
1829{
0aed257f
RH		 1830 [TCG_COND_NEVER] = "never",
1831 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1832 [TCG_COND_EQ] = "eq",
1833 [TCG_COND_NE] = "ne",
1834 [TCG_COND_LT] = "lt",
1835 [TCG_COND_GE] = "ge",
1836 [TCG_COND_LE] = "le",
1837 [TCG_COND_GT] = "gt",
1838 [TCG_COND_LTU] = "ltu",
1839 [TCG_COND_GEU] = "geu",
1840 [TCG_COND_LEU] = "leu",
1841 [TCG_COND_GTU] = "gtu"
1842};
1843
f713d6ad
RH		 1844static const char * const ldst_name[] =
1845{
1846 [MO_UB] = "ub",
1847 [MO_SB] = "sb",
1848 [MO_LEUW] = "leuw",
1849 [MO_LESW] = "lesw",
1850 [MO_LEUL] = "leul",
1851 [MO_LESL] = "lesl",
fc313c64 1852 [MO_LEUQ] = "leq",
f713d6ad
RH		 1853 [MO_BEUW] = "beuw",
1854 [MO_BESW] = "besw",
1855 [MO_BEUL] = "beul",
1856 [MO_BESL] = "besl",
fc313c64 1857 [MO_BEUQ] = "beq",
f713d6ad
RH		 1858};
1859
1f00b27f 1860static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 1861#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 1862 [MO_UNALN >> MO_ASHIFT] = "un+",
1863 [MO_ALIGN >> MO_ASHIFT] = "",
1864#else
1865 [MO_UNALN >> MO_ASHIFT] = "",
1866 [MO_ALIGN >> MO_ASHIFT] = "al+",
1867#endif
1868 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1869 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1870 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1871 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1872 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1873 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1874};
1875
587195bd
RH		 1876static const char bswap_flag_name[][6] = {
1877 [TCG_BSWAP_IZ] = "iz",
1878 [TCG_BSWAP_OZ] = "oz",
1879 [TCG_BSWAP_OS] = "os",
1880 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1881 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1882};
1883
b016486e
RH		 1884static inline bool tcg_regset_single(TCGRegSet d)
1885{
1886 return (d & (d - 1)) == 0;
1887}
1888
1889static inline TCGReg tcg_regset_first(TCGRegSet d)
1890{
1891 if (TCG_TARGET_NB_REGS <= 32) {
1892 return ctz32(d);
1893 } else {
1894 return ctz64(d);
1895 }
1896}
1897
b7a83ff8
RH		 1898/* Return only the number of characters output -- no error return. */
1899#define ne_fprintf(...) \
1900 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
1901
1902static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
c896fe29 1903{
c896fe29 1904 char buf[128];
c45cb8bb 1905 TCGOp *op;
c45cb8bb 1906
15fa08f8 1907 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 1908 int i, k, nb_oargs, nb_iargs, nb_cargs;
1909 const TCGOpDef *def;
c45cb8bb 1910 TCGOpcode c;
bdfb460e 1911 int col = 0;
c896fe29 1912
c45cb8bb 1913 c = op->opc;
c896fe29 1914 def = &tcg_op_defs[c];
c45cb8bb 1915
765b842a 1916 if (c == INDEX_op_insn_start) {
b016486e 1917 nb_oargs = 0;
b7a83ff8 1918 col += ne_fprintf(f, "\n ----");
9aef40ed
RH		 1919
1920 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1921 target_ulong a;
7e4597d7 1922#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 1923 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 1924#else
efee3746 1925 a = op->args[i];
7e4597d7 1926#endif
b7a83ff8 1927 col += ne_fprintf(f, " " TARGET_FMT_lx, a);
eeacee4d 1928 }
7e4597d7 1929 } else if (c == INDEX_op_call) {
3e92aa34 1930 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 1931 void *func = tcg_call_func(op);
3e92aa34 1932
c896fe29 1933 /* variable number of arguments */
cd9090aa
RH		 1934 nb_oargs = TCGOP_CALLO(op);
1935 nb_iargs = TCGOP_CALLI(op);
c896fe29 1936 nb_cargs = def->nb_cargs;
c896fe29 1937
b7a83ff8 1938 col += ne_fprintf(f, " %s ", def->name);
3e92aa34
RH		 1939
1940 /*
1941 * Print the function name from TCGHelperInfo, if available.
1942 * Note that plugins have a template function for the info,
1943 * but the actual function pointer comes from the plugin.
1944 */
3e92aa34 1945 if (func == info->func) {
b7a83ff8 1946 col += ne_fprintf(f, "%s", info->name);
3e92aa34 1947 } else {
b7a83ff8 1948 col += ne_fprintf(f, "plugin(%p)", func);
3e92aa34
RH		 1949 }
1950
b7a83ff8 1951 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
cf066674 1952 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1953 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1954 op->args[i]));
b03cce8e 1955 }
cf066674 1956 for (i = 0; i < nb_iargs; i++) {
efee3746 1957 TCGArg arg = op->args[nb_oargs + i];
39004a71 1958 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
b7a83ff8 1959 col += ne_fprintf(f, ",%s", t);
e8996ee0 1960 }
b03cce8e 1961 } else {
b7a83ff8 1962 col += ne_fprintf(f, " %s ", def->name);
c45cb8bb
RH		 1963
1964 nb_oargs = def->nb_oargs;
1965 nb_iargs = def->nb_iargs;
1966 nb_cargs = def->nb_cargs;
1967
d2fd745f 1968 if (def->flags & TCG_OPF_VECTOR) {
b7a83ff8
RH		 1969 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
1970 8 << TCGOP_VECE(op));
d2fd745f
RH		 1971 }
1972
b03cce8e 1973 k = 0;
c45cb8bb 1974 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1975 const char *sep = k ? "," : "";
1976 col += ne_fprintf(f, "%s%s", sep,
1977 tcg_get_arg_str(s, buf, sizeof(buf),
1978 op->args[k++]));
b03cce8e 1979 }
c45cb8bb 1980 for (i = 0; i < nb_iargs; i++) {
b7a83ff8
RH		 1981 const char *sep = k ? "," : "";
1982 col += ne_fprintf(f, "%s%s", sep,
1983 tcg_get_arg_str(s, buf, sizeof(buf),
1984 op->args[k++]));
b03cce8e 1985 }
be210acb
RH		 1986 switch (c) {
1987 case INDEX_op_brcond_i32:
be210acb 1988 case INDEX_op_setcond_i32:
ffc5ea09 1989 case INDEX_op_movcond_i32:
ffc5ea09 1990 case INDEX_op_brcond2_i32:
be210acb 1991 case INDEX_op_setcond2_i32:
ffc5ea09 1992 case INDEX_op_brcond_i64:
be210acb 1993 case INDEX_op_setcond_i64:
ffc5ea09 1994 case INDEX_op_movcond_i64:
212be173 1995 case INDEX_op_cmp_vec:
f75da298 1996 case INDEX_op_cmpsel_vec:
efee3746
RH		 1997 if (op->args[k] < ARRAY_SIZE(cond_name)
1998 && cond_name[op->args[k]]) {
b7a83ff8 1999 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
eeacee4d 2000 } else {
b7a83ff8 2001 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 2002 }
f48f3ede 2003 i = 1;
be210acb 2004 break;
f713d6ad
RH		 2005 case INDEX_op_qemu_ld_i32:
2006 case INDEX_op_qemu_st_i32:
07ce0b05 2007 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 2008 case INDEX_op_qemu_ld_i64:
2009 case INDEX_op_qemu_st_i64:
59227d5d 2010 {
9002ffcb 2011 MemOpIdx oi = op->args[k++];
14776ab5 2012 MemOp op = get_memop(oi);
59227d5d
RH		 2013 unsigned ix = get_mmuidx(oi);
2014
59c4b7e8 2015 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
b7a83ff8 2016 col += ne_fprintf(f, ",$0x%x,%u", op, ix);
59c4b7e8 2017 } else {
1f00b27f
SS		 2018 const char *s_al, *s_op;
2019 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 2020 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
b7a83ff8 2021 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 2022 }
2023 i = 1;
f713d6ad 2024 }
f713d6ad 2025 break;
587195bd
RH		 2026 case INDEX_op_bswap16_i32:
2027 case INDEX_op_bswap16_i64:
2028 case INDEX_op_bswap32_i32:
2029 case INDEX_op_bswap32_i64:
2030 case INDEX_op_bswap64_i64:
2031 {
2032 TCGArg flags = op->args[k];
2033 const char *name = NULL;
2034
2035 if (flags < ARRAY_SIZE(bswap_flag_name)) {
2036 name = bswap_flag_name[flags];
2037 }
2038 if (name) {
b7a83ff8 2039 col += ne_fprintf(f, ",%s", name);
587195bd 2040 } else {
b7a83ff8 2041 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
587195bd
RH		 2042 }
2043 i = k = 1;
2044 }
2045 break;
be210acb 2046 default:
f48f3ede 2047 i = 0;
be210acb
RH		 2048 break;
2049 }
51e3972c
RH		 2050 switch (c) {
2051 case INDEX_op_set_label:
2052 case INDEX_op_br:
2053 case INDEX_op_brcond_i32:
2054 case INDEX_op_brcond_i64:
2055 case INDEX_op_brcond2_i32:
b7a83ff8
RH		 2056 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
2057 arg_label(op->args[k])->id);
51e3972c
RH		 2058 i++, k++;
2059 break;
2060 default:
2061 break;
2062 }
2063 for (; i < nb_cargs; i++, k++) {
b7a83ff8
RH		 2064 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
2065 op->args[k]);
bdfb460e
RH		 2066 }
2067 }
bdfb460e 2068
1894f69a 2069 if (have_prefs || op->life) {
b7a83ff8
RH		 2070 for (; col < 40; ++col) {
2071 putc(' ', f);
bdfb460e 2072 }
1894f69a
RH		 2073 }
2074
2075 if (op->life) {
2076 unsigned life = op->life;
bdfb460e
RH		 2077
2078 if (life & (SYNC_ARG * 3)) {
b7a83ff8 2079 ne_fprintf(f, " sync:");
bdfb460e
RH		 2080 for (i = 0; i < 2; ++i) {
2081 if (life & (SYNC_ARG << i)) {
b7a83ff8 2082 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2083 }
2084 }
2085 }
2086 life /= DEAD_ARG;
2087 if (life) {
b7a83ff8 2088 ne_fprintf(f, " dead:");
bdfb460e
RH		 2089 for (i = 0; life; ++i, life >>= 1) {
2090 if (life & 1) {
b7a83ff8 2091 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2092 }
2093 }
b03cce8e 2094 }
c896fe29 2095 }
1894f69a
RH		 2096
2097 if (have_prefs) {
2098 for (i = 0; i < nb_oargs; ++i) {
31fd884b 2099 TCGRegSet set = output_pref(op, i);
1894f69a
RH		 2100
2101 if (i == 0) {
b7a83ff8 2102 ne_fprintf(f, " pref=");
1894f69a 2103 } else {
b7a83ff8 2104 ne_fprintf(f, ",");
1894f69a
RH		 2105 }
2106 if (set == 0) {
b7a83ff8 2107 ne_fprintf(f, "none");
1894f69a 2108 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
b7a83ff8 2109 ne_fprintf(f, "all");
1894f69a
RH		 2110#ifdef CONFIG_DEBUG_TCG
2111 } else if (tcg_regset_single(set)) {
2112 TCGReg reg = tcg_regset_first(set);
b7a83ff8 2113 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
1894f69a
RH		 2114#endif
2115 } else if (TCG_TARGET_NB_REGS <= 32) {
b7a83ff8 2116 ne_fprintf(f, "0x%x", (uint32_t)set);
1894f69a 2117 } else {
b7a83ff8 2118 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
1894f69a
RH		 2119 }
2120 }
2121 }
2122
b7a83ff8 2123 putc('\n', f);
c896fe29
FB		 2124 }
2125}
2126
2127/* we give more priority to constraints with less registers */
2128static int get_constraint_priority(const TCGOpDef *def, int k)
2129{
74a11790 2130 const TCGArgConstraint *arg_ct = &def->args_ct[k];
29f5e925 2131 int n = ctpop64(arg_ct->regs);
c896fe29 2132
29f5e925
RH		 2133 /*
2134 * Sort constraints of a single register first, which includes output
2135 * aliases (which must exactly match the input already allocated).
2136 */
2137 if (n == 1 || arg_ct->oalias) {
2138 return INT_MAX;
2139 }
2140
2141 /*
2142 * Sort register pairs next, first then second immediately after.
2143 * Arbitrarily sort multiple pairs by the index of the first reg;
2144 * there shouldn't be many pairs.
2145 */
2146 switch (arg_ct->pair) {
2147 case 1:
2148 case 3:
2149 return (k + 1) * 2;
2150 case 2:
2151 return (arg_ct->pair_index + 1) * 2 - 1;
c896fe29 2152 }
29f5e925
RH		 2153
2154 /* Finally, sort by decreasing register count. */
2155 assert(n > 1);
2156 return -n;
c896fe29
FB		 2157}
2158
2159/* sort from highest priority to lowest */
2160static void sort_constraints(TCGOpDef *def, int start, int n)
2161{
66792f90
RH		 2162 int i, j;
2163 TCGArgConstraint *a = def->args_ct;
c896fe29 2164
66792f90
RH		 2165 for (i = 0; i < n; i++) {
2166 a[start + i].sort_index = start + i;
2167 }
2168 if (n <= 1) {
c896fe29 2169 return;
66792f90
RH		 2170 }
2171 for (i = 0; i < n - 1; i++) {
2172 for (j = i + 1; j < n; j++) {
2173 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2174 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2175 if (p1 < p2) {
66792f90
RH		 2176 int tmp = a[start + i].sort_index;
2177 a[start + i].sort_index = a[start + j].sort_index;
2178 a[start + j].sort_index = tmp;
c896fe29
FB		 2179 }
2180 }
2181 }
2182}
2183
f69d277e 2184static void process_op_defs(TCGContext *s)
c896fe29 2185{
a9751609 2186 TCGOpcode op;
c896fe29 2187
f69d277e
RH		 2188 for (op = 0; op < NB_OPS; op++) {
2189 TCGOpDef *def = &tcg_op_defs[op];
2190 const TCGTargetOpDef *tdefs;
29f5e925
RH		 2191 bool saw_alias_pair = false;
2192 int i, o, i2, o2, nb_args;
f69d277e
RH		 2193
2194 if (def->flags & TCG_OPF_NOT_PRESENT) {
2195 continue;
2196 }
2197
c896fe29 2198 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2199 if (nb_args == 0) {
2200 continue;
2201 }
2202
4c22e840
RH		 2203 /*
2204 * Macro magic should make it impossible, but double-check that
2205 * the array index is in range. Since the signness of an enum
2206 * is implementation defined, force the result to unsigned.
2207 */
2208 unsigned con_set = tcg_target_op_def(op);
2209 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2210 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2211
2212 for (i = 0; i < nb_args; i++) {
2213 const char *ct_str = tdefs->args_ct_str[i];
8940ea0d
PMD		 2214 bool input_p = i >= def->nb_oargs;
2215
f69d277e 2216 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2217 tcg_debug_assert(ct_str != NULL);
f69d277e 2218
8940ea0d
PMD		 2219 switch (*ct_str) {
2220 case '0' ... '9':
2221 o = *ct_str - '0';
2222 tcg_debug_assert(input_p);
2223 tcg_debug_assert(o < def->nb_oargs);
2224 tcg_debug_assert(def->args_ct[o].regs != 0);
2225 tcg_debug_assert(!def->args_ct[o].oalias);
2226 def->args_ct[i] = def->args_ct[o];
2227 /* The output sets oalias. */
2228 def->args_ct[o].oalias = 1;
2229 def->args_ct[o].alias_index = i;
2230 /* The input sets ialias. */
2231 def->args_ct[i].ialias = 1;
2232 def->args_ct[i].alias_index = o;
29f5e925
RH		 2233 if (def->args_ct[i].pair) {
2234 saw_alias_pair = true;
2235 }
8940ea0d
PMD		 2236 tcg_debug_assert(ct_str[1] == '\0');
2237 continue;
2238
2239 case '&':
2240 tcg_debug_assert(!input_p);
2241 def->args_ct[i].newreg = true;
2242 ct_str++;
2243 break;
29f5e925
RH		 2244
2245 case 'p': /* plus */
2246 /* Allocate to the register after the previous. */
2247 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2248 o = i - 1;
2249 tcg_debug_assert(!def->args_ct[o].pair);
2250 tcg_debug_assert(!def->args_ct[o].ct);
2251 def->args_ct[i] = (TCGArgConstraint){
2252 .pair = 2,
2253 .pair_index = o,
2254 .regs = def->args_ct[o].regs << 1,
2255 };
2256 def->args_ct[o].pair = 1;
2257 def->args_ct[o].pair_index = i;
2258 tcg_debug_assert(ct_str[1] == '\0');
2259 continue;
2260
2261 case 'm': /* minus */
2262 /* Allocate to the register before the previous. */
2263 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2264 o = i - 1;
2265 tcg_debug_assert(!def->args_ct[o].pair);
2266 tcg_debug_assert(!def->args_ct[o].ct);
2267 def->args_ct[i] = (TCGArgConstraint){
2268 .pair = 1,
2269 .pair_index = o,
2270 .regs = def->args_ct[o].regs >> 1,
2271 };
2272 def->args_ct[o].pair = 2;
2273 def->args_ct[o].pair_index = i;
2274 tcg_debug_assert(ct_str[1] == '\0');
2275 continue;
8940ea0d
PMD		 2276 }
2277
2278 do {
2279 switch (*ct_str) {
17280ff4
RH		 2280 case 'i':
2281 def->args_ct[i].ct |= TCG_CT_CONST;
17280ff4 2282 break;
358b4923 2283
358b4923
RH		 2284 /* Include all of the target-specific constraints. */
2285
2286#undef CONST
2287#define CONST(CASE, MASK) \
8940ea0d 2288 case CASE: def->args_ct[i].ct |= MASK; break;
358b4923 2289#define REGS(CASE, MASK) \
8940ea0d 2290 case CASE: def->args_ct[i].regs |= MASK; break;
358b4923
RH		 2291
2292#include "tcg-target-con-str.h"
2293
2294#undef REGS
2295#undef CONST
17280ff4 2296 default:
8940ea0d
PMD		 2297 case '0' ... '9':
2298 case '&':
29f5e925
RH		 2299 case 'p':
2300 case 'm':
17280ff4 2301 /* Typo in TCGTargetOpDef constraint. */
358b4923 2302 g_assert_not_reached();
c896fe29 2303 }
8940ea0d 2304 } while (*++ct_str != '\0');
c896fe29
FB		 2305 }
2306
c68aaa18 2307 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2308 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2309
29f5e925
RH		 2310 /*
2311 * Fix up output pairs that are aliased with inputs.
2312 * When we created the alias, we copied pair from the output.
2313 * There are three cases:
2314 * (1a) Pairs of inputs alias pairs of outputs.
2315 * (1b) One input aliases the first of a pair of outputs.
2316 * (2) One input aliases the second of a pair of outputs.
2317 *
2318 * Case 1a is handled by making sure that the pair_index'es are
2319 * properly updated so that they appear the same as a pair of inputs.
2320 *
2321 * Case 1b is handled by setting the pair_index of the input to
2322 * itself, simply so it doesn't point to an unrelated argument.
2323 * Since we don't encounter the "second" during the input allocation
2324 * phase, nothing happens with the second half of the input pair.
2325 *
2326 * Case 2 is handled by setting the second input to pair=3, the
2327 * first output to pair=3, and the pair_index'es to match.
2328 */
2329 if (saw_alias_pair) {
2330 for (i = def->nb_oargs; i < nb_args; i++) {
2331 /*
2332 * Since [0-9pm] must be alone in the constraint string,
2333 * the only way they can both be set is if the pair comes
2334 * from the output alias.
2335 */
2336 if (!def->args_ct[i].ialias) {
2337 continue;
2338 }
2339 switch (def->args_ct[i].pair) {
2340 case 0:
2341 break;
2342 case 1:
2343 o = def->args_ct[i].alias_index;
2344 o2 = def->args_ct[o].pair_index;
2345 tcg_debug_assert(def->args_ct[o].pair == 1);
2346 tcg_debug_assert(def->args_ct[o2].pair == 2);
2347 if (def->args_ct[o2].oalias) {
2348 /* Case 1a */
2349 i2 = def->args_ct[o2].alias_index;
2350 tcg_debug_assert(def->args_ct[i2].pair == 2);
2351 def->args_ct[i2].pair_index = i;
2352 def->args_ct[i].pair_index = i2;
2353 } else {
2354 /* Case 1b */
2355 def->args_ct[i].pair_index = i;
2356 }
2357 break;
2358 case 2:
2359 o = def->args_ct[i].alias_index;
2360 o2 = def->args_ct[o].pair_index;
2361 tcg_debug_assert(def->args_ct[o].pair == 2);
2362 tcg_debug_assert(def->args_ct[o2].pair == 1);
2363 if (def->args_ct[o2].oalias) {
2364 /* Case 1a */
2365 i2 = def->args_ct[o2].alias_index;
2366 tcg_debug_assert(def->args_ct[i2].pair == 1);
2367 def->args_ct[i2].pair_index = i;
2368 def->args_ct[i].pair_index = i2;
2369 } else {
2370 /* Case 2 */
2371 def->args_ct[i].pair = 3;
2372 def->args_ct[o2].pair = 3;
2373 def->args_ct[i].pair_index = o2;
2374 def->args_ct[o2].pair_index = i;
2375 }
2376 break;
2377 default:
2378 g_assert_not_reached();
2379 }
2380 }
2381 }
2382
c896fe29
FB		 2383 /* sort the constraints (XXX: this is just an heuristic) */
2384 sort_constraints(def, 0, def->nb_oargs);
2385 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2386 }
c896fe29
FB		 2387}
2388
0c627cdc
RH		 2389void tcg_op_remove(TCGContext *s, TCGOp *op)
2390{
d88a117e
RH		 2391 TCGLabel *label;
2392
2393 switch (op->opc) {
2394 case INDEX_op_br:
2395 label = arg_label(op->args[0]);
2396 label->refs--;
2397 break;
2398 case INDEX_op_brcond_i32:
2399 case INDEX_op_brcond_i64:
2400 label = arg_label(op->args[3]);
2401 label->refs--;
2402 break;
2403 case INDEX_op_brcond2_i32:
2404 label = arg_label(op->args[5]);
2405 label->refs--;
2406 break;
2407 default:
2408 break;
2409 }
2410
15fa08f8
RH		 2411 QTAILQ_REMOVE(&s->ops, op, link);
2412 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2413 s->nb_ops--;
0c627cdc
RH		 2414
2415#ifdef CONFIG_PROFILER
d73415a3 2416 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2417#endif
2418}
2419
a80cdd31
RH		 2420void tcg_remove_ops_after(TCGOp *op)
2421{
2422 TCGContext *s = tcg_ctx;
2423
2424 while (true) {
2425 TCGOp *last = tcg_last_op();
2426 if (last == op) {
2427 return;
2428 }
2429 tcg_op_remove(s, last);
2430 }
2431}
2432
d4478943 2433static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
5a18407f 2434{
15fa08f8 2435 TCGContext *s = tcg_ctx;
cb10bc63
RH		 2436 TCGOp *op = NULL;
2437
2438 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
2439 QTAILQ_FOREACH(op, &s->free_ops, link) {
2440 if (nargs <= op->nargs) {
2441 QTAILQ_REMOVE(&s->free_ops, op, link);
2442 nargs = op->nargs;
2443 goto found;
2444 }
2445 }
15fa08f8 2446 }
cb10bc63
RH		 2447
2448 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
2449 nargs = MAX(4, nargs);
2450 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
2451
2452 found:
15fa08f8
RH		 2453 memset(op, 0, offsetof(TCGOp, link));
2454 op->opc = opc;
cb10bc63
RH		 2455 op->nargs = nargs;
2456
2457 /* Check for bitfield overflow. */
2458 tcg_debug_assert(op->nargs == nargs);
5a18407f 2459
cb10bc63 2460 s->nb_ops++;
15fa08f8
RH		 2461 return op;
2462}
2463
d4478943 2464TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
15fa08f8 2465{
d4478943 2466 TCGOp *op = tcg_op_alloc(opc, nargs);
15fa08f8
RH		 2467 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2468 return op;
2469}
5a18407f 2470
d4478943
PMD		 2471TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
2472 TCGOpcode opc, unsigned nargs)
15fa08f8 2473{
d4478943 2474 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 2475 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2476 return new_op;
2477}
2478
d4478943
PMD		 2479TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
2480 TCGOpcode opc, unsigned nargs)
5a18407f 2481{
d4478943 2482 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 2483 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2484 return new_op;
2485}
2486
b4fc67c7
RH		 2487/* Reachable analysis : remove unreachable code. */
2488static void reachable_code_pass(TCGContext *s)
2489{
2490 TCGOp *op, *op_next;
2491 bool dead = false;
2492
2493 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2494 bool remove = dead;
2495 TCGLabel *label;
b4fc67c7
RH		 2496
2497 switch (op->opc) {
2498 case INDEX_op_set_label:
2499 label = arg_label(op->args[0]);
2500 if (label->refs == 0) {
2501 /*
2502 * While there is an occasional backward branch, virtually
2503 * all branches generated by the translators are forward.
2504 * Which means that generally we will have already removed
2505 * all references to the label that will be, and there is
2506 * little to be gained by iterating.
2507 */
2508 remove = true;
2509 } else {
2510 /* Once we see a label, insns become live again. */
2511 dead = false;
2512 remove = false;
2513
2514 /*
2515 * Optimization can fold conditional branches to unconditional.
2516 * If we find a label with one reference which is preceded by
2517 * an unconditional branch to it, remove both. This needed to
2518 * wait until the dead code in between them was removed.
2519 */
2520 if (label->refs == 1) {
eae3eb3e 2521 TCGOp *op_prev = QTAILQ_PREV(op, link);
b4fc67c7
RH		 2522 if (op_prev->opc == INDEX_op_br &&
2523 label == arg_label(op_prev->args[0])) {
2524 tcg_op_remove(s, op_prev);
2525 remove = true;
2526 }
2527 }
2528 }
2529 break;
2530
2531 case INDEX_op_br:
2532 case INDEX_op_exit_tb:
2533 case INDEX_op_goto_ptr:
2534 /* Unconditional branches; everything following is dead. */
2535 dead = true;
2536 break;
2537
2538 case INDEX_op_call:
2539 /* Notice noreturn helper calls, raising exceptions. */
90163900 2540 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2541 dead = true;
2542 }
2543 break;
2544
2545 case INDEX_op_insn_start:
2546 /* Never remove -- we need to keep these for unwind. */
2547 remove = false;
2548 break;
2549
2550 default:
2551 break;
2552 }
2553
2554 if (remove) {
2555 tcg_op_remove(s, op);
2556 }
2557 }
2558}
2559
c70fbf0a
RH		 2560#define TS_DEAD 1
2561#define TS_MEM 2
2562
5a18407f
RH		 2563#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2564#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2565
25f49c5f
RH		 2566/* For liveness_pass_1, the register preferences for a given temp. */
2567static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2568{
2569 return ts->state_ptr;
2570}
2571
2572/* For liveness_pass_1, reset the preferences for a given temp to the
2573 * maximal regset for its type.
2574 */
2575static inline void la_reset_pref(TCGTemp *ts)
2576{
2577 *la_temp_pref(ts)
2578 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2579}
2580
9c43b68d
AJ		 2581/* liveness analysis: end of function: all temps are dead, and globals
2582 should be in memory. */
2616c808 2583static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2584{
b83eabea
RH		 2585 int i;
2586
2587 for (i = 0; i < ng; ++i) {
2588 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2589 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2590 }
2591 for (i = ng; i < nt; ++i) {
2592 s->temps[i].state = TS_DEAD;
25f49c5f 2593 la_reset_pref(&s->temps[i]);
b83eabea 2594 }
c896fe29
FB		 2595}
2596
9c43b68d
AJ		 2597/* liveness analysis: end of basic block: all temps are dead, globals
2598 and local temps should be in memory. */
2616c808 2599static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2600{
b83eabea 2601 int i;
641d5fbe 2602
ee17db83
RH		 2603 for (i = 0; i < nt; ++i) {
2604 TCGTemp *ts = &s->temps[i];
2605 int state;
2606
2607 switch (ts->kind) {
2608 case TEMP_FIXED:
2609 case TEMP_GLOBAL:
2610 case TEMP_LOCAL:
2611 state = TS_DEAD | TS_MEM;
2612 break;
2613 case TEMP_NORMAL:
c7482438 2614 case TEMP_EBB:
c0522136 2615 case TEMP_CONST:
ee17db83
RH		 2616 state = TS_DEAD;
2617 break;
2618 default:
2619 g_assert_not_reached();
2620 }
2621 ts->state = state;
2622 la_reset_pref(ts);
641d5fbe
FB		 2623 }
2624}
2625
f65a061c
RH		 2626/* liveness analysis: sync globals back to memory. */
2627static void la_global_sync(TCGContext *s, int ng)
2628{
2629 int i;
2630
2631 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2632 int state = s->temps[i].state;
2633 s->temps[i].state = state | TS_MEM;
2634 if (state == TS_DEAD) {
2635 /* If the global was previously dead, reset prefs. */
2636 la_reset_pref(&s->temps[i]);
2637 }
f65a061c
RH		 2638 }
2639}
2640
b4cb76e6 2641/*
c7482438
RH		 2642 * liveness analysis: conditional branch: all temps are dead unless
2643 * explicitly live-across-conditional-branch, globals and local temps
2644 * should be synced.
b4cb76e6
RH		 2645 */
2646static void la_bb_sync(TCGContext *s, int ng, int nt)
2647{
2648 la_global_sync(s, ng);
2649
2650 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2651 TCGTemp *ts = &s->temps[i];
2652 int state;
2653
2654 switch (ts->kind) {
2655 case TEMP_LOCAL:
2656 state = ts->state;
2657 ts->state = state | TS_MEM;
b4cb76e6
RH		 2658 if (state != TS_DEAD) {
2659 continue;
2660 }
c0522136
RH		 2661 break;
2662 case TEMP_NORMAL:
b4cb76e6 2663 s->temps[i].state = TS_DEAD;
c0522136 2664 break;
c7482438 2665 case TEMP_EBB:
c0522136
RH		 2666 case TEMP_CONST:
2667 continue;
2668 default:
2669 g_assert_not_reached();
b4cb76e6
RH		 2670 }
2671 la_reset_pref(&s->temps[i]);
2672 }
2673}
2674
f65a061c
RH		 2675/* liveness analysis: sync globals back to memory and kill. */
2676static void la_global_kill(TCGContext *s, int ng)
2677{
2678 int i;
2679
2680 for (i = 0; i < ng; i++) {
2681 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2682 la_reset_pref(&s->temps[i]);
2683 }
2684}
2685
2686/* liveness analysis: note live globals crossing calls. */
2687static void la_cross_call(TCGContext *s, int nt)
2688{
2689 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2690 int i;
2691
2692 for (i = 0; i < nt; i++) {
2693 TCGTemp *ts = &s->temps[i];
2694 if (!(ts->state & TS_DEAD)) {
2695 TCGRegSet *pset = la_temp_pref(ts);
2696 TCGRegSet set = *pset;
2697
2698 set &= mask;
2699 /* If the combination is not possible, restart. */
2700 if (set == 0) {
2701 set = tcg_target_available_regs[ts->type] & mask;
2702 }
2703 *pset = set;
2704 }
f65a061c
RH		 2705 }
2706}
2707
a1b3c48d 2708/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2709 given input arguments is dead. Instructions updating dead
2710 temporaries are removed. */
b83eabea 2711static void liveness_pass_1(TCGContext *s)
c896fe29 2712{
c70fbf0a 2713 int nb_globals = s->nb_globals;
2616c808 2714 int nb_temps = s->nb_temps;
15fa08f8 2715 TCGOp *op, *op_prev;
25f49c5f
RH		 2716 TCGRegSet *prefs;
2717 int i;
2718
2719 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2720 for (i = 0; i < nb_temps; ++i) {
2721 s->temps[i].state_ptr = prefs + i;
2722 }
a1b3c48d 2723
ae36a246 2724 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2725 la_func_end(s, nb_globals, nb_temps);
c896fe29 2726
eae3eb3e 2727 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2728 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2729 TCGOpcode opc_new, opc_new2;
2730 bool have_opc_new2;
a1b3c48d 2731 TCGLifeData arg_life = 0;
25f49c5f 2732 TCGTemp *ts;
c45cb8bb
RH		 2733 TCGOpcode opc = op->opc;
2734 const TCGOpDef *def = &tcg_op_defs[opc];
2735
c45cb8bb 2736 switch (opc) {
c896fe29 2737 case INDEX_op_call:
c6e113f5 2738 {
39004a71
RH		 2739 const TCGHelperInfo *info = tcg_call_info(op);
2740 int call_flags = tcg_call_flags(op);
c896fe29 2741
cd9090aa
RH		 2742 nb_oargs = TCGOP_CALLO(op);
2743 nb_iargs = TCGOP_CALLI(op);
c6e113f5 2744
c45cb8bb 2745 /* pure functions can be removed if their result is unused */
78505279 2746 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2747 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2748 ts = arg_temp(op->args[i]);
2749 if (ts->state != TS_DEAD) {
c6e113f5 2750 goto do_not_remove_call;
9c43b68d 2751 }
c6e113f5 2752 }
c45cb8bb 2753 goto do_remove;
152c35aa
RH		 2754 }
2755 do_not_remove_call:
c896fe29 2756
25f49c5f 2757 /* Output args are dead. */
152c35aa 2758 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2759 ts = arg_temp(op->args[i]);
2760 if (ts->state & TS_DEAD) {
152c35aa
RH		 2761 arg_life |= DEAD_ARG << i;
2762 }
25f49c5f 2763 if (ts->state & TS_MEM) {
152c35aa 2764 arg_life |= SYNC_ARG << i;
c6e113f5 2765 }
25f49c5f
RH		 2766 ts->state = TS_DEAD;
2767 la_reset_pref(ts);
152c35aa 2768 }
78505279 2769
31fd884b
RH		 2770 /* Not used -- it will be tcg_target_call_oarg_reg(). */
2771 memset(op->output_pref, 0, sizeof(op->output_pref));
2772
152c35aa
RH		 2773 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2774 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2775 la_global_kill(s, nb_globals);
152c35aa 2776 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2777 la_global_sync(s, nb_globals);
152c35aa 2778 }
b9c18f56 2779
25f49c5f 2780 /* Record arguments that die in this helper. */
152c35aa 2781 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f 2782 ts = arg_temp(op->args[i]);
39004a71 2783 if (ts->state & TS_DEAD) {
152c35aa 2784 arg_life |= DEAD_ARG << i;
c6e113f5 2785 }
152c35aa 2786 }
25f49c5f
RH		 2787
2788 /* For all live registers, remove call-clobbered prefs. */
2789 la_cross_call(s, nb_temps);
2790
39004a71
RH		 2791 /*
2792 * Input arguments are live for preceding opcodes.
2793 *
2794 * For those arguments that die, and will be allocated in
2795 * registers, clear the register set for that arg, to be
2796 * filled in below. For args that will be on the stack,
2797 * reset to any available reg. Process arguments in reverse
2798 * order so that if a temp is used more than once, the stack
2799 * reset to max happens before the register reset to 0.
2800 */
2801 for (i = nb_iargs - 1; i >= 0; i--) {
2802 const TCGCallArgumentLoc *loc = &info->in[i];
2803 ts = arg_temp(op->args[nb_oargs + i]);
25f49c5f 2804
39004a71
RH		 2805 if (ts->state & TS_DEAD) {
2806 switch (loc->kind) {
2807 case TCG_CALL_ARG_NORMAL:
2808 case TCG_CALL_ARG_EXTEND_U:
2809 case TCG_CALL_ARG_EXTEND_S:
2810 if (REG_P(loc)) {
2811 *la_temp_pref(ts) = 0;
2812 break;
2813 }
2814 /* fall through */
2815 default:
2816 *la_temp_pref(ts) =
2817 tcg_target_available_regs[ts->type];
2818 break;
2819 }
25f49c5f
RH		 2820 ts->state &= ~TS_DEAD;
2821 }
2822 }
2823
39004a71
RH		 2824 /*
2825 * For each input argument, add its input register to prefs.
2826 * If a temp is used once, this produces a single set bit;
2827 * if a temp is used multiple times, this produces a set.
2828 */
2829 for (i = 0; i < nb_iargs; i++) {
2830 const TCGCallArgumentLoc *loc = &info->in[i];
2831 ts = arg_temp(op->args[nb_oargs + i]);
2832
2833 switch (loc->kind) {
2834 case TCG_CALL_ARG_NORMAL:
2835 case TCG_CALL_ARG_EXTEND_U:
2836 case TCG_CALL_ARG_EXTEND_S:
2837 if (REG_P(loc)) {
2838 tcg_regset_set_reg(*la_temp_pref(ts),
2839 tcg_target_call_iarg_regs[loc->arg_slot]);
2840 }
2841 break;
2842 default:
2843 break;
c19f47bf 2844 }
c896fe29 2845 }
c896fe29 2846 }
c896fe29 2847 break;
765b842a 2848 case INDEX_op_insn_start:
c896fe29 2849 break;
5ff9d6a4 2850 case INDEX_op_discard:
5ff9d6a4 2851 /* mark the temporary as dead */
25f49c5f
RH		 2852 ts = arg_temp(op->args[0]);
2853 ts->state = TS_DEAD;
2854 la_reset_pref(ts);
5ff9d6a4 2855 break;
1305c451
RH		 2856
2857 case INDEX_op_add2_i32:
c45cb8bb 2858 opc_new = INDEX_op_add_i32;
f1fae40c 2859 goto do_addsub2;
1305c451 2860 case INDEX_op_sub2_i32:
c45cb8bb 2861 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 2862 goto do_addsub2;
2863 case INDEX_op_add2_i64:
c45cb8bb 2864 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 2865 goto do_addsub2;
2866 case INDEX_op_sub2_i64:
c45cb8bb 2867 opc_new = INDEX_op_sub_i64;
f1fae40c 2868 do_addsub2:
1305c451
RH		 2869 nb_iargs = 4;
2870 nb_oargs = 2;
2871 /* Test if the high part of the operation is dead, but not
2872 the low part. The result can be optimized to a simple
2873 add or sub. This happens often for x86_64 guest when the
2874 cpu mode is set to 32 bit. */
b83eabea
RH		 2875 if (arg_temp(op->args[1])->state == TS_DEAD) {
2876 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 2877 goto do_remove;
2878 }
c45cb8bb
RH		 2879 /* Replace the opcode and adjust the args in place,
2880 leaving 3 unused args at the end. */
2881 op->opc = opc = opc_new;
efee3746
RH		 2882 op->args[1] = op->args[2];
2883 op->args[2] = op->args[4];
1305c451
RH		 2884 /* Fall through and mark the single-word operation live. */
2885 nb_iargs = 2;
2886 nb_oargs = 1;
2887 }
2888 goto do_not_remove;
2889
1414968a 2890 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 2891 opc_new = INDEX_op_mul_i32;
2892 opc_new2 = INDEX_op_muluh_i32;
2893 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 2894 goto do_mul2;
f1fae40c 2895 case INDEX_op_muls2_i32:
c45cb8bb
RH		 2896 opc_new = INDEX_op_mul_i32;
2897 opc_new2 = INDEX_op_mulsh_i32;
2898 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 2899 goto do_mul2;
2900 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 2901 opc_new = INDEX_op_mul_i64;
2902 opc_new2 = INDEX_op_muluh_i64;
2903 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 2904 goto do_mul2;
f1fae40c 2905 case INDEX_op_muls2_i64:
c45cb8bb
RH		 2906 opc_new = INDEX_op_mul_i64;
2907 opc_new2 = INDEX_op_mulsh_i64;
2908 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 2909 goto do_mul2;
f1fae40c 2910 do_mul2:
1414968a
RH		 2911 nb_iargs = 2;
2912 nb_oargs = 2;
b83eabea
RH		 2913 if (arg_temp(op->args[1])->state == TS_DEAD) {
2914 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 2915 /* Both parts of the operation are dead. */
1414968a
RH		 2916 goto do_remove;
2917 }
03271524 2918 /* The high part of the operation is dead; generate the low. */
c45cb8bb 2919 op->opc = opc = opc_new;
efee3746
RH		 2920 op->args[1] = op->args[2];
2921 op->args[2] = op->args[3];
b83eabea 2922 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 2923 /* The low part of the operation is dead; generate the high. */
2924 op->opc = opc = opc_new2;
efee3746
RH		 2925 op->args[0] = op->args[1];
2926 op->args[1] = op->args[2];
2927 op->args[2] = op->args[3];
03271524
RH		 2928 } else {
2929 goto do_not_remove;
1414968a 2930 }
03271524
RH		 2931 /* Mark the single-word operation live. */
2932 nb_oargs = 1;
1414968a
RH		 2933 goto do_not_remove;
2934
c896fe29 2935 default:
1305c451 2936 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 2937 nb_iargs = def->nb_iargs;
2938 nb_oargs = def->nb_oargs;
c896fe29 2939
49516bc0
AJ		 2940 /* Test if the operation can be removed because all
2941 its outputs are dead. We assume that nb_oargs == 0
2942 implies side effects */
2943 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 2944 for (i = 0; i < nb_oargs; i++) {
b83eabea 2945 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 2946 goto do_not_remove;
9c43b68d 2947 }
49516bc0 2948 }
152c35aa
RH		 2949 goto do_remove;
2950 }
2951 goto do_not_remove;
49516bc0 2952
152c35aa
RH		 2953 do_remove:
2954 tcg_op_remove(s, op);
2955 break;
2956
2957 do_not_remove:
152c35aa 2958 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2959 ts = arg_temp(op->args[i]);
2960
2961 /* Remember the preference of the uses that followed. */
31fd884b
RH		 2962 if (i < ARRAY_SIZE(op->output_pref)) {
2963 op->output_pref[i] = *la_temp_pref(ts);
2964 }
25f49c5f
RH		 2965
2966 /* Output args are dead. */
2967 if (ts->state & TS_DEAD) {
152c35aa 2968 arg_life |= DEAD_ARG << i;
49516bc0 2969 }
25f49c5f 2970 if (ts->state & TS_MEM) {
152c35aa
RH		 2971 arg_life |= SYNC_ARG << i;
2972 }
25f49c5f
RH		 2973 ts->state = TS_DEAD;
2974 la_reset_pref(ts);
152c35aa 2975 }
49516bc0 2976
25f49c5f 2977 /* If end of basic block, update. */
ae36a246
RH		 2978 if (def->flags & TCG_OPF_BB_EXIT) {
2979 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 2980 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2981 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 2982 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 2983 la_bb_end(s, nb_globals, nb_temps);
152c35aa 2984 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 2985 la_global_sync(s, nb_globals);
25f49c5f
RH		 2986 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2987 la_cross_call(s, nb_temps);
2988 }
152c35aa
RH		 2989 }
2990
25f49c5f 2991 /* Record arguments that die in this opcode. */
152c35aa 2992 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2993 ts = arg_temp(op->args[i]);
2994 if (ts->state & TS_DEAD) {
152c35aa 2995 arg_life |= DEAD_ARG << i;
c896fe29 2996 }
c896fe29 2997 }
25f49c5f
RH		 2998
2999 /* Input arguments are live for preceding opcodes. */
152c35aa 3000 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 3001 ts = arg_temp(op->args[i]);
3002 if (ts->state & TS_DEAD) {
3003 /* For operands that were dead, initially allow
3004 all regs for the type. */
3005 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
3006 ts->state &= ~TS_DEAD;
3007 }
3008 }
3009
3010 /* Incorporate constraints for this operand. */
3011 switch (opc) {
3012 case INDEX_op_mov_i32:
3013 case INDEX_op_mov_i64:
3014 /* Note that these are TCG_OPF_NOT_PRESENT and do not
3015 have proper constraints. That said, special case
3016 moves to propagate preferences backward. */
3017 if (IS_DEAD_ARG(1)) {
3018 *la_temp_pref(arg_temp(op->args[0]))
3019 = *la_temp_pref(arg_temp(op->args[1]));
3020 }
3021 break;
3022
3023 default:
3024 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3025 const TCGArgConstraint *ct = &def->args_ct[i];
3026 TCGRegSet set, *pset;
3027
3028 ts = arg_temp(op->args[i]);
3029 pset = la_temp_pref(ts);
3030 set = *pset;
3031
9be0d080 3032 set &= ct->regs;
bc2b17e6 3033 if (ct->ialias) {
31fd884b 3034 set &= output_pref(op, ct->alias_index);
25f49c5f
RH		 3035 }
3036 /* If the combination is not possible, restart. */
3037 if (set == 0) {
9be0d080 3038 set = ct->regs;
25f49c5f
RH		 3039 }
3040 *pset = set;
3041 }
3042 break;
152c35aa 3043 }
c896fe29
FB		 3044 break;
3045 }
bee158cb 3046 op->life = arg_life;
1ff0a2c5 3047 }
c896fe29 3048}
c896fe29 3049
5a18407f 3050/* Liveness analysis: Convert indirect regs to direct temporaries. */
b83eabea 3051static bool liveness_pass_2(TCGContext *s)
5a18407f
RH		 3052{
3053 int nb_globals = s->nb_globals;
15fa08f8 3054 int nb_temps, i;
5a18407f 3055 bool changes = false;
15fa08f8 3056 TCGOp *op, *op_next;
5a18407f 3057
5a18407f
RH		 3058 /* Create a temporary for each indirect global. */
3059 for (i = 0; i < nb_globals; ++i) {
3060 TCGTemp *its = &s->temps[i];
3061 if (its->indirect_reg) {
3062 TCGTemp *dts = tcg_temp_alloc(s);
3063 dts->type = its->type;
3064 dts->base_type = its->base_type;
c7482438 3065 dts->kind = TEMP_EBB;
b83eabea
RH		 3066 its->state_ptr = dts;
3067 } else {
3068 its->state_ptr = NULL;
5a18407f 3069 }
b83eabea
RH		 3070 /* All globals begin dead. */
3071 its->state = TS_DEAD;
3072 }
3073 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3074 TCGTemp *its = &s->temps[i];
3075 its->state_ptr = NULL;
3076 its->state = TS_DEAD;
5a18407f 3077 }
5a18407f 3078
15fa08f8 3079 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 3080 TCGOpcode opc = op->opc;
3081 const TCGOpDef *def = &tcg_op_defs[opc];
3082 TCGLifeData arg_life = op->life;
3083 int nb_iargs, nb_oargs, call_flags;
b83eabea 3084 TCGTemp *arg_ts, *dir_ts;
5a18407f 3085
5a18407f 3086 if (opc == INDEX_op_call) {
cd9090aa
RH		 3087 nb_oargs = TCGOP_CALLO(op);
3088 nb_iargs = TCGOP_CALLI(op);
90163900 3089 call_flags = tcg_call_flags(op);
5a18407f
RH		 3090 } else {
3091 nb_iargs = def->nb_iargs;
3092 nb_oargs = def->nb_oargs;
3093
3094 /* Set flags similar to how calls require. */
b4cb76e6
RH		 3095 if (def->flags & TCG_OPF_COND_BRANCH) {
3096 /* Like reading globals: sync_globals */
3097 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3098 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 3099 /* Like writing globals: save_globals */
3100 call_flags = 0;
3101 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3102 /* Like reading globals: sync_globals */
3103 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3104 } else {
3105 /* No effect on globals. */
3106 call_flags = (TCG_CALL_NO_READ_GLOBALS |
3107 TCG_CALL_NO_WRITE_GLOBALS);
3108 }
3109 }
3110
3111 /* Make sure that input arguments are available. */
3112 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3113 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3114 dir_ts = arg_ts->state_ptr;
3115 if (dir_ts && arg_ts->state == TS_DEAD) {
3116 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
3117 ? INDEX_op_ld_i32
3118 : INDEX_op_ld_i64);
3119 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
3120
3121 lop->args[0] = temp_arg(dir_ts);
3122 lop->args[1] = temp_arg(arg_ts->mem_base);
3123 lop->args[2] = arg_ts->mem_offset;
3124
3125 /* Loaded, but synced with memory. */
3126 arg_ts->state = TS_MEM;
5a18407f
RH		 3127 }
3128 }
3129
3130 /* Perform input replacement, and mark inputs that became dead.
3131 No action is required except keeping temp_state up to date
3132 so that we reload when needed. */
3133 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3134 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3135 dir_ts = arg_ts->state_ptr;
3136 if (dir_ts) {
3137 op->args[i] = temp_arg(dir_ts);
3138 changes = true;
3139 if (IS_DEAD_ARG(i)) {
3140 arg_ts->state = TS_DEAD;
5a18407f
RH		 3141 }
3142 }
3143 }
3144
3145 /* Liveness analysis should ensure that the following are
3146 all correct, for call sites and basic block end points. */
3147 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
3148 /* Nothing to do */
3149 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
3150 for (i = 0; i < nb_globals; ++i) {
3151 /* Liveness should see that globals are synced back,
3152 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 3153 arg_ts = &s->temps[i];
3154 tcg_debug_assert(arg_ts->state_ptr == 0
3155 || arg_ts->state != 0);
5a18407f
RH		 3156 }
3157 } else {
3158 for (i = 0; i < nb_globals; ++i) {
3159 /* Liveness should see that globals are saved back,
3160 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 3161 arg_ts = &s->temps[i];
3162 tcg_debug_assert(arg_ts->state_ptr == 0
3163 || arg_ts->state == TS_DEAD);
5a18407f
RH		 3164 }
3165 }
3166
3167 /* Outputs become available. */
61f15c48
RH		 3168 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
3169 arg_ts = arg_temp(op->args[0]);
b83eabea 3170 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 3171 if (dir_ts) {
3172 op->args[0] = temp_arg(dir_ts);
3173 changes = true;
3174
3175 /* The output is now live and modified. */
3176 arg_ts->state = 0;
3177
3178 if (NEED_SYNC_ARG(0)) {
3179 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3180 ? INDEX_op_st_i32
3181 : INDEX_op_st_i64);
d4478943 3182 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
61f15c48
RH		 3183 TCGTemp *out_ts = dir_ts;
3184
3185 if (IS_DEAD_ARG(0)) {
3186 out_ts = arg_temp(op->args[1]);
3187 arg_ts->state = TS_DEAD;
3188 tcg_op_remove(s, op);
3189 } else {
3190 arg_ts->state = TS_MEM;
3191 }
3192
3193 sop->args[0] = temp_arg(out_ts);
3194 sop->args[1] = temp_arg(arg_ts->mem_base);
3195 sop->args[2] = arg_ts->mem_offset;
3196 } else {
3197 tcg_debug_assert(!IS_DEAD_ARG(0));
3198 }
5a18407f 3199 }
61f15c48
RH		 3200 } else {
3201 for (i = 0; i < nb_oargs; i++) {
3202 arg_ts = arg_temp(op->args[i]);
3203 dir_ts = arg_ts->state_ptr;
3204 if (!dir_ts) {
3205 continue;
3206 }
3207 op->args[i] = temp_arg(dir_ts);
3208 changes = true;
5a18407f 3209
61f15c48
RH		 3210 /* The output is now live and modified. */
3211 arg_ts->state = 0;
5a18407f 3212
61f15c48
RH		 3213 /* Sync outputs upon their last write. */
3214 if (NEED_SYNC_ARG(i)) {
3215 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3216 ? INDEX_op_st_i32
3217 : INDEX_op_st_i64);
d4478943 3218 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
5a18407f 3219
61f15c48
RH		 3220 sop->args[0] = temp_arg(dir_ts);
3221 sop->args[1] = temp_arg(arg_ts->mem_base);
3222 sop->args[2] = arg_ts->mem_offset;
5a18407f 3223
61f15c48
RH		 3224 arg_ts->state = TS_MEM;
3225 }
3226 /* Drop outputs that are dead. */
3227 if (IS_DEAD_ARG(i)) {
3228 arg_ts->state = TS_DEAD;
3229 }
5a18407f
RH		 3230 }
3231 }
3232 }
3233
3234 return changes;
3235}
3236
2272e4a7 3237static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 3238{
31c96417
RH		 3239 int size = tcg_type_size(ts->type);
3240 int align;
3241 intptr_t off;
c1c09194
RH		 3242
3243 switch (ts->type) {
3244 case TCG_TYPE_I32:
31c96417 3245 align = 4;
c1c09194
RH		 3246 break;
3247 case TCG_TYPE_I64:
3248 case TCG_TYPE_V64:
31c96417 3249 align = 8;
c1c09194
RH		 3250 break;
3251 case TCG_TYPE_V128:
c1c09194
RH		 3252 case TCG_TYPE_V256:
3253 /* Note that we do not require aligned storage for V256. */
31c96417 3254 align = 16;
c1c09194
RH		 3255 break;
3256 default:
3257 g_assert_not_reached();
b591dc59 3258 }
c1c09194 3259
b9537d59
RH		 3260 /*
3261 * Assume the stack is sufficiently aligned.
3262 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3263 * and do not require 16 byte vector alignment. This seems slightly
3264 * easier than fully parameterizing the above switch statement.
3265 */
3266 align = MIN(TCG_TARGET_STACK_ALIGN, align);
c1c09194 3267 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 3268
3269 /* If we've exhausted the stack frame, restart with a smaller TB. */
3270 if (off + size > s->frame_end) {
3271 tcg_raise_tb_overflow(s);
3272 }
c1c09194
RH		 3273 s->current_frame_offset = off + size;
3274
3275 ts->mem_offset = off;
9defd1bd
RH		 3276#if defined(__sparc__)
3277 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3278#endif
b3a62939 3279 ts->mem_base = s->frame_temp;
c896fe29 3280 ts->mem_allocated = 1;
c896fe29
FB		 3281}
3282
098859f1
RH		 3283/* Assign @reg to @ts, and update reg_to_temp[]. */
3284static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
3285{
3286 if (ts->val_type == TEMP_VAL_REG) {
3287 TCGReg old = ts->reg;
3288 tcg_debug_assert(s->reg_to_temp[old] == ts);
3289 if (old == reg) {
3290 return;
3291 }
3292 s->reg_to_temp[old] = NULL;
3293 }
3294 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
3295 s->reg_to_temp[reg] = ts;
3296 ts->val_type = TEMP_VAL_REG;
3297 ts->reg = reg;
3298}
3299
3300/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
3301static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
3302{
3303 tcg_debug_assert(type != TEMP_VAL_REG);
3304 if (ts->val_type == TEMP_VAL_REG) {
3305 TCGReg reg = ts->reg;
3306 tcg_debug_assert(s->reg_to_temp[reg] == ts);
3307 s->reg_to_temp[reg] = NULL;
3308 }
3309 ts->val_type = type;
3310}
3311
b722452a 3312static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3313
59d7c14e
RH		 3314/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3315 mark it free; otherwise mark it dead. */
3316static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3317{
c0522136
RH		 3318 TCGTempVal new_type;
3319
3320 switch (ts->kind) {
3321 case TEMP_FIXED:
59d7c14e 3322 return;
c0522136
RH		 3323 case TEMP_GLOBAL:
3324 case TEMP_LOCAL:
3325 new_type = TEMP_VAL_MEM;
3326 break;
3327 case TEMP_NORMAL:
c7482438 3328 case TEMP_EBB:
c0522136
RH		 3329 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3330 break;
3331 case TEMP_CONST:
3332 new_type = TEMP_VAL_CONST;
3333 break;
3334 default:
3335 g_assert_not_reached();
59d7c14e 3336 }
098859f1 3337 set_temp_val_nonreg(s, ts, new_type);
59d7c14e 3338}
7f6ceedf 3339
59d7c14e
RH		 3340/* Mark a temporary as dead. */
3341static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3342{
3343 temp_free_or_dead(s, ts, 1);
3344}
3345
3346/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3347 registers needs to be allocated to store a constant. If 'free_or_dead'
3348 is non-zero, subsequently release the temporary; if it is positive, the
3349 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3350static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3351 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3352{
c0522136 3353 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3354 if (!ts->mem_allocated) {
2272e4a7 3355 temp_allocate_frame(s, ts);
59d7c14e 3356 }
59d7c14e
RH		 3357 switch (ts->val_type) {
3358 case TEMP_VAL_CONST:
3359 /* If we're going to free the temp immediately, then we won't
3360 require it later in a register, so attempt to store the
3361 constant to memory directly. */
3362 if (free_or_dead
3363 && tcg_out_sti(s, ts->type, ts->val,
3364 ts->mem_base->reg, ts->mem_offset)) {
3365 break;
3366 }
3367 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3368 allocated_regs, preferred_regs);
59d7c14e
RH		 3369 /* fallthrough */
3370
3371 case TEMP_VAL_REG:
3372 tcg_out_st(s, ts->type, ts->reg,
3373 ts->mem_base->reg, ts->mem_offset);
3374 break;
3375
3376 case TEMP_VAL_MEM:
3377 break;
3378
3379 case TEMP_VAL_DEAD:
3380 default:
3381 tcg_abort();
3382 }
3383 ts->mem_coherent = 1;
3384 }
3385 if (free_or_dead) {
3386 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3387 }
7f6ceedf
AJ		 3388}
3389
c896fe29 3390/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3391static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3392{
f8b2f202 3393 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3394 if (ts != NULL) {
98b4e186 3395 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3396 }
3397}
3398
b016486e
RH		 3399/**
3400 * tcg_reg_alloc:
3401 * @required_regs: Set of registers in which we must allocate.
3402 * @allocated_regs: Set of registers which must be avoided.
3403 * @preferred_regs: Set of registers we should prefer.
3404 * @rev: True if we search the registers in "indirect" order.
3405 *
3406 * The allocated register must be in @required_regs & ~@allocated_regs,
3407 * but if we can put it in @preferred_regs we may save a move later.
3408 */
3409static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3410 TCGRegSet allocated_regs,
3411 TCGRegSet preferred_regs, bool rev)
c896fe29 3412{
b016486e
RH		 3413 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3414 TCGRegSet reg_ct[2];
91478cef 3415 const int *order;
c896fe29 3416
b016486e
RH		 3417 reg_ct[1] = required_regs & ~allocated_regs;
3418 tcg_debug_assert(reg_ct[1] != 0);
3419 reg_ct[0] = reg_ct[1] & preferred_regs;
3420
3421 /* Skip the preferred_regs option if it cannot be satisfied,
3422 or if the preference made no difference. */
3423 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3424
91478cef 3425 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3426
b016486e
RH		 3427 /* Try free registers, preferences first. */
3428 for (j = f; j < 2; j++) {
3429 TCGRegSet set = reg_ct[j];
3430
3431 if (tcg_regset_single(set)) {
3432 /* One register in the set. */
3433 TCGReg reg = tcg_regset_first(set);
3434 if (s->reg_to_temp[reg] == NULL) {
3435 return reg;
3436 }
3437 } else {
3438 for (i = 0; i < n; i++) {
3439 TCGReg reg = order[i];
3440 if (s->reg_to_temp[reg] == NULL &&
3441 tcg_regset_test_reg(set, reg)) {
3442 return reg;
3443 }
3444 }
3445 }
c896fe29
FB		 3446 }
3447
b016486e
RH		 3448 /* We must spill something. */
3449 for (j = f; j < 2; j++) {
3450 TCGRegSet set = reg_ct[j];
3451
3452 if (tcg_regset_single(set)) {
3453 /* One register in the set. */
3454 TCGReg reg = tcg_regset_first(set);
b3915dbb 3455 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3456 return reg;
b016486e
RH		 3457 } else {
3458 for (i = 0; i < n; i++) {
3459 TCGReg reg = order[i];
3460 if (tcg_regset_test_reg(set, reg)) {
3461 tcg_reg_free(s, reg, allocated_regs);
3462 return reg;
3463 }
3464 }
c896fe29
FB		 3465 }
3466 }
3467
3468 tcg_abort();
3469}
3470
29f5e925
RH		 3471static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
3472 TCGRegSet allocated_regs,
3473 TCGRegSet preferred_regs, bool rev)
3474{
3475 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3476 TCGRegSet reg_ct[2];
3477 const int *order;
3478
3479 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
3480 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
3481 tcg_debug_assert(reg_ct[1] != 0);
3482 reg_ct[0] = reg_ct[1] & preferred_regs;
3483
3484 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3485
3486 /*
3487 * Skip the preferred_regs option if it cannot be satisfied,
3488 * or if the preference made no difference.
3489 */
3490 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3491
3492 /*
3493 * Minimize the number of flushes by looking for 2 free registers first,
3494 * then a single flush, then two flushes.
3495 */
3496 for (fmin = 2; fmin >= 0; fmin--) {
3497 for (j = k; j < 2; j++) {
3498 TCGRegSet set = reg_ct[j];
3499
3500 for (i = 0; i < n; i++) {
3501 TCGReg reg = order[i];
3502
3503 if (tcg_regset_test_reg(set, reg)) {
3504 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
3505 if (f >= fmin) {
3506 tcg_reg_free(s, reg, allocated_regs);
3507 tcg_reg_free(s, reg + 1, allocated_regs);
3508 return reg;
3509 }
3510 }
3511 }
3512 }
3513 }
3514 tcg_abort();
3515}
3516
40ae5c62
RH		 3517/* Make sure the temporary is in a register. If needed, allocate the register
3518 from DESIRED while avoiding ALLOCATED. */
3519static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3520 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3521{
3522 TCGReg reg;
3523
3524 switch (ts->val_type) {
3525 case TEMP_VAL_REG:
3526 return;
3527 case TEMP_VAL_CONST:
b016486e 3528 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3529 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3530 if (ts->type <= TCG_TYPE_I64) {
3531 tcg_out_movi(s, ts->type, reg, ts->val);
3532 } else {
4e186175
RH		 3533 uint64_t val = ts->val;
3534 MemOp vece = MO_64;
3535
3536 /*
3537 * Find the minimal vector element that matches the constant.
3538 * The targets will, in general, have to do this search anyway,
3539 * do this generically.
3540 */
4e186175
RH		 3541 if (val == dup_const(MO_8, val)) {
3542 vece = MO_8;
3543 } else if (val == dup_const(MO_16, val)) {
3544 vece = MO_16;
0b4286dd 3545 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3546 vece = MO_32;
3547 }
3548
3549 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3550 }
40ae5c62
RH		 3551 ts->mem_coherent = 0;
3552 break;
3553 case TEMP_VAL_MEM:
b016486e 3554 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3555 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3556 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3557 ts->mem_coherent = 1;
3558 break;
3559 case TEMP_VAL_DEAD:
3560 default:
3561 tcg_abort();
3562 }
098859f1 3563 set_temp_val_reg(s, ts, reg);
40ae5c62
RH		 3564}
3565
59d7c14e
RH		 3566/* Save a temporary to memory. 'allocated_regs' is used in case a
3567 temporary registers needs to be allocated to store a constant. */
3568static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3569{
5a18407f
RH		 3570 /* The liveness analysis already ensures that globals are back
3571 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3572 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3573}
3574
9814dd27 3575/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3576 modified be the following code. 'allocated_regs' is used in case a
3577 temporary registers needs to be allocated to store a constant. */
3578static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3579{
ac3b8891 3580 int i, n;
c896fe29 3581
ac3b8891 3582 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3583 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3584 }
e5097dc8
FB		 3585}
3586
3d5c5f87
AJ		 3587/* sync globals to their canonical location and assume they can be
3588 read by the following code. 'allocated_regs' is used in case a
3589 temporary registers needs to be allocated to store a constant. */
3590static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3591{
ac3b8891 3592 int i, n;
3d5c5f87 3593
ac3b8891 3594 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3595 TCGTemp *ts = &s->temps[i];
5a18407f 3596 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3597 || ts->kind == TEMP_FIXED
5a18407f 3598 || ts->mem_coherent);
3d5c5f87
AJ		 3599 }
3600}
3601
e5097dc8 3602/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3603 all globals are stored at their canonical location. */
3604static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3605{
e5097dc8
FB		 3606 int i;
3607
b13eb728
RH		 3608 for (i = s->nb_globals; i < s->nb_temps; i++) {
3609 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3610
3611 switch (ts->kind) {
3612 case TEMP_LOCAL:
b13eb728 3613 temp_save(s, ts, allocated_regs);
c0522136
RH		 3614 break;
3615 case TEMP_NORMAL:
c7482438 3616 case TEMP_EBB:
5a18407f
RH		 3617 /* The liveness analysis already ensures that temps are dead.
3618 Keep an tcg_debug_assert for safety. */
3619 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3620 break;
3621 case TEMP_CONST:
3622 /* Similarly, we should have freed any allocated register. */
3623 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3624 break;
3625 default:
3626 g_assert_not_reached();
c896fe29
FB		 3627 }
3628 }
e8996ee0
FB		 3629
3630 save_globals(s, allocated_regs);
c896fe29
FB		 3631}
3632
b4cb76e6 3633/*
c7482438
RH		 3634 * At a conditional branch, we assume all temporaries are dead unless
3635 * explicitly live-across-conditional-branch; all globals and local
3636 * temps are synced to their location.
b4cb76e6
RH		 3637 */
3638static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3639{
3640 sync_globals(s, allocated_regs);
3641
3642 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3643 TCGTemp *ts = &s->temps[i];
3644 /*
3645 * The liveness analysis already ensures that temps are dead.
3646 * Keep tcg_debug_asserts for safety.
3647 */
c0522136
RH		 3648 switch (ts->kind) {
3649 case TEMP_LOCAL:
b4cb76e6 3650 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136
RH		 3651 break;
3652 case TEMP_NORMAL:
b4cb76e6 3653 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136 3654 break;
c7482438 3655 case TEMP_EBB:
c0522136
RH		 3656 case TEMP_CONST:
3657 break;
3658 default:
3659 g_assert_not_reached();
b4cb76e6
RH		 3660 }
3661 }
3662}
3663
bab1671f 3664/*
c58f4c97 3665 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3666 */
0fe4fca4 3667static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3668 tcg_target_ulong val, TCGLifeData arg_life,
3669 TCGRegSet preferred_regs)
e8996ee0 3670{
d63e3b6e 3671 /* ENV should not be modified. */
e01fa97d 3672 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3673
3674 /* The movi is not explicitly generated here. */
098859f1 3675 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
59d7c14e
RH		 3676 ots->val = val;
3677 ots->mem_coherent = 0;
3678 if (NEED_SYNC_ARG(0)) {
ba87719c 3679 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3680 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3681 temp_dead(s, ots);
4c4e1ab2 3682 }
e8996ee0
FB		 3683}
3684
bab1671f
RH		 3685/*
3686 * Specialized code generation for INDEX_op_mov_*.
3687 */
dd186292 3688static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3689{
dd186292 3690 const TCGLifeData arg_life = op->life;
69e3706d 3691 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3692 TCGTemp *ts, *ots;
450445d5 3693 TCGType otype, itype;
098859f1 3694 TCGReg oreg, ireg;
c896fe29 3695
d21369f5 3696 allocated_regs = s->reserved_regs;
31fd884b 3697 preferred_regs = output_pref(op, 0);
43439139
RH		 3698 ots = arg_temp(op->args[0]);
3699 ts = arg_temp(op->args[1]);
450445d5 3700
d63e3b6e 3701 /* ENV should not be modified. */
e01fa97d 3702 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3703
450445d5
RH		 3704 /* Note that otype != itype for no-op truncation. */
3705 otype = ots->type;
3706 itype = ts->type;
c29c1d7e 3707
0fe4fca4
PB		 3708 if (ts->val_type == TEMP_VAL_CONST) {
3709 /* propagate constant or generate sti */
3710 tcg_target_ulong val = ts->val;
3711 if (IS_DEAD_ARG(1)) {
3712 temp_dead(s, ts);
3713 }
69e3706d 3714 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3715 return;
3716 }
3717
3718 /* If the source value is in memory we're going to be forced
3719 to have it in a register in order to perform the copy. Copy
3720 the SOURCE value into its own register first, that way we
3721 don't have to reload SOURCE the next time it is used. */
3722 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3723 temp_load(s, ts, tcg_target_available_regs[itype],
3724 allocated_regs, preferred_regs);
c29c1d7e 3725 }
0fe4fca4 3726 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
098859f1
RH		 3727 ireg = ts->reg;
3728
d63e3b6e 3729 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3730 /* mov to a non-saved dead register makes no sense (even with
3731 liveness analysis disabled). */
eabb7b91 3732 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3733 if (!ots->mem_allocated) {
2272e4a7 3734 temp_allocate_frame(s, ots);
c29c1d7e 3735 }
098859f1 3736 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3737 if (IS_DEAD_ARG(1)) {
f8bf00f1 3738 temp_dead(s, ts);
c29c1d7e 3739 }
f8bf00f1 3740 temp_dead(s, ots);
098859f1
RH		 3741 return;
3742 }
3743
3744 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
3745 /*
3746 * The mov can be suppressed. Kill input first, so that it
3747 * is unlinked from reg_to_temp, then set the output to the
3748 * reg that we saved from the input.
3749 */
3750 temp_dead(s, ts);
3751 oreg = ireg;
c29c1d7e 3752 } else {
098859f1
RH		 3753 if (ots->val_type == TEMP_VAL_REG) {
3754 oreg = ots->reg;
c896fe29 3755 } else {
098859f1
RH		 3756 /* Make sure to not spill the input register during allocation. */
3757 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
3758 allocated_regs | ((TCGRegSet)1 << ireg),
3759 preferred_regs, ots->indirect_base);
c896fe29 3760 }
098859f1
RH		 3761 if (!tcg_out_mov(s, otype, oreg, ireg)) {
3762 /*
3763 * Cross register class move not supported.
3764 * Store the source register into the destination slot
3765 * and leave the destination temp as TEMP_VAL_MEM.
3766 */
3767 assert(!temp_readonly(ots));
3768 if (!ts->mem_allocated) {
3769 temp_allocate_frame(s, ots);
3770 }
3771 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
3772 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
3773 ots->mem_coherent = 1;
3774 return;
c896fe29 3775 }
ec7a869d 3776 }
098859f1
RH		 3777 set_temp_val_reg(s, ots, oreg);
3778 ots->mem_coherent = 0;
3779
3780 if (NEED_SYNC_ARG(0)) {
3781 temp_sync(s, ots, allocated_regs, 0, 0);
3782 }
c896fe29
FB		 3783}
3784
bab1671f
RH		 3785/*
3786 * Specialized code generation for INDEX_op_dup_vec.
3787 */
3788static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3789{
3790 const TCGLifeData arg_life = op->life;
3791 TCGRegSet dup_out_regs, dup_in_regs;
3792 TCGTemp *its, *ots;
3793 TCGType itype, vtype;
3794 unsigned vece;
31c96417 3795 int lowpart_ofs;
bab1671f
RH		 3796 bool ok;
3797
3798 ots = arg_temp(op->args[0]);
3799 its = arg_temp(op->args[1]);
3800
3801 /* ENV should not be modified. */
e01fa97d 3802 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 3803
3804 itype = its->type;
3805 vece = TCGOP_VECE(op);
3806 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3807
3808 if (its->val_type == TEMP_VAL_CONST) {
3809 /* Propagate constant via movi -> dupi. */
3810 tcg_target_ulong val = its->val;
3811 if (IS_DEAD_ARG(1)) {
3812 temp_dead(s, its);
3813 }
31fd884b 3814 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
bab1671f
RH		 3815 return;
3816 }
3817
9be0d080
RH		 3818 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3819 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 3820
3821 /* Allocate the output register now. */
3822 if (ots->val_type != TEMP_VAL_REG) {
3823 TCGRegSet allocated_regs = s->reserved_regs;
098859f1 3824 TCGReg oreg;
bab1671f
RH		 3825
3826 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3827 /* Make sure to not spill the input register. */
3828 tcg_regset_set_reg(allocated_regs, its->reg);
3829 }
098859f1 3830 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 3831 output_pref(op, 0), ots->indirect_base);
098859f1 3832 set_temp_val_reg(s, ots, oreg);
bab1671f
RH		 3833 }
3834
3835 switch (its->val_type) {
3836 case TEMP_VAL_REG:
3837 /*
3838 * The dup constriaints must be broad, covering all possible VECE.
3839 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3840 * to fail, indicating that extra moves are required for that case.
3841 */
3842 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3843 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3844 goto done;
3845 }
3846 /* Try again from memory or a vector input register. */
3847 }
3848 if (!its->mem_coherent) {
3849 /*
3850 * The input register is not synced, and so an extra store
3851 * would be required to use memory. Attempt an integer-vector
3852 * register move first. We do not have a TCGRegSet for this.
3853 */
3854 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3855 break;
3856 }
3857 /* Sync the temp back to its slot and load from there. */
3858 temp_sync(s, its, s->reserved_regs, 0, 0);
3859 }
3860 /* fall through */
3861
3862 case TEMP_VAL_MEM:
31c96417
RH		 3863 lowpart_ofs = 0;
3864 if (HOST_BIG_ENDIAN) {
3865 lowpart_ofs = tcg_type_size(itype) - (1 << vece);
3866 }
d6ecb4a9 3867 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
31c96417 3868 its->mem_offset + lowpart_ofs)) {
d6ecb4a9
RH		 3869 goto done;
3870 }
098859f1 3871 /* Load the input into the destination vector register. */
bab1671f
RH		 3872 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3873 break;
3874
3875 default:
3876 g_assert_not_reached();
3877 }
3878
3879 /* We now have a vector input register, so dup must succeed. */
3880 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3881 tcg_debug_assert(ok);
3882
3883 done:
36f5539c 3884 ots->mem_coherent = 0;
bab1671f
RH		 3885 if (IS_DEAD_ARG(1)) {
3886 temp_dead(s, its);
3887 }
3888 if (NEED_SYNC_ARG(0)) {
3889 temp_sync(s, ots, s->reserved_regs, 0, 0);
3890 }
3891 if (IS_DEAD_ARG(0)) {
3892 temp_dead(s, ots);
3893 }
3894}
3895
dd186292 3896static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 3897{
dd186292
RH		 3898 const TCGLifeData arg_life = op->life;
3899 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 3900 TCGRegSet i_allocated_regs;
3901 TCGRegSet o_allocated_regs;
b6638662
RH		 3902 int i, k, nb_iargs, nb_oargs;
3903 TCGReg reg;
c896fe29
FB		 3904 TCGArg arg;
3905 const TCGArgConstraint *arg_ct;
3906 TCGTemp *ts;
3907 TCGArg new_args[TCG_MAX_OP_ARGS];
3908 int const_args[TCG_MAX_OP_ARGS];
3909
3910 nb_oargs = def->nb_oargs;
3911 nb_iargs = def->nb_iargs;
3912
3913 /* copy constants */
a813e36f 3914 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 3915 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 3916 sizeof(TCGArg) * def->nb_cargs);
3917
d21369f5
RH		 3918 i_allocated_regs = s->reserved_regs;
3919 o_allocated_regs = s->reserved_regs;
82790a87 3920
a813e36f 3921 /* satisfy input constraints */
dd186292 3922 for (k = 0; k < nb_iargs; k++) {
29f5e925
RH		 3923 TCGRegSet i_preferred_regs, i_required_regs;
3924 bool allocate_new_reg, copyto_new_reg;
3925 TCGTemp *ts2;
3926 int i1, i2;
d62816f2 3927
66792f90 3928 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 3929 arg = op->args[i];
c896fe29 3930 arg_ct = &def->args_ct[i];
43439139 3931 ts = arg_temp(arg);
40ae5c62
RH		 3932
3933 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 3934 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 3935 /* constant is OK for instruction */
3936 const_args[i] = 1;
3937 new_args[i] = ts->val;
d62816f2 3938 continue;
c896fe29 3939 }
40ae5c62 3940
1c1824dc
RH		 3941 reg = ts->reg;
3942 i_preferred_regs = 0;
29f5e925 3943 i_required_regs = arg_ct->regs;
1c1824dc 3944 allocate_new_reg = false;
29f5e925
RH		 3945 copyto_new_reg = false;
3946
3947 switch (arg_ct->pair) {
3948 case 0: /* not paired */
3949 if (arg_ct->ialias) {
31fd884b 3950 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 3951
3952 /*
3953 * If the input is readonly, then it cannot also be an
3954 * output and aliased to itself. If the input is not
3955 * dead after the instruction, we must allocate a new
3956 * register and move it.
3957 */
3958 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3959 allocate_new_reg = true;
3960 } else if (ts->val_type == TEMP_VAL_REG) {
3961 /*
3962 * Check if the current register has already been
3963 * allocated for another input.
3964 */
3965 allocate_new_reg =
3966 tcg_regset_test_reg(i_allocated_regs, reg);
3967 }
3968 }
3969 if (!allocate_new_reg) {
3970 temp_load(s, ts, i_required_regs, i_allocated_regs,
3971 i_preferred_regs);
3972 reg = ts->reg;
3973 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
3974 }
3975 if (allocate_new_reg) {
3976 /*
3977 * Allocate a new register matching the constraint
3978 * and move the temporary register into it.
3979 */
3980 temp_load(s, ts, tcg_target_available_regs[ts->type],
3981 i_allocated_regs, 0);
3982 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
3983 i_preferred_regs, ts->indirect_base);
3984 copyto_new_reg = true;
3985 }
3986 break;
3987
3988 case 1:
3989 /* First of an input pair; if i1 == i2, the second is an output. */
3990 i1 = i;
3991 i2 = arg_ct->pair_index;
3992 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
3993
3994 /*
3995 * It is easier to default to allocating a new pair
3996 * and to identify a few cases where it's not required.
3997 */
3998 if (arg_ct->ialias) {
31fd884b 3999 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 4000 if (IS_DEAD_ARG(i1) &&
4001 IS_DEAD_ARG(i2) &&
4002 !temp_readonly(ts) &&
4003 ts->val_type == TEMP_VAL_REG &&
4004 ts->reg < TCG_TARGET_NB_REGS - 1 &&
4005 tcg_regset_test_reg(i_required_regs, reg) &&
4006 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4007 !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
4008 (ts2
4009 ? ts2->val_type == TEMP_VAL_REG &&
4010 ts2->reg == reg + 1 &&
4011 !temp_readonly(ts2)
4012 : s->reg_to_temp[reg + 1] == NULL)) {
4013 break;
4014 }
4015 } else {
4016 /* Without aliasing, the pair must also be an input. */
4017 tcg_debug_assert(ts2);
4018 if (ts->val_type == TEMP_VAL_REG &&
4019 ts2->val_type == TEMP_VAL_REG &&
4020 ts2->reg == reg + 1 &&
4021 tcg_regset_test_reg(i_required_regs, reg)) {
4022 break;
4023 }
4024 }
4025 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
4026 0, ts->indirect_base);
4027 goto do_pair;
4028
4029 case 2: /* pair second */
4030 reg = new_args[arg_ct->pair_index] + 1;
4031 goto do_pair;
1c1824dc 4032
29f5e925
RH		 4033 case 3: /* ialias with second output, no first input */
4034 tcg_debug_assert(arg_ct->ialias);
31fd884b 4035 i_preferred_regs = output_pref(op, arg_ct->alias_index);
d62816f2 4036
29f5e925
RH		 4037 if (IS_DEAD_ARG(i) &&
4038 !temp_readonly(ts) &&
4039 ts->val_type == TEMP_VAL_REG &&
4040 reg > 0 &&
4041 s->reg_to_temp[reg - 1] == NULL &&
4042 tcg_regset_test_reg(i_required_regs, reg) &&
4043 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4044 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
4045 tcg_regset_set_reg(i_allocated_regs, reg - 1);
4046 break;
4047 }
4048 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
4049 i_allocated_regs, 0,
4050 ts->indirect_base);
4051 tcg_regset_set_reg(i_allocated_regs, reg);
4052 reg += 1;
4053 goto do_pair;
4054
4055 do_pair:
c0522136 4056 /*
29f5e925
RH		 4057 * If an aliased input is not dead after the instruction,
4058 * we must allocate a new register and move it.
c0522136 4059 */
29f5e925
RH		 4060 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
4061 TCGRegSet t_allocated_regs = i_allocated_regs;
4062
1c1824dc 4063 /*
29f5e925
RH		 4064 * Because of the alias, and the continued life, make sure
4065 * that the temp is somewhere *other* than the reg pair,
4066 * and we get a copy in reg.
1c1824dc 4067 */
29f5e925
RH		 4068 tcg_regset_set_reg(t_allocated_regs, reg);
4069 tcg_regset_set_reg(t_allocated_regs, reg + 1);
4070 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
4071 /* If ts was already in reg, copy it somewhere else. */
4072 TCGReg nr;
4073 bool ok;
4074
4075 tcg_debug_assert(ts->kind != TEMP_FIXED);
4076 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
4077 t_allocated_regs, 0, ts->indirect_base);
4078 ok = tcg_out_mov(s, ts->type, nr, reg);
4079 tcg_debug_assert(ok);
4080
4081 set_temp_val_reg(s, ts, nr);
4082 } else {
4083 temp_load(s, ts, tcg_target_available_regs[ts->type],
4084 t_allocated_regs, 0);
4085 copyto_new_reg = true;
4086 }
4087 } else {
4088 /* Preferably allocate to reg, otherwise copy. */
4089 i_required_regs = (TCGRegSet)1 << reg;
4090 temp_load(s, ts, i_required_regs, i_allocated_regs,
4091 i_preferred_regs);
4092 copyto_new_reg = ts->reg != reg;
5ff9d6a4 4093 }
29f5e925 4094 break;
d62816f2 4095
29f5e925
RH		 4096 default:
4097 g_assert_not_reached();
1c1824dc 4098 }
d62816f2 4099
29f5e925 4100 if (copyto_new_reg) {
78113e83 4101 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 4102 /*
4103 * Cross register class move not supported. Sync the
4104 * temp back to its slot and load from there.
4105 */
4106 temp_sync(s, ts, i_allocated_regs, 0, 0);
4107 tcg_out_ld(s, ts->type, reg,
4108 ts->mem_base->reg, ts->mem_offset);
78113e83 4109 }
c896fe29 4110 }
c896fe29
FB		 4111 new_args[i] = reg;
4112 const_args[i] = 0;
82790a87 4113 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29 4114 }
a813e36f 4115
a52ad07e
AJ		 4116 /* mark dead temporaries and free the associated registers */
4117 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4118 if (IS_DEAD_ARG(i)) {
43439139 4119 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 4120 }
4121 }
4122
b4cb76e6
RH		 4123 if (def->flags & TCG_OPF_COND_BRANCH) {
4124 tcg_reg_alloc_cbranch(s, i_allocated_regs);
4125 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 4126 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 4127 } else {
e8996ee0 4128 if (def->flags & TCG_OPF_CALL_CLOBBER) {
a813e36f 4129 /* XXX: permit generic clobber register list ? */
c8074023
RH		 4130 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4131 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 4132 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 4133 }
c896fe29 4134 }
3d5c5f87
AJ		 4135 }
4136 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4137 /* sync globals if the op has side effects and might trigger
4138 an exception. */
82790a87 4139 sync_globals(s, i_allocated_regs);
c896fe29 4140 }
a813e36f 4141
e8996ee0 4142 /* satisfy the output constraints */
e8996ee0 4143 for(k = 0; k < nb_oargs; k++) {
66792f90 4144 i = def->args_ct[k].sort_index;
dd186292 4145 arg = op->args[i];
e8996ee0 4146 arg_ct = &def->args_ct[i];
43439139 4147 ts = arg_temp(arg);
d63e3b6e
RH		 4148
4149 /* ENV should not be modified. */
e01fa97d 4150 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4151
29f5e925
RH		 4152 switch (arg_ct->pair) {
4153 case 0: /* not paired */
4154 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
4155 reg = new_args[arg_ct->alias_index];
4156 } else if (arg_ct->newreg) {
4157 reg = tcg_reg_alloc(s, arg_ct->regs,
4158 i_allocated_regs | o_allocated_regs,
31fd884b 4159 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4160 } else {
4161 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
31fd884b 4162 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4163 }
4164 break;
4165
4166 case 1: /* first of pair */
4167 tcg_debug_assert(!arg_ct->newreg);
4168 if (arg_ct->oalias) {
4169 reg = new_args[arg_ct->alias_index];
4170 break;
4171 }
4172 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
31fd884b 4173 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4174 break;
4175
4176 case 2: /* second of pair */
4177 tcg_debug_assert(!arg_ct->newreg);
4178 if (arg_ct->oalias) {
4179 reg = new_args[arg_ct->alias_index];
4180 } else {
4181 reg = new_args[arg_ct->pair_index] + 1;
4182 }
4183 break;
4184
4185 case 3: /* first of pair, aliasing with a second input */
4186 tcg_debug_assert(!arg_ct->newreg);
4187 reg = new_args[arg_ct->pair_index] - 1;
4188 break;
4189
4190 default:
4191 g_assert_not_reached();
c896fe29 4192 }
82790a87 4193 tcg_regset_set_reg(o_allocated_regs, reg);
098859f1 4194 set_temp_val_reg(s, ts, reg);
d63e3b6e 4195 ts->mem_coherent = 0;
e8996ee0 4196 new_args[i] = reg;
c896fe29 4197 }
c896fe29
FB		 4198 }
4199
c896fe29 4200 /* emit instruction */
d2fd745f
RH		 4201 if (def->flags & TCG_OPF_VECTOR) {
4202 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
4203 new_args, const_args);
4204 } else {
4205 tcg_out_op(s, op->opc, new_args, const_args);
4206 }
4207
c896fe29
FB		 4208 /* move the outputs in the correct register if needed */
4209 for(i = 0; i < nb_oargs; i++) {
43439139 4210 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 4211
4212 /* ENV should not be modified. */
e01fa97d 4213 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4214
ec7a869d 4215 if (NEED_SYNC_ARG(i)) {
98b4e186 4216 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 4217 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 4218 temp_dead(s, ts);
ec7a869d 4219 }
c896fe29
FB		 4220 }
4221}
4222
efe86b21
RH		 4223static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
4224{
4225 const TCGLifeData arg_life = op->life;
4226 TCGTemp *ots, *itsl, *itsh;
4227 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4228
4229 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
4230 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
4231 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
4232
4233 ots = arg_temp(op->args[0]);
4234 itsl = arg_temp(op->args[1]);
4235 itsh = arg_temp(op->args[2]);
4236
4237 /* ENV should not be modified. */
4238 tcg_debug_assert(!temp_readonly(ots));
4239
4240 /* Allocate the output register now. */
4241 if (ots->val_type != TEMP_VAL_REG) {
4242 TCGRegSet allocated_regs = s->reserved_regs;
4243 TCGRegSet dup_out_regs =
4244 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
098859f1 4245 TCGReg oreg;
efe86b21
RH		 4246
4247 /* Make sure to not spill the input registers. */
4248 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
4249 tcg_regset_set_reg(allocated_regs, itsl->reg);
4250 }
4251 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
4252 tcg_regset_set_reg(allocated_regs, itsh->reg);
4253 }
4254
098859f1 4255 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 4256 output_pref(op, 0), ots->indirect_base);
098859f1 4257 set_temp_val_reg(s, ots, oreg);
efe86b21
RH		 4258 }
4259
4260 /* Promote dup2 of immediates to dupi_vec. */
4261 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
4262 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
4263 MemOp vece = MO_64;
4264
4265 if (val == dup_const(MO_8, val)) {
4266 vece = MO_8;
4267 } else if (val == dup_const(MO_16, val)) {
4268 vece = MO_16;
4269 } else if (val == dup_const(MO_32, val)) {
4270 vece = MO_32;
4271 }
4272
4273 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
4274 goto done;
4275 }
4276
4277 /* If the two inputs form one 64-bit value, try dupm_vec. */
aef85402
RH		 4278 if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
4279 itsh->temp_subindex == !HOST_BIG_ENDIAN &&
4280 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
4281 TCGTemp *its = itsl - HOST_BIG_ENDIAN;
4282
4283 temp_sync(s, its + 0, s->reserved_regs, 0, 0);
4284 temp_sync(s, its + 1, s->reserved_regs, 0, 0);
4285
efe86b21
RH		 4286 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
4287 its->mem_base->reg, its->mem_offset)) {
4288 goto done;
4289 }
4290 }
4291
4292 /* Fall back to generic expansion. */
4293 return false;
4294
4295 done:
36f5539c 4296 ots->mem_coherent = 0;
efe86b21
RH		 4297 if (IS_DEAD_ARG(1)) {
4298 temp_dead(s, itsl);
4299 }
4300 if (IS_DEAD_ARG(2)) {
4301 temp_dead(s, itsh);
4302 }
4303 if (NEED_SYNC_ARG(0)) {
4304 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
4305 } else if (IS_DEAD_ARG(0)) {
4306 temp_dead(s, ots);
4307 }
4308 return true;
4309}
4310
39004a71
RH		 4311static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
4312 TCGRegSet allocated_regs)
c896fe29 4313{
39004a71
RH		 4314 if (ts->val_type == TEMP_VAL_REG) {
4315 if (ts->reg != reg) {
4316 tcg_reg_free(s, reg, allocated_regs);
4317 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4318 /*
4319 * Cross register class move not supported. Sync the
4320 * temp back to its slot and load from there.
4321 */
4322 temp_sync(s, ts, allocated_regs, 0, 0);
4323 tcg_out_ld(s, ts->type, reg,
4324 ts->mem_base->reg, ts->mem_offset);
4325 }
4326 }
4327 } else {
4328 TCGRegSet arg_set = 0;
c896fe29 4329
39004a71
RH		 4330 tcg_reg_free(s, reg, allocated_regs);
4331 tcg_regset_set_reg(arg_set, reg);
4332 temp_load(s, ts, arg_set, allocated_regs, 0);
b03cce8e 4333 }
39004a71 4334}
39cf05d3 4335
39004a71
RH		 4336static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts,
4337 TCGRegSet allocated_regs)
4338{
4339 /*
4340 * When the destination is on the stack, load up the temp and store.
4341 * If there are many call-saved registers, the temp might live to
4342 * see another use; otherwise it'll be discarded.
4343 */
4344 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
4345 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
4346 TCG_TARGET_CALL_STACK_OFFSET +
4347 stk_slot * sizeof(tcg_target_long));
4348}
a813e36f 4349
39004a71
RH		 4350static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
4351 TCGTemp *ts, TCGRegSet *allocated_regs)
4352{
4353 if (REG_P(l)) {
4354 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
4355 load_arg_reg(s, reg, ts, *allocated_regs);
4356 tcg_regset_set_reg(*allocated_regs, reg);
4357 } else {
4358 load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs),
4359 ts, *allocated_regs);
4360 }
4361}
40ae5c62 4362
39004a71
RH		 4363static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
4364{
4365 const int nb_oargs = TCGOP_CALLO(op);
4366 const int nb_iargs = TCGOP_CALLI(op);
4367 const TCGLifeData arg_life = op->life;
4368 const TCGHelperInfo *info = tcg_call_info(op);
4369 TCGRegSet allocated_regs = s->reserved_regs;
4370 int i;
40ae5c62 4371
39004a71
RH		 4372 /*
4373 * Move inputs into place in reverse order,
4374 * so that we place stacked arguments first.
4375 */
4376 for (i = nb_iargs - 1; i >= 0; --i) {
4377 const TCGCallArgumentLoc *loc = &info->in[i];
4378 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
40ae5c62 4379
39004a71
RH		 4380 switch (loc->kind) {
4381 case TCG_CALL_ARG_NORMAL:
4382 case TCG_CALL_ARG_EXTEND_U:
4383 case TCG_CALL_ARG_EXTEND_S:
4384 load_arg_normal(s, loc, ts, &allocated_regs);
4385 break;
4386 default:
4387 g_assert_not_reached();
c896fe29 4388 }
c896fe29 4389 }
a813e36f 4390
39004a71 4391 /* Mark dead temporaries and free the associated registers. */
dd186292 4392 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 4393 if (IS_DEAD_ARG(i)) {
43439139 4394 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 4395 }
4396 }
a813e36f 4397
39004a71 4398 /* Clobber call registers. */
c8074023
RH		 4399 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4400 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 4401 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 4402 }
4403 }
78505279 4404
39004a71
RH		 4405 /*
4406 * Save globals if they might be written by the helper,
4407 * sync them if they might be read.
4408 */
4409 if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
78505279 4410 /* Nothing to do */
39004a71 4411 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
78505279
AJ		 4412 sync_globals(s, allocated_regs);
4413 } else {
b9c18f56
AJ		 4414 save_globals(s, allocated_regs);
4415 }
c896fe29 4416
cee44b03 4417 tcg_out_call(s, tcg_call_func(op), info);
c896fe29 4418
39004a71
RH		 4419 /* Assign output registers and emit moves if needed. */
4420 switch (info->out_kind) {
4421 case TCG_CALL_RET_NORMAL:
4422 for (i = 0; i < nb_oargs; i++) {
4423 TCGTemp *ts = arg_temp(op->args[i]);
4424 TCGReg reg = tcg_target_call_oarg_regs[i];
d63e3b6e 4425
39004a71
RH		 4426 /* ENV should not be modified. */
4427 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4428
39004a71
RH		 4429 set_temp_val_reg(s, ts, reg);
4430 ts->mem_coherent = 0;
4431 }
4432 break;
4433 default:
4434 g_assert_not_reached();
4435 }
4436
4437 /* Flush or discard output registers as needed. */
4438 for (i = 0; i < nb_oargs; i++) {
4439 TCGTemp *ts = arg_temp(op->args[i]);
d63e3b6e 4440 if (NEED_SYNC_ARG(i)) {
39004a71 4441 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
d63e3b6e
RH		 4442 } else if (IS_DEAD_ARG(i)) {
4443 temp_dead(s, ts);
c896fe29
FB		 4444 }
4445 }
c896fe29
FB		 4446}
4447
4448#ifdef CONFIG_PROFILER
4449
c3fac113
EC		 4450/* avoid copy/paste errors */
4451#define PROF_ADD(to, from, field) \
4452 do { \
d73415a3 4453 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 4454 } while (0)
4455
4456#define PROF_MAX(to, from, field) \
4457 do { \
d73415a3 4458 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 4459 if (val__ > (to)->field) { \
4460 (to)->field = val__; \
4461 } \
4462 } while (0)
4463
4464/* Pass in a zero'ed @prof */
4465static inline
4466void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4467{
0e2d61cf 4468 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 4469 unsigned int i;
4470
3468b59e 4471 for (i = 0; i < n_ctxs; i++) {
d73415a3 4472 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 4473 const TCGProfile *orig = &s->prof;
c3fac113
EC		 4474
4475 if (counters) {
72fd2efb 4476 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 4477 PROF_ADD(prof, orig, tb_count1);
4478 PROF_ADD(prof, orig, tb_count);
4479 PROF_ADD(prof, orig, op_count);
4480 PROF_MAX(prof, orig, op_count_max);
4481 PROF_ADD(prof, orig, temp_count);
4482 PROF_MAX(prof, orig, temp_count_max);
4483 PROF_ADD(prof, orig, del_op_count);
4484 PROF_ADD(prof, orig, code_in_len);
4485 PROF_ADD(prof, orig, code_out_len);
4486 PROF_ADD(prof, orig, search_out_len);
4487 PROF_ADD(prof, orig, interm_time);
4488 PROF_ADD(prof, orig, code_time);
4489 PROF_ADD(prof, orig, la_time);
4490 PROF_ADD(prof, orig, opt_time);
4491 PROF_ADD(prof, orig, restore_count);
4492 PROF_ADD(prof, orig, restore_time);
4493 }
4494 if (table) {
4495 int i;
4496
4497 for (i = 0; i < NB_OPS; i++) {
4498 PROF_ADD(prof, orig, table_op_count[i]);
4499 }
4500 }
4501 }
4502}
4503
4504#undef PROF_ADD
4505#undef PROF_MAX
4506
4507static void tcg_profile_snapshot_counters(TCGProfile *prof)
4508{
4509 tcg_profile_snapshot(prof, true, false);
4510}
4511
4512static void tcg_profile_snapshot_table(TCGProfile *prof)
4513{
4514 tcg_profile_snapshot(prof, false, true);
4515}
c896fe29 4516
b6a7f3e0 4517void tcg_dump_op_count(GString *buf)
c896fe29 4518{
c3fac113 4519 TCGProfile prof = {};
c896fe29 4520 int i;
d70724ce 4521
c3fac113 4522 tcg_profile_snapshot_table(&prof);
15fc7daa 4523 for (i = 0; i < NB_OPS; i++) {
b6a7f3e0
DB		 4524 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
4525 prof.table_op_count[i]);
c896fe29 4526 }
c896fe29 4527}
72fd2efb
EC		 4528
4529int64_t tcg_cpu_exec_time(void)
4530{
0e2d61cf 4531 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4532 unsigned int i;
4533 int64_t ret = 0;
4534
4535 for (i = 0; i < n_ctxs; i++) {
d73415a3 4536 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4537 const TCGProfile *prof = &s->prof;
4538
d73415a3 4539 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4540 }
4541 return ret;
4542}
246ae24d 4543#else
b6a7f3e0 4544void tcg_dump_op_count(GString *buf)
246ae24d 4545{
b6a7f3e0 4546 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
246ae24d 4547}
72fd2efb
EC		 4548
4549int64_t tcg_cpu_exec_time(void)
4550{
4551 error_report("%s: TCG profiler not compiled", __func__);
4552 exit(EXIT_FAILURE);
4553}
c896fe29
FB		 4554#endif
4555
4556
fbf59aad 4557int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
c896fe29 4558{
c3fac113
EC		 4559#ifdef CONFIG_PROFILER
4560 TCGProfile *prof = &s->prof;
4561#endif
15fa08f8
RH		 4562 int i, num_insns;
4563 TCGOp *op;
c896fe29 4564
04fe6400
RH		 4565#ifdef CONFIG_PROFILER
4566 {
c1f543b7 4567 int n = 0;
04fe6400 4568
15fa08f8
RH		 4569 QTAILQ_FOREACH(op, &s->ops, link) {
4570 n++;
4571 }
d73415a3 4572 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4573 if (n > prof->op_count_max) {
d73415a3 4574 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4575 }
4576
4577 n = s->nb_temps;
d73415a3 4578 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4579 if (n > prof->temp_count_max) {
d73415a3 4580 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4581 }
4582 }
4583#endif
4584
c896fe29 4585#ifdef DEBUG_DISAS
d977e1c2 4586 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
fbf59aad 4587 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4588 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4589 if (logfile) {
4590 fprintf(logfile, "OP:\n");
b7a83ff8 4591 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4592 fprintf(logfile, "\n");
4593 qemu_log_unlock(logfile);
4594 }
c896fe29
FB		 4595 }
4596#endif
4597
bef16ab4
RH		 4598#ifdef CONFIG_DEBUG_TCG
4599 /* Ensure all labels referenced have been emitted. */
4600 {
4601 TCGLabel *l;
4602 bool error = false;
4603
4604 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4605 if (unlikely(!l->present) && l->refs) {
4606 qemu_log_mask(CPU_LOG_TB_OP,
4607 "$L%d referenced but not present.\n", l->id);
4608 error = true;
4609 }
4610 }
4611 assert(!error);
4612 }
4613#endif
4614
c5cc28ff 4615#ifdef CONFIG_PROFILER
d73415a3 4616 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4617#endif
4618
8f2e8c07 4619#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4620 tcg_optimize(s);
8f2e8c07
KB		 4621#endif
4622
a23a9ec6 4623#ifdef CONFIG_PROFILER
d73415a3
SH		 4624 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4625 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4626#endif
c5cc28ff 4627
b4fc67c7 4628 reachable_code_pass(s);
b83eabea 4629 liveness_pass_1(s);
5a18407f 4630
b83eabea 4631 if (s->nb_indirects > 0) {
5a18407f 4632#ifdef DEBUG_DISAS
b83eabea 4633 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
fbf59aad 4634 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4635 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4636 if (logfile) {
4637 fprintf(logfile, "OP before indirect lowering:\n");
b7a83ff8 4638 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4639 fprintf(logfile, "\n");
4640 qemu_log_unlock(logfile);
4641 }
b83eabea 4642 }
5a18407f 4643#endif
b83eabea
RH		 4644 /* Replace indirect temps with direct temps. */
4645 if (liveness_pass_2(s)) {
4646 /* If changes were made, re-run liveness. */
4647 liveness_pass_1(s);
5a18407f
RH		 4648 }
4649 }
c5cc28ff 4650
a23a9ec6 4651#ifdef CONFIG_PROFILER
d73415a3 4652 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4653#endif
c896fe29
FB		 4654
4655#ifdef DEBUG_DISAS
d977e1c2 4656 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
fbf59aad 4657 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4658 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4659 if (logfile) {
4660 fprintf(logfile, "OP after optimization and liveness analysis:\n");
b7a83ff8 4661 tcg_dump_ops(s, logfile, true);
78b54858
RH		 4662 fprintf(logfile, "\n");
4663 qemu_log_unlock(logfile);
4664 }
c896fe29
FB		 4665 }
4666#endif
4667
35abb009
RH		 4668 /* Initialize goto_tb jump offsets. */
4669 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
4670 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
4671 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
4672 if (TCG_TARGET_HAS_direct_jump) {
4673 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
4674 tcg_ctx->tb_jmp_target_addr = NULL;
4675 } else {
4676 tcg_ctx->tb_jmp_insn_offset = NULL;
4677 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
4678 }
4679
c896fe29
FB		 4680 tcg_reg_alloc_start(s);
4681
db0c51a3
RH		 4682 /*
4683 * Reset the buffer pointers when restarting after overflow.
4684 * TODO: Move this into translate-all.c with the rest of the
4685 * buffer management. Having only this done here is confusing.
4686 */
4687 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4688 s->code_ptr = s->code_buf;
c896fe29 4689
659ef5cb 4690#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4691 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4692#endif
57a26946
RH		 4693#ifdef TCG_TARGET_NEED_POOL_LABELS
4694 s->pool_labels = NULL;
4695#endif
9ecefc84 4696
fca8a500 4697 num_insns = -1;
15fa08f8 4698 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4699 TCGOpcode opc = op->opc;
b3db8758 4700
c896fe29 4701#ifdef CONFIG_PROFILER
d73415a3 4702 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 4703#endif
c45cb8bb
RH		 4704
4705 switch (opc) {
c896fe29 4706 case INDEX_op_mov_i32:
c896fe29 4707 case INDEX_op_mov_i64:
d2fd745f 4708 case INDEX_op_mov_vec:
dd186292 4709 tcg_reg_alloc_mov(s, op);
c896fe29 4710 break;
bab1671f
RH		 4711 case INDEX_op_dup_vec:
4712 tcg_reg_alloc_dup(s, op);
4713 break;
765b842a 4714 case INDEX_op_insn_start:
fca8a500 4715 if (num_insns >= 0) {
9f754620
RH		 4716 size_t off = tcg_current_code_size(s);
4717 s->gen_insn_end_off[num_insns] = off;
4718 /* Assert that we do not overflow our stored offset. */
4719 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 4720 }
4721 num_insns++;
bad729e2
RH		 4722 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4723 target_ulong a;
4724#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 4725 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 4726#else
efee3746 4727 a = op->args[i];
bad729e2 4728#endif
fca8a500 4729 s->gen_insn_data[num_insns][i] = a;
bad729e2 4730 }
c896fe29 4731 break;
5ff9d6a4 4732 case INDEX_op_discard:
43439139 4733 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 4734 break;
c896fe29 4735 case INDEX_op_set_label:
e8996ee0 4736 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 4737 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 4738 break;
4739 case INDEX_op_call:
dd186292 4740 tcg_reg_alloc_call(s, op);
c45cb8bb 4741 break;
b55a8d9d
RH		 4742 case INDEX_op_exit_tb:
4743 tcg_out_exit_tb(s, op->args[0]);
4744 break;
cf7d6b8e
RH		 4745 case INDEX_op_goto_tb:
4746 tcg_out_goto_tb(s, op->args[0]);
4747 break;
efe86b21
RH		 4748 case INDEX_op_dup2_vec:
4749 if (tcg_reg_alloc_dup2(s, op)) {
4750 break;
4751 }
4752 /* fall through */
c896fe29 4753 default:
25c4d9cc 4754 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 4755 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 4756 /* Note: in order to speed up the code, it would be much
4757 faster to have specialized register allocator functions for
4758 some common argument patterns */
dd186292 4759 tcg_reg_alloc_op(s, op);
c896fe29
FB		 4760 break;
4761 }
b125f9dc
RH		 4762 /* Test for (pending) buffer overflow. The assumption is that any
4763 one operation beginning below the high water mark cannot overrun
4764 the buffer completely. Thus we can test for overflow after
4765 generating code without having to check during generation. */
644da9b3 4766 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 4767 return -1;
4768 }
6e6c4efe
RH		 4769 /* Test for TB overflow, as seen by gen_insn_end_off. */
4770 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4771 return -2;
4772 }
c896fe29 4773 }
fca8a500
RH		 4774 tcg_debug_assert(num_insns >= 0);
4775 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 4776
b76f0d8c 4777 /* Generate TB finalization at the end of block */
659ef5cb 4778#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 4779 i = tcg_out_ldst_finalize(s);
4780 if (i < 0) {
4781 return i;
23dceda6 4782 }
659ef5cb 4783#endif
57a26946 4784#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 4785 i = tcg_out_pool_finalize(s);
4786 if (i < 0) {
4787 return i;
57a26946
RH		 4788 }
4789#endif
7ecd02a0
RH		 4790 if (!tcg_resolve_relocs(s)) {
4791 return -2;
4792 }
c896fe29 4793
df5d2b16 4794#ifndef CONFIG_TCG_INTERPRETER
c896fe29 4795 /* flush instruction cache */
db0c51a3
RH		 4796 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4797 (uintptr_t)s->code_buf,
1da8de39 4798 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 4799#endif
2aeabc08 4800
1813e175 4801 return tcg_current_code_size(s);
c896fe29
FB		 4802}
4803
a23a9ec6 4804#ifdef CONFIG_PROFILER
3a841ab5 4805void tcg_dump_info(GString *buf)
a23a9ec6 4806{
c3fac113
EC		 4807 TCGProfile prof = {};
4808 const TCGProfile *s;
4809 int64_t tb_count;
4810 int64_t tb_div_count;
4811 int64_t tot;
4812
4813 tcg_profile_snapshot_counters(&prof);
4814 s = &prof;
4815 tb_count = s->tb_count;
4816 tb_div_count = tb_count ? tb_count : 1;
4817 tot = s->interm_time + s->code_time;
a23a9ec6 4818
3a841ab5
DB		 4819 g_string_append_printf(buf, "JIT cycles %" PRId64
4820 " (%0.3f s at 2.4 GHz)\n",
4821 tot, tot / 2.4e9);
4822 g_string_append_printf(buf, "translated TBs %" PRId64
4823 " (aborted=%" PRId64 " %0.1f%%)\n",
4824 tb_count, s->tb_count1 - tb_count,
4825 (double)(s->tb_count1 - s->tb_count)
4826 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4827 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
4828 (double)s->op_count / tb_div_count, s->op_count_max);
4829 g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
4830 (double)s->del_op_count / tb_div_count);
4831 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
4832 (double)s->temp_count / tb_div_count,
4833 s->temp_count_max);
4834 g_string_append_printf(buf, "avg host code/TB %0.1f\n",
4835 (double)s->code_out_len / tb_div_count);
4836 g_string_append_printf(buf, "avg search data/TB %0.1f\n",
4837 (double)s->search_out_len / tb_div_count);
a813e36f 4838
3a841ab5
DB		 4839 g_string_append_printf(buf, "cycles/op %0.1f\n",
4840 s->op_count ? (double)tot / s->op_count : 0);
4841 g_string_append_printf(buf, "cycles/in byte %0.1f\n",
4842 s->code_in_len ? (double)tot / s->code_in_len : 0);
4843 g_string_append_printf(buf, "cycles/out byte %0.1f\n",
4844 s->code_out_len ? (double)tot / s->code_out_len : 0);
4845 g_string_append_printf(buf, "cycles/search byte %0.1f\n",
4846 s->search_out_len ?
4847 (double)tot / s->search_out_len : 0);
fca8a500 4848 if (tot == 0) {
a23a9ec6 4849 tot = 1;
fca8a500 4850 }
3a841ab5
DB		 4851 g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
4852 (double)s->interm_time / tot * 100.0);
4853 g_string_append_printf(buf, " gen_code time %0.1f%%\n",
4854 (double)s->code_time / tot * 100.0);
4855 g_string_append_printf(buf, "optim./code time %0.1f%%\n",
4856 (double)s->opt_time / (s->code_time ?
4857 s->code_time : 1)
4858 * 100.0);
4859 g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
4860 (double)s->la_time / (s->code_time ?
4861 s->code_time : 1) * 100.0);
4862 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
4863 s->restore_count);
4864 g_string_append_printf(buf, " avg cycles %0.1f\n",
4865 s->restore_count ?
4866 (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 4867}
4868#else
3a841ab5 4869void tcg_dump_info(GString *buf)
a23a9ec6 4870{
3a841ab5 4871 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
a23a9ec6
FB		 4872}
4873#endif
813da627
RH		 4874
4875#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 4876/* In order to use this feature, the backend needs to do three things:
4877
4878 (1) Define ELF_HOST_MACHINE to indicate both what value to
4879 put into the ELF image and to indicate support for the feature.
4880
4881 (2) Define tcg_register_jit. This should create a buffer containing
4882 the contents of a .debug_frame section that describes the post-
4883 prologue unwind info for the tcg machine.
4884
4885 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4886*/
813da627
RH		 4887
4888/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4889typedef enum {
4890 JIT_NOACTION = 0,
4891 JIT_REGISTER_FN,
4892 JIT_UNREGISTER_FN
4893} jit_actions_t;
4894
4895struct jit_code_entry {
4896 struct jit_code_entry *next_entry;
4897 struct jit_code_entry *prev_entry;
4898 const void *symfile_addr;
4899 uint64_t symfile_size;
4900};
4901
4902struct jit_descriptor {
4903 uint32_t version;
4904 uint32_t action_flag;
4905 struct jit_code_entry *relevant_entry;
4906 struct jit_code_entry *first_entry;
4907};
4908
4909void __jit_debug_register_code(void) __attribute__((noinline));
4910void __jit_debug_register_code(void)
4911{
4912 asm("");
4913}
4914
4915/* Must statically initialize the version, because GDB may check
4916 the version before we can set it. */
4917struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4918
4919/* End GDB interface. */
4920
4921static int find_string(const char *strtab, const char *str)
4922{
4923 const char *p = strtab + 1;
4924
4925 while (1) {
4926 if (strcmp(p, str) == 0) {
4927 return p - strtab;
4928 }
4929 p += strlen(p) + 1;
4930 }
4931}
4932
755bf9e5 4933static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 4934 const void *debug_frame,
4935 size_t debug_frame_size)
813da627 4936{
5872bbf2
RH		 4937 struct __attribute__((packed)) DebugInfo {
4938 uint32_t len;
4939 uint16_t version;
4940 uint32_t abbrev;
4941 uint8_t ptr_size;
4942 uint8_t cu_die;
4943 uint16_t cu_lang;
4944 uintptr_t cu_low_pc;
4945 uintptr_t cu_high_pc;
4946 uint8_t fn_die;
4947 char fn_name[16];
4948 uintptr_t fn_low_pc;
4949 uintptr_t fn_high_pc;
4950 uint8_t cu_eoc;
4951 };
813da627
RH		 4952
4953 struct ElfImage {
4954 ElfW(Ehdr) ehdr;
4955 ElfW(Phdr) phdr;
5872bbf2
RH		 4956 ElfW(Shdr) shdr[7];
4957 ElfW(Sym) sym[2];
4958 struct DebugInfo di;
4959 uint8_t da[24];
4960 char str[80];
4961 };
4962
4963 struct ElfImage *img;
4964
4965 static const struct ElfImage img_template = {
4966 .ehdr = {
4967 .e_ident[EI_MAG0] = ELFMAG0,
4968 .e_ident[EI_MAG1] = ELFMAG1,
4969 .e_ident[EI_MAG2] = ELFMAG2,
4970 .e_ident[EI_MAG3] = ELFMAG3,
4971 .e_ident[EI_CLASS] = ELF_CLASS,
4972 .e_ident[EI_DATA] = ELF_DATA,
4973 .e_ident[EI_VERSION] = EV_CURRENT,
4974 .e_type = ET_EXEC,
4975 .e_machine = ELF_HOST_MACHINE,
4976 .e_version = EV_CURRENT,
4977 .e_phoff = offsetof(struct ElfImage, phdr),
4978 .e_shoff = offsetof(struct ElfImage, shdr),
4979 .e_ehsize = sizeof(ElfW(Shdr)),
4980 .e_phentsize = sizeof(ElfW(Phdr)),
4981 .e_phnum = 1,
4982 .e_shentsize = sizeof(ElfW(Shdr)),
4983 .e_shnum = ARRAY_SIZE(img->shdr),
4984 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 4985#ifdef ELF_HOST_FLAGS
4986 .e_flags = ELF_HOST_FLAGS,
4987#endif
4988#ifdef ELF_OSABI
4989 .e_ident[EI_OSABI] = ELF_OSABI,
4990#endif
5872bbf2
RH		 4991 },
4992 .phdr = {
4993 .p_type = PT_LOAD,
4994 .p_flags = PF_X,
4995 },
4996 .shdr = {
4997 [0] = { .sh_type = SHT_NULL },
4998 /* Trick: The contents of code_gen_buffer are not present in
4999 this fake ELF file; that got allocated elsewhere. Therefore
5000 we mark .text as SHT_NOBITS (similar to .bss) so that readers
5001 will not look for contents. We can record any address. */
5002 [1] = { /* .text */
5003 .sh_type = SHT_NOBITS,
5004 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
5005 },
5006 [2] = { /* .debug_info */
5007 .sh_type = SHT_PROGBITS,
5008 .sh_offset = offsetof(struct ElfImage, di),
5009 .sh_size = sizeof(struct DebugInfo),
5010 },
5011 [3] = { /* .debug_abbrev */
5012 .sh_type = SHT_PROGBITS,
5013 .sh_offset = offsetof(struct ElfImage, da),
5014 .sh_size = sizeof(img->da),
5015 },
5016 [4] = { /* .debug_frame */
5017 .sh_type = SHT_PROGBITS,
5018 .sh_offset = sizeof(struct ElfImage),
5019 },
5020 [5] = { /* .symtab */
5021 .sh_type = SHT_SYMTAB,
5022 .sh_offset = offsetof(struct ElfImage, sym),
5023 .sh_size = sizeof(img->sym),
5024 .sh_info = 1,
5025 .sh_link = ARRAY_SIZE(img->shdr) - 1,
5026 .sh_entsize = sizeof(ElfW(Sym)),
5027 },
5028 [6] = { /* .strtab */
5029 .sh_type = SHT_STRTAB,
5030 .sh_offset = offsetof(struct ElfImage, str),
5031 .sh_size = sizeof(img->str),
5032 }
5033 },
5034 .sym = {
5035 [1] = { /* code_gen_buffer */
5036 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
5037 .st_shndx = 1,
5038 }
5039 },
5040 .di = {
5041 .len = sizeof(struct DebugInfo) - 4,
5042 .version = 2,
5043 .ptr_size = sizeof(void *),
5044 .cu_die = 1,
5045 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
5046 .fn_die = 2,
5047 .fn_name = "code_gen_buffer"
5048 },
5049 .da = {
5050 1, /* abbrev number (the cu) */
5051 0x11, 1, /* DW_TAG_compile_unit, has children */
5052 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
5053 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5054 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5055 0, 0, /* end of abbrev */
5056 2, /* abbrev number (the fn) */
5057 0x2e, 0, /* DW_TAG_subprogram, no children */
5058 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
5059 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5060 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5061 0, 0, /* end of abbrev */
5062 0 /* no more abbrev */
5063 },
5064 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
5065 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 5066 };
5067
5068 /* We only need a single jit entry; statically allocate it. */
5069 static struct jit_code_entry one_entry;
5070
5872bbf2 5071 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 5072 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 5073 DebugFrameHeader *dfh;
813da627 5074
5872bbf2
RH		 5075 img = g_malloc(img_size);
5076 *img = img_template;
813da627 5077
5872bbf2
RH		 5078 img->phdr.p_vaddr = buf;
5079 img->phdr.p_paddr = buf;
5080 img->phdr.p_memsz = buf_size;
813da627 5081
813da627 5082 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 5083 img->shdr[1].sh_addr = buf;
813da627
RH		 5084 img->shdr[1].sh_size = buf_size;
5085
5872bbf2
RH		 5086 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
5087 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
5088
5089 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
5090 img->shdr[4].sh_size = debug_frame_size;
5091
5092 img->shdr[5].sh_name = find_string(img->str, ".symtab");
5093 img->shdr[6].sh_name = find_string(img->str, ".strtab");
5094
5095 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
5096 img->sym[1].st_value = buf;
5097 img->sym[1].st_size = buf_size;
813da627 5098
5872bbf2 5099 img->di.cu_low_pc = buf;
45aba097 5100 img->di.cu_high_pc = buf + buf_size;
5872bbf2 5101 img->di.fn_low_pc = buf;
45aba097 5102 img->di.fn_high_pc = buf + buf_size;
813da627 5103
2c90784a
RH		 5104 dfh = (DebugFrameHeader *)(img + 1);
5105 memcpy(dfh, debug_frame, debug_frame_size);
5106 dfh->fde.func_start = buf;
5107 dfh->fde.func_len = buf_size;
5108
813da627
RH		 5109#ifdef DEBUG_JIT
5110 /* Enable this block to be able to debug the ELF image file creation.
5111 One can use readelf, objdump, or other inspection utilities. */
5112 {
eb6b2edf
BM		 5113 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
5114 FILE *f = fopen(jit, "w+b");
813da627 5115 if (f) {
5872bbf2 5116 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 5117 /* Avoid stupid unused return value warning for fwrite. */
5118 }
5119 fclose(f);
5120 }
5121 }
5122#endif
5123
5124 one_entry.symfile_addr = img;
5125 one_entry.symfile_size = img_size;
5126
5127 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
5128 __jit_debug_descriptor.relevant_entry = &one_entry;
5129 __jit_debug_descriptor.first_entry = &one_entry;
5130 __jit_debug_register_code();
5131}
5132#else
5872bbf2
RH		 5133/* No support for the feature. Provide the entry point expected by exec.c,
5134 and implement the internal function we declared earlier. */
813da627 5135
755bf9e5 5136static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 5137 const void *debug_frame,
5138 size_t debug_frame_size)
813da627
RH		 5139{
5140}
5141
755bf9e5 5142void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 5143{
5144}
5145#endif /* ELF_HOST_MACHINE */
db432672
RH		 5146
5147#if !TCG_TARGET_MAYBE_vec
5148void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
5149{
5150 g_assert_not_reached();
5151}
5152#endif
QEMU mirror