]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg/tci: Move call-return regs to end of tcg_target_reg_alloc_order
[mirror_qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
1de7afc9 37#include "qemu/timer.h"
084cfca1 38#include "qemu/cacheflush.h"
c896fe29 39
c5d3c498 40/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 41 CPU definitions. Currently they are used for qemu_ld/st
42 instructions */
43#define NO_CPU_IO_DEFS
c896fe29 44
63c91552 45#include "exec/exec-all.h"
dcb32f1d 46#include "tcg/tcg-op.h"
813da627 47
edee2579 48#if UINTPTR_MAX == UINT32_MAX
813da627 49# define ELF_CLASS ELFCLASS32
edee2579
RH		 50#else
51# define ELF_CLASS ELFCLASS64
813da627
RH		 52#endif
53#ifdef HOST_WORDS_BIGENDIAN
54# define ELF_DATA ELFDATA2MSB
55#else
56# define ELF_DATA ELFDATA2LSB
57#endif
58
c896fe29 59#include "elf.h"
508127e2 60#include "exec/log.h"
5ff7258c 61#include "tcg-internal.h"
c896fe29 62
22f15579
RH		 63#ifdef CONFIG_TCG_INTERPRETER
64#include <ffi.h>
65#endif
66
139c1837 67/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 68 used here. */
e4d58b41
RH		 69static void tcg_target_init(TCGContext *s);
70static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 71static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 72 intptr_t value, intptr_t addend);
c896fe29 73
497a22eb
RH		 74/* The CIE and FDE header definitions will be common to all hosts. */
75typedef struct {
76 uint32_t len __attribute__((aligned((sizeof(void *)))));
77 uint32_t id;
78 uint8_t version;
79 char augmentation[1];
80 uint8_t code_align;
81 uint8_t data_align;
82 uint8_t return_column;
83} DebugFrameCIE;
84
85typedef struct QEMU_PACKED {
86 uint32_t len __attribute__((aligned((sizeof(void *)))));
87 uint32_t cie_offset;
edee2579
RH		 88 uintptr_t func_start;
89 uintptr_t func_len;
497a22eb
RH		 90} DebugFrameFDEHeader;
91
2c90784a
RH		 92typedef struct QEMU_PACKED {
93 DebugFrameCIE cie;
94 DebugFrameFDEHeader fde;
95} DebugFrameHeader;
96
755bf9e5 97static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 98 const void *debug_frame,
99 size_t debug_frame_size)
813da627
RH		 100 __attribute__((unused));
101
139c1837 102/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 103static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 104 intptr_t arg2);
78113e83 105static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 106static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 107 TCGReg ret, tcg_target_long arg);
5e8892db
MR		 108static void tcg_out_op(TCGContext *s, TCGOpcode opc,
109 const TCGArg args[TCG_MAX_OP_ARGS],
110 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 111#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 112static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
113 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 114static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 116static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
117 TCGReg dst, int64_t arg);
5e8892db
MR		 118static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
119 unsigned vecl, unsigned vece,
120 const TCGArg args[TCG_MAX_OP_ARGS],
121 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 122#else
e7632cfa
RH		 123static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
124 TCGReg dst, TCGReg src)
125{
126 g_assert_not_reached();
127}
d6ecb4a9
RH		 128static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
129 TCGReg dst, TCGReg base, intptr_t offset)
130{
131 g_assert_not_reached();
132}
4e186175
RH		 133static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
134 TCGReg dst, int64_t arg)
e7632cfa
RH		 135{
136 g_assert_not_reached();
137}
5e8892db
MR		 138static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
139 unsigned vecl, unsigned vece,
140 const TCGArg args[TCG_MAX_OP_ARGS],
141 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 142{
143 g_assert_not_reached();
144}
145#endif
2a534aff 146static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 147 intptr_t arg2);
59d7c14e
RH		 148static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
149 TCGReg base, intptr_t ofs);
2be7d76b 150static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
a4fbbd77 151static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 152#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 153static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 154#endif
c896fe29 155
42eb6dfc
RH		 156TCGContext tcg_init_ctx;
157__thread TCGContext *tcg_ctx;
158
5ff7258c 159TCGContext **tcg_ctxs;
0e2d61cf
RH		 160unsigned int tcg_cur_ctxs;
161unsigned int tcg_max_ctxs;
1c2adb95 162TCGv_env cpu_env = 0;
c8bc1168 163const void *tcg_code_gen_epilogue;
db0c51a3 164uintptr_t tcg_splitwx_diff;
df2cce29 165
b91ccb31
RH		 166#ifndef CONFIG_TCG_INTERPRETER
167tcg_prologue_fn *tcg_qemu_tb_exec;
168#endif
169
d2fd745f 170static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 171static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 172
1813e175 173#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 174static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 175{
176 *s->code_ptr++ = v;
177}
178
4196dca6
PM		 179static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
180 uint8_t v)
5c53bb81 181{
1813e175 182 *p = v;
5c53bb81 183}
1813e175 184#endif
5c53bb81 185
1813e175 186#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 187static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 188{
1813e175
RH		 189 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
190 *s->code_ptr++ = v;
191 } else {
192 tcg_insn_unit *p = s->code_ptr;
193 memcpy(p, &v, sizeof(v));
194 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
195 }
c896fe29
FB		 196}
197
4196dca6
PM		 198static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
199 uint16_t v)
5c53bb81 200{
1813e175
RH		 201 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
202 *p = v;
203 } else {
204 memcpy(p, &v, sizeof(v));
205 }
5c53bb81 206}
1813e175 207#endif
5c53bb81 208
1813e175 209#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 210static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 211{
1813e175
RH		 212 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
213 *s->code_ptr++ = v;
214 } else {
215 tcg_insn_unit *p = s->code_ptr;
216 memcpy(p, &v, sizeof(v));
217 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
218 }
c896fe29
FB		 219}
220
4196dca6
PM		 221static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
222 uint32_t v)
5c53bb81 223{
1813e175
RH		 224 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
225 *p = v;
226 } else {
227 memcpy(p, &v, sizeof(v));
228 }
5c53bb81 229}
1813e175 230#endif
5c53bb81 231
1813e175 232#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 233static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 234{
1813e175
RH		 235 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
236 *s->code_ptr++ = v;
237 } else {
238 tcg_insn_unit *p = s->code_ptr;
239 memcpy(p, &v, sizeof(v));
240 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
241 }
ac26eb69
RH		 242}
243
4196dca6
PM		 244static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
245 uint64_t v)
5c53bb81 246{
1813e175
RH		 247 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
248 *p = v;
249 } else {
250 memcpy(p, &v, sizeof(v));
251 }
5c53bb81 252}
1813e175 253#endif
5c53bb81 254
c896fe29
FB		 255/* label relocation processing */
256
1813e175 257static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 258 TCGLabel *l, intptr_t addend)
c896fe29 259{
7ecd02a0 260 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 261
7ecd02a0
RH		 262 r->type = type;
263 r->ptr = code_ptr;
264 r->addend = addend;
265 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 266}
267
92ab8e7d 268static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 269{
eabb7b91 270 tcg_debug_assert(!l->has_value);
c896fe29 271 l->has_value = 1;
92ab8e7d 272 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 273}
274
42a268c2 275TCGLabel *gen_new_label(void)
c896fe29 276{
b1311c4a 277 TCGContext *s = tcg_ctx;
51e3972c 278 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 279
7ecd02a0
RH		 280 memset(l, 0, sizeof(TCGLabel));
281 l->id = s->nb_labels++;
282 QSIMPLEQ_INIT(&l->relocs);
283
bef16ab4 284 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 285
286 return l;
c896fe29
FB		 287}
288
7ecd02a0
RH		 289static bool tcg_resolve_relocs(TCGContext *s)
290{
291 TCGLabel *l;
292
293 QSIMPLEQ_FOREACH(l, &s->labels, next) {
294 TCGRelocation *r;
295 uintptr_t value = l->u.value;
296
297 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
298 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
299 return false;
300 }
301 }
302 }
303 return true;
304}
305
9f754620
RH		 306static void set_jmp_reset_offset(TCGContext *s, int which)
307{
f14bed3f
RH		 308 /*
309 * We will check for overflow at the end of the opcode loop in
310 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
311 */
312 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 313}
314
db6b7d0c
RH		 315/* Signal overflow, starting over with fewer guest insns. */
316static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s)
317{
318 siglongjmp(s->jmp_trans, -2);
319}
320
4c22e840
RH		 321#define C_PFX1(P, A) P##A
322#define C_PFX2(P, A, B) P##A##_##B
323#define C_PFX3(P, A, B, C) P##A##_##B##_##C
324#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
325#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
326#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
327
328/* Define an enumeration for the various combinations. */
329
330#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
331#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
332#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
333#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
334
335#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
336#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
337#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
338#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
339
340#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
341
342#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
343#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
344#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
345#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
346
347typedef enum {
348#include "tcg-target-con-set.h"
349} TCGConstraintSetIndex;
350
351static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
352
353#undef C_O0_I1
354#undef C_O0_I2
355#undef C_O0_I3
356#undef C_O0_I4
357#undef C_O1_I1
358#undef C_O1_I2
359#undef C_O1_I3
360#undef C_O1_I4
361#undef C_N1_I2
362#undef C_O2_I1
363#undef C_O2_I2
364#undef C_O2_I3
365#undef C_O2_I4
366
367/* Put all of the constraint sets into an array, indexed by the enum. */
368
369#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
370#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
371#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
372#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
373
374#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
375#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
376#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
377#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
378
379#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
380
381#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
382#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
383#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
384#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
385
386static const TCGTargetOpDef constraint_sets[] = {
387#include "tcg-target-con-set.h"
388};
389
390
391#undef C_O0_I1
392#undef C_O0_I2
393#undef C_O0_I3
394#undef C_O0_I4
395#undef C_O1_I1
396#undef C_O1_I2
397#undef C_O1_I3
398#undef C_O1_I4
399#undef C_N1_I2
400#undef C_O2_I1
401#undef C_O2_I2
402#undef C_O2_I3
403#undef C_O2_I4
404
405/* Expand the enumerator to be returned from tcg_target_op_def(). */
406
407#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
408#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
409#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
410#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
411
412#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
413#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
414#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
415#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
416
417#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
418
419#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
420#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
421#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
422#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
423
139c1837 424#include "tcg-target.c.inc"
c896fe29 425
38b47b19
EC		 426static void alloc_tcg_plugin_context(TCGContext *s)
427{
428#ifdef CONFIG_PLUGIN
429 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
430 s->plugin_tb->insns =
431 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
432#endif
433}
434
3468b59e
EC		 435/*
436 * All TCG threads except the parent (i.e. the one that called tcg_context_init
437 * and registered the target's TCG globals) must register with this function
438 * before initiating translation.
439 *
440 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
441 * of tcg_region_init() for the reasoning behind this.
442 *
443 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
444 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
445 * is not used anymore for translation once this function is called.
446 *
447 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
448 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
449 */
450#ifdef CONFIG_USER_ONLY
451void tcg_register_thread(void)
452{
453 tcg_ctx = &tcg_init_ctx;
454}
455#else
456void tcg_register_thread(void)
457{
458 TCGContext *s = g_malloc(sizeof(*s));
459 unsigned int i, n;
3468b59e
EC		 460
461 *s = tcg_init_ctx;
462
463 /* Relink mem_base. */
464 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
465 if (tcg_init_ctx.temps[i].mem_base) {
466 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
467 tcg_debug_assert(b >= 0 && b < n);
468 s->temps[i].mem_base = &s->temps[b];
469 }
470 }
471
472 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 473 n = qatomic_fetch_inc(&tcg_cur_ctxs);
474 g_assert(n < tcg_max_ctxs);
d73415a3 475 qatomic_set(&tcg_ctxs[n], s);
3468b59e 476
38b47b19
EC		 477 if (n > 0) {
478 alloc_tcg_plugin_context(s);
bf042e8e 479 tcg_region_initial_alloc(s);
38b47b19
EC		 480 }
481
3468b59e 482 tcg_ctx = s;
e8feb96f 483}
3468b59e 484#endif /* !CONFIG_USER_ONLY */
e8feb96f 485
c896fe29
FB		 486/* pool based memory allocation */
487void *tcg_malloc_internal(TCGContext *s, int size)
488{
489 TCGPool *p;
490 int pool_size;
491
492 if (size > TCG_POOL_CHUNK_SIZE) {
493 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 494 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 495 p->size = size;
4055299e
KB		 496 p->next = s->pool_first_large;
497 s->pool_first_large = p;
498 return p->data;
c896fe29
FB		 499 } else {
500 p = s->pool_current;
501 if (!p) {
502 p = s->pool_first;
503 if (!p)
504 goto new_pool;
505 } else {
506 if (!p->next) {
507 new_pool:
508 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 509 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 510 p->size = pool_size;
511 p->next = NULL;
512 if (s->pool_current)
513 s->pool_current->next = p;
514 else
515 s->pool_first = p;
516 } else {
517 p = p->next;
518 }
519 }
520 }
521 s->pool_current = p;
522 s->pool_cur = p->data + size;
523 s->pool_end = p->data + p->size;
524 return p->data;
525}
526
527void tcg_pool_reset(TCGContext *s)
528{
4055299e
KB		 529 TCGPool *p, *t;
530 for (p = s->pool_first_large; p; p = t) {
531 t = p->next;
532 g_free(p);
533 }
534 s->pool_first_large = NULL;
c896fe29
FB		 535 s->pool_cur = s->pool_end = NULL;
536 s->pool_current = NULL;
537}
538
2ef6175a
RH		 539#include "exec/helper-proto.h"
540
100b5e01 541static const TCGHelperInfo all_helpers[] = {
2ef6175a 542#include "exec/helper-tcg.h"
100b5e01 543};
619205fd 544static GHashTable *helper_table;
100b5e01 545
22f15579
RH		 546#ifdef CONFIG_TCG_INTERPRETER
547static GHashTable *ffi_table;
548
549static ffi_type * const typecode_to_ffi[8] = {
550 [dh_typecode_void] = &ffi_type_void,
551 [dh_typecode_i32] = &ffi_type_uint32,
552 [dh_typecode_s32] = &ffi_type_sint32,
553 [dh_typecode_i64] = &ffi_type_uint64,
554 [dh_typecode_s64] = &ffi_type_sint64,
555 [dh_typecode_ptr] = &ffi_type_pointer,
556};
557#endif
558
91478cef 559static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 560static void process_op_defs(TCGContext *s);
1c2adb95
RH		 561static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
562 TCGReg reg, const char *name);
91478cef 563
43b972b7 564static void tcg_context_init(unsigned max_cpus)
c896fe29 565{
a76aabd3 566 TCGContext *s = &tcg_init_ctx;
100b5e01 567 int op, total_args, n, i;
c896fe29
FB		 568 TCGOpDef *def;
569 TCGArgConstraint *args_ct;
1c2adb95 570 TCGTemp *ts;
c896fe29
FB		 571
572 memset(s, 0, sizeof(*s));
c896fe29 573 s->nb_globals = 0;
c70fbf0a 574
c896fe29
FB		 575 /* Count total number of arguments and allocate the corresponding
576 space */
577 total_args = 0;
578 for(op = 0; op < NB_OPS; op++) {
579 def = &tcg_op_defs[op];
580 n = def->nb_iargs + def->nb_oargs;
581 total_args += n;
582 }
583
bc2b17e6 584 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 585
586 for(op = 0; op < NB_OPS; op++) {
587 def = &tcg_op_defs[op];
588 def->args_ct = args_ct;
c896fe29 589 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 590 args_ct += n;
591 }
5cd8f621
RH		 592
593 /* Register helpers. */
84fd9dd3 594 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 595 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 596
100b5e01 597 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
84fd9dd3 598 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 599 (gpointer)&all_helpers[i]);
100b5e01 600 }
5cd8f621 601
22f15579
RH		 602#ifdef CONFIG_TCG_INTERPRETER
603 /* g_direct_hash/equal for direct comparisons on uint32_t. */
604 ffi_table = g_hash_table_new(NULL, NULL);
605 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
606 struct {
607 ffi_cif cif;
608 ffi_type *args[];
609 } *ca;
610 uint32_t typemask = all_helpers[i].typemask;
611 gpointer hash = (gpointer)(uintptr_t)typemask;
612 ffi_status status;
613 int nargs;
614
615 if (g_hash_table_lookup(ffi_table, hash)) {
616 continue;
617 }
618
619 /* Ignoring the return type, find the last non-zero field. */
620 nargs = 32 - clz32(typemask >> 3);
621 nargs = DIV_ROUND_UP(nargs, 3);
622
623 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
624 ca->cif.rtype = typecode_to_ffi[typemask & 7];
625 ca->cif.nargs = nargs;
626
627 if (nargs != 0) {
628 ca->cif.arg_types = ca->args;
629 for (i = 0; i < nargs; ++i) {
630 int typecode = extract32(typemask, (i + 1) * 3, 3);
631 ca->args[i] = typecode_to_ffi[typecode];
632 }
633 }
634
635 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
636 ca->cif.rtype, ca->cif.arg_types);
637 assert(status == FFI_OK);
638
639 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
640 }
641#endif
642
c896fe29 643 tcg_target_init(s);
f69d277e 644 process_op_defs(s);
91478cef
RH		 645
646 /* Reverse the order of the saved registers, assuming they're all at
647 the start of tcg_target_reg_alloc_order. */
648 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
649 int r = tcg_target_reg_alloc_order[n];
650 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
651 break;
652 }
653 }
654 for (i = 0; i < n; ++i) {
655 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
656 }
657 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
658 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
659 }
b1311c4a 660
38b47b19
EC		 661 alloc_tcg_plugin_context(s);
662
b1311c4a 663 tcg_ctx = s;
3468b59e
EC		 664 /*
665 * In user-mode we simply share the init context among threads, since we
666 * use a single region. See the documentation tcg_region_init() for the
667 * reasoning behind this.
668 * In softmmu we will have at most max_cpus TCG threads.
669 */
670#ifdef CONFIG_USER_ONLY
df2cce29 671 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 672 tcg_cur_ctxs = 1;
673 tcg_max_ctxs = 1;
3468b59e 674#else
0e2d61cf
RH		 675 tcg_max_ctxs = max_cpus;
676 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 677#endif
1c2adb95
RH		 678
679 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
680 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
681 cpu_env = temp_tcgv_ptr(ts);
9002ec79 682}
b03cce8e 683
43b972b7 684void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 685{
43b972b7
RH		 686 tcg_context_init(max_cpus);
687 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 688}
689
6e3b2bfd
EC		 690/*
691 * Allocate TBs right before their corresponding translated code, making
692 * sure that TBs and code are on different cache lines.
693 */
694TranslationBlock *tcg_tb_alloc(TCGContext *s)
695{
696 uintptr_t align = qemu_icache_linesize;
697 TranslationBlock *tb;
698 void *next;
699
e8feb96f 700 retry:
6e3b2bfd
EC		 701 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
702 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
703
704 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 705 if (tcg_region_alloc(s)) {
706 return NULL;
707 }
708 goto retry;
6e3b2bfd 709 }
d73415a3 710 qatomic_set(&s->code_gen_ptr, next);
57a26946 711 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 712 return tb;
713}
714
9002ec79
RH		 715void tcg_prologue_init(TCGContext *s)
716{
b0a0794a 717 size_t prologue_size;
8163b749 718
b0a0794a
RH		 719 s->code_ptr = s->code_gen_ptr;
720 s->code_buf = s->code_gen_ptr;
5b38ee31 721 s->data_gen_ptr = NULL;
b91ccb31
RH		 722
723#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 724 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 725#endif
8163b749 726
5b38ee31
RH		 727#ifdef TCG_TARGET_NEED_POOL_LABELS
728 s->pool_labels = NULL;
729#endif
730
653b87eb 731 qemu_thread_jit_write();
8163b749 732 /* Generate the prologue. */
b03cce8e 733 tcg_target_qemu_prologue(s);
5b38ee31
RH		 734
735#ifdef TCG_TARGET_NEED_POOL_LABELS
736 /* Allow the prologue to put e.g. guest_base into a pool entry. */
737 {
1768987b
RH		 738 int result = tcg_out_pool_finalize(s);
739 tcg_debug_assert(result == 0);
5b38ee31
RH		 740 }
741#endif
742
b0a0794a
RH		 743 prologue_size = tcg_current_code_size(s);
744
df5d2b16 745#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 746 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
747 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 748#endif
8163b749 749
bf042e8e 750 tcg_region_prologue_set(s);
d6b64b2b
RH		 751
752#ifdef DEBUG_DISAS
753 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
fc59d2d8 754 FILE *logfile = qemu_log_lock();
8163b749 755 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
5b38ee31 756 if (s->data_gen_ptr) {
b0a0794a 757 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
5b38ee31
RH		 758 size_t data_size = prologue_size - code_size;
759 size_t i;
760
b0a0794a 761 log_disas(s->code_gen_ptr, code_size);
5b38ee31
RH		 762
763 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
764 if (sizeof(tcg_target_ulong) == 8) {
765 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
766 (uintptr_t)s->data_gen_ptr + i,
767 *(uint64_t *)(s->data_gen_ptr + i));
768 } else {
769 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
770 (uintptr_t)s->data_gen_ptr + i,
771 *(uint32_t *)(s->data_gen_ptr + i));
772 }
773 }
774 } else {
b0a0794a 775 log_disas(s->code_gen_ptr, prologue_size);
5b38ee31 776 }
d6b64b2b
RH		 777 qemu_log("\n");
778 qemu_log_flush();
fc59d2d8 779 qemu_log_unlock(logfile);
d6b64b2b
RH		 780 }
781#endif
cedbcb01
EC		 782
783 /* Assert that goto_ptr is implemented completely. */
784 if (TCG_TARGET_HAS_goto_ptr) {
8b5c2b62 785 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
cedbcb01 786 }
c896fe29
FB		 787}
788
c896fe29
FB		 789void tcg_func_start(TCGContext *s)
790{
791 tcg_pool_reset(s);
792 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 793
794 /* No temps have been previously allocated for size or locality. */
795 memset(s->free_temps, 0, sizeof(s->free_temps));
796
c0522136
RH		 797 /* No constant temps have been previously allocated. */
798 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
799 if (s->const_table[i]) {
800 g_hash_table_remove_all(s->const_table[i]);
801 }
802 }
803
abebf925 804 s->nb_ops = 0;
c896fe29
FB		 805 s->nb_labels = 0;
806 s->current_frame_offset = s->frame_start;
807
0a209d4b
RH		 808#ifdef CONFIG_DEBUG_TCG
809 s->goto_tb_issue_mask = 0;
810#endif
811
15fa08f8
RH		 812 QTAILQ_INIT(&s->ops);
813 QTAILQ_INIT(&s->free_ops);
bef16ab4 814 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 815}
816
ae30e866 817static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 818{
819 int n = s->nb_temps++;
ae30e866
RH		 820
821 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 822 tcg_raise_tb_overflow(s);
ae30e866 823 }
7ca4b752
RH		 824 return memset(&s->temps[n], 0, sizeof(TCGTemp));
825}
826
ae30e866 827static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 828{
fa477d25
RH		 829 TCGTemp *ts;
830
7ca4b752 831 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 832 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 833 s->nb_globals++;
fa477d25 834 ts = tcg_temp_alloc(s);
ee17db83 835 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 836
837 return ts;
c896fe29
FB		 838}
839
085272b3
RH		 840static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
841 TCGReg reg, const char *name)
c896fe29 842{
c896fe29 843 TCGTemp *ts;
c896fe29 844
b3a62939 845 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 846 tcg_abort();
b3a62939 847 }
7ca4b752
RH		 848
849 ts = tcg_global_alloc(s);
c896fe29
FB		 850 ts->base_type = type;
851 ts->type = type;
ee17db83 852 ts->kind = TEMP_FIXED;
c896fe29 853 ts->reg = reg;
c896fe29 854 ts->name = name;
c896fe29 855 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 856
085272b3 857 return ts;
a7812ae4
PB		 858}
859
b6638662 860void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 861{
b3a62939
RH		 862 s->frame_start = start;
863 s->frame_end = start + size;
085272b3
RH		 864 s->frame_temp
865 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 866}
867
085272b3
RH		 868TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
869 intptr_t offset, const char *name)
c896fe29 870{
b1311c4a 871 TCGContext *s = tcg_ctx;
dc41aa7d 872 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 873 TCGTemp *ts = tcg_global_alloc(s);
b3915dbb 874 int indirect_reg = 0, bigendian = 0;
7ca4b752
RH		 875#ifdef HOST_WORDS_BIGENDIAN
876 bigendian = 1;
877#endif
c896fe29 878
c0522136
RH		 879 switch (base_ts->kind) {
880 case TEMP_FIXED:
881 break;
882 case TEMP_GLOBAL:
5a18407f
RH		 883 /* We do not support double-indirect registers. */
884 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 885 base_ts->indirect_base = 1;
5a18407f
RH		 886 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
887 ? 2 : 1);
888 indirect_reg = 1;
c0522136
RH		 889 break;
890 default:
891 g_assert_not_reached();
b3915dbb
RH		 892 }
893
7ca4b752
RH		 894 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
895 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 896 char buf[64];
7ca4b752
RH		 897
898 ts->base_type = TCG_TYPE_I64;
c896fe29 899 ts->type = TCG_TYPE_I32;
b3915dbb 900 ts->indirect_reg = indirect_reg;
c896fe29 901 ts->mem_allocated = 1;
b3a62939 902 ts->mem_base = base_ts;
7ca4b752 903 ts->mem_offset = offset + bigendian * 4;
c896fe29
FB		 904 pstrcpy(buf, sizeof(buf), name);
905 pstrcat(buf, sizeof(buf), "_0");
906 ts->name = strdup(buf);
c896fe29 907
7ca4b752
RH		 908 tcg_debug_assert(ts2 == ts + 1);
909 ts2->base_type = TCG_TYPE_I64;
910 ts2->type = TCG_TYPE_I32;
b3915dbb 911 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 912 ts2->mem_allocated = 1;
913 ts2->mem_base = base_ts;
914 ts2->mem_offset = offset + (1 - bigendian) * 4;
c896fe29
FB		 915 pstrcpy(buf, sizeof(buf), name);
916 pstrcat(buf, sizeof(buf), "_1");
120c1084 917 ts2->name = strdup(buf);
7ca4b752 918 } else {
c896fe29
FB		 919 ts->base_type = type;
920 ts->type = type;
b3915dbb 921 ts->indirect_reg = indirect_reg;
c896fe29 922 ts->mem_allocated = 1;
b3a62939 923 ts->mem_base = base_ts;
c896fe29 924 ts->mem_offset = offset;
c896fe29 925 ts->name = name;
c896fe29 926 }
085272b3 927 return ts;
a7812ae4
PB		 928}
929
5bfa8034 930TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
c896fe29 931{
b1311c4a 932 TCGContext *s = tcg_ctx;
ee17db83 933 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
c896fe29 934 TCGTemp *ts;
641d5fbe 935 int idx, k;
c896fe29 936
0ec9eabc
RH		 937 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
938 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
939 if (idx < TCG_MAX_TEMPS) {
940 /* There is already an available temp with the right type. */
941 clear_bit(idx, s->free_temps[k].l);
942
e8996ee0 943 ts = &s->temps[idx];
e8996ee0 944 ts->temp_allocated = 1;
7ca4b752 945 tcg_debug_assert(ts->base_type == type);
ee17db83 946 tcg_debug_assert(ts->kind == kind);
e8996ee0 947 } else {
7ca4b752
RH		 948 ts = tcg_temp_alloc(s);
949 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
950 TCGTemp *ts2 = tcg_temp_alloc(s);
951
f6aa2f7d 952 ts->base_type = type;
e8996ee0
FB		 953 ts->type = TCG_TYPE_I32;
954 ts->temp_allocated = 1;
ee17db83 955 ts->kind = kind;
7ca4b752
RH		 956
957 tcg_debug_assert(ts2 == ts + 1);
958 ts2->base_type = TCG_TYPE_I64;
959 ts2->type = TCG_TYPE_I32;
960 ts2->temp_allocated = 1;
ee17db83 961 ts2->kind = kind;
7ca4b752 962 } else {
e8996ee0
FB		 963 ts->base_type = type;
964 ts->type = type;
965 ts->temp_allocated = 1;
ee17db83 966 ts->kind = kind;
e8996ee0 967 }
c896fe29 968 }
27bfd83c
PM		 969
970#if defined(CONFIG_DEBUG_TCG)
971 s->temps_in_use++;
972#endif
085272b3 973 return ts;
c896fe29
FB		 974}
975
d2fd745f
RH		 976TCGv_vec tcg_temp_new_vec(TCGType type)
977{
978 TCGTemp *t;
979
980#ifdef CONFIG_DEBUG_TCG
981 switch (type) {
982 case TCG_TYPE_V64:
983 assert(TCG_TARGET_HAS_v64);
984 break;
985 case TCG_TYPE_V128:
986 assert(TCG_TARGET_HAS_v128);
987 break;
988 case TCG_TYPE_V256:
989 assert(TCG_TARGET_HAS_v256);
990 break;
991 default:
992 g_assert_not_reached();
993 }
994#endif
995
996 t = tcg_temp_new_internal(type, 0);
997 return temp_tcgv_vec(t);
998}
999
1000/* Create a new temp of the same type as an existing temp. */
1001TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1002{
1003 TCGTemp *t = tcgv_vec_temp(match);
1004
1005 tcg_debug_assert(t->temp_allocated != 0);
1006
1007 t = tcg_temp_new_internal(t->base_type, 0);
1008 return temp_tcgv_vec(t);
1009}
1010
5bfa8034 1011void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1012{
b1311c4a 1013 TCGContext *s = tcg_ctx;
085272b3 1014 int k, idx;
c896fe29 1015
c0522136
RH		 1016 /* In order to simplify users of tcg_constant_*, silently ignore free. */
1017 if (ts->kind == TEMP_CONST) {
1018 return;
1019 }
1020
27bfd83c
PM		 1021#if defined(CONFIG_DEBUG_TCG)
1022 s->temps_in_use--;
1023 if (s->temps_in_use < 0) {
1024 fprintf(stderr, "More temporaries freed than allocated!\n");
1025 }
1026#endif
1027
ee17db83 1028 tcg_debug_assert(ts->kind < TEMP_GLOBAL);
eabb7b91 1029 tcg_debug_assert(ts->temp_allocated != 0);
e8996ee0 1030 ts->temp_allocated = 0;
0ec9eabc 1031
085272b3 1032 idx = temp_idx(ts);
ee17db83 1033 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1034 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1035}
1036
c0522136
RH		 1037TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1038{
1039 TCGContext *s = tcg_ctx;
1040 GHashTable *h = s->const_table[type];
1041 TCGTemp *ts;
1042
1043 if (h == NULL) {
1044 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1045 s->const_table[type] = h;
1046 }
1047
1048 ts = g_hash_table_lookup(h, &val);
1049 if (ts == NULL) {
1050 ts = tcg_temp_alloc(s);
1051
1052 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1053 TCGTemp *ts2 = tcg_temp_alloc(s);
1054
1055 ts->base_type = TCG_TYPE_I64;
1056 ts->type = TCG_TYPE_I32;
1057 ts->kind = TEMP_CONST;
1058 ts->temp_allocated = 1;
1059 /*
1060 * Retain the full value of the 64-bit constant in the low
1061 * part, so that the hash table works. Actual uses will
1062 * truncate the value to the low part.
1063 */
1064 ts->val = val;
1065
1066 tcg_debug_assert(ts2 == ts + 1);
1067 ts2->base_type = TCG_TYPE_I64;
1068 ts2->type = TCG_TYPE_I32;
1069 ts2->kind = TEMP_CONST;
1070 ts2->temp_allocated = 1;
1071 ts2->val = val >> 32;
1072 } else {
1073 ts->base_type = type;
1074 ts->type = type;
1075 ts->kind = TEMP_CONST;
1076 ts->temp_allocated = 1;
1077 ts->val = val;
1078 }
1079 g_hash_table_insert(h, &ts->val, ts);
1080 }
1081
1082 return ts;
1083}
1084
1085TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1086{
1087 val = dup_const(vece, val);
1088 return temp_tcgv_vec(tcg_constant_internal(type, val));
1089}
1090
88d4005b
RH		 1091TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1092{
1093 TCGTemp *t = tcgv_vec_temp(match);
1094
1095 tcg_debug_assert(t->temp_allocated != 0);
1096 return tcg_constant_vec(t->base_type, vece, val);
1097}
1098
a7812ae4 1099TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1100{
a7812ae4
PB		 1101 TCGv_i32 t0;
1102 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1103 tcg_gen_movi_i32(t0, val);
1104 return t0;
1105}
c896fe29 1106
a7812ae4 1107TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1108{
a7812ae4
PB		 1109 TCGv_i64 t0;
1110 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1111 tcg_gen_movi_i64(t0, val);
1112 return t0;
c896fe29
FB		 1113}
1114
a7812ae4 1115TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1116{
a7812ae4
PB		 1117 TCGv_i32 t0;
1118 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1119 tcg_gen_movi_i32(t0, val);
1120 return t0;
1121}
1122
a7812ae4 1123TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1124{
a7812ae4
PB		 1125 TCGv_i64 t0;
1126 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1127 tcg_gen_movi_i64(t0, val);
1128 return t0;
1129}
1130
27bfd83c
PM		 1131#if defined(CONFIG_DEBUG_TCG)
1132void tcg_clear_temp_count(void)
1133{
b1311c4a 1134 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1135 s->temps_in_use = 0;
1136}
1137
1138int tcg_check_temp_count(void)
1139{
b1311c4a 1140 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1141 if (s->temps_in_use) {
1142 /* Clear the count so that we don't give another
1143 * warning immediately next time around.
1144 */
1145 s->temps_in_use = 0;
1146 return 1;
1147 }
1148 return 0;
1149}
1150#endif
1151
be0f34b5
RH		 1152/* Return true if OP may appear in the opcode stream.
1153 Test the runtime variable that controls each opcode. */
1154bool tcg_op_supported(TCGOpcode op)
1155{
d2fd745f
RH		 1156 const bool have_vec
1157 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1158
be0f34b5
RH		 1159 switch (op) {
1160 case INDEX_op_discard:
1161 case INDEX_op_set_label:
1162 case INDEX_op_call:
1163 case INDEX_op_br:
1164 case INDEX_op_mb:
1165 case INDEX_op_insn_start:
1166 case INDEX_op_exit_tb:
1167 case INDEX_op_goto_tb:
1168 case INDEX_op_qemu_ld_i32:
1169 case INDEX_op_qemu_st_i32:
1170 case INDEX_op_qemu_ld_i64:
1171 case INDEX_op_qemu_st_i64:
1172 return true;
1173
07ce0b05
RH		 1174 case INDEX_op_qemu_st8_i32:
1175 return TCG_TARGET_HAS_qemu_st8_i32;
1176
be0f34b5
RH		 1177 case INDEX_op_goto_ptr:
1178 return TCG_TARGET_HAS_goto_ptr;
1179
1180 case INDEX_op_mov_i32:
be0f34b5
RH		 1181 case INDEX_op_setcond_i32:
1182 case INDEX_op_brcond_i32:
1183 case INDEX_op_ld8u_i32:
1184 case INDEX_op_ld8s_i32:
1185 case INDEX_op_ld16u_i32:
1186 case INDEX_op_ld16s_i32:
1187 case INDEX_op_ld_i32:
1188 case INDEX_op_st8_i32:
1189 case INDEX_op_st16_i32:
1190 case INDEX_op_st_i32:
1191 case INDEX_op_add_i32:
1192 case INDEX_op_sub_i32:
1193 case INDEX_op_mul_i32:
1194 case INDEX_op_and_i32:
1195 case INDEX_op_or_i32:
1196 case INDEX_op_xor_i32:
1197 case INDEX_op_shl_i32:
1198 case INDEX_op_shr_i32:
1199 case INDEX_op_sar_i32:
1200 return true;
1201
1202 case INDEX_op_movcond_i32:
1203 return TCG_TARGET_HAS_movcond_i32;
1204 case INDEX_op_div_i32:
1205 case INDEX_op_divu_i32:
1206 return TCG_TARGET_HAS_div_i32;
1207 case INDEX_op_rem_i32:
1208 case INDEX_op_remu_i32:
1209 return TCG_TARGET_HAS_rem_i32;
1210 case INDEX_op_div2_i32:
1211 case INDEX_op_divu2_i32:
1212 return TCG_TARGET_HAS_div2_i32;
1213 case INDEX_op_rotl_i32:
1214 case INDEX_op_rotr_i32:
1215 return TCG_TARGET_HAS_rot_i32;
1216 case INDEX_op_deposit_i32:
1217 return TCG_TARGET_HAS_deposit_i32;
1218 case INDEX_op_extract_i32:
1219 return TCG_TARGET_HAS_extract_i32;
1220 case INDEX_op_sextract_i32:
1221 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1222 case INDEX_op_extract2_i32:
1223 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1224 case INDEX_op_add2_i32:
1225 return TCG_TARGET_HAS_add2_i32;
1226 case INDEX_op_sub2_i32:
1227 return TCG_TARGET_HAS_sub2_i32;
1228 case INDEX_op_mulu2_i32:
1229 return TCG_TARGET_HAS_mulu2_i32;
1230 case INDEX_op_muls2_i32:
1231 return TCG_TARGET_HAS_muls2_i32;
1232 case INDEX_op_muluh_i32:
1233 return TCG_TARGET_HAS_muluh_i32;
1234 case INDEX_op_mulsh_i32:
1235 return TCG_TARGET_HAS_mulsh_i32;
1236 case INDEX_op_ext8s_i32:
1237 return TCG_TARGET_HAS_ext8s_i32;
1238 case INDEX_op_ext16s_i32:
1239 return TCG_TARGET_HAS_ext16s_i32;
1240 case INDEX_op_ext8u_i32:
1241 return TCG_TARGET_HAS_ext8u_i32;
1242 case INDEX_op_ext16u_i32:
1243 return TCG_TARGET_HAS_ext16u_i32;
1244 case INDEX_op_bswap16_i32:
1245 return TCG_TARGET_HAS_bswap16_i32;
1246 case INDEX_op_bswap32_i32:
1247 return TCG_TARGET_HAS_bswap32_i32;
1248 case INDEX_op_not_i32:
1249 return TCG_TARGET_HAS_not_i32;
1250 case INDEX_op_neg_i32:
1251 return TCG_TARGET_HAS_neg_i32;
1252 case INDEX_op_andc_i32:
1253 return TCG_TARGET_HAS_andc_i32;
1254 case INDEX_op_orc_i32:
1255 return TCG_TARGET_HAS_orc_i32;
1256 case INDEX_op_eqv_i32:
1257 return TCG_TARGET_HAS_eqv_i32;
1258 case INDEX_op_nand_i32:
1259 return TCG_TARGET_HAS_nand_i32;
1260 case INDEX_op_nor_i32:
1261 return TCG_TARGET_HAS_nor_i32;
1262 case INDEX_op_clz_i32:
1263 return TCG_TARGET_HAS_clz_i32;
1264 case INDEX_op_ctz_i32:
1265 return TCG_TARGET_HAS_ctz_i32;
1266 case INDEX_op_ctpop_i32:
1267 return TCG_TARGET_HAS_ctpop_i32;
1268
1269 case INDEX_op_brcond2_i32:
1270 case INDEX_op_setcond2_i32:
1271 return TCG_TARGET_REG_BITS == 32;
1272
1273 case INDEX_op_mov_i64:
be0f34b5
RH		 1274 case INDEX_op_setcond_i64:
1275 case INDEX_op_brcond_i64:
1276 case INDEX_op_ld8u_i64:
1277 case INDEX_op_ld8s_i64:
1278 case INDEX_op_ld16u_i64:
1279 case INDEX_op_ld16s_i64:
1280 case INDEX_op_ld32u_i64:
1281 case INDEX_op_ld32s_i64:
1282 case INDEX_op_ld_i64:
1283 case INDEX_op_st8_i64:
1284 case INDEX_op_st16_i64:
1285 case INDEX_op_st32_i64:
1286 case INDEX_op_st_i64:
1287 case INDEX_op_add_i64:
1288 case INDEX_op_sub_i64:
1289 case INDEX_op_mul_i64:
1290 case INDEX_op_and_i64:
1291 case INDEX_op_or_i64:
1292 case INDEX_op_xor_i64:
1293 case INDEX_op_shl_i64:
1294 case INDEX_op_shr_i64:
1295 case INDEX_op_sar_i64:
1296 case INDEX_op_ext_i32_i64:
1297 case INDEX_op_extu_i32_i64:
1298 return TCG_TARGET_REG_BITS == 64;
1299
1300 case INDEX_op_movcond_i64:
1301 return TCG_TARGET_HAS_movcond_i64;
1302 case INDEX_op_div_i64:
1303 case INDEX_op_divu_i64:
1304 return TCG_TARGET_HAS_div_i64;
1305 case INDEX_op_rem_i64:
1306 case INDEX_op_remu_i64:
1307 return TCG_TARGET_HAS_rem_i64;
1308 case INDEX_op_div2_i64:
1309 case INDEX_op_divu2_i64:
1310 return TCG_TARGET_HAS_div2_i64;
1311 case INDEX_op_rotl_i64:
1312 case INDEX_op_rotr_i64:
1313 return TCG_TARGET_HAS_rot_i64;
1314 case INDEX_op_deposit_i64:
1315 return TCG_TARGET_HAS_deposit_i64;
1316 case INDEX_op_extract_i64:
1317 return TCG_TARGET_HAS_extract_i64;
1318 case INDEX_op_sextract_i64:
1319 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1320 case INDEX_op_extract2_i64:
1321 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1322 case INDEX_op_extrl_i64_i32:
1323 return TCG_TARGET_HAS_extrl_i64_i32;
1324 case INDEX_op_extrh_i64_i32:
1325 return TCG_TARGET_HAS_extrh_i64_i32;
1326 case INDEX_op_ext8s_i64:
1327 return TCG_TARGET_HAS_ext8s_i64;
1328 case INDEX_op_ext16s_i64:
1329 return TCG_TARGET_HAS_ext16s_i64;
1330 case INDEX_op_ext32s_i64:
1331 return TCG_TARGET_HAS_ext32s_i64;
1332 case INDEX_op_ext8u_i64:
1333 return TCG_TARGET_HAS_ext8u_i64;
1334 case INDEX_op_ext16u_i64:
1335 return TCG_TARGET_HAS_ext16u_i64;
1336 case INDEX_op_ext32u_i64:
1337 return TCG_TARGET_HAS_ext32u_i64;
1338 case INDEX_op_bswap16_i64:
1339 return TCG_TARGET_HAS_bswap16_i64;
1340 case INDEX_op_bswap32_i64:
1341 return TCG_TARGET_HAS_bswap32_i64;
1342 case INDEX_op_bswap64_i64:
1343 return TCG_TARGET_HAS_bswap64_i64;
1344 case INDEX_op_not_i64:
1345 return TCG_TARGET_HAS_not_i64;
1346 case INDEX_op_neg_i64:
1347 return TCG_TARGET_HAS_neg_i64;
1348 case INDEX_op_andc_i64:
1349 return TCG_TARGET_HAS_andc_i64;
1350 case INDEX_op_orc_i64:
1351 return TCG_TARGET_HAS_orc_i64;
1352 case INDEX_op_eqv_i64:
1353 return TCG_TARGET_HAS_eqv_i64;
1354 case INDEX_op_nand_i64:
1355 return TCG_TARGET_HAS_nand_i64;
1356 case INDEX_op_nor_i64:
1357 return TCG_TARGET_HAS_nor_i64;
1358 case INDEX_op_clz_i64:
1359 return TCG_TARGET_HAS_clz_i64;
1360 case INDEX_op_ctz_i64:
1361 return TCG_TARGET_HAS_ctz_i64;
1362 case INDEX_op_ctpop_i64:
1363 return TCG_TARGET_HAS_ctpop_i64;
1364 case INDEX_op_add2_i64:
1365 return TCG_TARGET_HAS_add2_i64;
1366 case INDEX_op_sub2_i64:
1367 return TCG_TARGET_HAS_sub2_i64;
1368 case INDEX_op_mulu2_i64:
1369 return TCG_TARGET_HAS_mulu2_i64;
1370 case INDEX_op_muls2_i64:
1371 return TCG_TARGET_HAS_muls2_i64;
1372 case INDEX_op_muluh_i64:
1373 return TCG_TARGET_HAS_muluh_i64;
1374 case INDEX_op_mulsh_i64:
1375 return TCG_TARGET_HAS_mulsh_i64;
1376
d2fd745f
RH		 1377 case INDEX_op_mov_vec:
1378 case INDEX_op_dup_vec:
37ee55a0 1379 case INDEX_op_dupm_vec:
d2fd745f
RH		 1380 case INDEX_op_ld_vec:
1381 case INDEX_op_st_vec:
1382 case INDEX_op_add_vec:
1383 case INDEX_op_sub_vec:
1384 case INDEX_op_and_vec:
1385 case INDEX_op_or_vec:
1386 case INDEX_op_xor_vec:
212be173 1387 case INDEX_op_cmp_vec:
d2fd745f
RH		 1388 return have_vec;
1389 case INDEX_op_dup2_vec:
1390 return have_vec && TCG_TARGET_REG_BITS == 32;
1391 case INDEX_op_not_vec:
1392 return have_vec && TCG_TARGET_HAS_not_vec;
1393 case INDEX_op_neg_vec:
1394 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1395 case INDEX_op_abs_vec:
1396 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1397 case INDEX_op_andc_vec:
1398 return have_vec && TCG_TARGET_HAS_andc_vec;
1399 case INDEX_op_orc_vec:
1400 return have_vec && TCG_TARGET_HAS_orc_vec;
3774030a
RH		 1401 case INDEX_op_mul_vec:
1402 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1403 case INDEX_op_shli_vec:
1404 case INDEX_op_shri_vec:
1405 case INDEX_op_sari_vec:
1406 return have_vec && TCG_TARGET_HAS_shi_vec;
1407 case INDEX_op_shls_vec:
1408 case INDEX_op_shrs_vec:
1409 case INDEX_op_sars_vec:
1410 return have_vec && TCG_TARGET_HAS_shs_vec;
1411 case INDEX_op_shlv_vec:
1412 case INDEX_op_shrv_vec:
1413 case INDEX_op_sarv_vec:
1414 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1415 case INDEX_op_rotli_vec:
1416 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1417 case INDEX_op_rotls_vec:
1418 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1419 case INDEX_op_rotlv_vec:
1420 case INDEX_op_rotrv_vec:
1421 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1422 case INDEX_op_ssadd_vec:
1423 case INDEX_op_usadd_vec:
1424 case INDEX_op_sssub_vec:
1425 case INDEX_op_ussub_vec:
1426 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1427 case INDEX_op_smin_vec:
1428 case INDEX_op_umin_vec:
1429 case INDEX_op_smax_vec:
1430 case INDEX_op_umax_vec:
1431 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1432 case INDEX_op_bitsel_vec:
1433 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1434 case INDEX_op_cmpsel_vec:
1435 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1436
db432672
RH		 1437 default:
1438 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1439 return true;
be0f34b5 1440 }
be0f34b5
RH		 1441}
1442
39cf05d3
FB		 1443/* Note: we convert the 64 bit args to 32 bit and do some alignment
1444 and endian swap. Maybe it would be better to do the alignment
1445 and endian swap in tcg_reg_alloc_call(). */
ae8b75dc 1446void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1447{
75e8b9b7 1448 int i, real_args, nb_rets, pi;
3e92aa34
RH		 1449 unsigned typemask;
1450 const TCGHelperInfo *info;
75e8b9b7 1451 TCGOp *op;
afb49896 1452
619205fd 1453 info = g_hash_table_lookup(helper_table, (gpointer)func);
7319d83a 1454 typemask = info->typemask;
2bece2c8 1455
38b47b19
EC		 1456#ifdef CONFIG_PLUGIN
1457 /* detect non-plugin helpers */
1458 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1459 tcg_ctx->plugin_insn->calls_helpers = true;
1460 }
1461#endif
1462
34b1a49c
RH		 1463#if defined(__sparc__) && !defined(__arch64__) \
1464 && !defined(CONFIG_TCG_INTERPRETER)
1465 /* We have 64-bit values in one register, but need to pass as two
1466 separate parameters. Split them. */
7319d83a 1467 int orig_typemask = typemask;
34b1a49c
RH		 1468 int orig_nargs = nargs;
1469 TCGv_i64 retl, reth;
ae8b75dc 1470 TCGTemp *split_args[MAX_OPC_PARAM];
34b1a49c 1471
f764718d
RH		 1472 retl = NULL;
1473 reth = NULL;
7319d83a
RH		 1474 typemask = 0;
1475 for (i = real_args = 0; i < nargs; ++i) {
1476 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1477 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1478
1479 if (is_64bit) {
1480 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1481 TCGv_i32 h = tcg_temp_new_i32();
1482 TCGv_i32 l = tcg_temp_new_i32();
1483 tcg_gen_extr_i64_i32(l, h, orig);
1484 split_args[real_args++] = tcgv_i32_temp(h);
1485 typemask |= dh_typecode_i32 << (real_args * 3);
1486 split_args[real_args++] = tcgv_i32_temp(l);
1487 typemask |= dh_typecode_i32 << (real_args * 3);
1488 } else {
1489 split_args[real_args++] = args[i];
1490 typemask |= argtype << (real_args * 3);
34b1a49c 1491 }
34b1a49c 1492 }
7319d83a
RH		 1493 nargs = real_args;
1494 args = split_args;
34b1a49c 1495#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1496 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1497 int argtype = extract32(typemask, (i + 1) * 3, 3);
1498 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1499 bool is_signed = argtype & 1;
1500
1501 if (is_32bit) {
2bece2c8 1502 TCGv_i64 temp = tcg_temp_new_i64();
085272b3 1503 TCGv_i64 orig = temp_tcgv_i64(args[i]);
2bece2c8
RH		 1504 if (is_signed) {
1505 tcg_gen_ext32s_i64(temp, orig);
1506 } else {
1507 tcg_gen_ext32u_i64(temp, orig);
1508 }
ae8b75dc 1509 args[i] = tcgv_i64_temp(temp);
2bece2c8
RH		 1510 }
1511 }
1512#endif /* TCG_TARGET_EXTEND_ARGS */
1513
15fa08f8 1514 op = tcg_emit_op(INDEX_op_call);
75e8b9b7
RH		 1515
1516 pi = 0;
ae8b75dc 1517 if (ret != NULL) {
34b1a49c
RH		 1518#if defined(__sparc__) && !defined(__arch64__) \
1519 && !defined(CONFIG_TCG_INTERPRETER)
7319d83a 1520 if ((typemask & 6) == dh_typecode_i64) {
34b1a49c
RH		 1521 /* The 32-bit ABI is going to return the 64-bit value in
1522 the %o0/%o1 register pair. Prepare for this by using
1523 two return temporaries, and reassemble below. */
1524 retl = tcg_temp_new_i64();
1525 reth = tcg_temp_new_i64();
ae8b75dc
RH		 1526 op->args[pi++] = tcgv_i64_arg(reth);
1527 op->args[pi++] = tcgv_i64_arg(retl);
34b1a49c
RH		 1528 nb_rets = 2;
1529 } else {
ae8b75dc 1530 op->args[pi++] = temp_arg(ret);
34b1a49c
RH		 1531 nb_rets = 1;
1532 }
1533#else
7319d83a 1534 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
02eb19d0 1535#ifdef HOST_WORDS_BIGENDIAN
ae8b75dc
RH		 1536 op->args[pi++] = temp_arg(ret + 1);
1537 op->args[pi++] = temp_arg(ret);
39cf05d3 1538#else
ae8b75dc
RH		 1539 op->args[pi++] = temp_arg(ret);
1540 op->args[pi++] = temp_arg(ret + 1);
39cf05d3 1541#endif
a7812ae4 1542 nb_rets = 2;
34b1a49c 1543 } else {
ae8b75dc 1544 op->args[pi++] = temp_arg(ret);
a7812ae4 1545 nb_rets = 1;
c896fe29 1546 }
34b1a49c 1547#endif
a7812ae4
PB		 1548 } else {
1549 nb_rets = 0;
c896fe29 1550 }
cd9090aa 1551 TCGOP_CALLO(op) = nb_rets;
75e8b9b7 1552
a7812ae4
PB		 1553 real_args = 0;
1554 for (i = 0; i < nargs; i++) {
7319d83a
RH		 1555 int argtype = extract32(typemask, (i + 1) * 3, 3);
1556 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1557
bbb8a1b4 1558 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
39cf05d3
FB		 1559#ifdef TCG_TARGET_CALL_ALIGN_ARGS
1560 /* some targets want aligned 64 bit args */
ebd486d5 1561 if (real_args & 1) {
75e8b9b7 1562 op->args[pi++] = TCG_CALL_DUMMY_ARG;
ebd486d5 1563 real_args++;
39cf05d3
FB		 1564 }
1565#endif
c70fbf0a
RH		 1566 /* If stack grows up, then we will be placing successive
1567 arguments at lower addresses, which means we need to
1568 reverse the order compared to how we would normally
1569 treat either big or little-endian. For those arguments
1570 that will wind up in registers, this still works for
1571 HPPA (the only current STACK_GROWSUP target) since the
1572 argument registers are *also* allocated in decreasing
1573 order. If another such target is added, this logic may
1574 have to get more complicated to differentiate between
1575 stack arguments and register arguments. */
02eb19d0 1576#if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
ae8b75dc
RH		 1577 op->args[pi++] = temp_arg(args[i] + 1);
1578 op->args[pi++] = temp_arg(args[i]);
c896fe29 1579#else
ae8b75dc
RH		 1580 op->args[pi++] = temp_arg(args[i]);
1581 op->args[pi++] = temp_arg(args[i] + 1);
c896fe29 1582#endif
a7812ae4 1583 real_args += 2;
2bece2c8 1584 continue;
c896fe29 1585 }
2bece2c8 1586
ae8b75dc 1587 op->args[pi++] = temp_arg(args[i]);
2bece2c8 1588 real_args++;
c896fe29 1589 }
75e8b9b7 1590 op->args[pi++] = (uintptr_t)func;
3e92aa34 1591 op->args[pi++] = (uintptr_t)info;
cd9090aa 1592 TCGOP_CALLI(op) = real_args;
a7812ae4 1593
75e8b9b7 1594 /* Make sure the fields didn't overflow. */
cd9090aa 1595 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
75e8b9b7 1596 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
2bece2c8 1597
34b1a49c
RH		 1598#if defined(__sparc__) && !defined(__arch64__) \
1599 && !defined(CONFIG_TCG_INTERPRETER)
1600 /* Free all of the parts we allocated above. */
1601 for (i = real_args = 0; i < orig_nargs; ++i) {
7319d83a
RH		 1602 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1603 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1604
34b1a49c 1605 if (is_64bit) {
085272b3
RH		 1606 tcg_temp_free_internal(args[real_args++]);
1607 tcg_temp_free_internal(args[real_args++]);
34b1a49c
RH		 1608 } else {
1609 real_args++;
1610 }
1611 }
7319d83a 1612 if ((orig_typemask & 6) == dh_typecode_i64) {
34b1a49c
RH		 1613 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1614 Note that describing these as TCGv_i64 eliminates an unnecessary
1615 zero-extension that tcg_gen_concat_i32_i64 would create. */
085272b3 1616 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
34b1a49c
RH		 1617 tcg_temp_free_i64(retl);
1618 tcg_temp_free_i64(reth);
1619 }
1620#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1621 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1622 int argtype = extract32(typemask, (i + 1) * 3, 3);
1623 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1624
1625 if (is_32bit) {
085272b3 1626 tcg_temp_free_internal(args[i]);
2bece2c8
RH		 1627 }
1628 }
1629#endif /* TCG_TARGET_EXTEND_ARGS */
c896fe29 1630}
c896fe29 1631
8fcd3692 1632static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1633{
ac3b8891 1634 int i, n;
ac3b8891 1635
ee17db83
RH		 1636 for (i = 0, n = s->nb_temps; i < n; i++) {
1637 TCGTemp *ts = &s->temps[i];
1638 TCGTempVal val = TEMP_VAL_MEM;
1639
1640 switch (ts->kind) {
c0522136
RH		 1641 case TEMP_CONST:
1642 val = TEMP_VAL_CONST;
1643 break;
ee17db83
RH		 1644 case TEMP_FIXED:
1645 val = TEMP_VAL_REG;
1646 break;
1647 case TEMP_GLOBAL:
1648 break;
1649 case TEMP_NORMAL:
1650 val = TEMP_VAL_DEAD;
1651 /* fall through */
1652 case TEMP_LOCAL:
1653 ts->mem_allocated = 0;
1654 break;
1655 default:
1656 g_assert_not_reached();
1657 }
1658 ts->val_type = val;
e8996ee0 1659 }
f8b2f202
RH		 1660
1661 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1662}
1663
f8b2f202
RH		 1664static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1665 TCGTemp *ts)
c896fe29 1666{
1807f4c4 1667 int idx = temp_idx(ts);
ac56dd48 1668
ee17db83
RH		 1669 switch (ts->kind) {
1670 case TEMP_FIXED:
1671 case TEMP_GLOBAL:
ac56dd48 1672 pstrcpy(buf, buf_size, ts->name);
ee17db83
RH		 1673 break;
1674 case TEMP_LOCAL:
f8b2f202 1675 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83
RH		 1676 break;
1677 case TEMP_NORMAL:
f8b2f202 1678 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1679 break;
c0522136
RH		 1680 case TEMP_CONST:
1681 switch (ts->type) {
1682 case TCG_TYPE_I32:
1683 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1684 break;
1685#if TCG_TARGET_REG_BITS > 32
1686 case TCG_TYPE_I64:
1687 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1688 break;
1689#endif
1690 case TCG_TYPE_V64:
1691 case TCG_TYPE_V128:
1692 case TCG_TYPE_V256:
1693 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1694 64 << (ts->type - TCG_TYPE_V64), ts->val);
1695 break;
1696 default:
1697 g_assert_not_reached();
1698 }
1699 break;
c896fe29
FB		 1700 }
1701 return buf;
1702}
1703
43439139
RH		 1704static char *tcg_get_arg_str(TCGContext *s, char *buf,
1705 int buf_size, TCGArg arg)
f8b2f202 1706{
43439139 1707 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1708}
1709
f48f3ede
BS		 1710static const char * const cond_name[] =
1711{
0aed257f
RH		 1712 [TCG_COND_NEVER] = "never",
1713 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1714 [TCG_COND_EQ] = "eq",
1715 [TCG_COND_NE] = "ne",
1716 [TCG_COND_LT] = "lt",
1717 [TCG_COND_GE] = "ge",
1718 [TCG_COND_LE] = "le",
1719 [TCG_COND_GT] = "gt",
1720 [TCG_COND_LTU] = "ltu",
1721 [TCG_COND_GEU] = "geu",
1722 [TCG_COND_LEU] = "leu",
1723 [TCG_COND_GTU] = "gtu"
1724};
1725
f713d6ad
RH		 1726static const char * const ldst_name[] =
1727{
1728 [MO_UB] = "ub",
1729 [MO_SB] = "sb",
1730 [MO_LEUW] = "leuw",
1731 [MO_LESW] = "lesw",
1732 [MO_LEUL] = "leul",
1733 [MO_LESL] = "lesl",
1734 [MO_LEQ] = "leq",
1735 [MO_BEUW] = "beuw",
1736 [MO_BESW] = "besw",
1737 [MO_BEUL] = "beul",
1738 [MO_BESL] = "besl",
1739 [MO_BEQ] = "beq",
1740};
1741
1f00b27f 1742static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 1743#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 1744 [MO_UNALN >> MO_ASHIFT] = "un+",
1745 [MO_ALIGN >> MO_ASHIFT] = "",
1746#else
1747 [MO_UNALN >> MO_ASHIFT] = "",
1748 [MO_ALIGN >> MO_ASHIFT] = "al+",
1749#endif
1750 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1751 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1752 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1753 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1754 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1755 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1756};
1757
b016486e
RH		 1758static inline bool tcg_regset_single(TCGRegSet d)
1759{
1760 return (d & (d - 1)) == 0;
1761}
1762
1763static inline TCGReg tcg_regset_first(TCGRegSet d)
1764{
1765 if (TCG_TARGET_NB_REGS <= 32) {
1766 return ctz32(d);
1767 } else {
1768 return ctz64(d);
1769 }
1770}
1771
1894f69a 1772static void tcg_dump_ops(TCGContext *s, bool have_prefs)
c896fe29 1773{
c896fe29 1774 char buf[128];
c45cb8bb 1775 TCGOp *op;
c45cb8bb 1776
15fa08f8 1777 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 1778 int i, k, nb_oargs, nb_iargs, nb_cargs;
1779 const TCGOpDef *def;
c45cb8bb 1780 TCGOpcode c;
bdfb460e 1781 int col = 0;
c896fe29 1782
c45cb8bb 1783 c = op->opc;
c896fe29 1784 def = &tcg_op_defs[c];
c45cb8bb 1785
765b842a 1786 if (c == INDEX_op_insn_start) {
b016486e 1787 nb_oargs = 0;
15fa08f8 1788 col += qemu_log("\n ----");
9aef40ed
RH		 1789
1790 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1791 target_ulong a;
7e4597d7 1792#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 1793 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 1794#else
efee3746 1795 a = op->args[i];
7e4597d7 1796#endif
bdfb460e 1797 col += qemu_log(" " TARGET_FMT_lx, a);
eeacee4d 1798 }
7e4597d7 1799 } else if (c == INDEX_op_call) {
3e92aa34 1800 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 1801 void *func = tcg_call_func(op);
3e92aa34 1802
c896fe29 1803 /* variable number of arguments */
cd9090aa
RH		 1804 nb_oargs = TCGOP_CALLO(op);
1805 nb_iargs = TCGOP_CALLI(op);
c896fe29 1806 nb_cargs = def->nb_cargs;
c896fe29 1807
3e92aa34
RH		 1808 col += qemu_log(" %s ", def->name);
1809
1810 /*
1811 * Print the function name from TCGHelperInfo, if available.
1812 * Note that plugins have a template function for the info,
1813 * but the actual function pointer comes from the plugin.
1814 */
3e92aa34
RH		 1815 if (func == info->func) {
1816 col += qemu_log("%s", info->name);
1817 } else {
1818 col += qemu_log("plugin(%p)", func);
1819 }
1820
1821 col += qemu_log("$0x%x,$%d", info->flags, nb_oargs);
cf066674 1822 for (i = 0; i < nb_oargs; i++) {
43439139
RH		 1823 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1824 op->args[i]));
b03cce8e 1825 }
cf066674 1826 for (i = 0; i < nb_iargs; i++) {
efee3746 1827 TCGArg arg = op->args[nb_oargs + i];
cf066674
RH		 1828 const char *t = "<dummy>";
1829 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 1830 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
eeacee4d 1831 }
bdfb460e 1832 col += qemu_log(",%s", t);
e8996ee0 1833 }
b03cce8e 1834 } else {
bdfb460e 1835 col += qemu_log(" %s ", def->name);
c45cb8bb
RH		 1836
1837 nb_oargs = def->nb_oargs;
1838 nb_iargs = def->nb_iargs;
1839 nb_cargs = def->nb_cargs;
1840
d2fd745f
RH		 1841 if (def->flags & TCG_OPF_VECTOR) {
1842 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op),
1843 8 << TCGOP_VECE(op));
1844 }
1845
b03cce8e 1846 k = 0;
c45cb8bb 1847 for (i = 0; i < nb_oargs; i++) {
eeacee4d 1848 if (k != 0) {
bdfb460e 1849 col += qemu_log(",");
eeacee4d 1850 }
43439139
RH		 1851 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1852 op->args[k++]));
b03cce8e 1853 }
c45cb8bb 1854 for (i = 0; i < nb_iargs; i++) {
eeacee4d 1855 if (k != 0) {
bdfb460e 1856 col += qemu_log(",");
eeacee4d 1857 }
43439139
RH		 1858 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1859 op->args[k++]));
b03cce8e 1860 }
be210acb
RH		 1861 switch (c) {
1862 case INDEX_op_brcond_i32:
be210acb 1863 case INDEX_op_setcond_i32:
ffc5ea09 1864 case INDEX_op_movcond_i32:
ffc5ea09 1865 case INDEX_op_brcond2_i32:
be210acb 1866 case INDEX_op_setcond2_i32:
ffc5ea09 1867 case INDEX_op_brcond_i64:
be210acb 1868 case INDEX_op_setcond_i64:
ffc5ea09 1869 case INDEX_op_movcond_i64:
212be173 1870 case INDEX_op_cmp_vec:
f75da298 1871 case INDEX_op_cmpsel_vec:
efee3746
RH		 1872 if (op->args[k] < ARRAY_SIZE(cond_name)
1873 && cond_name[op->args[k]]) {
1874 col += qemu_log(",%s", cond_name[op->args[k++]]);
eeacee4d 1875 } else {
efee3746 1876 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 1877 }
f48f3ede 1878 i = 1;
be210acb 1879 break;
f713d6ad
RH		 1880 case INDEX_op_qemu_ld_i32:
1881 case INDEX_op_qemu_st_i32:
07ce0b05 1882 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 1883 case INDEX_op_qemu_ld_i64:
1884 case INDEX_op_qemu_st_i64:
59227d5d 1885 {
efee3746 1886 TCGMemOpIdx oi = op->args[k++];
14776ab5 1887 MemOp op = get_memop(oi);
59227d5d
RH		 1888 unsigned ix = get_mmuidx(oi);
1889
59c4b7e8 1890 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
bdfb460e 1891 col += qemu_log(",$0x%x,%u", op, ix);
59c4b7e8 1892 } else {
1f00b27f
SS		 1893 const char *s_al, *s_op;
1894 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 1895 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
bdfb460e 1896 col += qemu_log(",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 1897 }
1898 i = 1;
f713d6ad 1899 }
f713d6ad 1900 break;
be210acb 1901 default:
f48f3ede 1902 i = 0;
be210acb
RH		 1903 break;
1904 }
51e3972c
RH		 1905 switch (c) {
1906 case INDEX_op_set_label:
1907 case INDEX_op_br:
1908 case INDEX_op_brcond_i32:
1909 case INDEX_op_brcond_i64:
1910 case INDEX_op_brcond2_i32:
efee3746
RH		 1911 col += qemu_log("%s$L%d", k ? "," : "",
1912 arg_label(op->args[k])->id);
51e3972c
RH		 1913 i++, k++;
1914 break;
1915 default:
1916 break;
1917 }
1918 for (; i < nb_cargs; i++, k++) {
efee3746 1919 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]);
bdfb460e
RH		 1920 }
1921 }
bdfb460e 1922
1894f69a 1923 if (have_prefs || op->life) {
7606488c
RF		 1924
1925 QemuLogFile *logfile;
1926
1927 rcu_read_lock();
d73415a3 1928 logfile = qatomic_rcu_read(&qemu_logfile);
7606488c
RF		 1929 if (logfile) {
1930 for (; col < 40; ++col) {
1931 putc(' ', logfile->fd);
1932 }
bdfb460e 1933 }
7606488c 1934 rcu_read_unlock();
1894f69a
RH		 1935 }
1936
1937 if (op->life) {
1938 unsigned life = op->life;
bdfb460e
RH		 1939
1940 if (life & (SYNC_ARG * 3)) {
1941 qemu_log(" sync:");
1942 for (i = 0; i < 2; ++i) {
1943 if (life & (SYNC_ARG << i)) {
1944 qemu_log(" %d", i);
1945 }
1946 }
1947 }
1948 life /= DEAD_ARG;
1949 if (life) {
1950 qemu_log(" dead:");
1951 for (i = 0; life; ++i, life >>= 1) {
1952 if (life & 1) {
1953 qemu_log(" %d", i);
1954 }
1955 }
b03cce8e 1956 }
c896fe29 1957 }
1894f69a
RH		 1958
1959 if (have_prefs) {
1960 for (i = 0; i < nb_oargs; ++i) {
1961 TCGRegSet set = op->output_pref[i];
1962
1963 if (i == 0) {
1964 qemu_log(" pref=");
1965 } else {
1966 qemu_log(",");
1967 }
1968 if (set == 0) {
1969 qemu_log("none");
1970 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
1971 qemu_log("all");
1972#ifdef CONFIG_DEBUG_TCG
1973 } else if (tcg_regset_single(set)) {
1974 TCGReg reg = tcg_regset_first(set);
1975 qemu_log("%s", tcg_target_reg_names[reg]);
1976#endif
1977 } else if (TCG_TARGET_NB_REGS <= 32) {
1978 qemu_log("%#x", (uint32_t)set);
1979 } else {
1980 qemu_log("%#" PRIx64, (uint64_t)set);
1981 }
1982 }
1983 }
1984
eeacee4d 1985 qemu_log("\n");
c896fe29
FB		 1986 }
1987}
1988
1989/* we give more priority to constraints with less registers */
1990static int get_constraint_priority(const TCGOpDef *def, int k)
1991{
74a11790
RH		 1992 const TCGArgConstraint *arg_ct = &def->args_ct[k];
1993 int n;
c896fe29 1994
bc2b17e6 1995 if (arg_ct->oalias) {
c896fe29
FB		 1996 /* an alias is equivalent to a single register */
1997 n = 1;
1998 } else {
74a11790 1999 n = ctpop64(arg_ct->regs);
c896fe29
FB		 2000 }
2001 return TCG_TARGET_NB_REGS - n + 1;
2002}
2003
2004/* sort from highest priority to lowest */
2005static void sort_constraints(TCGOpDef *def, int start, int n)
2006{
66792f90
RH		 2007 int i, j;
2008 TCGArgConstraint *a = def->args_ct;
c896fe29 2009
66792f90
RH		 2010 for (i = 0; i < n; i++) {
2011 a[start + i].sort_index = start + i;
2012 }
2013 if (n <= 1) {
c896fe29 2014 return;
66792f90
RH		 2015 }
2016 for (i = 0; i < n - 1; i++) {
2017 for (j = i + 1; j < n; j++) {
2018 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2019 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2020 if (p1 < p2) {
66792f90
RH		 2021 int tmp = a[start + i].sort_index;
2022 a[start + i].sort_index = a[start + j].sort_index;
2023 a[start + j].sort_index = tmp;
c896fe29
FB		 2024 }
2025 }
2026 }
2027}
2028
f69d277e 2029static void process_op_defs(TCGContext *s)
c896fe29 2030{
a9751609 2031 TCGOpcode op;
c896fe29 2032
f69d277e
RH		 2033 for (op = 0; op < NB_OPS; op++) {
2034 TCGOpDef *def = &tcg_op_defs[op];
2035 const TCGTargetOpDef *tdefs;
069ea736 2036 int i, nb_args;
f69d277e
RH		 2037
2038 if (def->flags & TCG_OPF_NOT_PRESENT) {
2039 continue;
2040 }
2041
c896fe29 2042 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2043 if (nb_args == 0) {
2044 continue;
2045 }
2046
4c22e840
RH		 2047 /*
2048 * Macro magic should make it impossible, but double-check that
2049 * the array index is in range. Since the signness of an enum
2050 * is implementation defined, force the result to unsigned.
2051 */
2052 unsigned con_set = tcg_target_op_def(op);
2053 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2054 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2055
2056 for (i = 0; i < nb_args; i++) {
2057 const char *ct_str = tdefs->args_ct_str[i];
2058 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2059 tcg_debug_assert(ct_str != NULL);
f69d277e 2060
17280ff4
RH		 2061 while (*ct_str != '\0') {
2062 switch(*ct_str) {
2063 case '0' ... '9':
2064 {
2065 int oarg = *ct_str - '0';
2066 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2067 tcg_debug_assert(oarg < def->nb_oargs);
74a11790 2068 tcg_debug_assert(def->args_ct[oarg].regs != 0);
17280ff4 2069 def->args_ct[i] = def->args_ct[oarg];
bc2b17e6
RH		 2070 /* The output sets oalias. */
2071 def->args_ct[oarg].oalias = true;
17280ff4 2072 def->args_ct[oarg].alias_index = i;
bc2b17e6
RH		 2073 /* The input sets ialias. */
2074 def->args_ct[i].ialias = true;
17280ff4 2075 def->args_ct[i].alias_index = oarg;
c896fe29 2076 }
17280ff4
RH		 2077 ct_str++;
2078 break;
2079 case '&':
bc2b17e6 2080 def->args_ct[i].newreg = true;
17280ff4
RH		 2081 ct_str++;
2082 break;
2083 case 'i':
2084 def->args_ct[i].ct |= TCG_CT_CONST;
2085 ct_str++;
2086 break;
358b4923 2087
358b4923
RH		 2088 /* Include all of the target-specific constraints. */
2089
2090#undef CONST
2091#define CONST(CASE, MASK) \
2092 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2093#define REGS(CASE, MASK) \
2094 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2095
2096#include "tcg-target-con-str.h"
2097
2098#undef REGS
2099#undef CONST
17280ff4 2100 default:
17280ff4 2101 /* Typo in TCGTargetOpDef constraint. */
358b4923 2102 g_assert_not_reached();
c896fe29
FB		 2103 }
2104 }
2105 }
2106
c68aaa18 2107 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2108 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2109
c896fe29
FB		 2110 /* sort the constraints (XXX: this is just an heuristic) */
2111 sort_constraints(def, 0, def->nb_oargs);
2112 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2113 }
c896fe29
FB		 2114}
2115
0c627cdc
RH		 2116void tcg_op_remove(TCGContext *s, TCGOp *op)
2117{
d88a117e
RH		 2118 TCGLabel *label;
2119
2120 switch (op->opc) {
2121 case INDEX_op_br:
2122 label = arg_label(op->args[0]);
2123 label->refs--;
2124 break;
2125 case INDEX_op_brcond_i32:
2126 case INDEX_op_brcond_i64:
2127 label = arg_label(op->args[3]);
2128 label->refs--;
2129 break;
2130 case INDEX_op_brcond2_i32:
2131 label = arg_label(op->args[5]);
2132 label->refs--;
2133 break;
2134 default:
2135 break;
2136 }
2137
15fa08f8
RH		 2138 QTAILQ_REMOVE(&s->ops, op, link);
2139 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2140 s->nb_ops--;
0c627cdc
RH		 2141
2142#ifdef CONFIG_PROFILER
d73415a3 2143 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2144#endif
2145}
2146
a80cdd31
RH		 2147void tcg_remove_ops_after(TCGOp *op)
2148{
2149 TCGContext *s = tcg_ctx;
2150
2151 while (true) {
2152 TCGOp *last = tcg_last_op();
2153 if (last == op) {
2154 return;
2155 }
2156 tcg_op_remove(s, last);
2157 }
2158}
2159
15fa08f8 2160static TCGOp *tcg_op_alloc(TCGOpcode opc)
5a18407f 2161{
15fa08f8
RH		 2162 TCGContext *s = tcg_ctx;
2163 TCGOp *op;
5a18407f 2164
15fa08f8
RH		 2165 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2166 op = tcg_malloc(sizeof(TCGOp));
2167 } else {
2168 op = QTAILQ_FIRST(&s->free_ops);
2169 QTAILQ_REMOVE(&s->free_ops, op, link);
2170 }
2171 memset(op, 0, offsetof(TCGOp, link));
2172 op->opc = opc;
abebf925 2173 s->nb_ops++;
5a18407f 2174
15fa08f8
RH		 2175 return op;
2176}
2177
2178TCGOp *tcg_emit_op(TCGOpcode opc)
2179{
2180 TCGOp *op = tcg_op_alloc(opc);
2181 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2182 return op;
2183}
5a18407f 2184
ac1043f6 2185TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
15fa08f8
RH		 2186{
2187 TCGOp *new_op = tcg_op_alloc(opc);
2188 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2189 return new_op;
2190}
2191
ac1043f6 2192TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
5a18407f 2193{
15fa08f8
RH		 2194 TCGOp *new_op = tcg_op_alloc(opc);
2195 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2196 return new_op;
2197}
2198
b4fc67c7
RH		 2199/* Reachable analysis : remove unreachable code. */
2200static void reachable_code_pass(TCGContext *s)
2201{
2202 TCGOp *op, *op_next;
2203 bool dead = false;
2204
2205 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2206 bool remove = dead;
2207 TCGLabel *label;
b4fc67c7
RH		 2208
2209 switch (op->opc) {
2210 case INDEX_op_set_label:
2211 label = arg_label(op->args[0]);
2212 if (label->refs == 0) {
2213 /*
2214 * While there is an occasional backward branch, virtually
2215 * all branches generated by the translators are forward.
2216 * Which means that generally we will have already removed
2217 * all references to the label that will be, and there is
2218 * little to be gained by iterating.
2219 */
2220 remove = true;
2221 } else {
2222 /* Once we see a label, insns become live again. */
2223 dead = false;
2224 remove = false;
2225
2226 /*
2227 * Optimization can fold conditional branches to unconditional.
2228 * If we find a label with one reference which is preceded by
2229 * an unconditional branch to it, remove both. This needed to
2230 * wait until the dead code in between them was removed.
2231 */
2232 if (label->refs == 1) {
eae3eb3e 2233 TCGOp *op_prev = QTAILQ_PREV(op, link);
b4fc67c7
RH		 2234 if (op_prev->opc == INDEX_op_br &&
2235 label == arg_label(op_prev->args[0])) {
2236 tcg_op_remove(s, op_prev);
2237 remove = true;
2238 }
2239 }
2240 }
2241 break;
2242
2243 case INDEX_op_br:
2244 case INDEX_op_exit_tb:
2245 case INDEX_op_goto_ptr:
2246 /* Unconditional branches; everything following is dead. */
2247 dead = true;
2248 break;
2249
2250 case INDEX_op_call:
2251 /* Notice noreturn helper calls, raising exceptions. */
90163900 2252 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2253 dead = true;
2254 }
2255 break;
2256
2257 case INDEX_op_insn_start:
2258 /* Never remove -- we need to keep these for unwind. */
2259 remove = false;
2260 break;
2261
2262 default:
2263 break;
2264 }
2265
2266 if (remove) {
2267 tcg_op_remove(s, op);
2268 }
2269 }
2270}
2271
c70fbf0a
RH		 2272#define TS_DEAD 1
2273#define TS_MEM 2
2274
5a18407f
RH		 2275#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2276#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2277
25f49c5f
RH		 2278/* For liveness_pass_1, the register preferences for a given temp. */
2279static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2280{
2281 return ts->state_ptr;
2282}
2283
2284/* For liveness_pass_1, reset the preferences for a given temp to the
2285 * maximal regset for its type.
2286 */
2287static inline void la_reset_pref(TCGTemp *ts)
2288{
2289 *la_temp_pref(ts)
2290 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2291}
2292
9c43b68d
AJ		 2293/* liveness analysis: end of function: all temps are dead, and globals
2294 should be in memory. */
2616c808 2295static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2296{
b83eabea
RH		 2297 int i;
2298
2299 for (i = 0; i < ng; ++i) {
2300 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2301 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2302 }
2303 for (i = ng; i < nt; ++i) {
2304 s->temps[i].state = TS_DEAD;
25f49c5f 2305 la_reset_pref(&s->temps[i]);
b83eabea 2306 }
c896fe29
FB		 2307}
2308
9c43b68d
AJ		 2309/* liveness analysis: end of basic block: all temps are dead, globals
2310 and local temps should be in memory. */
2616c808 2311static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2312{
b83eabea 2313 int i;
641d5fbe 2314
ee17db83
RH		 2315 for (i = 0; i < nt; ++i) {
2316 TCGTemp *ts = &s->temps[i];
2317 int state;
2318
2319 switch (ts->kind) {
2320 case TEMP_FIXED:
2321 case TEMP_GLOBAL:
2322 case TEMP_LOCAL:
2323 state = TS_DEAD | TS_MEM;
2324 break;
2325 case TEMP_NORMAL:
c0522136 2326 case TEMP_CONST:
ee17db83
RH		 2327 state = TS_DEAD;
2328 break;
2329 default:
2330 g_assert_not_reached();
2331 }
2332 ts->state = state;
2333 la_reset_pref(ts);
641d5fbe
FB		 2334 }
2335}
2336
f65a061c
RH		 2337/* liveness analysis: sync globals back to memory. */
2338static void la_global_sync(TCGContext *s, int ng)
2339{
2340 int i;
2341
2342 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2343 int state = s->temps[i].state;
2344 s->temps[i].state = state | TS_MEM;
2345 if (state == TS_DEAD) {
2346 /* If the global was previously dead, reset prefs. */
2347 la_reset_pref(&s->temps[i]);
2348 }
f65a061c
RH		 2349 }
2350}
2351
b4cb76e6
RH		 2352/*
2353 * liveness analysis: conditional branch: all temps are dead,
2354 * globals and local temps should be synced.
2355 */
2356static void la_bb_sync(TCGContext *s, int ng, int nt)
2357{
2358 la_global_sync(s, ng);
2359
2360 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2361 TCGTemp *ts = &s->temps[i];
2362 int state;
2363
2364 switch (ts->kind) {
2365 case TEMP_LOCAL:
2366 state = ts->state;
2367 ts->state = state | TS_MEM;
b4cb76e6
RH		 2368 if (state != TS_DEAD) {
2369 continue;
2370 }
c0522136
RH		 2371 break;
2372 case TEMP_NORMAL:
b4cb76e6 2373 s->temps[i].state = TS_DEAD;
c0522136
RH		 2374 break;
2375 case TEMP_CONST:
2376 continue;
2377 default:
2378 g_assert_not_reached();
b4cb76e6
RH		 2379 }
2380 la_reset_pref(&s->temps[i]);
2381 }
2382}
2383
f65a061c
RH		 2384/* liveness analysis: sync globals back to memory and kill. */
2385static void la_global_kill(TCGContext *s, int ng)
2386{
2387 int i;
2388
2389 for (i = 0; i < ng; i++) {
2390 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2391 la_reset_pref(&s->temps[i]);
2392 }
2393}
2394
2395/* liveness analysis: note live globals crossing calls. */
2396static void la_cross_call(TCGContext *s, int nt)
2397{
2398 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2399 int i;
2400
2401 for (i = 0; i < nt; i++) {
2402 TCGTemp *ts = &s->temps[i];
2403 if (!(ts->state & TS_DEAD)) {
2404 TCGRegSet *pset = la_temp_pref(ts);
2405 TCGRegSet set = *pset;
2406
2407 set &= mask;
2408 /* If the combination is not possible, restart. */
2409 if (set == 0) {
2410 set = tcg_target_available_regs[ts->type] & mask;
2411 }
2412 *pset = set;
2413 }
f65a061c
RH		 2414 }
2415}
2416
a1b3c48d 2417/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2418 given input arguments is dead. Instructions updating dead
2419 temporaries are removed. */
b83eabea 2420static void liveness_pass_1(TCGContext *s)
c896fe29 2421{
c70fbf0a 2422 int nb_globals = s->nb_globals;
2616c808 2423 int nb_temps = s->nb_temps;
15fa08f8 2424 TCGOp *op, *op_prev;
25f49c5f
RH		 2425 TCGRegSet *prefs;
2426 int i;
2427
2428 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2429 for (i = 0; i < nb_temps; ++i) {
2430 s->temps[i].state_ptr = prefs + i;
2431 }
a1b3c48d 2432
ae36a246 2433 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2434 la_func_end(s, nb_globals, nb_temps);
c896fe29 2435
eae3eb3e 2436 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2437 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2438 TCGOpcode opc_new, opc_new2;
2439 bool have_opc_new2;
a1b3c48d 2440 TCGLifeData arg_life = 0;
25f49c5f 2441 TCGTemp *ts;
c45cb8bb
RH		 2442 TCGOpcode opc = op->opc;
2443 const TCGOpDef *def = &tcg_op_defs[opc];
2444
c45cb8bb 2445 switch (opc) {
c896fe29 2446 case INDEX_op_call:
c6e113f5
FB		 2447 {
2448 int call_flags;
25f49c5f 2449 int nb_call_regs;
c896fe29 2450
cd9090aa
RH		 2451 nb_oargs = TCGOP_CALLO(op);
2452 nb_iargs = TCGOP_CALLI(op);
90163900 2453 call_flags = tcg_call_flags(op);
c6e113f5 2454
c45cb8bb 2455 /* pure functions can be removed if their result is unused */
78505279 2456 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2457 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2458 ts = arg_temp(op->args[i]);
2459 if (ts->state != TS_DEAD) {
c6e113f5 2460 goto do_not_remove_call;
9c43b68d 2461 }
c6e113f5 2462 }
c45cb8bb 2463 goto do_remove;
152c35aa
RH		 2464 }
2465 do_not_remove_call:
c896fe29 2466
25f49c5f 2467 /* Output args are dead. */
152c35aa 2468 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2469 ts = arg_temp(op->args[i]);
2470 if (ts->state & TS_DEAD) {
152c35aa
RH		 2471 arg_life |= DEAD_ARG << i;
2472 }
25f49c5f 2473 if (ts->state & TS_MEM) {
152c35aa 2474 arg_life |= SYNC_ARG << i;
c6e113f5 2475 }
25f49c5f
RH		 2476 ts->state = TS_DEAD;
2477 la_reset_pref(ts);
2478
2479 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2480 op->output_pref[i] = 0;
152c35aa 2481 }
78505279 2482
152c35aa
RH		 2483 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2484 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2485 la_global_kill(s, nb_globals);
152c35aa 2486 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2487 la_global_sync(s, nb_globals);
152c35aa 2488 }
b9c18f56 2489
25f49c5f 2490 /* Record arguments that die in this helper. */
152c35aa 2491 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f
RH		 2492 ts = arg_temp(op->args[i]);
2493 if (ts && ts->state & TS_DEAD) {
152c35aa 2494 arg_life |= DEAD_ARG << i;
c6e113f5 2495 }
152c35aa 2496 }
25f49c5f
RH		 2497
2498 /* For all live registers, remove call-clobbered prefs. */
2499 la_cross_call(s, nb_temps);
2500
2501 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2502
2503 /* Input arguments are live for preceding opcodes. */
2504 for (i = 0; i < nb_iargs; i++) {
2505 ts = arg_temp(op->args[i + nb_oargs]);
2506 if (ts && ts->state & TS_DEAD) {
2507 /* For those arguments that die, and will be allocated
2508 * in registers, clear the register set for that arg,
2509 * to be filled in below. For args that will be on
2510 * the stack, reset to any available reg.
2511 */
2512 *la_temp_pref(ts)
2513 = (i < nb_call_regs ? 0 :
2514 tcg_target_available_regs[ts->type]);
2515 ts->state &= ~TS_DEAD;
2516 }
2517 }
2518
2519 /* For each input argument, add its input register to prefs.
2520 If a temp is used once, this produces a single set bit. */
2521 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2522 ts = arg_temp(op->args[i + nb_oargs]);
2523 if (ts) {
2524 tcg_regset_set_reg(*la_temp_pref(ts),
2525 tcg_target_call_iarg_regs[i]);
c19f47bf 2526 }
c896fe29 2527 }
c896fe29 2528 }
c896fe29 2529 break;
765b842a 2530 case INDEX_op_insn_start:
c896fe29 2531 break;
5ff9d6a4 2532 case INDEX_op_discard:
5ff9d6a4 2533 /* mark the temporary as dead */
25f49c5f
RH		 2534 ts = arg_temp(op->args[0]);
2535 ts->state = TS_DEAD;
2536 la_reset_pref(ts);
5ff9d6a4 2537 break;
1305c451
RH		 2538
2539 case INDEX_op_add2_i32:
c45cb8bb 2540 opc_new = INDEX_op_add_i32;
f1fae40c 2541 goto do_addsub2;
1305c451 2542 case INDEX_op_sub2_i32:
c45cb8bb 2543 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 2544 goto do_addsub2;
2545 case INDEX_op_add2_i64:
c45cb8bb 2546 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 2547 goto do_addsub2;
2548 case INDEX_op_sub2_i64:
c45cb8bb 2549 opc_new = INDEX_op_sub_i64;
f1fae40c 2550 do_addsub2:
1305c451
RH		 2551 nb_iargs = 4;
2552 nb_oargs = 2;
2553 /* Test if the high part of the operation is dead, but not
2554 the low part. The result can be optimized to a simple
2555 add or sub. This happens often for x86_64 guest when the
2556 cpu mode is set to 32 bit. */
b83eabea
RH		 2557 if (arg_temp(op->args[1])->state == TS_DEAD) {
2558 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 2559 goto do_remove;
2560 }
c45cb8bb
RH		 2561 /* Replace the opcode and adjust the args in place,
2562 leaving 3 unused args at the end. */
2563 op->opc = opc = opc_new;
efee3746
RH		 2564 op->args[1] = op->args[2];
2565 op->args[2] = op->args[4];
1305c451
RH		 2566 /* Fall through and mark the single-word operation live. */
2567 nb_iargs = 2;
2568 nb_oargs = 1;
2569 }
2570 goto do_not_remove;
2571
1414968a 2572 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 2573 opc_new = INDEX_op_mul_i32;
2574 opc_new2 = INDEX_op_muluh_i32;
2575 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 2576 goto do_mul2;
f1fae40c 2577 case INDEX_op_muls2_i32:
c45cb8bb
RH		 2578 opc_new = INDEX_op_mul_i32;
2579 opc_new2 = INDEX_op_mulsh_i32;
2580 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 2581 goto do_mul2;
2582 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 2583 opc_new = INDEX_op_mul_i64;
2584 opc_new2 = INDEX_op_muluh_i64;
2585 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 2586 goto do_mul2;
f1fae40c 2587 case INDEX_op_muls2_i64:
c45cb8bb
RH		 2588 opc_new = INDEX_op_mul_i64;
2589 opc_new2 = INDEX_op_mulsh_i64;
2590 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 2591 goto do_mul2;
f1fae40c 2592 do_mul2:
1414968a
RH		 2593 nb_iargs = 2;
2594 nb_oargs = 2;
b83eabea
RH		 2595 if (arg_temp(op->args[1])->state == TS_DEAD) {
2596 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 2597 /* Both parts of the operation are dead. */
1414968a
RH		 2598 goto do_remove;
2599 }
03271524 2600 /* The high part of the operation is dead; generate the low. */
c45cb8bb 2601 op->opc = opc = opc_new;
efee3746
RH		 2602 op->args[1] = op->args[2];
2603 op->args[2] = op->args[3];
b83eabea 2604 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 2605 /* The low part of the operation is dead; generate the high. */
2606 op->opc = opc = opc_new2;
efee3746
RH		 2607 op->args[0] = op->args[1];
2608 op->args[1] = op->args[2];
2609 op->args[2] = op->args[3];
03271524
RH		 2610 } else {
2611 goto do_not_remove;
1414968a 2612 }
03271524
RH		 2613 /* Mark the single-word operation live. */
2614 nb_oargs = 1;
1414968a
RH		 2615 goto do_not_remove;
2616
c896fe29 2617 default:
1305c451 2618 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 2619 nb_iargs = def->nb_iargs;
2620 nb_oargs = def->nb_oargs;
c896fe29 2621
49516bc0
AJ		 2622 /* Test if the operation can be removed because all
2623 its outputs are dead. We assume that nb_oargs == 0
2624 implies side effects */
2625 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 2626 for (i = 0; i < nb_oargs; i++) {
b83eabea 2627 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 2628 goto do_not_remove;
9c43b68d 2629 }
49516bc0 2630 }
152c35aa
RH		 2631 goto do_remove;
2632 }
2633 goto do_not_remove;
49516bc0 2634
152c35aa
RH		 2635 do_remove:
2636 tcg_op_remove(s, op);
2637 break;
2638
2639 do_not_remove:
152c35aa 2640 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2641 ts = arg_temp(op->args[i]);
2642
2643 /* Remember the preference of the uses that followed. */
2644 op->output_pref[i] = *la_temp_pref(ts);
2645
2646 /* Output args are dead. */
2647 if (ts->state & TS_DEAD) {
152c35aa 2648 arg_life |= DEAD_ARG << i;
49516bc0 2649 }
25f49c5f 2650 if (ts->state & TS_MEM) {
152c35aa
RH		 2651 arg_life |= SYNC_ARG << i;
2652 }
25f49c5f
RH		 2653 ts->state = TS_DEAD;
2654 la_reset_pref(ts);
152c35aa 2655 }
49516bc0 2656
25f49c5f 2657 /* If end of basic block, update. */
ae36a246
RH		 2658 if (def->flags & TCG_OPF_BB_EXIT) {
2659 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 2660 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2661 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 2662 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 2663 la_bb_end(s, nb_globals, nb_temps);
152c35aa 2664 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 2665 la_global_sync(s, nb_globals);
25f49c5f
RH		 2666 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2667 la_cross_call(s, nb_temps);
2668 }
152c35aa
RH		 2669 }
2670
25f49c5f 2671 /* Record arguments that die in this opcode. */
152c35aa 2672 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2673 ts = arg_temp(op->args[i]);
2674 if (ts->state & TS_DEAD) {
152c35aa 2675 arg_life |= DEAD_ARG << i;
c896fe29 2676 }
c896fe29 2677 }
25f49c5f
RH		 2678
2679 /* Input arguments are live for preceding opcodes. */
152c35aa 2680 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2681 ts = arg_temp(op->args[i]);
2682 if (ts->state & TS_DEAD) {
2683 /* For operands that were dead, initially allow
2684 all regs for the type. */
2685 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2686 ts->state &= ~TS_DEAD;
2687 }
2688 }
2689
2690 /* Incorporate constraints for this operand. */
2691 switch (opc) {
2692 case INDEX_op_mov_i32:
2693 case INDEX_op_mov_i64:
2694 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2695 have proper constraints. That said, special case
2696 moves to propagate preferences backward. */
2697 if (IS_DEAD_ARG(1)) {
2698 *la_temp_pref(arg_temp(op->args[0]))
2699 = *la_temp_pref(arg_temp(op->args[1]));
2700 }
2701 break;
2702
2703 default:
2704 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2705 const TCGArgConstraint *ct = &def->args_ct[i];
2706 TCGRegSet set, *pset;
2707
2708 ts = arg_temp(op->args[i]);
2709 pset = la_temp_pref(ts);
2710 set = *pset;
2711
9be0d080 2712 set &= ct->regs;
bc2b17e6 2713 if (ct->ialias) {
25f49c5f
RH		 2714 set &= op->output_pref[ct->alias_index];
2715 }
2716 /* If the combination is not possible, restart. */
2717 if (set == 0) {
9be0d080 2718 set = ct->regs;
25f49c5f
RH		 2719 }
2720 *pset = set;
2721 }
2722 break;
152c35aa 2723 }
c896fe29
FB		 2724 break;
2725 }
bee158cb 2726 op->life = arg_life;
1ff0a2c5 2727 }
c896fe29 2728}
c896fe29 2729
5a18407f 2730/* Liveness analysis: Convert indirect regs to direct temporaries. */
b83eabea 2731static bool liveness_pass_2(TCGContext *s)
5a18407f
RH		 2732{
2733 int nb_globals = s->nb_globals;
15fa08f8 2734 int nb_temps, i;
5a18407f 2735 bool changes = false;
15fa08f8 2736 TCGOp *op, *op_next;
5a18407f 2737
5a18407f
RH		 2738 /* Create a temporary for each indirect global. */
2739 for (i = 0; i < nb_globals; ++i) {
2740 TCGTemp *its = &s->temps[i];
2741 if (its->indirect_reg) {
2742 TCGTemp *dts = tcg_temp_alloc(s);
2743 dts->type = its->type;
2744 dts->base_type = its->base_type;
b83eabea
RH		 2745 its->state_ptr = dts;
2746 } else {
2747 its->state_ptr = NULL;
5a18407f 2748 }
b83eabea
RH		 2749 /* All globals begin dead. */
2750 its->state = TS_DEAD;
2751 }
2752 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2753 TCGTemp *its = &s->temps[i];
2754 its->state_ptr = NULL;
2755 its->state = TS_DEAD;
5a18407f 2756 }
5a18407f 2757
15fa08f8 2758 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 2759 TCGOpcode opc = op->opc;
2760 const TCGOpDef *def = &tcg_op_defs[opc];
2761 TCGLifeData arg_life = op->life;
2762 int nb_iargs, nb_oargs, call_flags;
b83eabea 2763 TCGTemp *arg_ts, *dir_ts;
5a18407f 2764
5a18407f 2765 if (opc == INDEX_op_call) {
cd9090aa
RH		 2766 nb_oargs = TCGOP_CALLO(op);
2767 nb_iargs = TCGOP_CALLI(op);
90163900 2768 call_flags = tcg_call_flags(op);
5a18407f
RH		 2769 } else {
2770 nb_iargs = def->nb_iargs;
2771 nb_oargs = def->nb_oargs;
2772
2773 /* Set flags similar to how calls require. */
b4cb76e6
RH		 2774 if (def->flags & TCG_OPF_COND_BRANCH) {
2775 /* Like reading globals: sync_globals */
2776 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2777 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 2778 /* Like writing globals: save_globals */
2779 call_flags = 0;
2780 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2781 /* Like reading globals: sync_globals */
2782 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2783 } else {
2784 /* No effect on globals. */
2785 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2786 TCG_CALL_NO_WRITE_GLOBALS);
2787 }
2788 }
2789
2790 /* Make sure that input arguments are available. */
2791 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2792 arg_ts = arg_temp(op->args[i]);
2793 if (arg_ts) {
2794 dir_ts = arg_ts->state_ptr;
2795 if (dir_ts && arg_ts->state == TS_DEAD) {
2796 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
5a18407f
RH		 2797 ? INDEX_op_ld_i32
2798 : INDEX_op_ld_i64);
ac1043f6 2799 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
5a18407f 2800
b83eabea
RH		 2801 lop->args[0] = temp_arg(dir_ts);
2802 lop->args[1] = temp_arg(arg_ts->mem_base);
2803 lop->args[2] = arg_ts->mem_offset;
5a18407f
RH		 2804
2805 /* Loaded, but synced with memory. */
b83eabea 2806 arg_ts->state = TS_MEM;
5a18407f
RH		 2807 }
2808 }
2809 }
2810
2811 /* Perform input replacement, and mark inputs that became dead.
2812 No action is required except keeping temp_state up to date
2813 so that we reload when needed. */
2814 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2815 arg_ts = arg_temp(op->args[i]);
2816 if (arg_ts) {
2817 dir_ts = arg_ts->state_ptr;
2818 if (dir_ts) {
2819 op->args[i] = temp_arg(dir_ts);
5a18407f
RH		 2820 changes = true;
2821 if (IS_DEAD_ARG(i)) {
b83eabea 2822 arg_ts->state = TS_DEAD;
5a18407f
RH		 2823 }
2824 }
2825 }
2826 }
2827
2828 /* Liveness analysis should ensure that the following are
2829 all correct, for call sites and basic block end points. */
2830 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2831 /* Nothing to do */
2832 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2833 for (i = 0; i < nb_globals; ++i) {
2834 /* Liveness should see that globals are synced back,
2835 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 2836 arg_ts = &s->temps[i];
2837 tcg_debug_assert(arg_ts->state_ptr == 0
2838 || arg_ts->state != 0);
5a18407f
RH		 2839 }
2840 } else {
2841 for (i = 0; i < nb_globals; ++i) {
2842 /* Liveness should see that globals are saved back,
2843 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 2844 arg_ts = &s->temps[i];
2845 tcg_debug_assert(arg_ts->state_ptr == 0
2846 || arg_ts->state == TS_DEAD);
5a18407f
RH		 2847 }
2848 }
2849
2850 /* Outputs become available. */
61f15c48
RH		 2851 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2852 arg_ts = arg_temp(op->args[0]);
b83eabea 2853 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 2854 if (dir_ts) {
2855 op->args[0] = temp_arg(dir_ts);
2856 changes = true;
2857
2858 /* The output is now live and modified. */
2859 arg_ts->state = 0;
2860
2861 if (NEED_SYNC_ARG(0)) {
2862 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2863 ? INDEX_op_st_i32
2864 : INDEX_op_st_i64);
2865 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2866 TCGTemp *out_ts = dir_ts;
2867
2868 if (IS_DEAD_ARG(0)) {
2869 out_ts = arg_temp(op->args[1]);
2870 arg_ts->state = TS_DEAD;
2871 tcg_op_remove(s, op);
2872 } else {
2873 arg_ts->state = TS_MEM;
2874 }
2875
2876 sop->args[0] = temp_arg(out_ts);
2877 sop->args[1] = temp_arg(arg_ts->mem_base);
2878 sop->args[2] = arg_ts->mem_offset;
2879 } else {
2880 tcg_debug_assert(!IS_DEAD_ARG(0));
2881 }
5a18407f 2882 }
61f15c48
RH		 2883 } else {
2884 for (i = 0; i < nb_oargs; i++) {
2885 arg_ts = arg_temp(op->args[i]);
2886 dir_ts = arg_ts->state_ptr;
2887 if (!dir_ts) {
2888 continue;
2889 }
2890 op->args[i] = temp_arg(dir_ts);
2891 changes = true;
5a18407f 2892
61f15c48
RH		 2893 /* The output is now live and modified. */
2894 arg_ts->state = 0;
5a18407f 2895
61f15c48
RH		 2896 /* Sync outputs upon their last write. */
2897 if (NEED_SYNC_ARG(i)) {
2898 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2899 ? INDEX_op_st_i32
2900 : INDEX_op_st_i64);
2901 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
5a18407f 2902
61f15c48
RH		 2903 sop->args[0] = temp_arg(dir_ts);
2904 sop->args[1] = temp_arg(arg_ts->mem_base);
2905 sop->args[2] = arg_ts->mem_offset;
5a18407f 2906
61f15c48
RH		 2907 arg_ts->state = TS_MEM;
2908 }
2909 /* Drop outputs that are dead. */
2910 if (IS_DEAD_ARG(i)) {
2911 arg_ts->state = TS_DEAD;
2912 }
5a18407f
RH		 2913 }
2914 }
2915 }
2916
2917 return changes;
2918}
2919
8d8fdbae 2920#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 2921static void dump_regs(TCGContext *s)
2922{
2923 TCGTemp *ts;
2924 int i;
2925 char buf[64];
2926
2927 for(i = 0; i < s->nb_temps; i++) {
2928 ts = &s->temps[i];
43439139 2929 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
c896fe29
FB		 2930 switch(ts->val_type) {
2931 case TEMP_VAL_REG:
2932 printf("%s", tcg_target_reg_names[ts->reg]);
2933 break;
2934 case TEMP_VAL_MEM:
b3a62939
RH		 2935 printf("%d(%s)", (int)ts->mem_offset,
2936 tcg_target_reg_names[ts->mem_base->reg]);
c896fe29
FB		 2937 break;
2938 case TEMP_VAL_CONST:
bdb38b95 2939 printf("$0x%" PRIx64, ts->val);
c896fe29
FB		 2940 break;
2941 case TEMP_VAL_DEAD:
2942 printf("D");
2943 break;
2944 default:
2945 printf("???");
2946 break;
2947 }
2948 printf("\n");
2949 }
2950
2951 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
f8b2f202 2952 if (s->reg_to_temp[i] != NULL) {
c896fe29
FB		 2953 printf("%s: %s\n",
2954 tcg_target_reg_names[i],
f8b2f202 2955 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
c896fe29
FB		 2956 }
2957 }
2958}
2959
2960static void check_regs(TCGContext *s)
2961{
869938ae 2962 int reg;
b6638662 2963 int k;
c896fe29
FB		 2964 TCGTemp *ts;
2965 char buf[64];
2966
f8b2f202
RH		 2967 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
2968 ts = s->reg_to_temp[reg];
2969 if (ts != NULL) {
2970 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
c896fe29
FB		 2971 printf("Inconsistency for register %s:\n",
2972 tcg_target_reg_names[reg]);
b03cce8e 2973 goto fail;
c896fe29
FB		 2974 }
2975 }
2976 }
f8b2f202 2977 for (k = 0; k < s->nb_temps; k++) {
c896fe29 2978 ts = &s->temps[k];
ee17db83
RH		 2979 if (ts->val_type == TEMP_VAL_REG
2980 && ts->kind != TEMP_FIXED
f8b2f202
RH		 2981 && s->reg_to_temp[ts->reg] != ts) {
2982 printf("Inconsistency for temp %s:\n",
2983 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
b03cce8e 2984 fail:
f8b2f202
RH		 2985 printf("reg state:\n");
2986 dump_regs(s);
2987 tcg_abort();
c896fe29
FB		 2988 }
2989 }
2990}
2991#endif
2992
2272e4a7 2993static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 2994{
9b9c37c3
RH		 2995#if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
2996 /* Sparc64 stack is accessed with offset of 2047 */
b591dc59
BS		 2997 s->current_frame_offset = (s->current_frame_offset +
2998 (tcg_target_long)sizeof(tcg_target_long) - 1) &
2999 ~(sizeof(tcg_target_long) - 1);
f44c9960 3000#endif
b591dc59
BS		 3001 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) >
3002 s->frame_end) {
5ff9d6a4 3003 tcg_abort();
b591dc59 3004 }
c896fe29 3005 ts->mem_offset = s->current_frame_offset;
b3a62939 3006 ts->mem_base = s->frame_temp;
c896fe29 3007 ts->mem_allocated = 1;
e2c6d1b4 3008 s->current_frame_offset += sizeof(tcg_target_long);
c896fe29
FB		 3009}
3010
b722452a 3011static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3012
59d7c14e
RH		 3013/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3014 mark it free; otherwise mark it dead. */
3015static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3016{
c0522136
RH		 3017 TCGTempVal new_type;
3018
3019 switch (ts->kind) {
3020 case TEMP_FIXED:
59d7c14e 3021 return;
c0522136
RH		 3022 case TEMP_GLOBAL:
3023 case TEMP_LOCAL:
3024 new_type = TEMP_VAL_MEM;
3025 break;
3026 case TEMP_NORMAL:
3027 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3028 break;
3029 case TEMP_CONST:
3030 new_type = TEMP_VAL_CONST;
3031 break;
3032 default:
3033 g_assert_not_reached();
59d7c14e
RH		 3034 }
3035 if (ts->val_type == TEMP_VAL_REG) {
3036 s->reg_to_temp[ts->reg] = NULL;
3037 }
c0522136 3038 ts->val_type = new_type;
59d7c14e 3039}
7f6ceedf 3040
59d7c14e
RH		 3041/* Mark a temporary as dead. */
3042static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3043{
3044 temp_free_or_dead(s, ts, 1);
3045}
3046
3047/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3048 registers needs to be allocated to store a constant. If 'free_or_dead'
3049 is non-zero, subsequently release the temporary; if it is positive, the
3050 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3051static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3052 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3053{
c0522136 3054 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3055 if (!ts->mem_allocated) {
2272e4a7 3056 temp_allocate_frame(s, ts);
59d7c14e 3057 }
59d7c14e
RH		 3058 switch (ts->val_type) {
3059 case TEMP_VAL_CONST:
3060 /* If we're going to free the temp immediately, then we won't
3061 require it later in a register, so attempt to store the
3062 constant to memory directly. */
3063 if (free_or_dead
3064 && tcg_out_sti(s, ts->type, ts->val,
3065 ts->mem_base->reg, ts->mem_offset)) {
3066 break;
3067 }
3068 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3069 allocated_regs, preferred_regs);
59d7c14e
RH		 3070 /* fallthrough */
3071
3072 case TEMP_VAL_REG:
3073 tcg_out_st(s, ts->type, ts->reg,
3074 ts->mem_base->reg, ts->mem_offset);
3075 break;
3076
3077 case TEMP_VAL_MEM:
3078 break;
3079
3080 case TEMP_VAL_DEAD:
3081 default:
3082 tcg_abort();
3083 }
3084 ts->mem_coherent = 1;
3085 }
3086 if (free_or_dead) {
3087 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3088 }
7f6ceedf
AJ		 3089}
3090
c896fe29 3091/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3092static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3093{
f8b2f202 3094 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3095 if (ts != NULL) {
98b4e186 3096 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3097 }
3098}
3099
b016486e
RH		 3100/**
3101 * tcg_reg_alloc:
3102 * @required_regs: Set of registers in which we must allocate.
3103 * @allocated_regs: Set of registers which must be avoided.
3104 * @preferred_regs: Set of registers we should prefer.
3105 * @rev: True if we search the registers in "indirect" order.
3106 *
3107 * The allocated register must be in @required_regs & ~@allocated_regs,
3108 * but if we can put it in @preferred_regs we may save a move later.
3109 */
3110static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3111 TCGRegSet allocated_regs,
3112 TCGRegSet preferred_regs, bool rev)
c896fe29 3113{
b016486e
RH		 3114 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3115 TCGRegSet reg_ct[2];
91478cef 3116 const int *order;
c896fe29 3117
b016486e
RH		 3118 reg_ct[1] = required_regs & ~allocated_regs;
3119 tcg_debug_assert(reg_ct[1] != 0);
3120 reg_ct[0] = reg_ct[1] & preferred_regs;
3121
3122 /* Skip the preferred_regs option if it cannot be satisfied,
3123 or if the preference made no difference. */
3124 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3125
91478cef 3126 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3127
b016486e
RH		 3128 /* Try free registers, preferences first. */
3129 for (j = f; j < 2; j++) {
3130 TCGRegSet set = reg_ct[j];
3131
3132 if (tcg_regset_single(set)) {
3133 /* One register in the set. */
3134 TCGReg reg = tcg_regset_first(set);
3135 if (s->reg_to_temp[reg] == NULL) {
3136 return reg;
3137 }
3138 } else {
3139 for (i = 0; i < n; i++) {
3140 TCGReg reg = order[i];
3141 if (s->reg_to_temp[reg] == NULL &&
3142 tcg_regset_test_reg(set, reg)) {
3143 return reg;
3144 }
3145 }
3146 }
c896fe29
FB		 3147 }
3148
b016486e
RH		 3149 /* We must spill something. */
3150 for (j = f; j < 2; j++) {
3151 TCGRegSet set = reg_ct[j];
3152
3153 if (tcg_regset_single(set)) {
3154 /* One register in the set. */
3155 TCGReg reg = tcg_regset_first(set);
b3915dbb 3156 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3157 return reg;
b016486e
RH		 3158 } else {
3159 for (i = 0; i < n; i++) {
3160 TCGReg reg = order[i];
3161 if (tcg_regset_test_reg(set, reg)) {
3162 tcg_reg_free(s, reg, allocated_regs);
3163 return reg;
3164 }
3165 }
c896fe29
FB		 3166 }
3167 }
3168
3169 tcg_abort();
3170}
3171
40ae5c62
RH		 3172/* Make sure the temporary is in a register. If needed, allocate the register
3173 from DESIRED while avoiding ALLOCATED. */
3174static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3175 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3176{
3177 TCGReg reg;
3178
3179 switch (ts->val_type) {
3180 case TEMP_VAL_REG:
3181 return;
3182 case TEMP_VAL_CONST:
b016486e 3183 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3184 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3185 if (ts->type <= TCG_TYPE_I64) {
3186 tcg_out_movi(s, ts->type, reg, ts->val);
3187 } else {
4e186175
RH		 3188 uint64_t val = ts->val;
3189 MemOp vece = MO_64;
3190
3191 /*
3192 * Find the minimal vector element that matches the constant.
3193 * The targets will, in general, have to do this search anyway,
3194 * do this generically.
3195 */
4e186175
RH		 3196 if (val == dup_const(MO_8, val)) {
3197 vece = MO_8;
3198 } else if (val == dup_const(MO_16, val)) {
3199 vece = MO_16;
0b4286dd 3200 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3201 vece = MO_32;
3202 }
3203
3204 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3205 }
40ae5c62
RH		 3206 ts->mem_coherent = 0;
3207 break;
3208 case TEMP_VAL_MEM:
b016486e 3209 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3210 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3211 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3212 ts->mem_coherent = 1;
3213 break;
3214 case TEMP_VAL_DEAD:
3215 default:
3216 tcg_abort();
3217 }
3218 ts->reg = reg;
3219 ts->val_type = TEMP_VAL_REG;
3220 s->reg_to_temp[reg] = ts;
3221}
3222
59d7c14e
RH		 3223/* Save a temporary to memory. 'allocated_regs' is used in case a
3224 temporary registers needs to be allocated to store a constant. */
3225static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3226{
5a18407f
RH		 3227 /* The liveness analysis already ensures that globals are back
3228 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3229 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3230}
3231
9814dd27 3232/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3233 modified be the following code. 'allocated_regs' is used in case a
3234 temporary registers needs to be allocated to store a constant. */
3235static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3236{
ac3b8891 3237 int i, n;
c896fe29 3238
ac3b8891 3239 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3240 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3241 }
e5097dc8
FB		 3242}
3243
3d5c5f87
AJ		 3244/* sync globals to their canonical location and assume they can be
3245 read by the following code. 'allocated_regs' is used in case a
3246 temporary registers needs to be allocated to store a constant. */
3247static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3248{
ac3b8891 3249 int i, n;
3d5c5f87 3250
ac3b8891 3251 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3252 TCGTemp *ts = &s->temps[i];
5a18407f 3253 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3254 || ts->kind == TEMP_FIXED
5a18407f 3255 || ts->mem_coherent);
3d5c5f87
AJ		 3256 }
3257}
3258
e5097dc8 3259/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3260 all globals are stored at their canonical location. */
3261static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3262{
e5097dc8
FB		 3263 int i;
3264
b13eb728
RH		 3265 for (i = s->nb_globals; i < s->nb_temps; i++) {
3266 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3267
3268 switch (ts->kind) {
3269 case TEMP_LOCAL:
b13eb728 3270 temp_save(s, ts, allocated_regs);
c0522136
RH		 3271 break;
3272 case TEMP_NORMAL:
5a18407f
RH		 3273 /* The liveness analysis already ensures that temps are dead.
3274 Keep an tcg_debug_assert for safety. */
3275 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3276 break;
3277 case TEMP_CONST:
3278 /* Similarly, we should have freed any allocated register. */
3279 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3280 break;
3281 default:
3282 g_assert_not_reached();
c896fe29
FB		 3283 }
3284 }
e8996ee0
FB		 3285
3286 save_globals(s, allocated_regs);
c896fe29
FB		 3287}
3288
b4cb76e6
RH		 3289/*
3290 * At a conditional branch, we assume all temporaries are dead and
3291 * all globals and local temps are synced to their location.
3292 */
3293static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3294{
3295 sync_globals(s, allocated_regs);
3296
3297 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3298 TCGTemp *ts = &s->temps[i];
3299 /*
3300 * The liveness analysis already ensures that temps are dead.
3301 * Keep tcg_debug_asserts for safety.
3302 */
c0522136
RH		 3303 switch (ts->kind) {
3304 case TEMP_LOCAL:
b4cb76e6 3305 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136
RH		 3306 break;
3307 case TEMP_NORMAL:
b4cb76e6 3308 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3309 break;
3310 case TEMP_CONST:
3311 break;
3312 default:
3313 g_assert_not_reached();
b4cb76e6
RH		 3314 }
3315 }
3316}
3317
bab1671f 3318/*
c58f4c97 3319 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3320 */
0fe4fca4 3321static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3322 tcg_target_ulong val, TCGLifeData arg_life,
3323 TCGRegSet preferred_regs)
e8996ee0 3324{
d63e3b6e 3325 /* ENV should not be modified. */
e01fa97d 3326 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3327
3328 /* The movi is not explicitly generated here. */
3329 if (ots->val_type == TEMP_VAL_REG) {
3330 s->reg_to_temp[ots->reg] = NULL;
ec7a869d 3331 }
59d7c14e
RH		 3332 ots->val_type = TEMP_VAL_CONST;
3333 ots->val = val;
3334 ots->mem_coherent = 0;
3335 if (NEED_SYNC_ARG(0)) {
ba87719c 3336 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3337 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3338 temp_dead(s, ots);
4c4e1ab2 3339 }
e8996ee0
FB		 3340}
3341
bab1671f
RH		 3342/*
3343 * Specialized code generation for INDEX_op_mov_*.
3344 */
dd186292 3345static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3346{
dd186292 3347 const TCGLifeData arg_life = op->life;
69e3706d 3348 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3349 TCGTemp *ts, *ots;
450445d5 3350 TCGType otype, itype;
c896fe29 3351
d21369f5 3352 allocated_regs = s->reserved_regs;
69e3706d 3353 preferred_regs = op->output_pref[0];
43439139
RH		 3354 ots = arg_temp(op->args[0]);
3355 ts = arg_temp(op->args[1]);
450445d5 3356
d63e3b6e 3357 /* ENV should not be modified. */
e01fa97d 3358 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3359
450445d5
RH		 3360 /* Note that otype != itype for no-op truncation. */
3361 otype = ots->type;
3362 itype = ts->type;
c29c1d7e 3363
0fe4fca4
PB		 3364 if (ts->val_type == TEMP_VAL_CONST) {
3365 /* propagate constant or generate sti */
3366 tcg_target_ulong val = ts->val;
3367 if (IS_DEAD_ARG(1)) {
3368 temp_dead(s, ts);
3369 }
69e3706d 3370 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3371 return;
3372 }
3373
3374 /* If the source value is in memory we're going to be forced
3375 to have it in a register in order to perform the copy. Copy
3376 the SOURCE value into its own register first, that way we
3377 don't have to reload SOURCE the next time it is used. */
3378 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3379 temp_load(s, ts, tcg_target_available_regs[itype],
3380 allocated_regs, preferred_regs);
c29c1d7e 3381 }
c896fe29 3382
0fe4fca4 3383 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
d63e3b6e 3384 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3385 /* mov to a non-saved dead register makes no sense (even with
3386 liveness analysis disabled). */
eabb7b91 3387 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3388 if (!ots->mem_allocated) {
2272e4a7 3389 temp_allocate_frame(s, ots);
c29c1d7e 3390 }
b3a62939 3391 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3392 if (IS_DEAD_ARG(1)) {
f8bf00f1 3393 temp_dead(s, ts);
c29c1d7e 3394 }
f8bf00f1 3395 temp_dead(s, ots);
c29c1d7e 3396 } else {
ee17db83 3397 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
c896fe29 3398 /* the mov can be suppressed */
c29c1d7e 3399 if (ots->val_type == TEMP_VAL_REG) {
f8b2f202 3400 s->reg_to_temp[ots->reg] = NULL;
c29c1d7e
AJ		 3401 }
3402 ots->reg = ts->reg;
f8bf00f1 3403 temp_dead(s, ts);
c896fe29 3404 } else {
c29c1d7e
AJ		 3405 if (ots->val_type != TEMP_VAL_REG) {
3406 /* When allocating a new register, make sure to not spill the
3407 input one. */
3408 tcg_regset_set_reg(allocated_regs, ts->reg);
450445d5 3409 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
69e3706d 3410 allocated_regs, preferred_regs,
b016486e 3411 ots->indirect_base);
c896fe29 3412 }
78113e83 3413 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) {
240c08d0
RH		 3414 /*
3415 * Cross register class move not supported.
3416 * Store the source register into the destination slot
3417 * and leave the destination temp as TEMP_VAL_MEM.
3418 */
e01fa97d 3419 assert(!temp_readonly(ots));
240c08d0
RH		 3420 if (!ts->mem_allocated) {
3421 temp_allocate_frame(s, ots);
3422 }
3423 tcg_out_st(s, ts->type, ts->reg,
3424 ots->mem_base->reg, ots->mem_offset);
3425 ots->mem_coherent = 1;
3426 temp_free_or_dead(s, ots, -1);
3427 return;
78113e83 3428 }
c896fe29 3429 }
c29c1d7e
AJ		 3430 ots->val_type = TEMP_VAL_REG;
3431 ots->mem_coherent = 0;
f8b2f202 3432 s->reg_to_temp[ots->reg] = ots;
c29c1d7e 3433 if (NEED_SYNC_ARG(0)) {
98b4e186 3434 temp_sync(s, ots, allocated_regs, 0, 0);
c896fe29 3435 }
ec7a869d 3436 }
c896fe29
FB		 3437}
3438
bab1671f
RH		 3439/*
3440 * Specialized code generation for INDEX_op_dup_vec.
3441 */
3442static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3443{
3444 const TCGLifeData arg_life = op->life;
3445 TCGRegSet dup_out_regs, dup_in_regs;
3446 TCGTemp *its, *ots;
3447 TCGType itype, vtype;
d6ecb4a9 3448 intptr_t endian_fixup;
bab1671f
RH		 3449 unsigned vece;
3450 bool ok;
3451
3452 ots = arg_temp(op->args[0]);
3453 its = arg_temp(op->args[1]);
3454
3455 /* ENV should not be modified. */
e01fa97d 3456 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 3457
3458 itype = its->type;
3459 vece = TCGOP_VECE(op);
3460 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3461
3462 if (its->val_type == TEMP_VAL_CONST) {
3463 /* Propagate constant via movi -> dupi. */
3464 tcg_target_ulong val = its->val;
3465 if (IS_DEAD_ARG(1)) {
3466 temp_dead(s, its);
3467 }
3468 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3469 return;
3470 }
3471
9be0d080
RH		 3472 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3473 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 3474
3475 /* Allocate the output register now. */
3476 if (ots->val_type != TEMP_VAL_REG) {
3477 TCGRegSet allocated_regs = s->reserved_regs;
3478
3479 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3480 /* Make sure to not spill the input register. */
3481 tcg_regset_set_reg(allocated_regs, its->reg);
3482 }
3483 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3484 op->output_pref[0], ots->indirect_base);
3485 ots->val_type = TEMP_VAL_REG;
3486 ots->mem_coherent = 0;
3487 s->reg_to_temp[ots->reg] = ots;
3488 }
3489
3490 switch (its->val_type) {
3491 case TEMP_VAL_REG:
3492 /*
3493 * The dup constriaints must be broad, covering all possible VECE.
3494 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3495 * to fail, indicating that extra moves are required for that case.
3496 */
3497 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3498 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3499 goto done;
3500 }
3501 /* Try again from memory or a vector input register. */
3502 }
3503 if (!its->mem_coherent) {
3504 /*
3505 * The input register is not synced, and so an extra store
3506 * would be required to use memory. Attempt an integer-vector
3507 * register move first. We do not have a TCGRegSet for this.
3508 */
3509 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3510 break;
3511 }
3512 /* Sync the temp back to its slot and load from there. */
3513 temp_sync(s, its, s->reserved_regs, 0, 0);
3514 }
3515 /* fall through */
3516
3517 case TEMP_VAL_MEM:
d6ecb4a9
RH		 3518#ifdef HOST_WORDS_BIGENDIAN
3519 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3520 endian_fixup -= 1 << vece;
3521#else
3522 endian_fixup = 0;
3523#endif
3524 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3525 its->mem_offset + endian_fixup)) {
3526 goto done;
3527 }
bab1671f
RH		 3528 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3529 break;
3530
3531 default:
3532 g_assert_not_reached();
3533 }
3534
3535 /* We now have a vector input register, so dup must succeed. */
3536 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3537 tcg_debug_assert(ok);
3538
3539 done:
3540 if (IS_DEAD_ARG(1)) {
3541 temp_dead(s, its);
3542 }
3543 if (NEED_SYNC_ARG(0)) {
3544 temp_sync(s, ots, s->reserved_regs, 0, 0);
3545 }
3546 if (IS_DEAD_ARG(0)) {
3547 temp_dead(s, ots);
3548 }
3549}
3550
dd186292 3551static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 3552{
dd186292
RH		 3553 const TCGLifeData arg_life = op->life;
3554 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 3555 TCGRegSet i_allocated_regs;
3556 TCGRegSet o_allocated_regs;
b6638662
RH		 3557 int i, k, nb_iargs, nb_oargs;
3558 TCGReg reg;
c896fe29
FB		 3559 TCGArg arg;
3560 const TCGArgConstraint *arg_ct;
3561 TCGTemp *ts;
3562 TCGArg new_args[TCG_MAX_OP_ARGS];
3563 int const_args[TCG_MAX_OP_ARGS];
3564
3565 nb_oargs = def->nb_oargs;
3566 nb_iargs = def->nb_iargs;
3567
3568 /* copy constants */
3569 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 3570 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 3571 sizeof(TCGArg) * def->nb_cargs);
3572
d21369f5
RH		 3573 i_allocated_regs = s->reserved_regs;
3574 o_allocated_regs = s->reserved_regs;
82790a87 3575
c896fe29 3576 /* satisfy input constraints */
dd186292 3577 for (k = 0; k < nb_iargs; k++) {
d62816f2
RH		 3578 TCGRegSet i_preferred_regs, o_preferred_regs;
3579
66792f90 3580 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 3581 arg = op->args[i];
c896fe29 3582 arg_ct = &def->args_ct[i];
43439139 3583 ts = arg_temp(arg);
40ae5c62
RH		 3584
3585 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 3586 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 3587 /* constant is OK for instruction */
3588 const_args[i] = 1;
3589 new_args[i] = ts->val;
d62816f2 3590 continue;
c896fe29 3591 }
40ae5c62 3592
d62816f2 3593 i_preferred_regs = o_preferred_regs = 0;
bc2b17e6 3594 if (arg_ct->ialias) {
d62816f2 3595 o_preferred_regs = op->output_pref[arg_ct->alias_index];
d62816f2 3596
c0522136
RH		 3597 /*
3598 * If the input is readonly, then it cannot also be an
3599 * output and aliased to itself. If the input is not
3600 * dead after the instruction, we must allocate a new
3601 * register and move it.
3602 */
3603 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3604 goto allocate_in_reg;
3605 }
3606
3607 /*
3608 * Check if the current register has already been allocated
3609 * for another input aliased to an output.
3610 */
3611 if (ts->val_type == TEMP_VAL_REG) {
3612 reg = ts->reg;
3613 for (int k2 = 0; k2 < k; k2++) {
3614 int i2 = def->args_ct[nb_oargs + k2].sort_index;
3615 if (def->args_ct[i2].ialias && reg == new_args[i2]) {
3616 goto allocate_in_reg;
7e1df267
AJ		 3617 }
3618 }
5ff9d6a4 3619 }
c0522136 3620 i_preferred_regs = o_preferred_regs;
c896fe29 3621 }
d62816f2 3622
9be0d080 3623 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
c896fe29 3624 reg = ts->reg;
d62816f2 3625
c0522136
RH		 3626 if (!tcg_regset_test_reg(arg_ct->regs, reg)) {
3627 allocate_in_reg:
3628 /*
3629 * Allocate a new register matching the constraint
3630 * and move the temporary register into it.
3631 */
d62816f2
RH		 3632 temp_load(s, ts, tcg_target_available_regs[ts->type],
3633 i_allocated_regs, 0);
9be0d080 3634 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
d62816f2 3635 o_preferred_regs, ts->indirect_base);
78113e83 3636 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3637 /*
3638 * Cross register class move not supported. Sync the
3639 * temp back to its slot and load from there.
3640 */
3641 temp_sync(s, ts, i_allocated_regs, 0, 0);
3642 tcg_out_ld(s, ts->type, reg,
3643 ts->mem_base->reg, ts->mem_offset);
78113e83 3644 }
c896fe29 3645 }
c896fe29
FB		 3646 new_args[i] = reg;
3647 const_args[i] = 0;
82790a87 3648 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29
FB		 3649 }
3650
a52ad07e
AJ		 3651 /* mark dead temporaries and free the associated registers */
3652 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3653 if (IS_DEAD_ARG(i)) {
43439139 3654 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 3655 }
3656 }
3657
b4cb76e6
RH		 3658 if (def->flags & TCG_OPF_COND_BRANCH) {
3659 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3660 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 3661 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 3662 } else {
e8996ee0
FB		 3663 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3664 /* XXX: permit generic clobber register list ? */
c8074023
RH		 3665 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3666 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 3667 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 3668 }
c896fe29 3669 }
3d5c5f87
AJ		 3670 }
3671 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3672 /* sync globals if the op has side effects and might trigger
3673 an exception. */
82790a87 3674 sync_globals(s, i_allocated_regs);
c896fe29 3675 }
e8996ee0
FB		 3676
3677 /* satisfy the output constraints */
e8996ee0 3678 for(k = 0; k < nb_oargs; k++) {
66792f90 3679 i = def->args_ct[k].sort_index;
dd186292 3680 arg = op->args[i];
e8996ee0 3681 arg_ct = &def->args_ct[i];
43439139 3682 ts = arg_temp(arg);
d63e3b6e
RH		 3683
3684 /* ENV should not be modified. */
e01fa97d 3685 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3686
bc2b17e6 3687 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
e8996ee0 3688 reg = new_args[arg_ct->alias_index];
bc2b17e6 3689 } else if (arg_ct->newreg) {
9be0d080 3690 reg = tcg_reg_alloc(s, arg_ct->regs,
82790a87 3691 i_allocated_regs | o_allocated_regs,
69e3706d 3692 op->output_pref[k], ts->indirect_base);
e8996ee0 3693 } else {
9be0d080 3694 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
69e3706d 3695 op->output_pref[k], ts->indirect_base);
c896fe29 3696 }
82790a87 3697 tcg_regset_set_reg(o_allocated_regs, reg);
d63e3b6e
RH		 3698 if (ts->val_type == TEMP_VAL_REG) {
3699 s->reg_to_temp[ts->reg] = NULL;
e8996ee0 3700 }
d63e3b6e
RH		 3701 ts->val_type = TEMP_VAL_REG;
3702 ts->reg = reg;
3703 /*
3704 * Temp value is modified, so the value kept in memory is
3705 * potentially not the same.
3706 */
3707 ts->mem_coherent = 0;
3708 s->reg_to_temp[reg] = ts;
e8996ee0 3709 new_args[i] = reg;
c896fe29 3710 }
c896fe29
FB		 3711 }
3712
c896fe29 3713 /* emit instruction */
d2fd745f
RH		 3714 if (def->flags & TCG_OPF_VECTOR) {
3715 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
3716 new_args, const_args);
3717 } else {
3718 tcg_out_op(s, op->opc, new_args, const_args);
3719 }
3720
c896fe29
FB		 3721 /* move the outputs in the correct register if needed */
3722 for(i = 0; i < nb_oargs; i++) {
43439139 3723 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 3724
3725 /* ENV should not be modified. */
e01fa97d 3726 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3727
ec7a869d 3728 if (NEED_SYNC_ARG(i)) {
98b4e186 3729 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 3730 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 3731 temp_dead(s, ts);
ec7a869d 3732 }
c896fe29
FB		 3733 }
3734}
3735
efe86b21
RH		 3736static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
3737{
3738 const TCGLifeData arg_life = op->life;
3739 TCGTemp *ots, *itsl, *itsh;
3740 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3741
3742 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
3743 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
3744 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
3745
3746 ots = arg_temp(op->args[0]);
3747 itsl = arg_temp(op->args[1]);
3748 itsh = arg_temp(op->args[2]);
3749
3750 /* ENV should not be modified. */
3751 tcg_debug_assert(!temp_readonly(ots));
3752
3753 /* Allocate the output register now. */
3754 if (ots->val_type != TEMP_VAL_REG) {
3755 TCGRegSet allocated_regs = s->reserved_regs;
3756 TCGRegSet dup_out_regs =
3757 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3758
3759 /* Make sure to not spill the input registers. */
3760 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
3761 tcg_regset_set_reg(allocated_regs, itsl->reg);
3762 }
3763 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
3764 tcg_regset_set_reg(allocated_regs, itsh->reg);
3765 }
3766
3767 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3768 op->output_pref[0], ots->indirect_base);
3769 ots->val_type = TEMP_VAL_REG;
3770 ots->mem_coherent = 0;
3771 s->reg_to_temp[ots->reg] = ots;
3772 }
3773
3774 /* Promote dup2 of immediates to dupi_vec. */
3775 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
3776 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
3777 MemOp vece = MO_64;
3778
3779 if (val == dup_const(MO_8, val)) {
3780 vece = MO_8;
3781 } else if (val == dup_const(MO_16, val)) {
3782 vece = MO_16;
3783 } else if (val == dup_const(MO_32, val)) {
3784 vece = MO_32;
3785 }
3786
3787 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
3788 goto done;
3789 }
3790
3791 /* If the two inputs form one 64-bit value, try dupm_vec. */
3792 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
3793 if (!itsl->mem_coherent) {
3794 temp_sync(s, itsl, s->reserved_regs, 0, 0);
3795 }
3796 if (!itsh->mem_coherent) {
3797 temp_sync(s, itsh, s->reserved_regs, 0, 0);
3798 }
3799#ifdef HOST_WORDS_BIGENDIAN
3800 TCGTemp *its = itsh;
3801#else
3802 TCGTemp *its = itsl;
3803#endif
3804 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
3805 its->mem_base->reg, its->mem_offset)) {
3806 goto done;
3807 }
3808 }
3809
3810 /* Fall back to generic expansion. */
3811 return false;
3812
3813 done:
3814 if (IS_DEAD_ARG(1)) {
3815 temp_dead(s, itsl);
3816 }
3817 if (IS_DEAD_ARG(2)) {
3818 temp_dead(s, itsh);
3819 }
3820 if (NEED_SYNC_ARG(0)) {
3821 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
3822 } else if (IS_DEAD_ARG(0)) {
3823 temp_dead(s, ots);
3824 }
3825 return true;
3826}
3827
b03cce8e
FB		 3828#ifdef TCG_TARGET_STACK_GROWSUP
3829#define STACK_DIR(x) (-(x))
3830#else
3831#define STACK_DIR(x) (x)
3832#endif
3833
dd186292 3834static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
c896fe29 3835{
cd9090aa
RH		 3836 const int nb_oargs = TCGOP_CALLO(op);
3837 const int nb_iargs = TCGOP_CALLI(op);
dd186292 3838 const TCGLifeData arg_life = op->life;
b6638662
RH		 3839 int flags, nb_regs, i;
3840 TCGReg reg;
cf066674 3841 TCGArg arg;
c896fe29 3842 TCGTemp *ts;
d3452f1f
RH		 3843 intptr_t stack_offset;
3844 size_t call_stack_size;
cf066674
RH		 3845 tcg_insn_unit *func_addr;
3846 int allocate_args;
c896fe29 3847 TCGRegSet allocated_regs;
c896fe29 3848
fa52e660 3849 func_addr = tcg_call_func(op);
90163900 3850 flags = tcg_call_flags(op);
c896fe29 3851
6e17d0c5 3852 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
c45cb8bb
RH		 3853 if (nb_regs > nb_iargs) {
3854 nb_regs = nb_iargs;
cf066674 3855 }
c896fe29
FB		 3856
3857 /* assign stack slots first */
c45cb8bb 3858 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
c896fe29
FB		 3859 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
3860 ~(TCG_TARGET_STACK_ALIGN - 1);
b03cce8e
FB		 3861 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
3862 if (allocate_args) {
345649c0
BS		 3863 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3864 preallocate call stack */
3865 tcg_abort();
b03cce8e 3866 }
39cf05d3
FB		 3867
3868 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
dd186292
RH		 3869 for (i = nb_regs; i < nb_iargs; i++) {
3870 arg = op->args[nb_oargs + i];
39cf05d3
FB		 3871#ifdef TCG_TARGET_STACK_GROWSUP
3872 stack_offset -= sizeof(tcg_target_long);
3873#endif
3874 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3875 ts = arg_temp(arg);
40ae5c62 3876 temp_load(s, ts, tcg_target_available_regs[ts->type],
b722452a 3877 s->reserved_regs, 0);
40ae5c62 3878 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
c896fe29 3879 }
39cf05d3
FB		 3880#ifndef TCG_TARGET_STACK_GROWSUP
3881 stack_offset += sizeof(tcg_target_long);
3882#endif
c896fe29
FB		 3883 }
3884
3885 /* assign input registers */
d21369f5 3886 allocated_regs = s->reserved_regs;
dd186292
RH		 3887 for (i = 0; i < nb_regs; i++) {
3888 arg = op->args[nb_oargs + i];
39cf05d3 3889 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3890 ts = arg_temp(arg);
39cf05d3 3891 reg = tcg_target_call_iarg_regs[i];
40ae5c62 3892
39cf05d3
FB		 3893 if (ts->val_type == TEMP_VAL_REG) {
3894 if (ts->reg != reg) {
4250da10 3895 tcg_reg_free(s, reg, allocated_regs);
78113e83 3896 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3897 /*
3898 * Cross register class move not supported. Sync the
3899 * temp back to its slot and load from there.
3900 */
3901 temp_sync(s, ts, allocated_regs, 0, 0);
3902 tcg_out_ld(s, ts->type, reg,
3903 ts->mem_base->reg, ts->mem_offset);
78113e83 3904 }
39cf05d3 3905 }
39cf05d3 3906 } else {
ccb1bb66 3907 TCGRegSet arg_set = 0;
40ae5c62 3908
4250da10 3909 tcg_reg_free(s, reg, allocated_regs);
40ae5c62 3910 tcg_regset_set_reg(arg_set, reg);
b722452a 3911 temp_load(s, ts, arg_set, allocated_regs, 0);
c896fe29 3912 }
40ae5c62 3913
39cf05d3 3914 tcg_regset_set_reg(allocated_regs, reg);
c896fe29 3915 }
c896fe29
FB		 3916 }
3917
c896fe29 3918 /* mark dead temporaries and free the associated registers */
dd186292 3919 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 3920 if (IS_DEAD_ARG(i)) {
43439139 3921 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 3922 }
3923 }
3924
3925 /* clobber call registers */
c8074023
RH		 3926 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3927 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 3928 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 3929 }
3930 }
78505279
AJ		 3931
3932 /* Save globals if they might be written by the helper, sync them if
3933 they might be read. */
3934 if (flags & TCG_CALL_NO_READ_GLOBALS) {
3935 /* Nothing to do */
3936 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
3937 sync_globals(s, allocated_regs);
3938 } else {
b9c18f56
AJ		 3939 save_globals(s, allocated_regs);
3940 }
c896fe29 3941
cf066674 3942 tcg_out_call(s, func_addr);
c896fe29
FB		 3943
3944 /* assign output registers and emit moves if needed */
3945 for(i = 0; i < nb_oargs; i++) {
dd186292 3946 arg = op->args[i];
43439139 3947 ts = arg_temp(arg);
d63e3b6e
RH		 3948
3949 /* ENV should not be modified. */
e01fa97d 3950 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3951
c896fe29 3952 reg = tcg_target_call_oarg_regs[i];
eabb7b91 3953 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
d63e3b6e
RH		 3954 if (ts->val_type == TEMP_VAL_REG) {
3955 s->reg_to_temp[ts->reg] = NULL;
3956 }
3957 ts->val_type = TEMP_VAL_REG;
3958 ts->reg = reg;
3959 ts->mem_coherent = 0;
3960 s->reg_to_temp[reg] = ts;
3961 if (NEED_SYNC_ARG(i)) {
3962 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
3963 } else if (IS_DEAD_ARG(i)) {
3964 temp_dead(s, ts);
c896fe29
FB		 3965 }
3966 }
c896fe29
FB		 3967}
3968
3969#ifdef CONFIG_PROFILER
3970
c3fac113
EC		 3971/* avoid copy/paste errors */
3972#define PROF_ADD(to, from, field) \
3973 do { \
d73415a3 3974 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 3975 } while (0)
3976
3977#define PROF_MAX(to, from, field) \
3978 do { \
d73415a3 3979 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 3980 if (val__ > (to)->field) { \
3981 (to)->field = val__; \
3982 } \
3983 } while (0)
3984
3985/* Pass in a zero'ed @prof */
3986static inline
3987void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
3988{
0e2d61cf 3989 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 3990 unsigned int i;
3991
3468b59e 3992 for (i = 0; i < n_ctxs; i++) {
d73415a3 3993 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 3994 const TCGProfile *orig = &s->prof;
c3fac113
EC		 3995
3996 if (counters) {
72fd2efb 3997 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 3998 PROF_ADD(prof, orig, tb_count1);
3999 PROF_ADD(prof, orig, tb_count);
4000 PROF_ADD(prof, orig, op_count);
4001 PROF_MAX(prof, orig, op_count_max);
4002 PROF_ADD(prof, orig, temp_count);
4003 PROF_MAX(prof, orig, temp_count_max);
4004 PROF_ADD(prof, orig, del_op_count);
4005 PROF_ADD(prof, orig, code_in_len);
4006 PROF_ADD(prof, orig, code_out_len);
4007 PROF_ADD(prof, orig, search_out_len);
4008 PROF_ADD(prof, orig, interm_time);
4009 PROF_ADD(prof, orig, code_time);
4010 PROF_ADD(prof, orig, la_time);
4011 PROF_ADD(prof, orig, opt_time);
4012 PROF_ADD(prof, orig, restore_count);
4013 PROF_ADD(prof, orig, restore_time);
4014 }
4015 if (table) {
4016 int i;
4017
4018 for (i = 0; i < NB_OPS; i++) {
4019 PROF_ADD(prof, orig, table_op_count[i]);
4020 }
4021 }
4022 }
4023}
4024
4025#undef PROF_ADD
4026#undef PROF_MAX
4027
4028static void tcg_profile_snapshot_counters(TCGProfile *prof)
4029{
4030 tcg_profile_snapshot(prof, true, false);
4031}
4032
4033static void tcg_profile_snapshot_table(TCGProfile *prof)
4034{
4035 tcg_profile_snapshot(prof, false, true);
4036}
c896fe29 4037
d4c51a0a 4038void tcg_dump_op_count(void)
c896fe29 4039{
c3fac113 4040 TCGProfile prof = {};
c896fe29 4041 int i;
d70724ce 4042
c3fac113 4043 tcg_profile_snapshot_table(&prof);
15fc7daa 4044 for (i = 0; i < NB_OPS; i++) {
d4c51a0a 4045 qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name,
c3fac113 4046 prof.table_op_count[i]);
c896fe29 4047 }
c896fe29 4048}
72fd2efb
EC		 4049
4050int64_t tcg_cpu_exec_time(void)
4051{
0e2d61cf 4052 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4053 unsigned int i;
4054 int64_t ret = 0;
4055
4056 for (i = 0; i < n_ctxs; i++) {
d73415a3 4057 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4058 const TCGProfile *prof = &s->prof;
4059
d73415a3 4060 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4061 }
4062 return ret;
4063}
246ae24d 4064#else
d4c51a0a 4065void tcg_dump_op_count(void)
246ae24d 4066{
d4c51a0a 4067 qemu_printf("[TCG profiler not compiled]\n");
246ae24d 4068}
72fd2efb
EC		 4069
4070int64_t tcg_cpu_exec_time(void)
4071{
4072 error_report("%s: TCG profiler not compiled", __func__);
4073 exit(EXIT_FAILURE);
4074}
c896fe29
FB		 4075#endif
4076
4077
5bd2ec3d 4078int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
c896fe29 4079{
c3fac113
EC		 4080#ifdef CONFIG_PROFILER
4081 TCGProfile *prof = &s->prof;
4082#endif
15fa08f8
RH		 4083 int i, num_insns;
4084 TCGOp *op;
c896fe29 4085
04fe6400
RH		 4086#ifdef CONFIG_PROFILER
4087 {
c1f543b7 4088 int n = 0;
04fe6400 4089
15fa08f8
RH		 4090 QTAILQ_FOREACH(op, &s->ops, link) {
4091 n++;
4092 }
d73415a3 4093 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4094 if (n > prof->op_count_max) {
d73415a3 4095 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4096 }
4097
4098 n = s->nb_temps;
d73415a3 4099 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4100 if (n > prof->temp_count_max) {
d73415a3 4101 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4102 }
4103 }
4104#endif
4105
c896fe29 4106#ifdef DEBUG_DISAS
d977e1c2
AB		 4107 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4108 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4109 FILE *logfile = qemu_log_lock();
93fcfe39 4110 qemu_log("OP:\n");
1894f69a 4111 tcg_dump_ops(s, false);
93fcfe39 4112 qemu_log("\n");
fc59d2d8 4113 qemu_log_unlock(logfile);
c896fe29
FB		 4114 }
4115#endif
4116
bef16ab4
RH		 4117#ifdef CONFIG_DEBUG_TCG
4118 /* Ensure all labels referenced have been emitted. */
4119 {
4120 TCGLabel *l;
4121 bool error = false;
4122
4123 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4124 if (unlikely(!l->present) && l->refs) {
4125 qemu_log_mask(CPU_LOG_TB_OP,
4126 "$L%d referenced but not present.\n", l->id);
4127 error = true;
4128 }
4129 }
4130 assert(!error);
4131 }
4132#endif
4133
c5cc28ff 4134#ifdef CONFIG_PROFILER
d73415a3 4135 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4136#endif
4137
8f2e8c07 4138#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4139 tcg_optimize(s);
8f2e8c07
KB		 4140#endif
4141
a23a9ec6 4142#ifdef CONFIG_PROFILER
d73415a3
SH		 4143 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4144 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4145#endif
c5cc28ff 4146
b4fc67c7 4147 reachable_code_pass(s);
b83eabea 4148 liveness_pass_1(s);
5a18407f 4149
b83eabea 4150 if (s->nb_indirects > 0) {
5a18407f 4151#ifdef DEBUG_DISAS
b83eabea
RH		 4152 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
4153 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4154 FILE *logfile = qemu_log_lock();
b83eabea 4155 qemu_log("OP before indirect lowering:\n");
1894f69a 4156 tcg_dump_ops(s, false);
b83eabea 4157 qemu_log("\n");
fc59d2d8 4158 qemu_log_unlock(logfile);
b83eabea 4159 }
5a18407f 4160#endif
b83eabea
RH		 4161 /* Replace indirect temps with direct temps. */
4162 if (liveness_pass_2(s)) {
4163 /* If changes were made, re-run liveness. */
4164 liveness_pass_1(s);
5a18407f
RH		 4165 }
4166 }
c5cc28ff 4167
a23a9ec6 4168#ifdef CONFIG_PROFILER
d73415a3 4169 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4170#endif
c896fe29
FB		 4171
4172#ifdef DEBUG_DISAS
d977e1c2
AB		 4173 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
4174 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4175 FILE *logfile = qemu_log_lock();
c5cc28ff 4176 qemu_log("OP after optimization and liveness analysis:\n");
1894f69a 4177 tcg_dump_ops(s, true);
93fcfe39 4178 qemu_log("\n");
fc59d2d8 4179 qemu_log_unlock(logfile);
c896fe29
FB		 4180 }
4181#endif
4182
4183 tcg_reg_alloc_start(s);
4184
db0c51a3
RH		 4185 /*
4186 * Reset the buffer pointers when restarting after overflow.
4187 * TODO: Move this into translate-all.c with the rest of the
4188 * buffer management. Having only this done here is confusing.
4189 */
4190 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4191 s->code_ptr = s->code_buf;
c896fe29 4192
659ef5cb 4193#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4194 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4195#endif
57a26946
RH		 4196#ifdef TCG_TARGET_NEED_POOL_LABELS
4197 s->pool_labels = NULL;
4198#endif
9ecefc84 4199
fca8a500 4200 num_insns = -1;
15fa08f8 4201 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4202 TCGOpcode opc = op->opc;
b3db8758 4203
c896fe29 4204#ifdef CONFIG_PROFILER
d73415a3 4205 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 4206#endif
c45cb8bb
RH		 4207
4208 switch (opc) {
c896fe29 4209 case INDEX_op_mov_i32:
c896fe29 4210 case INDEX_op_mov_i64:
d2fd745f 4211 case INDEX_op_mov_vec:
dd186292 4212 tcg_reg_alloc_mov(s, op);
c896fe29 4213 break;
bab1671f
RH		 4214 case INDEX_op_dup_vec:
4215 tcg_reg_alloc_dup(s, op);
4216 break;
765b842a 4217 case INDEX_op_insn_start:
fca8a500 4218 if (num_insns >= 0) {
9f754620
RH		 4219 size_t off = tcg_current_code_size(s);
4220 s->gen_insn_end_off[num_insns] = off;
4221 /* Assert that we do not overflow our stored offset. */
4222 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 4223 }
4224 num_insns++;
bad729e2
RH		 4225 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4226 target_ulong a;
4227#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 4228 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 4229#else
efee3746 4230 a = op->args[i];
bad729e2 4231#endif
fca8a500 4232 s->gen_insn_data[num_insns][i] = a;
bad729e2 4233 }
c896fe29 4234 break;
5ff9d6a4 4235 case INDEX_op_discard:
43439139 4236 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 4237 break;
c896fe29 4238 case INDEX_op_set_label:
e8996ee0 4239 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 4240 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 4241 break;
4242 case INDEX_op_call:
dd186292 4243 tcg_reg_alloc_call(s, op);
c45cb8bb 4244 break;
efe86b21
RH		 4245 case INDEX_op_dup2_vec:
4246 if (tcg_reg_alloc_dup2(s, op)) {
4247 break;
4248 }
4249 /* fall through */
c896fe29 4250 default:
25c4d9cc 4251 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 4252 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 4253 /* Note: in order to speed up the code, it would be much
4254 faster to have specialized register allocator functions for
4255 some common argument patterns */
dd186292 4256 tcg_reg_alloc_op(s, op);
c896fe29
FB		 4257 break;
4258 }
8d8fdbae 4259#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 4260 check_regs(s);
4261#endif
b125f9dc
RH		 4262 /* Test for (pending) buffer overflow. The assumption is that any
4263 one operation beginning below the high water mark cannot overrun
4264 the buffer completely. Thus we can test for overflow after
4265 generating code without having to check during generation. */
644da9b3 4266 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 4267 return -1;
4268 }
6e6c4efe
RH		 4269 /* Test for TB overflow, as seen by gen_insn_end_off. */
4270 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4271 return -2;
4272 }
c896fe29 4273 }
fca8a500
RH		 4274 tcg_debug_assert(num_insns >= 0);
4275 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 4276
b76f0d8c 4277 /* Generate TB finalization at the end of block */
659ef5cb 4278#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 4279 i = tcg_out_ldst_finalize(s);
4280 if (i < 0) {
4281 return i;
23dceda6 4282 }
659ef5cb 4283#endif
57a26946 4284#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 4285 i = tcg_out_pool_finalize(s);
4286 if (i < 0) {
4287 return i;
57a26946
RH		 4288 }
4289#endif
7ecd02a0
RH		 4290 if (!tcg_resolve_relocs(s)) {
4291 return -2;
4292 }
c896fe29 4293
df5d2b16 4294#ifndef CONFIG_TCG_INTERPRETER
c896fe29 4295 /* flush instruction cache */
db0c51a3
RH		 4296 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4297 (uintptr_t)s->code_buf,
1da8de39 4298 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 4299#endif
2aeabc08 4300
1813e175 4301 return tcg_current_code_size(s);
c896fe29
FB		 4302}
4303
a23a9ec6 4304#ifdef CONFIG_PROFILER
3de2faa9 4305void tcg_dump_info(void)
a23a9ec6 4306{
c3fac113
EC		 4307 TCGProfile prof = {};
4308 const TCGProfile *s;
4309 int64_t tb_count;
4310 int64_t tb_div_count;
4311 int64_t tot;
4312
4313 tcg_profile_snapshot_counters(&prof);
4314 s = &prof;
4315 tb_count = s->tb_count;
4316 tb_div_count = tb_count ? tb_count : 1;
4317 tot = s->interm_time + s->code_time;
a23a9ec6 4318
3de2faa9 4319 qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n",
a23a9ec6 4320 tot, tot / 2.4e9);
3de2faa9
MA		 4321 qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64
4322 " %0.1f%%)\n",
fca8a500
RH		 4323 tb_count, s->tb_count1 - tb_count,
4324 (double)(s->tb_count1 - s->tb_count)
4325 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
3de2faa9 4326 qemu_printf("avg ops/TB %0.1f max=%d\n",
fca8a500 4327 (double)s->op_count / tb_div_count, s->op_count_max);
3de2faa9 4328 qemu_printf("deleted ops/TB %0.2f\n",
fca8a500 4329 (double)s->del_op_count / tb_div_count);
3de2faa9 4330 qemu_printf("avg temps/TB %0.2f max=%d\n",
fca8a500 4331 (double)s->temp_count / tb_div_count, s->temp_count_max);
3de2faa9 4332 qemu_printf("avg host code/TB %0.1f\n",
fca8a500 4333 (double)s->code_out_len / tb_div_count);
3de2faa9 4334 qemu_printf("avg search data/TB %0.1f\n",
fca8a500 4335 (double)s->search_out_len / tb_div_count);
a23a9ec6 4336
3de2faa9 4337 qemu_printf("cycles/op %0.1f\n",
a23a9ec6 4338 s->op_count ? (double)tot / s->op_count : 0);
3de2faa9 4339 qemu_printf("cycles/in byte %0.1f\n",
a23a9ec6 4340 s->code_in_len ? (double)tot / s->code_in_len : 0);
3de2faa9 4341 qemu_printf("cycles/out byte %0.1f\n",
a23a9ec6 4342 s->code_out_len ? (double)tot / s->code_out_len : 0);
3de2faa9 4343 qemu_printf("cycles/search byte %0.1f\n",
fca8a500
RH		 4344 s->search_out_len ? (double)tot / s->search_out_len : 0);
4345 if (tot == 0) {
a23a9ec6 4346 tot = 1;
fca8a500 4347 }
3de2faa9 4348 qemu_printf(" gen_interm time %0.1f%%\n",
a23a9ec6 4349 (double)s->interm_time / tot * 100.0);
3de2faa9 4350 qemu_printf(" gen_code time %0.1f%%\n",
a23a9ec6 4351 (double)s->code_time / tot * 100.0);
3de2faa9 4352 qemu_printf("optim./code time %0.1f%%\n",
c5cc28ff
AJ		 4353 (double)s->opt_time / (s->code_time ? s->code_time : 1)
4354 * 100.0);
3de2faa9 4355 qemu_printf("liveness/code time %0.1f%%\n",
a23a9ec6 4356 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
3de2faa9 4357 qemu_printf("cpu_restore count %" PRId64 "\n",
a23a9ec6 4358 s->restore_count);
3de2faa9 4359 qemu_printf(" avg cycles %0.1f\n",
a23a9ec6 4360 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 4361}
4362#else
3de2faa9 4363void tcg_dump_info(void)
a23a9ec6 4364{
3de2faa9 4365 qemu_printf("[TCG profiler not compiled]\n");
a23a9ec6
FB		 4366}
4367#endif
813da627
RH		 4368
4369#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 4370/* In order to use this feature, the backend needs to do three things:
4371
4372 (1) Define ELF_HOST_MACHINE to indicate both what value to
4373 put into the ELF image and to indicate support for the feature.
4374
4375 (2) Define tcg_register_jit. This should create a buffer containing
4376 the contents of a .debug_frame section that describes the post-
4377 prologue unwind info for the tcg machine.
4378
4379 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4380*/
813da627
RH		 4381
4382/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4383typedef enum {
4384 JIT_NOACTION = 0,
4385 JIT_REGISTER_FN,
4386 JIT_UNREGISTER_FN
4387} jit_actions_t;
4388
4389struct jit_code_entry {
4390 struct jit_code_entry *next_entry;
4391 struct jit_code_entry *prev_entry;
4392 const void *symfile_addr;
4393 uint64_t symfile_size;
4394};
4395
4396struct jit_descriptor {
4397 uint32_t version;
4398 uint32_t action_flag;
4399 struct jit_code_entry *relevant_entry;
4400 struct jit_code_entry *first_entry;
4401};
4402
4403void __jit_debug_register_code(void) __attribute__((noinline));
4404void __jit_debug_register_code(void)
4405{
4406 asm("");
4407}
4408
4409/* Must statically initialize the version, because GDB may check
4410 the version before we can set it. */
4411struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4412
4413/* End GDB interface. */
4414
4415static int find_string(const char *strtab, const char *str)
4416{
4417 const char *p = strtab + 1;
4418
4419 while (1) {
4420 if (strcmp(p, str) == 0) {
4421 return p - strtab;
4422 }
4423 p += strlen(p) + 1;
4424 }
4425}
4426
755bf9e5 4427static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 4428 const void *debug_frame,
4429 size_t debug_frame_size)
813da627 4430{
5872bbf2
RH		 4431 struct __attribute__((packed)) DebugInfo {
4432 uint32_t len;
4433 uint16_t version;
4434 uint32_t abbrev;
4435 uint8_t ptr_size;
4436 uint8_t cu_die;
4437 uint16_t cu_lang;
4438 uintptr_t cu_low_pc;
4439 uintptr_t cu_high_pc;
4440 uint8_t fn_die;
4441 char fn_name[16];
4442 uintptr_t fn_low_pc;
4443 uintptr_t fn_high_pc;
4444 uint8_t cu_eoc;
4445 };
813da627
RH		 4446
4447 struct ElfImage {
4448 ElfW(Ehdr) ehdr;
4449 ElfW(Phdr) phdr;
5872bbf2
RH		 4450 ElfW(Shdr) shdr[7];
4451 ElfW(Sym) sym[2];
4452 struct DebugInfo di;
4453 uint8_t da[24];
4454 char str[80];
4455 };
4456
4457 struct ElfImage *img;
4458
4459 static const struct ElfImage img_template = {
4460 .ehdr = {
4461 .e_ident[EI_MAG0] = ELFMAG0,
4462 .e_ident[EI_MAG1] = ELFMAG1,
4463 .e_ident[EI_MAG2] = ELFMAG2,
4464 .e_ident[EI_MAG3] = ELFMAG3,
4465 .e_ident[EI_CLASS] = ELF_CLASS,
4466 .e_ident[EI_DATA] = ELF_DATA,
4467 .e_ident[EI_VERSION] = EV_CURRENT,
4468 .e_type = ET_EXEC,
4469 .e_machine = ELF_HOST_MACHINE,
4470 .e_version = EV_CURRENT,
4471 .e_phoff = offsetof(struct ElfImage, phdr),
4472 .e_shoff = offsetof(struct ElfImage, shdr),
4473 .e_ehsize = sizeof(ElfW(Shdr)),
4474 .e_phentsize = sizeof(ElfW(Phdr)),
4475 .e_phnum = 1,
4476 .e_shentsize = sizeof(ElfW(Shdr)),
4477 .e_shnum = ARRAY_SIZE(img->shdr),
4478 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 4479#ifdef ELF_HOST_FLAGS
4480 .e_flags = ELF_HOST_FLAGS,
4481#endif
4482#ifdef ELF_OSABI
4483 .e_ident[EI_OSABI] = ELF_OSABI,
4484#endif
5872bbf2
RH		 4485 },
4486 .phdr = {
4487 .p_type = PT_LOAD,
4488 .p_flags = PF_X,
4489 },
4490 .shdr = {
4491 [0] = { .sh_type = SHT_NULL },
4492 /* Trick: The contents of code_gen_buffer are not present in
4493 this fake ELF file; that got allocated elsewhere. Therefore
4494 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4495 will not look for contents. We can record any address. */
4496 [1] = { /* .text */
4497 .sh_type = SHT_NOBITS,
4498 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4499 },
4500 [2] = { /* .debug_info */
4501 .sh_type = SHT_PROGBITS,
4502 .sh_offset = offsetof(struct ElfImage, di),
4503 .sh_size = sizeof(struct DebugInfo),
4504 },
4505 [3] = { /* .debug_abbrev */
4506 .sh_type = SHT_PROGBITS,
4507 .sh_offset = offsetof(struct ElfImage, da),
4508 .sh_size = sizeof(img->da),
4509 },
4510 [4] = { /* .debug_frame */
4511 .sh_type = SHT_PROGBITS,
4512 .sh_offset = sizeof(struct ElfImage),
4513 },
4514 [5] = { /* .symtab */
4515 .sh_type = SHT_SYMTAB,
4516 .sh_offset = offsetof(struct ElfImage, sym),
4517 .sh_size = sizeof(img->sym),
4518 .sh_info = 1,
4519 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4520 .sh_entsize = sizeof(ElfW(Sym)),
4521 },
4522 [6] = { /* .strtab */
4523 .sh_type = SHT_STRTAB,
4524 .sh_offset = offsetof(struct ElfImage, str),
4525 .sh_size = sizeof(img->str),
4526 }
4527 },
4528 .sym = {
4529 [1] = { /* code_gen_buffer */
4530 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4531 .st_shndx = 1,
4532 }
4533 },
4534 .di = {
4535 .len = sizeof(struct DebugInfo) - 4,
4536 .version = 2,
4537 .ptr_size = sizeof(void *),
4538 .cu_die = 1,
4539 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4540 .fn_die = 2,
4541 .fn_name = "code_gen_buffer"
4542 },
4543 .da = {
4544 1, /* abbrev number (the cu) */
4545 0x11, 1, /* DW_TAG_compile_unit, has children */
4546 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4547 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4548 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4549 0, 0, /* end of abbrev */
4550 2, /* abbrev number (the fn) */
4551 0x2e, 0, /* DW_TAG_subprogram, no children */
4552 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4553 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4554 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4555 0, 0, /* end of abbrev */
4556 0 /* no more abbrev */
4557 },
4558 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4559 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 4560 };
4561
4562 /* We only need a single jit entry; statically allocate it. */
4563 static struct jit_code_entry one_entry;
4564
5872bbf2 4565 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 4566 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 4567 DebugFrameHeader *dfh;
813da627 4568
5872bbf2
RH		 4569 img = g_malloc(img_size);
4570 *img = img_template;
813da627 4571
5872bbf2
RH		 4572 img->phdr.p_vaddr = buf;
4573 img->phdr.p_paddr = buf;
4574 img->phdr.p_memsz = buf_size;
813da627 4575
813da627 4576 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 4577 img->shdr[1].sh_addr = buf;
813da627
RH		 4578 img->shdr[1].sh_size = buf_size;
4579
5872bbf2
RH		 4580 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4581 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4582
4583 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4584 img->shdr[4].sh_size = debug_frame_size;
4585
4586 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4587 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4588
4589 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4590 img->sym[1].st_value = buf;
4591 img->sym[1].st_size = buf_size;
813da627 4592
5872bbf2 4593 img->di.cu_low_pc = buf;
45aba097 4594 img->di.cu_high_pc = buf + buf_size;
5872bbf2 4595 img->di.fn_low_pc = buf;
45aba097 4596 img->di.fn_high_pc = buf + buf_size;
813da627 4597
2c90784a
RH		 4598 dfh = (DebugFrameHeader *)(img + 1);
4599 memcpy(dfh, debug_frame, debug_frame_size);
4600 dfh->fde.func_start = buf;
4601 dfh->fde.func_len = buf_size;
4602
813da627
RH		 4603#ifdef DEBUG_JIT
4604 /* Enable this block to be able to debug the ELF image file creation.
4605 One can use readelf, objdump, or other inspection utilities. */
4606 {
4607 FILE *f = fopen("/tmp/qemu.jit", "w+b");
4608 if (f) {
5872bbf2 4609 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 4610 /* Avoid stupid unused return value warning for fwrite. */
4611 }
4612 fclose(f);
4613 }
4614 }
4615#endif
4616
4617 one_entry.symfile_addr = img;
4618 one_entry.symfile_size = img_size;
4619
4620 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4621 __jit_debug_descriptor.relevant_entry = &one_entry;
4622 __jit_debug_descriptor.first_entry = &one_entry;
4623 __jit_debug_register_code();
4624}
4625#else
5872bbf2
RH		 4626/* No support for the feature. Provide the entry point expected by exec.c,
4627 and implement the internal function we declared earlier. */
813da627 4628
755bf9e5 4629static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 4630 const void *debug_frame,
4631 size_t debug_frame_size)
813da627
RH		 4632{
4633}
4634
755bf9e5 4635void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 4636{
4637}
4638#endif /* ELF_HOST_MACHINE */
db432672
RH		 4639
4640#if !TCG_TARGET_MAYBE_vec
4641void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4642{
4643 g_assert_not_reached();
4644}
4645#endif
QEMU mirror