]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
accel/tcg: Log tb->cflags with -d exec
[mirror_qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
1de7afc9 37#include "qemu/timer.h"
084cfca1 38#include "qemu/cacheflush.h"
c896fe29 39
c5d3c498 40/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 41 CPU definitions. Currently they are used for qemu_ld/st
42 instructions */
43#define NO_CPU_IO_DEFS
c896fe29 44
63c91552 45#include "exec/exec-all.h"
dcb32f1d 46#include "tcg/tcg-op.h"
813da627 47
edee2579 48#if UINTPTR_MAX == UINT32_MAX
813da627 49# define ELF_CLASS ELFCLASS32
edee2579
RH		 50#else
51# define ELF_CLASS ELFCLASS64
813da627
RH		 52#endif
53#ifdef HOST_WORDS_BIGENDIAN
54# define ELF_DATA ELFDATA2MSB
55#else
56# define ELF_DATA ELFDATA2LSB
57#endif
58
c896fe29 59#include "elf.h"
508127e2 60#include "exec/log.h"
5ff7258c 61#include "tcg-internal.h"
c896fe29 62
22f15579
RH		 63#ifdef CONFIG_TCG_INTERPRETER
64#include <ffi.h>
65#endif
66
139c1837 67/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 68 used here. */
e4d58b41
RH		 69static void tcg_target_init(TCGContext *s);
70static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 71static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 72 intptr_t value, intptr_t addend);
c896fe29 73
497a22eb
RH		 74/* The CIE and FDE header definitions will be common to all hosts. */
75typedef struct {
76 uint32_t len __attribute__((aligned((sizeof(void *)))));
77 uint32_t id;
78 uint8_t version;
79 char augmentation[1];
80 uint8_t code_align;
81 uint8_t data_align;
82 uint8_t return_column;
83} DebugFrameCIE;
84
85typedef struct QEMU_PACKED {
86 uint32_t len __attribute__((aligned((sizeof(void *)))));
87 uint32_t cie_offset;
edee2579
RH		 88 uintptr_t func_start;
89 uintptr_t func_len;
497a22eb
RH		 90} DebugFrameFDEHeader;
91
2c90784a
RH		 92typedef struct QEMU_PACKED {
93 DebugFrameCIE cie;
94 DebugFrameFDEHeader fde;
95} DebugFrameHeader;
96
755bf9e5 97static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 98 const void *debug_frame,
99 size_t debug_frame_size)
813da627
RH		 100 __attribute__((unused));
101
139c1837 102/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 103static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 104 intptr_t arg2);
78113e83 105static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 106static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 107 TCGReg ret, tcg_target_long arg);
5e8892db
MR		 108static void tcg_out_op(TCGContext *s, TCGOpcode opc,
109 const TCGArg args[TCG_MAX_OP_ARGS],
110 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 111#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 112static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
113 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 114static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 116static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
117 TCGReg dst, int64_t arg);
5e8892db
MR		 118static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
119 unsigned vecl, unsigned vece,
120 const TCGArg args[TCG_MAX_OP_ARGS],
121 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 122#else
e7632cfa
RH		 123static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
124 TCGReg dst, TCGReg src)
125{
126 g_assert_not_reached();
127}
d6ecb4a9
RH		 128static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
129 TCGReg dst, TCGReg base, intptr_t offset)
130{
131 g_assert_not_reached();
132}
4e186175
RH		 133static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
134 TCGReg dst, int64_t arg)
e7632cfa
RH		 135{
136 g_assert_not_reached();
137}
5e8892db
MR		 138static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
139 unsigned vecl, unsigned vece,
140 const TCGArg args[TCG_MAX_OP_ARGS],
141 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 142{
143 g_assert_not_reached();
144}
145#endif
2a534aff 146static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 147 intptr_t arg2);
59d7c14e
RH		 148static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
149 TCGReg base, intptr_t ofs);
7b7d8b2d
RH		 150#ifdef CONFIG_TCG_INTERPRETER
151static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
152 ffi_cif *cif);
153#else
2be7d76b 154static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
7b7d8b2d 155#endif
a4fbbd77 156static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 157#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 158static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 159#endif
c896fe29 160
42eb6dfc
RH		 161TCGContext tcg_init_ctx;
162__thread TCGContext *tcg_ctx;
163
5ff7258c 164TCGContext **tcg_ctxs;
0e2d61cf
RH		 165unsigned int tcg_cur_ctxs;
166unsigned int tcg_max_ctxs;
1c2adb95 167TCGv_env cpu_env = 0;
c8bc1168 168const void *tcg_code_gen_epilogue;
db0c51a3 169uintptr_t tcg_splitwx_diff;
df2cce29 170
b91ccb31
RH		 171#ifndef CONFIG_TCG_INTERPRETER
172tcg_prologue_fn *tcg_qemu_tb_exec;
173#endif
174
d2fd745f 175static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 176static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 177
1813e175 178#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 179static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 180{
181 *s->code_ptr++ = v;
182}
183
4196dca6
PM		 184static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
185 uint8_t v)
5c53bb81 186{
1813e175 187 *p = v;
5c53bb81 188}
1813e175 189#endif
5c53bb81 190
1813e175 191#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 192static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 193{
1813e175
RH		 194 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
195 *s->code_ptr++ = v;
196 } else {
197 tcg_insn_unit *p = s->code_ptr;
198 memcpy(p, &v, sizeof(v));
199 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
200 }
c896fe29
FB		 201}
202
4196dca6
PM		 203static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
204 uint16_t v)
5c53bb81 205{
1813e175
RH		 206 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
207 *p = v;
208 } else {
209 memcpy(p, &v, sizeof(v));
210 }
5c53bb81 211}
1813e175 212#endif
5c53bb81 213
1813e175 214#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 215static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 216{
1813e175
RH		 217 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
218 *s->code_ptr++ = v;
219 } else {
220 tcg_insn_unit *p = s->code_ptr;
221 memcpy(p, &v, sizeof(v));
222 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
223 }
c896fe29
FB		 224}
225
4196dca6
PM		 226static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
227 uint32_t v)
5c53bb81 228{
1813e175
RH		 229 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
230 *p = v;
231 } else {
232 memcpy(p, &v, sizeof(v));
233 }
5c53bb81 234}
1813e175 235#endif
5c53bb81 236
1813e175 237#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 238static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 239{
1813e175
RH		 240 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
241 *s->code_ptr++ = v;
242 } else {
243 tcg_insn_unit *p = s->code_ptr;
244 memcpy(p, &v, sizeof(v));
245 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
246 }
ac26eb69
RH		 247}
248
4196dca6
PM		 249static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
250 uint64_t v)
5c53bb81 251{
1813e175
RH		 252 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
253 *p = v;
254 } else {
255 memcpy(p, &v, sizeof(v));
256 }
5c53bb81 257}
1813e175 258#endif
5c53bb81 259
c896fe29
FB		 260/* label relocation processing */
261
1813e175 262static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 263 TCGLabel *l, intptr_t addend)
c896fe29 264{
7ecd02a0 265 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 266
7ecd02a0
RH		 267 r->type = type;
268 r->ptr = code_ptr;
269 r->addend = addend;
270 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 271}
272
92ab8e7d 273static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 274{
eabb7b91 275 tcg_debug_assert(!l->has_value);
c896fe29 276 l->has_value = 1;
92ab8e7d 277 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 278}
279
42a268c2 280TCGLabel *gen_new_label(void)
c896fe29 281{
b1311c4a 282 TCGContext *s = tcg_ctx;
51e3972c 283 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 284
7ecd02a0
RH		 285 memset(l, 0, sizeof(TCGLabel));
286 l->id = s->nb_labels++;
287 QSIMPLEQ_INIT(&l->relocs);
288
bef16ab4 289 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 290
291 return l;
c896fe29
FB		 292}
293
7ecd02a0
RH		 294static bool tcg_resolve_relocs(TCGContext *s)
295{
296 TCGLabel *l;
297
298 QSIMPLEQ_FOREACH(l, &s->labels, next) {
299 TCGRelocation *r;
300 uintptr_t value = l->u.value;
301
302 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
303 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
304 return false;
305 }
306 }
307 }
308 return true;
309}
310
9f754620
RH		 311static void set_jmp_reset_offset(TCGContext *s, int which)
312{
f14bed3f
RH		 313 /*
314 * We will check for overflow at the end of the opcode loop in
315 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
316 */
317 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 318}
319
db6b7d0c
RH		 320/* Signal overflow, starting over with fewer guest insns. */
321static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s)
322{
323 siglongjmp(s->jmp_trans, -2);
324}
325
4c22e840
RH		 326#define C_PFX1(P, A) P##A
327#define C_PFX2(P, A, B) P##A##_##B
328#define C_PFX3(P, A, B, C) P##A##_##B##_##C
329#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
330#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
331#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
332
333/* Define an enumeration for the various combinations. */
334
335#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
336#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
337#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
338#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
339
340#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
341#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
342#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
343#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
344
345#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
346
347#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
348#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
349#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
350#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
351
352typedef enum {
353#include "tcg-target-con-set.h"
354} TCGConstraintSetIndex;
355
356static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
357
358#undef C_O0_I1
359#undef C_O0_I2
360#undef C_O0_I3
361#undef C_O0_I4
362#undef C_O1_I1
363#undef C_O1_I2
364#undef C_O1_I3
365#undef C_O1_I4
366#undef C_N1_I2
367#undef C_O2_I1
368#undef C_O2_I2
369#undef C_O2_I3
370#undef C_O2_I4
371
372/* Put all of the constraint sets into an array, indexed by the enum. */
373
374#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
375#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
376#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
377#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
378
379#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
380#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
381#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
382#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
383
384#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
385
386#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
387#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
388#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
389#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
390
391static const TCGTargetOpDef constraint_sets[] = {
392#include "tcg-target-con-set.h"
393};
394
395
396#undef C_O0_I1
397#undef C_O0_I2
398#undef C_O0_I3
399#undef C_O0_I4
400#undef C_O1_I1
401#undef C_O1_I2
402#undef C_O1_I3
403#undef C_O1_I4
404#undef C_N1_I2
405#undef C_O2_I1
406#undef C_O2_I2
407#undef C_O2_I3
408#undef C_O2_I4
409
410/* Expand the enumerator to be returned from tcg_target_op_def(). */
411
412#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
413#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
414#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
415#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
416
417#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
418#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
419#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
420#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
421
422#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
423
424#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
425#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
426#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
427#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
428
139c1837 429#include "tcg-target.c.inc"
c896fe29 430
38b47b19
EC		 431static void alloc_tcg_plugin_context(TCGContext *s)
432{
433#ifdef CONFIG_PLUGIN
434 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
435 s->plugin_tb->insns =
436 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
437#endif
438}
439
3468b59e
EC		 440/*
441 * All TCG threads except the parent (i.e. the one that called tcg_context_init
442 * and registered the target's TCG globals) must register with this function
443 * before initiating translation.
444 *
445 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
446 * of tcg_region_init() for the reasoning behind this.
447 *
448 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
449 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
450 * is not used anymore for translation once this function is called.
451 *
452 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
453 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
454 */
455#ifdef CONFIG_USER_ONLY
456void tcg_register_thread(void)
457{
458 tcg_ctx = &tcg_init_ctx;
459}
460#else
461void tcg_register_thread(void)
462{
463 TCGContext *s = g_malloc(sizeof(*s));
464 unsigned int i, n;
3468b59e
EC		 465
466 *s = tcg_init_ctx;
467
468 /* Relink mem_base. */
469 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
470 if (tcg_init_ctx.temps[i].mem_base) {
471 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
472 tcg_debug_assert(b >= 0 && b < n);
473 s->temps[i].mem_base = &s->temps[b];
474 }
475 }
476
477 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 478 n = qatomic_fetch_inc(&tcg_cur_ctxs);
479 g_assert(n < tcg_max_ctxs);
d73415a3 480 qatomic_set(&tcg_ctxs[n], s);
3468b59e 481
38b47b19
EC		 482 if (n > 0) {
483 alloc_tcg_plugin_context(s);
bf042e8e 484 tcg_region_initial_alloc(s);
38b47b19
EC		 485 }
486
3468b59e 487 tcg_ctx = s;
e8feb96f 488}
3468b59e 489#endif /* !CONFIG_USER_ONLY */
e8feb96f 490
c896fe29
FB		 491/* pool based memory allocation */
492void *tcg_malloc_internal(TCGContext *s, int size)
493{
494 TCGPool *p;
495 int pool_size;
496
497 if (size > TCG_POOL_CHUNK_SIZE) {
498 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 499 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 500 p->size = size;
4055299e
KB		 501 p->next = s->pool_first_large;
502 s->pool_first_large = p;
503 return p->data;
c896fe29
FB		 504 } else {
505 p = s->pool_current;
506 if (!p) {
507 p = s->pool_first;
508 if (!p)
509 goto new_pool;
510 } else {
511 if (!p->next) {
512 new_pool:
513 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 514 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 515 p->size = pool_size;
516 p->next = NULL;
517 if (s->pool_current)
518 s->pool_current->next = p;
519 else
520 s->pool_first = p;
521 } else {
522 p = p->next;
523 }
524 }
525 }
526 s->pool_current = p;
527 s->pool_cur = p->data + size;
528 s->pool_end = p->data + p->size;
529 return p->data;
530}
531
532void tcg_pool_reset(TCGContext *s)
533{
4055299e
KB		 534 TCGPool *p, *t;
535 for (p = s->pool_first_large; p; p = t) {
536 t = p->next;
537 g_free(p);
538 }
539 s->pool_first_large = NULL;
c896fe29
FB		 540 s->pool_cur = s->pool_end = NULL;
541 s->pool_current = NULL;
542}
543
2ef6175a
RH		 544#include "exec/helper-proto.h"
545
100b5e01 546static const TCGHelperInfo all_helpers[] = {
2ef6175a 547#include "exec/helper-tcg.h"
100b5e01 548};
619205fd 549static GHashTable *helper_table;
100b5e01 550
22f15579
RH		 551#ifdef CONFIG_TCG_INTERPRETER
552static GHashTable *ffi_table;
553
554static ffi_type * const typecode_to_ffi[8] = {
555 [dh_typecode_void] = &ffi_type_void,
556 [dh_typecode_i32] = &ffi_type_uint32,
557 [dh_typecode_s32] = &ffi_type_sint32,
558 [dh_typecode_i64] = &ffi_type_uint64,
559 [dh_typecode_s64] = &ffi_type_sint64,
560 [dh_typecode_ptr] = &ffi_type_pointer,
561};
562#endif
563
91478cef 564static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 565static void process_op_defs(TCGContext *s);
1c2adb95
RH		 566static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
567 TCGReg reg, const char *name);
91478cef 568
43b972b7 569static void tcg_context_init(unsigned max_cpus)
c896fe29 570{
a76aabd3 571 TCGContext *s = &tcg_init_ctx;
100b5e01 572 int op, total_args, n, i;
c896fe29
FB		 573 TCGOpDef *def;
574 TCGArgConstraint *args_ct;
1c2adb95 575 TCGTemp *ts;
c896fe29
FB		 576
577 memset(s, 0, sizeof(*s));
c896fe29 578 s->nb_globals = 0;
c70fbf0a 579
c896fe29
FB		 580 /* Count total number of arguments and allocate the corresponding
581 space */
582 total_args = 0;
583 for(op = 0; op < NB_OPS; op++) {
584 def = &tcg_op_defs[op];
585 n = def->nb_iargs + def->nb_oargs;
586 total_args += n;
587 }
588
bc2b17e6 589 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 590
591 for(op = 0; op < NB_OPS; op++) {
592 def = &tcg_op_defs[op];
593 def->args_ct = args_ct;
c896fe29 594 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 595 args_ct += n;
596 }
5cd8f621
RH		 597
598 /* Register helpers. */
84fd9dd3 599 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 600 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 601
100b5e01 602 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
84fd9dd3 603 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 604 (gpointer)&all_helpers[i]);
100b5e01 605 }
5cd8f621 606
22f15579
RH		 607#ifdef CONFIG_TCG_INTERPRETER
608 /* g_direct_hash/equal for direct comparisons on uint32_t. */
609 ffi_table = g_hash_table_new(NULL, NULL);
610 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
611 struct {
612 ffi_cif cif;
613 ffi_type *args[];
614 } *ca;
615 uint32_t typemask = all_helpers[i].typemask;
616 gpointer hash = (gpointer)(uintptr_t)typemask;
617 ffi_status status;
618 int nargs;
619
620 if (g_hash_table_lookup(ffi_table, hash)) {
621 continue;
622 }
623
624 /* Ignoring the return type, find the last non-zero field. */
625 nargs = 32 - clz32(typemask >> 3);
626 nargs = DIV_ROUND_UP(nargs, 3);
627
628 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
629 ca->cif.rtype = typecode_to_ffi[typemask & 7];
630 ca->cif.nargs = nargs;
631
632 if (nargs != 0) {
633 ca->cif.arg_types = ca->args;
634 for (i = 0; i < nargs; ++i) {
635 int typecode = extract32(typemask, (i + 1) * 3, 3);
636 ca->args[i] = typecode_to_ffi[typecode];
637 }
638 }
639
640 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
641 ca->cif.rtype, ca->cif.arg_types);
642 assert(status == FFI_OK);
643
644 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
645 }
646#endif
647
c896fe29 648 tcg_target_init(s);
f69d277e 649 process_op_defs(s);
91478cef
RH		 650
651 /* Reverse the order of the saved registers, assuming they're all at
652 the start of tcg_target_reg_alloc_order. */
653 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
654 int r = tcg_target_reg_alloc_order[n];
655 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
656 break;
657 }
658 }
659 for (i = 0; i < n; ++i) {
660 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
661 }
662 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
663 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
664 }
b1311c4a 665
38b47b19
EC		 666 alloc_tcg_plugin_context(s);
667
b1311c4a 668 tcg_ctx = s;
3468b59e
EC		 669 /*
670 * In user-mode we simply share the init context among threads, since we
671 * use a single region. See the documentation tcg_region_init() for the
672 * reasoning behind this.
673 * In softmmu we will have at most max_cpus TCG threads.
674 */
675#ifdef CONFIG_USER_ONLY
df2cce29 676 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 677 tcg_cur_ctxs = 1;
678 tcg_max_ctxs = 1;
3468b59e 679#else
0e2d61cf
RH		 680 tcg_max_ctxs = max_cpus;
681 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 682#endif
1c2adb95
RH		 683
684 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
685 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
686 cpu_env = temp_tcgv_ptr(ts);
9002ec79 687}
b03cce8e 688
43b972b7 689void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 690{
43b972b7
RH		 691 tcg_context_init(max_cpus);
692 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 693}
694
6e3b2bfd
EC		 695/*
696 * Allocate TBs right before their corresponding translated code, making
697 * sure that TBs and code are on different cache lines.
698 */
699TranslationBlock *tcg_tb_alloc(TCGContext *s)
700{
701 uintptr_t align = qemu_icache_linesize;
702 TranslationBlock *tb;
703 void *next;
704
e8feb96f 705 retry:
6e3b2bfd
EC		 706 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
707 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
708
709 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 710 if (tcg_region_alloc(s)) {
711 return NULL;
712 }
713 goto retry;
6e3b2bfd 714 }
d73415a3 715 qatomic_set(&s->code_gen_ptr, next);
57a26946 716 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 717 return tb;
718}
719
9002ec79
RH		 720void tcg_prologue_init(TCGContext *s)
721{
b0a0794a 722 size_t prologue_size;
8163b749 723
b0a0794a
RH		 724 s->code_ptr = s->code_gen_ptr;
725 s->code_buf = s->code_gen_ptr;
5b38ee31 726 s->data_gen_ptr = NULL;
b91ccb31
RH		 727
728#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 729 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 730#endif
8163b749 731
5b38ee31
RH		 732#ifdef TCG_TARGET_NEED_POOL_LABELS
733 s->pool_labels = NULL;
734#endif
735
653b87eb 736 qemu_thread_jit_write();
8163b749 737 /* Generate the prologue. */
b03cce8e 738 tcg_target_qemu_prologue(s);
5b38ee31
RH		 739
740#ifdef TCG_TARGET_NEED_POOL_LABELS
741 /* Allow the prologue to put e.g. guest_base into a pool entry. */
742 {
1768987b
RH		 743 int result = tcg_out_pool_finalize(s);
744 tcg_debug_assert(result == 0);
5b38ee31
RH		 745 }
746#endif
747
b0a0794a
RH		 748 prologue_size = tcg_current_code_size(s);
749
df5d2b16 750#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 751 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
752 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 753#endif
8163b749 754
d6b64b2b
RH		 755#ifdef DEBUG_DISAS
756 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
fc59d2d8 757 FILE *logfile = qemu_log_lock();
8163b749 758 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
5b38ee31 759 if (s->data_gen_ptr) {
b0a0794a 760 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
5b38ee31
RH		 761 size_t data_size = prologue_size - code_size;
762 size_t i;
763
b0a0794a 764 log_disas(s->code_gen_ptr, code_size);
5b38ee31
RH		 765
766 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
767 if (sizeof(tcg_target_ulong) == 8) {
768 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
769 (uintptr_t)s->data_gen_ptr + i,
770 *(uint64_t *)(s->data_gen_ptr + i));
771 } else {
772 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
773 (uintptr_t)s->data_gen_ptr + i,
774 *(uint32_t *)(s->data_gen_ptr + i));
775 }
776 }
777 } else {
b0a0794a 778 log_disas(s->code_gen_ptr, prologue_size);
5b38ee31 779 }
d6b64b2b
RH		 780 qemu_log("\n");
781 qemu_log_flush();
fc59d2d8 782 qemu_log_unlock(logfile);
d6b64b2b
RH		 783 }
784#endif
cedbcb01 785
6eea0434
RH		 786#ifndef CONFIG_TCG_INTERPRETER
787 /*
788 * Assert that goto_ptr is implemented completely, setting an epilogue.
789 * For tci, we use NULL as the signal to return from the interpreter,
790 * so skip this check.
791 */
cedbcb01 792 if (TCG_TARGET_HAS_goto_ptr) {
8b5c2b62 793 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
cedbcb01 794 }
6eea0434 795#endif
d1c74ab3
RH		 796
797 tcg_region_prologue_set(s);
c896fe29
FB		 798}
799
c896fe29
FB		 800void tcg_func_start(TCGContext *s)
801{
802 tcg_pool_reset(s);
803 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 804
805 /* No temps have been previously allocated for size or locality. */
806 memset(s->free_temps, 0, sizeof(s->free_temps));
807
c0522136
RH		 808 /* No constant temps have been previously allocated. */
809 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
810 if (s->const_table[i]) {
811 g_hash_table_remove_all(s->const_table[i]);
812 }
813 }
814
abebf925 815 s->nb_ops = 0;
c896fe29
FB		 816 s->nb_labels = 0;
817 s->current_frame_offset = s->frame_start;
818
0a209d4b
RH		 819#ifdef CONFIG_DEBUG_TCG
820 s->goto_tb_issue_mask = 0;
821#endif
822
15fa08f8
RH		 823 QTAILQ_INIT(&s->ops);
824 QTAILQ_INIT(&s->free_ops);
bef16ab4 825 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 826}
827
ae30e866 828static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 829{
830 int n = s->nb_temps++;
ae30e866
RH		 831
832 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 833 tcg_raise_tb_overflow(s);
ae30e866 834 }
7ca4b752
RH		 835 return memset(&s->temps[n], 0, sizeof(TCGTemp));
836}
837
ae30e866 838static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 839{
fa477d25
RH		 840 TCGTemp *ts;
841
7ca4b752 842 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 843 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 844 s->nb_globals++;
fa477d25 845 ts = tcg_temp_alloc(s);
ee17db83 846 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 847
848 return ts;
c896fe29
FB		 849}
850
085272b3
RH		 851static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
852 TCGReg reg, const char *name)
c896fe29 853{
c896fe29 854 TCGTemp *ts;
c896fe29 855
b3a62939 856 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 857 tcg_abort();
b3a62939 858 }
7ca4b752
RH		 859
860 ts = tcg_global_alloc(s);
c896fe29
FB		 861 ts->base_type = type;
862 ts->type = type;
ee17db83 863 ts->kind = TEMP_FIXED;
c896fe29 864 ts->reg = reg;
c896fe29 865 ts->name = name;
c896fe29 866 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 867
085272b3 868 return ts;
a7812ae4
PB		 869}
870
b6638662 871void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 872{
b3a62939
RH		 873 s->frame_start = start;
874 s->frame_end = start + size;
085272b3
RH		 875 s->frame_temp
876 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 877}
878
085272b3
RH		 879TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
880 intptr_t offset, const char *name)
c896fe29 881{
b1311c4a 882 TCGContext *s = tcg_ctx;
dc41aa7d 883 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 884 TCGTemp *ts = tcg_global_alloc(s);
b3915dbb 885 int indirect_reg = 0, bigendian = 0;
7ca4b752
RH		 886#ifdef HOST_WORDS_BIGENDIAN
887 bigendian = 1;
888#endif
c896fe29 889
c0522136
RH		 890 switch (base_ts->kind) {
891 case TEMP_FIXED:
892 break;
893 case TEMP_GLOBAL:
5a18407f
RH		 894 /* We do not support double-indirect registers. */
895 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 896 base_ts->indirect_base = 1;
5a18407f
RH		 897 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
898 ? 2 : 1);
899 indirect_reg = 1;
c0522136
RH		 900 break;
901 default:
902 g_assert_not_reached();
b3915dbb
RH		 903 }
904
7ca4b752
RH		 905 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
906 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 907 char buf[64];
7ca4b752
RH		 908
909 ts->base_type = TCG_TYPE_I64;
c896fe29 910 ts->type = TCG_TYPE_I32;
b3915dbb 911 ts->indirect_reg = indirect_reg;
c896fe29 912 ts->mem_allocated = 1;
b3a62939 913 ts->mem_base = base_ts;
7ca4b752 914 ts->mem_offset = offset + bigendian * 4;
c896fe29
FB		 915 pstrcpy(buf, sizeof(buf), name);
916 pstrcat(buf, sizeof(buf), "_0");
917 ts->name = strdup(buf);
c896fe29 918
7ca4b752
RH		 919 tcg_debug_assert(ts2 == ts + 1);
920 ts2->base_type = TCG_TYPE_I64;
921 ts2->type = TCG_TYPE_I32;
b3915dbb 922 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 923 ts2->mem_allocated = 1;
924 ts2->mem_base = base_ts;
925 ts2->mem_offset = offset + (1 - bigendian) * 4;
c896fe29
FB		 926 pstrcpy(buf, sizeof(buf), name);
927 pstrcat(buf, sizeof(buf), "_1");
120c1084 928 ts2->name = strdup(buf);
7ca4b752 929 } else {
c896fe29
FB		 930 ts->base_type = type;
931 ts->type = type;
b3915dbb 932 ts->indirect_reg = indirect_reg;
c896fe29 933 ts->mem_allocated = 1;
b3a62939 934 ts->mem_base = base_ts;
c896fe29 935 ts->mem_offset = offset;
c896fe29 936 ts->name = name;
c896fe29 937 }
085272b3 938 return ts;
a7812ae4
PB		 939}
940
5bfa8034 941TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
c896fe29 942{
b1311c4a 943 TCGContext *s = tcg_ctx;
ee17db83 944 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
c896fe29 945 TCGTemp *ts;
641d5fbe 946 int idx, k;
c896fe29 947
0ec9eabc
RH		 948 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
949 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
950 if (idx < TCG_MAX_TEMPS) {
951 /* There is already an available temp with the right type. */
952 clear_bit(idx, s->free_temps[k].l);
953
e8996ee0 954 ts = &s->temps[idx];
e8996ee0 955 ts->temp_allocated = 1;
7ca4b752 956 tcg_debug_assert(ts->base_type == type);
ee17db83 957 tcg_debug_assert(ts->kind == kind);
e8996ee0 958 } else {
7ca4b752
RH		 959 ts = tcg_temp_alloc(s);
960 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
961 TCGTemp *ts2 = tcg_temp_alloc(s);
962
f6aa2f7d 963 ts->base_type = type;
e8996ee0
FB		 964 ts->type = TCG_TYPE_I32;
965 ts->temp_allocated = 1;
ee17db83 966 ts->kind = kind;
7ca4b752
RH		 967
968 tcg_debug_assert(ts2 == ts + 1);
969 ts2->base_type = TCG_TYPE_I64;
970 ts2->type = TCG_TYPE_I32;
971 ts2->temp_allocated = 1;
ee17db83 972 ts2->kind = kind;
7ca4b752 973 } else {
e8996ee0
FB		 974 ts->base_type = type;
975 ts->type = type;
976 ts->temp_allocated = 1;
ee17db83 977 ts->kind = kind;
e8996ee0 978 }
c896fe29 979 }
27bfd83c
PM		 980
981#if defined(CONFIG_DEBUG_TCG)
982 s->temps_in_use++;
983#endif
085272b3 984 return ts;
c896fe29
FB		 985}
986
d2fd745f
RH		 987TCGv_vec tcg_temp_new_vec(TCGType type)
988{
989 TCGTemp *t;
990
991#ifdef CONFIG_DEBUG_TCG
992 switch (type) {
993 case TCG_TYPE_V64:
994 assert(TCG_TARGET_HAS_v64);
995 break;
996 case TCG_TYPE_V128:
997 assert(TCG_TARGET_HAS_v128);
998 break;
999 case TCG_TYPE_V256:
1000 assert(TCG_TARGET_HAS_v256);
1001 break;
1002 default:
1003 g_assert_not_reached();
1004 }
1005#endif
1006
1007 t = tcg_temp_new_internal(type, 0);
1008 return temp_tcgv_vec(t);
1009}
1010
1011/* Create a new temp of the same type as an existing temp. */
1012TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1013{
1014 TCGTemp *t = tcgv_vec_temp(match);
1015
1016 tcg_debug_assert(t->temp_allocated != 0);
1017
1018 t = tcg_temp_new_internal(t->base_type, 0);
1019 return temp_tcgv_vec(t);
1020}
1021
5bfa8034 1022void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1023{
b1311c4a 1024 TCGContext *s = tcg_ctx;
085272b3 1025 int k, idx;
c896fe29 1026
c0522136
RH		 1027 /* In order to simplify users of tcg_constant_*, silently ignore free. */
1028 if (ts->kind == TEMP_CONST) {
1029 return;
1030 }
1031
27bfd83c
PM		 1032#if defined(CONFIG_DEBUG_TCG)
1033 s->temps_in_use--;
1034 if (s->temps_in_use < 0) {
1035 fprintf(stderr, "More temporaries freed than allocated!\n");
1036 }
1037#endif
1038
ee17db83 1039 tcg_debug_assert(ts->kind < TEMP_GLOBAL);
eabb7b91 1040 tcg_debug_assert(ts->temp_allocated != 0);
e8996ee0 1041 ts->temp_allocated = 0;
0ec9eabc 1042
085272b3 1043 idx = temp_idx(ts);
ee17db83 1044 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1045 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1046}
1047
c0522136
RH		 1048TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1049{
1050 TCGContext *s = tcg_ctx;
1051 GHashTable *h = s->const_table[type];
1052 TCGTemp *ts;
1053
1054 if (h == NULL) {
1055 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1056 s->const_table[type] = h;
1057 }
1058
1059 ts = g_hash_table_lookup(h, &val);
1060 if (ts == NULL) {
1061 ts = tcg_temp_alloc(s);
1062
1063 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1064 TCGTemp *ts2 = tcg_temp_alloc(s);
1065
1066 ts->base_type = TCG_TYPE_I64;
1067 ts->type = TCG_TYPE_I32;
1068 ts->kind = TEMP_CONST;
1069 ts->temp_allocated = 1;
1070 /*
1071 * Retain the full value of the 64-bit constant in the low
1072 * part, so that the hash table works. Actual uses will
1073 * truncate the value to the low part.
1074 */
1075 ts->val = val;
1076
1077 tcg_debug_assert(ts2 == ts + 1);
1078 ts2->base_type = TCG_TYPE_I64;
1079 ts2->type = TCG_TYPE_I32;
1080 ts2->kind = TEMP_CONST;
1081 ts2->temp_allocated = 1;
1082 ts2->val = val >> 32;
1083 } else {
1084 ts->base_type = type;
1085 ts->type = type;
1086 ts->kind = TEMP_CONST;
1087 ts->temp_allocated = 1;
1088 ts->val = val;
1089 }
1090 g_hash_table_insert(h, &ts->val, ts);
1091 }
1092
1093 return ts;
1094}
1095
1096TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1097{
1098 val = dup_const(vece, val);
1099 return temp_tcgv_vec(tcg_constant_internal(type, val));
1100}
1101
88d4005b
RH		 1102TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1103{
1104 TCGTemp *t = tcgv_vec_temp(match);
1105
1106 tcg_debug_assert(t->temp_allocated != 0);
1107 return tcg_constant_vec(t->base_type, vece, val);
1108}
1109
a7812ae4 1110TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1111{
a7812ae4
PB		 1112 TCGv_i32 t0;
1113 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1114 tcg_gen_movi_i32(t0, val);
1115 return t0;
1116}
c896fe29 1117
a7812ae4 1118TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1119{
a7812ae4
PB		 1120 TCGv_i64 t0;
1121 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1122 tcg_gen_movi_i64(t0, val);
1123 return t0;
c896fe29
FB		 1124}
1125
a7812ae4 1126TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1127{
a7812ae4
PB		 1128 TCGv_i32 t0;
1129 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1130 tcg_gen_movi_i32(t0, val);
1131 return t0;
1132}
1133
a7812ae4 1134TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1135{
a7812ae4
PB		 1136 TCGv_i64 t0;
1137 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1138 tcg_gen_movi_i64(t0, val);
1139 return t0;
1140}
1141
27bfd83c
PM		 1142#if defined(CONFIG_DEBUG_TCG)
1143void tcg_clear_temp_count(void)
1144{
b1311c4a 1145 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1146 s->temps_in_use = 0;
1147}
1148
1149int tcg_check_temp_count(void)
1150{
b1311c4a 1151 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1152 if (s->temps_in_use) {
1153 /* Clear the count so that we don't give another
1154 * warning immediately next time around.
1155 */
1156 s->temps_in_use = 0;
1157 return 1;
1158 }
1159 return 0;
1160}
1161#endif
1162
be0f34b5
RH		 1163/* Return true if OP may appear in the opcode stream.
1164 Test the runtime variable that controls each opcode. */
1165bool tcg_op_supported(TCGOpcode op)
1166{
d2fd745f
RH		 1167 const bool have_vec
1168 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1169
be0f34b5
RH		 1170 switch (op) {
1171 case INDEX_op_discard:
1172 case INDEX_op_set_label:
1173 case INDEX_op_call:
1174 case INDEX_op_br:
1175 case INDEX_op_mb:
1176 case INDEX_op_insn_start:
1177 case INDEX_op_exit_tb:
1178 case INDEX_op_goto_tb:
1179 case INDEX_op_qemu_ld_i32:
1180 case INDEX_op_qemu_st_i32:
1181 case INDEX_op_qemu_ld_i64:
1182 case INDEX_op_qemu_st_i64:
1183 return true;
1184
07ce0b05
RH		 1185 case INDEX_op_qemu_st8_i32:
1186 return TCG_TARGET_HAS_qemu_st8_i32;
1187
be0f34b5
RH		 1188 case INDEX_op_goto_ptr:
1189 return TCG_TARGET_HAS_goto_ptr;
1190
1191 case INDEX_op_mov_i32:
be0f34b5
RH		 1192 case INDEX_op_setcond_i32:
1193 case INDEX_op_brcond_i32:
1194 case INDEX_op_ld8u_i32:
1195 case INDEX_op_ld8s_i32:
1196 case INDEX_op_ld16u_i32:
1197 case INDEX_op_ld16s_i32:
1198 case INDEX_op_ld_i32:
1199 case INDEX_op_st8_i32:
1200 case INDEX_op_st16_i32:
1201 case INDEX_op_st_i32:
1202 case INDEX_op_add_i32:
1203 case INDEX_op_sub_i32:
1204 case INDEX_op_mul_i32:
1205 case INDEX_op_and_i32:
1206 case INDEX_op_or_i32:
1207 case INDEX_op_xor_i32:
1208 case INDEX_op_shl_i32:
1209 case INDEX_op_shr_i32:
1210 case INDEX_op_sar_i32:
1211 return true;
1212
1213 case INDEX_op_movcond_i32:
1214 return TCG_TARGET_HAS_movcond_i32;
1215 case INDEX_op_div_i32:
1216 case INDEX_op_divu_i32:
1217 return TCG_TARGET_HAS_div_i32;
1218 case INDEX_op_rem_i32:
1219 case INDEX_op_remu_i32:
1220 return TCG_TARGET_HAS_rem_i32;
1221 case INDEX_op_div2_i32:
1222 case INDEX_op_divu2_i32:
1223 return TCG_TARGET_HAS_div2_i32;
1224 case INDEX_op_rotl_i32:
1225 case INDEX_op_rotr_i32:
1226 return TCG_TARGET_HAS_rot_i32;
1227 case INDEX_op_deposit_i32:
1228 return TCG_TARGET_HAS_deposit_i32;
1229 case INDEX_op_extract_i32:
1230 return TCG_TARGET_HAS_extract_i32;
1231 case INDEX_op_sextract_i32:
1232 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1233 case INDEX_op_extract2_i32:
1234 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1235 case INDEX_op_add2_i32:
1236 return TCG_TARGET_HAS_add2_i32;
1237 case INDEX_op_sub2_i32:
1238 return TCG_TARGET_HAS_sub2_i32;
1239 case INDEX_op_mulu2_i32:
1240 return TCG_TARGET_HAS_mulu2_i32;
1241 case INDEX_op_muls2_i32:
1242 return TCG_TARGET_HAS_muls2_i32;
1243 case INDEX_op_muluh_i32:
1244 return TCG_TARGET_HAS_muluh_i32;
1245 case INDEX_op_mulsh_i32:
1246 return TCG_TARGET_HAS_mulsh_i32;
1247 case INDEX_op_ext8s_i32:
1248 return TCG_TARGET_HAS_ext8s_i32;
1249 case INDEX_op_ext16s_i32:
1250 return TCG_TARGET_HAS_ext16s_i32;
1251 case INDEX_op_ext8u_i32:
1252 return TCG_TARGET_HAS_ext8u_i32;
1253 case INDEX_op_ext16u_i32:
1254 return TCG_TARGET_HAS_ext16u_i32;
1255 case INDEX_op_bswap16_i32:
1256 return TCG_TARGET_HAS_bswap16_i32;
1257 case INDEX_op_bswap32_i32:
1258 return TCG_TARGET_HAS_bswap32_i32;
1259 case INDEX_op_not_i32:
1260 return TCG_TARGET_HAS_not_i32;
1261 case INDEX_op_neg_i32:
1262 return TCG_TARGET_HAS_neg_i32;
1263 case INDEX_op_andc_i32:
1264 return TCG_TARGET_HAS_andc_i32;
1265 case INDEX_op_orc_i32:
1266 return TCG_TARGET_HAS_orc_i32;
1267 case INDEX_op_eqv_i32:
1268 return TCG_TARGET_HAS_eqv_i32;
1269 case INDEX_op_nand_i32:
1270 return TCG_TARGET_HAS_nand_i32;
1271 case INDEX_op_nor_i32:
1272 return TCG_TARGET_HAS_nor_i32;
1273 case INDEX_op_clz_i32:
1274 return TCG_TARGET_HAS_clz_i32;
1275 case INDEX_op_ctz_i32:
1276 return TCG_TARGET_HAS_ctz_i32;
1277 case INDEX_op_ctpop_i32:
1278 return TCG_TARGET_HAS_ctpop_i32;
1279
1280 case INDEX_op_brcond2_i32:
1281 case INDEX_op_setcond2_i32:
1282 return TCG_TARGET_REG_BITS == 32;
1283
1284 case INDEX_op_mov_i64:
be0f34b5
RH		 1285 case INDEX_op_setcond_i64:
1286 case INDEX_op_brcond_i64:
1287 case INDEX_op_ld8u_i64:
1288 case INDEX_op_ld8s_i64:
1289 case INDEX_op_ld16u_i64:
1290 case INDEX_op_ld16s_i64:
1291 case INDEX_op_ld32u_i64:
1292 case INDEX_op_ld32s_i64:
1293 case INDEX_op_ld_i64:
1294 case INDEX_op_st8_i64:
1295 case INDEX_op_st16_i64:
1296 case INDEX_op_st32_i64:
1297 case INDEX_op_st_i64:
1298 case INDEX_op_add_i64:
1299 case INDEX_op_sub_i64:
1300 case INDEX_op_mul_i64:
1301 case INDEX_op_and_i64:
1302 case INDEX_op_or_i64:
1303 case INDEX_op_xor_i64:
1304 case INDEX_op_shl_i64:
1305 case INDEX_op_shr_i64:
1306 case INDEX_op_sar_i64:
1307 case INDEX_op_ext_i32_i64:
1308 case INDEX_op_extu_i32_i64:
1309 return TCG_TARGET_REG_BITS == 64;
1310
1311 case INDEX_op_movcond_i64:
1312 return TCG_TARGET_HAS_movcond_i64;
1313 case INDEX_op_div_i64:
1314 case INDEX_op_divu_i64:
1315 return TCG_TARGET_HAS_div_i64;
1316 case INDEX_op_rem_i64:
1317 case INDEX_op_remu_i64:
1318 return TCG_TARGET_HAS_rem_i64;
1319 case INDEX_op_div2_i64:
1320 case INDEX_op_divu2_i64:
1321 return TCG_TARGET_HAS_div2_i64;
1322 case INDEX_op_rotl_i64:
1323 case INDEX_op_rotr_i64:
1324 return TCG_TARGET_HAS_rot_i64;
1325 case INDEX_op_deposit_i64:
1326 return TCG_TARGET_HAS_deposit_i64;
1327 case INDEX_op_extract_i64:
1328 return TCG_TARGET_HAS_extract_i64;
1329 case INDEX_op_sextract_i64:
1330 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1331 case INDEX_op_extract2_i64:
1332 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1333 case INDEX_op_extrl_i64_i32:
1334 return TCG_TARGET_HAS_extrl_i64_i32;
1335 case INDEX_op_extrh_i64_i32:
1336 return TCG_TARGET_HAS_extrh_i64_i32;
1337 case INDEX_op_ext8s_i64:
1338 return TCG_TARGET_HAS_ext8s_i64;
1339 case INDEX_op_ext16s_i64:
1340 return TCG_TARGET_HAS_ext16s_i64;
1341 case INDEX_op_ext32s_i64:
1342 return TCG_TARGET_HAS_ext32s_i64;
1343 case INDEX_op_ext8u_i64:
1344 return TCG_TARGET_HAS_ext8u_i64;
1345 case INDEX_op_ext16u_i64:
1346 return TCG_TARGET_HAS_ext16u_i64;
1347 case INDEX_op_ext32u_i64:
1348 return TCG_TARGET_HAS_ext32u_i64;
1349 case INDEX_op_bswap16_i64:
1350 return TCG_TARGET_HAS_bswap16_i64;
1351 case INDEX_op_bswap32_i64:
1352 return TCG_TARGET_HAS_bswap32_i64;
1353 case INDEX_op_bswap64_i64:
1354 return TCG_TARGET_HAS_bswap64_i64;
1355 case INDEX_op_not_i64:
1356 return TCG_TARGET_HAS_not_i64;
1357 case INDEX_op_neg_i64:
1358 return TCG_TARGET_HAS_neg_i64;
1359 case INDEX_op_andc_i64:
1360 return TCG_TARGET_HAS_andc_i64;
1361 case INDEX_op_orc_i64:
1362 return TCG_TARGET_HAS_orc_i64;
1363 case INDEX_op_eqv_i64:
1364 return TCG_TARGET_HAS_eqv_i64;
1365 case INDEX_op_nand_i64:
1366 return TCG_TARGET_HAS_nand_i64;
1367 case INDEX_op_nor_i64:
1368 return TCG_TARGET_HAS_nor_i64;
1369 case INDEX_op_clz_i64:
1370 return TCG_TARGET_HAS_clz_i64;
1371 case INDEX_op_ctz_i64:
1372 return TCG_TARGET_HAS_ctz_i64;
1373 case INDEX_op_ctpop_i64:
1374 return TCG_TARGET_HAS_ctpop_i64;
1375 case INDEX_op_add2_i64:
1376 return TCG_TARGET_HAS_add2_i64;
1377 case INDEX_op_sub2_i64:
1378 return TCG_TARGET_HAS_sub2_i64;
1379 case INDEX_op_mulu2_i64:
1380 return TCG_TARGET_HAS_mulu2_i64;
1381 case INDEX_op_muls2_i64:
1382 return TCG_TARGET_HAS_muls2_i64;
1383 case INDEX_op_muluh_i64:
1384 return TCG_TARGET_HAS_muluh_i64;
1385 case INDEX_op_mulsh_i64:
1386 return TCG_TARGET_HAS_mulsh_i64;
1387
d2fd745f
RH		 1388 case INDEX_op_mov_vec:
1389 case INDEX_op_dup_vec:
37ee55a0 1390 case INDEX_op_dupm_vec:
d2fd745f
RH		 1391 case INDEX_op_ld_vec:
1392 case INDEX_op_st_vec:
1393 case INDEX_op_add_vec:
1394 case INDEX_op_sub_vec:
1395 case INDEX_op_and_vec:
1396 case INDEX_op_or_vec:
1397 case INDEX_op_xor_vec:
212be173 1398 case INDEX_op_cmp_vec:
d2fd745f
RH		 1399 return have_vec;
1400 case INDEX_op_dup2_vec:
1401 return have_vec && TCG_TARGET_REG_BITS == 32;
1402 case INDEX_op_not_vec:
1403 return have_vec && TCG_TARGET_HAS_not_vec;
1404 case INDEX_op_neg_vec:
1405 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1406 case INDEX_op_abs_vec:
1407 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1408 case INDEX_op_andc_vec:
1409 return have_vec && TCG_TARGET_HAS_andc_vec;
1410 case INDEX_op_orc_vec:
1411 return have_vec && TCG_TARGET_HAS_orc_vec;
3774030a
RH		 1412 case INDEX_op_mul_vec:
1413 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1414 case INDEX_op_shli_vec:
1415 case INDEX_op_shri_vec:
1416 case INDEX_op_sari_vec:
1417 return have_vec && TCG_TARGET_HAS_shi_vec;
1418 case INDEX_op_shls_vec:
1419 case INDEX_op_shrs_vec:
1420 case INDEX_op_sars_vec:
1421 return have_vec && TCG_TARGET_HAS_shs_vec;
1422 case INDEX_op_shlv_vec:
1423 case INDEX_op_shrv_vec:
1424 case INDEX_op_sarv_vec:
1425 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1426 case INDEX_op_rotli_vec:
1427 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1428 case INDEX_op_rotls_vec:
1429 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1430 case INDEX_op_rotlv_vec:
1431 case INDEX_op_rotrv_vec:
1432 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1433 case INDEX_op_ssadd_vec:
1434 case INDEX_op_usadd_vec:
1435 case INDEX_op_sssub_vec:
1436 case INDEX_op_ussub_vec:
1437 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1438 case INDEX_op_smin_vec:
1439 case INDEX_op_umin_vec:
1440 case INDEX_op_smax_vec:
1441 case INDEX_op_umax_vec:
1442 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1443 case INDEX_op_bitsel_vec:
1444 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1445 case INDEX_op_cmpsel_vec:
1446 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1447
db432672
RH		 1448 default:
1449 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1450 return true;
be0f34b5 1451 }
be0f34b5
RH		 1452}
1453
39cf05d3
FB		 1454/* Note: we convert the 64 bit args to 32 bit and do some alignment
1455 and endian swap. Maybe it would be better to do the alignment
1456 and endian swap in tcg_reg_alloc_call(). */
ae8b75dc 1457void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1458{
75e8b9b7 1459 int i, real_args, nb_rets, pi;
3e92aa34
RH		 1460 unsigned typemask;
1461 const TCGHelperInfo *info;
75e8b9b7 1462 TCGOp *op;
afb49896 1463
619205fd 1464 info = g_hash_table_lookup(helper_table, (gpointer)func);
7319d83a 1465 typemask = info->typemask;
2bece2c8 1466
38b47b19
EC		 1467#ifdef CONFIG_PLUGIN
1468 /* detect non-plugin helpers */
1469 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1470 tcg_ctx->plugin_insn->calls_helpers = true;
1471 }
1472#endif
1473
34b1a49c
RH		 1474#if defined(__sparc__) && !defined(__arch64__) \
1475 && !defined(CONFIG_TCG_INTERPRETER)
1476 /* We have 64-bit values in one register, but need to pass as two
1477 separate parameters. Split them. */
7319d83a 1478 int orig_typemask = typemask;
34b1a49c
RH		 1479 int orig_nargs = nargs;
1480 TCGv_i64 retl, reth;
ae8b75dc 1481 TCGTemp *split_args[MAX_OPC_PARAM];
34b1a49c 1482
f764718d
RH		 1483 retl = NULL;
1484 reth = NULL;
7319d83a
RH		 1485 typemask = 0;
1486 for (i = real_args = 0; i < nargs; ++i) {
1487 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1488 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1489
1490 if (is_64bit) {
1491 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1492 TCGv_i32 h = tcg_temp_new_i32();
1493 TCGv_i32 l = tcg_temp_new_i32();
1494 tcg_gen_extr_i64_i32(l, h, orig);
1495 split_args[real_args++] = tcgv_i32_temp(h);
1496 typemask |= dh_typecode_i32 << (real_args * 3);
1497 split_args[real_args++] = tcgv_i32_temp(l);
1498 typemask |= dh_typecode_i32 << (real_args * 3);
1499 } else {
1500 split_args[real_args++] = args[i];
1501 typemask |= argtype << (real_args * 3);
34b1a49c 1502 }
34b1a49c 1503 }
7319d83a
RH		 1504 nargs = real_args;
1505 args = split_args;
34b1a49c 1506#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1507 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1508 int argtype = extract32(typemask, (i + 1) * 3, 3);
1509 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1510 bool is_signed = argtype & 1;
1511
1512 if (is_32bit) {
2bece2c8 1513 TCGv_i64 temp = tcg_temp_new_i64();
085272b3 1514 TCGv_i64 orig = temp_tcgv_i64(args[i]);
2bece2c8
RH		 1515 if (is_signed) {
1516 tcg_gen_ext32s_i64(temp, orig);
1517 } else {
1518 tcg_gen_ext32u_i64(temp, orig);
1519 }
ae8b75dc 1520 args[i] = tcgv_i64_temp(temp);
2bece2c8
RH		 1521 }
1522 }
1523#endif /* TCG_TARGET_EXTEND_ARGS */
1524
15fa08f8 1525 op = tcg_emit_op(INDEX_op_call);
75e8b9b7
RH		 1526
1527 pi = 0;
ae8b75dc 1528 if (ret != NULL) {
34b1a49c
RH		 1529#if defined(__sparc__) && !defined(__arch64__) \
1530 && !defined(CONFIG_TCG_INTERPRETER)
7319d83a 1531 if ((typemask & 6) == dh_typecode_i64) {
34b1a49c
RH		 1532 /* The 32-bit ABI is going to return the 64-bit value in
1533 the %o0/%o1 register pair. Prepare for this by using
1534 two return temporaries, and reassemble below. */
1535 retl = tcg_temp_new_i64();
1536 reth = tcg_temp_new_i64();
ae8b75dc
RH		 1537 op->args[pi++] = tcgv_i64_arg(reth);
1538 op->args[pi++] = tcgv_i64_arg(retl);
34b1a49c
RH		 1539 nb_rets = 2;
1540 } else {
ae8b75dc 1541 op->args[pi++] = temp_arg(ret);
34b1a49c
RH		 1542 nb_rets = 1;
1543 }
1544#else
7319d83a 1545 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
02eb19d0 1546#ifdef HOST_WORDS_BIGENDIAN
ae8b75dc
RH		 1547 op->args[pi++] = temp_arg(ret + 1);
1548 op->args[pi++] = temp_arg(ret);
39cf05d3 1549#else
ae8b75dc
RH		 1550 op->args[pi++] = temp_arg(ret);
1551 op->args[pi++] = temp_arg(ret + 1);
39cf05d3 1552#endif
a7812ae4 1553 nb_rets = 2;
34b1a49c 1554 } else {
ae8b75dc 1555 op->args[pi++] = temp_arg(ret);
a7812ae4 1556 nb_rets = 1;
c896fe29 1557 }
34b1a49c 1558#endif
a7812ae4
PB		 1559 } else {
1560 nb_rets = 0;
c896fe29 1561 }
cd9090aa 1562 TCGOP_CALLO(op) = nb_rets;
75e8b9b7 1563
a7812ae4
PB		 1564 real_args = 0;
1565 for (i = 0; i < nargs; i++) {
7319d83a
RH		 1566 int argtype = extract32(typemask, (i + 1) * 3, 3);
1567 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
7b7d8b2d 1568 bool want_align = false;
7319d83a 1569
7b7d8b2d
RH		 1570#if defined(CONFIG_TCG_INTERPRETER)
1571 /*
1572 * Align all arguments, so that they land in predictable places
1573 * for passing off to ffi_call.
1574 */
1575 want_align = true;
1576#elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
1577 /* Some targets want aligned 64 bit args */
1578 want_align = is_64bit;
39cf05d3 1579#endif
7b7d8b2d
RH		 1580
1581 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
1582 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1583 real_args++;
1584 }
1585
1586 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
1587 /*
1588 * If stack grows up, then we will be placing successive
1589 * arguments at lower addresses, which means we need to
1590 * reverse the order compared to how we would normally
1591 * treat either big or little-endian. For those arguments
1592 * that will wind up in registers, this still works for
1593 * HPPA (the only current STACK_GROWSUP target) since the
1594 * argument registers are *also* allocated in decreasing
1595 * order. If another such target is added, this logic may
1596 * have to get more complicated to differentiate between
1597 * stack arguments and register arguments.
1598 */
02eb19d0 1599#if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
ae8b75dc
RH		 1600 op->args[pi++] = temp_arg(args[i] + 1);
1601 op->args[pi++] = temp_arg(args[i]);
c896fe29 1602#else
ae8b75dc
RH		 1603 op->args[pi++] = temp_arg(args[i]);
1604 op->args[pi++] = temp_arg(args[i] + 1);
c896fe29 1605#endif
a7812ae4 1606 real_args += 2;
2bece2c8 1607 continue;
c896fe29 1608 }
2bece2c8 1609
ae8b75dc 1610 op->args[pi++] = temp_arg(args[i]);
2bece2c8 1611 real_args++;
c896fe29 1612 }
75e8b9b7 1613 op->args[pi++] = (uintptr_t)func;
3e92aa34 1614 op->args[pi++] = (uintptr_t)info;
cd9090aa 1615 TCGOP_CALLI(op) = real_args;
a7812ae4 1616
75e8b9b7 1617 /* Make sure the fields didn't overflow. */
cd9090aa 1618 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
75e8b9b7 1619 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
2bece2c8 1620
34b1a49c
RH		 1621#if defined(__sparc__) && !defined(__arch64__) \
1622 && !defined(CONFIG_TCG_INTERPRETER)
1623 /* Free all of the parts we allocated above. */
1624 for (i = real_args = 0; i < orig_nargs; ++i) {
7319d83a
RH		 1625 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1626 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1627
34b1a49c 1628 if (is_64bit) {
085272b3
RH		 1629 tcg_temp_free_internal(args[real_args++]);
1630 tcg_temp_free_internal(args[real_args++]);
34b1a49c
RH		 1631 } else {
1632 real_args++;
1633 }
1634 }
7319d83a 1635 if ((orig_typemask & 6) == dh_typecode_i64) {
34b1a49c
RH		 1636 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1637 Note that describing these as TCGv_i64 eliminates an unnecessary
1638 zero-extension that tcg_gen_concat_i32_i64 would create. */
085272b3 1639 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
34b1a49c
RH		 1640 tcg_temp_free_i64(retl);
1641 tcg_temp_free_i64(reth);
1642 }
1643#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1644 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1645 int argtype = extract32(typemask, (i + 1) * 3, 3);
1646 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1647
1648 if (is_32bit) {
085272b3 1649 tcg_temp_free_internal(args[i]);
2bece2c8
RH		 1650 }
1651 }
1652#endif /* TCG_TARGET_EXTEND_ARGS */
c896fe29 1653}
c896fe29 1654
8fcd3692 1655static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1656{
ac3b8891 1657 int i, n;
ac3b8891 1658
ee17db83
RH		 1659 for (i = 0, n = s->nb_temps; i < n; i++) {
1660 TCGTemp *ts = &s->temps[i];
1661 TCGTempVal val = TEMP_VAL_MEM;
1662
1663 switch (ts->kind) {
c0522136
RH		 1664 case TEMP_CONST:
1665 val = TEMP_VAL_CONST;
1666 break;
ee17db83
RH		 1667 case TEMP_FIXED:
1668 val = TEMP_VAL_REG;
1669 break;
1670 case TEMP_GLOBAL:
1671 break;
1672 case TEMP_NORMAL:
1673 val = TEMP_VAL_DEAD;
1674 /* fall through */
1675 case TEMP_LOCAL:
1676 ts->mem_allocated = 0;
1677 break;
1678 default:
1679 g_assert_not_reached();
1680 }
1681 ts->val_type = val;
e8996ee0 1682 }
f8b2f202
RH		 1683
1684 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1685}
1686
f8b2f202
RH		 1687static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1688 TCGTemp *ts)
c896fe29 1689{
1807f4c4 1690 int idx = temp_idx(ts);
ac56dd48 1691
ee17db83
RH		 1692 switch (ts->kind) {
1693 case TEMP_FIXED:
1694 case TEMP_GLOBAL:
ac56dd48 1695 pstrcpy(buf, buf_size, ts->name);
ee17db83
RH		 1696 break;
1697 case TEMP_LOCAL:
f8b2f202 1698 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83
RH		 1699 break;
1700 case TEMP_NORMAL:
f8b2f202 1701 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1702 break;
c0522136
RH		 1703 case TEMP_CONST:
1704 switch (ts->type) {
1705 case TCG_TYPE_I32:
1706 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1707 break;
1708#if TCG_TARGET_REG_BITS > 32
1709 case TCG_TYPE_I64:
1710 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1711 break;
1712#endif
1713 case TCG_TYPE_V64:
1714 case TCG_TYPE_V128:
1715 case TCG_TYPE_V256:
1716 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1717 64 << (ts->type - TCG_TYPE_V64), ts->val);
1718 break;
1719 default:
1720 g_assert_not_reached();
1721 }
1722 break;
c896fe29
FB		 1723 }
1724 return buf;
1725}
1726
43439139
RH		 1727static char *tcg_get_arg_str(TCGContext *s, char *buf,
1728 int buf_size, TCGArg arg)
f8b2f202 1729{
43439139 1730 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1731}
1732
f48f3ede
BS		 1733static const char * const cond_name[] =
1734{
0aed257f
RH		 1735 [TCG_COND_NEVER] = "never",
1736 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1737 [TCG_COND_EQ] = "eq",
1738 [TCG_COND_NE] = "ne",
1739 [TCG_COND_LT] = "lt",
1740 [TCG_COND_GE] = "ge",
1741 [TCG_COND_LE] = "le",
1742 [TCG_COND_GT] = "gt",
1743 [TCG_COND_LTU] = "ltu",
1744 [TCG_COND_GEU] = "geu",
1745 [TCG_COND_LEU] = "leu",
1746 [TCG_COND_GTU] = "gtu"
1747};
1748
f713d6ad
RH		 1749static const char * const ldst_name[] =
1750{
1751 [MO_UB] = "ub",
1752 [MO_SB] = "sb",
1753 [MO_LEUW] = "leuw",
1754 [MO_LESW] = "lesw",
1755 [MO_LEUL] = "leul",
1756 [MO_LESL] = "lesl",
1757 [MO_LEQ] = "leq",
1758 [MO_BEUW] = "beuw",
1759 [MO_BESW] = "besw",
1760 [MO_BEUL] = "beul",
1761 [MO_BESL] = "besl",
1762 [MO_BEQ] = "beq",
1763};
1764
1f00b27f 1765static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 1766#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 1767 [MO_UNALN >> MO_ASHIFT] = "un+",
1768 [MO_ALIGN >> MO_ASHIFT] = "",
1769#else
1770 [MO_UNALN >> MO_ASHIFT] = "",
1771 [MO_ALIGN >> MO_ASHIFT] = "al+",
1772#endif
1773 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1774 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1775 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1776 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1777 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1778 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1779};
1780
587195bd
RH		 1781static const char bswap_flag_name[][6] = {
1782 [TCG_BSWAP_IZ] = "iz",
1783 [TCG_BSWAP_OZ] = "oz",
1784 [TCG_BSWAP_OS] = "os",
1785 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1786 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1787};
1788
b016486e
RH		 1789static inline bool tcg_regset_single(TCGRegSet d)
1790{
1791 return (d & (d - 1)) == 0;
1792}
1793
1794static inline TCGReg tcg_regset_first(TCGRegSet d)
1795{
1796 if (TCG_TARGET_NB_REGS <= 32) {
1797 return ctz32(d);
1798 } else {
1799 return ctz64(d);
1800 }
1801}
1802
1894f69a 1803static void tcg_dump_ops(TCGContext *s, bool have_prefs)
c896fe29 1804{
c896fe29 1805 char buf[128];
c45cb8bb 1806 TCGOp *op;
c45cb8bb 1807
15fa08f8 1808 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 1809 int i, k, nb_oargs, nb_iargs, nb_cargs;
1810 const TCGOpDef *def;
c45cb8bb 1811 TCGOpcode c;
bdfb460e 1812 int col = 0;
c896fe29 1813
c45cb8bb 1814 c = op->opc;
c896fe29 1815 def = &tcg_op_defs[c];
c45cb8bb 1816
765b842a 1817 if (c == INDEX_op_insn_start) {
b016486e 1818 nb_oargs = 0;
15fa08f8 1819 col += qemu_log("\n ----");
9aef40ed
RH		 1820
1821 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1822 target_ulong a;
7e4597d7 1823#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 1824 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 1825#else
efee3746 1826 a = op->args[i];
7e4597d7 1827#endif
bdfb460e 1828 col += qemu_log(" " TARGET_FMT_lx, a);
eeacee4d 1829 }
7e4597d7 1830 } else if (c == INDEX_op_call) {
3e92aa34 1831 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 1832 void *func = tcg_call_func(op);
3e92aa34 1833
c896fe29 1834 /* variable number of arguments */
cd9090aa
RH		 1835 nb_oargs = TCGOP_CALLO(op);
1836 nb_iargs = TCGOP_CALLI(op);
c896fe29 1837 nb_cargs = def->nb_cargs;
c896fe29 1838
3e92aa34
RH		 1839 col += qemu_log(" %s ", def->name);
1840
1841 /*
1842 * Print the function name from TCGHelperInfo, if available.
1843 * Note that plugins have a template function for the info,
1844 * but the actual function pointer comes from the plugin.
1845 */
3e92aa34
RH		 1846 if (func == info->func) {
1847 col += qemu_log("%s", info->name);
1848 } else {
1849 col += qemu_log("plugin(%p)", func);
1850 }
1851
8973fe43 1852 col += qemu_log(",$0x%x,$%d", info->flags, nb_oargs);
cf066674 1853 for (i = 0; i < nb_oargs; i++) {
43439139
RH		 1854 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1855 op->args[i]));
b03cce8e 1856 }
cf066674 1857 for (i = 0; i < nb_iargs; i++) {
efee3746 1858 TCGArg arg = op->args[nb_oargs + i];
cf066674
RH		 1859 const char *t = "<dummy>";
1860 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 1861 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
eeacee4d 1862 }
bdfb460e 1863 col += qemu_log(",%s", t);
e8996ee0 1864 }
b03cce8e 1865 } else {
bdfb460e 1866 col += qemu_log(" %s ", def->name);
c45cb8bb
RH		 1867
1868 nb_oargs = def->nb_oargs;
1869 nb_iargs = def->nb_iargs;
1870 nb_cargs = def->nb_cargs;
1871
d2fd745f
RH		 1872 if (def->flags & TCG_OPF_VECTOR) {
1873 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op),
1874 8 << TCGOP_VECE(op));
1875 }
1876
b03cce8e 1877 k = 0;
c45cb8bb 1878 for (i = 0; i < nb_oargs; i++) {
eeacee4d 1879 if (k != 0) {
bdfb460e 1880 col += qemu_log(",");
eeacee4d 1881 }
43439139
RH		 1882 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1883 op->args[k++]));
b03cce8e 1884 }
c45cb8bb 1885 for (i = 0; i < nb_iargs; i++) {
eeacee4d 1886 if (k != 0) {
bdfb460e 1887 col += qemu_log(",");
eeacee4d 1888 }
43439139
RH		 1889 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1890 op->args[k++]));
b03cce8e 1891 }
be210acb
RH		 1892 switch (c) {
1893 case INDEX_op_brcond_i32:
be210acb 1894 case INDEX_op_setcond_i32:
ffc5ea09 1895 case INDEX_op_movcond_i32:
ffc5ea09 1896 case INDEX_op_brcond2_i32:
be210acb 1897 case INDEX_op_setcond2_i32:
ffc5ea09 1898 case INDEX_op_brcond_i64:
be210acb 1899 case INDEX_op_setcond_i64:
ffc5ea09 1900 case INDEX_op_movcond_i64:
212be173 1901 case INDEX_op_cmp_vec:
f75da298 1902 case INDEX_op_cmpsel_vec:
efee3746
RH		 1903 if (op->args[k] < ARRAY_SIZE(cond_name)
1904 && cond_name[op->args[k]]) {
1905 col += qemu_log(",%s", cond_name[op->args[k++]]);
eeacee4d 1906 } else {
efee3746 1907 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 1908 }
f48f3ede 1909 i = 1;
be210acb 1910 break;
f713d6ad
RH		 1911 case INDEX_op_qemu_ld_i32:
1912 case INDEX_op_qemu_st_i32:
07ce0b05 1913 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 1914 case INDEX_op_qemu_ld_i64:
1915 case INDEX_op_qemu_st_i64:
59227d5d 1916 {
efee3746 1917 TCGMemOpIdx oi = op->args[k++];
14776ab5 1918 MemOp op = get_memop(oi);
59227d5d
RH		 1919 unsigned ix = get_mmuidx(oi);
1920
59c4b7e8 1921 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
bdfb460e 1922 col += qemu_log(",$0x%x,%u", op, ix);
59c4b7e8 1923 } else {
1f00b27f
SS		 1924 const char *s_al, *s_op;
1925 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 1926 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
bdfb460e 1927 col += qemu_log(",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 1928 }
1929 i = 1;
f713d6ad 1930 }
f713d6ad 1931 break;
587195bd
RH		 1932 case INDEX_op_bswap16_i32:
1933 case INDEX_op_bswap16_i64:
1934 case INDEX_op_bswap32_i32:
1935 case INDEX_op_bswap32_i64:
1936 case INDEX_op_bswap64_i64:
1937 {
1938 TCGArg flags = op->args[k];
1939 const char *name = NULL;
1940
1941 if (flags < ARRAY_SIZE(bswap_flag_name)) {
1942 name = bswap_flag_name[flags];
1943 }
1944 if (name) {
1945 col += qemu_log(",%s", name);
1946 } else {
1947 col += qemu_log(",$0x%" TCG_PRIlx, flags);
1948 }
1949 i = k = 1;
1950 }
1951 break;
be210acb 1952 default:
f48f3ede 1953 i = 0;
be210acb
RH		 1954 break;
1955 }
51e3972c
RH		 1956 switch (c) {
1957 case INDEX_op_set_label:
1958 case INDEX_op_br:
1959 case INDEX_op_brcond_i32:
1960 case INDEX_op_brcond_i64:
1961 case INDEX_op_brcond2_i32:
efee3746
RH		 1962 col += qemu_log("%s$L%d", k ? "," : "",
1963 arg_label(op->args[k])->id);
51e3972c
RH		 1964 i++, k++;
1965 break;
1966 default:
1967 break;
1968 }
1969 for (; i < nb_cargs; i++, k++) {
efee3746 1970 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]);
bdfb460e
RH		 1971 }
1972 }
bdfb460e 1973
1894f69a 1974 if (have_prefs || op->life) {
7606488c
RF		 1975
1976 QemuLogFile *logfile;
1977
1978 rcu_read_lock();
d73415a3 1979 logfile = qatomic_rcu_read(&qemu_logfile);
7606488c
RF		 1980 if (logfile) {
1981 for (; col < 40; ++col) {
1982 putc(' ', logfile->fd);
1983 }
bdfb460e 1984 }
7606488c 1985 rcu_read_unlock();
1894f69a
RH		 1986 }
1987
1988 if (op->life) {
1989 unsigned life = op->life;
bdfb460e
RH		 1990
1991 if (life & (SYNC_ARG * 3)) {
1992 qemu_log(" sync:");
1993 for (i = 0; i < 2; ++i) {
1994 if (life & (SYNC_ARG << i)) {
1995 qemu_log(" %d", i);
1996 }
1997 }
1998 }
1999 life /= DEAD_ARG;
2000 if (life) {
2001 qemu_log(" dead:");
2002 for (i = 0; life; ++i, life >>= 1) {
2003 if (life & 1) {
2004 qemu_log(" %d", i);
2005 }
2006 }
b03cce8e 2007 }
c896fe29 2008 }
1894f69a
RH		 2009
2010 if (have_prefs) {
2011 for (i = 0; i < nb_oargs; ++i) {
2012 TCGRegSet set = op->output_pref[i];
2013
2014 if (i == 0) {
2015 qemu_log(" pref=");
2016 } else {
2017 qemu_log(",");
2018 }
2019 if (set == 0) {
2020 qemu_log("none");
2021 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2022 qemu_log("all");
2023#ifdef CONFIG_DEBUG_TCG
2024 } else if (tcg_regset_single(set)) {
2025 TCGReg reg = tcg_regset_first(set);
2026 qemu_log("%s", tcg_target_reg_names[reg]);
2027#endif
2028 } else if (TCG_TARGET_NB_REGS <= 32) {
2029 qemu_log("%#x", (uint32_t)set);
2030 } else {
2031 qemu_log("%#" PRIx64, (uint64_t)set);
2032 }
2033 }
2034 }
2035
eeacee4d 2036 qemu_log("\n");
c896fe29
FB		 2037 }
2038}
2039
2040/* we give more priority to constraints with less registers */
2041static int get_constraint_priority(const TCGOpDef *def, int k)
2042{
74a11790
RH		 2043 const TCGArgConstraint *arg_ct = &def->args_ct[k];
2044 int n;
c896fe29 2045
bc2b17e6 2046 if (arg_ct->oalias) {
c896fe29
FB		 2047 /* an alias is equivalent to a single register */
2048 n = 1;
2049 } else {
74a11790 2050 n = ctpop64(arg_ct->regs);
c896fe29
FB		 2051 }
2052 return TCG_TARGET_NB_REGS - n + 1;
2053}
2054
2055/* sort from highest priority to lowest */
2056static void sort_constraints(TCGOpDef *def, int start, int n)
2057{
66792f90
RH		 2058 int i, j;
2059 TCGArgConstraint *a = def->args_ct;
c896fe29 2060
66792f90
RH		 2061 for (i = 0; i < n; i++) {
2062 a[start + i].sort_index = start + i;
2063 }
2064 if (n <= 1) {
c896fe29 2065 return;
66792f90
RH		 2066 }
2067 for (i = 0; i < n - 1; i++) {
2068 for (j = i + 1; j < n; j++) {
2069 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2070 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2071 if (p1 < p2) {
66792f90
RH		 2072 int tmp = a[start + i].sort_index;
2073 a[start + i].sort_index = a[start + j].sort_index;
2074 a[start + j].sort_index = tmp;
c896fe29
FB		 2075 }
2076 }
2077 }
2078}
2079
f69d277e 2080static void process_op_defs(TCGContext *s)
c896fe29 2081{
a9751609 2082 TCGOpcode op;
c896fe29 2083
f69d277e
RH		 2084 for (op = 0; op < NB_OPS; op++) {
2085 TCGOpDef *def = &tcg_op_defs[op];
2086 const TCGTargetOpDef *tdefs;
069ea736 2087 int i, nb_args;
f69d277e
RH		 2088
2089 if (def->flags & TCG_OPF_NOT_PRESENT) {
2090 continue;
2091 }
2092
c896fe29 2093 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2094 if (nb_args == 0) {
2095 continue;
2096 }
2097
4c22e840
RH		 2098 /*
2099 * Macro magic should make it impossible, but double-check that
2100 * the array index is in range. Since the signness of an enum
2101 * is implementation defined, force the result to unsigned.
2102 */
2103 unsigned con_set = tcg_target_op_def(op);
2104 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2105 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2106
2107 for (i = 0; i < nb_args; i++) {
2108 const char *ct_str = tdefs->args_ct_str[i];
2109 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2110 tcg_debug_assert(ct_str != NULL);
f69d277e 2111
17280ff4
RH		 2112 while (*ct_str != '\0') {
2113 switch(*ct_str) {
2114 case '0' ... '9':
2115 {
2116 int oarg = *ct_str - '0';
2117 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2118 tcg_debug_assert(oarg < def->nb_oargs);
74a11790 2119 tcg_debug_assert(def->args_ct[oarg].regs != 0);
17280ff4 2120 def->args_ct[i] = def->args_ct[oarg];
bc2b17e6
RH		 2121 /* The output sets oalias. */
2122 def->args_ct[oarg].oalias = true;
17280ff4 2123 def->args_ct[oarg].alias_index = i;
bc2b17e6
RH		 2124 /* The input sets ialias. */
2125 def->args_ct[i].ialias = true;
17280ff4 2126 def->args_ct[i].alias_index = oarg;
c896fe29 2127 }
17280ff4
RH		 2128 ct_str++;
2129 break;
2130 case '&':
bc2b17e6 2131 def->args_ct[i].newreg = true;
17280ff4
RH		 2132 ct_str++;
2133 break;
2134 case 'i':
2135 def->args_ct[i].ct |= TCG_CT_CONST;
2136 ct_str++;
2137 break;
358b4923 2138
358b4923
RH		 2139 /* Include all of the target-specific constraints. */
2140
2141#undef CONST
2142#define CONST(CASE, MASK) \
2143 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2144#define REGS(CASE, MASK) \
2145 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2146
2147#include "tcg-target-con-str.h"
2148
2149#undef REGS
2150#undef CONST
17280ff4 2151 default:
17280ff4 2152 /* Typo in TCGTargetOpDef constraint. */
358b4923 2153 g_assert_not_reached();
c896fe29
FB		 2154 }
2155 }
2156 }
2157
c68aaa18 2158 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2159 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2160
c896fe29
FB		 2161 /* sort the constraints (XXX: this is just an heuristic) */
2162 sort_constraints(def, 0, def->nb_oargs);
2163 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2164 }
c896fe29
FB		 2165}
2166
0c627cdc
RH		 2167void tcg_op_remove(TCGContext *s, TCGOp *op)
2168{
d88a117e
RH		 2169 TCGLabel *label;
2170
2171 switch (op->opc) {
2172 case INDEX_op_br:
2173 label = arg_label(op->args[0]);
2174 label->refs--;
2175 break;
2176 case INDEX_op_brcond_i32:
2177 case INDEX_op_brcond_i64:
2178 label = arg_label(op->args[3]);
2179 label->refs--;
2180 break;
2181 case INDEX_op_brcond2_i32:
2182 label = arg_label(op->args[5]);
2183 label->refs--;
2184 break;
2185 default:
2186 break;
2187 }
2188
15fa08f8
RH		 2189 QTAILQ_REMOVE(&s->ops, op, link);
2190 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2191 s->nb_ops--;
0c627cdc
RH		 2192
2193#ifdef CONFIG_PROFILER
d73415a3 2194 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2195#endif
2196}
2197
a80cdd31
RH		 2198void tcg_remove_ops_after(TCGOp *op)
2199{
2200 TCGContext *s = tcg_ctx;
2201
2202 while (true) {
2203 TCGOp *last = tcg_last_op();
2204 if (last == op) {
2205 return;
2206 }
2207 tcg_op_remove(s, last);
2208 }
2209}
2210
15fa08f8 2211static TCGOp *tcg_op_alloc(TCGOpcode opc)
5a18407f 2212{
15fa08f8
RH		 2213 TCGContext *s = tcg_ctx;
2214 TCGOp *op;
5a18407f 2215
15fa08f8
RH		 2216 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2217 op = tcg_malloc(sizeof(TCGOp));
2218 } else {
2219 op = QTAILQ_FIRST(&s->free_ops);
2220 QTAILQ_REMOVE(&s->free_ops, op, link);
2221 }
2222 memset(op, 0, offsetof(TCGOp, link));
2223 op->opc = opc;
abebf925 2224 s->nb_ops++;
5a18407f 2225
15fa08f8
RH		 2226 return op;
2227}
2228
2229TCGOp *tcg_emit_op(TCGOpcode opc)
2230{
2231 TCGOp *op = tcg_op_alloc(opc);
2232 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2233 return op;
2234}
5a18407f 2235
ac1043f6 2236TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
15fa08f8
RH		 2237{
2238 TCGOp *new_op = tcg_op_alloc(opc);
2239 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2240 return new_op;
2241}
2242
ac1043f6 2243TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
5a18407f 2244{
15fa08f8
RH		 2245 TCGOp *new_op = tcg_op_alloc(opc);
2246 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2247 return new_op;
2248}
2249
b4fc67c7
RH		 2250/* Reachable analysis : remove unreachable code. */
2251static void reachable_code_pass(TCGContext *s)
2252{
2253 TCGOp *op, *op_next;
2254 bool dead = false;
2255
2256 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2257 bool remove = dead;
2258 TCGLabel *label;
b4fc67c7
RH		 2259
2260 switch (op->opc) {
2261 case INDEX_op_set_label:
2262 label = arg_label(op->args[0]);
2263 if (label->refs == 0) {
2264 /*
2265 * While there is an occasional backward branch, virtually
2266 * all branches generated by the translators are forward.
2267 * Which means that generally we will have already removed
2268 * all references to the label that will be, and there is
2269 * little to be gained by iterating.
2270 */
2271 remove = true;
2272 } else {
2273 /* Once we see a label, insns become live again. */
2274 dead = false;
2275 remove = false;
2276
2277 /*
2278 * Optimization can fold conditional branches to unconditional.
2279 * If we find a label with one reference which is preceded by
2280 * an unconditional branch to it, remove both. This needed to
2281 * wait until the dead code in between them was removed.
2282 */
2283 if (label->refs == 1) {
eae3eb3e 2284 TCGOp *op_prev = QTAILQ_PREV(op, link);
b4fc67c7
RH		 2285 if (op_prev->opc == INDEX_op_br &&
2286 label == arg_label(op_prev->args[0])) {
2287 tcg_op_remove(s, op_prev);
2288 remove = true;
2289 }
2290 }
2291 }
2292 break;
2293
2294 case INDEX_op_br:
2295 case INDEX_op_exit_tb:
2296 case INDEX_op_goto_ptr:
2297 /* Unconditional branches; everything following is dead. */
2298 dead = true;
2299 break;
2300
2301 case INDEX_op_call:
2302 /* Notice noreturn helper calls, raising exceptions. */
90163900 2303 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2304 dead = true;
2305 }
2306 break;
2307
2308 case INDEX_op_insn_start:
2309 /* Never remove -- we need to keep these for unwind. */
2310 remove = false;
2311 break;
2312
2313 default:
2314 break;
2315 }
2316
2317 if (remove) {
2318 tcg_op_remove(s, op);
2319 }
2320 }
2321}
2322
c70fbf0a
RH		 2323#define TS_DEAD 1
2324#define TS_MEM 2
2325
5a18407f
RH		 2326#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2327#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2328
25f49c5f
RH		 2329/* For liveness_pass_1, the register preferences for a given temp. */
2330static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2331{
2332 return ts->state_ptr;
2333}
2334
2335/* For liveness_pass_1, reset the preferences for a given temp to the
2336 * maximal regset for its type.
2337 */
2338static inline void la_reset_pref(TCGTemp *ts)
2339{
2340 *la_temp_pref(ts)
2341 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2342}
2343
9c43b68d
AJ		 2344/* liveness analysis: end of function: all temps are dead, and globals
2345 should be in memory. */
2616c808 2346static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2347{
b83eabea
RH		 2348 int i;
2349
2350 for (i = 0; i < ng; ++i) {
2351 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2352 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2353 }
2354 for (i = ng; i < nt; ++i) {
2355 s->temps[i].state = TS_DEAD;
25f49c5f 2356 la_reset_pref(&s->temps[i]);
b83eabea 2357 }
c896fe29
FB		 2358}
2359
9c43b68d
AJ		 2360/* liveness analysis: end of basic block: all temps are dead, globals
2361 and local temps should be in memory. */
2616c808 2362static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2363{
b83eabea 2364 int i;
641d5fbe 2365
ee17db83
RH		 2366 for (i = 0; i < nt; ++i) {
2367 TCGTemp *ts = &s->temps[i];
2368 int state;
2369
2370 switch (ts->kind) {
2371 case TEMP_FIXED:
2372 case TEMP_GLOBAL:
2373 case TEMP_LOCAL:
2374 state = TS_DEAD | TS_MEM;
2375 break;
2376 case TEMP_NORMAL:
c0522136 2377 case TEMP_CONST:
ee17db83
RH		 2378 state = TS_DEAD;
2379 break;
2380 default:
2381 g_assert_not_reached();
2382 }
2383 ts->state = state;
2384 la_reset_pref(ts);
641d5fbe
FB		 2385 }
2386}
2387
f65a061c
RH		 2388/* liveness analysis: sync globals back to memory. */
2389static void la_global_sync(TCGContext *s, int ng)
2390{
2391 int i;
2392
2393 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2394 int state = s->temps[i].state;
2395 s->temps[i].state = state | TS_MEM;
2396 if (state == TS_DEAD) {
2397 /* If the global was previously dead, reset prefs. */
2398 la_reset_pref(&s->temps[i]);
2399 }
f65a061c
RH		 2400 }
2401}
2402
b4cb76e6
RH		 2403/*
2404 * liveness analysis: conditional branch: all temps are dead,
2405 * globals and local temps should be synced.
2406 */
2407static void la_bb_sync(TCGContext *s, int ng, int nt)
2408{
2409 la_global_sync(s, ng);
2410
2411 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2412 TCGTemp *ts = &s->temps[i];
2413 int state;
2414
2415 switch (ts->kind) {
2416 case TEMP_LOCAL:
2417 state = ts->state;
2418 ts->state = state | TS_MEM;
b4cb76e6
RH		 2419 if (state != TS_DEAD) {
2420 continue;
2421 }
c0522136
RH		 2422 break;
2423 case TEMP_NORMAL:
b4cb76e6 2424 s->temps[i].state = TS_DEAD;
c0522136
RH		 2425 break;
2426 case TEMP_CONST:
2427 continue;
2428 default:
2429 g_assert_not_reached();
b4cb76e6
RH		 2430 }
2431 la_reset_pref(&s->temps[i]);
2432 }
2433}
2434
f65a061c
RH		 2435/* liveness analysis: sync globals back to memory and kill. */
2436static void la_global_kill(TCGContext *s, int ng)
2437{
2438 int i;
2439
2440 for (i = 0; i < ng; i++) {
2441 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2442 la_reset_pref(&s->temps[i]);
2443 }
2444}
2445
2446/* liveness analysis: note live globals crossing calls. */
2447static void la_cross_call(TCGContext *s, int nt)
2448{
2449 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2450 int i;
2451
2452 for (i = 0; i < nt; i++) {
2453 TCGTemp *ts = &s->temps[i];
2454 if (!(ts->state & TS_DEAD)) {
2455 TCGRegSet *pset = la_temp_pref(ts);
2456 TCGRegSet set = *pset;
2457
2458 set &= mask;
2459 /* If the combination is not possible, restart. */
2460 if (set == 0) {
2461 set = tcg_target_available_regs[ts->type] & mask;
2462 }
2463 *pset = set;
2464 }
f65a061c
RH		 2465 }
2466}
2467
a1b3c48d 2468/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2469 given input arguments is dead. Instructions updating dead
2470 temporaries are removed. */
b83eabea 2471static void liveness_pass_1(TCGContext *s)
c896fe29 2472{
c70fbf0a 2473 int nb_globals = s->nb_globals;
2616c808 2474 int nb_temps = s->nb_temps;
15fa08f8 2475 TCGOp *op, *op_prev;
25f49c5f
RH		 2476 TCGRegSet *prefs;
2477 int i;
2478
2479 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2480 for (i = 0; i < nb_temps; ++i) {
2481 s->temps[i].state_ptr = prefs + i;
2482 }
a1b3c48d 2483
ae36a246 2484 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2485 la_func_end(s, nb_globals, nb_temps);
c896fe29 2486
eae3eb3e 2487 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2488 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2489 TCGOpcode opc_new, opc_new2;
2490 bool have_opc_new2;
a1b3c48d 2491 TCGLifeData arg_life = 0;
25f49c5f 2492 TCGTemp *ts;
c45cb8bb
RH		 2493 TCGOpcode opc = op->opc;
2494 const TCGOpDef *def = &tcg_op_defs[opc];
2495
c45cb8bb 2496 switch (opc) {
c896fe29 2497 case INDEX_op_call:
c6e113f5
FB		 2498 {
2499 int call_flags;
25f49c5f 2500 int nb_call_regs;
c896fe29 2501
cd9090aa
RH		 2502 nb_oargs = TCGOP_CALLO(op);
2503 nb_iargs = TCGOP_CALLI(op);
90163900 2504 call_flags = tcg_call_flags(op);
c6e113f5 2505
c45cb8bb 2506 /* pure functions can be removed if their result is unused */
78505279 2507 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2508 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2509 ts = arg_temp(op->args[i]);
2510 if (ts->state != TS_DEAD) {
c6e113f5 2511 goto do_not_remove_call;
9c43b68d 2512 }
c6e113f5 2513 }
c45cb8bb 2514 goto do_remove;
152c35aa
RH		 2515 }
2516 do_not_remove_call:
c896fe29 2517
25f49c5f 2518 /* Output args are dead. */
152c35aa 2519 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2520 ts = arg_temp(op->args[i]);
2521 if (ts->state & TS_DEAD) {
152c35aa
RH		 2522 arg_life |= DEAD_ARG << i;
2523 }
25f49c5f 2524 if (ts->state & TS_MEM) {
152c35aa 2525 arg_life |= SYNC_ARG << i;
c6e113f5 2526 }
25f49c5f
RH		 2527 ts->state = TS_DEAD;
2528 la_reset_pref(ts);
2529
2530 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2531 op->output_pref[i] = 0;
152c35aa 2532 }
78505279 2533
152c35aa
RH		 2534 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2535 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2536 la_global_kill(s, nb_globals);
152c35aa 2537 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2538 la_global_sync(s, nb_globals);
152c35aa 2539 }
b9c18f56 2540
25f49c5f 2541 /* Record arguments that die in this helper. */
152c35aa 2542 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f
RH		 2543 ts = arg_temp(op->args[i]);
2544 if (ts && ts->state & TS_DEAD) {
152c35aa 2545 arg_life |= DEAD_ARG << i;
c6e113f5 2546 }
152c35aa 2547 }
25f49c5f
RH		 2548
2549 /* For all live registers, remove call-clobbered prefs. */
2550 la_cross_call(s, nb_temps);
2551
2552 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2553
2554 /* Input arguments are live for preceding opcodes. */
2555 for (i = 0; i < nb_iargs; i++) {
2556 ts = arg_temp(op->args[i + nb_oargs]);
2557 if (ts && ts->state & TS_DEAD) {
2558 /* For those arguments that die, and will be allocated
2559 * in registers, clear the register set for that arg,
2560 * to be filled in below. For args that will be on
2561 * the stack, reset to any available reg.
2562 */
2563 *la_temp_pref(ts)
2564 = (i < nb_call_regs ? 0 :
2565 tcg_target_available_regs[ts->type]);
2566 ts->state &= ~TS_DEAD;
2567 }
2568 }
2569
2570 /* For each input argument, add its input register to prefs.
2571 If a temp is used once, this produces a single set bit. */
2572 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2573 ts = arg_temp(op->args[i + nb_oargs]);
2574 if (ts) {
2575 tcg_regset_set_reg(*la_temp_pref(ts),
2576 tcg_target_call_iarg_regs[i]);
c19f47bf 2577 }
c896fe29 2578 }
c896fe29 2579 }
c896fe29 2580 break;
765b842a 2581 case INDEX_op_insn_start:
c896fe29 2582 break;
5ff9d6a4 2583 case INDEX_op_discard:
5ff9d6a4 2584 /* mark the temporary as dead */
25f49c5f
RH		 2585 ts = arg_temp(op->args[0]);
2586 ts->state = TS_DEAD;
2587 la_reset_pref(ts);
5ff9d6a4 2588 break;
1305c451
RH		 2589
2590 case INDEX_op_add2_i32:
c45cb8bb 2591 opc_new = INDEX_op_add_i32;
f1fae40c 2592 goto do_addsub2;
1305c451 2593 case INDEX_op_sub2_i32:
c45cb8bb 2594 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 2595 goto do_addsub2;
2596 case INDEX_op_add2_i64:
c45cb8bb 2597 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 2598 goto do_addsub2;
2599 case INDEX_op_sub2_i64:
c45cb8bb 2600 opc_new = INDEX_op_sub_i64;
f1fae40c 2601 do_addsub2:
1305c451
RH		 2602 nb_iargs = 4;
2603 nb_oargs = 2;
2604 /* Test if the high part of the operation is dead, but not
2605 the low part. The result can be optimized to a simple
2606 add or sub. This happens often for x86_64 guest when the
2607 cpu mode is set to 32 bit. */
b83eabea
RH		 2608 if (arg_temp(op->args[1])->state == TS_DEAD) {
2609 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 2610 goto do_remove;
2611 }
c45cb8bb
RH		 2612 /* Replace the opcode and adjust the args in place,
2613 leaving 3 unused args at the end. */
2614 op->opc = opc = opc_new;
efee3746
RH		 2615 op->args[1] = op->args[2];
2616 op->args[2] = op->args[4];
1305c451
RH		 2617 /* Fall through and mark the single-word operation live. */
2618 nb_iargs = 2;
2619 nb_oargs = 1;
2620 }
2621 goto do_not_remove;
2622
1414968a 2623 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 2624 opc_new = INDEX_op_mul_i32;
2625 opc_new2 = INDEX_op_muluh_i32;
2626 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 2627 goto do_mul2;
f1fae40c 2628 case INDEX_op_muls2_i32:
c45cb8bb
RH		 2629 opc_new = INDEX_op_mul_i32;
2630 opc_new2 = INDEX_op_mulsh_i32;
2631 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 2632 goto do_mul2;
2633 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 2634 opc_new = INDEX_op_mul_i64;
2635 opc_new2 = INDEX_op_muluh_i64;
2636 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 2637 goto do_mul2;
f1fae40c 2638 case INDEX_op_muls2_i64:
c45cb8bb
RH		 2639 opc_new = INDEX_op_mul_i64;
2640 opc_new2 = INDEX_op_mulsh_i64;
2641 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 2642 goto do_mul2;
f1fae40c 2643 do_mul2:
1414968a
RH		 2644 nb_iargs = 2;
2645 nb_oargs = 2;
b83eabea
RH		 2646 if (arg_temp(op->args[1])->state == TS_DEAD) {
2647 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 2648 /* Both parts of the operation are dead. */
1414968a
RH		 2649 goto do_remove;
2650 }
03271524 2651 /* The high part of the operation is dead; generate the low. */
c45cb8bb 2652 op->opc = opc = opc_new;
efee3746
RH		 2653 op->args[1] = op->args[2];
2654 op->args[2] = op->args[3];
b83eabea 2655 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 2656 /* The low part of the operation is dead; generate the high. */
2657 op->opc = opc = opc_new2;
efee3746
RH		 2658 op->args[0] = op->args[1];
2659 op->args[1] = op->args[2];
2660 op->args[2] = op->args[3];
03271524
RH		 2661 } else {
2662 goto do_not_remove;
1414968a 2663 }
03271524
RH		 2664 /* Mark the single-word operation live. */
2665 nb_oargs = 1;
1414968a
RH		 2666 goto do_not_remove;
2667
c896fe29 2668 default:
1305c451 2669 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 2670 nb_iargs = def->nb_iargs;
2671 nb_oargs = def->nb_oargs;
c896fe29 2672
49516bc0
AJ		 2673 /* Test if the operation can be removed because all
2674 its outputs are dead. We assume that nb_oargs == 0
2675 implies side effects */
2676 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 2677 for (i = 0; i < nb_oargs; i++) {
b83eabea 2678 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 2679 goto do_not_remove;
9c43b68d 2680 }
49516bc0 2681 }
152c35aa
RH		 2682 goto do_remove;
2683 }
2684 goto do_not_remove;
49516bc0 2685
152c35aa
RH		 2686 do_remove:
2687 tcg_op_remove(s, op);
2688 break;
2689
2690 do_not_remove:
152c35aa 2691 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2692 ts = arg_temp(op->args[i]);
2693
2694 /* Remember the preference of the uses that followed. */
2695 op->output_pref[i] = *la_temp_pref(ts);
2696
2697 /* Output args are dead. */
2698 if (ts->state & TS_DEAD) {
152c35aa 2699 arg_life |= DEAD_ARG << i;
49516bc0 2700 }
25f49c5f 2701 if (ts->state & TS_MEM) {
152c35aa
RH		 2702 arg_life |= SYNC_ARG << i;
2703 }
25f49c5f
RH		 2704 ts->state = TS_DEAD;
2705 la_reset_pref(ts);
152c35aa 2706 }
49516bc0 2707
25f49c5f 2708 /* If end of basic block, update. */
ae36a246
RH		 2709 if (def->flags & TCG_OPF_BB_EXIT) {
2710 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 2711 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2712 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 2713 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 2714 la_bb_end(s, nb_globals, nb_temps);
152c35aa 2715 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 2716 la_global_sync(s, nb_globals);
25f49c5f
RH		 2717 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2718 la_cross_call(s, nb_temps);
2719 }
152c35aa
RH		 2720 }
2721
25f49c5f 2722 /* Record arguments that die in this opcode. */
152c35aa 2723 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2724 ts = arg_temp(op->args[i]);
2725 if (ts->state & TS_DEAD) {
152c35aa 2726 arg_life |= DEAD_ARG << i;
c896fe29 2727 }
c896fe29 2728 }
25f49c5f
RH		 2729
2730 /* Input arguments are live for preceding opcodes. */
152c35aa 2731 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2732 ts = arg_temp(op->args[i]);
2733 if (ts->state & TS_DEAD) {
2734 /* For operands that were dead, initially allow
2735 all regs for the type. */
2736 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2737 ts->state &= ~TS_DEAD;
2738 }
2739 }
2740
2741 /* Incorporate constraints for this operand. */
2742 switch (opc) {
2743 case INDEX_op_mov_i32:
2744 case INDEX_op_mov_i64:
2745 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2746 have proper constraints. That said, special case
2747 moves to propagate preferences backward. */
2748 if (IS_DEAD_ARG(1)) {
2749 *la_temp_pref(arg_temp(op->args[0]))
2750 = *la_temp_pref(arg_temp(op->args[1]));
2751 }
2752 break;
2753
2754 default:
2755 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2756 const TCGArgConstraint *ct = &def->args_ct[i];
2757 TCGRegSet set, *pset;
2758
2759 ts = arg_temp(op->args[i]);
2760 pset = la_temp_pref(ts);
2761 set = *pset;
2762
9be0d080 2763 set &= ct->regs;
bc2b17e6 2764 if (ct->ialias) {
25f49c5f
RH		 2765 set &= op->output_pref[ct->alias_index];
2766 }
2767 /* If the combination is not possible, restart. */
2768 if (set == 0) {
9be0d080 2769 set = ct->regs;
25f49c5f
RH		 2770 }
2771 *pset = set;
2772 }
2773 break;
152c35aa 2774 }
c896fe29
FB		 2775 break;
2776 }
bee158cb 2777 op->life = arg_life;
1ff0a2c5 2778 }
c896fe29 2779}
c896fe29 2780
5a18407f 2781/* Liveness analysis: Convert indirect regs to direct temporaries. */
b83eabea 2782static bool liveness_pass_2(TCGContext *s)
5a18407f
RH		 2783{
2784 int nb_globals = s->nb_globals;
15fa08f8 2785 int nb_temps, i;
5a18407f 2786 bool changes = false;
15fa08f8 2787 TCGOp *op, *op_next;
5a18407f 2788
5a18407f
RH		 2789 /* Create a temporary for each indirect global. */
2790 for (i = 0; i < nb_globals; ++i) {
2791 TCGTemp *its = &s->temps[i];
2792 if (its->indirect_reg) {
2793 TCGTemp *dts = tcg_temp_alloc(s);
2794 dts->type = its->type;
2795 dts->base_type = its->base_type;
b83eabea
RH		 2796 its->state_ptr = dts;
2797 } else {
2798 its->state_ptr = NULL;
5a18407f 2799 }
b83eabea
RH		 2800 /* All globals begin dead. */
2801 its->state = TS_DEAD;
2802 }
2803 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2804 TCGTemp *its = &s->temps[i];
2805 its->state_ptr = NULL;
2806 its->state = TS_DEAD;
5a18407f 2807 }
5a18407f 2808
15fa08f8 2809 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 2810 TCGOpcode opc = op->opc;
2811 const TCGOpDef *def = &tcg_op_defs[opc];
2812 TCGLifeData arg_life = op->life;
2813 int nb_iargs, nb_oargs, call_flags;
b83eabea 2814 TCGTemp *arg_ts, *dir_ts;
5a18407f 2815
5a18407f 2816 if (opc == INDEX_op_call) {
cd9090aa
RH		 2817 nb_oargs = TCGOP_CALLO(op);
2818 nb_iargs = TCGOP_CALLI(op);
90163900 2819 call_flags = tcg_call_flags(op);
5a18407f
RH		 2820 } else {
2821 nb_iargs = def->nb_iargs;
2822 nb_oargs = def->nb_oargs;
2823
2824 /* Set flags similar to how calls require. */
b4cb76e6
RH		 2825 if (def->flags & TCG_OPF_COND_BRANCH) {
2826 /* Like reading globals: sync_globals */
2827 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2828 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 2829 /* Like writing globals: save_globals */
2830 call_flags = 0;
2831 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2832 /* Like reading globals: sync_globals */
2833 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2834 } else {
2835 /* No effect on globals. */
2836 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2837 TCG_CALL_NO_WRITE_GLOBALS);
2838 }
2839 }
2840
2841 /* Make sure that input arguments are available. */
2842 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2843 arg_ts = arg_temp(op->args[i]);
2844 if (arg_ts) {
2845 dir_ts = arg_ts->state_ptr;
2846 if (dir_ts && arg_ts->state == TS_DEAD) {
2847 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
5a18407f
RH		 2848 ? INDEX_op_ld_i32
2849 : INDEX_op_ld_i64);
ac1043f6 2850 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
5a18407f 2851
b83eabea
RH		 2852 lop->args[0] = temp_arg(dir_ts);
2853 lop->args[1] = temp_arg(arg_ts->mem_base);
2854 lop->args[2] = arg_ts->mem_offset;
5a18407f
RH		 2855
2856 /* Loaded, but synced with memory. */
b83eabea 2857 arg_ts->state = TS_MEM;
5a18407f
RH		 2858 }
2859 }
2860 }
2861
2862 /* Perform input replacement, and mark inputs that became dead.
2863 No action is required except keeping temp_state up to date
2864 so that we reload when needed. */
2865 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2866 arg_ts = arg_temp(op->args[i]);
2867 if (arg_ts) {
2868 dir_ts = arg_ts->state_ptr;
2869 if (dir_ts) {
2870 op->args[i] = temp_arg(dir_ts);
5a18407f
RH		 2871 changes = true;
2872 if (IS_DEAD_ARG(i)) {
b83eabea 2873 arg_ts->state = TS_DEAD;
5a18407f
RH		 2874 }
2875 }
2876 }
2877 }
2878
2879 /* Liveness analysis should ensure that the following are
2880 all correct, for call sites and basic block end points. */
2881 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2882 /* Nothing to do */
2883 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2884 for (i = 0; i < nb_globals; ++i) {
2885 /* Liveness should see that globals are synced back,
2886 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 2887 arg_ts = &s->temps[i];
2888 tcg_debug_assert(arg_ts->state_ptr == 0
2889 || arg_ts->state != 0);
5a18407f
RH		 2890 }
2891 } else {
2892 for (i = 0; i < nb_globals; ++i) {
2893 /* Liveness should see that globals are saved back,
2894 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 2895 arg_ts = &s->temps[i];
2896 tcg_debug_assert(arg_ts->state_ptr == 0
2897 || arg_ts->state == TS_DEAD);
5a18407f
RH		 2898 }
2899 }
2900
2901 /* Outputs become available. */
61f15c48
RH		 2902 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2903 arg_ts = arg_temp(op->args[0]);
b83eabea 2904 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 2905 if (dir_ts) {
2906 op->args[0] = temp_arg(dir_ts);
2907 changes = true;
2908
2909 /* The output is now live and modified. */
2910 arg_ts->state = 0;
2911
2912 if (NEED_SYNC_ARG(0)) {
2913 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2914 ? INDEX_op_st_i32
2915 : INDEX_op_st_i64);
2916 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2917 TCGTemp *out_ts = dir_ts;
2918
2919 if (IS_DEAD_ARG(0)) {
2920 out_ts = arg_temp(op->args[1]);
2921 arg_ts->state = TS_DEAD;
2922 tcg_op_remove(s, op);
2923 } else {
2924 arg_ts->state = TS_MEM;
2925 }
2926
2927 sop->args[0] = temp_arg(out_ts);
2928 sop->args[1] = temp_arg(arg_ts->mem_base);
2929 sop->args[2] = arg_ts->mem_offset;
2930 } else {
2931 tcg_debug_assert(!IS_DEAD_ARG(0));
2932 }
5a18407f 2933 }
61f15c48
RH		 2934 } else {
2935 for (i = 0; i < nb_oargs; i++) {
2936 arg_ts = arg_temp(op->args[i]);
2937 dir_ts = arg_ts->state_ptr;
2938 if (!dir_ts) {
2939 continue;
2940 }
2941 op->args[i] = temp_arg(dir_ts);
2942 changes = true;
5a18407f 2943
61f15c48
RH		 2944 /* The output is now live and modified. */
2945 arg_ts->state = 0;
5a18407f 2946
61f15c48
RH		 2947 /* Sync outputs upon their last write. */
2948 if (NEED_SYNC_ARG(i)) {
2949 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2950 ? INDEX_op_st_i32
2951 : INDEX_op_st_i64);
2952 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
5a18407f 2953
61f15c48
RH		 2954 sop->args[0] = temp_arg(dir_ts);
2955 sop->args[1] = temp_arg(arg_ts->mem_base);
2956 sop->args[2] = arg_ts->mem_offset;
5a18407f 2957
61f15c48
RH		 2958 arg_ts->state = TS_MEM;
2959 }
2960 /* Drop outputs that are dead. */
2961 if (IS_DEAD_ARG(i)) {
2962 arg_ts->state = TS_DEAD;
2963 }
5a18407f
RH		 2964 }
2965 }
2966 }
2967
2968 return changes;
2969}
2970
8d8fdbae 2971#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 2972static void dump_regs(TCGContext *s)
2973{
2974 TCGTemp *ts;
2975 int i;
2976 char buf[64];
2977
2978 for(i = 0; i < s->nb_temps; i++) {
2979 ts = &s->temps[i];
43439139 2980 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
c896fe29
FB		 2981 switch(ts->val_type) {
2982 case TEMP_VAL_REG:
2983 printf("%s", tcg_target_reg_names[ts->reg]);
2984 break;
2985 case TEMP_VAL_MEM:
b3a62939
RH		 2986 printf("%d(%s)", (int)ts->mem_offset,
2987 tcg_target_reg_names[ts->mem_base->reg]);
c896fe29
FB		 2988 break;
2989 case TEMP_VAL_CONST:
bdb38b95 2990 printf("$0x%" PRIx64, ts->val);
c896fe29
FB		 2991 break;
2992 case TEMP_VAL_DEAD:
2993 printf("D");
2994 break;
2995 default:
2996 printf("???");
2997 break;
2998 }
2999 printf("\n");
3000 }
3001
3002 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
f8b2f202 3003 if (s->reg_to_temp[i] != NULL) {
c896fe29
FB		 3004 printf("%s: %s\n",
3005 tcg_target_reg_names[i],
f8b2f202 3006 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
c896fe29
FB		 3007 }
3008 }
3009}
3010
3011static void check_regs(TCGContext *s)
3012{
869938ae 3013 int reg;
b6638662 3014 int k;
c896fe29
FB		 3015 TCGTemp *ts;
3016 char buf[64];
3017
f8b2f202
RH		 3018 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
3019 ts = s->reg_to_temp[reg];
3020 if (ts != NULL) {
3021 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
c896fe29
FB		 3022 printf("Inconsistency for register %s:\n",
3023 tcg_target_reg_names[reg]);
b03cce8e 3024 goto fail;
c896fe29
FB		 3025 }
3026 }
3027 }
f8b2f202 3028 for (k = 0; k < s->nb_temps; k++) {
c896fe29 3029 ts = &s->temps[k];
ee17db83
RH		 3030 if (ts->val_type == TEMP_VAL_REG
3031 && ts->kind != TEMP_FIXED
f8b2f202
RH		 3032 && s->reg_to_temp[ts->reg] != ts) {
3033 printf("Inconsistency for temp %s:\n",
3034 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
b03cce8e 3035 fail:
f8b2f202
RH		 3036 printf("reg state:\n");
3037 dump_regs(s);
3038 tcg_abort();
c896fe29
FB		 3039 }
3040 }
3041}
3042#endif
3043
2272e4a7 3044static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 3045{
c1c09194
RH		 3046 intptr_t off, size, align;
3047
3048 switch (ts->type) {
3049 case TCG_TYPE_I32:
3050 size = align = 4;
3051 break;
3052 case TCG_TYPE_I64:
3053 case TCG_TYPE_V64:
3054 size = align = 8;
3055 break;
3056 case TCG_TYPE_V128:
3057 size = align = 16;
3058 break;
3059 case TCG_TYPE_V256:
3060 /* Note that we do not require aligned storage for V256. */
3061 size = 32, align = 16;
3062 break;
3063 default:
3064 g_assert_not_reached();
b591dc59 3065 }
c1c09194
RH		 3066
3067 assert(align <= TCG_TARGET_STACK_ALIGN);
3068 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 3069
3070 /* If we've exhausted the stack frame, restart with a smaller TB. */
3071 if (off + size > s->frame_end) {
3072 tcg_raise_tb_overflow(s);
3073 }
c1c09194
RH		 3074 s->current_frame_offset = off + size;
3075
3076 ts->mem_offset = off;
9defd1bd
RH		 3077#if defined(__sparc__)
3078 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3079#endif
b3a62939 3080 ts->mem_base = s->frame_temp;
c896fe29 3081 ts->mem_allocated = 1;
c896fe29
FB		 3082}
3083
b722452a 3084static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3085
59d7c14e
RH		 3086/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3087 mark it free; otherwise mark it dead. */
3088static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3089{
c0522136
RH		 3090 TCGTempVal new_type;
3091
3092 switch (ts->kind) {
3093 case TEMP_FIXED:
59d7c14e 3094 return;
c0522136
RH		 3095 case TEMP_GLOBAL:
3096 case TEMP_LOCAL:
3097 new_type = TEMP_VAL_MEM;
3098 break;
3099 case TEMP_NORMAL:
3100 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3101 break;
3102 case TEMP_CONST:
3103 new_type = TEMP_VAL_CONST;
3104 break;
3105 default:
3106 g_assert_not_reached();
59d7c14e
RH		 3107 }
3108 if (ts->val_type == TEMP_VAL_REG) {
3109 s->reg_to_temp[ts->reg] = NULL;
3110 }
c0522136 3111 ts->val_type = new_type;
59d7c14e 3112}
7f6ceedf 3113
59d7c14e
RH		 3114/* Mark a temporary as dead. */
3115static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3116{
3117 temp_free_or_dead(s, ts, 1);
3118}
3119
3120/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3121 registers needs to be allocated to store a constant. If 'free_or_dead'
3122 is non-zero, subsequently release the temporary; if it is positive, the
3123 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3124static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3125 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3126{
c0522136 3127 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3128 if (!ts->mem_allocated) {
2272e4a7 3129 temp_allocate_frame(s, ts);
59d7c14e 3130 }
59d7c14e
RH		 3131 switch (ts->val_type) {
3132 case TEMP_VAL_CONST:
3133 /* If we're going to free the temp immediately, then we won't
3134 require it later in a register, so attempt to store the
3135 constant to memory directly. */
3136 if (free_or_dead
3137 && tcg_out_sti(s, ts->type, ts->val,
3138 ts->mem_base->reg, ts->mem_offset)) {
3139 break;
3140 }
3141 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3142 allocated_regs, preferred_regs);
59d7c14e
RH		 3143 /* fallthrough */
3144
3145 case TEMP_VAL_REG:
3146 tcg_out_st(s, ts->type, ts->reg,
3147 ts->mem_base->reg, ts->mem_offset);
3148 break;
3149
3150 case TEMP_VAL_MEM:
3151 break;
3152
3153 case TEMP_VAL_DEAD:
3154 default:
3155 tcg_abort();
3156 }
3157 ts->mem_coherent = 1;
3158 }
3159 if (free_or_dead) {
3160 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3161 }
7f6ceedf
AJ		 3162}
3163
c896fe29 3164/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3165static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3166{
f8b2f202 3167 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3168 if (ts != NULL) {
98b4e186 3169 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3170 }
3171}
3172
b016486e
RH		 3173/**
3174 * tcg_reg_alloc:
3175 * @required_regs: Set of registers in which we must allocate.
3176 * @allocated_regs: Set of registers which must be avoided.
3177 * @preferred_regs: Set of registers we should prefer.
3178 * @rev: True if we search the registers in "indirect" order.
3179 *
3180 * The allocated register must be in @required_regs & ~@allocated_regs,
3181 * but if we can put it in @preferred_regs we may save a move later.
3182 */
3183static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3184 TCGRegSet allocated_regs,
3185 TCGRegSet preferred_regs, bool rev)
c896fe29 3186{
b016486e
RH		 3187 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3188 TCGRegSet reg_ct[2];
91478cef 3189 const int *order;
c896fe29 3190
b016486e
RH		 3191 reg_ct[1] = required_regs & ~allocated_regs;
3192 tcg_debug_assert(reg_ct[1] != 0);
3193 reg_ct[0] = reg_ct[1] & preferred_regs;
3194
3195 /* Skip the preferred_regs option if it cannot be satisfied,
3196 or if the preference made no difference. */
3197 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3198
91478cef 3199 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3200
b016486e
RH		 3201 /* Try free registers, preferences first. */
3202 for (j = f; j < 2; j++) {
3203 TCGRegSet set = reg_ct[j];
3204
3205 if (tcg_regset_single(set)) {
3206 /* One register in the set. */
3207 TCGReg reg = tcg_regset_first(set);
3208 if (s->reg_to_temp[reg] == NULL) {
3209 return reg;
3210 }
3211 } else {
3212 for (i = 0; i < n; i++) {
3213 TCGReg reg = order[i];
3214 if (s->reg_to_temp[reg] == NULL &&
3215 tcg_regset_test_reg(set, reg)) {
3216 return reg;
3217 }
3218 }
3219 }
c896fe29
FB		 3220 }
3221
b016486e
RH		 3222 /* We must spill something. */
3223 for (j = f; j < 2; j++) {
3224 TCGRegSet set = reg_ct[j];
3225
3226 if (tcg_regset_single(set)) {
3227 /* One register in the set. */
3228 TCGReg reg = tcg_regset_first(set);
b3915dbb 3229 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3230 return reg;
b016486e
RH		 3231 } else {
3232 for (i = 0; i < n; i++) {
3233 TCGReg reg = order[i];
3234 if (tcg_regset_test_reg(set, reg)) {
3235 tcg_reg_free(s, reg, allocated_regs);
3236 return reg;
3237 }
3238 }
c896fe29
FB		 3239 }
3240 }
3241
3242 tcg_abort();
3243}
3244
40ae5c62
RH		 3245/* Make sure the temporary is in a register. If needed, allocate the register
3246 from DESIRED while avoiding ALLOCATED. */
3247static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3248 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3249{
3250 TCGReg reg;
3251
3252 switch (ts->val_type) {
3253 case TEMP_VAL_REG:
3254 return;
3255 case TEMP_VAL_CONST:
b016486e 3256 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3257 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3258 if (ts->type <= TCG_TYPE_I64) {
3259 tcg_out_movi(s, ts->type, reg, ts->val);
3260 } else {
4e186175
RH		 3261 uint64_t val = ts->val;
3262 MemOp vece = MO_64;
3263
3264 /*
3265 * Find the minimal vector element that matches the constant.
3266 * The targets will, in general, have to do this search anyway,
3267 * do this generically.
3268 */
4e186175
RH		 3269 if (val == dup_const(MO_8, val)) {
3270 vece = MO_8;
3271 } else if (val == dup_const(MO_16, val)) {
3272 vece = MO_16;
0b4286dd 3273 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3274 vece = MO_32;
3275 }
3276
3277 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3278 }
40ae5c62
RH		 3279 ts->mem_coherent = 0;
3280 break;
3281 case TEMP_VAL_MEM:
b016486e 3282 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3283 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3284 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3285 ts->mem_coherent = 1;
3286 break;
3287 case TEMP_VAL_DEAD:
3288 default:
3289 tcg_abort();
3290 }
3291 ts->reg = reg;
3292 ts->val_type = TEMP_VAL_REG;
3293 s->reg_to_temp[reg] = ts;
3294}
3295
59d7c14e
RH		 3296/* Save a temporary to memory. 'allocated_regs' is used in case a
3297 temporary registers needs to be allocated to store a constant. */
3298static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3299{
5a18407f
RH		 3300 /* The liveness analysis already ensures that globals are back
3301 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3302 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3303}
3304
9814dd27 3305/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3306 modified be the following code. 'allocated_regs' is used in case a
3307 temporary registers needs to be allocated to store a constant. */
3308static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3309{
ac3b8891 3310 int i, n;
c896fe29 3311
ac3b8891 3312 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3313 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3314 }
e5097dc8
FB		 3315}
3316
3d5c5f87
AJ		 3317/* sync globals to their canonical location and assume they can be
3318 read by the following code. 'allocated_regs' is used in case a
3319 temporary registers needs to be allocated to store a constant. */
3320static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3321{
ac3b8891 3322 int i, n;
3d5c5f87 3323
ac3b8891 3324 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3325 TCGTemp *ts = &s->temps[i];
5a18407f 3326 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3327 || ts->kind == TEMP_FIXED
5a18407f 3328 || ts->mem_coherent);
3d5c5f87
AJ		 3329 }
3330}
3331
e5097dc8 3332/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3333 all globals are stored at their canonical location. */
3334static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3335{
e5097dc8
FB		 3336 int i;
3337
b13eb728
RH		 3338 for (i = s->nb_globals; i < s->nb_temps; i++) {
3339 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3340
3341 switch (ts->kind) {
3342 case TEMP_LOCAL:
b13eb728 3343 temp_save(s, ts, allocated_regs);
c0522136
RH		 3344 break;
3345 case TEMP_NORMAL:
5a18407f
RH		 3346 /* The liveness analysis already ensures that temps are dead.
3347 Keep an tcg_debug_assert for safety. */
3348 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3349 break;
3350 case TEMP_CONST:
3351 /* Similarly, we should have freed any allocated register. */
3352 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3353 break;
3354 default:
3355 g_assert_not_reached();
c896fe29
FB		 3356 }
3357 }
e8996ee0
FB		 3358
3359 save_globals(s, allocated_regs);
c896fe29
FB		 3360}
3361
b4cb76e6
RH		 3362/*
3363 * At a conditional branch, we assume all temporaries are dead and
3364 * all globals and local temps are synced to their location.
3365 */
3366static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3367{
3368 sync_globals(s, allocated_regs);
3369
3370 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3371 TCGTemp *ts = &s->temps[i];
3372 /*
3373 * The liveness analysis already ensures that temps are dead.
3374 * Keep tcg_debug_asserts for safety.
3375 */
c0522136
RH		 3376 switch (ts->kind) {
3377 case TEMP_LOCAL:
b4cb76e6 3378 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136
RH		 3379 break;
3380 case TEMP_NORMAL:
b4cb76e6 3381 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3382 break;
3383 case TEMP_CONST:
3384 break;
3385 default:
3386 g_assert_not_reached();
b4cb76e6
RH		 3387 }
3388 }
3389}
3390
bab1671f 3391/*
c58f4c97 3392 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3393 */
0fe4fca4 3394static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3395 tcg_target_ulong val, TCGLifeData arg_life,
3396 TCGRegSet preferred_regs)
e8996ee0 3397{
d63e3b6e 3398 /* ENV should not be modified. */
e01fa97d 3399 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3400
3401 /* The movi is not explicitly generated here. */
3402 if (ots->val_type == TEMP_VAL_REG) {
3403 s->reg_to_temp[ots->reg] = NULL;
ec7a869d 3404 }
59d7c14e
RH		 3405 ots->val_type = TEMP_VAL_CONST;
3406 ots->val = val;
3407 ots->mem_coherent = 0;
3408 if (NEED_SYNC_ARG(0)) {
ba87719c 3409 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3410 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3411 temp_dead(s, ots);
4c4e1ab2 3412 }
e8996ee0
FB		 3413}
3414
bab1671f
RH		 3415/*
3416 * Specialized code generation for INDEX_op_mov_*.
3417 */
dd186292 3418static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3419{
dd186292 3420 const TCGLifeData arg_life = op->life;
69e3706d 3421 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3422 TCGTemp *ts, *ots;
450445d5 3423 TCGType otype, itype;
c896fe29 3424
d21369f5 3425 allocated_regs = s->reserved_regs;
69e3706d 3426 preferred_regs = op->output_pref[0];
43439139
RH		 3427 ots = arg_temp(op->args[0]);
3428 ts = arg_temp(op->args[1]);
450445d5 3429
d63e3b6e 3430 /* ENV should not be modified. */
e01fa97d 3431 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3432
450445d5
RH		 3433 /* Note that otype != itype for no-op truncation. */
3434 otype = ots->type;
3435 itype = ts->type;
c29c1d7e 3436
0fe4fca4
PB		 3437 if (ts->val_type == TEMP_VAL_CONST) {
3438 /* propagate constant or generate sti */
3439 tcg_target_ulong val = ts->val;
3440 if (IS_DEAD_ARG(1)) {
3441 temp_dead(s, ts);
3442 }
69e3706d 3443 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3444 return;
3445 }
3446
3447 /* If the source value is in memory we're going to be forced
3448 to have it in a register in order to perform the copy. Copy
3449 the SOURCE value into its own register first, that way we
3450 don't have to reload SOURCE the next time it is used. */
3451 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3452 temp_load(s, ts, tcg_target_available_regs[itype],
3453 allocated_regs, preferred_regs);
c29c1d7e 3454 }
c896fe29 3455
0fe4fca4 3456 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
d63e3b6e 3457 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3458 /* mov to a non-saved dead register makes no sense (even with
3459 liveness analysis disabled). */
eabb7b91 3460 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3461 if (!ots->mem_allocated) {
2272e4a7 3462 temp_allocate_frame(s, ots);
c29c1d7e 3463 }
b3a62939 3464 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3465 if (IS_DEAD_ARG(1)) {
f8bf00f1 3466 temp_dead(s, ts);
c29c1d7e 3467 }
f8bf00f1 3468 temp_dead(s, ots);
c29c1d7e 3469 } else {
ee17db83 3470 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
c896fe29 3471 /* the mov can be suppressed */
c29c1d7e 3472 if (ots->val_type == TEMP_VAL_REG) {
f8b2f202 3473 s->reg_to_temp[ots->reg] = NULL;
c29c1d7e
AJ		 3474 }
3475 ots->reg = ts->reg;
f8bf00f1 3476 temp_dead(s, ts);
c896fe29 3477 } else {
c29c1d7e
AJ		 3478 if (ots->val_type != TEMP_VAL_REG) {
3479 /* When allocating a new register, make sure to not spill the
3480 input one. */
3481 tcg_regset_set_reg(allocated_regs, ts->reg);
450445d5 3482 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
69e3706d 3483 allocated_regs, preferred_regs,
b016486e 3484 ots->indirect_base);
c896fe29 3485 }
78113e83 3486 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) {
240c08d0
RH		 3487 /*
3488 * Cross register class move not supported.
3489 * Store the source register into the destination slot
3490 * and leave the destination temp as TEMP_VAL_MEM.
3491 */
e01fa97d 3492 assert(!temp_readonly(ots));
240c08d0
RH		 3493 if (!ts->mem_allocated) {
3494 temp_allocate_frame(s, ots);
3495 }
3496 tcg_out_st(s, ts->type, ts->reg,
3497 ots->mem_base->reg, ots->mem_offset);
3498 ots->mem_coherent = 1;
3499 temp_free_or_dead(s, ots, -1);
3500 return;
78113e83 3501 }
c896fe29 3502 }
c29c1d7e
AJ		 3503 ots->val_type = TEMP_VAL_REG;
3504 ots->mem_coherent = 0;
f8b2f202 3505 s->reg_to_temp[ots->reg] = ots;
c29c1d7e 3506 if (NEED_SYNC_ARG(0)) {
98b4e186 3507 temp_sync(s, ots, allocated_regs, 0, 0);
c896fe29 3508 }
ec7a869d 3509 }
c896fe29
FB		 3510}
3511
bab1671f
RH		 3512/*
3513 * Specialized code generation for INDEX_op_dup_vec.
3514 */
3515static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3516{
3517 const TCGLifeData arg_life = op->life;
3518 TCGRegSet dup_out_regs, dup_in_regs;
3519 TCGTemp *its, *ots;
3520 TCGType itype, vtype;
d6ecb4a9 3521 intptr_t endian_fixup;
bab1671f
RH		 3522 unsigned vece;
3523 bool ok;
3524
3525 ots = arg_temp(op->args[0]);
3526 its = arg_temp(op->args[1]);
3527
3528 /* ENV should not be modified. */
e01fa97d 3529 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 3530
3531 itype = its->type;
3532 vece = TCGOP_VECE(op);
3533 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3534
3535 if (its->val_type == TEMP_VAL_CONST) {
3536 /* Propagate constant via movi -> dupi. */
3537 tcg_target_ulong val = its->val;
3538 if (IS_DEAD_ARG(1)) {
3539 temp_dead(s, its);
3540 }
3541 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3542 return;
3543 }
3544
9be0d080
RH		 3545 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3546 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 3547
3548 /* Allocate the output register now. */
3549 if (ots->val_type != TEMP_VAL_REG) {
3550 TCGRegSet allocated_regs = s->reserved_regs;
3551
3552 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3553 /* Make sure to not spill the input register. */
3554 tcg_regset_set_reg(allocated_regs, its->reg);
3555 }
3556 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3557 op->output_pref[0], ots->indirect_base);
3558 ots->val_type = TEMP_VAL_REG;
3559 ots->mem_coherent = 0;
3560 s->reg_to_temp[ots->reg] = ots;
3561 }
3562
3563 switch (its->val_type) {
3564 case TEMP_VAL_REG:
3565 /*
3566 * The dup constriaints must be broad, covering all possible VECE.
3567 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3568 * to fail, indicating that extra moves are required for that case.
3569 */
3570 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3571 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3572 goto done;
3573 }
3574 /* Try again from memory or a vector input register. */
3575 }
3576 if (!its->mem_coherent) {
3577 /*
3578 * The input register is not synced, and so an extra store
3579 * would be required to use memory. Attempt an integer-vector
3580 * register move first. We do not have a TCGRegSet for this.
3581 */
3582 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3583 break;
3584 }
3585 /* Sync the temp back to its slot and load from there. */
3586 temp_sync(s, its, s->reserved_regs, 0, 0);
3587 }
3588 /* fall through */
3589
3590 case TEMP_VAL_MEM:
d6ecb4a9
RH		 3591#ifdef HOST_WORDS_BIGENDIAN
3592 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3593 endian_fixup -= 1 << vece;
3594#else
3595 endian_fixup = 0;
3596#endif
3597 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3598 its->mem_offset + endian_fixup)) {
3599 goto done;
3600 }
bab1671f
RH		 3601 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3602 break;
3603
3604 default:
3605 g_assert_not_reached();
3606 }
3607
3608 /* We now have a vector input register, so dup must succeed. */
3609 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3610 tcg_debug_assert(ok);
3611
3612 done:
3613 if (IS_DEAD_ARG(1)) {
3614 temp_dead(s, its);
3615 }
3616 if (NEED_SYNC_ARG(0)) {
3617 temp_sync(s, ots, s->reserved_regs, 0, 0);
3618 }
3619 if (IS_DEAD_ARG(0)) {
3620 temp_dead(s, ots);
3621 }
3622}
3623
dd186292 3624static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 3625{
dd186292
RH		 3626 const TCGLifeData arg_life = op->life;
3627 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 3628 TCGRegSet i_allocated_regs;
3629 TCGRegSet o_allocated_regs;
b6638662
RH		 3630 int i, k, nb_iargs, nb_oargs;
3631 TCGReg reg;
c896fe29
FB		 3632 TCGArg arg;
3633 const TCGArgConstraint *arg_ct;
3634 TCGTemp *ts;
3635 TCGArg new_args[TCG_MAX_OP_ARGS];
3636 int const_args[TCG_MAX_OP_ARGS];
3637
3638 nb_oargs = def->nb_oargs;
3639 nb_iargs = def->nb_iargs;
3640
3641 /* copy constants */
3642 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 3643 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 3644 sizeof(TCGArg) * def->nb_cargs);
3645
d21369f5
RH		 3646 i_allocated_regs = s->reserved_regs;
3647 o_allocated_regs = s->reserved_regs;
82790a87 3648
c896fe29 3649 /* satisfy input constraints */
dd186292 3650 for (k = 0; k < nb_iargs; k++) {
d62816f2
RH		 3651 TCGRegSet i_preferred_regs, o_preferred_regs;
3652
66792f90 3653 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 3654 arg = op->args[i];
c896fe29 3655 arg_ct = &def->args_ct[i];
43439139 3656 ts = arg_temp(arg);
40ae5c62
RH		 3657
3658 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 3659 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 3660 /* constant is OK for instruction */
3661 const_args[i] = 1;
3662 new_args[i] = ts->val;
d62816f2 3663 continue;
c896fe29 3664 }
40ae5c62 3665
d62816f2 3666 i_preferred_regs = o_preferred_regs = 0;
bc2b17e6 3667 if (arg_ct->ialias) {
d62816f2 3668 o_preferred_regs = op->output_pref[arg_ct->alias_index];
d62816f2 3669
c0522136
RH		 3670 /*
3671 * If the input is readonly, then it cannot also be an
3672 * output and aliased to itself. If the input is not
3673 * dead after the instruction, we must allocate a new
3674 * register and move it.
3675 */
3676 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3677 goto allocate_in_reg;
3678 }
3679
3680 /*
3681 * Check if the current register has already been allocated
3682 * for another input aliased to an output.
3683 */
3684 if (ts->val_type == TEMP_VAL_REG) {
3685 reg = ts->reg;
3686 for (int k2 = 0; k2 < k; k2++) {
3687 int i2 = def->args_ct[nb_oargs + k2].sort_index;
3688 if (def->args_ct[i2].ialias && reg == new_args[i2]) {
3689 goto allocate_in_reg;
7e1df267
AJ		 3690 }
3691 }
5ff9d6a4 3692 }
c0522136 3693 i_preferred_regs = o_preferred_regs;
c896fe29 3694 }
d62816f2 3695
9be0d080 3696 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
c896fe29 3697 reg = ts->reg;
d62816f2 3698
c0522136
RH		 3699 if (!tcg_regset_test_reg(arg_ct->regs, reg)) {
3700 allocate_in_reg:
3701 /*
3702 * Allocate a new register matching the constraint
3703 * and move the temporary register into it.
3704 */
d62816f2
RH		 3705 temp_load(s, ts, tcg_target_available_regs[ts->type],
3706 i_allocated_regs, 0);
9be0d080 3707 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
d62816f2 3708 o_preferred_regs, ts->indirect_base);
78113e83 3709 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3710 /*
3711 * Cross register class move not supported. Sync the
3712 * temp back to its slot and load from there.
3713 */
3714 temp_sync(s, ts, i_allocated_regs, 0, 0);
3715 tcg_out_ld(s, ts->type, reg,
3716 ts->mem_base->reg, ts->mem_offset);
78113e83 3717 }
c896fe29 3718 }
c896fe29
FB		 3719 new_args[i] = reg;
3720 const_args[i] = 0;
82790a87 3721 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29
FB		 3722 }
3723
a52ad07e
AJ		 3724 /* mark dead temporaries and free the associated registers */
3725 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3726 if (IS_DEAD_ARG(i)) {
43439139 3727 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 3728 }
3729 }
3730
b4cb76e6
RH		 3731 if (def->flags & TCG_OPF_COND_BRANCH) {
3732 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3733 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 3734 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 3735 } else {
e8996ee0
FB		 3736 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3737 /* XXX: permit generic clobber register list ? */
c8074023
RH		 3738 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3739 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 3740 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 3741 }
c896fe29 3742 }
3d5c5f87
AJ		 3743 }
3744 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3745 /* sync globals if the op has side effects and might trigger
3746 an exception. */
82790a87 3747 sync_globals(s, i_allocated_regs);
c896fe29 3748 }
e8996ee0
FB		 3749
3750 /* satisfy the output constraints */
e8996ee0 3751 for(k = 0; k < nb_oargs; k++) {
66792f90 3752 i = def->args_ct[k].sort_index;
dd186292 3753 arg = op->args[i];
e8996ee0 3754 arg_ct = &def->args_ct[i];
43439139 3755 ts = arg_temp(arg);
d63e3b6e
RH		 3756
3757 /* ENV should not be modified. */
e01fa97d 3758 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3759
bc2b17e6 3760 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
e8996ee0 3761 reg = new_args[arg_ct->alias_index];
bc2b17e6 3762 } else if (arg_ct->newreg) {
9be0d080 3763 reg = tcg_reg_alloc(s, arg_ct->regs,
82790a87 3764 i_allocated_regs | o_allocated_regs,
69e3706d 3765 op->output_pref[k], ts->indirect_base);
e8996ee0 3766 } else {
9be0d080 3767 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
69e3706d 3768 op->output_pref[k], ts->indirect_base);
c896fe29 3769 }
82790a87 3770 tcg_regset_set_reg(o_allocated_regs, reg);
d63e3b6e
RH		 3771 if (ts->val_type == TEMP_VAL_REG) {
3772 s->reg_to_temp[ts->reg] = NULL;
e8996ee0 3773 }
d63e3b6e
RH		 3774 ts->val_type = TEMP_VAL_REG;
3775 ts->reg = reg;
3776 /*
3777 * Temp value is modified, so the value kept in memory is
3778 * potentially not the same.
3779 */
3780 ts->mem_coherent = 0;
3781 s->reg_to_temp[reg] = ts;
e8996ee0 3782 new_args[i] = reg;
c896fe29 3783 }
c896fe29
FB		 3784 }
3785
c896fe29 3786 /* emit instruction */
d2fd745f
RH		 3787 if (def->flags & TCG_OPF_VECTOR) {
3788 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
3789 new_args, const_args);
3790 } else {
3791 tcg_out_op(s, op->opc, new_args, const_args);
3792 }
3793
c896fe29
FB		 3794 /* move the outputs in the correct register if needed */
3795 for(i = 0; i < nb_oargs; i++) {
43439139 3796 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 3797
3798 /* ENV should not be modified. */
e01fa97d 3799 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3800
ec7a869d 3801 if (NEED_SYNC_ARG(i)) {
98b4e186 3802 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 3803 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 3804 temp_dead(s, ts);
ec7a869d 3805 }
c896fe29
FB		 3806 }
3807}
3808
efe86b21
RH		 3809static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
3810{
3811 const TCGLifeData arg_life = op->life;
3812 TCGTemp *ots, *itsl, *itsh;
3813 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3814
3815 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
3816 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
3817 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
3818
3819 ots = arg_temp(op->args[0]);
3820 itsl = arg_temp(op->args[1]);
3821 itsh = arg_temp(op->args[2]);
3822
3823 /* ENV should not be modified. */
3824 tcg_debug_assert(!temp_readonly(ots));
3825
3826 /* Allocate the output register now. */
3827 if (ots->val_type != TEMP_VAL_REG) {
3828 TCGRegSet allocated_regs = s->reserved_regs;
3829 TCGRegSet dup_out_regs =
3830 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3831
3832 /* Make sure to not spill the input registers. */
3833 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
3834 tcg_regset_set_reg(allocated_regs, itsl->reg);
3835 }
3836 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
3837 tcg_regset_set_reg(allocated_regs, itsh->reg);
3838 }
3839
3840 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3841 op->output_pref[0], ots->indirect_base);
3842 ots->val_type = TEMP_VAL_REG;
3843 ots->mem_coherent = 0;
3844 s->reg_to_temp[ots->reg] = ots;
3845 }
3846
3847 /* Promote dup2 of immediates to dupi_vec. */
3848 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
3849 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
3850 MemOp vece = MO_64;
3851
3852 if (val == dup_const(MO_8, val)) {
3853 vece = MO_8;
3854 } else if (val == dup_const(MO_16, val)) {
3855 vece = MO_16;
3856 } else if (val == dup_const(MO_32, val)) {
3857 vece = MO_32;
3858 }
3859
3860 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
3861 goto done;
3862 }
3863
3864 /* If the two inputs form one 64-bit value, try dupm_vec. */
3865 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
3866 if (!itsl->mem_coherent) {
3867 temp_sync(s, itsl, s->reserved_regs, 0, 0);
3868 }
3869 if (!itsh->mem_coherent) {
3870 temp_sync(s, itsh, s->reserved_regs, 0, 0);
3871 }
3872#ifdef HOST_WORDS_BIGENDIAN
3873 TCGTemp *its = itsh;
3874#else
3875 TCGTemp *its = itsl;
3876#endif
3877 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
3878 its->mem_base->reg, its->mem_offset)) {
3879 goto done;
3880 }
3881 }
3882
3883 /* Fall back to generic expansion. */
3884 return false;
3885
3886 done:
3887 if (IS_DEAD_ARG(1)) {
3888 temp_dead(s, itsl);
3889 }
3890 if (IS_DEAD_ARG(2)) {
3891 temp_dead(s, itsh);
3892 }
3893 if (NEED_SYNC_ARG(0)) {
3894 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
3895 } else if (IS_DEAD_ARG(0)) {
3896 temp_dead(s, ots);
3897 }
3898 return true;
3899}
3900
b03cce8e
FB		 3901#ifdef TCG_TARGET_STACK_GROWSUP
3902#define STACK_DIR(x) (-(x))
3903#else
3904#define STACK_DIR(x) (x)
3905#endif
3906
dd186292 3907static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
c896fe29 3908{
cd9090aa
RH		 3909 const int nb_oargs = TCGOP_CALLO(op);
3910 const int nb_iargs = TCGOP_CALLI(op);
dd186292 3911 const TCGLifeData arg_life = op->life;
7b7d8b2d 3912 const TCGHelperInfo *info;
b6638662
RH		 3913 int flags, nb_regs, i;
3914 TCGReg reg;
cf066674 3915 TCGArg arg;
c896fe29 3916 TCGTemp *ts;
d3452f1f
RH		 3917 intptr_t stack_offset;
3918 size_t call_stack_size;
cf066674
RH		 3919 tcg_insn_unit *func_addr;
3920 int allocate_args;
c896fe29 3921 TCGRegSet allocated_regs;
c896fe29 3922
fa52e660 3923 func_addr = tcg_call_func(op);
7b7d8b2d
RH		 3924 info = tcg_call_info(op);
3925 flags = info->flags;
c896fe29 3926
6e17d0c5 3927 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
c45cb8bb
RH		 3928 if (nb_regs > nb_iargs) {
3929 nb_regs = nb_iargs;
cf066674 3930 }
c896fe29
FB		 3931
3932 /* assign stack slots first */
c45cb8bb 3933 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
c896fe29
FB		 3934 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
3935 ~(TCG_TARGET_STACK_ALIGN - 1);
b03cce8e
FB		 3936 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
3937 if (allocate_args) {
345649c0
BS		 3938 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3939 preallocate call stack */
3940 tcg_abort();
b03cce8e 3941 }
39cf05d3
FB		 3942
3943 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
dd186292
RH		 3944 for (i = nb_regs; i < nb_iargs; i++) {
3945 arg = op->args[nb_oargs + i];
39cf05d3
FB		 3946#ifdef TCG_TARGET_STACK_GROWSUP
3947 stack_offset -= sizeof(tcg_target_long);
3948#endif
3949 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3950 ts = arg_temp(arg);
40ae5c62 3951 temp_load(s, ts, tcg_target_available_regs[ts->type],
b722452a 3952 s->reserved_regs, 0);
40ae5c62 3953 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
c896fe29 3954 }
39cf05d3
FB		 3955#ifndef TCG_TARGET_STACK_GROWSUP
3956 stack_offset += sizeof(tcg_target_long);
3957#endif
c896fe29
FB		 3958 }
3959
3960 /* assign input registers */
d21369f5 3961 allocated_regs = s->reserved_regs;
dd186292
RH		 3962 for (i = 0; i < nb_regs; i++) {
3963 arg = op->args[nb_oargs + i];
39cf05d3 3964 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3965 ts = arg_temp(arg);
39cf05d3 3966 reg = tcg_target_call_iarg_regs[i];
40ae5c62 3967
39cf05d3
FB		 3968 if (ts->val_type == TEMP_VAL_REG) {
3969 if (ts->reg != reg) {
4250da10 3970 tcg_reg_free(s, reg, allocated_regs);
78113e83 3971 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3972 /*
3973 * Cross register class move not supported. Sync the
3974 * temp back to its slot and load from there.
3975 */
3976 temp_sync(s, ts, allocated_regs, 0, 0);
3977 tcg_out_ld(s, ts->type, reg,
3978 ts->mem_base->reg, ts->mem_offset);
78113e83 3979 }
39cf05d3 3980 }
39cf05d3 3981 } else {
ccb1bb66 3982 TCGRegSet arg_set = 0;
40ae5c62 3983
4250da10 3984 tcg_reg_free(s, reg, allocated_regs);
40ae5c62 3985 tcg_regset_set_reg(arg_set, reg);
b722452a 3986 temp_load(s, ts, arg_set, allocated_regs, 0);
c896fe29 3987 }
40ae5c62 3988
39cf05d3 3989 tcg_regset_set_reg(allocated_regs, reg);
c896fe29 3990 }
c896fe29
FB		 3991 }
3992
c896fe29 3993 /* mark dead temporaries and free the associated registers */
dd186292 3994 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 3995 if (IS_DEAD_ARG(i)) {
43439139 3996 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 3997 }
3998 }
3999
4000 /* clobber call registers */
c8074023
RH		 4001 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4002 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 4003 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 4004 }
4005 }
78505279
AJ		 4006
4007 /* Save globals if they might be written by the helper, sync them if
4008 they might be read. */
4009 if (flags & TCG_CALL_NO_READ_GLOBALS) {
4010 /* Nothing to do */
4011 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
4012 sync_globals(s, allocated_regs);
4013 } else {
b9c18f56
AJ		 4014 save_globals(s, allocated_regs);
4015 }
c896fe29 4016
7b7d8b2d
RH		 4017#ifdef CONFIG_TCG_INTERPRETER
4018 {
4019 gpointer hash = (gpointer)(uintptr_t)info->typemask;
4020 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash);
4021 assert(cif != NULL);
4022 tcg_out_call(s, func_addr, cif);
4023 }
4024#else
cf066674 4025 tcg_out_call(s, func_addr);
7b7d8b2d 4026#endif
c896fe29
FB		 4027
4028 /* assign output registers and emit moves if needed */
4029 for(i = 0; i < nb_oargs; i++) {
dd186292 4030 arg = op->args[i];
43439139 4031 ts = arg_temp(arg);
d63e3b6e
RH		 4032
4033 /* ENV should not be modified. */
e01fa97d 4034 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4035
c896fe29 4036 reg = tcg_target_call_oarg_regs[i];
eabb7b91 4037 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
d63e3b6e
RH		 4038 if (ts->val_type == TEMP_VAL_REG) {
4039 s->reg_to_temp[ts->reg] = NULL;
4040 }
4041 ts->val_type = TEMP_VAL_REG;
4042 ts->reg = reg;
4043 ts->mem_coherent = 0;
4044 s->reg_to_temp[reg] = ts;
4045 if (NEED_SYNC_ARG(i)) {
4046 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
4047 } else if (IS_DEAD_ARG(i)) {
4048 temp_dead(s, ts);
c896fe29
FB		 4049 }
4050 }
c896fe29
FB		 4051}
4052
4053#ifdef CONFIG_PROFILER
4054
c3fac113
EC		 4055/* avoid copy/paste errors */
4056#define PROF_ADD(to, from, field) \
4057 do { \
d73415a3 4058 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 4059 } while (0)
4060
4061#define PROF_MAX(to, from, field) \
4062 do { \
d73415a3 4063 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 4064 if (val__ > (to)->field) { \
4065 (to)->field = val__; \
4066 } \
4067 } while (0)
4068
4069/* Pass in a zero'ed @prof */
4070static inline
4071void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4072{
0e2d61cf 4073 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 4074 unsigned int i;
4075
3468b59e 4076 for (i = 0; i < n_ctxs; i++) {
d73415a3 4077 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 4078 const TCGProfile *orig = &s->prof;
c3fac113
EC		 4079
4080 if (counters) {
72fd2efb 4081 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 4082 PROF_ADD(prof, orig, tb_count1);
4083 PROF_ADD(prof, orig, tb_count);
4084 PROF_ADD(prof, orig, op_count);
4085 PROF_MAX(prof, orig, op_count_max);
4086 PROF_ADD(prof, orig, temp_count);
4087 PROF_MAX(prof, orig, temp_count_max);
4088 PROF_ADD(prof, orig, del_op_count);
4089 PROF_ADD(prof, orig, code_in_len);
4090 PROF_ADD(prof, orig, code_out_len);
4091 PROF_ADD(prof, orig, search_out_len);
4092 PROF_ADD(prof, orig, interm_time);
4093 PROF_ADD(prof, orig, code_time);
4094 PROF_ADD(prof, orig, la_time);
4095 PROF_ADD(prof, orig, opt_time);
4096 PROF_ADD(prof, orig, restore_count);
4097 PROF_ADD(prof, orig, restore_time);
4098 }
4099 if (table) {
4100 int i;
4101
4102 for (i = 0; i < NB_OPS; i++) {
4103 PROF_ADD(prof, orig, table_op_count[i]);
4104 }
4105 }
4106 }
4107}
4108
4109#undef PROF_ADD
4110#undef PROF_MAX
4111
4112static void tcg_profile_snapshot_counters(TCGProfile *prof)
4113{
4114 tcg_profile_snapshot(prof, true, false);
4115}
4116
4117static void tcg_profile_snapshot_table(TCGProfile *prof)
4118{
4119 tcg_profile_snapshot(prof, false, true);
4120}
c896fe29 4121
d4c51a0a 4122void tcg_dump_op_count(void)
c896fe29 4123{
c3fac113 4124 TCGProfile prof = {};
c896fe29 4125 int i;
d70724ce 4126
c3fac113 4127 tcg_profile_snapshot_table(&prof);
15fc7daa 4128 for (i = 0; i < NB_OPS; i++) {
d4c51a0a 4129 qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name,
c3fac113 4130 prof.table_op_count[i]);
c896fe29 4131 }
c896fe29 4132}
72fd2efb
EC		 4133
4134int64_t tcg_cpu_exec_time(void)
4135{
0e2d61cf 4136 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4137 unsigned int i;
4138 int64_t ret = 0;
4139
4140 for (i = 0; i < n_ctxs; i++) {
d73415a3 4141 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4142 const TCGProfile *prof = &s->prof;
4143
d73415a3 4144 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4145 }
4146 return ret;
4147}
246ae24d 4148#else
d4c51a0a 4149void tcg_dump_op_count(void)
246ae24d 4150{
d4c51a0a 4151 qemu_printf("[TCG profiler not compiled]\n");
246ae24d 4152}
72fd2efb
EC		 4153
4154int64_t tcg_cpu_exec_time(void)
4155{
4156 error_report("%s: TCG profiler not compiled", __func__);
4157 exit(EXIT_FAILURE);
4158}
c896fe29
FB		 4159#endif
4160
4161
5bd2ec3d 4162int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
c896fe29 4163{
c3fac113
EC		 4164#ifdef CONFIG_PROFILER
4165 TCGProfile *prof = &s->prof;
4166#endif
15fa08f8
RH		 4167 int i, num_insns;
4168 TCGOp *op;
c896fe29 4169
04fe6400
RH		 4170#ifdef CONFIG_PROFILER
4171 {
c1f543b7 4172 int n = 0;
04fe6400 4173
15fa08f8
RH		 4174 QTAILQ_FOREACH(op, &s->ops, link) {
4175 n++;
4176 }
d73415a3 4177 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4178 if (n > prof->op_count_max) {
d73415a3 4179 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4180 }
4181
4182 n = s->nb_temps;
d73415a3 4183 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4184 if (n > prof->temp_count_max) {
d73415a3 4185 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4186 }
4187 }
4188#endif
4189
c896fe29 4190#ifdef DEBUG_DISAS
d977e1c2
AB		 4191 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4192 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4193 FILE *logfile = qemu_log_lock();
93fcfe39 4194 qemu_log("OP:\n");
1894f69a 4195 tcg_dump_ops(s, false);
93fcfe39 4196 qemu_log("\n");
fc59d2d8 4197 qemu_log_unlock(logfile);
c896fe29
FB		 4198 }
4199#endif
4200
bef16ab4
RH		 4201#ifdef CONFIG_DEBUG_TCG
4202 /* Ensure all labels referenced have been emitted. */
4203 {
4204 TCGLabel *l;
4205 bool error = false;
4206
4207 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4208 if (unlikely(!l->present) && l->refs) {
4209 qemu_log_mask(CPU_LOG_TB_OP,
4210 "$L%d referenced but not present.\n", l->id);
4211 error = true;
4212 }
4213 }
4214 assert(!error);
4215 }
4216#endif
4217
c5cc28ff 4218#ifdef CONFIG_PROFILER
d73415a3 4219 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4220#endif
4221
8f2e8c07 4222#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4223 tcg_optimize(s);
8f2e8c07
KB		 4224#endif
4225
a23a9ec6 4226#ifdef CONFIG_PROFILER
d73415a3
SH		 4227 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4228 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4229#endif
c5cc28ff 4230
b4fc67c7 4231 reachable_code_pass(s);
b83eabea 4232 liveness_pass_1(s);
5a18407f 4233
b83eabea 4234 if (s->nb_indirects > 0) {
5a18407f 4235#ifdef DEBUG_DISAS
b83eabea
RH		 4236 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
4237 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4238 FILE *logfile = qemu_log_lock();
b83eabea 4239 qemu_log("OP before indirect lowering:\n");
1894f69a 4240 tcg_dump_ops(s, false);
b83eabea 4241 qemu_log("\n");
fc59d2d8 4242 qemu_log_unlock(logfile);
b83eabea 4243 }
5a18407f 4244#endif
b83eabea
RH		 4245 /* Replace indirect temps with direct temps. */
4246 if (liveness_pass_2(s)) {
4247 /* If changes were made, re-run liveness. */
4248 liveness_pass_1(s);
5a18407f
RH		 4249 }
4250 }
c5cc28ff 4251
a23a9ec6 4252#ifdef CONFIG_PROFILER
d73415a3 4253 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4254#endif
c896fe29
FB		 4255
4256#ifdef DEBUG_DISAS
d977e1c2
AB		 4257 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
4258 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4259 FILE *logfile = qemu_log_lock();
c5cc28ff 4260 qemu_log("OP after optimization and liveness analysis:\n");
1894f69a 4261 tcg_dump_ops(s, true);
93fcfe39 4262 qemu_log("\n");
fc59d2d8 4263 qemu_log_unlock(logfile);
c896fe29
FB		 4264 }
4265#endif
4266
4267 tcg_reg_alloc_start(s);
4268
db0c51a3
RH		 4269 /*
4270 * Reset the buffer pointers when restarting after overflow.
4271 * TODO: Move this into translate-all.c with the rest of the
4272 * buffer management. Having only this done here is confusing.
4273 */
4274 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4275 s->code_ptr = s->code_buf;
c896fe29 4276
659ef5cb 4277#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4278 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4279#endif
57a26946
RH		 4280#ifdef TCG_TARGET_NEED_POOL_LABELS
4281 s->pool_labels = NULL;
4282#endif
9ecefc84 4283
fca8a500 4284 num_insns = -1;
15fa08f8 4285 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4286 TCGOpcode opc = op->opc;
b3db8758 4287
c896fe29 4288#ifdef CONFIG_PROFILER
d73415a3 4289 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 4290#endif
c45cb8bb
RH		 4291
4292 switch (opc) {
c896fe29 4293 case INDEX_op_mov_i32:
c896fe29 4294 case INDEX_op_mov_i64:
d2fd745f 4295 case INDEX_op_mov_vec:
dd186292 4296 tcg_reg_alloc_mov(s, op);
c896fe29 4297 break;
bab1671f
RH		 4298 case INDEX_op_dup_vec:
4299 tcg_reg_alloc_dup(s, op);
4300 break;
765b842a 4301 case INDEX_op_insn_start:
fca8a500 4302 if (num_insns >= 0) {
9f754620
RH		 4303 size_t off = tcg_current_code_size(s);
4304 s->gen_insn_end_off[num_insns] = off;
4305 /* Assert that we do not overflow our stored offset. */
4306 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 4307 }
4308 num_insns++;
bad729e2
RH		 4309 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4310 target_ulong a;
4311#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 4312 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 4313#else
efee3746 4314 a = op->args[i];
bad729e2 4315#endif
fca8a500 4316 s->gen_insn_data[num_insns][i] = a;
bad729e2 4317 }
c896fe29 4318 break;
5ff9d6a4 4319 case INDEX_op_discard:
43439139 4320 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 4321 break;
c896fe29 4322 case INDEX_op_set_label:
e8996ee0 4323 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 4324 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 4325 break;
4326 case INDEX_op_call:
dd186292 4327 tcg_reg_alloc_call(s, op);
c45cb8bb 4328 break;
efe86b21
RH		 4329 case INDEX_op_dup2_vec:
4330 if (tcg_reg_alloc_dup2(s, op)) {
4331 break;
4332 }
4333 /* fall through */
c896fe29 4334 default:
25c4d9cc 4335 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 4336 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 4337 /* Note: in order to speed up the code, it would be much
4338 faster to have specialized register allocator functions for
4339 some common argument patterns */
dd186292 4340 tcg_reg_alloc_op(s, op);
c896fe29
FB		 4341 break;
4342 }
8d8fdbae 4343#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 4344 check_regs(s);
4345#endif
b125f9dc
RH		 4346 /* Test for (pending) buffer overflow. The assumption is that any
4347 one operation beginning below the high water mark cannot overrun
4348 the buffer completely. Thus we can test for overflow after
4349 generating code without having to check during generation. */
644da9b3 4350 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 4351 return -1;
4352 }
6e6c4efe
RH		 4353 /* Test for TB overflow, as seen by gen_insn_end_off. */
4354 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4355 return -2;
4356 }
c896fe29 4357 }
fca8a500
RH		 4358 tcg_debug_assert(num_insns >= 0);
4359 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 4360
b76f0d8c 4361 /* Generate TB finalization at the end of block */
659ef5cb 4362#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 4363 i = tcg_out_ldst_finalize(s);
4364 if (i < 0) {
4365 return i;
23dceda6 4366 }
659ef5cb 4367#endif
57a26946 4368#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 4369 i = tcg_out_pool_finalize(s);
4370 if (i < 0) {
4371 return i;
57a26946
RH		 4372 }
4373#endif
7ecd02a0
RH		 4374 if (!tcg_resolve_relocs(s)) {
4375 return -2;
4376 }
c896fe29 4377
df5d2b16 4378#ifndef CONFIG_TCG_INTERPRETER
c896fe29 4379 /* flush instruction cache */
db0c51a3
RH		 4380 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4381 (uintptr_t)s->code_buf,
1da8de39 4382 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 4383#endif
2aeabc08 4384
1813e175 4385 return tcg_current_code_size(s);
c896fe29
FB		 4386}
4387
a23a9ec6 4388#ifdef CONFIG_PROFILER
3de2faa9 4389void tcg_dump_info(void)
a23a9ec6 4390{
c3fac113
EC		 4391 TCGProfile prof = {};
4392 const TCGProfile *s;
4393 int64_t tb_count;
4394 int64_t tb_div_count;
4395 int64_t tot;
4396
4397 tcg_profile_snapshot_counters(&prof);
4398 s = &prof;
4399 tb_count = s->tb_count;
4400 tb_div_count = tb_count ? tb_count : 1;
4401 tot = s->interm_time + s->code_time;
a23a9ec6 4402
3de2faa9 4403 qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n",
a23a9ec6 4404 tot, tot / 2.4e9);
3de2faa9
MA		 4405 qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64
4406 " %0.1f%%)\n",
fca8a500
RH		 4407 tb_count, s->tb_count1 - tb_count,
4408 (double)(s->tb_count1 - s->tb_count)
4409 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
3de2faa9 4410 qemu_printf("avg ops/TB %0.1f max=%d\n",
fca8a500 4411 (double)s->op_count / tb_div_count, s->op_count_max);
3de2faa9 4412 qemu_printf("deleted ops/TB %0.2f\n",
fca8a500 4413 (double)s->del_op_count / tb_div_count);
3de2faa9 4414 qemu_printf("avg temps/TB %0.2f max=%d\n",
fca8a500 4415 (double)s->temp_count / tb_div_count, s->temp_count_max);
3de2faa9 4416 qemu_printf("avg host code/TB %0.1f\n",
fca8a500 4417 (double)s->code_out_len / tb_div_count);
3de2faa9 4418 qemu_printf("avg search data/TB %0.1f\n",
fca8a500 4419 (double)s->search_out_len / tb_div_count);
a23a9ec6 4420
3de2faa9 4421 qemu_printf("cycles/op %0.1f\n",
a23a9ec6 4422 s->op_count ? (double)tot / s->op_count : 0);
3de2faa9 4423 qemu_printf("cycles/in byte %0.1f\n",
a23a9ec6 4424 s->code_in_len ? (double)tot / s->code_in_len : 0);
3de2faa9 4425 qemu_printf("cycles/out byte %0.1f\n",
a23a9ec6 4426 s->code_out_len ? (double)tot / s->code_out_len : 0);
3de2faa9 4427 qemu_printf("cycles/search byte %0.1f\n",
fca8a500
RH		 4428 s->search_out_len ? (double)tot / s->search_out_len : 0);
4429 if (tot == 0) {
a23a9ec6 4430 tot = 1;
fca8a500 4431 }
3de2faa9 4432 qemu_printf(" gen_interm time %0.1f%%\n",
a23a9ec6 4433 (double)s->interm_time / tot * 100.0);
3de2faa9 4434 qemu_printf(" gen_code time %0.1f%%\n",
a23a9ec6 4435 (double)s->code_time / tot * 100.0);
3de2faa9 4436 qemu_printf("optim./code time %0.1f%%\n",
c5cc28ff
AJ		 4437 (double)s->opt_time / (s->code_time ? s->code_time : 1)
4438 * 100.0);
3de2faa9 4439 qemu_printf("liveness/code time %0.1f%%\n",
a23a9ec6 4440 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
3de2faa9 4441 qemu_printf("cpu_restore count %" PRId64 "\n",
a23a9ec6 4442 s->restore_count);
3de2faa9 4443 qemu_printf(" avg cycles %0.1f\n",
a23a9ec6 4444 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 4445}
4446#else
3de2faa9 4447void tcg_dump_info(void)
a23a9ec6 4448{
3de2faa9 4449 qemu_printf("[TCG profiler not compiled]\n");
a23a9ec6
FB		 4450}
4451#endif
813da627
RH		 4452
4453#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 4454/* In order to use this feature, the backend needs to do three things:
4455
4456 (1) Define ELF_HOST_MACHINE to indicate both what value to
4457 put into the ELF image and to indicate support for the feature.
4458
4459 (2) Define tcg_register_jit. This should create a buffer containing
4460 the contents of a .debug_frame section that describes the post-
4461 prologue unwind info for the tcg machine.
4462
4463 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4464*/
813da627
RH		 4465
4466/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4467typedef enum {
4468 JIT_NOACTION = 0,
4469 JIT_REGISTER_FN,
4470 JIT_UNREGISTER_FN
4471} jit_actions_t;
4472
4473struct jit_code_entry {
4474 struct jit_code_entry *next_entry;
4475 struct jit_code_entry *prev_entry;
4476 const void *symfile_addr;
4477 uint64_t symfile_size;
4478};
4479
4480struct jit_descriptor {
4481 uint32_t version;
4482 uint32_t action_flag;
4483 struct jit_code_entry *relevant_entry;
4484 struct jit_code_entry *first_entry;
4485};
4486
4487void __jit_debug_register_code(void) __attribute__((noinline));
4488void __jit_debug_register_code(void)
4489{
4490 asm("");
4491}
4492
4493/* Must statically initialize the version, because GDB may check
4494 the version before we can set it. */
4495struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4496
4497/* End GDB interface. */
4498
4499static int find_string(const char *strtab, const char *str)
4500{
4501 const char *p = strtab + 1;
4502
4503 while (1) {
4504 if (strcmp(p, str) == 0) {
4505 return p - strtab;
4506 }
4507 p += strlen(p) + 1;
4508 }
4509}
4510
755bf9e5 4511static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 4512 const void *debug_frame,
4513 size_t debug_frame_size)
813da627 4514{
5872bbf2
RH		 4515 struct __attribute__((packed)) DebugInfo {
4516 uint32_t len;
4517 uint16_t version;
4518 uint32_t abbrev;
4519 uint8_t ptr_size;
4520 uint8_t cu_die;
4521 uint16_t cu_lang;
4522 uintptr_t cu_low_pc;
4523 uintptr_t cu_high_pc;
4524 uint8_t fn_die;
4525 char fn_name[16];
4526 uintptr_t fn_low_pc;
4527 uintptr_t fn_high_pc;
4528 uint8_t cu_eoc;
4529 };
813da627
RH		 4530
4531 struct ElfImage {
4532 ElfW(Ehdr) ehdr;
4533 ElfW(Phdr) phdr;
5872bbf2
RH		 4534 ElfW(Shdr) shdr[7];
4535 ElfW(Sym) sym[2];
4536 struct DebugInfo di;
4537 uint8_t da[24];
4538 char str[80];
4539 };
4540
4541 struct ElfImage *img;
4542
4543 static const struct ElfImage img_template = {
4544 .ehdr = {
4545 .e_ident[EI_MAG0] = ELFMAG0,
4546 .e_ident[EI_MAG1] = ELFMAG1,
4547 .e_ident[EI_MAG2] = ELFMAG2,
4548 .e_ident[EI_MAG3] = ELFMAG3,
4549 .e_ident[EI_CLASS] = ELF_CLASS,
4550 .e_ident[EI_DATA] = ELF_DATA,
4551 .e_ident[EI_VERSION] = EV_CURRENT,
4552 .e_type = ET_EXEC,
4553 .e_machine = ELF_HOST_MACHINE,
4554 .e_version = EV_CURRENT,
4555 .e_phoff = offsetof(struct ElfImage, phdr),
4556 .e_shoff = offsetof(struct ElfImage, shdr),
4557 .e_ehsize = sizeof(ElfW(Shdr)),
4558 .e_phentsize = sizeof(ElfW(Phdr)),
4559 .e_phnum = 1,
4560 .e_shentsize = sizeof(ElfW(Shdr)),
4561 .e_shnum = ARRAY_SIZE(img->shdr),
4562 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 4563#ifdef ELF_HOST_FLAGS
4564 .e_flags = ELF_HOST_FLAGS,
4565#endif
4566#ifdef ELF_OSABI
4567 .e_ident[EI_OSABI] = ELF_OSABI,
4568#endif
5872bbf2
RH		 4569 },
4570 .phdr = {
4571 .p_type = PT_LOAD,
4572 .p_flags = PF_X,
4573 },
4574 .shdr = {
4575 [0] = { .sh_type = SHT_NULL },
4576 /* Trick: The contents of code_gen_buffer are not present in
4577 this fake ELF file; that got allocated elsewhere. Therefore
4578 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4579 will not look for contents. We can record any address. */
4580 [1] = { /* .text */
4581 .sh_type = SHT_NOBITS,
4582 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4583 },
4584 [2] = { /* .debug_info */
4585 .sh_type = SHT_PROGBITS,
4586 .sh_offset = offsetof(struct ElfImage, di),
4587 .sh_size = sizeof(struct DebugInfo),
4588 },
4589 [3] = { /* .debug_abbrev */
4590 .sh_type = SHT_PROGBITS,
4591 .sh_offset = offsetof(struct ElfImage, da),
4592 .sh_size = sizeof(img->da),
4593 },
4594 [4] = { /* .debug_frame */
4595 .sh_type = SHT_PROGBITS,
4596 .sh_offset = sizeof(struct ElfImage),
4597 },
4598 [5] = { /* .symtab */
4599 .sh_type = SHT_SYMTAB,
4600 .sh_offset = offsetof(struct ElfImage, sym),
4601 .sh_size = sizeof(img->sym),
4602 .sh_info = 1,
4603 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4604 .sh_entsize = sizeof(ElfW(Sym)),
4605 },
4606 [6] = { /* .strtab */
4607 .sh_type = SHT_STRTAB,
4608 .sh_offset = offsetof(struct ElfImage, str),
4609 .sh_size = sizeof(img->str),
4610 }
4611 },
4612 .sym = {
4613 [1] = { /* code_gen_buffer */
4614 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4615 .st_shndx = 1,
4616 }
4617 },
4618 .di = {
4619 .len = sizeof(struct DebugInfo) - 4,
4620 .version = 2,
4621 .ptr_size = sizeof(void *),
4622 .cu_die = 1,
4623 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4624 .fn_die = 2,
4625 .fn_name = "code_gen_buffer"
4626 },
4627 .da = {
4628 1, /* abbrev number (the cu) */
4629 0x11, 1, /* DW_TAG_compile_unit, has children */
4630 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4631 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4632 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4633 0, 0, /* end of abbrev */
4634 2, /* abbrev number (the fn) */
4635 0x2e, 0, /* DW_TAG_subprogram, no children */
4636 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4637 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4638 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4639 0, 0, /* end of abbrev */
4640 0 /* no more abbrev */
4641 },
4642 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4643 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 4644 };
4645
4646 /* We only need a single jit entry; statically allocate it. */
4647 static struct jit_code_entry one_entry;
4648
5872bbf2 4649 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 4650 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 4651 DebugFrameHeader *dfh;
813da627 4652
5872bbf2
RH		 4653 img = g_malloc(img_size);
4654 *img = img_template;
813da627 4655
5872bbf2
RH		 4656 img->phdr.p_vaddr = buf;
4657 img->phdr.p_paddr = buf;
4658 img->phdr.p_memsz = buf_size;
813da627 4659
813da627 4660 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 4661 img->shdr[1].sh_addr = buf;
813da627
RH		 4662 img->shdr[1].sh_size = buf_size;
4663
5872bbf2
RH		 4664 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4665 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4666
4667 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4668 img->shdr[4].sh_size = debug_frame_size;
4669
4670 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4671 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4672
4673 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4674 img->sym[1].st_value = buf;
4675 img->sym[1].st_size = buf_size;
813da627 4676
5872bbf2 4677 img->di.cu_low_pc = buf;
45aba097 4678 img->di.cu_high_pc = buf + buf_size;
5872bbf2 4679 img->di.fn_low_pc = buf;
45aba097 4680 img->di.fn_high_pc = buf + buf_size;
813da627 4681
2c90784a
RH		 4682 dfh = (DebugFrameHeader *)(img + 1);
4683 memcpy(dfh, debug_frame, debug_frame_size);
4684 dfh->fde.func_start = buf;
4685 dfh->fde.func_len = buf_size;
4686
813da627
RH		 4687#ifdef DEBUG_JIT
4688 /* Enable this block to be able to debug the ELF image file creation.
4689 One can use readelf, objdump, or other inspection utilities. */
4690 {
4691 FILE *f = fopen("/tmp/qemu.jit", "w+b");
4692 if (f) {
5872bbf2 4693 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 4694 /* Avoid stupid unused return value warning for fwrite. */
4695 }
4696 fclose(f);
4697 }
4698 }
4699#endif
4700
4701 one_entry.symfile_addr = img;
4702 one_entry.symfile_size = img_size;
4703
4704 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4705 __jit_debug_descriptor.relevant_entry = &one_entry;
4706 __jit_debug_descriptor.first_entry = &one_entry;
4707 __jit_debug_register_code();
4708}
4709#else
5872bbf2
RH		 4710/* No support for the feature. Provide the entry point expected by exec.c,
4711 and implement the internal function we declared earlier. */
813da627 4712
755bf9e5 4713static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 4714 const void *debug_frame,
4715 size_t debug_frame_size)
813da627
RH		 4716{
4717}
4718
755bf9e5 4719void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 4720{
4721}
4722#endif /* ELF_HOST_MACHINE */
db432672
RH		 4723
4724#if !TCG_TARGET_MAYBE_vec
4725void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4726{
4727 g_assert_not_reached();
4728}
4729#endif
QEMU mirror