]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg: Move TCG_{LOW,HIGH} to tcg-internal.h
[mirror_qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
1de7afc9 37#include "qemu/timer.h"
084cfca1 38#include "qemu/cacheflush.h"
ad768e6f 39#include "qemu/cacheinfo.h"
c896fe29 40
c5d3c498 41/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 42 CPU definitions. Currently they are used for qemu_ld/st
43 instructions */
44#define NO_CPU_IO_DEFS
c896fe29 45
63c91552 46#include "exec/exec-all.h"
dcb32f1d 47#include "tcg/tcg-op.h"
813da627 48
edee2579 49#if UINTPTR_MAX == UINT32_MAX
813da627 50# define ELF_CLASS ELFCLASS32
edee2579
RH		 51#else
52# define ELF_CLASS ELFCLASS64
813da627 53#endif
e03b5686 54#if HOST_BIG_ENDIAN
813da627
RH		 55# define ELF_DATA ELFDATA2MSB
56#else
57# define ELF_DATA ELFDATA2LSB
58#endif
59
c896fe29 60#include "elf.h"
508127e2 61#include "exec/log.h"
d2ba8026 62#include "tcg/tcg-ldst.h"
5ff7258c 63#include "tcg-internal.h"
c896fe29 64
22f15579
RH		 65#ifdef CONFIG_TCG_INTERPRETER
66#include <ffi.h>
67#endif
68
139c1837 69/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 70 used here. */
e4d58b41
RH		 71static void tcg_target_init(TCGContext *s);
72static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 73static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 74 intptr_t value, intptr_t addend);
c896fe29 75
497a22eb
RH		 76/* The CIE and FDE header definitions will be common to all hosts. */
77typedef struct {
78 uint32_t len __attribute__((aligned((sizeof(void *)))));
79 uint32_t id;
80 uint8_t version;
81 char augmentation[1];
82 uint8_t code_align;
83 uint8_t data_align;
84 uint8_t return_column;
85} DebugFrameCIE;
86
87typedef struct QEMU_PACKED {
88 uint32_t len __attribute__((aligned((sizeof(void *)))));
89 uint32_t cie_offset;
edee2579
RH		 90 uintptr_t func_start;
91 uintptr_t func_len;
497a22eb
RH		 92} DebugFrameFDEHeader;
93
2c90784a
RH		 94typedef struct QEMU_PACKED {
95 DebugFrameCIE cie;
96 DebugFrameFDEHeader fde;
97} DebugFrameHeader;
98
755bf9e5 99static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 100 const void *debug_frame,
101 size_t debug_frame_size)
813da627
RH		 102 __attribute__((unused));
103
139c1837 104/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 105static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 106 intptr_t arg2);
78113e83 107static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 108static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 109 TCGReg ret, tcg_target_long arg);
5e8892db
MR		 110static void tcg_out_op(TCGContext *s, TCGOpcode opc,
111 const TCGArg args[TCG_MAX_OP_ARGS],
112 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 113#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 114static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 116static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
117 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 118static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
119 TCGReg dst, int64_t arg);
5e8892db
MR		 120static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
121 unsigned vecl, unsigned vece,
122 const TCGArg args[TCG_MAX_OP_ARGS],
123 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 124#else
e7632cfa
RH		 125static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
126 TCGReg dst, TCGReg src)
127{
128 g_assert_not_reached();
129}
d6ecb4a9
RH		 130static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
131 TCGReg dst, TCGReg base, intptr_t offset)
132{
133 g_assert_not_reached();
134}
4e186175
RH		 135static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
136 TCGReg dst, int64_t arg)
e7632cfa
RH		 137{
138 g_assert_not_reached();
139}
5e8892db
MR		 140static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
141 unsigned vecl, unsigned vece,
142 const TCGArg args[TCG_MAX_OP_ARGS],
143 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 144{
145 g_assert_not_reached();
146}
147#endif
2a534aff 148static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 149 intptr_t arg2);
59d7c14e
RH		 150static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
151 TCGReg base, intptr_t ofs);
7b7d8b2d
RH		 152#ifdef CONFIG_TCG_INTERPRETER
153static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
154 ffi_cif *cif);
155#else
2be7d76b 156static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
7b7d8b2d 157#endif
a4fbbd77 158static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 159#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 160static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 161#endif
c896fe29 162
42eb6dfc
RH		 163TCGContext tcg_init_ctx;
164__thread TCGContext *tcg_ctx;
165
5ff7258c 166TCGContext **tcg_ctxs;
0e2d61cf
RH		 167unsigned int tcg_cur_ctxs;
168unsigned int tcg_max_ctxs;
1c2adb95 169TCGv_env cpu_env = 0;
c8bc1168 170const void *tcg_code_gen_epilogue;
db0c51a3 171uintptr_t tcg_splitwx_diff;
df2cce29 172
b91ccb31
RH		 173#ifndef CONFIG_TCG_INTERPRETER
174tcg_prologue_fn *tcg_qemu_tb_exec;
175#endif
176
d2fd745f 177static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 178static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 179
1813e175 180#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 181static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 182{
183 *s->code_ptr++ = v;
184}
185
4196dca6
PM		 186static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
187 uint8_t v)
5c53bb81 188{
1813e175 189 *p = v;
5c53bb81 190}
1813e175 191#endif
5c53bb81 192
1813e175 193#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 194static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 195{
1813e175
RH		 196 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
197 *s->code_ptr++ = v;
198 } else {
199 tcg_insn_unit *p = s->code_ptr;
200 memcpy(p, &v, sizeof(v));
201 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
202 }
c896fe29
FB		 203}
204
4196dca6
PM		 205static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
206 uint16_t v)
5c53bb81 207{
1813e175
RH		 208 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
209 *p = v;
210 } else {
211 memcpy(p, &v, sizeof(v));
212 }
5c53bb81 213}
1813e175 214#endif
5c53bb81 215
1813e175 216#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 217static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 218{
1813e175
RH		 219 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
220 *s->code_ptr++ = v;
221 } else {
222 tcg_insn_unit *p = s->code_ptr;
223 memcpy(p, &v, sizeof(v));
224 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
225 }
c896fe29
FB		 226}
227
4196dca6
PM		 228static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
229 uint32_t v)
5c53bb81 230{
1813e175
RH		 231 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
232 *p = v;
233 } else {
234 memcpy(p, &v, sizeof(v));
235 }
5c53bb81 236}
1813e175 237#endif
5c53bb81 238
1813e175 239#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 240static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 241{
1813e175
RH		 242 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
243 *s->code_ptr++ = v;
244 } else {
245 tcg_insn_unit *p = s->code_ptr;
246 memcpy(p, &v, sizeof(v));
247 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
248 }
ac26eb69
RH		 249}
250
4196dca6
PM		 251static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
252 uint64_t v)
5c53bb81 253{
1813e175
RH		 254 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
255 *p = v;
256 } else {
257 memcpy(p, &v, sizeof(v));
258 }
5c53bb81 259}
1813e175 260#endif
5c53bb81 261
c896fe29
FB		 262/* label relocation processing */
263
1813e175 264static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 265 TCGLabel *l, intptr_t addend)
c896fe29 266{
7ecd02a0 267 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 268
7ecd02a0
RH		 269 r->type = type;
270 r->ptr = code_ptr;
271 r->addend = addend;
272 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 273}
274
92ab8e7d 275static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 276{
eabb7b91 277 tcg_debug_assert(!l->has_value);
c896fe29 278 l->has_value = 1;
92ab8e7d 279 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 280}
281
42a268c2 282TCGLabel *gen_new_label(void)
c896fe29 283{
b1311c4a 284 TCGContext *s = tcg_ctx;
51e3972c 285 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 286
7ecd02a0
RH		 287 memset(l, 0, sizeof(TCGLabel));
288 l->id = s->nb_labels++;
289 QSIMPLEQ_INIT(&l->relocs);
290
bef16ab4 291 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 292
293 return l;
c896fe29
FB		 294}
295
7ecd02a0
RH		 296static bool tcg_resolve_relocs(TCGContext *s)
297{
298 TCGLabel *l;
299
300 QSIMPLEQ_FOREACH(l, &s->labels, next) {
301 TCGRelocation *r;
302 uintptr_t value = l->u.value;
303
304 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
305 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
306 return false;
307 }
308 }
309 }
310 return true;
311}
312
9f754620
RH		 313static void set_jmp_reset_offset(TCGContext *s, int which)
314{
f14bed3f
RH		 315 /*
316 * We will check for overflow at the end of the opcode loop in
317 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
318 */
319 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 320}
321
db6b7d0c 322/* Signal overflow, starting over with fewer guest insns. */
8905770b
MAL		 323static G_NORETURN
324void tcg_raise_tb_overflow(TCGContext *s)
db6b7d0c
RH		 325{
326 siglongjmp(s->jmp_trans, -2);
327}
328
4c22e840
RH		 329#define C_PFX1(P, A) P##A
330#define C_PFX2(P, A, B) P##A##_##B
331#define C_PFX3(P, A, B, C) P##A##_##B##_##C
332#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
333#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
334#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
335
336/* Define an enumeration for the various combinations. */
337
338#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
339#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
340#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
341#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
342
343#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
344#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
345#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
346#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
347
348#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
349
350#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
351#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
352#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
353#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
354
355typedef enum {
356#include "tcg-target-con-set.h"
357} TCGConstraintSetIndex;
358
359static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
360
361#undef C_O0_I1
362#undef C_O0_I2
363#undef C_O0_I3
364#undef C_O0_I4
365#undef C_O1_I1
366#undef C_O1_I2
367#undef C_O1_I3
368#undef C_O1_I4
369#undef C_N1_I2
370#undef C_O2_I1
371#undef C_O2_I2
372#undef C_O2_I3
373#undef C_O2_I4
374
375/* Put all of the constraint sets into an array, indexed by the enum. */
376
377#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
378#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
379#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
380#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
381
382#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
383#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
384#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
385#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
386
387#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
388
389#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
390#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
391#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
392#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
393
394static const TCGTargetOpDef constraint_sets[] = {
395#include "tcg-target-con-set.h"
396};
397
398
399#undef C_O0_I1
400#undef C_O0_I2
401#undef C_O0_I3
402#undef C_O0_I4
403#undef C_O1_I1
404#undef C_O1_I2
405#undef C_O1_I3
406#undef C_O1_I4
407#undef C_N1_I2
408#undef C_O2_I1
409#undef C_O2_I2
410#undef C_O2_I3
411#undef C_O2_I4
412
413/* Expand the enumerator to be returned from tcg_target_op_def(). */
414
415#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
416#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
417#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
418#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
419
420#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
421#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
422#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
423#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
424
425#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
426
427#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
428#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
429#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
430#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
431
139c1837 432#include "tcg-target.c.inc"
c896fe29 433
38b47b19
EC		 434static void alloc_tcg_plugin_context(TCGContext *s)
435{
436#ifdef CONFIG_PLUGIN
437 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
438 s->plugin_tb->insns =
439 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
440#endif
441}
442
3468b59e
EC		 443/*
444 * All TCG threads except the parent (i.e. the one that called tcg_context_init
445 * and registered the target's TCG globals) must register with this function
446 * before initiating translation.
447 *
448 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
449 * of tcg_region_init() for the reasoning behind this.
450 *
451 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
452 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
453 * is not used anymore for translation once this function is called.
454 *
455 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
456 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
457 */
458#ifdef CONFIG_USER_ONLY
459void tcg_register_thread(void)
460{
461 tcg_ctx = &tcg_init_ctx;
462}
463#else
464void tcg_register_thread(void)
465{
466 TCGContext *s = g_malloc(sizeof(*s));
467 unsigned int i, n;
3468b59e
EC		 468
469 *s = tcg_init_ctx;
470
471 /* Relink mem_base. */
472 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
473 if (tcg_init_ctx.temps[i].mem_base) {
474 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
475 tcg_debug_assert(b >= 0 && b < n);
476 s->temps[i].mem_base = &s->temps[b];
477 }
478 }
479
480 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 481 n = qatomic_fetch_inc(&tcg_cur_ctxs);
482 g_assert(n < tcg_max_ctxs);
d73415a3 483 qatomic_set(&tcg_ctxs[n], s);
3468b59e 484
38b47b19
EC		 485 if (n > 0) {
486 alloc_tcg_plugin_context(s);
bf042e8e 487 tcg_region_initial_alloc(s);
38b47b19
EC		 488 }
489
3468b59e 490 tcg_ctx = s;
e8feb96f 491}
3468b59e 492#endif /* !CONFIG_USER_ONLY */
e8feb96f 493
c896fe29
FB		 494/* pool based memory allocation */
495void *tcg_malloc_internal(TCGContext *s, int size)
496{
497 TCGPool *p;
498 int pool_size;
a813e36f 499
c896fe29
FB		 500 if (size > TCG_POOL_CHUNK_SIZE) {
501 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 502 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 503 p->size = size;
4055299e
KB		 504 p->next = s->pool_first_large;
505 s->pool_first_large = p;
506 return p->data;
c896fe29
FB		 507 } else {
508 p = s->pool_current;
509 if (!p) {
510 p = s->pool_first;
511 if (!p)
512 goto new_pool;
513 } else {
514 if (!p->next) {
515 new_pool:
516 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 517 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 518 p->size = pool_size;
519 p->next = NULL;
a813e36f 520 if (s->pool_current) {
c896fe29 521 s->pool_current->next = p;
a813e36f 522 } else {
c896fe29 523 s->pool_first = p;
a813e36f 524 }
c896fe29
FB		 525 } else {
526 p = p->next;
527 }
528 }
529 }
530 s->pool_current = p;
531 s->pool_cur = p->data + size;
532 s->pool_end = p->data + p->size;
533 return p->data;
534}
535
536void tcg_pool_reset(TCGContext *s)
537{
4055299e
KB		 538 TCGPool *p, *t;
539 for (p = s->pool_first_large; p; p = t) {
540 t = p->next;
541 g_free(p);
542 }
543 s->pool_first_large = NULL;
c896fe29
FB		 544 s->pool_cur = s->pool_end = NULL;
545 s->pool_current = NULL;
546}
547
2ef6175a
RH		 548#include "exec/helper-proto.h"
549
100b5e01 550static const TCGHelperInfo all_helpers[] = {
2ef6175a 551#include "exec/helper-tcg.h"
100b5e01 552};
619205fd 553static GHashTable *helper_table;
100b5e01 554
22f15579
RH		 555#ifdef CONFIG_TCG_INTERPRETER
556static GHashTable *ffi_table;
557
558static ffi_type * const typecode_to_ffi[8] = {
559 [dh_typecode_void] = &ffi_type_void,
560 [dh_typecode_i32] = &ffi_type_uint32,
561 [dh_typecode_s32] = &ffi_type_sint32,
562 [dh_typecode_i64] = &ffi_type_uint64,
563 [dh_typecode_s64] = &ffi_type_sint64,
564 [dh_typecode_ptr] = &ffi_type_pointer,
565};
566#endif
567
91478cef 568static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 569static void process_op_defs(TCGContext *s);
1c2adb95
RH		 570static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
571 TCGReg reg, const char *name);
91478cef 572
43b972b7 573static void tcg_context_init(unsigned max_cpus)
c896fe29 574{
a76aabd3 575 TCGContext *s = &tcg_init_ctx;
100b5e01 576 int op, total_args, n, i;
c896fe29
FB		 577 TCGOpDef *def;
578 TCGArgConstraint *args_ct;
1c2adb95 579 TCGTemp *ts;
c896fe29
FB		 580
581 memset(s, 0, sizeof(*s));
c896fe29 582 s->nb_globals = 0;
c70fbf0a 583
c896fe29
FB		 584 /* Count total number of arguments and allocate the corresponding
585 space */
586 total_args = 0;
587 for(op = 0; op < NB_OPS; op++) {
588 def = &tcg_op_defs[op];
589 n = def->nb_iargs + def->nb_oargs;
590 total_args += n;
591 }
592
bc2b17e6 593 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 594
595 for(op = 0; op < NB_OPS; op++) {
596 def = &tcg_op_defs[op];
597 def->args_ct = args_ct;
c896fe29 598 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 599 args_ct += n;
600 }
5cd8f621
RH		 601
602 /* Register helpers. */
84fd9dd3 603 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 604 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 605
100b5e01 606 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
84fd9dd3 607 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 608 (gpointer)&all_helpers[i]);
100b5e01 609 }
5cd8f621 610
22f15579
RH		 611#ifdef CONFIG_TCG_INTERPRETER
612 /* g_direct_hash/equal for direct comparisons on uint32_t. */
613 ffi_table = g_hash_table_new(NULL, NULL);
614 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
615 struct {
616 ffi_cif cif;
617 ffi_type *args[];
618 } *ca;
619 uint32_t typemask = all_helpers[i].typemask;
620 gpointer hash = (gpointer)(uintptr_t)typemask;
621 ffi_status status;
622 int nargs;
623
624 if (g_hash_table_lookup(ffi_table, hash)) {
625 continue;
626 }
627
628 /* Ignoring the return type, find the last non-zero field. */
629 nargs = 32 - clz32(typemask >> 3);
630 nargs = DIV_ROUND_UP(nargs, 3);
631
632 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
633 ca->cif.rtype = typecode_to_ffi[typemask & 7];
634 ca->cif.nargs = nargs;
635
636 if (nargs != 0) {
637 ca->cif.arg_types = ca->args;
9dd1d56e
IZ		 638 for (int j = 0; j < nargs; ++j) {
639 int typecode = extract32(typemask, (j + 1) * 3, 3);
640 ca->args[j] = typecode_to_ffi[typecode];
22f15579
RH		 641 }
642 }
643
644 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
645 ca->cif.rtype, ca->cif.arg_types);
646 assert(status == FFI_OK);
647
648 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
649 }
650#endif
651
c896fe29 652 tcg_target_init(s);
f69d277e 653 process_op_defs(s);
91478cef
RH		 654
655 /* Reverse the order of the saved registers, assuming they're all at
656 the start of tcg_target_reg_alloc_order. */
657 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
658 int r = tcg_target_reg_alloc_order[n];
659 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
660 break;
661 }
662 }
663 for (i = 0; i < n; ++i) {
664 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
665 }
666 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
667 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
668 }
b1311c4a 669
38b47b19
EC		 670 alloc_tcg_plugin_context(s);
671
b1311c4a 672 tcg_ctx = s;
3468b59e
EC		 673 /*
674 * In user-mode we simply share the init context among threads, since we
675 * use a single region. See the documentation tcg_region_init() for the
676 * reasoning behind this.
677 * In softmmu we will have at most max_cpus TCG threads.
678 */
679#ifdef CONFIG_USER_ONLY
df2cce29 680 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 681 tcg_cur_ctxs = 1;
682 tcg_max_ctxs = 1;
3468b59e 683#else
0e2d61cf
RH		 684 tcg_max_ctxs = max_cpus;
685 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 686#endif
1c2adb95
RH		 687
688 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
689 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
690 cpu_env = temp_tcgv_ptr(ts);
9002ec79 691}
b03cce8e 692
43b972b7 693void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 694{
43b972b7
RH		 695 tcg_context_init(max_cpus);
696 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 697}
698
6e3b2bfd
EC		 699/*
700 * Allocate TBs right before their corresponding translated code, making
701 * sure that TBs and code are on different cache lines.
702 */
703TranslationBlock *tcg_tb_alloc(TCGContext *s)
704{
705 uintptr_t align = qemu_icache_linesize;
706 TranslationBlock *tb;
707 void *next;
708
e8feb96f 709 retry:
6e3b2bfd
EC		 710 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
711 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
712
713 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 714 if (tcg_region_alloc(s)) {
715 return NULL;
716 }
717 goto retry;
6e3b2bfd 718 }
d73415a3 719 qatomic_set(&s->code_gen_ptr, next);
57a26946 720 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 721 return tb;
722}
723
9002ec79
RH		 724void tcg_prologue_init(TCGContext *s)
725{
b0a0794a 726 size_t prologue_size;
8163b749 727
b0a0794a
RH		 728 s->code_ptr = s->code_gen_ptr;
729 s->code_buf = s->code_gen_ptr;
5b38ee31 730 s->data_gen_ptr = NULL;
b91ccb31
RH		 731
732#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 733 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 734#endif
8163b749 735
5b38ee31
RH		 736#ifdef TCG_TARGET_NEED_POOL_LABELS
737 s->pool_labels = NULL;
738#endif
739
653b87eb 740 qemu_thread_jit_write();
8163b749 741 /* Generate the prologue. */
b03cce8e 742 tcg_target_qemu_prologue(s);
5b38ee31
RH		 743
744#ifdef TCG_TARGET_NEED_POOL_LABELS
745 /* Allow the prologue to put e.g. guest_base into a pool entry. */
746 {
1768987b
RH		 747 int result = tcg_out_pool_finalize(s);
748 tcg_debug_assert(result == 0);
5b38ee31
RH		 749 }
750#endif
751
b0a0794a
RH		 752 prologue_size = tcg_current_code_size(s);
753
df5d2b16 754#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 755 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
756 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 757#endif
8163b749 758
d6b64b2b
RH		 759#ifdef DEBUG_DISAS
760 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
c60f599b 761 FILE *logfile = qemu_log_trylock();
78b54858
RH		 762 if (logfile) {
763 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
764 if (s->data_gen_ptr) {
765 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
766 size_t data_size = prologue_size - code_size;
767 size_t i;
768
769 disas(logfile, s->code_gen_ptr, code_size);
770
771 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
772 if (sizeof(tcg_target_ulong) == 8) {
773 fprintf(logfile,
774 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
775 (uintptr_t)s->data_gen_ptr + i,
776 *(uint64_t *)(s->data_gen_ptr + i));
777 } else {
778 fprintf(logfile,
779 "0x%08" PRIxPTR ": .long 0x%08x\n",
780 (uintptr_t)s->data_gen_ptr + i,
781 *(uint32_t *)(s->data_gen_ptr + i));
782 }
5b38ee31 783 }
78b54858
RH		 784 } else {
785 disas(logfile, s->code_gen_ptr, prologue_size);
5b38ee31 786 }
78b54858 787 fprintf(logfile, "\n");
78b54858 788 qemu_log_unlock(logfile);
5b38ee31 789 }
d6b64b2b
RH		 790 }
791#endif
cedbcb01 792
6eea0434
RH		 793#ifndef CONFIG_TCG_INTERPRETER
794 /*
795 * Assert that goto_ptr is implemented completely, setting an epilogue.
796 * For tci, we use NULL as the signal to return from the interpreter,
797 * so skip this check.
798 */
f4e01e30 799 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 800#endif
d1c74ab3
RH		 801
802 tcg_region_prologue_set(s);
c896fe29
FB		 803}
804
c896fe29
FB		 805void tcg_func_start(TCGContext *s)
806{
807 tcg_pool_reset(s);
808 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 809
810 /* No temps have been previously allocated for size or locality. */
811 memset(s->free_temps, 0, sizeof(s->free_temps));
812
c0522136
RH		 813 /* No constant temps have been previously allocated. */
814 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
815 if (s->const_table[i]) {
816 g_hash_table_remove_all(s->const_table[i]);
817 }
818 }
819
abebf925 820 s->nb_ops = 0;
c896fe29
FB		 821 s->nb_labels = 0;
822 s->current_frame_offset = s->frame_start;
823
0a209d4b
RH		 824#ifdef CONFIG_DEBUG_TCG
825 s->goto_tb_issue_mask = 0;
826#endif
827
15fa08f8
RH		 828 QTAILQ_INIT(&s->ops);
829 QTAILQ_INIT(&s->free_ops);
bef16ab4 830 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 831}
832
ae30e866 833static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 834{
835 int n = s->nb_temps++;
ae30e866
RH		 836
837 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 838 tcg_raise_tb_overflow(s);
ae30e866 839 }
7ca4b752
RH		 840 return memset(&s->temps[n], 0, sizeof(TCGTemp));
841}
842
ae30e866 843static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 844{
fa477d25
RH		 845 TCGTemp *ts;
846
7ca4b752 847 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 848 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 849 s->nb_globals++;
fa477d25 850 ts = tcg_temp_alloc(s);
ee17db83 851 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 852
853 return ts;
c896fe29
FB		 854}
855
085272b3
RH		 856static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
857 TCGReg reg, const char *name)
c896fe29 858{
c896fe29 859 TCGTemp *ts;
c896fe29 860
b3a62939 861 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 862 tcg_abort();
b3a62939 863 }
7ca4b752
RH		 864
865 ts = tcg_global_alloc(s);
c896fe29
FB		 866 ts->base_type = type;
867 ts->type = type;
ee17db83 868 ts->kind = TEMP_FIXED;
c896fe29 869 ts->reg = reg;
c896fe29 870 ts->name = name;
c896fe29 871 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 872
085272b3 873 return ts;
a7812ae4
PB		 874}
875
b6638662 876void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 877{
b3a62939
RH		 878 s->frame_start = start;
879 s->frame_end = start + size;
085272b3
RH		 880 s->frame_temp
881 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 882}
883
085272b3
RH		 884TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
885 intptr_t offset, const char *name)
c896fe29 886{
b1311c4a 887 TCGContext *s = tcg_ctx;
dc41aa7d 888 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 889 TCGTemp *ts = tcg_global_alloc(s);
b3915dbb 890 int indirect_reg = 0, bigendian = 0;
e03b5686 891#if HOST_BIG_ENDIAN
7ca4b752
RH		 892 bigendian = 1;
893#endif
c896fe29 894
c0522136
RH		 895 switch (base_ts->kind) {
896 case TEMP_FIXED:
897 break;
898 case TEMP_GLOBAL:
5a18407f
RH		 899 /* We do not support double-indirect registers. */
900 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 901 base_ts->indirect_base = 1;
5a18407f
RH		 902 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
903 ? 2 : 1);
904 indirect_reg = 1;
c0522136
RH		 905 break;
906 default:
907 g_assert_not_reached();
b3915dbb
RH		 908 }
909
7ca4b752
RH		 910 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
911 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 912 char buf[64];
7ca4b752
RH		 913
914 ts->base_type = TCG_TYPE_I64;
c896fe29 915 ts->type = TCG_TYPE_I32;
b3915dbb 916 ts->indirect_reg = indirect_reg;
c896fe29 917 ts->mem_allocated = 1;
b3a62939 918 ts->mem_base = base_ts;
7ca4b752 919 ts->mem_offset = offset + bigendian * 4;
c896fe29
FB		 920 pstrcpy(buf, sizeof(buf), name);
921 pstrcat(buf, sizeof(buf), "_0");
922 ts->name = strdup(buf);
c896fe29 923
7ca4b752
RH		 924 tcg_debug_assert(ts2 == ts + 1);
925 ts2->base_type = TCG_TYPE_I64;
926 ts2->type = TCG_TYPE_I32;
b3915dbb 927 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 928 ts2->mem_allocated = 1;
929 ts2->mem_base = base_ts;
930 ts2->mem_offset = offset + (1 - bigendian) * 4;
c896fe29
FB		 931 pstrcpy(buf, sizeof(buf), name);
932 pstrcat(buf, sizeof(buf), "_1");
120c1084 933 ts2->name = strdup(buf);
7ca4b752 934 } else {
c896fe29
FB		 935 ts->base_type = type;
936 ts->type = type;
b3915dbb 937 ts->indirect_reg = indirect_reg;
c896fe29 938 ts->mem_allocated = 1;
b3a62939 939 ts->mem_base = base_ts;
c896fe29 940 ts->mem_offset = offset;
c896fe29 941 ts->name = name;
c896fe29 942 }
085272b3 943 return ts;
a7812ae4
PB		 944}
945
5bfa8034 946TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
c896fe29 947{
b1311c4a 948 TCGContext *s = tcg_ctx;
ee17db83 949 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
c896fe29 950 TCGTemp *ts;
641d5fbe 951 int idx, k;
c896fe29 952
0ec9eabc
RH		 953 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
954 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
955 if (idx < TCG_MAX_TEMPS) {
956 /* There is already an available temp with the right type. */
957 clear_bit(idx, s->free_temps[k].l);
958
e8996ee0 959 ts = &s->temps[idx];
e8996ee0 960 ts->temp_allocated = 1;
7ca4b752 961 tcg_debug_assert(ts->base_type == type);
ee17db83 962 tcg_debug_assert(ts->kind == kind);
e8996ee0 963 } else {
7ca4b752
RH		 964 ts = tcg_temp_alloc(s);
965 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
966 TCGTemp *ts2 = tcg_temp_alloc(s);
967
f6aa2f7d 968 ts->base_type = type;
e8996ee0
FB		 969 ts->type = TCG_TYPE_I32;
970 ts->temp_allocated = 1;
ee17db83 971 ts->kind = kind;
7ca4b752
RH		 972
973 tcg_debug_assert(ts2 == ts + 1);
974 ts2->base_type = TCG_TYPE_I64;
975 ts2->type = TCG_TYPE_I32;
976 ts2->temp_allocated = 1;
ee17db83 977 ts2->kind = kind;
7ca4b752 978 } else {
e8996ee0
FB		 979 ts->base_type = type;
980 ts->type = type;
981 ts->temp_allocated = 1;
ee17db83 982 ts->kind = kind;
e8996ee0 983 }
c896fe29 984 }
27bfd83c
PM		 985
986#if defined(CONFIG_DEBUG_TCG)
987 s->temps_in_use++;
988#endif
085272b3 989 return ts;
c896fe29
FB		 990}
991
d2fd745f
RH		 992TCGv_vec tcg_temp_new_vec(TCGType type)
993{
994 TCGTemp *t;
995
996#ifdef CONFIG_DEBUG_TCG
997 switch (type) {
998 case TCG_TYPE_V64:
999 assert(TCG_TARGET_HAS_v64);
1000 break;
1001 case TCG_TYPE_V128:
1002 assert(TCG_TARGET_HAS_v128);
1003 break;
1004 case TCG_TYPE_V256:
1005 assert(TCG_TARGET_HAS_v256);
1006 break;
1007 default:
1008 g_assert_not_reached();
1009 }
1010#endif
1011
1012 t = tcg_temp_new_internal(type, 0);
1013 return temp_tcgv_vec(t);
1014}
1015
1016/* Create a new temp of the same type as an existing temp. */
1017TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1018{
1019 TCGTemp *t = tcgv_vec_temp(match);
1020
1021 tcg_debug_assert(t->temp_allocated != 0);
1022
1023 t = tcg_temp_new_internal(t->base_type, 0);
1024 return temp_tcgv_vec(t);
1025}
1026
5bfa8034 1027void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1028{
b1311c4a 1029 TCGContext *s = tcg_ctx;
085272b3 1030 int k, idx;
c896fe29 1031
c7482438
RH		 1032 switch (ts->kind) {
1033 case TEMP_CONST:
1034 /*
1035 * In order to simplify users of tcg_constant_*,
1036 * silently ignore free.
1037 */
c0522136 1038 return;
c7482438
RH		 1039 case TEMP_NORMAL:
1040 case TEMP_LOCAL:
1041 break;
1042 default:
1043 g_assert_not_reached();
c0522136
RH		 1044 }
1045
27bfd83c
PM		 1046#if defined(CONFIG_DEBUG_TCG)
1047 s->temps_in_use--;
1048 if (s->temps_in_use < 0) {
1049 fprintf(stderr, "More temporaries freed than allocated!\n");
1050 }
1051#endif
1052
eabb7b91 1053 tcg_debug_assert(ts->temp_allocated != 0);
e8996ee0 1054 ts->temp_allocated = 0;
0ec9eabc 1055
085272b3 1056 idx = temp_idx(ts);
ee17db83 1057 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1058 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1059}
1060
c0522136
RH		 1061TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1062{
1063 TCGContext *s = tcg_ctx;
1064 GHashTable *h = s->const_table[type];
1065 TCGTemp *ts;
1066
1067 if (h == NULL) {
1068 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1069 s->const_table[type] = h;
1070 }
1071
1072 ts = g_hash_table_lookup(h, &val);
1073 if (ts == NULL) {
1074 ts = tcg_temp_alloc(s);
1075
1076 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1077 TCGTemp *ts2 = tcg_temp_alloc(s);
1078
1079 ts->base_type = TCG_TYPE_I64;
1080 ts->type = TCG_TYPE_I32;
1081 ts->kind = TEMP_CONST;
1082 ts->temp_allocated = 1;
1083 /*
1084 * Retain the full value of the 64-bit constant in the low
1085 * part, so that the hash table works. Actual uses will
1086 * truncate the value to the low part.
1087 */
1088 ts->val = val;
1089
1090 tcg_debug_assert(ts2 == ts + 1);
1091 ts2->base_type = TCG_TYPE_I64;
1092 ts2->type = TCG_TYPE_I32;
1093 ts2->kind = TEMP_CONST;
1094 ts2->temp_allocated = 1;
1095 ts2->val = val >> 32;
1096 } else {
1097 ts->base_type = type;
1098 ts->type = type;
1099 ts->kind = TEMP_CONST;
1100 ts->temp_allocated = 1;
1101 ts->val = val;
1102 }
1103 g_hash_table_insert(h, &ts->val, ts);
1104 }
1105
1106 return ts;
1107}
1108
1109TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1110{
1111 val = dup_const(vece, val);
1112 return temp_tcgv_vec(tcg_constant_internal(type, val));
1113}
1114
88d4005b
RH		 1115TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1116{
1117 TCGTemp *t = tcgv_vec_temp(match);
1118
1119 tcg_debug_assert(t->temp_allocated != 0);
1120 return tcg_constant_vec(t->base_type, vece, val);
1121}
1122
a7812ae4 1123TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1124{
a7812ae4
PB		 1125 TCGv_i32 t0;
1126 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1127 tcg_gen_movi_i32(t0, val);
1128 return t0;
1129}
c896fe29 1130
a7812ae4 1131TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1132{
a7812ae4
PB		 1133 TCGv_i64 t0;
1134 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1135 tcg_gen_movi_i64(t0, val);
1136 return t0;
c896fe29
FB		 1137}
1138
a7812ae4 1139TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1140{
a7812ae4
PB		 1141 TCGv_i32 t0;
1142 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1143 tcg_gen_movi_i32(t0, val);
1144 return t0;
1145}
1146
a7812ae4 1147TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1148{
a7812ae4
PB		 1149 TCGv_i64 t0;
1150 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1151 tcg_gen_movi_i64(t0, val);
1152 return t0;
1153}
1154
27bfd83c
PM		 1155#if defined(CONFIG_DEBUG_TCG)
1156void tcg_clear_temp_count(void)
1157{
b1311c4a 1158 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1159 s->temps_in_use = 0;
1160}
1161
1162int tcg_check_temp_count(void)
1163{
b1311c4a 1164 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1165 if (s->temps_in_use) {
1166 /* Clear the count so that we don't give another
1167 * warning immediately next time around.
1168 */
1169 s->temps_in_use = 0;
1170 return 1;
1171 }
1172 return 0;
1173}
1174#endif
1175
be0f34b5
RH		 1176/* Return true if OP may appear in the opcode stream.
1177 Test the runtime variable that controls each opcode. */
1178bool tcg_op_supported(TCGOpcode op)
1179{
d2fd745f
RH		 1180 const bool have_vec
1181 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1182
be0f34b5
RH		 1183 switch (op) {
1184 case INDEX_op_discard:
1185 case INDEX_op_set_label:
1186 case INDEX_op_call:
1187 case INDEX_op_br:
1188 case INDEX_op_mb:
1189 case INDEX_op_insn_start:
1190 case INDEX_op_exit_tb:
1191 case INDEX_op_goto_tb:
f4e01e30 1192 case INDEX_op_goto_ptr:
be0f34b5
RH		 1193 case INDEX_op_qemu_ld_i32:
1194 case INDEX_op_qemu_st_i32:
1195 case INDEX_op_qemu_ld_i64:
1196 case INDEX_op_qemu_st_i64:
1197 return true;
1198
07ce0b05
RH		 1199 case INDEX_op_qemu_st8_i32:
1200 return TCG_TARGET_HAS_qemu_st8_i32;
1201
be0f34b5 1202 case INDEX_op_mov_i32:
be0f34b5
RH		 1203 case INDEX_op_setcond_i32:
1204 case INDEX_op_brcond_i32:
1205 case INDEX_op_ld8u_i32:
1206 case INDEX_op_ld8s_i32:
1207 case INDEX_op_ld16u_i32:
1208 case INDEX_op_ld16s_i32:
1209 case INDEX_op_ld_i32:
1210 case INDEX_op_st8_i32:
1211 case INDEX_op_st16_i32:
1212 case INDEX_op_st_i32:
1213 case INDEX_op_add_i32:
1214 case INDEX_op_sub_i32:
1215 case INDEX_op_mul_i32:
1216 case INDEX_op_and_i32:
1217 case INDEX_op_or_i32:
1218 case INDEX_op_xor_i32:
1219 case INDEX_op_shl_i32:
1220 case INDEX_op_shr_i32:
1221 case INDEX_op_sar_i32:
1222 return true;
1223
1224 case INDEX_op_movcond_i32:
1225 return TCG_TARGET_HAS_movcond_i32;
1226 case INDEX_op_div_i32:
1227 case INDEX_op_divu_i32:
1228 return TCG_TARGET_HAS_div_i32;
1229 case INDEX_op_rem_i32:
1230 case INDEX_op_remu_i32:
1231 return TCG_TARGET_HAS_rem_i32;
1232 case INDEX_op_div2_i32:
1233 case INDEX_op_divu2_i32:
1234 return TCG_TARGET_HAS_div2_i32;
1235 case INDEX_op_rotl_i32:
1236 case INDEX_op_rotr_i32:
1237 return TCG_TARGET_HAS_rot_i32;
1238 case INDEX_op_deposit_i32:
1239 return TCG_TARGET_HAS_deposit_i32;
1240 case INDEX_op_extract_i32:
1241 return TCG_TARGET_HAS_extract_i32;
1242 case INDEX_op_sextract_i32:
1243 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1244 case INDEX_op_extract2_i32:
1245 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1246 case INDEX_op_add2_i32:
1247 return TCG_TARGET_HAS_add2_i32;
1248 case INDEX_op_sub2_i32:
1249 return TCG_TARGET_HAS_sub2_i32;
1250 case INDEX_op_mulu2_i32:
1251 return TCG_TARGET_HAS_mulu2_i32;
1252 case INDEX_op_muls2_i32:
1253 return TCG_TARGET_HAS_muls2_i32;
1254 case INDEX_op_muluh_i32:
1255 return TCG_TARGET_HAS_muluh_i32;
1256 case INDEX_op_mulsh_i32:
1257 return TCG_TARGET_HAS_mulsh_i32;
1258 case INDEX_op_ext8s_i32:
1259 return TCG_TARGET_HAS_ext8s_i32;
1260 case INDEX_op_ext16s_i32:
1261 return TCG_TARGET_HAS_ext16s_i32;
1262 case INDEX_op_ext8u_i32:
1263 return TCG_TARGET_HAS_ext8u_i32;
1264 case INDEX_op_ext16u_i32:
1265 return TCG_TARGET_HAS_ext16u_i32;
1266 case INDEX_op_bswap16_i32:
1267 return TCG_TARGET_HAS_bswap16_i32;
1268 case INDEX_op_bswap32_i32:
1269 return TCG_TARGET_HAS_bswap32_i32;
1270 case INDEX_op_not_i32:
1271 return TCG_TARGET_HAS_not_i32;
1272 case INDEX_op_neg_i32:
1273 return TCG_TARGET_HAS_neg_i32;
1274 case INDEX_op_andc_i32:
1275 return TCG_TARGET_HAS_andc_i32;
1276 case INDEX_op_orc_i32:
1277 return TCG_TARGET_HAS_orc_i32;
1278 case INDEX_op_eqv_i32:
1279 return TCG_TARGET_HAS_eqv_i32;
1280 case INDEX_op_nand_i32:
1281 return TCG_TARGET_HAS_nand_i32;
1282 case INDEX_op_nor_i32:
1283 return TCG_TARGET_HAS_nor_i32;
1284 case INDEX_op_clz_i32:
1285 return TCG_TARGET_HAS_clz_i32;
1286 case INDEX_op_ctz_i32:
1287 return TCG_TARGET_HAS_ctz_i32;
1288 case INDEX_op_ctpop_i32:
1289 return TCG_TARGET_HAS_ctpop_i32;
1290
1291 case INDEX_op_brcond2_i32:
1292 case INDEX_op_setcond2_i32:
1293 return TCG_TARGET_REG_BITS == 32;
1294
1295 case INDEX_op_mov_i64:
be0f34b5
RH		 1296 case INDEX_op_setcond_i64:
1297 case INDEX_op_brcond_i64:
1298 case INDEX_op_ld8u_i64:
1299 case INDEX_op_ld8s_i64:
1300 case INDEX_op_ld16u_i64:
1301 case INDEX_op_ld16s_i64:
1302 case INDEX_op_ld32u_i64:
1303 case INDEX_op_ld32s_i64:
1304 case INDEX_op_ld_i64:
1305 case INDEX_op_st8_i64:
1306 case INDEX_op_st16_i64:
1307 case INDEX_op_st32_i64:
1308 case INDEX_op_st_i64:
1309 case INDEX_op_add_i64:
1310 case INDEX_op_sub_i64:
1311 case INDEX_op_mul_i64:
1312 case INDEX_op_and_i64:
1313 case INDEX_op_or_i64:
1314 case INDEX_op_xor_i64:
1315 case INDEX_op_shl_i64:
1316 case INDEX_op_shr_i64:
1317 case INDEX_op_sar_i64:
1318 case INDEX_op_ext_i32_i64:
1319 case INDEX_op_extu_i32_i64:
1320 return TCG_TARGET_REG_BITS == 64;
1321
1322 case INDEX_op_movcond_i64:
1323 return TCG_TARGET_HAS_movcond_i64;
1324 case INDEX_op_div_i64:
1325 case INDEX_op_divu_i64:
1326 return TCG_TARGET_HAS_div_i64;
1327 case INDEX_op_rem_i64:
1328 case INDEX_op_remu_i64:
1329 return TCG_TARGET_HAS_rem_i64;
1330 case INDEX_op_div2_i64:
1331 case INDEX_op_divu2_i64:
1332 return TCG_TARGET_HAS_div2_i64;
1333 case INDEX_op_rotl_i64:
1334 case INDEX_op_rotr_i64:
1335 return TCG_TARGET_HAS_rot_i64;
1336 case INDEX_op_deposit_i64:
1337 return TCG_TARGET_HAS_deposit_i64;
1338 case INDEX_op_extract_i64:
1339 return TCG_TARGET_HAS_extract_i64;
1340 case INDEX_op_sextract_i64:
1341 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1342 case INDEX_op_extract2_i64:
1343 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1344 case INDEX_op_extrl_i64_i32:
1345 return TCG_TARGET_HAS_extrl_i64_i32;
1346 case INDEX_op_extrh_i64_i32:
1347 return TCG_TARGET_HAS_extrh_i64_i32;
1348 case INDEX_op_ext8s_i64:
1349 return TCG_TARGET_HAS_ext8s_i64;
1350 case INDEX_op_ext16s_i64:
1351 return TCG_TARGET_HAS_ext16s_i64;
1352 case INDEX_op_ext32s_i64:
1353 return TCG_TARGET_HAS_ext32s_i64;
1354 case INDEX_op_ext8u_i64:
1355 return TCG_TARGET_HAS_ext8u_i64;
1356 case INDEX_op_ext16u_i64:
1357 return TCG_TARGET_HAS_ext16u_i64;
1358 case INDEX_op_ext32u_i64:
1359 return TCG_TARGET_HAS_ext32u_i64;
1360 case INDEX_op_bswap16_i64:
1361 return TCG_TARGET_HAS_bswap16_i64;
1362 case INDEX_op_bswap32_i64:
1363 return TCG_TARGET_HAS_bswap32_i64;
1364 case INDEX_op_bswap64_i64:
1365 return TCG_TARGET_HAS_bswap64_i64;
1366 case INDEX_op_not_i64:
1367 return TCG_TARGET_HAS_not_i64;
1368 case INDEX_op_neg_i64:
1369 return TCG_TARGET_HAS_neg_i64;
1370 case INDEX_op_andc_i64:
1371 return TCG_TARGET_HAS_andc_i64;
1372 case INDEX_op_orc_i64:
1373 return TCG_TARGET_HAS_orc_i64;
1374 case INDEX_op_eqv_i64:
1375 return TCG_TARGET_HAS_eqv_i64;
1376 case INDEX_op_nand_i64:
1377 return TCG_TARGET_HAS_nand_i64;
1378 case INDEX_op_nor_i64:
1379 return TCG_TARGET_HAS_nor_i64;
1380 case INDEX_op_clz_i64:
1381 return TCG_TARGET_HAS_clz_i64;
1382 case INDEX_op_ctz_i64:
1383 return TCG_TARGET_HAS_ctz_i64;
1384 case INDEX_op_ctpop_i64:
1385 return TCG_TARGET_HAS_ctpop_i64;
1386 case INDEX_op_add2_i64:
1387 return TCG_TARGET_HAS_add2_i64;
1388 case INDEX_op_sub2_i64:
1389 return TCG_TARGET_HAS_sub2_i64;
1390 case INDEX_op_mulu2_i64:
1391 return TCG_TARGET_HAS_mulu2_i64;
1392 case INDEX_op_muls2_i64:
1393 return TCG_TARGET_HAS_muls2_i64;
1394 case INDEX_op_muluh_i64:
1395 return TCG_TARGET_HAS_muluh_i64;
1396 case INDEX_op_mulsh_i64:
1397 return TCG_TARGET_HAS_mulsh_i64;
1398
d2fd745f
RH		 1399 case INDEX_op_mov_vec:
1400 case INDEX_op_dup_vec:
37ee55a0 1401 case INDEX_op_dupm_vec:
d2fd745f
RH		 1402 case INDEX_op_ld_vec:
1403 case INDEX_op_st_vec:
1404 case INDEX_op_add_vec:
1405 case INDEX_op_sub_vec:
1406 case INDEX_op_and_vec:
1407 case INDEX_op_or_vec:
1408 case INDEX_op_xor_vec:
212be173 1409 case INDEX_op_cmp_vec:
d2fd745f
RH		 1410 return have_vec;
1411 case INDEX_op_dup2_vec:
1412 return have_vec && TCG_TARGET_REG_BITS == 32;
1413 case INDEX_op_not_vec:
1414 return have_vec && TCG_TARGET_HAS_not_vec;
1415 case INDEX_op_neg_vec:
1416 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1417 case INDEX_op_abs_vec:
1418 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1419 case INDEX_op_andc_vec:
1420 return have_vec && TCG_TARGET_HAS_andc_vec;
1421 case INDEX_op_orc_vec:
1422 return have_vec && TCG_TARGET_HAS_orc_vec;
ed523473
RH		 1423 case INDEX_op_nand_vec:
1424 return have_vec && TCG_TARGET_HAS_nand_vec;
1425 case INDEX_op_nor_vec:
1426 return have_vec && TCG_TARGET_HAS_nor_vec;
1427 case INDEX_op_eqv_vec:
1428 return have_vec && TCG_TARGET_HAS_eqv_vec;
3774030a
RH		 1429 case INDEX_op_mul_vec:
1430 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1431 case INDEX_op_shli_vec:
1432 case INDEX_op_shri_vec:
1433 case INDEX_op_sari_vec:
1434 return have_vec && TCG_TARGET_HAS_shi_vec;
1435 case INDEX_op_shls_vec:
1436 case INDEX_op_shrs_vec:
1437 case INDEX_op_sars_vec:
1438 return have_vec && TCG_TARGET_HAS_shs_vec;
1439 case INDEX_op_shlv_vec:
1440 case INDEX_op_shrv_vec:
1441 case INDEX_op_sarv_vec:
1442 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1443 case INDEX_op_rotli_vec:
1444 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1445 case INDEX_op_rotls_vec:
1446 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1447 case INDEX_op_rotlv_vec:
1448 case INDEX_op_rotrv_vec:
1449 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1450 case INDEX_op_ssadd_vec:
1451 case INDEX_op_usadd_vec:
1452 case INDEX_op_sssub_vec:
1453 case INDEX_op_ussub_vec:
1454 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1455 case INDEX_op_smin_vec:
1456 case INDEX_op_umin_vec:
1457 case INDEX_op_smax_vec:
1458 case INDEX_op_umax_vec:
1459 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1460 case INDEX_op_bitsel_vec:
1461 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1462 case INDEX_op_cmpsel_vec:
1463 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1464
db432672
RH		 1465 default:
1466 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1467 return true;
be0f34b5 1468 }
be0f34b5
RH		 1469}
1470
39cf05d3
FB		 1471/* Note: we convert the 64 bit args to 32 bit and do some alignment
1472 and endian swap. Maybe it would be better to do the alignment
1473 and endian swap in tcg_reg_alloc_call(). */
ae8b75dc 1474void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1475{
75e8b9b7 1476 int i, real_args, nb_rets, pi;
3e92aa34
RH		 1477 unsigned typemask;
1478 const TCGHelperInfo *info;
75e8b9b7 1479 TCGOp *op;
afb49896 1480
619205fd 1481 info = g_hash_table_lookup(helper_table, (gpointer)func);
7319d83a 1482 typemask = info->typemask;
2bece2c8 1483
38b47b19
EC		 1484#ifdef CONFIG_PLUGIN
1485 /* detect non-plugin helpers */
1486 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1487 tcg_ctx->plugin_insn->calls_helpers = true;
1488 }
1489#endif
1490
3a5f6805 1491#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1492 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1493 int argtype = extract32(typemask, (i + 1) * 3, 3);
1494 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1495 bool is_signed = argtype & 1;
1496
1497 if (is_32bit) {
2bece2c8 1498 TCGv_i64 temp = tcg_temp_new_i64();
18cf3d07 1499 TCGv_i32 orig = temp_tcgv_i32(args[i]);
2bece2c8 1500 if (is_signed) {
18cf3d07 1501 tcg_gen_ext_i32_i64(temp, orig);
2bece2c8 1502 } else {
18cf3d07 1503 tcg_gen_extu_i32_i64(temp, orig);
2bece2c8 1504 }
ae8b75dc 1505 args[i] = tcgv_i64_temp(temp);
2bece2c8
RH		 1506 }
1507 }
1508#endif /* TCG_TARGET_EXTEND_ARGS */
1509
15fa08f8 1510 op = tcg_emit_op(INDEX_op_call);
75e8b9b7
RH		 1511
1512 pi = 0;
ae8b75dc 1513 if (ret != NULL) {
7319d83a 1514 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
e03b5686 1515#if HOST_BIG_ENDIAN
ae8b75dc
RH		 1516 op->args[pi++] = temp_arg(ret + 1);
1517 op->args[pi++] = temp_arg(ret);
39cf05d3 1518#else
ae8b75dc
RH		 1519 op->args[pi++] = temp_arg(ret);
1520 op->args[pi++] = temp_arg(ret + 1);
39cf05d3 1521#endif
a7812ae4 1522 nb_rets = 2;
34b1a49c 1523 } else {
ae8b75dc 1524 op->args[pi++] = temp_arg(ret);
a7812ae4 1525 nb_rets = 1;
c896fe29 1526 }
a7812ae4
PB		 1527 } else {
1528 nb_rets = 0;
c896fe29 1529 }
cd9090aa 1530 TCGOP_CALLO(op) = nb_rets;
75e8b9b7 1531
a7812ae4
PB		 1532 real_args = 0;
1533 for (i = 0; i < nargs; i++) {
7319d83a
RH		 1534 int argtype = extract32(typemask, (i + 1) * 3, 3);
1535 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
7b7d8b2d 1536 bool want_align = false;
7319d83a 1537
7b7d8b2d
RH		 1538#if defined(CONFIG_TCG_INTERPRETER)
1539 /*
1540 * Align all arguments, so that they land in predictable places
1541 * for passing off to ffi_call.
1542 */
1543 want_align = true;
1544#elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
1545 /* Some targets want aligned 64 bit args */
1546 want_align = is_64bit;
39cf05d3 1547#endif
7b7d8b2d
RH		 1548
1549 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
1550 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1551 real_args++;
1552 }
1553
1554 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
8d21de51
RH		 1555 op->args[pi++] = temp_arg(args[i] + HOST_BIG_ENDIAN);
1556 op->args[pi++] = temp_arg(args[i] + !HOST_BIG_ENDIAN);
a7812ae4 1557 real_args += 2;
2bece2c8 1558 continue;
c896fe29 1559 }
2bece2c8 1560
ae8b75dc 1561 op->args[pi++] = temp_arg(args[i]);
2bece2c8 1562 real_args++;
c896fe29 1563 }
75e8b9b7 1564 op->args[pi++] = (uintptr_t)func;
3e92aa34 1565 op->args[pi++] = (uintptr_t)info;
cd9090aa 1566 TCGOP_CALLI(op) = real_args;
a7812ae4 1567
75e8b9b7 1568 /* Make sure the fields didn't overflow. */
cd9090aa 1569 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
75e8b9b7 1570 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
2bece2c8 1571
3a5f6805 1572#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1573 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1574 int argtype = extract32(typemask, (i + 1) * 3, 3);
1575 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1576
1577 if (is_32bit) {
085272b3 1578 tcg_temp_free_internal(args[i]);
2bece2c8
RH		 1579 }
1580 }
1581#endif /* TCG_TARGET_EXTEND_ARGS */
c896fe29 1582}
c896fe29 1583
8fcd3692 1584static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1585{
ac3b8891 1586 int i, n;
ac3b8891 1587
ee17db83
RH		 1588 for (i = 0, n = s->nb_temps; i < n; i++) {
1589 TCGTemp *ts = &s->temps[i];
1590 TCGTempVal val = TEMP_VAL_MEM;
1591
1592 switch (ts->kind) {
c0522136
RH		 1593 case TEMP_CONST:
1594 val = TEMP_VAL_CONST;
1595 break;
ee17db83
RH		 1596 case TEMP_FIXED:
1597 val = TEMP_VAL_REG;
1598 break;
1599 case TEMP_GLOBAL:
1600 break;
1601 case TEMP_NORMAL:
c7482438 1602 case TEMP_EBB:
ee17db83
RH		 1603 val = TEMP_VAL_DEAD;
1604 /* fall through */
1605 case TEMP_LOCAL:
1606 ts->mem_allocated = 0;
1607 break;
1608 default:
1609 g_assert_not_reached();
1610 }
1611 ts->val_type = val;
e8996ee0 1612 }
f8b2f202
RH		 1613
1614 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1615}
1616
f8b2f202
RH		 1617static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1618 TCGTemp *ts)
c896fe29 1619{
1807f4c4 1620 int idx = temp_idx(ts);
ac56dd48 1621
ee17db83
RH		 1622 switch (ts->kind) {
1623 case TEMP_FIXED:
1624 case TEMP_GLOBAL:
ac56dd48 1625 pstrcpy(buf, buf_size, ts->name);
ee17db83
RH		 1626 break;
1627 case TEMP_LOCAL:
f8b2f202 1628 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83 1629 break;
c7482438
RH		 1630 case TEMP_EBB:
1631 snprintf(buf, buf_size, "ebb%d", idx - s->nb_globals);
1632 break;
ee17db83 1633 case TEMP_NORMAL:
f8b2f202 1634 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1635 break;
c0522136
RH		 1636 case TEMP_CONST:
1637 switch (ts->type) {
1638 case TCG_TYPE_I32:
1639 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1640 break;
1641#if TCG_TARGET_REG_BITS > 32
1642 case TCG_TYPE_I64:
1643 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1644 break;
1645#endif
1646 case TCG_TYPE_V64:
1647 case TCG_TYPE_V128:
1648 case TCG_TYPE_V256:
1649 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1650 64 << (ts->type - TCG_TYPE_V64), ts->val);
1651 break;
1652 default:
1653 g_assert_not_reached();
1654 }
1655 break;
c896fe29
FB		 1656 }
1657 return buf;
1658}
1659
43439139
RH		 1660static char *tcg_get_arg_str(TCGContext *s, char *buf,
1661 int buf_size, TCGArg arg)
f8b2f202 1662{
43439139 1663 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1664}
1665
f48f3ede
BS		 1666static const char * const cond_name[] =
1667{
0aed257f
RH		 1668 [TCG_COND_NEVER] = "never",
1669 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1670 [TCG_COND_EQ] = "eq",
1671 [TCG_COND_NE] = "ne",
1672 [TCG_COND_LT] = "lt",
1673 [TCG_COND_GE] = "ge",
1674 [TCG_COND_LE] = "le",
1675 [TCG_COND_GT] = "gt",
1676 [TCG_COND_LTU] = "ltu",
1677 [TCG_COND_GEU] = "geu",
1678 [TCG_COND_LEU] = "leu",
1679 [TCG_COND_GTU] = "gtu"
1680};
1681
f713d6ad
RH		 1682static const char * const ldst_name[] =
1683{
1684 [MO_UB] = "ub",
1685 [MO_SB] = "sb",
1686 [MO_LEUW] = "leuw",
1687 [MO_LESW] = "lesw",
1688 [MO_LEUL] = "leul",
1689 [MO_LESL] = "lesl",
fc313c64 1690 [MO_LEUQ] = "leq",
f713d6ad
RH		 1691 [MO_BEUW] = "beuw",
1692 [MO_BESW] = "besw",
1693 [MO_BEUL] = "beul",
1694 [MO_BESL] = "besl",
fc313c64 1695 [MO_BEUQ] = "beq",
f713d6ad
RH		 1696};
1697
1f00b27f 1698static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 1699#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 1700 [MO_UNALN >> MO_ASHIFT] = "un+",
1701 [MO_ALIGN >> MO_ASHIFT] = "",
1702#else
1703 [MO_UNALN >> MO_ASHIFT] = "",
1704 [MO_ALIGN >> MO_ASHIFT] = "al+",
1705#endif
1706 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1707 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1708 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1709 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1710 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1711 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1712};
1713
587195bd
RH		 1714static const char bswap_flag_name[][6] = {
1715 [TCG_BSWAP_IZ] = "iz",
1716 [TCG_BSWAP_OZ] = "oz",
1717 [TCG_BSWAP_OS] = "os",
1718 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1719 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1720};
1721
b016486e
RH		 1722static inline bool tcg_regset_single(TCGRegSet d)
1723{
1724 return (d & (d - 1)) == 0;
1725}
1726
1727static inline TCGReg tcg_regset_first(TCGRegSet d)
1728{
1729 if (TCG_TARGET_NB_REGS <= 32) {
1730 return ctz32(d);
1731 } else {
1732 return ctz64(d);
1733 }
1734}
1735
b7a83ff8
RH		 1736/* Return only the number of characters output -- no error return. */
1737#define ne_fprintf(...) \
1738 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
1739
1740static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
c896fe29 1741{
c896fe29 1742 char buf[128];
c45cb8bb 1743 TCGOp *op;
c45cb8bb 1744
15fa08f8 1745 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 1746 int i, k, nb_oargs, nb_iargs, nb_cargs;
1747 const TCGOpDef *def;
c45cb8bb 1748 TCGOpcode c;
bdfb460e 1749 int col = 0;
c896fe29 1750
c45cb8bb 1751 c = op->opc;
c896fe29 1752 def = &tcg_op_defs[c];
c45cb8bb 1753
765b842a 1754 if (c == INDEX_op_insn_start) {
b016486e 1755 nb_oargs = 0;
b7a83ff8 1756 col += ne_fprintf(f, "\n ----");
9aef40ed
RH		 1757
1758 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1759 target_ulong a;
7e4597d7 1760#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 1761 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 1762#else
efee3746 1763 a = op->args[i];
7e4597d7 1764#endif
b7a83ff8 1765 col += ne_fprintf(f, " " TARGET_FMT_lx, a);
eeacee4d 1766 }
7e4597d7 1767 } else if (c == INDEX_op_call) {
3e92aa34 1768 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 1769 void *func = tcg_call_func(op);
3e92aa34 1770
c896fe29 1771 /* variable number of arguments */
cd9090aa
RH		 1772 nb_oargs = TCGOP_CALLO(op);
1773 nb_iargs = TCGOP_CALLI(op);
c896fe29 1774 nb_cargs = def->nb_cargs;
c896fe29 1775
b7a83ff8 1776 col += ne_fprintf(f, " %s ", def->name);
3e92aa34
RH		 1777
1778 /*
1779 * Print the function name from TCGHelperInfo, if available.
1780 * Note that plugins have a template function for the info,
1781 * but the actual function pointer comes from the plugin.
1782 */
3e92aa34 1783 if (func == info->func) {
b7a83ff8 1784 col += ne_fprintf(f, "%s", info->name);
3e92aa34 1785 } else {
b7a83ff8 1786 col += ne_fprintf(f, "plugin(%p)", func);
3e92aa34
RH		 1787 }
1788
b7a83ff8 1789 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
cf066674 1790 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1791 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1792 op->args[i]));
b03cce8e 1793 }
cf066674 1794 for (i = 0; i < nb_iargs; i++) {
efee3746 1795 TCGArg arg = op->args[nb_oargs + i];
cf066674
RH		 1796 const char *t = "<dummy>";
1797 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 1798 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
eeacee4d 1799 }
b7a83ff8 1800 col += ne_fprintf(f, ",%s", t);
e8996ee0 1801 }
b03cce8e 1802 } else {
b7a83ff8 1803 col += ne_fprintf(f, " %s ", def->name);
c45cb8bb
RH		 1804
1805 nb_oargs = def->nb_oargs;
1806 nb_iargs = def->nb_iargs;
1807 nb_cargs = def->nb_cargs;
1808
d2fd745f 1809 if (def->flags & TCG_OPF_VECTOR) {
b7a83ff8
RH		 1810 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
1811 8 << TCGOP_VECE(op));
d2fd745f
RH		 1812 }
1813
b03cce8e 1814 k = 0;
c45cb8bb 1815 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1816 const char *sep = k ? "," : "";
1817 col += ne_fprintf(f, "%s%s", sep,
1818 tcg_get_arg_str(s, buf, sizeof(buf),
1819 op->args[k++]));
b03cce8e 1820 }
c45cb8bb 1821 for (i = 0; i < nb_iargs; i++) {
b7a83ff8
RH		 1822 const char *sep = k ? "," : "";
1823 col += ne_fprintf(f, "%s%s", sep,
1824 tcg_get_arg_str(s, buf, sizeof(buf),
1825 op->args[k++]));
b03cce8e 1826 }
be210acb
RH		 1827 switch (c) {
1828 case INDEX_op_brcond_i32:
be210acb 1829 case INDEX_op_setcond_i32:
ffc5ea09 1830 case INDEX_op_movcond_i32:
ffc5ea09 1831 case INDEX_op_brcond2_i32:
be210acb 1832 case INDEX_op_setcond2_i32:
ffc5ea09 1833 case INDEX_op_brcond_i64:
be210acb 1834 case INDEX_op_setcond_i64:
ffc5ea09 1835 case INDEX_op_movcond_i64:
212be173 1836 case INDEX_op_cmp_vec:
f75da298 1837 case INDEX_op_cmpsel_vec:
efee3746
RH		 1838 if (op->args[k] < ARRAY_SIZE(cond_name)
1839 && cond_name[op->args[k]]) {
b7a83ff8 1840 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
eeacee4d 1841 } else {
b7a83ff8 1842 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 1843 }
f48f3ede 1844 i = 1;
be210acb 1845 break;
f713d6ad
RH		 1846 case INDEX_op_qemu_ld_i32:
1847 case INDEX_op_qemu_st_i32:
07ce0b05 1848 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 1849 case INDEX_op_qemu_ld_i64:
1850 case INDEX_op_qemu_st_i64:
59227d5d 1851 {
9002ffcb 1852 MemOpIdx oi = op->args[k++];
14776ab5 1853 MemOp op = get_memop(oi);
59227d5d
RH		 1854 unsigned ix = get_mmuidx(oi);
1855
59c4b7e8 1856 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
b7a83ff8 1857 col += ne_fprintf(f, ",$0x%x,%u", op, ix);
59c4b7e8 1858 } else {
1f00b27f
SS		 1859 const char *s_al, *s_op;
1860 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 1861 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
b7a83ff8 1862 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 1863 }
1864 i = 1;
f713d6ad 1865 }
f713d6ad 1866 break;
587195bd
RH		 1867 case INDEX_op_bswap16_i32:
1868 case INDEX_op_bswap16_i64:
1869 case INDEX_op_bswap32_i32:
1870 case INDEX_op_bswap32_i64:
1871 case INDEX_op_bswap64_i64:
1872 {
1873 TCGArg flags = op->args[k];
1874 const char *name = NULL;
1875
1876 if (flags < ARRAY_SIZE(bswap_flag_name)) {
1877 name = bswap_flag_name[flags];
1878 }
1879 if (name) {
b7a83ff8 1880 col += ne_fprintf(f, ",%s", name);
587195bd 1881 } else {
b7a83ff8 1882 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
587195bd
RH		 1883 }
1884 i = k = 1;
1885 }
1886 break;
be210acb 1887 default:
f48f3ede 1888 i = 0;
be210acb
RH		 1889 break;
1890 }
51e3972c
RH		 1891 switch (c) {
1892 case INDEX_op_set_label:
1893 case INDEX_op_br:
1894 case INDEX_op_brcond_i32:
1895 case INDEX_op_brcond_i64:
1896 case INDEX_op_brcond2_i32:
b7a83ff8
RH		 1897 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
1898 arg_label(op->args[k])->id);
51e3972c
RH		 1899 i++, k++;
1900 break;
1901 default:
1902 break;
1903 }
1904 for (; i < nb_cargs; i++, k++) {
b7a83ff8
RH		 1905 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
1906 op->args[k]);
bdfb460e
RH		 1907 }
1908 }
bdfb460e 1909
1894f69a 1910 if (have_prefs || op->life) {
b7a83ff8
RH		 1911 for (; col < 40; ++col) {
1912 putc(' ', f);
bdfb460e 1913 }
1894f69a
RH		 1914 }
1915
1916 if (op->life) {
1917 unsigned life = op->life;
bdfb460e
RH		 1918
1919 if (life & (SYNC_ARG * 3)) {
b7a83ff8 1920 ne_fprintf(f, " sync:");
bdfb460e
RH		 1921 for (i = 0; i < 2; ++i) {
1922 if (life & (SYNC_ARG << i)) {
b7a83ff8 1923 ne_fprintf(f, " %d", i);
bdfb460e
RH		 1924 }
1925 }
1926 }
1927 life /= DEAD_ARG;
1928 if (life) {
b7a83ff8 1929 ne_fprintf(f, " dead:");
bdfb460e
RH		 1930 for (i = 0; life; ++i, life >>= 1) {
1931 if (life & 1) {
b7a83ff8 1932 ne_fprintf(f, " %d", i);
bdfb460e
RH		 1933 }
1934 }
b03cce8e 1935 }
c896fe29 1936 }
1894f69a
RH		 1937
1938 if (have_prefs) {
1939 for (i = 0; i < nb_oargs; ++i) {
1940 TCGRegSet set = op->output_pref[i];
1941
1942 if (i == 0) {
b7a83ff8 1943 ne_fprintf(f, " pref=");
1894f69a 1944 } else {
b7a83ff8 1945 ne_fprintf(f, ",");
1894f69a
RH		 1946 }
1947 if (set == 0) {
b7a83ff8 1948 ne_fprintf(f, "none");
1894f69a 1949 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
b7a83ff8 1950 ne_fprintf(f, "all");
1894f69a
RH		 1951#ifdef CONFIG_DEBUG_TCG
1952 } else if (tcg_regset_single(set)) {
1953 TCGReg reg = tcg_regset_first(set);
b7a83ff8 1954 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
1894f69a
RH		 1955#endif
1956 } else if (TCG_TARGET_NB_REGS <= 32) {
b7a83ff8 1957 ne_fprintf(f, "0x%x", (uint32_t)set);
1894f69a 1958 } else {
b7a83ff8 1959 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
1894f69a
RH		 1960 }
1961 }
1962 }
1963
b7a83ff8 1964 putc('\n', f);
c896fe29
FB		 1965 }
1966}
1967
1968/* we give more priority to constraints with less registers */
1969static int get_constraint_priority(const TCGOpDef *def, int k)
1970{
74a11790 1971 const TCGArgConstraint *arg_ct = &def->args_ct[k];
29f5e925 1972 int n = ctpop64(arg_ct->regs);
c896fe29 1973
29f5e925
RH		 1974 /*
1975 * Sort constraints of a single register first, which includes output
1976 * aliases (which must exactly match the input already allocated).
1977 */
1978 if (n == 1 || arg_ct->oalias) {
1979 return INT_MAX;
1980 }
1981
1982 /*
1983 * Sort register pairs next, first then second immediately after.
1984 * Arbitrarily sort multiple pairs by the index of the first reg;
1985 * there shouldn't be many pairs.
1986 */
1987 switch (arg_ct->pair) {
1988 case 1:
1989 case 3:
1990 return (k + 1) * 2;
1991 case 2:
1992 return (arg_ct->pair_index + 1) * 2 - 1;
c896fe29 1993 }
29f5e925
RH		 1994
1995 /* Finally, sort by decreasing register count. */
1996 assert(n > 1);
1997 return -n;
c896fe29
FB		 1998}
1999
2000/* sort from highest priority to lowest */
2001static void sort_constraints(TCGOpDef *def, int start, int n)
2002{
66792f90
RH		 2003 int i, j;
2004 TCGArgConstraint *a = def->args_ct;
c896fe29 2005
66792f90
RH		 2006 for (i = 0; i < n; i++) {
2007 a[start + i].sort_index = start + i;
2008 }
2009 if (n <= 1) {
c896fe29 2010 return;
66792f90
RH		 2011 }
2012 for (i = 0; i < n - 1; i++) {
2013 for (j = i + 1; j < n; j++) {
2014 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2015 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2016 if (p1 < p2) {
66792f90
RH		 2017 int tmp = a[start + i].sort_index;
2018 a[start + i].sort_index = a[start + j].sort_index;
2019 a[start + j].sort_index = tmp;
c896fe29
FB		 2020 }
2021 }
2022 }
2023}
2024
f69d277e 2025static void process_op_defs(TCGContext *s)
c896fe29 2026{
a9751609 2027 TCGOpcode op;
c896fe29 2028
f69d277e
RH		 2029 for (op = 0; op < NB_OPS; op++) {
2030 TCGOpDef *def = &tcg_op_defs[op];
2031 const TCGTargetOpDef *tdefs;
29f5e925
RH		 2032 bool saw_alias_pair = false;
2033 int i, o, i2, o2, nb_args;
f69d277e
RH		 2034
2035 if (def->flags & TCG_OPF_NOT_PRESENT) {
2036 continue;
2037 }
2038
c896fe29 2039 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2040 if (nb_args == 0) {
2041 continue;
2042 }
2043
4c22e840
RH		 2044 /*
2045 * Macro magic should make it impossible, but double-check that
2046 * the array index is in range. Since the signness of an enum
2047 * is implementation defined, force the result to unsigned.
2048 */
2049 unsigned con_set = tcg_target_op_def(op);
2050 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2051 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2052
2053 for (i = 0; i < nb_args; i++) {
2054 const char *ct_str = tdefs->args_ct_str[i];
8940ea0d
PMD		 2055 bool input_p = i >= def->nb_oargs;
2056
f69d277e 2057 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2058 tcg_debug_assert(ct_str != NULL);
f69d277e 2059
8940ea0d
PMD		 2060 switch (*ct_str) {
2061 case '0' ... '9':
2062 o = *ct_str - '0';
2063 tcg_debug_assert(input_p);
2064 tcg_debug_assert(o < def->nb_oargs);
2065 tcg_debug_assert(def->args_ct[o].regs != 0);
2066 tcg_debug_assert(!def->args_ct[o].oalias);
2067 def->args_ct[i] = def->args_ct[o];
2068 /* The output sets oalias. */
2069 def->args_ct[o].oalias = 1;
2070 def->args_ct[o].alias_index = i;
2071 /* The input sets ialias. */
2072 def->args_ct[i].ialias = 1;
2073 def->args_ct[i].alias_index = o;
29f5e925
RH		 2074 if (def->args_ct[i].pair) {
2075 saw_alias_pair = true;
2076 }
8940ea0d
PMD		 2077 tcg_debug_assert(ct_str[1] == '\0');
2078 continue;
2079
2080 case '&':
2081 tcg_debug_assert(!input_p);
2082 def->args_ct[i].newreg = true;
2083 ct_str++;
2084 break;
29f5e925
RH		 2085
2086 case 'p': /* plus */
2087 /* Allocate to the register after the previous. */
2088 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2089 o = i - 1;
2090 tcg_debug_assert(!def->args_ct[o].pair);
2091 tcg_debug_assert(!def->args_ct[o].ct);
2092 def->args_ct[i] = (TCGArgConstraint){
2093 .pair = 2,
2094 .pair_index = o,
2095 .regs = def->args_ct[o].regs << 1,
2096 };
2097 def->args_ct[o].pair = 1;
2098 def->args_ct[o].pair_index = i;
2099 tcg_debug_assert(ct_str[1] == '\0');
2100 continue;
2101
2102 case 'm': /* minus */
2103 /* Allocate to the register before the previous. */
2104 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2105 o = i - 1;
2106 tcg_debug_assert(!def->args_ct[o].pair);
2107 tcg_debug_assert(!def->args_ct[o].ct);
2108 def->args_ct[i] = (TCGArgConstraint){
2109 .pair = 1,
2110 .pair_index = o,
2111 .regs = def->args_ct[o].regs >> 1,
2112 };
2113 def->args_ct[o].pair = 2;
2114 def->args_ct[o].pair_index = i;
2115 tcg_debug_assert(ct_str[1] == '\0');
2116 continue;
8940ea0d
PMD		 2117 }
2118
2119 do {
2120 switch (*ct_str) {
17280ff4
RH		 2121 case 'i':
2122 def->args_ct[i].ct |= TCG_CT_CONST;
17280ff4 2123 break;
358b4923 2124
358b4923
RH		 2125 /* Include all of the target-specific constraints. */
2126
2127#undef CONST
2128#define CONST(CASE, MASK) \
8940ea0d 2129 case CASE: def->args_ct[i].ct |= MASK; break;
358b4923 2130#define REGS(CASE, MASK) \
8940ea0d 2131 case CASE: def->args_ct[i].regs |= MASK; break;
358b4923
RH		 2132
2133#include "tcg-target-con-str.h"
2134
2135#undef REGS
2136#undef CONST
17280ff4 2137 default:
8940ea0d
PMD		 2138 case '0' ... '9':
2139 case '&':
29f5e925
RH		 2140 case 'p':
2141 case 'm':
17280ff4 2142 /* Typo in TCGTargetOpDef constraint. */
358b4923 2143 g_assert_not_reached();
c896fe29 2144 }
8940ea0d 2145 } while (*++ct_str != '\0');
c896fe29
FB		 2146 }
2147
c68aaa18 2148 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2149 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2150
29f5e925
RH		 2151 /*
2152 * Fix up output pairs that are aliased with inputs.
2153 * When we created the alias, we copied pair from the output.
2154 * There are three cases:
2155 * (1a) Pairs of inputs alias pairs of outputs.
2156 * (1b) One input aliases the first of a pair of outputs.
2157 * (2) One input aliases the second of a pair of outputs.
2158 *
2159 * Case 1a is handled by making sure that the pair_index'es are
2160 * properly updated so that they appear the same as a pair of inputs.
2161 *
2162 * Case 1b is handled by setting the pair_index of the input to
2163 * itself, simply so it doesn't point to an unrelated argument.
2164 * Since we don't encounter the "second" during the input allocation
2165 * phase, nothing happens with the second half of the input pair.
2166 *
2167 * Case 2 is handled by setting the second input to pair=3, the
2168 * first output to pair=3, and the pair_index'es to match.
2169 */
2170 if (saw_alias_pair) {
2171 for (i = def->nb_oargs; i < nb_args; i++) {
2172 /*
2173 * Since [0-9pm] must be alone in the constraint string,
2174 * the only way they can both be set is if the pair comes
2175 * from the output alias.
2176 */
2177 if (!def->args_ct[i].ialias) {
2178 continue;
2179 }
2180 switch (def->args_ct[i].pair) {
2181 case 0:
2182 break;
2183 case 1:
2184 o = def->args_ct[i].alias_index;
2185 o2 = def->args_ct[o].pair_index;
2186 tcg_debug_assert(def->args_ct[o].pair == 1);
2187 tcg_debug_assert(def->args_ct[o2].pair == 2);
2188 if (def->args_ct[o2].oalias) {
2189 /* Case 1a */
2190 i2 = def->args_ct[o2].alias_index;
2191 tcg_debug_assert(def->args_ct[i2].pair == 2);
2192 def->args_ct[i2].pair_index = i;
2193 def->args_ct[i].pair_index = i2;
2194 } else {
2195 /* Case 1b */
2196 def->args_ct[i].pair_index = i;
2197 }
2198 break;
2199 case 2:
2200 o = def->args_ct[i].alias_index;
2201 o2 = def->args_ct[o].pair_index;
2202 tcg_debug_assert(def->args_ct[o].pair == 2);
2203 tcg_debug_assert(def->args_ct[o2].pair == 1);
2204 if (def->args_ct[o2].oalias) {
2205 /* Case 1a */
2206 i2 = def->args_ct[o2].alias_index;
2207 tcg_debug_assert(def->args_ct[i2].pair == 1);
2208 def->args_ct[i2].pair_index = i;
2209 def->args_ct[i].pair_index = i2;
2210 } else {
2211 /* Case 2 */
2212 def->args_ct[i].pair = 3;
2213 def->args_ct[o2].pair = 3;
2214 def->args_ct[i].pair_index = o2;
2215 def->args_ct[o2].pair_index = i;
2216 }
2217 break;
2218 default:
2219 g_assert_not_reached();
2220 }
2221 }
2222 }
2223
c896fe29
FB		 2224 /* sort the constraints (XXX: this is just an heuristic) */
2225 sort_constraints(def, 0, def->nb_oargs);
2226 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2227 }
c896fe29
FB		 2228}
2229
0c627cdc
RH		 2230void tcg_op_remove(TCGContext *s, TCGOp *op)
2231{
d88a117e
RH		 2232 TCGLabel *label;
2233
2234 switch (op->opc) {
2235 case INDEX_op_br:
2236 label = arg_label(op->args[0]);
2237 label->refs--;
2238 break;
2239 case INDEX_op_brcond_i32:
2240 case INDEX_op_brcond_i64:
2241 label = arg_label(op->args[3]);
2242 label->refs--;
2243 break;
2244 case INDEX_op_brcond2_i32:
2245 label = arg_label(op->args[5]);
2246 label->refs--;
2247 break;
2248 default:
2249 break;
2250 }
2251
15fa08f8
RH		 2252 QTAILQ_REMOVE(&s->ops, op, link);
2253 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2254 s->nb_ops--;
0c627cdc
RH		 2255
2256#ifdef CONFIG_PROFILER
d73415a3 2257 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2258#endif
2259}
2260
a80cdd31
RH		 2261void tcg_remove_ops_after(TCGOp *op)
2262{
2263 TCGContext *s = tcg_ctx;
2264
2265 while (true) {
2266 TCGOp *last = tcg_last_op();
2267 if (last == op) {
2268 return;
2269 }
2270 tcg_op_remove(s, last);
2271 }
2272}
2273
15fa08f8 2274static TCGOp *tcg_op_alloc(TCGOpcode opc)
5a18407f 2275{
15fa08f8
RH		 2276 TCGContext *s = tcg_ctx;
2277 TCGOp *op;
5a18407f 2278
15fa08f8
RH		 2279 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2280 op = tcg_malloc(sizeof(TCGOp));
2281 } else {
2282 op = QTAILQ_FIRST(&s->free_ops);
2283 QTAILQ_REMOVE(&s->free_ops, op, link);
2284 }
2285 memset(op, 0, offsetof(TCGOp, link));
2286 op->opc = opc;
abebf925 2287 s->nb_ops++;
5a18407f 2288
15fa08f8
RH		 2289 return op;
2290}
2291
2292TCGOp *tcg_emit_op(TCGOpcode opc)
2293{
2294 TCGOp *op = tcg_op_alloc(opc);
2295 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2296 return op;
2297}
5a18407f 2298
ac1043f6 2299TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
15fa08f8
RH		 2300{
2301 TCGOp *new_op = tcg_op_alloc(opc);
2302 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2303 return new_op;
2304}
2305
ac1043f6 2306TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
5a18407f 2307{
15fa08f8
RH		 2308 TCGOp *new_op = tcg_op_alloc(opc);
2309 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2310 return new_op;
2311}
2312
b4fc67c7
RH		 2313/* Reachable analysis : remove unreachable code. */
2314static void reachable_code_pass(TCGContext *s)
2315{
2316 TCGOp *op, *op_next;
2317 bool dead = false;
2318
2319 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2320 bool remove = dead;
2321 TCGLabel *label;
b4fc67c7
RH		 2322
2323 switch (op->opc) {
2324 case INDEX_op_set_label:
2325 label = arg_label(op->args[0]);
2326 if (label->refs == 0) {
2327 /*
2328 * While there is an occasional backward branch, virtually
2329 * all branches generated by the translators are forward.
2330 * Which means that generally we will have already removed
2331 * all references to the label that will be, and there is
2332 * little to be gained by iterating.
2333 */
2334 remove = true;
2335 } else {
2336 /* Once we see a label, insns become live again. */
2337 dead = false;
2338 remove = false;
2339
2340 /*
2341 * Optimization can fold conditional branches to unconditional.
2342 * If we find a label with one reference which is preceded by
2343 * an unconditional branch to it, remove both. This needed to
2344 * wait until the dead code in between them was removed.
2345 */
2346 if (label->refs == 1) {
eae3eb3e 2347 TCGOp *op_prev = QTAILQ_PREV(op, link);
b4fc67c7
RH		 2348 if (op_prev->opc == INDEX_op_br &&
2349 label == arg_label(op_prev->args[0])) {
2350 tcg_op_remove(s, op_prev);
2351 remove = true;
2352 }
2353 }
2354 }
2355 break;
2356
2357 case INDEX_op_br:
2358 case INDEX_op_exit_tb:
2359 case INDEX_op_goto_ptr:
2360 /* Unconditional branches; everything following is dead. */
2361 dead = true;
2362 break;
2363
2364 case INDEX_op_call:
2365 /* Notice noreturn helper calls, raising exceptions. */
90163900 2366 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2367 dead = true;
2368 }
2369 break;
2370
2371 case INDEX_op_insn_start:
2372 /* Never remove -- we need to keep these for unwind. */
2373 remove = false;
2374 break;
2375
2376 default:
2377 break;
2378 }
2379
2380 if (remove) {
2381 tcg_op_remove(s, op);
2382 }
2383 }
2384}
2385
c70fbf0a
RH		 2386#define TS_DEAD 1
2387#define TS_MEM 2
2388
5a18407f
RH		 2389#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2390#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2391
25f49c5f
RH		 2392/* For liveness_pass_1, the register preferences for a given temp. */
2393static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2394{
2395 return ts->state_ptr;
2396}
2397
2398/* For liveness_pass_1, reset the preferences for a given temp to the
2399 * maximal regset for its type.
2400 */
2401static inline void la_reset_pref(TCGTemp *ts)
2402{
2403 *la_temp_pref(ts)
2404 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2405}
2406
9c43b68d
AJ		 2407/* liveness analysis: end of function: all temps are dead, and globals
2408 should be in memory. */
2616c808 2409static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2410{
b83eabea
RH		 2411 int i;
2412
2413 for (i = 0; i < ng; ++i) {
2414 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2415 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2416 }
2417 for (i = ng; i < nt; ++i) {
2418 s->temps[i].state = TS_DEAD;
25f49c5f 2419 la_reset_pref(&s->temps[i]);
b83eabea 2420 }
c896fe29
FB		 2421}
2422
9c43b68d
AJ		 2423/* liveness analysis: end of basic block: all temps are dead, globals
2424 and local temps should be in memory. */
2616c808 2425static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2426{
b83eabea 2427 int i;
641d5fbe 2428
ee17db83
RH		 2429 for (i = 0; i < nt; ++i) {
2430 TCGTemp *ts = &s->temps[i];
2431 int state;
2432
2433 switch (ts->kind) {
2434 case TEMP_FIXED:
2435 case TEMP_GLOBAL:
2436 case TEMP_LOCAL:
2437 state = TS_DEAD | TS_MEM;
2438 break;
2439 case TEMP_NORMAL:
c7482438 2440 case TEMP_EBB:
c0522136 2441 case TEMP_CONST:
ee17db83
RH		 2442 state = TS_DEAD;
2443 break;
2444 default:
2445 g_assert_not_reached();
2446 }
2447 ts->state = state;
2448 la_reset_pref(ts);
641d5fbe
FB		 2449 }
2450}
2451
f65a061c
RH		 2452/* liveness analysis: sync globals back to memory. */
2453static void la_global_sync(TCGContext *s, int ng)
2454{
2455 int i;
2456
2457 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2458 int state = s->temps[i].state;
2459 s->temps[i].state = state | TS_MEM;
2460 if (state == TS_DEAD) {
2461 /* If the global was previously dead, reset prefs. */
2462 la_reset_pref(&s->temps[i]);
2463 }
f65a061c
RH		 2464 }
2465}
2466
b4cb76e6 2467/*
c7482438
RH		 2468 * liveness analysis: conditional branch: all temps are dead unless
2469 * explicitly live-across-conditional-branch, globals and local temps
2470 * should be synced.
b4cb76e6
RH		 2471 */
2472static void la_bb_sync(TCGContext *s, int ng, int nt)
2473{
2474 la_global_sync(s, ng);
2475
2476 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2477 TCGTemp *ts = &s->temps[i];
2478 int state;
2479
2480 switch (ts->kind) {
2481 case TEMP_LOCAL:
2482 state = ts->state;
2483 ts->state = state | TS_MEM;
b4cb76e6
RH		 2484 if (state != TS_DEAD) {
2485 continue;
2486 }
c0522136
RH		 2487 break;
2488 case TEMP_NORMAL:
b4cb76e6 2489 s->temps[i].state = TS_DEAD;
c0522136 2490 break;
c7482438 2491 case TEMP_EBB:
c0522136
RH		 2492 case TEMP_CONST:
2493 continue;
2494 default:
2495 g_assert_not_reached();
b4cb76e6
RH		 2496 }
2497 la_reset_pref(&s->temps[i]);
2498 }
2499}
2500
f65a061c
RH		 2501/* liveness analysis: sync globals back to memory and kill. */
2502static void la_global_kill(TCGContext *s, int ng)
2503{
2504 int i;
2505
2506 for (i = 0; i < ng; i++) {
2507 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2508 la_reset_pref(&s->temps[i]);
2509 }
2510}
2511
2512/* liveness analysis: note live globals crossing calls. */
2513static void la_cross_call(TCGContext *s, int nt)
2514{
2515 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2516 int i;
2517
2518 for (i = 0; i < nt; i++) {
2519 TCGTemp *ts = &s->temps[i];
2520 if (!(ts->state & TS_DEAD)) {
2521 TCGRegSet *pset = la_temp_pref(ts);
2522 TCGRegSet set = *pset;
2523
2524 set &= mask;
2525 /* If the combination is not possible, restart. */
2526 if (set == 0) {
2527 set = tcg_target_available_regs[ts->type] & mask;
2528 }
2529 *pset = set;
2530 }
f65a061c
RH		 2531 }
2532}
2533
a1b3c48d 2534/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2535 given input arguments is dead. Instructions updating dead
2536 temporaries are removed. */
b83eabea 2537static void liveness_pass_1(TCGContext *s)
c896fe29 2538{
c70fbf0a 2539 int nb_globals = s->nb_globals;
2616c808 2540 int nb_temps = s->nb_temps;
15fa08f8 2541 TCGOp *op, *op_prev;
25f49c5f
RH		 2542 TCGRegSet *prefs;
2543 int i;
2544
2545 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2546 for (i = 0; i < nb_temps; ++i) {
2547 s->temps[i].state_ptr = prefs + i;
2548 }
a1b3c48d 2549
ae36a246 2550 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2551 la_func_end(s, nb_globals, nb_temps);
c896fe29 2552
eae3eb3e 2553 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2554 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2555 TCGOpcode opc_new, opc_new2;
2556 bool have_opc_new2;
a1b3c48d 2557 TCGLifeData arg_life = 0;
25f49c5f 2558 TCGTemp *ts;
c45cb8bb
RH		 2559 TCGOpcode opc = op->opc;
2560 const TCGOpDef *def = &tcg_op_defs[opc];
2561
c45cb8bb 2562 switch (opc) {
c896fe29 2563 case INDEX_op_call:
c6e113f5
FB		 2564 {
2565 int call_flags;
25f49c5f 2566 int nb_call_regs;
c896fe29 2567
cd9090aa
RH		 2568 nb_oargs = TCGOP_CALLO(op);
2569 nb_iargs = TCGOP_CALLI(op);
90163900 2570 call_flags = tcg_call_flags(op);
c6e113f5 2571
c45cb8bb 2572 /* pure functions can be removed if their result is unused */
78505279 2573 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2574 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2575 ts = arg_temp(op->args[i]);
2576 if (ts->state != TS_DEAD) {
c6e113f5 2577 goto do_not_remove_call;
9c43b68d 2578 }
c6e113f5 2579 }
c45cb8bb 2580 goto do_remove;
152c35aa
RH		 2581 }
2582 do_not_remove_call:
c896fe29 2583
25f49c5f 2584 /* Output args are dead. */
152c35aa 2585 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2586 ts = arg_temp(op->args[i]);
2587 if (ts->state & TS_DEAD) {
152c35aa
RH		 2588 arg_life |= DEAD_ARG << i;
2589 }
25f49c5f 2590 if (ts->state & TS_MEM) {
152c35aa 2591 arg_life |= SYNC_ARG << i;
c6e113f5 2592 }
25f49c5f
RH		 2593 ts->state = TS_DEAD;
2594 la_reset_pref(ts);
2595
2596 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2597 op->output_pref[i] = 0;
152c35aa 2598 }
78505279 2599
152c35aa
RH		 2600 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2601 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2602 la_global_kill(s, nb_globals);
152c35aa 2603 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2604 la_global_sync(s, nb_globals);
152c35aa 2605 }
b9c18f56 2606
25f49c5f 2607 /* Record arguments that die in this helper. */
152c35aa 2608 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f
RH		 2609 ts = arg_temp(op->args[i]);
2610 if (ts && ts->state & TS_DEAD) {
152c35aa 2611 arg_life |= DEAD_ARG << i;
c6e113f5 2612 }
152c35aa 2613 }
25f49c5f
RH		 2614
2615 /* For all live registers, remove call-clobbered prefs. */
2616 la_cross_call(s, nb_temps);
2617
2618 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2619
2620 /* Input arguments are live for preceding opcodes. */
2621 for (i = 0; i < nb_iargs; i++) {
2622 ts = arg_temp(op->args[i + nb_oargs]);
2623 if (ts && ts->state & TS_DEAD) {
2624 /* For those arguments that die, and will be allocated
2625 * in registers, clear the register set for that arg,
2626 * to be filled in below. For args that will be on
2627 * the stack, reset to any available reg.
2628 */
2629 *la_temp_pref(ts)
2630 = (i < nb_call_regs ? 0 :
2631 tcg_target_available_regs[ts->type]);
2632 ts->state &= ~TS_DEAD;
2633 }
2634 }
2635
2636 /* For each input argument, add its input register to prefs.
2637 If a temp is used once, this produces a single set bit. */
2638 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2639 ts = arg_temp(op->args[i + nb_oargs]);
2640 if (ts) {
2641 tcg_regset_set_reg(*la_temp_pref(ts),
2642 tcg_target_call_iarg_regs[i]);
c19f47bf 2643 }
c896fe29 2644 }
c896fe29 2645 }
c896fe29 2646 break;
765b842a 2647 case INDEX_op_insn_start:
c896fe29 2648 break;
5ff9d6a4 2649 case INDEX_op_discard:
5ff9d6a4 2650 /* mark the temporary as dead */
25f49c5f
RH		 2651 ts = arg_temp(op->args[0]);
2652 ts->state = TS_DEAD;
2653 la_reset_pref(ts);
5ff9d6a4 2654 break;
1305c451
RH		 2655
2656 case INDEX_op_add2_i32:
c45cb8bb 2657 opc_new = INDEX_op_add_i32;
f1fae40c 2658 goto do_addsub2;
1305c451 2659 case INDEX_op_sub2_i32:
c45cb8bb 2660 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 2661 goto do_addsub2;
2662 case INDEX_op_add2_i64:
c45cb8bb 2663 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 2664 goto do_addsub2;
2665 case INDEX_op_sub2_i64:
c45cb8bb 2666 opc_new = INDEX_op_sub_i64;
f1fae40c 2667 do_addsub2:
1305c451
RH		 2668 nb_iargs = 4;
2669 nb_oargs = 2;
2670 /* Test if the high part of the operation is dead, but not
2671 the low part. The result can be optimized to a simple
2672 add or sub. This happens often for x86_64 guest when the
2673 cpu mode is set to 32 bit. */
b83eabea
RH		 2674 if (arg_temp(op->args[1])->state == TS_DEAD) {
2675 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 2676 goto do_remove;
2677 }
c45cb8bb
RH		 2678 /* Replace the opcode and adjust the args in place,
2679 leaving 3 unused args at the end. */
2680 op->opc = opc = opc_new;
efee3746
RH		 2681 op->args[1] = op->args[2];
2682 op->args[2] = op->args[4];
1305c451
RH		 2683 /* Fall through and mark the single-word operation live. */
2684 nb_iargs = 2;
2685 nb_oargs = 1;
2686 }
2687 goto do_not_remove;
2688
1414968a 2689 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 2690 opc_new = INDEX_op_mul_i32;
2691 opc_new2 = INDEX_op_muluh_i32;
2692 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 2693 goto do_mul2;
f1fae40c 2694 case INDEX_op_muls2_i32:
c45cb8bb
RH		 2695 opc_new = INDEX_op_mul_i32;
2696 opc_new2 = INDEX_op_mulsh_i32;
2697 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 2698 goto do_mul2;
2699 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 2700 opc_new = INDEX_op_mul_i64;
2701 opc_new2 = INDEX_op_muluh_i64;
2702 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 2703 goto do_mul2;
f1fae40c 2704 case INDEX_op_muls2_i64:
c45cb8bb
RH		 2705 opc_new = INDEX_op_mul_i64;
2706 opc_new2 = INDEX_op_mulsh_i64;
2707 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 2708 goto do_mul2;
f1fae40c 2709 do_mul2:
1414968a
RH		 2710 nb_iargs = 2;
2711 nb_oargs = 2;
b83eabea
RH		 2712 if (arg_temp(op->args[1])->state == TS_DEAD) {
2713 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 2714 /* Both parts of the operation are dead. */
1414968a
RH		 2715 goto do_remove;
2716 }
03271524 2717 /* The high part of the operation is dead; generate the low. */
c45cb8bb 2718 op->opc = opc = opc_new;
efee3746
RH		 2719 op->args[1] = op->args[2];
2720 op->args[2] = op->args[3];
b83eabea 2721 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 2722 /* The low part of the operation is dead; generate the high. */
2723 op->opc = opc = opc_new2;
efee3746
RH		 2724 op->args[0] = op->args[1];
2725 op->args[1] = op->args[2];
2726 op->args[2] = op->args[3];
03271524
RH		 2727 } else {
2728 goto do_not_remove;
1414968a 2729 }
03271524
RH		 2730 /* Mark the single-word operation live. */
2731 nb_oargs = 1;
1414968a
RH		 2732 goto do_not_remove;
2733
c896fe29 2734 default:
1305c451 2735 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 2736 nb_iargs = def->nb_iargs;
2737 nb_oargs = def->nb_oargs;
c896fe29 2738
49516bc0
AJ		 2739 /* Test if the operation can be removed because all
2740 its outputs are dead. We assume that nb_oargs == 0
2741 implies side effects */
2742 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 2743 for (i = 0; i < nb_oargs; i++) {
b83eabea 2744 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 2745 goto do_not_remove;
9c43b68d 2746 }
49516bc0 2747 }
152c35aa
RH		 2748 goto do_remove;
2749 }
2750 goto do_not_remove;
49516bc0 2751
152c35aa
RH		 2752 do_remove:
2753 tcg_op_remove(s, op);
2754 break;
2755
2756 do_not_remove:
152c35aa 2757 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2758 ts = arg_temp(op->args[i]);
2759
2760 /* Remember the preference of the uses that followed. */
2761 op->output_pref[i] = *la_temp_pref(ts);
2762
2763 /* Output args are dead. */
2764 if (ts->state & TS_DEAD) {
152c35aa 2765 arg_life |= DEAD_ARG << i;
49516bc0 2766 }
25f49c5f 2767 if (ts->state & TS_MEM) {
152c35aa
RH		 2768 arg_life |= SYNC_ARG << i;
2769 }
25f49c5f
RH		 2770 ts->state = TS_DEAD;
2771 la_reset_pref(ts);
152c35aa 2772 }
49516bc0 2773
25f49c5f 2774 /* If end of basic block, update. */
ae36a246
RH		 2775 if (def->flags & TCG_OPF_BB_EXIT) {
2776 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 2777 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2778 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 2779 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 2780 la_bb_end(s, nb_globals, nb_temps);
152c35aa 2781 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 2782 la_global_sync(s, nb_globals);
25f49c5f
RH		 2783 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2784 la_cross_call(s, nb_temps);
2785 }
152c35aa
RH		 2786 }
2787
25f49c5f 2788 /* Record arguments that die in this opcode. */
152c35aa 2789 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2790 ts = arg_temp(op->args[i]);
2791 if (ts->state & TS_DEAD) {
152c35aa 2792 arg_life |= DEAD_ARG << i;
c896fe29 2793 }
c896fe29 2794 }
25f49c5f
RH		 2795
2796 /* Input arguments are live for preceding opcodes. */
152c35aa 2797 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2798 ts = arg_temp(op->args[i]);
2799 if (ts->state & TS_DEAD) {
2800 /* For operands that were dead, initially allow
2801 all regs for the type. */
2802 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2803 ts->state &= ~TS_DEAD;
2804 }
2805 }
2806
2807 /* Incorporate constraints for this operand. */
2808 switch (opc) {
2809 case INDEX_op_mov_i32:
2810 case INDEX_op_mov_i64:
2811 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2812 have proper constraints. That said, special case
2813 moves to propagate preferences backward. */
2814 if (IS_DEAD_ARG(1)) {
2815 *la_temp_pref(arg_temp(op->args[0]))
2816 = *la_temp_pref(arg_temp(op->args[1]));
2817 }
2818 break;
2819
2820 default:
2821 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2822 const TCGArgConstraint *ct = &def->args_ct[i];
2823 TCGRegSet set, *pset;
2824
2825 ts = arg_temp(op->args[i]);
2826 pset = la_temp_pref(ts);
2827 set = *pset;
2828
9be0d080 2829 set &= ct->regs;
bc2b17e6 2830 if (ct->ialias) {
25f49c5f
RH		 2831 set &= op->output_pref[ct->alias_index];
2832 }
2833 /* If the combination is not possible, restart. */
2834 if (set == 0) {
9be0d080 2835 set = ct->regs;
25f49c5f
RH		 2836 }
2837 *pset = set;
2838 }
2839 break;
152c35aa 2840 }
c896fe29
FB		 2841 break;
2842 }
bee158cb 2843 op->life = arg_life;
1ff0a2c5 2844 }
c896fe29 2845}
c896fe29 2846
5a18407f 2847/* Liveness analysis: Convert indirect regs to direct temporaries. */
b83eabea 2848static bool liveness_pass_2(TCGContext *s)
5a18407f
RH		 2849{
2850 int nb_globals = s->nb_globals;
15fa08f8 2851 int nb_temps, i;
5a18407f 2852 bool changes = false;
15fa08f8 2853 TCGOp *op, *op_next;
5a18407f 2854
5a18407f
RH		 2855 /* Create a temporary for each indirect global. */
2856 for (i = 0; i < nb_globals; ++i) {
2857 TCGTemp *its = &s->temps[i];
2858 if (its->indirect_reg) {
2859 TCGTemp *dts = tcg_temp_alloc(s);
2860 dts->type = its->type;
2861 dts->base_type = its->base_type;
c7482438 2862 dts->kind = TEMP_EBB;
b83eabea
RH		 2863 its->state_ptr = dts;
2864 } else {
2865 its->state_ptr = NULL;
5a18407f 2866 }
b83eabea
RH		 2867 /* All globals begin dead. */
2868 its->state = TS_DEAD;
2869 }
2870 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2871 TCGTemp *its = &s->temps[i];
2872 its->state_ptr = NULL;
2873 its->state = TS_DEAD;
5a18407f 2874 }
5a18407f 2875
15fa08f8 2876 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 2877 TCGOpcode opc = op->opc;
2878 const TCGOpDef *def = &tcg_op_defs[opc];
2879 TCGLifeData arg_life = op->life;
2880 int nb_iargs, nb_oargs, call_flags;
b83eabea 2881 TCGTemp *arg_ts, *dir_ts;
5a18407f 2882
5a18407f 2883 if (opc == INDEX_op_call) {
cd9090aa
RH		 2884 nb_oargs = TCGOP_CALLO(op);
2885 nb_iargs = TCGOP_CALLI(op);
90163900 2886 call_flags = tcg_call_flags(op);
5a18407f
RH		 2887 } else {
2888 nb_iargs = def->nb_iargs;
2889 nb_oargs = def->nb_oargs;
2890
2891 /* Set flags similar to how calls require. */
b4cb76e6
RH		 2892 if (def->flags & TCG_OPF_COND_BRANCH) {
2893 /* Like reading globals: sync_globals */
2894 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2895 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 2896 /* Like writing globals: save_globals */
2897 call_flags = 0;
2898 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2899 /* Like reading globals: sync_globals */
2900 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2901 } else {
2902 /* No effect on globals. */
2903 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2904 TCG_CALL_NO_WRITE_GLOBALS);
2905 }
2906 }
2907
2908 /* Make sure that input arguments are available. */
2909 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2910 arg_ts = arg_temp(op->args[i]);
2911 if (arg_ts) {
2912 dir_ts = arg_ts->state_ptr;
2913 if (dir_ts && arg_ts->state == TS_DEAD) {
2914 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
5a18407f
RH		 2915 ? INDEX_op_ld_i32
2916 : INDEX_op_ld_i64);
ac1043f6 2917 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
5a18407f 2918
b83eabea
RH		 2919 lop->args[0] = temp_arg(dir_ts);
2920 lop->args[1] = temp_arg(arg_ts->mem_base);
2921 lop->args[2] = arg_ts->mem_offset;
5a18407f
RH		 2922
2923 /* Loaded, but synced with memory. */
b83eabea 2924 arg_ts->state = TS_MEM;
5a18407f
RH		 2925 }
2926 }
2927 }
2928
2929 /* Perform input replacement, and mark inputs that became dead.
2930 No action is required except keeping temp_state up to date
2931 so that we reload when needed. */
2932 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2933 arg_ts = arg_temp(op->args[i]);
2934 if (arg_ts) {
2935 dir_ts = arg_ts->state_ptr;
2936 if (dir_ts) {
2937 op->args[i] = temp_arg(dir_ts);
5a18407f
RH		 2938 changes = true;
2939 if (IS_DEAD_ARG(i)) {
b83eabea 2940 arg_ts->state = TS_DEAD;
5a18407f
RH		 2941 }
2942 }
2943 }
2944 }
2945
2946 /* Liveness analysis should ensure that the following are
2947 all correct, for call sites and basic block end points. */
2948 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2949 /* Nothing to do */
2950 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2951 for (i = 0; i < nb_globals; ++i) {
2952 /* Liveness should see that globals are synced back,
2953 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 2954 arg_ts = &s->temps[i];
2955 tcg_debug_assert(arg_ts->state_ptr == 0
2956 || arg_ts->state != 0);
5a18407f
RH		 2957 }
2958 } else {
2959 for (i = 0; i < nb_globals; ++i) {
2960 /* Liveness should see that globals are saved back,
2961 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 2962 arg_ts = &s->temps[i];
2963 tcg_debug_assert(arg_ts->state_ptr == 0
2964 || arg_ts->state == TS_DEAD);
5a18407f
RH		 2965 }
2966 }
2967
2968 /* Outputs become available. */
61f15c48
RH		 2969 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2970 arg_ts = arg_temp(op->args[0]);
b83eabea 2971 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 2972 if (dir_ts) {
2973 op->args[0] = temp_arg(dir_ts);
2974 changes = true;
2975
2976 /* The output is now live and modified. */
2977 arg_ts->state = 0;
2978
2979 if (NEED_SYNC_ARG(0)) {
2980 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2981 ? INDEX_op_st_i32
2982 : INDEX_op_st_i64);
2983 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2984 TCGTemp *out_ts = dir_ts;
2985
2986 if (IS_DEAD_ARG(0)) {
2987 out_ts = arg_temp(op->args[1]);
2988 arg_ts->state = TS_DEAD;
2989 tcg_op_remove(s, op);
2990 } else {
2991 arg_ts->state = TS_MEM;
2992 }
2993
2994 sop->args[0] = temp_arg(out_ts);
2995 sop->args[1] = temp_arg(arg_ts->mem_base);
2996 sop->args[2] = arg_ts->mem_offset;
2997 } else {
2998 tcg_debug_assert(!IS_DEAD_ARG(0));
2999 }
5a18407f 3000 }
61f15c48
RH		 3001 } else {
3002 for (i = 0; i < nb_oargs; i++) {
3003 arg_ts = arg_temp(op->args[i]);
3004 dir_ts = arg_ts->state_ptr;
3005 if (!dir_ts) {
3006 continue;
3007 }
3008 op->args[i] = temp_arg(dir_ts);
3009 changes = true;
5a18407f 3010
61f15c48
RH		 3011 /* The output is now live and modified. */
3012 arg_ts->state = 0;
5a18407f 3013
61f15c48
RH		 3014 /* Sync outputs upon their last write. */
3015 if (NEED_SYNC_ARG(i)) {
3016 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3017 ? INDEX_op_st_i32
3018 : INDEX_op_st_i64);
3019 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
5a18407f 3020
61f15c48
RH		 3021 sop->args[0] = temp_arg(dir_ts);
3022 sop->args[1] = temp_arg(arg_ts->mem_base);
3023 sop->args[2] = arg_ts->mem_offset;
5a18407f 3024
61f15c48
RH		 3025 arg_ts->state = TS_MEM;
3026 }
3027 /* Drop outputs that are dead. */
3028 if (IS_DEAD_ARG(i)) {
3029 arg_ts->state = TS_DEAD;
3030 }
5a18407f
RH		 3031 }
3032 }
3033 }
3034
3035 return changes;
3036}
3037
2272e4a7 3038static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 3039{
c1c09194
RH		 3040 intptr_t off, size, align;
3041
3042 switch (ts->type) {
3043 case TCG_TYPE_I32:
3044 size = align = 4;
3045 break;
3046 case TCG_TYPE_I64:
3047 case TCG_TYPE_V64:
3048 size = align = 8;
3049 break;
3050 case TCG_TYPE_V128:
3051 size = align = 16;
3052 break;
3053 case TCG_TYPE_V256:
3054 /* Note that we do not require aligned storage for V256. */
3055 size = 32, align = 16;
3056 break;
3057 default:
3058 g_assert_not_reached();
b591dc59 3059 }
c1c09194 3060
b9537d59
RH		 3061 /*
3062 * Assume the stack is sufficiently aligned.
3063 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3064 * and do not require 16 byte vector alignment. This seems slightly
3065 * easier than fully parameterizing the above switch statement.
3066 */
3067 align = MIN(TCG_TARGET_STACK_ALIGN, align);
c1c09194 3068 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 3069
3070 /* If we've exhausted the stack frame, restart with a smaller TB. */
3071 if (off + size > s->frame_end) {
3072 tcg_raise_tb_overflow(s);
3073 }
c1c09194
RH		 3074 s->current_frame_offset = off + size;
3075
3076 ts->mem_offset = off;
9defd1bd
RH		 3077#if defined(__sparc__)
3078 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3079#endif
b3a62939 3080 ts->mem_base = s->frame_temp;
c896fe29 3081 ts->mem_allocated = 1;
c896fe29
FB		 3082}
3083
098859f1
RH		 3084/* Assign @reg to @ts, and update reg_to_temp[]. */
3085static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
3086{
3087 if (ts->val_type == TEMP_VAL_REG) {
3088 TCGReg old = ts->reg;
3089 tcg_debug_assert(s->reg_to_temp[old] == ts);
3090 if (old == reg) {
3091 return;
3092 }
3093 s->reg_to_temp[old] = NULL;
3094 }
3095 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
3096 s->reg_to_temp[reg] = ts;
3097 ts->val_type = TEMP_VAL_REG;
3098 ts->reg = reg;
3099}
3100
3101/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
3102static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
3103{
3104 tcg_debug_assert(type != TEMP_VAL_REG);
3105 if (ts->val_type == TEMP_VAL_REG) {
3106 TCGReg reg = ts->reg;
3107 tcg_debug_assert(s->reg_to_temp[reg] == ts);
3108 s->reg_to_temp[reg] = NULL;
3109 }
3110 ts->val_type = type;
3111}
3112
b722452a 3113static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3114
59d7c14e
RH		 3115/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3116 mark it free; otherwise mark it dead. */
3117static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3118{
c0522136
RH		 3119 TCGTempVal new_type;
3120
3121 switch (ts->kind) {
3122 case TEMP_FIXED:
59d7c14e 3123 return;
c0522136
RH		 3124 case TEMP_GLOBAL:
3125 case TEMP_LOCAL:
3126 new_type = TEMP_VAL_MEM;
3127 break;
3128 case TEMP_NORMAL:
c7482438 3129 case TEMP_EBB:
c0522136
RH		 3130 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3131 break;
3132 case TEMP_CONST:
3133 new_type = TEMP_VAL_CONST;
3134 break;
3135 default:
3136 g_assert_not_reached();
59d7c14e 3137 }
098859f1 3138 set_temp_val_nonreg(s, ts, new_type);
59d7c14e 3139}
7f6ceedf 3140
59d7c14e
RH		 3141/* Mark a temporary as dead. */
3142static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3143{
3144 temp_free_or_dead(s, ts, 1);
3145}
3146
3147/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3148 registers needs to be allocated to store a constant. If 'free_or_dead'
3149 is non-zero, subsequently release the temporary; if it is positive, the
3150 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3151static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3152 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3153{
c0522136 3154 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3155 if (!ts->mem_allocated) {
2272e4a7 3156 temp_allocate_frame(s, ts);
59d7c14e 3157 }
59d7c14e
RH		 3158 switch (ts->val_type) {
3159 case TEMP_VAL_CONST:
3160 /* If we're going to free the temp immediately, then we won't
3161 require it later in a register, so attempt to store the
3162 constant to memory directly. */
3163 if (free_or_dead
3164 && tcg_out_sti(s, ts->type, ts->val,
3165 ts->mem_base->reg, ts->mem_offset)) {
3166 break;
3167 }
3168 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3169 allocated_regs, preferred_regs);
59d7c14e
RH		 3170 /* fallthrough */
3171
3172 case TEMP_VAL_REG:
3173 tcg_out_st(s, ts->type, ts->reg,
3174 ts->mem_base->reg, ts->mem_offset);
3175 break;
3176
3177 case TEMP_VAL_MEM:
3178 break;
3179
3180 case TEMP_VAL_DEAD:
3181 default:
3182 tcg_abort();
3183 }
3184 ts->mem_coherent = 1;
3185 }
3186 if (free_or_dead) {
3187 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3188 }
7f6ceedf
AJ		 3189}
3190
c896fe29 3191/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3192static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3193{
f8b2f202 3194 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3195 if (ts != NULL) {
98b4e186 3196 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3197 }
3198}
3199
b016486e
RH		 3200/**
3201 * tcg_reg_alloc:
3202 * @required_regs: Set of registers in which we must allocate.
3203 * @allocated_regs: Set of registers which must be avoided.
3204 * @preferred_regs: Set of registers we should prefer.
3205 * @rev: True if we search the registers in "indirect" order.
3206 *
3207 * The allocated register must be in @required_regs & ~@allocated_regs,
3208 * but if we can put it in @preferred_regs we may save a move later.
3209 */
3210static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3211 TCGRegSet allocated_regs,
3212 TCGRegSet preferred_regs, bool rev)
c896fe29 3213{
b016486e
RH		 3214 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3215 TCGRegSet reg_ct[2];
91478cef 3216 const int *order;
c896fe29 3217
b016486e
RH		 3218 reg_ct[1] = required_regs & ~allocated_regs;
3219 tcg_debug_assert(reg_ct[1] != 0);
3220 reg_ct[0] = reg_ct[1] & preferred_regs;
3221
3222 /* Skip the preferred_regs option if it cannot be satisfied,
3223 or if the preference made no difference. */
3224 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3225
91478cef 3226 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3227
b016486e
RH		 3228 /* Try free registers, preferences first. */
3229 for (j = f; j < 2; j++) {
3230 TCGRegSet set = reg_ct[j];
3231
3232 if (tcg_regset_single(set)) {
3233 /* One register in the set. */
3234 TCGReg reg = tcg_regset_first(set);
3235 if (s->reg_to_temp[reg] == NULL) {
3236 return reg;
3237 }
3238 } else {
3239 for (i = 0; i < n; i++) {
3240 TCGReg reg = order[i];
3241 if (s->reg_to_temp[reg] == NULL &&
3242 tcg_regset_test_reg(set, reg)) {
3243 return reg;
3244 }
3245 }
3246 }
c896fe29
FB		 3247 }
3248
b016486e
RH		 3249 /* We must spill something. */
3250 for (j = f; j < 2; j++) {
3251 TCGRegSet set = reg_ct[j];
3252
3253 if (tcg_regset_single(set)) {
3254 /* One register in the set. */
3255 TCGReg reg = tcg_regset_first(set);
b3915dbb 3256 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3257 return reg;
b016486e
RH		 3258 } else {
3259 for (i = 0; i < n; i++) {
3260 TCGReg reg = order[i];
3261 if (tcg_regset_test_reg(set, reg)) {
3262 tcg_reg_free(s, reg, allocated_regs);
3263 return reg;
3264 }
3265 }
c896fe29
FB		 3266 }
3267 }
3268
3269 tcg_abort();
3270}
3271
29f5e925
RH		 3272static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
3273 TCGRegSet allocated_regs,
3274 TCGRegSet preferred_regs, bool rev)
3275{
3276 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3277 TCGRegSet reg_ct[2];
3278 const int *order;
3279
3280 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
3281 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
3282 tcg_debug_assert(reg_ct[1] != 0);
3283 reg_ct[0] = reg_ct[1] & preferred_regs;
3284
3285 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3286
3287 /*
3288 * Skip the preferred_regs option if it cannot be satisfied,
3289 * or if the preference made no difference.
3290 */
3291 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3292
3293 /*
3294 * Minimize the number of flushes by looking for 2 free registers first,
3295 * then a single flush, then two flushes.
3296 */
3297 for (fmin = 2; fmin >= 0; fmin--) {
3298 for (j = k; j < 2; j++) {
3299 TCGRegSet set = reg_ct[j];
3300
3301 for (i = 0; i < n; i++) {
3302 TCGReg reg = order[i];
3303
3304 if (tcg_regset_test_reg(set, reg)) {
3305 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
3306 if (f >= fmin) {
3307 tcg_reg_free(s, reg, allocated_regs);
3308 tcg_reg_free(s, reg + 1, allocated_regs);
3309 return reg;
3310 }
3311 }
3312 }
3313 }
3314 }
3315 tcg_abort();
3316}
3317
40ae5c62
RH		 3318/* Make sure the temporary is in a register. If needed, allocate the register
3319 from DESIRED while avoiding ALLOCATED. */
3320static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3321 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3322{
3323 TCGReg reg;
3324
3325 switch (ts->val_type) {
3326 case TEMP_VAL_REG:
3327 return;
3328 case TEMP_VAL_CONST:
b016486e 3329 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3330 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3331 if (ts->type <= TCG_TYPE_I64) {
3332 tcg_out_movi(s, ts->type, reg, ts->val);
3333 } else {
4e186175
RH		 3334 uint64_t val = ts->val;
3335 MemOp vece = MO_64;
3336
3337 /*
3338 * Find the minimal vector element that matches the constant.
3339 * The targets will, in general, have to do this search anyway,
3340 * do this generically.
3341 */
4e186175
RH		 3342 if (val == dup_const(MO_8, val)) {
3343 vece = MO_8;
3344 } else if (val == dup_const(MO_16, val)) {
3345 vece = MO_16;
0b4286dd 3346 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3347 vece = MO_32;
3348 }
3349
3350 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3351 }
40ae5c62
RH		 3352 ts->mem_coherent = 0;
3353 break;
3354 case TEMP_VAL_MEM:
b016486e 3355 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3356 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3357 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3358 ts->mem_coherent = 1;
3359 break;
3360 case TEMP_VAL_DEAD:
3361 default:
3362 tcg_abort();
3363 }
098859f1 3364 set_temp_val_reg(s, ts, reg);
40ae5c62
RH		 3365}
3366
59d7c14e
RH		 3367/* Save a temporary to memory. 'allocated_regs' is used in case a
3368 temporary registers needs to be allocated to store a constant. */
3369static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3370{
5a18407f
RH		 3371 /* The liveness analysis already ensures that globals are back
3372 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3373 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3374}
3375
9814dd27 3376/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3377 modified be the following code. 'allocated_regs' is used in case a
3378 temporary registers needs to be allocated to store a constant. */
3379static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3380{
ac3b8891 3381 int i, n;
c896fe29 3382
ac3b8891 3383 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3384 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3385 }
e5097dc8
FB		 3386}
3387
3d5c5f87
AJ		 3388/* sync globals to their canonical location and assume they can be
3389 read by the following code. 'allocated_regs' is used in case a
3390 temporary registers needs to be allocated to store a constant. */
3391static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3392{
ac3b8891 3393 int i, n;
3d5c5f87 3394
ac3b8891 3395 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3396 TCGTemp *ts = &s->temps[i];
5a18407f 3397 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3398 || ts->kind == TEMP_FIXED
5a18407f 3399 || ts->mem_coherent);
3d5c5f87
AJ		 3400 }
3401}
3402
e5097dc8 3403/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3404 all globals are stored at their canonical location. */
3405static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3406{
e5097dc8
FB		 3407 int i;
3408
b13eb728
RH		 3409 for (i = s->nb_globals; i < s->nb_temps; i++) {
3410 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3411
3412 switch (ts->kind) {
3413 case TEMP_LOCAL:
b13eb728 3414 temp_save(s, ts, allocated_regs);
c0522136
RH		 3415 break;
3416 case TEMP_NORMAL:
c7482438 3417 case TEMP_EBB:
5a18407f
RH		 3418 /* The liveness analysis already ensures that temps are dead.
3419 Keep an tcg_debug_assert for safety. */
3420 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3421 break;
3422 case TEMP_CONST:
3423 /* Similarly, we should have freed any allocated register. */
3424 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3425 break;
3426 default:
3427 g_assert_not_reached();
c896fe29
FB		 3428 }
3429 }
e8996ee0
FB		 3430
3431 save_globals(s, allocated_regs);
c896fe29
FB		 3432}
3433
b4cb76e6 3434/*
c7482438
RH		 3435 * At a conditional branch, we assume all temporaries are dead unless
3436 * explicitly live-across-conditional-branch; all globals and local
3437 * temps are synced to their location.
b4cb76e6
RH		 3438 */
3439static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3440{
3441 sync_globals(s, allocated_regs);
3442
3443 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3444 TCGTemp *ts = &s->temps[i];
3445 /*
3446 * The liveness analysis already ensures that temps are dead.
3447 * Keep tcg_debug_asserts for safety.
3448 */
c0522136
RH		 3449 switch (ts->kind) {
3450 case TEMP_LOCAL:
b4cb76e6 3451 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136
RH		 3452 break;
3453 case TEMP_NORMAL:
b4cb76e6 3454 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136 3455 break;
c7482438 3456 case TEMP_EBB:
c0522136
RH		 3457 case TEMP_CONST:
3458 break;
3459 default:
3460 g_assert_not_reached();
b4cb76e6
RH		 3461 }
3462 }
3463}
3464
bab1671f 3465/*
c58f4c97 3466 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3467 */
0fe4fca4 3468static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3469 tcg_target_ulong val, TCGLifeData arg_life,
3470 TCGRegSet preferred_regs)
e8996ee0 3471{
d63e3b6e 3472 /* ENV should not be modified. */
e01fa97d 3473 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3474
3475 /* The movi is not explicitly generated here. */
098859f1 3476 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
59d7c14e
RH		 3477 ots->val = val;
3478 ots->mem_coherent = 0;
3479 if (NEED_SYNC_ARG(0)) {
ba87719c 3480 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3481 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3482 temp_dead(s, ots);
4c4e1ab2 3483 }
e8996ee0
FB		 3484}
3485
bab1671f
RH		 3486/*
3487 * Specialized code generation for INDEX_op_mov_*.
3488 */
dd186292 3489static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3490{
dd186292 3491 const TCGLifeData arg_life = op->life;
69e3706d 3492 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3493 TCGTemp *ts, *ots;
450445d5 3494 TCGType otype, itype;
098859f1 3495 TCGReg oreg, ireg;
c896fe29 3496
d21369f5 3497 allocated_regs = s->reserved_regs;
69e3706d 3498 preferred_regs = op->output_pref[0];
43439139
RH		 3499 ots = arg_temp(op->args[0]);
3500 ts = arg_temp(op->args[1]);
450445d5 3501
d63e3b6e 3502 /* ENV should not be modified. */
e01fa97d 3503 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3504
450445d5
RH		 3505 /* Note that otype != itype for no-op truncation. */
3506 otype = ots->type;
3507 itype = ts->type;
c29c1d7e 3508
0fe4fca4
PB		 3509 if (ts->val_type == TEMP_VAL_CONST) {
3510 /* propagate constant or generate sti */
3511 tcg_target_ulong val = ts->val;
3512 if (IS_DEAD_ARG(1)) {
3513 temp_dead(s, ts);
3514 }
69e3706d 3515 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3516 return;
3517 }
3518
3519 /* If the source value is in memory we're going to be forced
3520 to have it in a register in order to perform the copy. Copy
3521 the SOURCE value into its own register first, that way we
3522 don't have to reload SOURCE the next time it is used. */
3523 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3524 temp_load(s, ts, tcg_target_available_regs[itype],
3525 allocated_regs, preferred_regs);
c29c1d7e 3526 }
0fe4fca4 3527 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
098859f1
RH		 3528 ireg = ts->reg;
3529
d63e3b6e 3530 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3531 /* mov to a non-saved dead register makes no sense (even with
3532 liveness analysis disabled). */
eabb7b91 3533 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3534 if (!ots->mem_allocated) {
2272e4a7 3535 temp_allocate_frame(s, ots);
c29c1d7e 3536 }
098859f1 3537 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3538 if (IS_DEAD_ARG(1)) {
f8bf00f1 3539 temp_dead(s, ts);
c29c1d7e 3540 }
f8bf00f1 3541 temp_dead(s, ots);
098859f1
RH		 3542 return;
3543 }
3544
3545 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
3546 /*
3547 * The mov can be suppressed. Kill input first, so that it
3548 * is unlinked from reg_to_temp, then set the output to the
3549 * reg that we saved from the input.
3550 */
3551 temp_dead(s, ts);
3552 oreg = ireg;
c29c1d7e 3553 } else {
098859f1
RH		 3554 if (ots->val_type == TEMP_VAL_REG) {
3555 oreg = ots->reg;
c896fe29 3556 } else {
098859f1
RH		 3557 /* Make sure to not spill the input register during allocation. */
3558 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
3559 allocated_regs | ((TCGRegSet)1 << ireg),
3560 preferred_regs, ots->indirect_base);
c896fe29 3561 }
098859f1
RH		 3562 if (!tcg_out_mov(s, otype, oreg, ireg)) {
3563 /*
3564 * Cross register class move not supported.
3565 * Store the source register into the destination slot
3566 * and leave the destination temp as TEMP_VAL_MEM.
3567 */
3568 assert(!temp_readonly(ots));
3569 if (!ts->mem_allocated) {
3570 temp_allocate_frame(s, ots);
3571 }
3572 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
3573 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
3574 ots->mem_coherent = 1;
3575 return;
c896fe29 3576 }
ec7a869d 3577 }
098859f1
RH		 3578 set_temp_val_reg(s, ots, oreg);
3579 ots->mem_coherent = 0;
3580
3581 if (NEED_SYNC_ARG(0)) {
3582 temp_sync(s, ots, allocated_regs, 0, 0);
3583 }
c896fe29
FB		 3584}
3585
bab1671f
RH		 3586/*
3587 * Specialized code generation for INDEX_op_dup_vec.
3588 */
3589static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3590{
3591 const TCGLifeData arg_life = op->life;
3592 TCGRegSet dup_out_regs, dup_in_regs;
3593 TCGTemp *its, *ots;
3594 TCGType itype, vtype;
d6ecb4a9 3595 intptr_t endian_fixup;
bab1671f
RH		 3596 unsigned vece;
3597 bool ok;
3598
3599 ots = arg_temp(op->args[0]);
3600 its = arg_temp(op->args[1]);
3601
3602 /* ENV should not be modified. */
e01fa97d 3603 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 3604
3605 itype = its->type;
3606 vece = TCGOP_VECE(op);
3607 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3608
3609 if (its->val_type == TEMP_VAL_CONST) {
3610 /* Propagate constant via movi -> dupi. */
3611 tcg_target_ulong val = its->val;
3612 if (IS_DEAD_ARG(1)) {
3613 temp_dead(s, its);
3614 }
3615 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3616 return;
3617 }
3618
9be0d080
RH		 3619 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3620 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 3621
3622 /* Allocate the output register now. */
3623 if (ots->val_type != TEMP_VAL_REG) {
3624 TCGRegSet allocated_regs = s->reserved_regs;
098859f1 3625 TCGReg oreg;
bab1671f
RH		 3626
3627 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3628 /* Make sure to not spill the input register. */
3629 tcg_regset_set_reg(allocated_regs, its->reg);
3630 }
098859f1
RH		 3631 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3632 op->output_pref[0], ots->indirect_base);
3633 set_temp_val_reg(s, ots, oreg);
bab1671f
RH		 3634 }
3635
3636 switch (its->val_type) {
3637 case TEMP_VAL_REG:
3638 /*
3639 * The dup constriaints must be broad, covering all possible VECE.
3640 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3641 * to fail, indicating that extra moves are required for that case.
3642 */
3643 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3644 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3645 goto done;
3646 }
3647 /* Try again from memory or a vector input register. */
3648 }
3649 if (!its->mem_coherent) {
3650 /*
3651 * The input register is not synced, and so an extra store
3652 * would be required to use memory. Attempt an integer-vector
3653 * register move first. We do not have a TCGRegSet for this.
3654 */
3655 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3656 break;
3657 }
3658 /* Sync the temp back to its slot and load from there. */
3659 temp_sync(s, its, s->reserved_regs, 0, 0);
3660 }
3661 /* fall through */
3662
3663 case TEMP_VAL_MEM:
e03b5686 3664#if HOST_BIG_ENDIAN
d6ecb4a9
RH		 3665 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3666 endian_fixup -= 1 << vece;
3667#else
3668 endian_fixup = 0;
3669#endif
098859f1 3670 /* Attempt to dup directly from the input memory slot. */
d6ecb4a9
RH		 3671 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3672 its->mem_offset + endian_fixup)) {
3673 goto done;
3674 }
098859f1 3675 /* Load the input into the destination vector register. */
bab1671f
RH		 3676 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3677 break;
3678
3679 default:
3680 g_assert_not_reached();
3681 }
3682
3683 /* We now have a vector input register, so dup must succeed. */
3684 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3685 tcg_debug_assert(ok);
3686
3687 done:
36f5539c 3688 ots->mem_coherent = 0;
bab1671f
RH		 3689 if (IS_DEAD_ARG(1)) {
3690 temp_dead(s, its);
3691 }
3692 if (NEED_SYNC_ARG(0)) {
3693 temp_sync(s, ots, s->reserved_regs, 0, 0);
3694 }
3695 if (IS_DEAD_ARG(0)) {
3696 temp_dead(s, ots);
3697 }
3698}
3699
dd186292 3700static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 3701{
dd186292
RH		 3702 const TCGLifeData arg_life = op->life;
3703 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 3704 TCGRegSet i_allocated_regs;
3705 TCGRegSet o_allocated_regs;
b6638662
RH		 3706 int i, k, nb_iargs, nb_oargs;
3707 TCGReg reg;
c896fe29
FB		 3708 TCGArg arg;
3709 const TCGArgConstraint *arg_ct;
3710 TCGTemp *ts;
3711 TCGArg new_args[TCG_MAX_OP_ARGS];
3712 int const_args[TCG_MAX_OP_ARGS];
3713
3714 nb_oargs = def->nb_oargs;
3715 nb_iargs = def->nb_iargs;
3716
3717 /* copy constants */
a813e36f 3718 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 3719 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 3720 sizeof(TCGArg) * def->nb_cargs);
3721
d21369f5
RH		 3722 i_allocated_regs = s->reserved_regs;
3723 o_allocated_regs = s->reserved_regs;
82790a87 3724
a813e36f 3725 /* satisfy input constraints */
dd186292 3726 for (k = 0; k < nb_iargs; k++) {
29f5e925
RH		 3727 TCGRegSet i_preferred_regs, i_required_regs;
3728 bool allocate_new_reg, copyto_new_reg;
3729 TCGTemp *ts2;
3730 int i1, i2;
d62816f2 3731
66792f90 3732 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 3733 arg = op->args[i];
c896fe29 3734 arg_ct = &def->args_ct[i];
43439139 3735 ts = arg_temp(arg);
40ae5c62
RH		 3736
3737 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 3738 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 3739 /* constant is OK for instruction */
3740 const_args[i] = 1;
3741 new_args[i] = ts->val;
d62816f2 3742 continue;
c896fe29 3743 }
40ae5c62 3744
1c1824dc
RH		 3745 reg = ts->reg;
3746 i_preferred_regs = 0;
29f5e925 3747 i_required_regs = arg_ct->regs;
1c1824dc 3748 allocate_new_reg = false;
29f5e925
RH		 3749 copyto_new_reg = false;
3750
3751 switch (arg_ct->pair) {
3752 case 0: /* not paired */
3753 if (arg_ct->ialias) {
3754 i_preferred_regs = op->output_pref[arg_ct->alias_index];
3755
3756 /*
3757 * If the input is readonly, then it cannot also be an
3758 * output and aliased to itself. If the input is not
3759 * dead after the instruction, we must allocate a new
3760 * register and move it.
3761 */
3762 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3763 allocate_new_reg = true;
3764 } else if (ts->val_type == TEMP_VAL_REG) {
3765 /*
3766 * Check if the current register has already been
3767 * allocated for another input.
3768 */
3769 allocate_new_reg =
3770 tcg_regset_test_reg(i_allocated_regs, reg);
3771 }
3772 }
3773 if (!allocate_new_reg) {
3774 temp_load(s, ts, i_required_regs, i_allocated_regs,
3775 i_preferred_regs);
3776 reg = ts->reg;
3777 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
3778 }
3779 if (allocate_new_reg) {
3780 /*
3781 * Allocate a new register matching the constraint
3782 * and move the temporary register into it.
3783 */
3784 temp_load(s, ts, tcg_target_available_regs[ts->type],
3785 i_allocated_regs, 0);
3786 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
3787 i_preferred_regs, ts->indirect_base);
3788 copyto_new_reg = true;
3789 }
3790 break;
3791
3792 case 1:
3793 /* First of an input pair; if i1 == i2, the second is an output. */
3794 i1 = i;
3795 i2 = arg_ct->pair_index;
3796 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
3797
3798 /*
3799 * It is easier to default to allocating a new pair
3800 * and to identify a few cases where it's not required.
3801 */
3802 if (arg_ct->ialias) {
3803 i_preferred_regs = op->output_pref[arg_ct->alias_index];
3804 if (IS_DEAD_ARG(i1) &&
3805 IS_DEAD_ARG(i2) &&
3806 !temp_readonly(ts) &&
3807 ts->val_type == TEMP_VAL_REG &&
3808 ts->reg < TCG_TARGET_NB_REGS - 1 &&
3809 tcg_regset_test_reg(i_required_regs, reg) &&
3810 !tcg_regset_test_reg(i_allocated_regs, reg) &&
3811 !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
3812 (ts2
3813 ? ts2->val_type == TEMP_VAL_REG &&
3814 ts2->reg == reg + 1 &&
3815 !temp_readonly(ts2)
3816 : s->reg_to_temp[reg + 1] == NULL)) {
3817 break;
3818 }
3819 } else {
3820 /* Without aliasing, the pair must also be an input. */
3821 tcg_debug_assert(ts2);
3822 if (ts->val_type == TEMP_VAL_REG &&
3823 ts2->val_type == TEMP_VAL_REG &&
3824 ts2->reg == reg + 1 &&
3825 tcg_regset_test_reg(i_required_regs, reg)) {
3826 break;
3827 }
3828 }
3829 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
3830 0, ts->indirect_base);
3831 goto do_pair;
3832
3833 case 2: /* pair second */
3834 reg = new_args[arg_ct->pair_index] + 1;
3835 goto do_pair;
1c1824dc 3836
29f5e925
RH		 3837 case 3: /* ialias with second output, no first input */
3838 tcg_debug_assert(arg_ct->ialias);
1c1824dc 3839 i_preferred_regs = op->output_pref[arg_ct->alias_index];
d62816f2 3840
29f5e925
RH		 3841 if (IS_DEAD_ARG(i) &&
3842 !temp_readonly(ts) &&
3843 ts->val_type == TEMP_VAL_REG &&
3844 reg > 0 &&
3845 s->reg_to_temp[reg - 1] == NULL &&
3846 tcg_regset_test_reg(i_required_regs, reg) &&
3847 !tcg_regset_test_reg(i_allocated_regs, reg) &&
3848 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
3849 tcg_regset_set_reg(i_allocated_regs, reg - 1);
3850 break;
3851 }
3852 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
3853 i_allocated_regs, 0,
3854 ts->indirect_base);
3855 tcg_regset_set_reg(i_allocated_regs, reg);
3856 reg += 1;
3857 goto do_pair;
3858
3859 do_pair:
c0522136 3860 /*
29f5e925
RH		 3861 * If an aliased input is not dead after the instruction,
3862 * we must allocate a new register and move it.
c0522136 3863 */
29f5e925
RH		 3864 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
3865 TCGRegSet t_allocated_regs = i_allocated_regs;
3866
1c1824dc 3867 /*
29f5e925
RH		 3868 * Because of the alias, and the continued life, make sure
3869 * that the temp is somewhere *other* than the reg pair,
3870 * and we get a copy in reg.
1c1824dc 3871 */
29f5e925
RH		 3872 tcg_regset_set_reg(t_allocated_regs, reg);
3873 tcg_regset_set_reg(t_allocated_regs, reg + 1);
3874 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
3875 /* If ts was already in reg, copy it somewhere else. */
3876 TCGReg nr;
3877 bool ok;
3878
3879 tcg_debug_assert(ts->kind != TEMP_FIXED);
3880 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
3881 t_allocated_regs, 0, ts->indirect_base);
3882 ok = tcg_out_mov(s, ts->type, nr, reg);
3883 tcg_debug_assert(ok);
3884
3885 set_temp_val_reg(s, ts, nr);
3886 } else {
3887 temp_load(s, ts, tcg_target_available_regs[ts->type],
3888 t_allocated_regs, 0);
3889 copyto_new_reg = true;
3890 }
3891 } else {
3892 /* Preferably allocate to reg, otherwise copy. */
3893 i_required_regs = (TCGRegSet)1 << reg;
3894 temp_load(s, ts, i_required_regs, i_allocated_regs,
3895 i_preferred_regs);
3896 copyto_new_reg = ts->reg != reg;
5ff9d6a4 3897 }
29f5e925 3898 break;
d62816f2 3899
29f5e925
RH		 3900 default:
3901 g_assert_not_reached();
1c1824dc 3902 }
d62816f2 3903
29f5e925 3904 if (copyto_new_reg) {
78113e83 3905 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3906 /*
3907 * Cross register class move not supported. Sync the
3908 * temp back to its slot and load from there.
3909 */
3910 temp_sync(s, ts, i_allocated_regs, 0, 0);
3911 tcg_out_ld(s, ts->type, reg,
3912 ts->mem_base->reg, ts->mem_offset);
78113e83 3913 }
c896fe29 3914 }
c896fe29
FB		 3915 new_args[i] = reg;
3916 const_args[i] = 0;
82790a87 3917 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29 3918 }
a813e36f 3919
a52ad07e
AJ		 3920 /* mark dead temporaries and free the associated registers */
3921 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3922 if (IS_DEAD_ARG(i)) {
43439139 3923 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 3924 }
3925 }
3926
b4cb76e6
RH		 3927 if (def->flags & TCG_OPF_COND_BRANCH) {
3928 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3929 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 3930 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 3931 } else {
e8996ee0 3932 if (def->flags & TCG_OPF_CALL_CLOBBER) {
a813e36f 3933 /* XXX: permit generic clobber register list ? */
c8074023
RH		 3934 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3935 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 3936 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 3937 }
c896fe29 3938 }
3d5c5f87
AJ		 3939 }
3940 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3941 /* sync globals if the op has side effects and might trigger
3942 an exception. */
82790a87 3943 sync_globals(s, i_allocated_regs);
c896fe29 3944 }
a813e36f 3945
e8996ee0 3946 /* satisfy the output constraints */
e8996ee0 3947 for(k = 0; k < nb_oargs; k++) {
66792f90 3948 i = def->args_ct[k].sort_index;
dd186292 3949 arg = op->args[i];
e8996ee0 3950 arg_ct = &def->args_ct[i];
43439139 3951 ts = arg_temp(arg);
d63e3b6e
RH		 3952
3953 /* ENV should not be modified. */
e01fa97d 3954 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3955
29f5e925
RH		 3956 switch (arg_ct->pair) {
3957 case 0: /* not paired */
3958 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
3959 reg = new_args[arg_ct->alias_index];
3960 } else if (arg_ct->newreg) {
3961 reg = tcg_reg_alloc(s, arg_ct->regs,
3962 i_allocated_regs | o_allocated_regs,
3963 op->output_pref[k], ts->indirect_base);
3964 } else {
3965 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
3966 op->output_pref[k], ts->indirect_base);
3967 }
3968 break;
3969
3970 case 1: /* first of pair */
3971 tcg_debug_assert(!arg_ct->newreg);
3972 if (arg_ct->oalias) {
3973 reg = new_args[arg_ct->alias_index];
3974 break;
3975 }
3976 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
3977 op->output_pref[k], ts->indirect_base);
3978 break;
3979
3980 case 2: /* second of pair */
3981 tcg_debug_assert(!arg_ct->newreg);
3982 if (arg_ct->oalias) {
3983 reg = new_args[arg_ct->alias_index];
3984 } else {
3985 reg = new_args[arg_ct->pair_index] + 1;
3986 }
3987 break;
3988
3989 case 3: /* first of pair, aliasing with a second input */
3990 tcg_debug_assert(!arg_ct->newreg);
3991 reg = new_args[arg_ct->pair_index] - 1;
3992 break;
3993
3994 default:
3995 g_assert_not_reached();
c896fe29 3996 }
82790a87 3997 tcg_regset_set_reg(o_allocated_regs, reg);
098859f1 3998 set_temp_val_reg(s, ts, reg);
d63e3b6e 3999 ts->mem_coherent = 0;
e8996ee0 4000 new_args[i] = reg;
c896fe29 4001 }
c896fe29
FB		 4002 }
4003
c896fe29 4004 /* emit instruction */
d2fd745f
RH		 4005 if (def->flags & TCG_OPF_VECTOR) {
4006 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
4007 new_args, const_args);
4008 } else {
4009 tcg_out_op(s, op->opc, new_args, const_args);
4010 }
4011
c896fe29
FB		 4012 /* move the outputs in the correct register if needed */
4013 for(i = 0; i < nb_oargs; i++) {
43439139 4014 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 4015
4016 /* ENV should not be modified. */
e01fa97d 4017 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4018
ec7a869d 4019 if (NEED_SYNC_ARG(i)) {
98b4e186 4020 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 4021 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 4022 temp_dead(s, ts);
ec7a869d 4023 }
c896fe29
FB		 4024 }
4025}
4026
efe86b21
RH		 4027static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
4028{
4029 const TCGLifeData arg_life = op->life;
4030 TCGTemp *ots, *itsl, *itsh;
4031 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4032
4033 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
4034 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
4035 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
4036
4037 ots = arg_temp(op->args[0]);
4038 itsl = arg_temp(op->args[1]);
4039 itsh = arg_temp(op->args[2]);
4040
4041 /* ENV should not be modified. */
4042 tcg_debug_assert(!temp_readonly(ots));
4043
4044 /* Allocate the output register now. */
4045 if (ots->val_type != TEMP_VAL_REG) {
4046 TCGRegSet allocated_regs = s->reserved_regs;
4047 TCGRegSet dup_out_regs =
4048 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
098859f1 4049 TCGReg oreg;
efe86b21
RH		 4050
4051 /* Make sure to not spill the input registers. */
4052 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
4053 tcg_regset_set_reg(allocated_regs, itsl->reg);
4054 }
4055 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
4056 tcg_regset_set_reg(allocated_regs, itsh->reg);
4057 }
4058
098859f1
RH		 4059 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
4060 op->output_pref[0], ots->indirect_base);
4061 set_temp_val_reg(s, ots, oreg);
efe86b21
RH		 4062 }
4063
4064 /* Promote dup2 of immediates to dupi_vec. */
4065 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
4066 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
4067 MemOp vece = MO_64;
4068
4069 if (val == dup_const(MO_8, val)) {
4070 vece = MO_8;
4071 } else if (val == dup_const(MO_16, val)) {
4072 vece = MO_16;
4073 } else if (val == dup_const(MO_32, val)) {
4074 vece = MO_32;
4075 }
4076
4077 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
4078 goto done;
4079 }
4080
4081 /* If the two inputs form one 64-bit value, try dupm_vec. */
4082 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
4083 if (!itsl->mem_coherent) {
4084 temp_sync(s, itsl, s->reserved_regs, 0, 0);
4085 }
4086 if (!itsh->mem_coherent) {
4087 temp_sync(s, itsh, s->reserved_regs, 0, 0);
4088 }
e03b5686 4089#if HOST_BIG_ENDIAN
efe86b21
RH		 4090 TCGTemp *its = itsh;
4091#else
4092 TCGTemp *its = itsl;
4093#endif
4094 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
4095 its->mem_base->reg, its->mem_offset)) {
4096 goto done;
4097 }
4098 }
4099
4100 /* Fall back to generic expansion. */
4101 return false;
4102
4103 done:
36f5539c 4104 ots->mem_coherent = 0;
efe86b21
RH		 4105 if (IS_DEAD_ARG(1)) {
4106 temp_dead(s, itsl);
4107 }
4108 if (IS_DEAD_ARG(2)) {
4109 temp_dead(s, itsh);
4110 }
4111 if (NEED_SYNC_ARG(0)) {
4112 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
4113 } else if (IS_DEAD_ARG(0)) {
4114 temp_dead(s, ots);
4115 }
4116 return true;
4117}
4118
dd186292 4119static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
c896fe29 4120{
cd9090aa
RH		 4121 const int nb_oargs = TCGOP_CALLO(op);
4122 const int nb_iargs = TCGOP_CALLI(op);
dd186292 4123 const TCGLifeData arg_life = op->life;
7b7d8b2d 4124 const TCGHelperInfo *info;
b6638662
RH		 4125 int flags, nb_regs, i;
4126 TCGReg reg;
cf066674 4127 TCGArg arg;
c896fe29 4128 TCGTemp *ts;
d3452f1f
RH		 4129 intptr_t stack_offset;
4130 size_t call_stack_size;
cf066674
RH		 4131 tcg_insn_unit *func_addr;
4132 int allocate_args;
c896fe29 4133 TCGRegSet allocated_regs;
c896fe29 4134
fa52e660 4135 func_addr = tcg_call_func(op);
7b7d8b2d
RH		 4136 info = tcg_call_info(op);
4137 flags = info->flags;
c896fe29 4138
6e17d0c5 4139 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
c45cb8bb
RH		 4140 if (nb_regs > nb_iargs) {
4141 nb_regs = nb_iargs;
cf066674 4142 }
c896fe29
FB		 4143
4144 /* assign stack slots first */
c45cb8bb 4145 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
a813e36f 4146 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
c896fe29 4147 ~(TCG_TARGET_STACK_ALIGN - 1);
b03cce8e
FB		 4148 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
4149 if (allocate_args) {
345649c0
BS		 4150 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
4151 preallocate call stack */
4152 tcg_abort();
b03cce8e 4153 }
39cf05d3
FB		 4154
4155 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
dd186292
RH		 4156 for (i = nb_regs; i < nb_iargs; i++) {
4157 arg = op->args[nb_oargs + i];
39cf05d3 4158 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 4159 ts = arg_temp(arg);
40ae5c62 4160 temp_load(s, ts, tcg_target_available_regs[ts->type],
b722452a 4161 s->reserved_regs, 0);
40ae5c62 4162 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
c896fe29 4163 }
39cf05d3 4164 stack_offset += sizeof(tcg_target_long);
c896fe29 4165 }
a813e36f 4166
c896fe29 4167 /* assign input registers */
d21369f5 4168 allocated_regs = s->reserved_regs;
dd186292
RH		 4169 for (i = 0; i < nb_regs; i++) {
4170 arg = op->args[nb_oargs + i];
39cf05d3 4171 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 4172 ts = arg_temp(arg);
39cf05d3 4173 reg = tcg_target_call_iarg_regs[i];
40ae5c62 4174
39cf05d3
FB		 4175 if (ts->val_type == TEMP_VAL_REG) {
4176 if (ts->reg != reg) {
4250da10 4177 tcg_reg_free(s, reg, allocated_regs);
78113e83 4178 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 4179 /*
4180 * Cross register class move not supported. Sync the
4181 * temp back to its slot and load from there.
4182 */
4183 temp_sync(s, ts, allocated_regs, 0, 0);
4184 tcg_out_ld(s, ts->type, reg,
4185 ts->mem_base->reg, ts->mem_offset);
78113e83 4186 }
39cf05d3 4187 }
39cf05d3 4188 } else {
ccb1bb66 4189 TCGRegSet arg_set = 0;
40ae5c62 4190
4250da10 4191 tcg_reg_free(s, reg, allocated_regs);
40ae5c62 4192 tcg_regset_set_reg(arg_set, reg);
b722452a 4193 temp_load(s, ts, arg_set, allocated_regs, 0);
c896fe29 4194 }
40ae5c62 4195
39cf05d3 4196 tcg_regset_set_reg(allocated_regs, reg);
c896fe29 4197 }
c896fe29 4198 }
a813e36f 4199
c896fe29 4200 /* mark dead temporaries and free the associated registers */
dd186292 4201 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 4202 if (IS_DEAD_ARG(i)) {
43439139 4203 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 4204 }
4205 }
a813e36f 4206
c896fe29 4207 /* clobber call registers */
c8074023
RH		 4208 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4209 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 4210 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 4211 }
4212 }
78505279
AJ		 4213
4214 /* Save globals if they might be written by the helper, sync them if
4215 they might be read. */
4216 if (flags & TCG_CALL_NO_READ_GLOBALS) {
4217 /* Nothing to do */
4218 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
4219 sync_globals(s, allocated_regs);
4220 } else {
b9c18f56
AJ		 4221 save_globals(s, allocated_regs);
4222 }
c896fe29 4223
7b7d8b2d
RH		 4224#ifdef CONFIG_TCG_INTERPRETER
4225 {
4226 gpointer hash = (gpointer)(uintptr_t)info->typemask;
4227 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash);
4228 assert(cif != NULL);
4229 tcg_out_call(s, func_addr, cif);
4230 }
4231#else
cf066674 4232 tcg_out_call(s, func_addr);
7b7d8b2d 4233#endif
c896fe29
FB		 4234
4235 /* assign output registers and emit moves if needed */
4236 for(i = 0; i < nb_oargs; i++) {
dd186292 4237 arg = op->args[i];
43439139 4238 ts = arg_temp(arg);
d63e3b6e
RH		 4239
4240 /* ENV should not be modified. */
e01fa97d 4241 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4242
c896fe29 4243 reg = tcg_target_call_oarg_regs[i];
098859f1 4244 set_temp_val_reg(s, ts, reg);
d63e3b6e 4245 ts->mem_coherent = 0;
d63e3b6e
RH		 4246 if (NEED_SYNC_ARG(i)) {
4247 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
4248 } else if (IS_DEAD_ARG(i)) {
4249 temp_dead(s, ts);
c896fe29
FB		 4250 }
4251 }
c896fe29
FB		 4252}
4253
4254#ifdef CONFIG_PROFILER
4255
c3fac113
EC		 4256/* avoid copy/paste errors */
4257#define PROF_ADD(to, from, field) \
4258 do { \
d73415a3 4259 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 4260 } while (0)
4261
4262#define PROF_MAX(to, from, field) \
4263 do { \
d73415a3 4264 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 4265 if (val__ > (to)->field) { \
4266 (to)->field = val__; \
4267 } \
4268 } while (0)
4269
4270/* Pass in a zero'ed @prof */
4271static inline
4272void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4273{
0e2d61cf 4274 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 4275 unsigned int i;
4276
3468b59e 4277 for (i = 0; i < n_ctxs; i++) {
d73415a3 4278 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 4279 const TCGProfile *orig = &s->prof;
c3fac113
EC		 4280
4281 if (counters) {
72fd2efb 4282 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 4283 PROF_ADD(prof, orig, tb_count1);
4284 PROF_ADD(prof, orig, tb_count);
4285 PROF_ADD(prof, orig, op_count);
4286 PROF_MAX(prof, orig, op_count_max);
4287 PROF_ADD(prof, orig, temp_count);
4288 PROF_MAX(prof, orig, temp_count_max);
4289 PROF_ADD(prof, orig, del_op_count);
4290 PROF_ADD(prof, orig, code_in_len);
4291 PROF_ADD(prof, orig, code_out_len);
4292 PROF_ADD(prof, orig, search_out_len);
4293 PROF_ADD(prof, orig, interm_time);
4294 PROF_ADD(prof, orig, code_time);
4295 PROF_ADD(prof, orig, la_time);
4296 PROF_ADD(prof, orig, opt_time);
4297 PROF_ADD(prof, orig, restore_count);
4298 PROF_ADD(prof, orig, restore_time);
4299 }
4300 if (table) {
4301 int i;
4302
4303 for (i = 0; i < NB_OPS; i++) {
4304 PROF_ADD(prof, orig, table_op_count[i]);
4305 }
4306 }
4307 }
4308}
4309
4310#undef PROF_ADD
4311#undef PROF_MAX
4312
4313static void tcg_profile_snapshot_counters(TCGProfile *prof)
4314{
4315 tcg_profile_snapshot(prof, true, false);
4316}
4317
4318static void tcg_profile_snapshot_table(TCGProfile *prof)
4319{
4320 tcg_profile_snapshot(prof, false, true);
4321}
c896fe29 4322
b6a7f3e0 4323void tcg_dump_op_count(GString *buf)
c896fe29 4324{
c3fac113 4325 TCGProfile prof = {};
c896fe29 4326 int i;
d70724ce 4327
c3fac113 4328 tcg_profile_snapshot_table(&prof);
15fc7daa 4329 for (i = 0; i < NB_OPS; i++) {
b6a7f3e0
DB		 4330 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
4331 prof.table_op_count[i]);
c896fe29 4332 }
c896fe29 4333}
72fd2efb
EC		 4334
4335int64_t tcg_cpu_exec_time(void)
4336{
0e2d61cf 4337 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4338 unsigned int i;
4339 int64_t ret = 0;
4340
4341 for (i = 0; i < n_ctxs; i++) {
d73415a3 4342 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4343 const TCGProfile *prof = &s->prof;
4344
d73415a3 4345 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4346 }
4347 return ret;
4348}
246ae24d 4349#else
b6a7f3e0 4350void tcg_dump_op_count(GString *buf)
246ae24d 4351{
b6a7f3e0 4352 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
246ae24d 4353}
72fd2efb
EC		 4354
4355int64_t tcg_cpu_exec_time(void)
4356{
4357 error_report("%s: TCG profiler not compiled", __func__);
4358 exit(EXIT_FAILURE);
4359}
c896fe29
FB		 4360#endif
4361
4362
fbf59aad 4363int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
c896fe29 4364{
c3fac113
EC		 4365#ifdef CONFIG_PROFILER
4366 TCGProfile *prof = &s->prof;
4367#endif
15fa08f8
RH		 4368 int i, num_insns;
4369 TCGOp *op;
c896fe29 4370
04fe6400
RH		 4371#ifdef CONFIG_PROFILER
4372 {
c1f543b7 4373 int n = 0;
04fe6400 4374
15fa08f8
RH		 4375 QTAILQ_FOREACH(op, &s->ops, link) {
4376 n++;
4377 }
d73415a3 4378 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4379 if (n > prof->op_count_max) {
d73415a3 4380 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4381 }
4382
4383 n = s->nb_temps;
d73415a3 4384 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4385 if (n > prof->temp_count_max) {
d73415a3 4386 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4387 }
4388 }
4389#endif
4390
c896fe29 4391#ifdef DEBUG_DISAS
d977e1c2 4392 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
fbf59aad 4393 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4394 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4395 if (logfile) {
4396 fprintf(logfile, "OP:\n");
b7a83ff8 4397 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4398 fprintf(logfile, "\n");
4399 qemu_log_unlock(logfile);
4400 }
c896fe29
FB		 4401 }
4402#endif
4403
bef16ab4
RH		 4404#ifdef CONFIG_DEBUG_TCG
4405 /* Ensure all labels referenced have been emitted. */
4406 {
4407 TCGLabel *l;
4408 bool error = false;
4409
4410 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4411 if (unlikely(!l->present) && l->refs) {
4412 qemu_log_mask(CPU_LOG_TB_OP,
4413 "$L%d referenced but not present.\n", l->id);
4414 error = true;
4415 }
4416 }
4417 assert(!error);
4418 }
4419#endif
4420
c5cc28ff 4421#ifdef CONFIG_PROFILER
d73415a3 4422 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4423#endif
4424
8f2e8c07 4425#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4426 tcg_optimize(s);
8f2e8c07
KB		 4427#endif
4428
a23a9ec6 4429#ifdef CONFIG_PROFILER
d73415a3
SH		 4430 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4431 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4432#endif
c5cc28ff 4433
b4fc67c7 4434 reachable_code_pass(s);
b83eabea 4435 liveness_pass_1(s);
5a18407f 4436
b83eabea 4437 if (s->nb_indirects > 0) {
5a18407f 4438#ifdef DEBUG_DISAS
b83eabea 4439 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
fbf59aad 4440 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4441 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4442 if (logfile) {
4443 fprintf(logfile, "OP before indirect lowering:\n");
b7a83ff8 4444 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4445 fprintf(logfile, "\n");
4446 qemu_log_unlock(logfile);
4447 }
b83eabea 4448 }
5a18407f 4449#endif
b83eabea
RH		 4450 /* Replace indirect temps with direct temps. */
4451 if (liveness_pass_2(s)) {
4452 /* If changes were made, re-run liveness. */
4453 liveness_pass_1(s);
5a18407f
RH		 4454 }
4455 }
c5cc28ff 4456
a23a9ec6 4457#ifdef CONFIG_PROFILER
d73415a3 4458 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4459#endif
c896fe29
FB		 4460
4461#ifdef DEBUG_DISAS
d977e1c2 4462 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
fbf59aad 4463 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4464 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4465 if (logfile) {
4466 fprintf(logfile, "OP after optimization and liveness analysis:\n");
b7a83ff8 4467 tcg_dump_ops(s, logfile, true);
78b54858
RH		 4468 fprintf(logfile, "\n");
4469 qemu_log_unlock(logfile);
4470 }
c896fe29
FB		 4471 }
4472#endif
4473
35abb009
RH		 4474 /* Initialize goto_tb jump offsets. */
4475 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
4476 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
4477 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
4478 if (TCG_TARGET_HAS_direct_jump) {
4479 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
4480 tcg_ctx->tb_jmp_target_addr = NULL;
4481 } else {
4482 tcg_ctx->tb_jmp_insn_offset = NULL;
4483 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
4484 }
4485
c896fe29
FB		 4486 tcg_reg_alloc_start(s);
4487
db0c51a3
RH		 4488 /*
4489 * Reset the buffer pointers when restarting after overflow.
4490 * TODO: Move this into translate-all.c with the rest of the
4491 * buffer management. Having only this done here is confusing.
4492 */
4493 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4494 s->code_ptr = s->code_buf;
c896fe29 4495
659ef5cb 4496#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4497 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4498#endif
57a26946
RH		 4499#ifdef TCG_TARGET_NEED_POOL_LABELS
4500 s->pool_labels = NULL;
4501#endif
9ecefc84 4502
fca8a500 4503 num_insns = -1;
15fa08f8 4504 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4505 TCGOpcode opc = op->opc;
b3db8758 4506
c896fe29 4507#ifdef CONFIG_PROFILER
d73415a3 4508 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 4509#endif
c45cb8bb
RH		 4510
4511 switch (opc) {
c896fe29 4512 case INDEX_op_mov_i32:
c896fe29 4513 case INDEX_op_mov_i64:
d2fd745f 4514 case INDEX_op_mov_vec:
dd186292 4515 tcg_reg_alloc_mov(s, op);
c896fe29 4516 break;
bab1671f
RH		 4517 case INDEX_op_dup_vec:
4518 tcg_reg_alloc_dup(s, op);
4519 break;
765b842a 4520 case INDEX_op_insn_start:
fca8a500 4521 if (num_insns >= 0) {
9f754620
RH		 4522 size_t off = tcg_current_code_size(s);
4523 s->gen_insn_end_off[num_insns] = off;
4524 /* Assert that we do not overflow our stored offset. */
4525 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 4526 }
4527 num_insns++;
bad729e2
RH		 4528 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4529 target_ulong a;
4530#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 4531 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 4532#else
efee3746 4533 a = op->args[i];
bad729e2 4534#endif
fca8a500 4535 s->gen_insn_data[num_insns][i] = a;
bad729e2 4536 }
c896fe29 4537 break;
5ff9d6a4 4538 case INDEX_op_discard:
43439139 4539 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 4540 break;
c896fe29 4541 case INDEX_op_set_label:
e8996ee0 4542 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 4543 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 4544 break;
4545 case INDEX_op_call:
dd186292 4546 tcg_reg_alloc_call(s, op);
c45cb8bb 4547 break;
efe86b21
RH		 4548 case INDEX_op_dup2_vec:
4549 if (tcg_reg_alloc_dup2(s, op)) {
4550 break;
4551 }
4552 /* fall through */
c896fe29 4553 default:
25c4d9cc 4554 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 4555 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 4556 /* Note: in order to speed up the code, it would be much
4557 faster to have specialized register allocator functions for
4558 some common argument patterns */
dd186292 4559 tcg_reg_alloc_op(s, op);
c896fe29
FB		 4560 break;
4561 }
b125f9dc
RH		 4562 /* Test for (pending) buffer overflow. The assumption is that any
4563 one operation beginning below the high water mark cannot overrun
4564 the buffer completely. Thus we can test for overflow after
4565 generating code without having to check during generation. */
644da9b3 4566 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 4567 return -1;
4568 }
6e6c4efe
RH		 4569 /* Test for TB overflow, as seen by gen_insn_end_off. */
4570 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4571 return -2;
4572 }
c896fe29 4573 }
fca8a500
RH		 4574 tcg_debug_assert(num_insns >= 0);
4575 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 4576
b76f0d8c 4577 /* Generate TB finalization at the end of block */
659ef5cb 4578#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 4579 i = tcg_out_ldst_finalize(s);
4580 if (i < 0) {
4581 return i;
23dceda6 4582 }
659ef5cb 4583#endif
57a26946 4584#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 4585 i = tcg_out_pool_finalize(s);
4586 if (i < 0) {
4587 return i;
57a26946
RH		 4588 }
4589#endif
7ecd02a0
RH		 4590 if (!tcg_resolve_relocs(s)) {
4591 return -2;
4592 }
c896fe29 4593
df5d2b16 4594#ifndef CONFIG_TCG_INTERPRETER
c896fe29 4595 /* flush instruction cache */
db0c51a3
RH		 4596 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4597 (uintptr_t)s->code_buf,
1da8de39 4598 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 4599#endif
2aeabc08 4600
1813e175 4601 return tcg_current_code_size(s);
c896fe29
FB		 4602}
4603
a23a9ec6 4604#ifdef CONFIG_PROFILER
3a841ab5 4605void tcg_dump_info(GString *buf)
a23a9ec6 4606{
c3fac113
EC		 4607 TCGProfile prof = {};
4608 const TCGProfile *s;
4609 int64_t tb_count;
4610 int64_t tb_div_count;
4611 int64_t tot;
4612
4613 tcg_profile_snapshot_counters(&prof);
4614 s = &prof;
4615 tb_count = s->tb_count;
4616 tb_div_count = tb_count ? tb_count : 1;
4617 tot = s->interm_time + s->code_time;
a23a9ec6 4618
3a841ab5
DB		 4619 g_string_append_printf(buf, "JIT cycles %" PRId64
4620 " (%0.3f s at 2.4 GHz)\n",
4621 tot, tot / 2.4e9);
4622 g_string_append_printf(buf, "translated TBs %" PRId64
4623 " (aborted=%" PRId64 " %0.1f%%)\n",
4624 tb_count, s->tb_count1 - tb_count,
4625 (double)(s->tb_count1 - s->tb_count)
4626 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4627 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
4628 (double)s->op_count / tb_div_count, s->op_count_max);
4629 g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
4630 (double)s->del_op_count / tb_div_count);
4631 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
4632 (double)s->temp_count / tb_div_count,
4633 s->temp_count_max);
4634 g_string_append_printf(buf, "avg host code/TB %0.1f\n",
4635 (double)s->code_out_len / tb_div_count);
4636 g_string_append_printf(buf, "avg search data/TB %0.1f\n",
4637 (double)s->search_out_len / tb_div_count);
a813e36f 4638
3a841ab5
DB		 4639 g_string_append_printf(buf, "cycles/op %0.1f\n",
4640 s->op_count ? (double)tot / s->op_count : 0);
4641 g_string_append_printf(buf, "cycles/in byte %0.1f\n",
4642 s->code_in_len ? (double)tot / s->code_in_len : 0);
4643 g_string_append_printf(buf, "cycles/out byte %0.1f\n",
4644 s->code_out_len ? (double)tot / s->code_out_len : 0);
4645 g_string_append_printf(buf, "cycles/search byte %0.1f\n",
4646 s->search_out_len ?
4647 (double)tot / s->search_out_len : 0);
fca8a500 4648 if (tot == 0) {
a23a9ec6 4649 tot = 1;
fca8a500 4650 }
3a841ab5
DB		 4651 g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
4652 (double)s->interm_time / tot * 100.0);
4653 g_string_append_printf(buf, " gen_code time %0.1f%%\n",
4654 (double)s->code_time / tot * 100.0);
4655 g_string_append_printf(buf, "optim./code time %0.1f%%\n",
4656 (double)s->opt_time / (s->code_time ?
4657 s->code_time : 1)
4658 * 100.0);
4659 g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
4660 (double)s->la_time / (s->code_time ?
4661 s->code_time : 1) * 100.0);
4662 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
4663 s->restore_count);
4664 g_string_append_printf(buf, " avg cycles %0.1f\n",
4665 s->restore_count ?
4666 (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 4667}
4668#else
3a841ab5 4669void tcg_dump_info(GString *buf)
a23a9ec6 4670{
3a841ab5 4671 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
a23a9ec6
FB		 4672}
4673#endif
813da627
RH		 4674
4675#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 4676/* In order to use this feature, the backend needs to do three things:
4677
4678 (1) Define ELF_HOST_MACHINE to indicate both what value to
4679 put into the ELF image and to indicate support for the feature.
4680
4681 (2) Define tcg_register_jit. This should create a buffer containing
4682 the contents of a .debug_frame section that describes the post-
4683 prologue unwind info for the tcg machine.
4684
4685 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4686*/
813da627
RH		 4687
4688/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4689typedef enum {
4690 JIT_NOACTION = 0,
4691 JIT_REGISTER_FN,
4692 JIT_UNREGISTER_FN
4693} jit_actions_t;
4694
4695struct jit_code_entry {
4696 struct jit_code_entry *next_entry;
4697 struct jit_code_entry *prev_entry;
4698 const void *symfile_addr;
4699 uint64_t symfile_size;
4700};
4701
4702struct jit_descriptor {
4703 uint32_t version;
4704 uint32_t action_flag;
4705 struct jit_code_entry *relevant_entry;
4706 struct jit_code_entry *first_entry;
4707};
4708
4709void __jit_debug_register_code(void) __attribute__((noinline));
4710void __jit_debug_register_code(void)
4711{
4712 asm("");
4713}
4714
4715/* Must statically initialize the version, because GDB may check
4716 the version before we can set it. */
4717struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4718
4719/* End GDB interface. */
4720
4721static int find_string(const char *strtab, const char *str)
4722{
4723 const char *p = strtab + 1;
4724
4725 while (1) {
4726 if (strcmp(p, str) == 0) {
4727 return p - strtab;
4728 }
4729 p += strlen(p) + 1;
4730 }
4731}
4732
755bf9e5 4733static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 4734 const void *debug_frame,
4735 size_t debug_frame_size)
813da627 4736{
5872bbf2
RH		 4737 struct __attribute__((packed)) DebugInfo {
4738 uint32_t len;
4739 uint16_t version;
4740 uint32_t abbrev;
4741 uint8_t ptr_size;
4742 uint8_t cu_die;
4743 uint16_t cu_lang;
4744 uintptr_t cu_low_pc;
4745 uintptr_t cu_high_pc;
4746 uint8_t fn_die;
4747 char fn_name[16];
4748 uintptr_t fn_low_pc;
4749 uintptr_t fn_high_pc;
4750 uint8_t cu_eoc;
4751 };
813da627
RH		 4752
4753 struct ElfImage {
4754 ElfW(Ehdr) ehdr;
4755 ElfW(Phdr) phdr;
5872bbf2
RH		 4756 ElfW(Shdr) shdr[7];
4757 ElfW(Sym) sym[2];
4758 struct DebugInfo di;
4759 uint8_t da[24];
4760 char str[80];
4761 };
4762
4763 struct ElfImage *img;
4764
4765 static const struct ElfImage img_template = {
4766 .ehdr = {
4767 .e_ident[EI_MAG0] = ELFMAG0,
4768 .e_ident[EI_MAG1] = ELFMAG1,
4769 .e_ident[EI_MAG2] = ELFMAG2,
4770 .e_ident[EI_MAG3] = ELFMAG3,
4771 .e_ident[EI_CLASS] = ELF_CLASS,
4772 .e_ident[EI_DATA] = ELF_DATA,
4773 .e_ident[EI_VERSION] = EV_CURRENT,
4774 .e_type = ET_EXEC,
4775 .e_machine = ELF_HOST_MACHINE,
4776 .e_version = EV_CURRENT,
4777 .e_phoff = offsetof(struct ElfImage, phdr),
4778 .e_shoff = offsetof(struct ElfImage, shdr),
4779 .e_ehsize = sizeof(ElfW(Shdr)),
4780 .e_phentsize = sizeof(ElfW(Phdr)),
4781 .e_phnum = 1,
4782 .e_shentsize = sizeof(ElfW(Shdr)),
4783 .e_shnum = ARRAY_SIZE(img->shdr),
4784 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 4785#ifdef ELF_HOST_FLAGS
4786 .e_flags = ELF_HOST_FLAGS,
4787#endif
4788#ifdef ELF_OSABI
4789 .e_ident[EI_OSABI] = ELF_OSABI,
4790#endif
5872bbf2
RH		 4791 },
4792 .phdr = {
4793 .p_type = PT_LOAD,
4794 .p_flags = PF_X,
4795 },
4796 .shdr = {
4797 [0] = { .sh_type = SHT_NULL },
4798 /* Trick: The contents of code_gen_buffer are not present in
4799 this fake ELF file; that got allocated elsewhere. Therefore
4800 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4801 will not look for contents. We can record any address. */
4802 [1] = { /* .text */
4803 .sh_type = SHT_NOBITS,
4804 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4805 },
4806 [2] = { /* .debug_info */
4807 .sh_type = SHT_PROGBITS,
4808 .sh_offset = offsetof(struct ElfImage, di),
4809 .sh_size = sizeof(struct DebugInfo),
4810 },
4811 [3] = { /* .debug_abbrev */
4812 .sh_type = SHT_PROGBITS,
4813 .sh_offset = offsetof(struct ElfImage, da),
4814 .sh_size = sizeof(img->da),
4815 },
4816 [4] = { /* .debug_frame */
4817 .sh_type = SHT_PROGBITS,
4818 .sh_offset = sizeof(struct ElfImage),
4819 },
4820 [5] = { /* .symtab */
4821 .sh_type = SHT_SYMTAB,
4822 .sh_offset = offsetof(struct ElfImage, sym),
4823 .sh_size = sizeof(img->sym),
4824 .sh_info = 1,
4825 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4826 .sh_entsize = sizeof(ElfW(Sym)),
4827 },
4828 [6] = { /* .strtab */
4829 .sh_type = SHT_STRTAB,
4830 .sh_offset = offsetof(struct ElfImage, str),
4831 .sh_size = sizeof(img->str),
4832 }
4833 },
4834 .sym = {
4835 [1] = { /* code_gen_buffer */
4836 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4837 .st_shndx = 1,
4838 }
4839 },
4840 .di = {
4841 .len = sizeof(struct DebugInfo) - 4,
4842 .version = 2,
4843 .ptr_size = sizeof(void *),
4844 .cu_die = 1,
4845 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4846 .fn_die = 2,
4847 .fn_name = "code_gen_buffer"
4848 },
4849 .da = {
4850 1, /* abbrev number (the cu) */
4851 0x11, 1, /* DW_TAG_compile_unit, has children */
4852 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4853 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4854 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4855 0, 0, /* end of abbrev */
4856 2, /* abbrev number (the fn) */
4857 0x2e, 0, /* DW_TAG_subprogram, no children */
4858 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4859 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4860 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4861 0, 0, /* end of abbrev */
4862 0 /* no more abbrev */
4863 },
4864 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4865 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 4866 };
4867
4868 /* We only need a single jit entry; statically allocate it. */
4869 static struct jit_code_entry one_entry;
4870
5872bbf2 4871 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 4872 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 4873 DebugFrameHeader *dfh;
813da627 4874
5872bbf2
RH		 4875 img = g_malloc(img_size);
4876 *img = img_template;
813da627 4877
5872bbf2
RH		 4878 img->phdr.p_vaddr = buf;
4879 img->phdr.p_paddr = buf;
4880 img->phdr.p_memsz = buf_size;
813da627 4881
813da627 4882 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 4883 img->shdr[1].sh_addr = buf;
813da627
RH		 4884 img->shdr[1].sh_size = buf_size;
4885
5872bbf2
RH		 4886 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4887 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4888
4889 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4890 img->shdr[4].sh_size = debug_frame_size;
4891
4892 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4893 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4894
4895 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4896 img->sym[1].st_value = buf;
4897 img->sym[1].st_size = buf_size;
813da627 4898
5872bbf2 4899 img->di.cu_low_pc = buf;
45aba097 4900 img->di.cu_high_pc = buf + buf_size;
5872bbf2 4901 img->di.fn_low_pc = buf;
45aba097 4902 img->di.fn_high_pc = buf + buf_size;
813da627 4903
2c90784a
RH		 4904 dfh = (DebugFrameHeader *)(img + 1);
4905 memcpy(dfh, debug_frame, debug_frame_size);
4906 dfh->fde.func_start = buf;
4907 dfh->fde.func_len = buf_size;
4908
813da627
RH		 4909#ifdef DEBUG_JIT
4910 /* Enable this block to be able to debug the ELF image file creation.
4911 One can use readelf, objdump, or other inspection utilities. */
4912 {
eb6b2edf
BM		 4913 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
4914 FILE *f = fopen(jit, "w+b");
813da627 4915 if (f) {
5872bbf2 4916 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 4917 /* Avoid stupid unused return value warning for fwrite. */
4918 }
4919 fclose(f);
4920 }
4921 }
4922#endif
4923
4924 one_entry.symfile_addr = img;
4925 one_entry.symfile_size = img_size;
4926
4927 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4928 __jit_debug_descriptor.relevant_entry = &one_entry;
4929 __jit_debug_descriptor.first_entry = &one_entry;
4930 __jit_debug_register_code();
4931}
4932#else
5872bbf2
RH		 4933/* No support for the feature. Provide the entry point expected by exec.c,
4934 and implement the internal function we declared earlier. */
813da627 4935
755bf9e5 4936static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 4937 const void *debug_frame,
4938 size_t debug_frame_size)
813da627
RH		 4939{
4940}
4941
755bf9e5 4942void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 4943{
4944}
4945#endif /* ELF_HOST_MACHINE */
db432672
RH		 4946
4947#if !TCG_TARGET_MAYBE_vec
4948void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4949{
4950 g_assert_not_reached();
4951}
4952#endif
QEMU mirror