]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg: Remove check_regs
[mirror_qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
1de7afc9 37#include "qemu/timer.h"
084cfca1 38#include "qemu/cacheflush.h"
ad768e6f 39#include "qemu/cacheinfo.h"
c896fe29 40
c5d3c498 41/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 42 CPU definitions. Currently they are used for qemu_ld/st
43 instructions */
44#define NO_CPU_IO_DEFS
c896fe29 45
63c91552 46#include "exec/exec-all.h"
dcb32f1d 47#include "tcg/tcg-op.h"
813da627 48
edee2579 49#if UINTPTR_MAX == UINT32_MAX
813da627 50# define ELF_CLASS ELFCLASS32
edee2579
RH		 51#else
52# define ELF_CLASS ELFCLASS64
813da627 53#endif
e03b5686 54#if HOST_BIG_ENDIAN
813da627
RH		 55# define ELF_DATA ELFDATA2MSB
56#else
57# define ELF_DATA ELFDATA2LSB
58#endif
59
c896fe29 60#include "elf.h"
508127e2 61#include "exec/log.h"
d2ba8026 62#include "tcg/tcg-ldst.h"
5ff7258c 63#include "tcg-internal.h"
c896fe29 64
22f15579
RH		 65#ifdef CONFIG_TCG_INTERPRETER
66#include <ffi.h>
67#endif
68
139c1837 69/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 70 used here. */
e4d58b41
RH		 71static void tcg_target_init(TCGContext *s);
72static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 73static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 74 intptr_t value, intptr_t addend);
c896fe29 75
497a22eb
RH		 76/* The CIE and FDE header definitions will be common to all hosts. */
77typedef struct {
78 uint32_t len __attribute__((aligned((sizeof(void *)))));
79 uint32_t id;
80 uint8_t version;
81 char augmentation[1];
82 uint8_t code_align;
83 uint8_t data_align;
84 uint8_t return_column;
85} DebugFrameCIE;
86
87typedef struct QEMU_PACKED {
88 uint32_t len __attribute__((aligned((sizeof(void *)))));
89 uint32_t cie_offset;
edee2579
RH		 90 uintptr_t func_start;
91 uintptr_t func_len;
497a22eb
RH		 92} DebugFrameFDEHeader;
93
2c90784a
RH		 94typedef struct QEMU_PACKED {
95 DebugFrameCIE cie;
96 DebugFrameFDEHeader fde;
97} DebugFrameHeader;
98
755bf9e5 99static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 100 const void *debug_frame,
101 size_t debug_frame_size)
813da627
RH		 102 __attribute__((unused));
103
139c1837 104/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 105static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 106 intptr_t arg2);
78113e83 107static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 108static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 109 TCGReg ret, tcg_target_long arg);
5e8892db
MR		 110static void tcg_out_op(TCGContext *s, TCGOpcode opc,
111 const TCGArg args[TCG_MAX_OP_ARGS],
112 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 113#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 114static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 116static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
117 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 118static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
119 TCGReg dst, int64_t arg);
5e8892db
MR		 120static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
121 unsigned vecl, unsigned vece,
122 const TCGArg args[TCG_MAX_OP_ARGS],
123 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 124#else
e7632cfa
RH		 125static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
126 TCGReg dst, TCGReg src)
127{
128 g_assert_not_reached();
129}
d6ecb4a9
RH		 130static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
131 TCGReg dst, TCGReg base, intptr_t offset)
132{
133 g_assert_not_reached();
134}
4e186175
RH		 135static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
136 TCGReg dst, int64_t arg)
e7632cfa
RH		 137{
138 g_assert_not_reached();
139}
5e8892db
MR		 140static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
141 unsigned vecl, unsigned vece,
142 const TCGArg args[TCG_MAX_OP_ARGS],
143 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 144{
145 g_assert_not_reached();
146}
147#endif
2a534aff 148static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 149 intptr_t arg2);
59d7c14e
RH		 150static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
151 TCGReg base, intptr_t ofs);
7b7d8b2d
RH		 152#ifdef CONFIG_TCG_INTERPRETER
153static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
154 ffi_cif *cif);
155#else
2be7d76b 156static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
7b7d8b2d 157#endif
a4fbbd77 158static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 159#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 160static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 161#endif
c896fe29 162
42eb6dfc
RH		 163TCGContext tcg_init_ctx;
164__thread TCGContext *tcg_ctx;
165
5ff7258c 166TCGContext **tcg_ctxs;
0e2d61cf
RH		 167unsigned int tcg_cur_ctxs;
168unsigned int tcg_max_ctxs;
1c2adb95 169TCGv_env cpu_env = 0;
c8bc1168 170const void *tcg_code_gen_epilogue;
db0c51a3 171uintptr_t tcg_splitwx_diff;
df2cce29 172
b91ccb31
RH		 173#ifndef CONFIG_TCG_INTERPRETER
174tcg_prologue_fn *tcg_qemu_tb_exec;
175#endif
176
d2fd745f 177static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 178static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 179
1813e175 180#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 181static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 182{
183 *s->code_ptr++ = v;
184}
185
4196dca6
PM		 186static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
187 uint8_t v)
5c53bb81 188{
1813e175 189 *p = v;
5c53bb81 190}
1813e175 191#endif
5c53bb81 192
1813e175 193#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 194static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 195{
1813e175
RH		 196 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
197 *s->code_ptr++ = v;
198 } else {
199 tcg_insn_unit *p = s->code_ptr;
200 memcpy(p, &v, sizeof(v));
201 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
202 }
c896fe29
FB		 203}
204
4196dca6
PM		 205static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
206 uint16_t v)
5c53bb81 207{
1813e175
RH		 208 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
209 *p = v;
210 } else {
211 memcpy(p, &v, sizeof(v));
212 }
5c53bb81 213}
1813e175 214#endif
5c53bb81 215
1813e175 216#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 217static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 218{
1813e175
RH		 219 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
220 *s->code_ptr++ = v;
221 } else {
222 tcg_insn_unit *p = s->code_ptr;
223 memcpy(p, &v, sizeof(v));
224 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
225 }
c896fe29
FB		 226}
227
4196dca6
PM		 228static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
229 uint32_t v)
5c53bb81 230{
1813e175
RH		 231 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
232 *p = v;
233 } else {
234 memcpy(p, &v, sizeof(v));
235 }
5c53bb81 236}
1813e175 237#endif
5c53bb81 238
1813e175 239#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 240static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 241{
1813e175
RH		 242 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
243 *s->code_ptr++ = v;
244 } else {
245 tcg_insn_unit *p = s->code_ptr;
246 memcpy(p, &v, sizeof(v));
247 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
248 }
ac26eb69
RH		 249}
250
4196dca6
PM		 251static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
252 uint64_t v)
5c53bb81 253{
1813e175
RH		 254 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
255 *p = v;
256 } else {
257 memcpy(p, &v, sizeof(v));
258 }
5c53bb81 259}
1813e175 260#endif
5c53bb81 261
c896fe29
FB		 262/* label relocation processing */
263
1813e175 264static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 265 TCGLabel *l, intptr_t addend)
c896fe29 266{
7ecd02a0 267 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 268
7ecd02a0
RH		 269 r->type = type;
270 r->ptr = code_ptr;
271 r->addend = addend;
272 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 273}
274
92ab8e7d 275static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 276{
eabb7b91 277 tcg_debug_assert(!l->has_value);
c896fe29 278 l->has_value = 1;
92ab8e7d 279 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 280}
281
42a268c2 282TCGLabel *gen_new_label(void)
c896fe29 283{
b1311c4a 284 TCGContext *s = tcg_ctx;
51e3972c 285 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 286
7ecd02a0
RH		 287 memset(l, 0, sizeof(TCGLabel));
288 l->id = s->nb_labels++;
289 QSIMPLEQ_INIT(&l->relocs);
290
bef16ab4 291 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 292
293 return l;
c896fe29
FB		 294}
295
7ecd02a0
RH		 296static bool tcg_resolve_relocs(TCGContext *s)
297{
298 TCGLabel *l;
299
300 QSIMPLEQ_FOREACH(l, &s->labels, next) {
301 TCGRelocation *r;
302 uintptr_t value = l->u.value;
303
304 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
305 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
306 return false;
307 }
308 }
309 }
310 return true;
311}
312
9f754620
RH		 313static void set_jmp_reset_offset(TCGContext *s, int which)
314{
f14bed3f
RH		 315 /*
316 * We will check for overflow at the end of the opcode loop in
317 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
318 */
319 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 320}
321
db6b7d0c 322/* Signal overflow, starting over with fewer guest insns. */
8905770b
MAL		 323static G_NORETURN
324void tcg_raise_tb_overflow(TCGContext *s)
db6b7d0c
RH		 325{
326 siglongjmp(s->jmp_trans, -2);
327}
328
4c22e840
RH		 329#define C_PFX1(P, A) P##A
330#define C_PFX2(P, A, B) P##A##_##B
331#define C_PFX3(P, A, B, C) P##A##_##B##_##C
332#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
333#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
334#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
335
336/* Define an enumeration for the various combinations. */
337
338#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
339#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
340#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
341#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
342
343#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
344#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
345#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
346#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
347
348#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
349
350#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
351#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
352#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
353#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
354
355typedef enum {
356#include "tcg-target-con-set.h"
357} TCGConstraintSetIndex;
358
359static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
360
361#undef C_O0_I1
362#undef C_O0_I2
363#undef C_O0_I3
364#undef C_O0_I4
365#undef C_O1_I1
366#undef C_O1_I2
367#undef C_O1_I3
368#undef C_O1_I4
369#undef C_N1_I2
370#undef C_O2_I1
371#undef C_O2_I2
372#undef C_O2_I3
373#undef C_O2_I4
374
375/* Put all of the constraint sets into an array, indexed by the enum. */
376
377#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
378#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
379#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
380#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
381
382#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
383#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
384#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
385#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
386
387#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
388
389#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
390#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
391#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
392#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
393
394static const TCGTargetOpDef constraint_sets[] = {
395#include "tcg-target-con-set.h"
396};
397
398
399#undef C_O0_I1
400#undef C_O0_I2
401#undef C_O0_I3
402#undef C_O0_I4
403#undef C_O1_I1
404#undef C_O1_I2
405#undef C_O1_I3
406#undef C_O1_I4
407#undef C_N1_I2
408#undef C_O2_I1
409#undef C_O2_I2
410#undef C_O2_I3
411#undef C_O2_I4
412
413/* Expand the enumerator to be returned from tcg_target_op_def(). */
414
415#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
416#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
417#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
418#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
419
420#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
421#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
422#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
423#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
424
425#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
426
427#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
428#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
429#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
430#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
431
139c1837 432#include "tcg-target.c.inc"
c896fe29 433
38b47b19
EC		 434static void alloc_tcg_plugin_context(TCGContext *s)
435{
436#ifdef CONFIG_PLUGIN
437 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
438 s->plugin_tb->insns =
439 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
440#endif
441}
442
3468b59e
EC		 443/*
444 * All TCG threads except the parent (i.e. the one that called tcg_context_init
445 * and registered the target's TCG globals) must register with this function
446 * before initiating translation.
447 *
448 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
449 * of tcg_region_init() for the reasoning behind this.
450 *
451 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
452 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
453 * is not used anymore for translation once this function is called.
454 *
455 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
456 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
457 */
458#ifdef CONFIG_USER_ONLY
459void tcg_register_thread(void)
460{
461 tcg_ctx = &tcg_init_ctx;
462}
463#else
464void tcg_register_thread(void)
465{
466 TCGContext *s = g_malloc(sizeof(*s));
467 unsigned int i, n;
3468b59e
EC		 468
469 *s = tcg_init_ctx;
470
471 /* Relink mem_base. */
472 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
473 if (tcg_init_ctx.temps[i].mem_base) {
474 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
475 tcg_debug_assert(b >= 0 && b < n);
476 s->temps[i].mem_base = &s->temps[b];
477 }
478 }
479
480 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 481 n = qatomic_fetch_inc(&tcg_cur_ctxs);
482 g_assert(n < tcg_max_ctxs);
d73415a3 483 qatomic_set(&tcg_ctxs[n], s);
3468b59e 484
38b47b19
EC		 485 if (n > 0) {
486 alloc_tcg_plugin_context(s);
bf042e8e 487 tcg_region_initial_alloc(s);
38b47b19
EC		 488 }
489
3468b59e 490 tcg_ctx = s;
e8feb96f 491}
3468b59e 492#endif /* !CONFIG_USER_ONLY */
e8feb96f 493
c896fe29
FB		 494/* pool based memory allocation */
495void *tcg_malloc_internal(TCGContext *s, int size)
496{
497 TCGPool *p;
498 int pool_size;
a813e36f 499
c896fe29
FB		 500 if (size > TCG_POOL_CHUNK_SIZE) {
501 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 502 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 503 p->size = size;
4055299e
KB		 504 p->next = s->pool_first_large;
505 s->pool_first_large = p;
506 return p->data;
c896fe29
FB		 507 } else {
508 p = s->pool_current;
509 if (!p) {
510 p = s->pool_first;
511 if (!p)
512 goto new_pool;
513 } else {
514 if (!p->next) {
515 new_pool:
516 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 517 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 518 p->size = pool_size;
519 p->next = NULL;
a813e36f 520 if (s->pool_current) {
c896fe29 521 s->pool_current->next = p;
a813e36f 522 } else {
c896fe29 523 s->pool_first = p;
a813e36f 524 }
c896fe29
FB		 525 } else {
526 p = p->next;
527 }
528 }
529 }
530 s->pool_current = p;
531 s->pool_cur = p->data + size;
532 s->pool_end = p->data + p->size;
533 return p->data;
534}
535
536void tcg_pool_reset(TCGContext *s)
537{
4055299e
KB		 538 TCGPool *p, *t;
539 for (p = s->pool_first_large; p; p = t) {
540 t = p->next;
541 g_free(p);
542 }
543 s->pool_first_large = NULL;
c896fe29
FB		 544 s->pool_cur = s->pool_end = NULL;
545 s->pool_current = NULL;
546}
547
2ef6175a
RH		 548#include "exec/helper-proto.h"
549
100b5e01 550static const TCGHelperInfo all_helpers[] = {
2ef6175a 551#include "exec/helper-tcg.h"
100b5e01 552};
619205fd 553static GHashTable *helper_table;
100b5e01 554
22f15579
RH		 555#ifdef CONFIG_TCG_INTERPRETER
556static GHashTable *ffi_table;
557
558static ffi_type * const typecode_to_ffi[8] = {
559 [dh_typecode_void] = &ffi_type_void,
560 [dh_typecode_i32] = &ffi_type_uint32,
561 [dh_typecode_s32] = &ffi_type_sint32,
562 [dh_typecode_i64] = &ffi_type_uint64,
563 [dh_typecode_s64] = &ffi_type_sint64,
564 [dh_typecode_ptr] = &ffi_type_pointer,
565};
566#endif
567
91478cef 568static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 569static void process_op_defs(TCGContext *s);
1c2adb95
RH		 570static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
571 TCGReg reg, const char *name);
91478cef 572
43b972b7 573static void tcg_context_init(unsigned max_cpus)
c896fe29 574{
a76aabd3 575 TCGContext *s = &tcg_init_ctx;
100b5e01 576 int op, total_args, n, i;
c896fe29
FB		 577 TCGOpDef *def;
578 TCGArgConstraint *args_ct;
1c2adb95 579 TCGTemp *ts;
c896fe29
FB		 580
581 memset(s, 0, sizeof(*s));
c896fe29 582 s->nb_globals = 0;
c70fbf0a 583
c896fe29
FB		 584 /* Count total number of arguments and allocate the corresponding
585 space */
586 total_args = 0;
587 for(op = 0; op < NB_OPS; op++) {
588 def = &tcg_op_defs[op];
589 n = def->nb_iargs + def->nb_oargs;
590 total_args += n;
591 }
592
bc2b17e6 593 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 594
595 for(op = 0; op < NB_OPS; op++) {
596 def = &tcg_op_defs[op];
597 def->args_ct = args_ct;
c896fe29 598 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 599 args_ct += n;
600 }
5cd8f621
RH		 601
602 /* Register helpers. */
84fd9dd3 603 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 604 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 605
100b5e01 606 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
84fd9dd3 607 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 608 (gpointer)&all_helpers[i]);
100b5e01 609 }
5cd8f621 610
22f15579
RH		 611#ifdef CONFIG_TCG_INTERPRETER
612 /* g_direct_hash/equal for direct comparisons on uint32_t. */
613 ffi_table = g_hash_table_new(NULL, NULL);
614 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
615 struct {
616 ffi_cif cif;
617 ffi_type *args[];
618 } *ca;
619 uint32_t typemask = all_helpers[i].typemask;
620 gpointer hash = (gpointer)(uintptr_t)typemask;
621 ffi_status status;
622 int nargs;
623
624 if (g_hash_table_lookup(ffi_table, hash)) {
625 continue;
626 }
627
628 /* Ignoring the return type, find the last non-zero field. */
629 nargs = 32 - clz32(typemask >> 3);
630 nargs = DIV_ROUND_UP(nargs, 3);
631
632 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
633 ca->cif.rtype = typecode_to_ffi[typemask & 7];
634 ca->cif.nargs = nargs;
635
636 if (nargs != 0) {
637 ca->cif.arg_types = ca->args;
9dd1d56e
IZ		 638 for (int j = 0; j < nargs; ++j) {
639 int typecode = extract32(typemask, (j + 1) * 3, 3);
640 ca->args[j] = typecode_to_ffi[typecode];
22f15579
RH		 641 }
642 }
643
644 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
645 ca->cif.rtype, ca->cif.arg_types);
646 assert(status == FFI_OK);
647
648 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
649 }
650#endif
651
c896fe29 652 tcg_target_init(s);
f69d277e 653 process_op_defs(s);
91478cef
RH		 654
655 /* Reverse the order of the saved registers, assuming they're all at
656 the start of tcg_target_reg_alloc_order. */
657 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
658 int r = tcg_target_reg_alloc_order[n];
659 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
660 break;
661 }
662 }
663 for (i = 0; i < n; ++i) {
664 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
665 }
666 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
667 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
668 }
b1311c4a 669
38b47b19
EC		 670 alloc_tcg_plugin_context(s);
671
b1311c4a 672 tcg_ctx = s;
3468b59e
EC		 673 /*
674 * In user-mode we simply share the init context among threads, since we
675 * use a single region. See the documentation tcg_region_init() for the
676 * reasoning behind this.
677 * In softmmu we will have at most max_cpus TCG threads.
678 */
679#ifdef CONFIG_USER_ONLY
df2cce29 680 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 681 tcg_cur_ctxs = 1;
682 tcg_max_ctxs = 1;
3468b59e 683#else
0e2d61cf
RH		 684 tcg_max_ctxs = max_cpus;
685 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 686#endif
1c2adb95
RH		 687
688 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
689 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
690 cpu_env = temp_tcgv_ptr(ts);
9002ec79 691}
b03cce8e 692
43b972b7 693void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 694{
43b972b7
RH		 695 tcg_context_init(max_cpus);
696 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 697}
698
6e3b2bfd
EC		 699/*
700 * Allocate TBs right before their corresponding translated code, making
701 * sure that TBs and code are on different cache lines.
702 */
703TranslationBlock *tcg_tb_alloc(TCGContext *s)
704{
705 uintptr_t align = qemu_icache_linesize;
706 TranslationBlock *tb;
707 void *next;
708
e8feb96f 709 retry:
6e3b2bfd
EC		 710 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
711 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
712
713 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 714 if (tcg_region_alloc(s)) {
715 return NULL;
716 }
717 goto retry;
6e3b2bfd 718 }
d73415a3 719 qatomic_set(&s->code_gen_ptr, next);
57a26946 720 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 721 return tb;
722}
723
9002ec79
RH		 724void tcg_prologue_init(TCGContext *s)
725{
b0a0794a 726 size_t prologue_size;
8163b749 727
b0a0794a
RH		 728 s->code_ptr = s->code_gen_ptr;
729 s->code_buf = s->code_gen_ptr;
5b38ee31 730 s->data_gen_ptr = NULL;
b91ccb31
RH		 731
732#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 733 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 734#endif
8163b749 735
5b38ee31
RH		 736#ifdef TCG_TARGET_NEED_POOL_LABELS
737 s->pool_labels = NULL;
738#endif
739
653b87eb 740 qemu_thread_jit_write();
8163b749 741 /* Generate the prologue. */
b03cce8e 742 tcg_target_qemu_prologue(s);
5b38ee31
RH		 743
744#ifdef TCG_TARGET_NEED_POOL_LABELS
745 /* Allow the prologue to put e.g. guest_base into a pool entry. */
746 {
1768987b
RH		 747 int result = tcg_out_pool_finalize(s);
748 tcg_debug_assert(result == 0);
5b38ee31
RH		 749 }
750#endif
751
b0a0794a
RH		 752 prologue_size = tcg_current_code_size(s);
753
df5d2b16 754#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 755 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
756 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 757#endif
8163b749 758
d6b64b2b
RH		 759#ifdef DEBUG_DISAS
760 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
c60f599b 761 FILE *logfile = qemu_log_trylock();
78b54858
RH		 762 if (logfile) {
763 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
764 if (s->data_gen_ptr) {
765 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
766 size_t data_size = prologue_size - code_size;
767 size_t i;
768
769 disas(logfile, s->code_gen_ptr, code_size);
770
771 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
772 if (sizeof(tcg_target_ulong) == 8) {
773 fprintf(logfile,
774 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
775 (uintptr_t)s->data_gen_ptr + i,
776 *(uint64_t *)(s->data_gen_ptr + i));
777 } else {
778 fprintf(logfile,
779 "0x%08" PRIxPTR ": .long 0x%08x\n",
780 (uintptr_t)s->data_gen_ptr + i,
781 *(uint32_t *)(s->data_gen_ptr + i));
782 }
5b38ee31 783 }
78b54858
RH		 784 } else {
785 disas(logfile, s->code_gen_ptr, prologue_size);
5b38ee31 786 }
78b54858 787 fprintf(logfile, "\n");
78b54858 788 qemu_log_unlock(logfile);
5b38ee31 789 }
d6b64b2b
RH		 790 }
791#endif
cedbcb01 792
6eea0434
RH		 793#ifndef CONFIG_TCG_INTERPRETER
794 /*
795 * Assert that goto_ptr is implemented completely, setting an epilogue.
796 * For tci, we use NULL as the signal to return from the interpreter,
797 * so skip this check.
798 */
f4e01e30 799 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 800#endif
d1c74ab3
RH		 801
802 tcg_region_prologue_set(s);
c896fe29
FB		 803}
804
c896fe29
FB		 805void tcg_func_start(TCGContext *s)
806{
807 tcg_pool_reset(s);
808 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 809
810 /* No temps have been previously allocated for size or locality. */
811 memset(s->free_temps, 0, sizeof(s->free_temps));
812
c0522136
RH		 813 /* No constant temps have been previously allocated. */
814 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
815 if (s->const_table[i]) {
816 g_hash_table_remove_all(s->const_table[i]);
817 }
818 }
819
abebf925 820 s->nb_ops = 0;
c896fe29
FB		 821 s->nb_labels = 0;
822 s->current_frame_offset = s->frame_start;
823
0a209d4b
RH		 824#ifdef CONFIG_DEBUG_TCG
825 s->goto_tb_issue_mask = 0;
826#endif
827
15fa08f8
RH		 828 QTAILQ_INIT(&s->ops);
829 QTAILQ_INIT(&s->free_ops);
bef16ab4 830 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 831}
832
ae30e866 833static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 834{
835 int n = s->nb_temps++;
ae30e866
RH		 836
837 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 838 tcg_raise_tb_overflow(s);
ae30e866 839 }
7ca4b752
RH		 840 return memset(&s->temps[n], 0, sizeof(TCGTemp));
841}
842
ae30e866 843static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 844{
fa477d25
RH		 845 TCGTemp *ts;
846
7ca4b752 847 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 848 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 849 s->nb_globals++;
fa477d25 850 ts = tcg_temp_alloc(s);
ee17db83 851 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 852
853 return ts;
c896fe29
FB		 854}
855
085272b3
RH		 856static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
857 TCGReg reg, const char *name)
c896fe29 858{
c896fe29 859 TCGTemp *ts;
c896fe29 860
b3a62939 861 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 862 tcg_abort();
b3a62939 863 }
7ca4b752
RH		 864
865 ts = tcg_global_alloc(s);
c896fe29
FB		 866 ts->base_type = type;
867 ts->type = type;
ee17db83 868 ts->kind = TEMP_FIXED;
c896fe29 869 ts->reg = reg;
c896fe29 870 ts->name = name;
c896fe29 871 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 872
085272b3 873 return ts;
a7812ae4
PB		 874}
875
b6638662 876void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 877{
b3a62939
RH		 878 s->frame_start = start;
879 s->frame_end = start + size;
085272b3
RH		 880 s->frame_temp
881 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 882}
883
085272b3
RH		 884TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
885 intptr_t offset, const char *name)
c896fe29 886{
b1311c4a 887 TCGContext *s = tcg_ctx;
dc41aa7d 888 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 889 TCGTemp *ts = tcg_global_alloc(s);
b3915dbb 890 int indirect_reg = 0, bigendian = 0;
e03b5686 891#if HOST_BIG_ENDIAN
7ca4b752
RH		 892 bigendian = 1;
893#endif
c896fe29 894
c0522136
RH		 895 switch (base_ts->kind) {
896 case TEMP_FIXED:
897 break;
898 case TEMP_GLOBAL:
5a18407f
RH		 899 /* We do not support double-indirect registers. */
900 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 901 base_ts->indirect_base = 1;
5a18407f
RH		 902 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
903 ? 2 : 1);
904 indirect_reg = 1;
c0522136
RH		 905 break;
906 default:
907 g_assert_not_reached();
b3915dbb
RH		 908 }
909
7ca4b752
RH		 910 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
911 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 912 char buf[64];
7ca4b752
RH		 913
914 ts->base_type = TCG_TYPE_I64;
c896fe29 915 ts->type = TCG_TYPE_I32;
b3915dbb 916 ts->indirect_reg = indirect_reg;
c896fe29 917 ts->mem_allocated = 1;
b3a62939 918 ts->mem_base = base_ts;
7ca4b752 919 ts->mem_offset = offset + bigendian * 4;
c896fe29
FB		 920 pstrcpy(buf, sizeof(buf), name);
921 pstrcat(buf, sizeof(buf), "_0");
922 ts->name = strdup(buf);
c896fe29 923
7ca4b752
RH		 924 tcg_debug_assert(ts2 == ts + 1);
925 ts2->base_type = TCG_TYPE_I64;
926 ts2->type = TCG_TYPE_I32;
b3915dbb 927 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 928 ts2->mem_allocated = 1;
929 ts2->mem_base = base_ts;
930 ts2->mem_offset = offset + (1 - bigendian) * 4;
c896fe29
FB		 931 pstrcpy(buf, sizeof(buf), name);
932 pstrcat(buf, sizeof(buf), "_1");
120c1084 933 ts2->name = strdup(buf);
7ca4b752 934 } else {
c896fe29
FB		 935 ts->base_type = type;
936 ts->type = type;
b3915dbb 937 ts->indirect_reg = indirect_reg;
c896fe29 938 ts->mem_allocated = 1;
b3a62939 939 ts->mem_base = base_ts;
c896fe29 940 ts->mem_offset = offset;
c896fe29 941 ts->name = name;
c896fe29 942 }
085272b3 943 return ts;
a7812ae4
PB		 944}
945
5bfa8034 946TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
c896fe29 947{
b1311c4a 948 TCGContext *s = tcg_ctx;
ee17db83 949 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
c896fe29 950 TCGTemp *ts;
641d5fbe 951 int idx, k;
c896fe29 952
0ec9eabc
RH		 953 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
954 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
955 if (idx < TCG_MAX_TEMPS) {
956 /* There is already an available temp with the right type. */
957 clear_bit(idx, s->free_temps[k].l);
958
e8996ee0 959 ts = &s->temps[idx];
e8996ee0 960 ts->temp_allocated = 1;
7ca4b752 961 tcg_debug_assert(ts->base_type == type);
ee17db83 962 tcg_debug_assert(ts->kind == kind);
e8996ee0 963 } else {
7ca4b752
RH		 964 ts = tcg_temp_alloc(s);
965 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
966 TCGTemp *ts2 = tcg_temp_alloc(s);
967
f6aa2f7d 968 ts->base_type = type;
e8996ee0
FB		 969 ts->type = TCG_TYPE_I32;
970 ts->temp_allocated = 1;
ee17db83 971 ts->kind = kind;
7ca4b752
RH		 972
973 tcg_debug_assert(ts2 == ts + 1);
974 ts2->base_type = TCG_TYPE_I64;
975 ts2->type = TCG_TYPE_I32;
976 ts2->temp_allocated = 1;
ee17db83 977 ts2->kind = kind;
7ca4b752 978 } else {
e8996ee0
FB		 979 ts->base_type = type;
980 ts->type = type;
981 ts->temp_allocated = 1;
ee17db83 982 ts->kind = kind;
e8996ee0 983 }
c896fe29 984 }
27bfd83c
PM		 985
986#if defined(CONFIG_DEBUG_TCG)
987 s->temps_in_use++;
988#endif
085272b3 989 return ts;
c896fe29
FB		 990}
991
d2fd745f
RH		 992TCGv_vec tcg_temp_new_vec(TCGType type)
993{
994 TCGTemp *t;
995
996#ifdef CONFIG_DEBUG_TCG
997 switch (type) {
998 case TCG_TYPE_V64:
999 assert(TCG_TARGET_HAS_v64);
1000 break;
1001 case TCG_TYPE_V128:
1002 assert(TCG_TARGET_HAS_v128);
1003 break;
1004 case TCG_TYPE_V256:
1005 assert(TCG_TARGET_HAS_v256);
1006 break;
1007 default:
1008 g_assert_not_reached();
1009 }
1010#endif
1011
1012 t = tcg_temp_new_internal(type, 0);
1013 return temp_tcgv_vec(t);
1014}
1015
1016/* Create a new temp of the same type as an existing temp. */
1017TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1018{
1019 TCGTemp *t = tcgv_vec_temp(match);
1020
1021 tcg_debug_assert(t->temp_allocated != 0);
1022
1023 t = tcg_temp_new_internal(t->base_type, 0);
1024 return temp_tcgv_vec(t);
1025}
1026
5bfa8034 1027void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1028{
b1311c4a 1029 TCGContext *s = tcg_ctx;
085272b3 1030 int k, idx;
c896fe29 1031
c7482438
RH		 1032 switch (ts->kind) {
1033 case TEMP_CONST:
1034 /*
1035 * In order to simplify users of tcg_constant_*,
1036 * silently ignore free.
1037 */
c0522136 1038 return;
c7482438
RH		 1039 case TEMP_NORMAL:
1040 case TEMP_LOCAL:
1041 break;
1042 default:
1043 g_assert_not_reached();
c0522136
RH		 1044 }
1045
27bfd83c
PM		 1046#if defined(CONFIG_DEBUG_TCG)
1047 s->temps_in_use--;
1048 if (s->temps_in_use < 0) {
1049 fprintf(stderr, "More temporaries freed than allocated!\n");
1050 }
1051#endif
1052
eabb7b91 1053 tcg_debug_assert(ts->temp_allocated != 0);
e8996ee0 1054 ts->temp_allocated = 0;
0ec9eabc 1055
085272b3 1056 idx = temp_idx(ts);
ee17db83 1057 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1058 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1059}
1060
c0522136
RH		 1061TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1062{
1063 TCGContext *s = tcg_ctx;
1064 GHashTable *h = s->const_table[type];
1065 TCGTemp *ts;
1066
1067 if (h == NULL) {
1068 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1069 s->const_table[type] = h;
1070 }
1071
1072 ts = g_hash_table_lookup(h, &val);
1073 if (ts == NULL) {
1074 ts = tcg_temp_alloc(s);
1075
1076 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1077 TCGTemp *ts2 = tcg_temp_alloc(s);
1078
1079 ts->base_type = TCG_TYPE_I64;
1080 ts->type = TCG_TYPE_I32;
1081 ts->kind = TEMP_CONST;
1082 ts->temp_allocated = 1;
1083 /*
1084 * Retain the full value of the 64-bit constant in the low
1085 * part, so that the hash table works. Actual uses will
1086 * truncate the value to the low part.
1087 */
1088 ts->val = val;
1089
1090 tcg_debug_assert(ts2 == ts + 1);
1091 ts2->base_type = TCG_TYPE_I64;
1092 ts2->type = TCG_TYPE_I32;
1093 ts2->kind = TEMP_CONST;
1094 ts2->temp_allocated = 1;
1095 ts2->val = val >> 32;
1096 } else {
1097 ts->base_type = type;
1098 ts->type = type;
1099 ts->kind = TEMP_CONST;
1100 ts->temp_allocated = 1;
1101 ts->val = val;
1102 }
1103 g_hash_table_insert(h, &ts->val, ts);
1104 }
1105
1106 return ts;
1107}
1108
1109TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1110{
1111 val = dup_const(vece, val);
1112 return temp_tcgv_vec(tcg_constant_internal(type, val));
1113}
1114
88d4005b
RH		 1115TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1116{
1117 TCGTemp *t = tcgv_vec_temp(match);
1118
1119 tcg_debug_assert(t->temp_allocated != 0);
1120 return tcg_constant_vec(t->base_type, vece, val);
1121}
1122
a7812ae4 1123TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1124{
a7812ae4
PB		 1125 TCGv_i32 t0;
1126 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1127 tcg_gen_movi_i32(t0, val);
1128 return t0;
1129}
c896fe29 1130
a7812ae4 1131TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1132{
a7812ae4
PB		 1133 TCGv_i64 t0;
1134 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1135 tcg_gen_movi_i64(t0, val);
1136 return t0;
c896fe29
FB		 1137}
1138
a7812ae4 1139TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1140{
a7812ae4
PB		 1141 TCGv_i32 t0;
1142 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1143 tcg_gen_movi_i32(t0, val);
1144 return t0;
1145}
1146
a7812ae4 1147TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1148{
a7812ae4
PB		 1149 TCGv_i64 t0;
1150 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1151 tcg_gen_movi_i64(t0, val);
1152 return t0;
1153}
1154
27bfd83c
PM		 1155#if defined(CONFIG_DEBUG_TCG)
1156void tcg_clear_temp_count(void)
1157{
b1311c4a 1158 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1159 s->temps_in_use = 0;
1160}
1161
1162int tcg_check_temp_count(void)
1163{
b1311c4a 1164 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1165 if (s->temps_in_use) {
1166 /* Clear the count so that we don't give another
1167 * warning immediately next time around.
1168 */
1169 s->temps_in_use = 0;
1170 return 1;
1171 }
1172 return 0;
1173}
1174#endif
1175
be0f34b5
RH		 1176/* Return true if OP may appear in the opcode stream.
1177 Test the runtime variable that controls each opcode. */
1178bool tcg_op_supported(TCGOpcode op)
1179{
d2fd745f
RH		 1180 const bool have_vec
1181 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1182
be0f34b5
RH		 1183 switch (op) {
1184 case INDEX_op_discard:
1185 case INDEX_op_set_label:
1186 case INDEX_op_call:
1187 case INDEX_op_br:
1188 case INDEX_op_mb:
1189 case INDEX_op_insn_start:
1190 case INDEX_op_exit_tb:
1191 case INDEX_op_goto_tb:
f4e01e30 1192 case INDEX_op_goto_ptr:
be0f34b5
RH		 1193 case INDEX_op_qemu_ld_i32:
1194 case INDEX_op_qemu_st_i32:
1195 case INDEX_op_qemu_ld_i64:
1196 case INDEX_op_qemu_st_i64:
1197 return true;
1198
07ce0b05
RH		 1199 case INDEX_op_qemu_st8_i32:
1200 return TCG_TARGET_HAS_qemu_st8_i32;
1201
be0f34b5 1202 case INDEX_op_mov_i32:
be0f34b5
RH		 1203 case INDEX_op_setcond_i32:
1204 case INDEX_op_brcond_i32:
1205 case INDEX_op_ld8u_i32:
1206 case INDEX_op_ld8s_i32:
1207 case INDEX_op_ld16u_i32:
1208 case INDEX_op_ld16s_i32:
1209 case INDEX_op_ld_i32:
1210 case INDEX_op_st8_i32:
1211 case INDEX_op_st16_i32:
1212 case INDEX_op_st_i32:
1213 case INDEX_op_add_i32:
1214 case INDEX_op_sub_i32:
1215 case INDEX_op_mul_i32:
1216 case INDEX_op_and_i32:
1217 case INDEX_op_or_i32:
1218 case INDEX_op_xor_i32:
1219 case INDEX_op_shl_i32:
1220 case INDEX_op_shr_i32:
1221 case INDEX_op_sar_i32:
1222 return true;
1223
1224 case INDEX_op_movcond_i32:
1225 return TCG_TARGET_HAS_movcond_i32;
1226 case INDEX_op_div_i32:
1227 case INDEX_op_divu_i32:
1228 return TCG_TARGET_HAS_div_i32;
1229 case INDEX_op_rem_i32:
1230 case INDEX_op_remu_i32:
1231 return TCG_TARGET_HAS_rem_i32;
1232 case INDEX_op_div2_i32:
1233 case INDEX_op_divu2_i32:
1234 return TCG_TARGET_HAS_div2_i32;
1235 case INDEX_op_rotl_i32:
1236 case INDEX_op_rotr_i32:
1237 return TCG_TARGET_HAS_rot_i32;
1238 case INDEX_op_deposit_i32:
1239 return TCG_TARGET_HAS_deposit_i32;
1240 case INDEX_op_extract_i32:
1241 return TCG_TARGET_HAS_extract_i32;
1242 case INDEX_op_sextract_i32:
1243 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1244 case INDEX_op_extract2_i32:
1245 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1246 case INDEX_op_add2_i32:
1247 return TCG_TARGET_HAS_add2_i32;
1248 case INDEX_op_sub2_i32:
1249 return TCG_TARGET_HAS_sub2_i32;
1250 case INDEX_op_mulu2_i32:
1251 return TCG_TARGET_HAS_mulu2_i32;
1252 case INDEX_op_muls2_i32:
1253 return TCG_TARGET_HAS_muls2_i32;
1254 case INDEX_op_muluh_i32:
1255 return TCG_TARGET_HAS_muluh_i32;
1256 case INDEX_op_mulsh_i32:
1257 return TCG_TARGET_HAS_mulsh_i32;
1258 case INDEX_op_ext8s_i32:
1259 return TCG_TARGET_HAS_ext8s_i32;
1260 case INDEX_op_ext16s_i32:
1261 return TCG_TARGET_HAS_ext16s_i32;
1262 case INDEX_op_ext8u_i32:
1263 return TCG_TARGET_HAS_ext8u_i32;
1264 case INDEX_op_ext16u_i32:
1265 return TCG_TARGET_HAS_ext16u_i32;
1266 case INDEX_op_bswap16_i32:
1267 return TCG_TARGET_HAS_bswap16_i32;
1268 case INDEX_op_bswap32_i32:
1269 return TCG_TARGET_HAS_bswap32_i32;
1270 case INDEX_op_not_i32:
1271 return TCG_TARGET_HAS_not_i32;
1272 case INDEX_op_neg_i32:
1273 return TCG_TARGET_HAS_neg_i32;
1274 case INDEX_op_andc_i32:
1275 return TCG_TARGET_HAS_andc_i32;
1276 case INDEX_op_orc_i32:
1277 return TCG_TARGET_HAS_orc_i32;
1278 case INDEX_op_eqv_i32:
1279 return TCG_TARGET_HAS_eqv_i32;
1280 case INDEX_op_nand_i32:
1281 return TCG_TARGET_HAS_nand_i32;
1282 case INDEX_op_nor_i32:
1283 return TCG_TARGET_HAS_nor_i32;
1284 case INDEX_op_clz_i32:
1285 return TCG_TARGET_HAS_clz_i32;
1286 case INDEX_op_ctz_i32:
1287 return TCG_TARGET_HAS_ctz_i32;
1288 case INDEX_op_ctpop_i32:
1289 return TCG_TARGET_HAS_ctpop_i32;
1290
1291 case INDEX_op_brcond2_i32:
1292 case INDEX_op_setcond2_i32:
1293 return TCG_TARGET_REG_BITS == 32;
1294
1295 case INDEX_op_mov_i64:
be0f34b5
RH		 1296 case INDEX_op_setcond_i64:
1297 case INDEX_op_brcond_i64:
1298 case INDEX_op_ld8u_i64:
1299 case INDEX_op_ld8s_i64:
1300 case INDEX_op_ld16u_i64:
1301 case INDEX_op_ld16s_i64:
1302 case INDEX_op_ld32u_i64:
1303 case INDEX_op_ld32s_i64:
1304 case INDEX_op_ld_i64:
1305 case INDEX_op_st8_i64:
1306 case INDEX_op_st16_i64:
1307 case INDEX_op_st32_i64:
1308 case INDEX_op_st_i64:
1309 case INDEX_op_add_i64:
1310 case INDEX_op_sub_i64:
1311 case INDEX_op_mul_i64:
1312 case INDEX_op_and_i64:
1313 case INDEX_op_or_i64:
1314 case INDEX_op_xor_i64:
1315 case INDEX_op_shl_i64:
1316 case INDEX_op_shr_i64:
1317 case INDEX_op_sar_i64:
1318 case INDEX_op_ext_i32_i64:
1319 case INDEX_op_extu_i32_i64:
1320 return TCG_TARGET_REG_BITS == 64;
1321
1322 case INDEX_op_movcond_i64:
1323 return TCG_TARGET_HAS_movcond_i64;
1324 case INDEX_op_div_i64:
1325 case INDEX_op_divu_i64:
1326 return TCG_TARGET_HAS_div_i64;
1327 case INDEX_op_rem_i64:
1328 case INDEX_op_remu_i64:
1329 return TCG_TARGET_HAS_rem_i64;
1330 case INDEX_op_div2_i64:
1331 case INDEX_op_divu2_i64:
1332 return TCG_TARGET_HAS_div2_i64;
1333 case INDEX_op_rotl_i64:
1334 case INDEX_op_rotr_i64:
1335 return TCG_TARGET_HAS_rot_i64;
1336 case INDEX_op_deposit_i64:
1337 return TCG_TARGET_HAS_deposit_i64;
1338 case INDEX_op_extract_i64:
1339 return TCG_TARGET_HAS_extract_i64;
1340 case INDEX_op_sextract_i64:
1341 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1342 case INDEX_op_extract2_i64:
1343 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1344 case INDEX_op_extrl_i64_i32:
1345 return TCG_TARGET_HAS_extrl_i64_i32;
1346 case INDEX_op_extrh_i64_i32:
1347 return TCG_TARGET_HAS_extrh_i64_i32;
1348 case INDEX_op_ext8s_i64:
1349 return TCG_TARGET_HAS_ext8s_i64;
1350 case INDEX_op_ext16s_i64:
1351 return TCG_TARGET_HAS_ext16s_i64;
1352 case INDEX_op_ext32s_i64:
1353 return TCG_TARGET_HAS_ext32s_i64;
1354 case INDEX_op_ext8u_i64:
1355 return TCG_TARGET_HAS_ext8u_i64;
1356 case INDEX_op_ext16u_i64:
1357 return TCG_TARGET_HAS_ext16u_i64;
1358 case INDEX_op_ext32u_i64:
1359 return TCG_TARGET_HAS_ext32u_i64;
1360 case INDEX_op_bswap16_i64:
1361 return TCG_TARGET_HAS_bswap16_i64;
1362 case INDEX_op_bswap32_i64:
1363 return TCG_TARGET_HAS_bswap32_i64;
1364 case INDEX_op_bswap64_i64:
1365 return TCG_TARGET_HAS_bswap64_i64;
1366 case INDEX_op_not_i64:
1367 return TCG_TARGET_HAS_not_i64;
1368 case INDEX_op_neg_i64:
1369 return TCG_TARGET_HAS_neg_i64;
1370 case INDEX_op_andc_i64:
1371 return TCG_TARGET_HAS_andc_i64;
1372 case INDEX_op_orc_i64:
1373 return TCG_TARGET_HAS_orc_i64;
1374 case INDEX_op_eqv_i64:
1375 return TCG_TARGET_HAS_eqv_i64;
1376 case INDEX_op_nand_i64:
1377 return TCG_TARGET_HAS_nand_i64;
1378 case INDEX_op_nor_i64:
1379 return TCG_TARGET_HAS_nor_i64;
1380 case INDEX_op_clz_i64:
1381 return TCG_TARGET_HAS_clz_i64;
1382 case INDEX_op_ctz_i64:
1383 return TCG_TARGET_HAS_ctz_i64;
1384 case INDEX_op_ctpop_i64:
1385 return TCG_TARGET_HAS_ctpop_i64;
1386 case INDEX_op_add2_i64:
1387 return TCG_TARGET_HAS_add2_i64;
1388 case INDEX_op_sub2_i64:
1389 return TCG_TARGET_HAS_sub2_i64;
1390 case INDEX_op_mulu2_i64:
1391 return TCG_TARGET_HAS_mulu2_i64;
1392 case INDEX_op_muls2_i64:
1393 return TCG_TARGET_HAS_muls2_i64;
1394 case INDEX_op_muluh_i64:
1395 return TCG_TARGET_HAS_muluh_i64;
1396 case INDEX_op_mulsh_i64:
1397 return TCG_TARGET_HAS_mulsh_i64;
1398
d2fd745f
RH		 1399 case INDEX_op_mov_vec:
1400 case INDEX_op_dup_vec:
37ee55a0 1401 case INDEX_op_dupm_vec:
d2fd745f
RH		 1402 case INDEX_op_ld_vec:
1403 case INDEX_op_st_vec:
1404 case INDEX_op_add_vec:
1405 case INDEX_op_sub_vec:
1406 case INDEX_op_and_vec:
1407 case INDEX_op_or_vec:
1408 case INDEX_op_xor_vec:
212be173 1409 case INDEX_op_cmp_vec:
d2fd745f
RH		 1410 return have_vec;
1411 case INDEX_op_dup2_vec:
1412 return have_vec && TCG_TARGET_REG_BITS == 32;
1413 case INDEX_op_not_vec:
1414 return have_vec && TCG_TARGET_HAS_not_vec;
1415 case INDEX_op_neg_vec:
1416 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1417 case INDEX_op_abs_vec:
1418 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1419 case INDEX_op_andc_vec:
1420 return have_vec && TCG_TARGET_HAS_andc_vec;
1421 case INDEX_op_orc_vec:
1422 return have_vec && TCG_TARGET_HAS_orc_vec;
ed523473
RH		 1423 case INDEX_op_nand_vec:
1424 return have_vec && TCG_TARGET_HAS_nand_vec;
1425 case INDEX_op_nor_vec:
1426 return have_vec && TCG_TARGET_HAS_nor_vec;
1427 case INDEX_op_eqv_vec:
1428 return have_vec && TCG_TARGET_HAS_eqv_vec;
3774030a
RH		 1429 case INDEX_op_mul_vec:
1430 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1431 case INDEX_op_shli_vec:
1432 case INDEX_op_shri_vec:
1433 case INDEX_op_sari_vec:
1434 return have_vec && TCG_TARGET_HAS_shi_vec;
1435 case INDEX_op_shls_vec:
1436 case INDEX_op_shrs_vec:
1437 case INDEX_op_sars_vec:
1438 return have_vec && TCG_TARGET_HAS_shs_vec;
1439 case INDEX_op_shlv_vec:
1440 case INDEX_op_shrv_vec:
1441 case INDEX_op_sarv_vec:
1442 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1443 case INDEX_op_rotli_vec:
1444 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1445 case INDEX_op_rotls_vec:
1446 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1447 case INDEX_op_rotlv_vec:
1448 case INDEX_op_rotrv_vec:
1449 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1450 case INDEX_op_ssadd_vec:
1451 case INDEX_op_usadd_vec:
1452 case INDEX_op_sssub_vec:
1453 case INDEX_op_ussub_vec:
1454 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1455 case INDEX_op_smin_vec:
1456 case INDEX_op_umin_vec:
1457 case INDEX_op_smax_vec:
1458 case INDEX_op_umax_vec:
1459 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1460 case INDEX_op_bitsel_vec:
1461 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1462 case INDEX_op_cmpsel_vec:
1463 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1464
db432672
RH		 1465 default:
1466 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1467 return true;
be0f34b5 1468 }
be0f34b5
RH		 1469}
1470
39cf05d3
FB		 1471/* Note: we convert the 64 bit args to 32 bit and do some alignment
1472 and endian swap. Maybe it would be better to do the alignment
1473 and endian swap in tcg_reg_alloc_call(). */
ae8b75dc 1474void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1475{
75e8b9b7 1476 int i, real_args, nb_rets, pi;
3e92aa34
RH		 1477 unsigned typemask;
1478 const TCGHelperInfo *info;
75e8b9b7 1479 TCGOp *op;
afb49896 1480
619205fd 1481 info = g_hash_table_lookup(helper_table, (gpointer)func);
7319d83a 1482 typemask = info->typemask;
2bece2c8 1483
38b47b19
EC		 1484#ifdef CONFIG_PLUGIN
1485 /* detect non-plugin helpers */
1486 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1487 tcg_ctx->plugin_insn->calls_helpers = true;
1488 }
1489#endif
1490
3a5f6805 1491#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1492 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1493 int argtype = extract32(typemask, (i + 1) * 3, 3);
1494 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1495 bool is_signed = argtype & 1;
1496
1497 if (is_32bit) {
2bece2c8 1498 TCGv_i64 temp = tcg_temp_new_i64();
18cf3d07 1499 TCGv_i32 orig = temp_tcgv_i32(args[i]);
2bece2c8 1500 if (is_signed) {
18cf3d07 1501 tcg_gen_ext_i32_i64(temp, orig);
2bece2c8 1502 } else {
18cf3d07 1503 tcg_gen_extu_i32_i64(temp, orig);
2bece2c8 1504 }
ae8b75dc 1505 args[i] = tcgv_i64_temp(temp);
2bece2c8
RH		 1506 }
1507 }
1508#endif /* TCG_TARGET_EXTEND_ARGS */
1509
15fa08f8 1510 op = tcg_emit_op(INDEX_op_call);
75e8b9b7
RH		 1511
1512 pi = 0;
ae8b75dc 1513 if (ret != NULL) {
7319d83a 1514 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
e03b5686 1515#if HOST_BIG_ENDIAN
ae8b75dc
RH		 1516 op->args[pi++] = temp_arg(ret + 1);
1517 op->args[pi++] = temp_arg(ret);
39cf05d3 1518#else
ae8b75dc
RH		 1519 op->args[pi++] = temp_arg(ret);
1520 op->args[pi++] = temp_arg(ret + 1);
39cf05d3 1521#endif
a7812ae4 1522 nb_rets = 2;
34b1a49c 1523 } else {
ae8b75dc 1524 op->args[pi++] = temp_arg(ret);
a7812ae4 1525 nb_rets = 1;
c896fe29 1526 }
a7812ae4
PB		 1527 } else {
1528 nb_rets = 0;
c896fe29 1529 }
cd9090aa 1530 TCGOP_CALLO(op) = nb_rets;
75e8b9b7 1531
a7812ae4
PB		 1532 real_args = 0;
1533 for (i = 0; i < nargs; i++) {
7319d83a
RH		 1534 int argtype = extract32(typemask, (i + 1) * 3, 3);
1535 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
7b7d8b2d 1536 bool want_align = false;
7319d83a 1537
7b7d8b2d
RH		 1538#if defined(CONFIG_TCG_INTERPRETER)
1539 /*
1540 * Align all arguments, so that they land in predictable places
1541 * for passing off to ffi_call.
1542 */
1543 want_align = true;
1544#elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
1545 /* Some targets want aligned 64 bit args */
1546 want_align = is_64bit;
39cf05d3 1547#endif
7b7d8b2d
RH		 1548
1549 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
1550 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1551 real_args++;
1552 }
1553
1554 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
8d21de51
RH		 1555 op->args[pi++] = temp_arg(args[i] + HOST_BIG_ENDIAN);
1556 op->args[pi++] = temp_arg(args[i] + !HOST_BIG_ENDIAN);
a7812ae4 1557 real_args += 2;
2bece2c8 1558 continue;
c896fe29 1559 }
2bece2c8 1560
ae8b75dc 1561 op->args[pi++] = temp_arg(args[i]);
2bece2c8 1562 real_args++;
c896fe29 1563 }
75e8b9b7 1564 op->args[pi++] = (uintptr_t)func;
3e92aa34 1565 op->args[pi++] = (uintptr_t)info;
cd9090aa 1566 TCGOP_CALLI(op) = real_args;
a7812ae4 1567
75e8b9b7 1568 /* Make sure the fields didn't overflow. */
cd9090aa 1569 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
75e8b9b7 1570 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
2bece2c8 1571
3a5f6805 1572#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1573 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1574 int argtype = extract32(typemask, (i + 1) * 3, 3);
1575 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1576
1577 if (is_32bit) {
085272b3 1578 tcg_temp_free_internal(args[i]);
2bece2c8
RH		 1579 }
1580 }
1581#endif /* TCG_TARGET_EXTEND_ARGS */
c896fe29 1582}
c896fe29 1583
8fcd3692 1584static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1585{
ac3b8891 1586 int i, n;
ac3b8891 1587
ee17db83
RH		 1588 for (i = 0, n = s->nb_temps; i < n; i++) {
1589 TCGTemp *ts = &s->temps[i];
1590 TCGTempVal val = TEMP_VAL_MEM;
1591
1592 switch (ts->kind) {
c0522136
RH		 1593 case TEMP_CONST:
1594 val = TEMP_VAL_CONST;
1595 break;
ee17db83
RH		 1596 case TEMP_FIXED:
1597 val = TEMP_VAL_REG;
1598 break;
1599 case TEMP_GLOBAL:
1600 break;
1601 case TEMP_NORMAL:
c7482438 1602 case TEMP_EBB:
ee17db83
RH		 1603 val = TEMP_VAL_DEAD;
1604 /* fall through */
1605 case TEMP_LOCAL:
1606 ts->mem_allocated = 0;
1607 break;
1608 default:
1609 g_assert_not_reached();
1610 }
1611 ts->val_type = val;
e8996ee0 1612 }
f8b2f202
RH		 1613
1614 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1615}
1616
f8b2f202
RH		 1617static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1618 TCGTemp *ts)
c896fe29 1619{
1807f4c4 1620 int idx = temp_idx(ts);
ac56dd48 1621
ee17db83
RH		 1622 switch (ts->kind) {
1623 case TEMP_FIXED:
1624 case TEMP_GLOBAL:
ac56dd48 1625 pstrcpy(buf, buf_size, ts->name);
ee17db83
RH		 1626 break;
1627 case TEMP_LOCAL:
f8b2f202 1628 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83 1629 break;
c7482438
RH		 1630 case TEMP_EBB:
1631 snprintf(buf, buf_size, "ebb%d", idx - s->nb_globals);
1632 break;
ee17db83 1633 case TEMP_NORMAL:
f8b2f202 1634 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1635 break;
c0522136
RH		 1636 case TEMP_CONST:
1637 switch (ts->type) {
1638 case TCG_TYPE_I32:
1639 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1640 break;
1641#if TCG_TARGET_REG_BITS > 32
1642 case TCG_TYPE_I64:
1643 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1644 break;
1645#endif
1646 case TCG_TYPE_V64:
1647 case TCG_TYPE_V128:
1648 case TCG_TYPE_V256:
1649 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1650 64 << (ts->type - TCG_TYPE_V64), ts->val);
1651 break;
1652 default:
1653 g_assert_not_reached();
1654 }
1655 break;
c896fe29
FB		 1656 }
1657 return buf;
1658}
1659
43439139
RH		 1660static char *tcg_get_arg_str(TCGContext *s, char *buf,
1661 int buf_size, TCGArg arg)
f8b2f202 1662{
43439139 1663 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1664}
1665
f48f3ede
BS		 1666static const char * const cond_name[] =
1667{
0aed257f
RH		 1668 [TCG_COND_NEVER] = "never",
1669 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1670 [TCG_COND_EQ] = "eq",
1671 [TCG_COND_NE] = "ne",
1672 [TCG_COND_LT] = "lt",
1673 [TCG_COND_GE] = "ge",
1674 [TCG_COND_LE] = "le",
1675 [TCG_COND_GT] = "gt",
1676 [TCG_COND_LTU] = "ltu",
1677 [TCG_COND_GEU] = "geu",
1678 [TCG_COND_LEU] = "leu",
1679 [TCG_COND_GTU] = "gtu"
1680};
1681
f713d6ad
RH		 1682static const char * const ldst_name[] =
1683{
1684 [MO_UB] = "ub",
1685 [MO_SB] = "sb",
1686 [MO_LEUW] = "leuw",
1687 [MO_LESW] = "lesw",
1688 [MO_LEUL] = "leul",
1689 [MO_LESL] = "lesl",
fc313c64 1690 [MO_LEUQ] = "leq",
f713d6ad
RH		 1691 [MO_BEUW] = "beuw",
1692 [MO_BESW] = "besw",
1693 [MO_BEUL] = "beul",
1694 [MO_BESL] = "besl",
fc313c64 1695 [MO_BEUQ] = "beq",
f713d6ad
RH		 1696};
1697
1f00b27f 1698static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 1699#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 1700 [MO_UNALN >> MO_ASHIFT] = "un+",
1701 [MO_ALIGN >> MO_ASHIFT] = "",
1702#else
1703 [MO_UNALN >> MO_ASHIFT] = "",
1704 [MO_ALIGN >> MO_ASHIFT] = "al+",
1705#endif
1706 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1707 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1708 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1709 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1710 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1711 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1712};
1713
587195bd
RH		 1714static const char bswap_flag_name[][6] = {
1715 [TCG_BSWAP_IZ] = "iz",
1716 [TCG_BSWAP_OZ] = "oz",
1717 [TCG_BSWAP_OS] = "os",
1718 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1719 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1720};
1721
b016486e
RH		 1722static inline bool tcg_regset_single(TCGRegSet d)
1723{
1724 return (d & (d - 1)) == 0;
1725}
1726
1727static inline TCGReg tcg_regset_first(TCGRegSet d)
1728{
1729 if (TCG_TARGET_NB_REGS <= 32) {
1730 return ctz32(d);
1731 } else {
1732 return ctz64(d);
1733 }
1734}
1735
b7a83ff8
RH		 1736/* Return only the number of characters output -- no error return. */
1737#define ne_fprintf(...) \
1738 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
1739
1740static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
c896fe29 1741{
c896fe29 1742 char buf[128];
c45cb8bb 1743 TCGOp *op;
c45cb8bb 1744
15fa08f8 1745 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 1746 int i, k, nb_oargs, nb_iargs, nb_cargs;
1747 const TCGOpDef *def;
c45cb8bb 1748 TCGOpcode c;
bdfb460e 1749 int col = 0;
c896fe29 1750
c45cb8bb 1751 c = op->opc;
c896fe29 1752 def = &tcg_op_defs[c];
c45cb8bb 1753
765b842a 1754 if (c == INDEX_op_insn_start) {
b016486e 1755 nb_oargs = 0;
b7a83ff8 1756 col += ne_fprintf(f, "\n ----");
9aef40ed
RH		 1757
1758 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1759 target_ulong a;
7e4597d7 1760#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 1761 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 1762#else
efee3746 1763 a = op->args[i];
7e4597d7 1764#endif
b7a83ff8 1765 col += ne_fprintf(f, " " TARGET_FMT_lx, a);
eeacee4d 1766 }
7e4597d7 1767 } else if (c == INDEX_op_call) {
3e92aa34 1768 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 1769 void *func = tcg_call_func(op);
3e92aa34 1770
c896fe29 1771 /* variable number of arguments */
cd9090aa
RH		 1772 nb_oargs = TCGOP_CALLO(op);
1773 nb_iargs = TCGOP_CALLI(op);
c896fe29 1774 nb_cargs = def->nb_cargs;
c896fe29 1775
b7a83ff8 1776 col += ne_fprintf(f, " %s ", def->name);
3e92aa34
RH		 1777
1778 /*
1779 * Print the function name from TCGHelperInfo, if available.
1780 * Note that plugins have a template function for the info,
1781 * but the actual function pointer comes from the plugin.
1782 */
3e92aa34 1783 if (func == info->func) {
b7a83ff8 1784 col += ne_fprintf(f, "%s", info->name);
3e92aa34 1785 } else {
b7a83ff8 1786 col += ne_fprintf(f, "plugin(%p)", func);
3e92aa34
RH		 1787 }
1788
b7a83ff8 1789 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
cf066674 1790 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1791 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1792 op->args[i]));
b03cce8e 1793 }
cf066674 1794 for (i = 0; i < nb_iargs; i++) {
efee3746 1795 TCGArg arg = op->args[nb_oargs + i];
cf066674
RH		 1796 const char *t = "<dummy>";
1797 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 1798 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
eeacee4d 1799 }
b7a83ff8 1800 col += ne_fprintf(f, ",%s", t);
e8996ee0 1801 }
b03cce8e 1802 } else {
b7a83ff8 1803 col += ne_fprintf(f, " %s ", def->name);
c45cb8bb
RH		 1804
1805 nb_oargs = def->nb_oargs;
1806 nb_iargs = def->nb_iargs;
1807 nb_cargs = def->nb_cargs;
1808
d2fd745f 1809 if (def->flags & TCG_OPF_VECTOR) {
b7a83ff8
RH		 1810 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
1811 8 << TCGOP_VECE(op));
d2fd745f
RH		 1812 }
1813
b03cce8e 1814 k = 0;
c45cb8bb 1815 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1816 const char *sep = k ? "," : "";
1817 col += ne_fprintf(f, "%s%s", sep,
1818 tcg_get_arg_str(s, buf, sizeof(buf),
1819 op->args[k++]));
b03cce8e 1820 }
c45cb8bb 1821 for (i = 0; i < nb_iargs; i++) {
b7a83ff8
RH		 1822 const char *sep = k ? "," : "";
1823 col += ne_fprintf(f, "%s%s", sep,
1824 tcg_get_arg_str(s, buf, sizeof(buf),
1825 op->args[k++]));
b03cce8e 1826 }
be210acb
RH		 1827 switch (c) {
1828 case INDEX_op_brcond_i32:
be210acb 1829 case INDEX_op_setcond_i32:
ffc5ea09 1830 case INDEX_op_movcond_i32:
ffc5ea09 1831 case INDEX_op_brcond2_i32:
be210acb 1832 case INDEX_op_setcond2_i32:
ffc5ea09 1833 case INDEX_op_brcond_i64:
be210acb 1834 case INDEX_op_setcond_i64:
ffc5ea09 1835 case INDEX_op_movcond_i64:
212be173 1836 case INDEX_op_cmp_vec:
f75da298 1837 case INDEX_op_cmpsel_vec:
efee3746
RH		 1838 if (op->args[k] < ARRAY_SIZE(cond_name)
1839 && cond_name[op->args[k]]) {
b7a83ff8 1840 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
eeacee4d 1841 } else {
b7a83ff8 1842 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 1843 }
f48f3ede 1844 i = 1;
be210acb 1845 break;
f713d6ad
RH		 1846 case INDEX_op_qemu_ld_i32:
1847 case INDEX_op_qemu_st_i32:
07ce0b05 1848 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 1849 case INDEX_op_qemu_ld_i64:
1850 case INDEX_op_qemu_st_i64:
59227d5d 1851 {
9002ffcb 1852 MemOpIdx oi = op->args[k++];
14776ab5 1853 MemOp op = get_memop(oi);
59227d5d
RH		 1854 unsigned ix = get_mmuidx(oi);
1855
59c4b7e8 1856 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
b7a83ff8 1857 col += ne_fprintf(f, ",$0x%x,%u", op, ix);
59c4b7e8 1858 } else {
1f00b27f
SS		 1859 const char *s_al, *s_op;
1860 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 1861 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
b7a83ff8 1862 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 1863 }
1864 i = 1;
f713d6ad 1865 }
f713d6ad 1866 break;
587195bd
RH		 1867 case INDEX_op_bswap16_i32:
1868 case INDEX_op_bswap16_i64:
1869 case INDEX_op_bswap32_i32:
1870 case INDEX_op_bswap32_i64:
1871 case INDEX_op_bswap64_i64:
1872 {
1873 TCGArg flags = op->args[k];
1874 const char *name = NULL;
1875
1876 if (flags < ARRAY_SIZE(bswap_flag_name)) {
1877 name = bswap_flag_name[flags];
1878 }
1879 if (name) {
b7a83ff8 1880 col += ne_fprintf(f, ",%s", name);
587195bd 1881 } else {
b7a83ff8 1882 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
587195bd
RH		 1883 }
1884 i = k = 1;
1885 }
1886 break;
be210acb 1887 default:
f48f3ede 1888 i = 0;
be210acb
RH		 1889 break;
1890 }
51e3972c
RH		 1891 switch (c) {
1892 case INDEX_op_set_label:
1893 case INDEX_op_br:
1894 case INDEX_op_brcond_i32:
1895 case INDEX_op_brcond_i64:
1896 case INDEX_op_brcond2_i32:
b7a83ff8
RH		 1897 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
1898 arg_label(op->args[k])->id);
51e3972c
RH		 1899 i++, k++;
1900 break;
1901 default:
1902 break;
1903 }
1904 for (; i < nb_cargs; i++, k++) {
b7a83ff8
RH		 1905 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
1906 op->args[k]);
bdfb460e
RH		 1907 }
1908 }
bdfb460e 1909
1894f69a 1910 if (have_prefs || op->life) {
b7a83ff8
RH		 1911 for (; col < 40; ++col) {
1912 putc(' ', f);
bdfb460e 1913 }
1894f69a
RH		 1914 }
1915
1916 if (op->life) {
1917 unsigned life = op->life;
bdfb460e
RH		 1918
1919 if (life & (SYNC_ARG * 3)) {
b7a83ff8 1920 ne_fprintf(f, " sync:");
bdfb460e
RH		 1921 for (i = 0; i < 2; ++i) {
1922 if (life & (SYNC_ARG << i)) {
b7a83ff8 1923 ne_fprintf(f, " %d", i);
bdfb460e
RH		 1924 }
1925 }
1926 }
1927 life /= DEAD_ARG;
1928 if (life) {
b7a83ff8 1929 ne_fprintf(f, " dead:");
bdfb460e
RH		 1930 for (i = 0; life; ++i, life >>= 1) {
1931 if (life & 1) {
b7a83ff8 1932 ne_fprintf(f, " %d", i);
bdfb460e
RH		 1933 }
1934 }
b03cce8e 1935 }
c896fe29 1936 }
1894f69a
RH		 1937
1938 if (have_prefs) {
1939 for (i = 0; i < nb_oargs; ++i) {
1940 TCGRegSet set = op->output_pref[i];
1941
1942 if (i == 0) {
b7a83ff8 1943 ne_fprintf(f, " pref=");
1894f69a 1944 } else {
b7a83ff8 1945 ne_fprintf(f, ",");
1894f69a
RH		 1946 }
1947 if (set == 0) {
b7a83ff8 1948 ne_fprintf(f, "none");
1894f69a 1949 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
b7a83ff8 1950 ne_fprintf(f, "all");
1894f69a
RH		 1951#ifdef CONFIG_DEBUG_TCG
1952 } else if (tcg_regset_single(set)) {
1953 TCGReg reg = tcg_regset_first(set);
b7a83ff8 1954 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
1894f69a
RH		 1955#endif
1956 } else if (TCG_TARGET_NB_REGS <= 32) {
b7a83ff8 1957 ne_fprintf(f, "0x%x", (uint32_t)set);
1894f69a 1958 } else {
b7a83ff8 1959 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
1894f69a
RH		 1960 }
1961 }
1962 }
1963
b7a83ff8 1964 putc('\n', f);
c896fe29
FB		 1965 }
1966}
1967
1968/* we give more priority to constraints with less registers */
1969static int get_constraint_priority(const TCGOpDef *def, int k)
1970{
74a11790
RH		 1971 const TCGArgConstraint *arg_ct = &def->args_ct[k];
1972 int n;
c896fe29 1973
bc2b17e6 1974 if (arg_ct->oalias) {
c896fe29
FB		 1975 /* an alias is equivalent to a single register */
1976 n = 1;
1977 } else {
74a11790 1978 n = ctpop64(arg_ct->regs);
c896fe29
FB		 1979 }
1980 return TCG_TARGET_NB_REGS - n + 1;
1981}
1982
1983/* sort from highest priority to lowest */
1984static void sort_constraints(TCGOpDef *def, int start, int n)
1985{
66792f90
RH		 1986 int i, j;
1987 TCGArgConstraint *a = def->args_ct;
c896fe29 1988
66792f90
RH		 1989 for (i = 0; i < n; i++) {
1990 a[start + i].sort_index = start + i;
1991 }
1992 if (n <= 1) {
c896fe29 1993 return;
66792f90
RH		 1994 }
1995 for (i = 0; i < n - 1; i++) {
1996 for (j = i + 1; j < n; j++) {
1997 int p1 = get_constraint_priority(def, a[start + i].sort_index);
1998 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 1999 if (p1 < p2) {
66792f90
RH		 2000 int tmp = a[start + i].sort_index;
2001 a[start + i].sort_index = a[start + j].sort_index;
2002 a[start + j].sort_index = tmp;
c896fe29
FB		 2003 }
2004 }
2005 }
2006}
2007
f69d277e 2008static void process_op_defs(TCGContext *s)
c896fe29 2009{
a9751609 2010 TCGOpcode op;
c896fe29 2011
f69d277e
RH		 2012 for (op = 0; op < NB_OPS; op++) {
2013 TCGOpDef *def = &tcg_op_defs[op];
2014 const TCGTargetOpDef *tdefs;
069ea736 2015 int i, nb_args;
f69d277e
RH		 2016
2017 if (def->flags & TCG_OPF_NOT_PRESENT) {
2018 continue;
2019 }
2020
c896fe29 2021 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2022 if (nb_args == 0) {
2023 continue;
2024 }
2025
4c22e840
RH		 2026 /*
2027 * Macro magic should make it impossible, but double-check that
2028 * the array index is in range. Since the signness of an enum
2029 * is implementation defined, force the result to unsigned.
2030 */
2031 unsigned con_set = tcg_target_op_def(op);
2032 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2033 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2034
2035 for (i = 0; i < nb_args; i++) {
2036 const char *ct_str = tdefs->args_ct_str[i];
2037 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2038 tcg_debug_assert(ct_str != NULL);
f69d277e 2039
17280ff4
RH		 2040 while (*ct_str != '\0') {
2041 switch(*ct_str) {
2042 case '0' ... '9':
2043 {
2044 int oarg = *ct_str - '0';
2045 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2046 tcg_debug_assert(oarg < def->nb_oargs);
74a11790 2047 tcg_debug_assert(def->args_ct[oarg].regs != 0);
17280ff4 2048 def->args_ct[i] = def->args_ct[oarg];
bc2b17e6
RH		 2049 /* The output sets oalias. */
2050 def->args_ct[oarg].oalias = true;
17280ff4 2051 def->args_ct[oarg].alias_index = i;
bc2b17e6
RH		 2052 /* The input sets ialias. */
2053 def->args_ct[i].ialias = true;
17280ff4 2054 def->args_ct[i].alias_index = oarg;
c896fe29 2055 }
17280ff4
RH		 2056 ct_str++;
2057 break;
2058 case '&':
bc2b17e6 2059 def->args_ct[i].newreg = true;
17280ff4
RH		 2060 ct_str++;
2061 break;
2062 case 'i':
2063 def->args_ct[i].ct |= TCG_CT_CONST;
2064 ct_str++;
2065 break;
358b4923 2066
358b4923
RH		 2067 /* Include all of the target-specific constraints. */
2068
2069#undef CONST
2070#define CONST(CASE, MASK) \
2071 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2072#define REGS(CASE, MASK) \
2073 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2074
2075#include "tcg-target-con-str.h"
2076
2077#undef REGS
2078#undef CONST
17280ff4 2079 default:
17280ff4 2080 /* Typo in TCGTargetOpDef constraint. */
358b4923 2081 g_assert_not_reached();
c896fe29
FB		 2082 }
2083 }
2084 }
2085
c68aaa18 2086 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2087 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2088
c896fe29
FB		 2089 /* sort the constraints (XXX: this is just an heuristic) */
2090 sort_constraints(def, 0, def->nb_oargs);
2091 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2092 }
c896fe29
FB		 2093}
2094
0c627cdc
RH		 2095void tcg_op_remove(TCGContext *s, TCGOp *op)
2096{
d88a117e
RH		 2097 TCGLabel *label;
2098
2099 switch (op->opc) {
2100 case INDEX_op_br:
2101 label = arg_label(op->args[0]);
2102 label->refs--;
2103 break;
2104 case INDEX_op_brcond_i32:
2105 case INDEX_op_brcond_i64:
2106 label = arg_label(op->args[3]);
2107 label->refs--;
2108 break;
2109 case INDEX_op_brcond2_i32:
2110 label = arg_label(op->args[5]);
2111 label->refs--;
2112 break;
2113 default:
2114 break;
2115 }
2116
15fa08f8
RH		 2117 QTAILQ_REMOVE(&s->ops, op, link);
2118 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2119 s->nb_ops--;
0c627cdc
RH		 2120
2121#ifdef CONFIG_PROFILER
d73415a3 2122 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2123#endif
2124}
2125
a80cdd31
RH		 2126void tcg_remove_ops_after(TCGOp *op)
2127{
2128 TCGContext *s = tcg_ctx;
2129
2130 while (true) {
2131 TCGOp *last = tcg_last_op();
2132 if (last == op) {
2133 return;
2134 }
2135 tcg_op_remove(s, last);
2136 }
2137}
2138
15fa08f8 2139static TCGOp *tcg_op_alloc(TCGOpcode opc)
5a18407f 2140{
15fa08f8
RH		 2141 TCGContext *s = tcg_ctx;
2142 TCGOp *op;
5a18407f 2143
15fa08f8
RH		 2144 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2145 op = tcg_malloc(sizeof(TCGOp));
2146 } else {
2147 op = QTAILQ_FIRST(&s->free_ops);
2148 QTAILQ_REMOVE(&s->free_ops, op, link);
2149 }
2150 memset(op, 0, offsetof(TCGOp, link));
2151 op->opc = opc;
abebf925 2152 s->nb_ops++;
5a18407f 2153
15fa08f8
RH		 2154 return op;
2155}
2156
2157TCGOp *tcg_emit_op(TCGOpcode opc)
2158{
2159 TCGOp *op = tcg_op_alloc(opc);
2160 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2161 return op;
2162}
5a18407f 2163
ac1043f6 2164TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
15fa08f8
RH		 2165{
2166 TCGOp *new_op = tcg_op_alloc(opc);
2167 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2168 return new_op;
2169}
2170
ac1043f6 2171TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
5a18407f 2172{
15fa08f8
RH		 2173 TCGOp *new_op = tcg_op_alloc(opc);
2174 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2175 return new_op;
2176}
2177
b4fc67c7
RH		 2178/* Reachable analysis : remove unreachable code. */
2179static void reachable_code_pass(TCGContext *s)
2180{
2181 TCGOp *op, *op_next;
2182 bool dead = false;
2183
2184 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2185 bool remove = dead;
2186 TCGLabel *label;
b4fc67c7
RH		 2187
2188 switch (op->opc) {
2189 case INDEX_op_set_label:
2190 label = arg_label(op->args[0]);
2191 if (label->refs == 0) {
2192 /*
2193 * While there is an occasional backward branch, virtually
2194 * all branches generated by the translators are forward.
2195 * Which means that generally we will have already removed
2196 * all references to the label that will be, and there is
2197 * little to be gained by iterating.
2198 */
2199 remove = true;
2200 } else {
2201 /* Once we see a label, insns become live again. */
2202 dead = false;
2203 remove = false;
2204
2205 /*
2206 * Optimization can fold conditional branches to unconditional.
2207 * If we find a label with one reference which is preceded by
2208 * an unconditional branch to it, remove both. This needed to
2209 * wait until the dead code in between them was removed.
2210 */
2211 if (label->refs == 1) {
eae3eb3e 2212 TCGOp *op_prev = QTAILQ_PREV(op, link);
b4fc67c7
RH		 2213 if (op_prev->opc == INDEX_op_br &&
2214 label == arg_label(op_prev->args[0])) {
2215 tcg_op_remove(s, op_prev);
2216 remove = true;
2217 }
2218 }
2219 }
2220 break;
2221
2222 case INDEX_op_br:
2223 case INDEX_op_exit_tb:
2224 case INDEX_op_goto_ptr:
2225 /* Unconditional branches; everything following is dead. */
2226 dead = true;
2227 break;
2228
2229 case INDEX_op_call:
2230 /* Notice noreturn helper calls, raising exceptions. */
90163900 2231 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2232 dead = true;
2233 }
2234 break;
2235
2236 case INDEX_op_insn_start:
2237 /* Never remove -- we need to keep these for unwind. */
2238 remove = false;
2239 break;
2240
2241 default:
2242 break;
2243 }
2244
2245 if (remove) {
2246 tcg_op_remove(s, op);
2247 }
2248 }
2249}
2250
c70fbf0a
RH		 2251#define TS_DEAD 1
2252#define TS_MEM 2
2253
5a18407f
RH		 2254#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2255#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2256
25f49c5f
RH		 2257/* For liveness_pass_1, the register preferences for a given temp. */
2258static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2259{
2260 return ts->state_ptr;
2261}
2262
2263/* For liveness_pass_1, reset the preferences for a given temp to the
2264 * maximal regset for its type.
2265 */
2266static inline void la_reset_pref(TCGTemp *ts)
2267{
2268 *la_temp_pref(ts)
2269 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2270}
2271
9c43b68d
AJ		 2272/* liveness analysis: end of function: all temps are dead, and globals
2273 should be in memory. */
2616c808 2274static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2275{
b83eabea
RH		 2276 int i;
2277
2278 for (i = 0; i < ng; ++i) {
2279 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2280 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2281 }
2282 for (i = ng; i < nt; ++i) {
2283 s->temps[i].state = TS_DEAD;
25f49c5f 2284 la_reset_pref(&s->temps[i]);
b83eabea 2285 }
c896fe29
FB		 2286}
2287
9c43b68d
AJ		 2288/* liveness analysis: end of basic block: all temps are dead, globals
2289 and local temps should be in memory. */
2616c808 2290static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2291{
b83eabea 2292 int i;
641d5fbe 2293
ee17db83
RH		 2294 for (i = 0; i < nt; ++i) {
2295 TCGTemp *ts = &s->temps[i];
2296 int state;
2297
2298 switch (ts->kind) {
2299 case TEMP_FIXED:
2300 case TEMP_GLOBAL:
2301 case TEMP_LOCAL:
2302 state = TS_DEAD | TS_MEM;
2303 break;
2304 case TEMP_NORMAL:
c7482438 2305 case TEMP_EBB:
c0522136 2306 case TEMP_CONST:
ee17db83
RH		 2307 state = TS_DEAD;
2308 break;
2309 default:
2310 g_assert_not_reached();
2311 }
2312 ts->state = state;
2313 la_reset_pref(ts);
641d5fbe
FB		 2314 }
2315}
2316
f65a061c
RH		 2317/* liveness analysis: sync globals back to memory. */
2318static void la_global_sync(TCGContext *s, int ng)
2319{
2320 int i;
2321
2322 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2323 int state = s->temps[i].state;
2324 s->temps[i].state = state | TS_MEM;
2325 if (state == TS_DEAD) {
2326 /* If the global was previously dead, reset prefs. */
2327 la_reset_pref(&s->temps[i]);
2328 }
f65a061c
RH		 2329 }
2330}
2331
b4cb76e6 2332/*
c7482438
RH		 2333 * liveness analysis: conditional branch: all temps are dead unless
2334 * explicitly live-across-conditional-branch, globals and local temps
2335 * should be synced.
b4cb76e6
RH		 2336 */
2337static void la_bb_sync(TCGContext *s, int ng, int nt)
2338{
2339 la_global_sync(s, ng);
2340
2341 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2342 TCGTemp *ts = &s->temps[i];
2343 int state;
2344
2345 switch (ts->kind) {
2346 case TEMP_LOCAL:
2347 state = ts->state;
2348 ts->state = state | TS_MEM;
b4cb76e6
RH		 2349 if (state != TS_DEAD) {
2350 continue;
2351 }
c0522136
RH		 2352 break;
2353 case TEMP_NORMAL:
b4cb76e6 2354 s->temps[i].state = TS_DEAD;
c0522136 2355 break;
c7482438 2356 case TEMP_EBB:
c0522136
RH		 2357 case TEMP_CONST:
2358 continue;
2359 default:
2360 g_assert_not_reached();
b4cb76e6
RH		 2361 }
2362 la_reset_pref(&s->temps[i]);
2363 }
2364}
2365
f65a061c
RH		 2366/* liveness analysis: sync globals back to memory and kill. */
2367static void la_global_kill(TCGContext *s, int ng)
2368{
2369 int i;
2370
2371 for (i = 0; i < ng; i++) {
2372 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2373 la_reset_pref(&s->temps[i]);
2374 }
2375}
2376
2377/* liveness analysis: note live globals crossing calls. */
2378static void la_cross_call(TCGContext *s, int nt)
2379{
2380 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2381 int i;
2382
2383 for (i = 0; i < nt; i++) {
2384 TCGTemp *ts = &s->temps[i];
2385 if (!(ts->state & TS_DEAD)) {
2386 TCGRegSet *pset = la_temp_pref(ts);
2387 TCGRegSet set = *pset;
2388
2389 set &= mask;
2390 /* If the combination is not possible, restart. */
2391 if (set == 0) {
2392 set = tcg_target_available_regs[ts->type] & mask;
2393 }
2394 *pset = set;
2395 }
f65a061c
RH		 2396 }
2397}
2398
a1b3c48d 2399/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2400 given input arguments is dead. Instructions updating dead
2401 temporaries are removed. */
b83eabea 2402static void liveness_pass_1(TCGContext *s)
c896fe29 2403{
c70fbf0a 2404 int nb_globals = s->nb_globals;
2616c808 2405 int nb_temps = s->nb_temps;
15fa08f8 2406 TCGOp *op, *op_prev;
25f49c5f
RH		 2407 TCGRegSet *prefs;
2408 int i;
2409
2410 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2411 for (i = 0; i < nb_temps; ++i) {
2412 s->temps[i].state_ptr = prefs + i;
2413 }
a1b3c48d 2414
ae36a246 2415 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2416 la_func_end(s, nb_globals, nb_temps);
c896fe29 2417
eae3eb3e 2418 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2419 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2420 TCGOpcode opc_new, opc_new2;
2421 bool have_opc_new2;
a1b3c48d 2422 TCGLifeData arg_life = 0;
25f49c5f 2423 TCGTemp *ts;
c45cb8bb
RH		 2424 TCGOpcode opc = op->opc;
2425 const TCGOpDef *def = &tcg_op_defs[opc];
2426
c45cb8bb 2427 switch (opc) {
c896fe29 2428 case INDEX_op_call:
c6e113f5
FB		 2429 {
2430 int call_flags;
25f49c5f 2431 int nb_call_regs;
c896fe29 2432
cd9090aa
RH		 2433 nb_oargs = TCGOP_CALLO(op);
2434 nb_iargs = TCGOP_CALLI(op);
90163900 2435 call_flags = tcg_call_flags(op);
c6e113f5 2436
c45cb8bb 2437 /* pure functions can be removed if their result is unused */
78505279 2438 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2439 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2440 ts = arg_temp(op->args[i]);
2441 if (ts->state != TS_DEAD) {
c6e113f5 2442 goto do_not_remove_call;
9c43b68d 2443 }
c6e113f5 2444 }
c45cb8bb 2445 goto do_remove;
152c35aa
RH		 2446 }
2447 do_not_remove_call:
c896fe29 2448
25f49c5f 2449 /* Output args are dead. */
152c35aa 2450 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2451 ts = arg_temp(op->args[i]);
2452 if (ts->state & TS_DEAD) {
152c35aa
RH		 2453 arg_life |= DEAD_ARG << i;
2454 }
25f49c5f 2455 if (ts->state & TS_MEM) {
152c35aa 2456 arg_life |= SYNC_ARG << i;
c6e113f5 2457 }
25f49c5f
RH		 2458 ts->state = TS_DEAD;
2459 la_reset_pref(ts);
2460
2461 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2462 op->output_pref[i] = 0;
152c35aa 2463 }
78505279 2464
152c35aa
RH		 2465 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2466 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2467 la_global_kill(s, nb_globals);
152c35aa 2468 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2469 la_global_sync(s, nb_globals);
152c35aa 2470 }
b9c18f56 2471
25f49c5f 2472 /* Record arguments that die in this helper. */
152c35aa 2473 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f
RH		 2474 ts = arg_temp(op->args[i]);
2475 if (ts && ts->state & TS_DEAD) {
152c35aa 2476 arg_life |= DEAD_ARG << i;
c6e113f5 2477 }
152c35aa 2478 }
25f49c5f
RH		 2479
2480 /* For all live registers, remove call-clobbered prefs. */
2481 la_cross_call(s, nb_temps);
2482
2483 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2484
2485 /* Input arguments are live for preceding opcodes. */
2486 for (i = 0; i < nb_iargs; i++) {
2487 ts = arg_temp(op->args[i + nb_oargs]);
2488 if (ts && ts->state & TS_DEAD) {
2489 /* For those arguments that die, and will be allocated
2490 * in registers, clear the register set for that arg,
2491 * to be filled in below. For args that will be on
2492 * the stack, reset to any available reg.
2493 */
2494 *la_temp_pref(ts)
2495 = (i < nb_call_regs ? 0 :
2496 tcg_target_available_regs[ts->type]);
2497 ts->state &= ~TS_DEAD;
2498 }
2499 }
2500
2501 /* For each input argument, add its input register to prefs.
2502 If a temp is used once, this produces a single set bit. */
2503 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2504 ts = arg_temp(op->args[i + nb_oargs]);
2505 if (ts) {
2506 tcg_regset_set_reg(*la_temp_pref(ts),
2507 tcg_target_call_iarg_regs[i]);
c19f47bf 2508 }
c896fe29 2509 }
c896fe29 2510 }
c896fe29 2511 break;
765b842a 2512 case INDEX_op_insn_start:
c896fe29 2513 break;
5ff9d6a4 2514 case INDEX_op_discard:
5ff9d6a4 2515 /* mark the temporary as dead */
25f49c5f
RH		 2516 ts = arg_temp(op->args[0]);
2517 ts->state = TS_DEAD;
2518 la_reset_pref(ts);
5ff9d6a4 2519 break;
1305c451
RH		 2520
2521 case INDEX_op_add2_i32:
c45cb8bb 2522 opc_new = INDEX_op_add_i32;
f1fae40c 2523 goto do_addsub2;
1305c451 2524 case INDEX_op_sub2_i32:
c45cb8bb 2525 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 2526 goto do_addsub2;
2527 case INDEX_op_add2_i64:
c45cb8bb 2528 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 2529 goto do_addsub2;
2530 case INDEX_op_sub2_i64:
c45cb8bb 2531 opc_new = INDEX_op_sub_i64;
f1fae40c 2532 do_addsub2:
1305c451
RH		 2533 nb_iargs = 4;
2534 nb_oargs = 2;
2535 /* Test if the high part of the operation is dead, but not
2536 the low part. The result can be optimized to a simple
2537 add or sub. This happens often for x86_64 guest when the
2538 cpu mode is set to 32 bit. */
b83eabea
RH		 2539 if (arg_temp(op->args[1])->state == TS_DEAD) {
2540 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 2541 goto do_remove;
2542 }
c45cb8bb
RH		 2543 /* Replace the opcode and adjust the args in place,
2544 leaving 3 unused args at the end. */
2545 op->opc = opc = opc_new;
efee3746
RH		 2546 op->args[1] = op->args[2];
2547 op->args[2] = op->args[4];
1305c451
RH		 2548 /* Fall through and mark the single-word operation live. */
2549 nb_iargs = 2;
2550 nb_oargs = 1;
2551 }
2552 goto do_not_remove;
2553
1414968a 2554 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 2555 opc_new = INDEX_op_mul_i32;
2556 opc_new2 = INDEX_op_muluh_i32;
2557 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 2558 goto do_mul2;
f1fae40c 2559 case INDEX_op_muls2_i32:
c45cb8bb
RH		 2560 opc_new = INDEX_op_mul_i32;
2561 opc_new2 = INDEX_op_mulsh_i32;
2562 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 2563 goto do_mul2;
2564 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 2565 opc_new = INDEX_op_mul_i64;
2566 opc_new2 = INDEX_op_muluh_i64;
2567 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 2568 goto do_mul2;
f1fae40c 2569 case INDEX_op_muls2_i64:
c45cb8bb
RH		 2570 opc_new = INDEX_op_mul_i64;
2571 opc_new2 = INDEX_op_mulsh_i64;
2572 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 2573 goto do_mul2;
f1fae40c 2574 do_mul2:
1414968a
RH		 2575 nb_iargs = 2;
2576 nb_oargs = 2;
b83eabea
RH		 2577 if (arg_temp(op->args[1])->state == TS_DEAD) {
2578 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 2579 /* Both parts of the operation are dead. */
1414968a
RH		 2580 goto do_remove;
2581 }
03271524 2582 /* The high part of the operation is dead; generate the low. */
c45cb8bb 2583 op->opc = opc = opc_new;
efee3746
RH		 2584 op->args[1] = op->args[2];
2585 op->args[2] = op->args[3];
b83eabea 2586 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 2587 /* The low part of the operation is dead; generate the high. */
2588 op->opc = opc = opc_new2;
efee3746
RH		 2589 op->args[0] = op->args[1];
2590 op->args[1] = op->args[2];
2591 op->args[2] = op->args[3];
03271524
RH		 2592 } else {
2593 goto do_not_remove;
1414968a 2594 }
03271524
RH		 2595 /* Mark the single-word operation live. */
2596 nb_oargs = 1;
1414968a
RH		 2597 goto do_not_remove;
2598
c896fe29 2599 default:
1305c451 2600 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 2601 nb_iargs = def->nb_iargs;
2602 nb_oargs = def->nb_oargs;
c896fe29 2603
49516bc0
AJ		 2604 /* Test if the operation can be removed because all
2605 its outputs are dead. We assume that nb_oargs == 0
2606 implies side effects */
2607 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 2608 for (i = 0; i < nb_oargs; i++) {
b83eabea 2609 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 2610 goto do_not_remove;
9c43b68d 2611 }
49516bc0 2612 }
152c35aa
RH		 2613 goto do_remove;
2614 }
2615 goto do_not_remove;
49516bc0 2616
152c35aa
RH		 2617 do_remove:
2618 tcg_op_remove(s, op);
2619 break;
2620
2621 do_not_remove:
152c35aa 2622 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2623 ts = arg_temp(op->args[i]);
2624
2625 /* Remember the preference of the uses that followed. */
2626 op->output_pref[i] = *la_temp_pref(ts);
2627
2628 /* Output args are dead. */
2629 if (ts->state & TS_DEAD) {
152c35aa 2630 arg_life |= DEAD_ARG << i;
49516bc0 2631 }
25f49c5f 2632 if (ts->state & TS_MEM) {
152c35aa
RH		 2633 arg_life |= SYNC_ARG << i;
2634 }
25f49c5f
RH		 2635 ts->state = TS_DEAD;
2636 la_reset_pref(ts);
152c35aa 2637 }
49516bc0 2638
25f49c5f 2639 /* If end of basic block, update. */
ae36a246
RH		 2640 if (def->flags & TCG_OPF_BB_EXIT) {
2641 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 2642 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2643 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 2644 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 2645 la_bb_end(s, nb_globals, nb_temps);
152c35aa 2646 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 2647 la_global_sync(s, nb_globals);
25f49c5f
RH		 2648 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2649 la_cross_call(s, nb_temps);
2650 }
152c35aa
RH		 2651 }
2652
25f49c5f 2653 /* Record arguments that die in this opcode. */
152c35aa 2654 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2655 ts = arg_temp(op->args[i]);
2656 if (ts->state & TS_DEAD) {
152c35aa 2657 arg_life |= DEAD_ARG << i;
c896fe29 2658 }
c896fe29 2659 }
25f49c5f
RH		 2660
2661 /* Input arguments are live for preceding opcodes. */
152c35aa 2662 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2663 ts = arg_temp(op->args[i]);
2664 if (ts->state & TS_DEAD) {
2665 /* For operands that were dead, initially allow
2666 all regs for the type. */
2667 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2668 ts->state &= ~TS_DEAD;
2669 }
2670 }
2671
2672 /* Incorporate constraints for this operand. */
2673 switch (opc) {
2674 case INDEX_op_mov_i32:
2675 case INDEX_op_mov_i64:
2676 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2677 have proper constraints. That said, special case
2678 moves to propagate preferences backward. */
2679 if (IS_DEAD_ARG(1)) {
2680 *la_temp_pref(arg_temp(op->args[0]))
2681 = *la_temp_pref(arg_temp(op->args[1]));
2682 }
2683 break;
2684
2685 default:
2686 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2687 const TCGArgConstraint *ct = &def->args_ct[i];
2688 TCGRegSet set, *pset;
2689
2690 ts = arg_temp(op->args[i]);
2691 pset = la_temp_pref(ts);
2692 set = *pset;
2693
9be0d080 2694 set &= ct->regs;
bc2b17e6 2695 if (ct->ialias) {
25f49c5f
RH		 2696 set &= op->output_pref[ct->alias_index];
2697 }
2698 /* If the combination is not possible, restart. */
2699 if (set == 0) {
9be0d080 2700 set = ct->regs;
25f49c5f
RH		 2701 }
2702 *pset = set;
2703 }
2704 break;
152c35aa 2705 }
c896fe29
FB		 2706 break;
2707 }
bee158cb 2708 op->life = arg_life;
1ff0a2c5 2709 }
c896fe29 2710}
c896fe29 2711
5a18407f 2712/* Liveness analysis: Convert indirect regs to direct temporaries. */
b83eabea 2713static bool liveness_pass_2(TCGContext *s)
5a18407f
RH		 2714{
2715 int nb_globals = s->nb_globals;
15fa08f8 2716 int nb_temps, i;
5a18407f 2717 bool changes = false;
15fa08f8 2718 TCGOp *op, *op_next;
5a18407f 2719
5a18407f
RH		 2720 /* Create a temporary for each indirect global. */
2721 for (i = 0; i < nb_globals; ++i) {
2722 TCGTemp *its = &s->temps[i];
2723 if (its->indirect_reg) {
2724 TCGTemp *dts = tcg_temp_alloc(s);
2725 dts->type = its->type;
2726 dts->base_type = its->base_type;
c7482438 2727 dts->kind = TEMP_EBB;
b83eabea
RH		 2728 its->state_ptr = dts;
2729 } else {
2730 its->state_ptr = NULL;
5a18407f 2731 }
b83eabea
RH		 2732 /* All globals begin dead. */
2733 its->state = TS_DEAD;
2734 }
2735 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2736 TCGTemp *its = &s->temps[i];
2737 its->state_ptr = NULL;
2738 its->state = TS_DEAD;
5a18407f 2739 }
5a18407f 2740
15fa08f8 2741 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 2742 TCGOpcode opc = op->opc;
2743 const TCGOpDef *def = &tcg_op_defs[opc];
2744 TCGLifeData arg_life = op->life;
2745 int nb_iargs, nb_oargs, call_flags;
b83eabea 2746 TCGTemp *arg_ts, *dir_ts;
5a18407f 2747
5a18407f 2748 if (opc == INDEX_op_call) {
cd9090aa
RH		 2749 nb_oargs = TCGOP_CALLO(op);
2750 nb_iargs = TCGOP_CALLI(op);
90163900 2751 call_flags = tcg_call_flags(op);
5a18407f
RH		 2752 } else {
2753 nb_iargs = def->nb_iargs;
2754 nb_oargs = def->nb_oargs;
2755
2756 /* Set flags similar to how calls require. */
b4cb76e6
RH		 2757 if (def->flags & TCG_OPF_COND_BRANCH) {
2758 /* Like reading globals: sync_globals */
2759 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2760 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 2761 /* Like writing globals: save_globals */
2762 call_flags = 0;
2763 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2764 /* Like reading globals: sync_globals */
2765 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2766 } else {
2767 /* No effect on globals. */
2768 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2769 TCG_CALL_NO_WRITE_GLOBALS);
2770 }
2771 }
2772
2773 /* Make sure that input arguments are available. */
2774 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2775 arg_ts = arg_temp(op->args[i]);
2776 if (arg_ts) {
2777 dir_ts = arg_ts->state_ptr;
2778 if (dir_ts && arg_ts->state == TS_DEAD) {
2779 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
5a18407f
RH		 2780 ? INDEX_op_ld_i32
2781 : INDEX_op_ld_i64);
ac1043f6 2782 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
5a18407f 2783
b83eabea
RH		 2784 lop->args[0] = temp_arg(dir_ts);
2785 lop->args[1] = temp_arg(arg_ts->mem_base);
2786 lop->args[2] = arg_ts->mem_offset;
5a18407f
RH		 2787
2788 /* Loaded, but synced with memory. */
b83eabea 2789 arg_ts->state = TS_MEM;
5a18407f
RH		 2790 }
2791 }
2792 }
2793
2794 /* Perform input replacement, and mark inputs that became dead.
2795 No action is required except keeping temp_state up to date
2796 so that we reload when needed. */
2797 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2798 arg_ts = arg_temp(op->args[i]);
2799 if (arg_ts) {
2800 dir_ts = arg_ts->state_ptr;
2801 if (dir_ts) {
2802 op->args[i] = temp_arg(dir_ts);
5a18407f
RH		 2803 changes = true;
2804 if (IS_DEAD_ARG(i)) {
b83eabea 2805 arg_ts->state = TS_DEAD;
5a18407f
RH		 2806 }
2807 }
2808 }
2809 }
2810
2811 /* Liveness analysis should ensure that the following are
2812 all correct, for call sites and basic block end points. */
2813 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2814 /* Nothing to do */
2815 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2816 for (i = 0; i < nb_globals; ++i) {
2817 /* Liveness should see that globals are synced back,
2818 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 2819 arg_ts = &s->temps[i];
2820 tcg_debug_assert(arg_ts->state_ptr == 0
2821 || arg_ts->state != 0);
5a18407f
RH		 2822 }
2823 } else {
2824 for (i = 0; i < nb_globals; ++i) {
2825 /* Liveness should see that globals are saved back,
2826 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 2827 arg_ts = &s->temps[i];
2828 tcg_debug_assert(arg_ts->state_ptr == 0
2829 || arg_ts->state == TS_DEAD);
5a18407f
RH		 2830 }
2831 }
2832
2833 /* Outputs become available. */
61f15c48
RH		 2834 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2835 arg_ts = arg_temp(op->args[0]);
b83eabea 2836 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 2837 if (dir_ts) {
2838 op->args[0] = temp_arg(dir_ts);
2839 changes = true;
2840
2841 /* The output is now live and modified. */
2842 arg_ts->state = 0;
2843
2844 if (NEED_SYNC_ARG(0)) {
2845 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2846 ? INDEX_op_st_i32
2847 : INDEX_op_st_i64);
2848 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2849 TCGTemp *out_ts = dir_ts;
2850
2851 if (IS_DEAD_ARG(0)) {
2852 out_ts = arg_temp(op->args[1]);
2853 arg_ts->state = TS_DEAD;
2854 tcg_op_remove(s, op);
2855 } else {
2856 arg_ts->state = TS_MEM;
2857 }
2858
2859 sop->args[0] = temp_arg(out_ts);
2860 sop->args[1] = temp_arg(arg_ts->mem_base);
2861 sop->args[2] = arg_ts->mem_offset;
2862 } else {
2863 tcg_debug_assert(!IS_DEAD_ARG(0));
2864 }
5a18407f 2865 }
61f15c48
RH		 2866 } else {
2867 for (i = 0; i < nb_oargs; i++) {
2868 arg_ts = arg_temp(op->args[i]);
2869 dir_ts = arg_ts->state_ptr;
2870 if (!dir_ts) {
2871 continue;
2872 }
2873 op->args[i] = temp_arg(dir_ts);
2874 changes = true;
5a18407f 2875
61f15c48
RH		 2876 /* The output is now live and modified. */
2877 arg_ts->state = 0;
5a18407f 2878
61f15c48
RH		 2879 /* Sync outputs upon their last write. */
2880 if (NEED_SYNC_ARG(i)) {
2881 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2882 ? INDEX_op_st_i32
2883 : INDEX_op_st_i64);
2884 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
5a18407f 2885
61f15c48
RH		 2886 sop->args[0] = temp_arg(dir_ts);
2887 sop->args[1] = temp_arg(arg_ts->mem_base);
2888 sop->args[2] = arg_ts->mem_offset;
5a18407f 2889
61f15c48
RH		 2890 arg_ts->state = TS_MEM;
2891 }
2892 /* Drop outputs that are dead. */
2893 if (IS_DEAD_ARG(i)) {
2894 arg_ts->state = TS_DEAD;
2895 }
5a18407f
RH		 2896 }
2897 }
2898 }
2899
2900 return changes;
2901}
2902
2272e4a7 2903static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 2904{
c1c09194
RH		 2905 intptr_t off, size, align;
2906
2907 switch (ts->type) {
2908 case TCG_TYPE_I32:
2909 size = align = 4;
2910 break;
2911 case TCG_TYPE_I64:
2912 case TCG_TYPE_V64:
2913 size = align = 8;
2914 break;
2915 case TCG_TYPE_V128:
2916 size = align = 16;
2917 break;
2918 case TCG_TYPE_V256:
2919 /* Note that we do not require aligned storage for V256. */
2920 size = 32, align = 16;
2921 break;
2922 default:
2923 g_assert_not_reached();
b591dc59 2924 }
c1c09194 2925
b9537d59
RH		 2926 /*
2927 * Assume the stack is sufficiently aligned.
2928 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
2929 * and do not require 16 byte vector alignment. This seems slightly
2930 * easier than fully parameterizing the above switch statement.
2931 */
2932 align = MIN(TCG_TARGET_STACK_ALIGN, align);
c1c09194 2933 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 2934
2935 /* If we've exhausted the stack frame, restart with a smaller TB. */
2936 if (off + size > s->frame_end) {
2937 tcg_raise_tb_overflow(s);
2938 }
c1c09194
RH		 2939 s->current_frame_offset = off + size;
2940
2941 ts->mem_offset = off;
9defd1bd
RH		 2942#if defined(__sparc__)
2943 ts->mem_offset += TCG_TARGET_STACK_BIAS;
2944#endif
b3a62939 2945 ts->mem_base = s->frame_temp;
c896fe29 2946 ts->mem_allocated = 1;
c896fe29
FB		 2947}
2948
098859f1
RH		 2949/* Assign @reg to @ts, and update reg_to_temp[]. */
2950static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
2951{
2952 if (ts->val_type == TEMP_VAL_REG) {
2953 TCGReg old = ts->reg;
2954 tcg_debug_assert(s->reg_to_temp[old] == ts);
2955 if (old == reg) {
2956 return;
2957 }
2958 s->reg_to_temp[old] = NULL;
2959 }
2960 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
2961 s->reg_to_temp[reg] = ts;
2962 ts->val_type = TEMP_VAL_REG;
2963 ts->reg = reg;
2964}
2965
2966/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
2967static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
2968{
2969 tcg_debug_assert(type != TEMP_VAL_REG);
2970 if (ts->val_type == TEMP_VAL_REG) {
2971 TCGReg reg = ts->reg;
2972 tcg_debug_assert(s->reg_to_temp[reg] == ts);
2973 s->reg_to_temp[reg] = NULL;
2974 }
2975 ts->val_type = type;
2976}
2977
b722452a 2978static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 2979
59d7c14e
RH		 2980/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
2981 mark it free; otherwise mark it dead. */
2982static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 2983{
c0522136
RH		 2984 TCGTempVal new_type;
2985
2986 switch (ts->kind) {
2987 case TEMP_FIXED:
59d7c14e 2988 return;
c0522136
RH		 2989 case TEMP_GLOBAL:
2990 case TEMP_LOCAL:
2991 new_type = TEMP_VAL_MEM;
2992 break;
2993 case TEMP_NORMAL:
c7482438 2994 case TEMP_EBB:
c0522136
RH		 2995 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
2996 break;
2997 case TEMP_CONST:
2998 new_type = TEMP_VAL_CONST;
2999 break;
3000 default:
3001 g_assert_not_reached();
59d7c14e 3002 }
098859f1 3003 set_temp_val_nonreg(s, ts, new_type);
59d7c14e 3004}
7f6ceedf 3005
59d7c14e
RH		 3006/* Mark a temporary as dead. */
3007static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3008{
3009 temp_free_or_dead(s, ts, 1);
3010}
3011
3012/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3013 registers needs to be allocated to store a constant. If 'free_or_dead'
3014 is non-zero, subsequently release the temporary; if it is positive, the
3015 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3016static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3017 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3018{
c0522136 3019 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3020 if (!ts->mem_allocated) {
2272e4a7 3021 temp_allocate_frame(s, ts);
59d7c14e 3022 }
59d7c14e
RH		 3023 switch (ts->val_type) {
3024 case TEMP_VAL_CONST:
3025 /* If we're going to free the temp immediately, then we won't
3026 require it later in a register, so attempt to store the
3027 constant to memory directly. */
3028 if (free_or_dead
3029 && tcg_out_sti(s, ts->type, ts->val,
3030 ts->mem_base->reg, ts->mem_offset)) {
3031 break;
3032 }
3033 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3034 allocated_regs, preferred_regs);
59d7c14e
RH		 3035 /* fallthrough */
3036
3037 case TEMP_VAL_REG:
3038 tcg_out_st(s, ts->type, ts->reg,
3039 ts->mem_base->reg, ts->mem_offset);
3040 break;
3041
3042 case TEMP_VAL_MEM:
3043 break;
3044
3045 case TEMP_VAL_DEAD:
3046 default:
3047 tcg_abort();
3048 }
3049 ts->mem_coherent = 1;
3050 }
3051 if (free_or_dead) {
3052 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3053 }
7f6ceedf
AJ		 3054}
3055
c896fe29 3056/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3057static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3058{
f8b2f202 3059 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3060 if (ts != NULL) {
98b4e186 3061 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3062 }
3063}
3064
b016486e
RH		 3065/**
3066 * tcg_reg_alloc:
3067 * @required_regs: Set of registers in which we must allocate.
3068 * @allocated_regs: Set of registers which must be avoided.
3069 * @preferred_regs: Set of registers we should prefer.
3070 * @rev: True if we search the registers in "indirect" order.
3071 *
3072 * The allocated register must be in @required_regs & ~@allocated_regs,
3073 * but if we can put it in @preferred_regs we may save a move later.
3074 */
3075static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3076 TCGRegSet allocated_regs,
3077 TCGRegSet preferred_regs, bool rev)
c896fe29 3078{
b016486e
RH		 3079 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3080 TCGRegSet reg_ct[2];
91478cef 3081 const int *order;
c896fe29 3082
b016486e
RH		 3083 reg_ct[1] = required_regs & ~allocated_regs;
3084 tcg_debug_assert(reg_ct[1] != 0);
3085 reg_ct[0] = reg_ct[1] & preferred_regs;
3086
3087 /* Skip the preferred_regs option if it cannot be satisfied,
3088 or if the preference made no difference. */
3089 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3090
91478cef 3091 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3092
b016486e
RH		 3093 /* Try free registers, preferences first. */
3094 for (j = f; j < 2; j++) {
3095 TCGRegSet set = reg_ct[j];
3096
3097 if (tcg_regset_single(set)) {
3098 /* One register in the set. */
3099 TCGReg reg = tcg_regset_first(set);
3100 if (s->reg_to_temp[reg] == NULL) {
3101 return reg;
3102 }
3103 } else {
3104 for (i = 0; i < n; i++) {
3105 TCGReg reg = order[i];
3106 if (s->reg_to_temp[reg] == NULL &&
3107 tcg_regset_test_reg(set, reg)) {
3108 return reg;
3109 }
3110 }
3111 }
c896fe29
FB		 3112 }
3113
b016486e
RH		 3114 /* We must spill something. */
3115 for (j = f; j < 2; j++) {
3116 TCGRegSet set = reg_ct[j];
3117
3118 if (tcg_regset_single(set)) {
3119 /* One register in the set. */
3120 TCGReg reg = tcg_regset_first(set);
b3915dbb 3121 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3122 return reg;
b016486e
RH		 3123 } else {
3124 for (i = 0; i < n; i++) {
3125 TCGReg reg = order[i];
3126 if (tcg_regset_test_reg(set, reg)) {
3127 tcg_reg_free(s, reg, allocated_regs);
3128 return reg;
3129 }
3130 }
c896fe29
FB		 3131 }
3132 }
3133
3134 tcg_abort();
3135}
3136
40ae5c62
RH		 3137/* Make sure the temporary is in a register. If needed, allocate the register
3138 from DESIRED while avoiding ALLOCATED. */
3139static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3140 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3141{
3142 TCGReg reg;
3143
3144 switch (ts->val_type) {
3145 case TEMP_VAL_REG:
3146 return;
3147 case TEMP_VAL_CONST:
b016486e 3148 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3149 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3150 if (ts->type <= TCG_TYPE_I64) {
3151 tcg_out_movi(s, ts->type, reg, ts->val);
3152 } else {
4e186175
RH		 3153 uint64_t val = ts->val;
3154 MemOp vece = MO_64;
3155
3156 /*
3157 * Find the minimal vector element that matches the constant.
3158 * The targets will, in general, have to do this search anyway,
3159 * do this generically.
3160 */
4e186175
RH		 3161 if (val == dup_const(MO_8, val)) {
3162 vece = MO_8;
3163 } else if (val == dup_const(MO_16, val)) {
3164 vece = MO_16;
0b4286dd 3165 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3166 vece = MO_32;
3167 }
3168
3169 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3170 }
40ae5c62
RH		 3171 ts->mem_coherent = 0;
3172 break;
3173 case TEMP_VAL_MEM:
b016486e 3174 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3175 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3176 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3177 ts->mem_coherent = 1;
3178 break;
3179 case TEMP_VAL_DEAD:
3180 default:
3181 tcg_abort();
3182 }
098859f1 3183 set_temp_val_reg(s, ts, reg);
40ae5c62
RH		 3184}
3185
59d7c14e
RH		 3186/* Save a temporary to memory. 'allocated_regs' is used in case a
3187 temporary registers needs to be allocated to store a constant. */
3188static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3189{
5a18407f
RH		 3190 /* The liveness analysis already ensures that globals are back
3191 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3192 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3193}
3194
9814dd27 3195/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3196 modified be the following code. 'allocated_regs' is used in case a
3197 temporary registers needs to be allocated to store a constant. */
3198static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3199{
ac3b8891 3200 int i, n;
c896fe29 3201
ac3b8891 3202 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3203 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3204 }
e5097dc8
FB		 3205}
3206
3d5c5f87
AJ		 3207/* sync globals to their canonical location and assume they can be
3208 read by the following code. 'allocated_regs' is used in case a
3209 temporary registers needs to be allocated to store a constant. */
3210static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3211{
ac3b8891 3212 int i, n;
3d5c5f87 3213
ac3b8891 3214 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3215 TCGTemp *ts = &s->temps[i];
5a18407f 3216 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3217 || ts->kind == TEMP_FIXED
5a18407f 3218 || ts->mem_coherent);
3d5c5f87
AJ		 3219 }
3220}
3221
e5097dc8 3222/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3223 all globals are stored at their canonical location. */
3224static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3225{
e5097dc8
FB		 3226 int i;
3227
b13eb728
RH		 3228 for (i = s->nb_globals; i < s->nb_temps; i++) {
3229 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3230
3231 switch (ts->kind) {
3232 case TEMP_LOCAL:
b13eb728 3233 temp_save(s, ts, allocated_regs);
c0522136
RH		 3234 break;
3235 case TEMP_NORMAL:
c7482438 3236 case TEMP_EBB:
5a18407f
RH		 3237 /* The liveness analysis already ensures that temps are dead.
3238 Keep an tcg_debug_assert for safety. */
3239 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3240 break;
3241 case TEMP_CONST:
3242 /* Similarly, we should have freed any allocated register. */
3243 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3244 break;
3245 default:
3246 g_assert_not_reached();
c896fe29
FB		 3247 }
3248 }
e8996ee0
FB		 3249
3250 save_globals(s, allocated_regs);
c896fe29
FB		 3251}
3252
b4cb76e6 3253/*
c7482438
RH		 3254 * At a conditional branch, we assume all temporaries are dead unless
3255 * explicitly live-across-conditional-branch; all globals and local
3256 * temps are synced to their location.
b4cb76e6
RH		 3257 */
3258static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3259{
3260 sync_globals(s, allocated_regs);
3261
3262 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3263 TCGTemp *ts = &s->temps[i];
3264 /*
3265 * The liveness analysis already ensures that temps are dead.
3266 * Keep tcg_debug_asserts for safety.
3267 */
c0522136
RH		 3268 switch (ts->kind) {
3269 case TEMP_LOCAL:
b4cb76e6 3270 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136
RH		 3271 break;
3272 case TEMP_NORMAL:
b4cb76e6 3273 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136 3274 break;
c7482438 3275 case TEMP_EBB:
c0522136
RH		 3276 case TEMP_CONST:
3277 break;
3278 default:
3279 g_assert_not_reached();
b4cb76e6
RH		 3280 }
3281 }
3282}
3283
bab1671f 3284/*
c58f4c97 3285 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3286 */
0fe4fca4 3287static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3288 tcg_target_ulong val, TCGLifeData arg_life,
3289 TCGRegSet preferred_regs)
e8996ee0 3290{
d63e3b6e 3291 /* ENV should not be modified. */
e01fa97d 3292 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3293
3294 /* The movi is not explicitly generated here. */
098859f1 3295 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
59d7c14e
RH		 3296 ots->val = val;
3297 ots->mem_coherent = 0;
3298 if (NEED_SYNC_ARG(0)) {
ba87719c 3299 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3300 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3301 temp_dead(s, ots);
4c4e1ab2 3302 }
e8996ee0
FB		 3303}
3304
bab1671f
RH		 3305/*
3306 * Specialized code generation for INDEX_op_mov_*.
3307 */
dd186292 3308static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3309{
dd186292 3310 const TCGLifeData arg_life = op->life;
69e3706d 3311 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3312 TCGTemp *ts, *ots;
450445d5 3313 TCGType otype, itype;
098859f1 3314 TCGReg oreg, ireg;
c896fe29 3315
d21369f5 3316 allocated_regs = s->reserved_regs;
69e3706d 3317 preferred_regs = op->output_pref[0];
43439139
RH		 3318 ots = arg_temp(op->args[0]);
3319 ts = arg_temp(op->args[1]);
450445d5 3320
d63e3b6e 3321 /* ENV should not be modified. */
e01fa97d 3322 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3323
450445d5
RH		 3324 /* Note that otype != itype for no-op truncation. */
3325 otype = ots->type;
3326 itype = ts->type;
c29c1d7e 3327
0fe4fca4
PB		 3328 if (ts->val_type == TEMP_VAL_CONST) {
3329 /* propagate constant or generate sti */
3330 tcg_target_ulong val = ts->val;
3331 if (IS_DEAD_ARG(1)) {
3332 temp_dead(s, ts);
3333 }
69e3706d 3334 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3335 return;
3336 }
3337
3338 /* If the source value is in memory we're going to be forced
3339 to have it in a register in order to perform the copy. Copy
3340 the SOURCE value into its own register first, that way we
3341 don't have to reload SOURCE the next time it is used. */
3342 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3343 temp_load(s, ts, tcg_target_available_regs[itype],
3344 allocated_regs, preferred_regs);
c29c1d7e 3345 }
0fe4fca4 3346 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
098859f1
RH		 3347 ireg = ts->reg;
3348
d63e3b6e 3349 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3350 /* mov to a non-saved dead register makes no sense (even with
3351 liveness analysis disabled). */
eabb7b91 3352 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3353 if (!ots->mem_allocated) {
2272e4a7 3354 temp_allocate_frame(s, ots);
c29c1d7e 3355 }
098859f1 3356 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3357 if (IS_DEAD_ARG(1)) {
f8bf00f1 3358 temp_dead(s, ts);
c29c1d7e 3359 }
f8bf00f1 3360 temp_dead(s, ots);
098859f1
RH		 3361 return;
3362 }
3363
3364 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
3365 /*
3366 * The mov can be suppressed. Kill input first, so that it
3367 * is unlinked from reg_to_temp, then set the output to the
3368 * reg that we saved from the input.
3369 */
3370 temp_dead(s, ts);
3371 oreg = ireg;
c29c1d7e 3372 } else {
098859f1
RH		 3373 if (ots->val_type == TEMP_VAL_REG) {
3374 oreg = ots->reg;
c896fe29 3375 } else {
098859f1
RH		 3376 /* Make sure to not spill the input register during allocation. */
3377 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
3378 allocated_regs | ((TCGRegSet)1 << ireg),
3379 preferred_regs, ots->indirect_base);
c896fe29 3380 }
098859f1
RH		 3381 if (!tcg_out_mov(s, otype, oreg, ireg)) {
3382 /*
3383 * Cross register class move not supported.
3384 * Store the source register into the destination slot
3385 * and leave the destination temp as TEMP_VAL_MEM.
3386 */
3387 assert(!temp_readonly(ots));
3388 if (!ts->mem_allocated) {
3389 temp_allocate_frame(s, ots);
3390 }
3391 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
3392 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
3393 ots->mem_coherent = 1;
3394 return;
c896fe29 3395 }
ec7a869d 3396 }
098859f1
RH		 3397 set_temp_val_reg(s, ots, oreg);
3398 ots->mem_coherent = 0;
3399
3400 if (NEED_SYNC_ARG(0)) {
3401 temp_sync(s, ots, allocated_regs, 0, 0);
3402 }
c896fe29
FB		 3403}
3404
bab1671f
RH		 3405/*
3406 * Specialized code generation for INDEX_op_dup_vec.
3407 */
3408static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3409{
3410 const TCGLifeData arg_life = op->life;
3411 TCGRegSet dup_out_regs, dup_in_regs;
3412 TCGTemp *its, *ots;
3413 TCGType itype, vtype;
d6ecb4a9 3414 intptr_t endian_fixup;
bab1671f
RH		 3415 unsigned vece;
3416 bool ok;
3417
3418 ots = arg_temp(op->args[0]);
3419 its = arg_temp(op->args[1]);
3420
3421 /* ENV should not be modified. */
e01fa97d 3422 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 3423
3424 itype = its->type;
3425 vece = TCGOP_VECE(op);
3426 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3427
3428 if (its->val_type == TEMP_VAL_CONST) {
3429 /* Propagate constant via movi -> dupi. */
3430 tcg_target_ulong val = its->val;
3431 if (IS_DEAD_ARG(1)) {
3432 temp_dead(s, its);
3433 }
3434 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3435 return;
3436 }
3437
9be0d080
RH		 3438 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3439 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 3440
3441 /* Allocate the output register now. */
3442 if (ots->val_type != TEMP_VAL_REG) {
3443 TCGRegSet allocated_regs = s->reserved_regs;
098859f1 3444 TCGReg oreg;
bab1671f
RH		 3445
3446 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3447 /* Make sure to not spill the input register. */
3448 tcg_regset_set_reg(allocated_regs, its->reg);
3449 }
098859f1
RH		 3450 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3451 op->output_pref[0], ots->indirect_base);
3452 set_temp_val_reg(s, ots, oreg);
bab1671f
RH		 3453 }
3454
3455 switch (its->val_type) {
3456 case TEMP_VAL_REG:
3457 /*
3458 * The dup constriaints must be broad, covering all possible VECE.
3459 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3460 * to fail, indicating that extra moves are required for that case.
3461 */
3462 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3463 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3464 goto done;
3465 }
3466 /* Try again from memory or a vector input register. */
3467 }
3468 if (!its->mem_coherent) {
3469 /*
3470 * The input register is not synced, and so an extra store
3471 * would be required to use memory. Attempt an integer-vector
3472 * register move first. We do not have a TCGRegSet for this.
3473 */
3474 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3475 break;
3476 }
3477 /* Sync the temp back to its slot and load from there. */
3478 temp_sync(s, its, s->reserved_regs, 0, 0);
3479 }
3480 /* fall through */
3481
3482 case TEMP_VAL_MEM:
e03b5686 3483#if HOST_BIG_ENDIAN
d6ecb4a9
RH		 3484 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3485 endian_fixup -= 1 << vece;
3486#else
3487 endian_fixup = 0;
3488#endif
098859f1 3489 /* Attempt to dup directly from the input memory slot. */
d6ecb4a9
RH		 3490 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3491 its->mem_offset + endian_fixup)) {
3492 goto done;
3493 }
098859f1 3494 /* Load the input into the destination vector register. */
bab1671f
RH		 3495 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3496 break;
3497
3498 default:
3499 g_assert_not_reached();
3500 }
3501
3502 /* We now have a vector input register, so dup must succeed. */
3503 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3504 tcg_debug_assert(ok);
3505
3506 done:
36f5539c 3507 ots->mem_coherent = 0;
bab1671f
RH		 3508 if (IS_DEAD_ARG(1)) {
3509 temp_dead(s, its);
3510 }
3511 if (NEED_SYNC_ARG(0)) {
3512 temp_sync(s, ots, s->reserved_regs, 0, 0);
3513 }
3514 if (IS_DEAD_ARG(0)) {
3515 temp_dead(s, ots);
3516 }
3517}
3518
dd186292 3519static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 3520{
dd186292
RH		 3521 const TCGLifeData arg_life = op->life;
3522 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 3523 TCGRegSet i_allocated_regs;
3524 TCGRegSet o_allocated_regs;
b6638662
RH		 3525 int i, k, nb_iargs, nb_oargs;
3526 TCGReg reg;
c896fe29
FB		 3527 TCGArg arg;
3528 const TCGArgConstraint *arg_ct;
3529 TCGTemp *ts;
3530 TCGArg new_args[TCG_MAX_OP_ARGS];
3531 int const_args[TCG_MAX_OP_ARGS];
3532
3533 nb_oargs = def->nb_oargs;
3534 nb_iargs = def->nb_iargs;
3535
3536 /* copy constants */
a813e36f 3537 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 3538 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 3539 sizeof(TCGArg) * def->nb_cargs);
3540
d21369f5
RH		 3541 i_allocated_regs = s->reserved_regs;
3542 o_allocated_regs = s->reserved_regs;
82790a87 3543
a813e36f 3544 /* satisfy input constraints */
dd186292 3545 for (k = 0; k < nb_iargs; k++) {
1c1824dc
RH		 3546 TCGRegSet i_preferred_regs;
3547 bool allocate_new_reg;
d62816f2 3548
66792f90 3549 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 3550 arg = op->args[i];
c896fe29 3551 arg_ct = &def->args_ct[i];
43439139 3552 ts = arg_temp(arg);
40ae5c62
RH		 3553
3554 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 3555 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 3556 /* constant is OK for instruction */
3557 const_args[i] = 1;
3558 new_args[i] = ts->val;
d62816f2 3559 continue;
c896fe29 3560 }
40ae5c62 3561
1c1824dc
RH		 3562 reg = ts->reg;
3563 i_preferred_regs = 0;
3564 allocate_new_reg = false;
3565
bc2b17e6 3566 if (arg_ct->ialias) {
1c1824dc 3567 i_preferred_regs = op->output_pref[arg_ct->alias_index];
d62816f2 3568
c0522136
RH		 3569 /*
3570 * If the input is readonly, then it cannot also be an
3571 * output and aliased to itself. If the input is not
3572 * dead after the instruction, we must allocate a new
3573 * register and move it.
3574 */
3575 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
1c1824dc
RH		 3576 allocate_new_reg = true;
3577 } else if (ts->val_type == TEMP_VAL_REG) {
3578 /*
3579 * Check if the current register has already been
3580 * allocated for another input.
3581 */
3582 allocate_new_reg = tcg_regset_test_reg(i_allocated_regs, reg);
5ff9d6a4 3583 }
c896fe29 3584 }
d62816f2 3585
1c1824dc
RH		 3586 if (!allocate_new_reg) {
3587 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
3588 reg = ts->reg;
3589 allocate_new_reg = !tcg_regset_test_reg(arg_ct->regs, reg);
3590 }
d62816f2 3591
1c1824dc 3592 if (allocate_new_reg) {
c0522136
RH		 3593 /*
3594 * Allocate a new register matching the constraint
3595 * and move the temporary register into it.
3596 */
d62816f2
RH		 3597 temp_load(s, ts, tcg_target_available_regs[ts->type],
3598 i_allocated_regs, 0);
9be0d080 3599 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
1c1824dc 3600 i_preferred_regs, ts->indirect_base);
78113e83 3601 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3602 /*
3603 * Cross register class move not supported. Sync the
3604 * temp back to its slot and load from there.
3605 */
3606 temp_sync(s, ts, i_allocated_regs, 0, 0);
3607 tcg_out_ld(s, ts->type, reg,
3608 ts->mem_base->reg, ts->mem_offset);
78113e83 3609 }
c896fe29 3610 }
c896fe29
FB		 3611 new_args[i] = reg;
3612 const_args[i] = 0;
82790a87 3613 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29 3614 }
a813e36f 3615
a52ad07e
AJ		 3616 /* mark dead temporaries and free the associated registers */
3617 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3618 if (IS_DEAD_ARG(i)) {
43439139 3619 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 3620 }
3621 }
3622
b4cb76e6
RH		 3623 if (def->flags & TCG_OPF_COND_BRANCH) {
3624 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3625 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 3626 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 3627 } else {
e8996ee0 3628 if (def->flags & TCG_OPF_CALL_CLOBBER) {
a813e36f 3629 /* XXX: permit generic clobber register list ? */
c8074023
RH		 3630 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3631 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 3632 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 3633 }
c896fe29 3634 }
3d5c5f87
AJ		 3635 }
3636 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3637 /* sync globals if the op has side effects and might trigger
3638 an exception. */
82790a87 3639 sync_globals(s, i_allocated_regs);
c896fe29 3640 }
a813e36f 3641
e8996ee0 3642 /* satisfy the output constraints */
e8996ee0 3643 for(k = 0; k < nb_oargs; k++) {
66792f90 3644 i = def->args_ct[k].sort_index;
dd186292 3645 arg = op->args[i];
e8996ee0 3646 arg_ct = &def->args_ct[i];
43439139 3647 ts = arg_temp(arg);
d63e3b6e
RH		 3648
3649 /* ENV should not be modified. */
e01fa97d 3650 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3651
bc2b17e6 3652 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
e8996ee0 3653 reg = new_args[arg_ct->alias_index];
bc2b17e6 3654 } else if (arg_ct->newreg) {
9be0d080 3655 reg = tcg_reg_alloc(s, arg_ct->regs,
82790a87 3656 i_allocated_regs | o_allocated_regs,
69e3706d 3657 op->output_pref[k], ts->indirect_base);
e8996ee0 3658 } else {
9be0d080 3659 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
69e3706d 3660 op->output_pref[k], ts->indirect_base);
c896fe29 3661 }
82790a87 3662 tcg_regset_set_reg(o_allocated_regs, reg);
098859f1 3663 set_temp_val_reg(s, ts, reg);
d63e3b6e 3664 ts->mem_coherent = 0;
e8996ee0 3665 new_args[i] = reg;
c896fe29 3666 }
c896fe29
FB		 3667 }
3668
c896fe29 3669 /* emit instruction */
d2fd745f
RH		 3670 if (def->flags & TCG_OPF_VECTOR) {
3671 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
3672 new_args, const_args);
3673 } else {
3674 tcg_out_op(s, op->opc, new_args, const_args);
3675 }
3676
c896fe29
FB		 3677 /* move the outputs in the correct register if needed */
3678 for(i = 0; i < nb_oargs; i++) {
43439139 3679 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 3680
3681 /* ENV should not be modified. */
e01fa97d 3682 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3683
ec7a869d 3684 if (NEED_SYNC_ARG(i)) {
98b4e186 3685 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 3686 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 3687 temp_dead(s, ts);
ec7a869d 3688 }
c896fe29
FB		 3689 }
3690}
3691
efe86b21
RH		 3692static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
3693{
3694 const TCGLifeData arg_life = op->life;
3695 TCGTemp *ots, *itsl, *itsh;
3696 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3697
3698 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
3699 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
3700 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
3701
3702 ots = arg_temp(op->args[0]);
3703 itsl = arg_temp(op->args[1]);
3704 itsh = arg_temp(op->args[2]);
3705
3706 /* ENV should not be modified. */
3707 tcg_debug_assert(!temp_readonly(ots));
3708
3709 /* Allocate the output register now. */
3710 if (ots->val_type != TEMP_VAL_REG) {
3711 TCGRegSet allocated_regs = s->reserved_regs;
3712 TCGRegSet dup_out_regs =
3713 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
098859f1 3714 TCGReg oreg;
efe86b21
RH		 3715
3716 /* Make sure to not spill the input registers. */
3717 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
3718 tcg_regset_set_reg(allocated_regs, itsl->reg);
3719 }
3720 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
3721 tcg_regset_set_reg(allocated_regs, itsh->reg);
3722 }
3723
098859f1
RH		 3724 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3725 op->output_pref[0], ots->indirect_base);
3726 set_temp_val_reg(s, ots, oreg);
efe86b21
RH		 3727 }
3728
3729 /* Promote dup2 of immediates to dupi_vec. */
3730 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
3731 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
3732 MemOp vece = MO_64;
3733
3734 if (val == dup_const(MO_8, val)) {
3735 vece = MO_8;
3736 } else if (val == dup_const(MO_16, val)) {
3737 vece = MO_16;
3738 } else if (val == dup_const(MO_32, val)) {
3739 vece = MO_32;
3740 }
3741
3742 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
3743 goto done;
3744 }
3745
3746 /* If the two inputs form one 64-bit value, try dupm_vec. */
3747 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
3748 if (!itsl->mem_coherent) {
3749 temp_sync(s, itsl, s->reserved_regs, 0, 0);
3750 }
3751 if (!itsh->mem_coherent) {
3752 temp_sync(s, itsh, s->reserved_regs, 0, 0);
3753 }
e03b5686 3754#if HOST_BIG_ENDIAN
efe86b21
RH		 3755 TCGTemp *its = itsh;
3756#else
3757 TCGTemp *its = itsl;
3758#endif
3759 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
3760 its->mem_base->reg, its->mem_offset)) {
3761 goto done;
3762 }
3763 }
3764
3765 /* Fall back to generic expansion. */
3766 return false;
3767
3768 done:
36f5539c 3769 ots->mem_coherent = 0;
efe86b21
RH		 3770 if (IS_DEAD_ARG(1)) {
3771 temp_dead(s, itsl);
3772 }
3773 if (IS_DEAD_ARG(2)) {
3774 temp_dead(s, itsh);
3775 }
3776 if (NEED_SYNC_ARG(0)) {
3777 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
3778 } else if (IS_DEAD_ARG(0)) {
3779 temp_dead(s, ots);
3780 }
3781 return true;
3782}
3783
dd186292 3784static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
c896fe29 3785{
cd9090aa
RH		 3786 const int nb_oargs = TCGOP_CALLO(op);
3787 const int nb_iargs = TCGOP_CALLI(op);
dd186292 3788 const TCGLifeData arg_life = op->life;
7b7d8b2d 3789 const TCGHelperInfo *info;
b6638662
RH		 3790 int flags, nb_regs, i;
3791 TCGReg reg;
cf066674 3792 TCGArg arg;
c896fe29 3793 TCGTemp *ts;
d3452f1f
RH		 3794 intptr_t stack_offset;
3795 size_t call_stack_size;
cf066674
RH		 3796 tcg_insn_unit *func_addr;
3797 int allocate_args;
c896fe29 3798 TCGRegSet allocated_regs;
c896fe29 3799
fa52e660 3800 func_addr = tcg_call_func(op);
7b7d8b2d
RH		 3801 info = tcg_call_info(op);
3802 flags = info->flags;
c896fe29 3803
6e17d0c5 3804 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
c45cb8bb
RH		 3805 if (nb_regs > nb_iargs) {
3806 nb_regs = nb_iargs;
cf066674 3807 }
c896fe29
FB		 3808
3809 /* assign stack slots first */
c45cb8bb 3810 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
a813e36f 3811 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
c896fe29 3812 ~(TCG_TARGET_STACK_ALIGN - 1);
b03cce8e
FB		 3813 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
3814 if (allocate_args) {
345649c0
BS		 3815 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3816 preallocate call stack */
3817 tcg_abort();
b03cce8e 3818 }
39cf05d3
FB		 3819
3820 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
dd186292
RH		 3821 for (i = nb_regs; i < nb_iargs; i++) {
3822 arg = op->args[nb_oargs + i];
39cf05d3 3823 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3824 ts = arg_temp(arg);
40ae5c62 3825 temp_load(s, ts, tcg_target_available_regs[ts->type],
b722452a 3826 s->reserved_regs, 0);
40ae5c62 3827 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
c896fe29 3828 }
39cf05d3 3829 stack_offset += sizeof(tcg_target_long);
c896fe29 3830 }
a813e36f 3831
c896fe29 3832 /* assign input registers */
d21369f5 3833 allocated_regs = s->reserved_regs;
dd186292
RH		 3834 for (i = 0; i < nb_regs; i++) {
3835 arg = op->args[nb_oargs + i];
39cf05d3 3836 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3837 ts = arg_temp(arg);
39cf05d3 3838 reg = tcg_target_call_iarg_regs[i];
40ae5c62 3839
39cf05d3
FB		 3840 if (ts->val_type == TEMP_VAL_REG) {
3841 if (ts->reg != reg) {
4250da10 3842 tcg_reg_free(s, reg, allocated_regs);
78113e83 3843 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3844 /*
3845 * Cross register class move not supported. Sync the
3846 * temp back to its slot and load from there.
3847 */
3848 temp_sync(s, ts, allocated_regs, 0, 0);
3849 tcg_out_ld(s, ts->type, reg,
3850 ts->mem_base->reg, ts->mem_offset);
78113e83 3851 }
39cf05d3 3852 }
39cf05d3 3853 } else {
ccb1bb66 3854 TCGRegSet arg_set = 0;
40ae5c62 3855
4250da10 3856 tcg_reg_free(s, reg, allocated_regs);
40ae5c62 3857 tcg_regset_set_reg(arg_set, reg);
b722452a 3858 temp_load(s, ts, arg_set, allocated_regs, 0);
c896fe29 3859 }
40ae5c62 3860
39cf05d3 3861 tcg_regset_set_reg(allocated_regs, reg);
c896fe29 3862 }
c896fe29 3863 }
a813e36f 3864
c896fe29 3865 /* mark dead temporaries and free the associated registers */
dd186292 3866 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 3867 if (IS_DEAD_ARG(i)) {
43439139 3868 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 3869 }
3870 }
a813e36f 3871
c896fe29 3872 /* clobber call registers */
c8074023
RH		 3873 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3874 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 3875 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 3876 }
3877 }
78505279
AJ		 3878
3879 /* Save globals if they might be written by the helper, sync them if
3880 they might be read. */
3881 if (flags & TCG_CALL_NO_READ_GLOBALS) {
3882 /* Nothing to do */
3883 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
3884 sync_globals(s, allocated_regs);
3885 } else {
b9c18f56
AJ		 3886 save_globals(s, allocated_regs);
3887 }
c896fe29 3888
7b7d8b2d
RH		 3889#ifdef CONFIG_TCG_INTERPRETER
3890 {
3891 gpointer hash = (gpointer)(uintptr_t)info->typemask;
3892 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash);
3893 assert(cif != NULL);
3894 tcg_out_call(s, func_addr, cif);
3895 }
3896#else
cf066674 3897 tcg_out_call(s, func_addr);
7b7d8b2d 3898#endif
c896fe29
FB		 3899
3900 /* assign output registers and emit moves if needed */
3901 for(i = 0; i < nb_oargs; i++) {
dd186292 3902 arg = op->args[i];
43439139 3903 ts = arg_temp(arg);
d63e3b6e
RH		 3904
3905 /* ENV should not be modified. */
e01fa97d 3906 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3907
c896fe29 3908 reg = tcg_target_call_oarg_regs[i];
098859f1 3909 set_temp_val_reg(s, ts, reg);
d63e3b6e 3910 ts->mem_coherent = 0;
d63e3b6e
RH		 3911 if (NEED_SYNC_ARG(i)) {
3912 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
3913 } else if (IS_DEAD_ARG(i)) {
3914 temp_dead(s, ts);
c896fe29
FB		 3915 }
3916 }
c896fe29
FB		 3917}
3918
3919#ifdef CONFIG_PROFILER
3920
c3fac113
EC		 3921/* avoid copy/paste errors */
3922#define PROF_ADD(to, from, field) \
3923 do { \
d73415a3 3924 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 3925 } while (0)
3926
3927#define PROF_MAX(to, from, field) \
3928 do { \
d73415a3 3929 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 3930 if (val__ > (to)->field) { \
3931 (to)->field = val__; \
3932 } \
3933 } while (0)
3934
3935/* Pass in a zero'ed @prof */
3936static inline
3937void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
3938{
0e2d61cf 3939 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 3940 unsigned int i;
3941
3468b59e 3942 for (i = 0; i < n_ctxs; i++) {
d73415a3 3943 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 3944 const TCGProfile *orig = &s->prof;
c3fac113
EC		 3945
3946 if (counters) {
72fd2efb 3947 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 3948 PROF_ADD(prof, orig, tb_count1);
3949 PROF_ADD(prof, orig, tb_count);
3950 PROF_ADD(prof, orig, op_count);
3951 PROF_MAX(prof, orig, op_count_max);
3952 PROF_ADD(prof, orig, temp_count);
3953 PROF_MAX(prof, orig, temp_count_max);
3954 PROF_ADD(prof, orig, del_op_count);
3955 PROF_ADD(prof, orig, code_in_len);
3956 PROF_ADD(prof, orig, code_out_len);
3957 PROF_ADD(prof, orig, search_out_len);
3958 PROF_ADD(prof, orig, interm_time);
3959 PROF_ADD(prof, orig, code_time);
3960 PROF_ADD(prof, orig, la_time);
3961 PROF_ADD(prof, orig, opt_time);
3962 PROF_ADD(prof, orig, restore_count);
3963 PROF_ADD(prof, orig, restore_time);
3964 }
3965 if (table) {
3966 int i;
3967
3968 for (i = 0; i < NB_OPS; i++) {
3969 PROF_ADD(prof, orig, table_op_count[i]);
3970 }
3971 }
3972 }
3973}
3974
3975#undef PROF_ADD
3976#undef PROF_MAX
3977
3978static void tcg_profile_snapshot_counters(TCGProfile *prof)
3979{
3980 tcg_profile_snapshot(prof, true, false);
3981}
3982
3983static void tcg_profile_snapshot_table(TCGProfile *prof)
3984{
3985 tcg_profile_snapshot(prof, false, true);
3986}
c896fe29 3987
b6a7f3e0 3988void tcg_dump_op_count(GString *buf)
c896fe29 3989{
c3fac113 3990 TCGProfile prof = {};
c896fe29 3991 int i;
d70724ce 3992
c3fac113 3993 tcg_profile_snapshot_table(&prof);
15fc7daa 3994 for (i = 0; i < NB_OPS; i++) {
b6a7f3e0
DB		 3995 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
3996 prof.table_op_count[i]);
c896fe29 3997 }
c896fe29 3998}
72fd2efb
EC		 3999
4000int64_t tcg_cpu_exec_time(void)
4001{
0e2d61cf 4002 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4003 unsigned int i;
4004 int64_t ret = 0;
4005
4006 for (i = 0; i < n_ctxs; i++) {
d73415a3 4007 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4008 const TCGProfile *prof = &s->prof;
4009
d73415a3 4010 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4011 }
4012 return ret;
4013}
246ae24d 4014#else
b6a7f3e0 4015void tcg_dump_op_count(GString *buf)
246ae24d 4016{
b6a7f3e0 4017 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
246ae24d 4018}
72fd2efb
EC		 4019
4020int64_t tcg_cpu_exec_time(void)
4021{
4022 error_report("%s: TCG profiler not compiled", __func__);
4023 exit(EXIT_FAILURE);
4024}
c896fe29
FB		 4025#endif
4026
4027
fbf59aad 4028int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
c896fe29 4029{
c3fac113
EC		 4030#ifdef CONFIG_PROFILER
4031 TCGProfile *prof = &s->prof;
4032#endif
15fa08f8
RH		 4033 int i, num_insns;
4034 TCGOp *op;
c896fe29 4035
04fe6400
RH		 4036#ifdef CONFIG_PROFILER
4037 {
c1f543b7 4038 int n = 0;
04fe6400 4039
15fa08f8
RH		 4040 QTAILQ_FOREACH(op, &s->ops, link) {
4041 n++;
4042 }
d73415a3 4043 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4044 if (n > prof->op_count_max) {
d73415a3 4045 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4046 }
4047
4048 n = s->nb_temps;
d73415a3 4049 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4050 if (n > prof->temp_count_max) {
d73415a3 4051 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4052 }
4053 }
4054#endif
4055
c896fe29 4056#ifdef DEBUG_DISAS
d977e1c2 4057 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
fbf59aad 4058 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4059 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4060 if (logfile) {
4061 fprintf(logfile, "OP:\n");
b7a83ff8 4062 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4063 fprintf(logfile, "\n");
4064 qemu_log_unlock(logfile);
4065 }
c896fe29
FB		 4066 }
4067#endif
4068
bef16ab4
RH		 4069#ifdef CONFIG_DEBUG_TCG
4070 /* Ensure all labels referenced have been emitted. */
4071 {
4072 TCGLabel *l;
4073 bool error = false;
4074
4075 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4076 if (unlikely(!l->present) && l->refs) {
4077 qemu_log_mask(CPU_LOG_TB_OP,
4078 "$L%d referenced but not present.\n", l->id);
4079 error = true;
4080 }
4081 }
4082 assert(!error);
4083 }
4084#endif
4085
c5cc28ff 4086#ifdef CONFIG_PROFILER
d73415a3 4087 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4088#endif
4089
8f2e8c07 4090#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4091 tcg_optimize(s);
8f2e8c07
KB		 4092#endif
4093
a23a9ec6 4094#ifdef CONFIG_PROFILER
d73415a3
SH		 4095 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4096 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4097#endif
c5cc28ff 4098
b4fc67c7 4099 reachable_code_pass(s);
b83eabea 4100 liveness_pass_1(s);
5a18407f 4101
b83eabea 4102 if (s->nb_indirects > 0) {
5a18407f 4103#ifdef DEBUG_DISAS
b83eabea 4104 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
fbf59aad 4105 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4106 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4107 if (logfile) {
4108 fprintf(logfile, "OP before indirect lowering:\n");
b7a83ff8 4109 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4110 fprintf(logfile, "\n");
4111 qemu_log_unlock(logfile);
4112 }
b83eabea 4113 }
5a18407f 4114#endif
b83eabea
RH		 4115 /* Replace indirect temps with direct temps. */
4116 if (liveness_pass_2(s)) {
4117 /* If changes were made, re-run liveness. */
4118 liveness_pass_1(s);
5a18407f
RH		 4119 }
4120 }
c5cc28ff 4121
a23a9ec6 4122#ifdef CONFIG_PROFILER
d73415a3 4123 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4124#endif
c896fe29
FB		 4125
4126#ifdef DEBUG_DISAS
d977e1c2 4127 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
fbf59aad 4128 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4129 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4130 if (logfile) {
4131 fprintf(logfile, "OP after optimization and liveness analysis:\n");
b7a83ff8 4132 tcg_dump_ops(s, logfile, true);
78b54858
RH		 4133 fprintf(logfile, "\n");
4134 qemu_log_unlock(logfile);
4135 }
c896fe29
FB		 4136 }
4137#endif
4138
35abb009
RH		 4139 /* Initialize goto_tb jump offsets. */
4140 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
4141 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
4142 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
4143 if (TCG_TARGET_HAS_direct_jump) {
4144 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
4145 tcg_ctx->tb_jmp_target_addr = NULL;
4146 } else {
4147 tcg_ctx->tb_jmp_insn_offset = NULL;
4148 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
4149 }
4150
c896fe29
FB		 4151 tcg_reg_alloc_start(s);
4152
db0c51a3
RH		 4153 /*
4154 * Reset the buffer pointers when restarting after overflow.
4155 * TODO: Move this into translate-all.c with the rest of the
4156 * buffer management. Having only this done here is confusing.
4157 */
4158 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4159 s->code_ptr = s->code_buf;
c896fe29 4160
659ef5cb 4161#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4162 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4163#endif
57a26946
RH		 4164#ifdef TCG_TARGET_NEED_POOL_LABELS
4165 s->pool_labels = NULL;
4166#endif
9ecefc84 4167
fca8a500 4168 num_insns = -1;
15fa08f8 4169 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4170 TCGOpcode opc = op->opc;
b3db8758 4171
c896fe29 4172#ifdef CONFIG_PROFILER
d73415a3 4173 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 4174#endif
c45cb8bb
RH		 4175
4176 switch (opc) {
c896fe29 4177 case INDEX_op_mov_i32:
c896fe29 4178 case INDEX_op_mov_i64:
d2fd745f 4179 case INDEX_op_mov_vec:
dd186292 4180 tcg_reg_alloc_mov(s, op);
c896fe29 4181 break;
bab1671f
RH		 4182 case INDEX_op_dup_vec:
4183 tcg_reg_alloc_dup(s, op);
4184 break;
765b842a 4185 case INDEX_op_insn_start:
fca8a500 4186 if (num_insns >= 0) {
9f754620
RH		 4187 size_t off = tcg_current_code_size(s);
4188 s->gen_insn_end_off[num_insns] = off;
4189 /* Assert that we do not overflow our stored offset. */
4190 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 4191 }
4192 num_insns++;
bad729e2
RH		 4193 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4194 target_ulong a;
4195#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 4196 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 4197#else
efee3746 4198 a = op->args[i];
bad729e2 4199#endif
fca8a500 4200 s->gen_insn_data[num_insns][i] = a;
bad729e2 4201 }
c896fe29 4202 break;
5ff9d6a4 4203 case INDEX_op_discard:
43439139 4204 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 4205 break;
c896fe29 4206 case INDEX_op_set_label:
e8996ee0 4207 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 4208 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 4209 break;
4210 case INDEX_op_call:
dd186292 4211 tcg_reg_alloc_call(s, op);
c45cb8bb 4212 break;
efe86b21
RH		 4213 case INDEX_op_dup2_vec:
4214 if (tcg_reg_alloc_dup2(s, op)) {
4215 break;
4216 }
4217 /* fall through */
c896fe29 4218 default:
25c4d9cc 4219 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 4220 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 4221 /* Note: in order to speed up the code, it would be much
4222 faster to have specialized register allocator functions for
4223 some common argument patterns */
dd186292 4224 tcg_reg_alloc_op(s, op);
c896fe29
FB		 4225 break;
4226 }
b125f9dc
RH		 4227 /* Test for (pending) buffer overflow. The assumption is that any
4228 one operation beginning below the high water mark cannot overrun
4229 the buffer completely. Thus we can test for overflow after
4230 generating code without having to check during generation. */
644da9b3 4231 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 4232 return -1;
4233 }
6e6c4efe
RH		 4234 /* Test for TB overflow, as seen by gen_insn_end_off. */
4235 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4236 return -2;
4237 }
c896fe29 4238 }
fca8a500
RH		 4239 tcg_debug_assert(num_insns >= 0);
4240 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 4241
b76f0d8c 4242 /* Generate TB finalization at the end of block */
659ef5cb 4243#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 4244 i = tcg_out_ldst_finalize(s);
4245 if (i < 0) {
4246 return i;
23dceda6 4247 }
659ef5cb 4248#endif
57a26946 4249#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 4250 i = tcg_out_pool_finalize(s);
4251 if (i < 0) {
4252 return i;
57a26946
RH		 4253 }
4254#endif
7ecd02a0
RH		 4255 if (!tcg_resolve_relocs(s)) {
4256 return -2;
4257 }
c896fe29 4258
df5d2b16 4259#ifndef CONFIG_TCG_INTERPRETER
c896fe29 4260 /* flush instruction cache */
db0c51a3
RH		 4261 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4262 (uintptr_t)s->code_buf,
1da8de39 4263 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 4264#endif
2aeabc08 4265
1813e175 4266 return tcg_current_code_size(s);
c896fe29
FB		 4267}
4268
a23a9ec6 4269#ifdef CONFIG_PROFILER
3a841ab5 4270void tcg_dump_info(GString *buf)
a23a9ec6 4271{
c3fac113
EC		 4272 TCGProfile prof = {};
4273 const TCGProfile *s;
4274 int64_t tb_count;
4275 int64_t tb_div_count;
4276 int64_t tot;
4277
4278 tcg_profile_snapshot_counters(&prof);
4279 s = &prof;
4280 tb_count = s->tb_count;
4281 tb_div_count = tb_count ? tb_count : 1;
4282 tot = s->interm_time + s->code_time;
a23a9ec6 4283
3a841ab5
DB		 4284 g_string_append_printf(buf, "JIT cycles %" PRId64
4285 " (%0.3f s at 2.4 GHz)\n",
4286 tot, tot / 2.4e9);
4287 g_string_append_printf(buf, "translated TBs %" PRId64
4288 " (aborted=%" PRId64 " %0.1f%%)\n",
4289 tb_count, s->tb_count1 - tb_count,
4290 (double)(s->tb_count1 - s->tb_count)
4291 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4292 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
4293 (double)s->op_count / tb_div_count, s->op_count_max);
4294 g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
4295 (double)s->del_op_count / tb_div_count);
4296 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
4297 (double)s->temp_count / tb_div_count,
4298 s->temp_count_max);
4299 g_string_append_printf(buf, "avg host code/TB %0.1f\n",
4300 (double)s->code_out_len / tb_div_count);
4301 g_string_append_printf(buf, "avg search data/TB %0.1f\n",
4302 (double)s->search_out_len / tb_div_count);
a813e36f 4303
3a841ab5
DB		 4304 g_string_append_printf(buf, "cycles/op %0.1f\n",
4305 s->op_count ? (double)tot / s->op_count : 0);
4306 g_string_append_printf(buf, "cycles/in byte %0.1f\n",
4307 s->code_in_len ? (double)tot / s->code_in_len : 0);
4308 g_string_append_printf(buf, "cycles/out byte %0.1f\n",
4309 s->code_out_len ? (double)tot / s->code_out_len : 0);
4310 g_string_append_printf(buf, "cycles/search byte %0.1f\n",
4311 s->search_out_len ?
4312 (double)tot / s->search_out_len : 0);
fca8a500 4313 if (tot == 0) {
a23a9ec6 4314 tot = 1;
fca8a500 4315 }
3a841ab5
DB		 4316 g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
4317 (double)s->interm_time / tot * 100.0);
4318 g_string_append_printf(buf, " gen_code time %0.1f%%\n",
4319 (double)s->code_time / tot * 100.0);
4320 g_string_append_printf(buf, "optim./code time %0.1f%%\n",
4321 (double)s->opt_time / (s->code_time ?
4322 s->code_time : 1)
4323 * 100.0);
4324 g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
4325 (double)s->la_time / (s->code_time ?
4326 s->code_time : 1) * 100.0);
4327 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
4328 s->restore_count);
4329 g_string_append_printf(buf, " avg cycles %0.1f\n",
4330 s->restore_count ?
4331 (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 4332}
4333#else
3a841ab5 4334void tcg_dump_info(GString *buf)
a23a9ec6 4335{
3a841ab5 4336 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
a23a9ec6
FB		 4337}
4338#endif
813da627
RH		 4339
4340#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 4341/* In order to use this feature, the backend needs to do three things:
4342
4343 (1) Define ELF_HOST_MACHINE to indicate both what value to
4344 put into the ELF image and to indicate support for the feature.
4345
4346 (2) Define tcg_register_jit. This should create a buffer containing
4347 the contents of a .debug_frame section that describes the post-
4348 prologue unwind info for the tcg machine.
4349
4350 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4351*/
813da627
RH		 4352
4353/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4354typedef enum {
4355 JIT_NOACTION = 0,
4356 JIT_REGISTER_FN,
4357 JIT_UNREGISTER_FN
4358} jit_actions_t;
4359
4360struct jit_code_entry {
4361 struct jit_code_entry *next_entry;
4362 struct jit_code_entry *prev_entry;
4363 const void *symfile_addr;
4364 uint64_t symfile_size;
4365};
4366
4367struct jit_descriptor {
4368 uint32_t version;
4369 uint32_t action_flag;
4370 struct jit_code_entry *relevant_entry;
4371 struct jit_code_entry *first_entry;
4372};
4373
4374void __jit_debug_register_code(void) __attribute__((noinline));
4375void __jit_debug_register_code(void)
4376{
4377 asm("");
4378}
4379
4380/* Must statically initialize the version, because GDB may check
4381 the version before we can set it. */
4382struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4383
4384/* End GDB interface. */
4385
4386static int find_string(const char *strtab, const char *str)
4387{
4388 const char *p = strtab + 1;
4389
4390 while (1) {
4391 if (strcmp(p, str) == 0) {
4392 return p - strtab;
4393 }
4394 p += strlen(p) + 1;
4395 }
4396}
4397
755bf9e5 4398static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 4399 const void *debug_frame,
4400 size_t debug_frame_size)
813da627 4401{
5872bbf2
RH		 4402 struct __attribute__((packed)) DebugInfo {
4403 uint32_t len;
4404 uint16_t version;
4405 uint32_t abbrev;
4406 uint8_t ptr_size;
4407 uint8_t cu_die;
4408 uint16_t cu_lang;
4409 uintptr_t cu_low_pc;
4410 uintptr_t cu_high_pc;
4411 uint8_t fn_die;
4412 char fn_name[16];
4413 uintptr_t fn_low_pc;
4414 uintptr_t fn_high_pc;
4415 uint8_t cu_eoc;
4416 };
813da627
RH		 4417
4418 struct ElfImage {
4419 ElfW(Ehdr) ehdr;
4420 ElfW(Phdr) phdr;
5872bbf2
RH		 4421 ElfW(Shdr) shdr[7];
4422 ElfW(Sym) sym[2];
4423 struct DebugInfo di;
4424 uint8_t da[24];
4425 char str[80];
4426 };
4427
4428 struct ElfImage *img;
4429
4430 static const struct ElfImage img_template = {
4431 .ehdr = {
4432 .e_ident[EI_MAG0] = ELFMAG0,
4433 .e_ident[EI_MAG1] = ELFMAG1,
4434 .e_ident[EI_MAG2] = ELFMAG2,
4435 .e_ident[EI_MAG3] = ELFMAG3,
4436 .e_ident[EI_CLASS] = ELF_CLASS,
4437 .e_ident[EI_DATA] = ELF_DATA,
4438 .e_ident[EI_VERSION] = EV_CURRENT,
4439 .e_type = ET_EXEC,
4440 .e_machine = ELF_HOST_MACHINE,
4441 .e_version = EV_CURRENT,
4442 .e_phoff = offsetof(struct ElfImage, phdr),
4443 .e_shoff = offsetof(struct ElfImage, shdr),
4444 .e_ehsize = sizeof(ElfW(Shdr)),
4445 .e_phentsize = sizeof(ElfW(Phdr)),
4446 .e_phnum = 1,
4447 .e_shentsize = sizeof(ElfW(Shdr)),
4448 .e_shnum = ARRAY_SIZE(img->shdr),
4449 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 4450#ifdef ELF_HOST_FLAGS
4451 .e_flags = ELF_HOST_FLAGS,
4452#endif
4453#ifdef ELF_OSABI
4454 .e_ident[EI_OSABI] = ELF_OSABI,
4455#endif
5872bbf2
RH		 4456 },
4457 .phdr = {
4458 .p_type = PT_LOAD,
4459 .p_flags = PF_X,
4460 },
4461 .shdr = {
4462 [0] = { .sh_type = SHT_NULL },
4463 /* Trick: The contents of code_gen_buffer are not present in
4464 this fake ELF file; that got allocated elsewhere. Therefore
4465 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4466 will not look for contents. We can record any address. */
4467 [1] = { /* .text */
4468 .sh_type = SHT_NOBITS,
4469 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4470 },
4471 [2] = { /* .debug_info */
4472 .sh_type = SHT_PROGBITS,
4473 .sh_offset = offsetof(struct ElfImage, di),
4474 .sh_size = sizeof(struct DebugInfo),
4475 },
4476 [3] = { /* .debug_abbrev */
4477 .sh_type = SHT_PROGBITS,
4478 .sh_offset = offsetof(struct ElfImage, da),
4479 .sh_size = sizeof(img->da),
4480 },
4481 [4] = { /* .debug_frame */
4482 .sh_type = SHT_PROGBITS,
4483 .sh_offset = sizeof(struct ElfImage),
4484 },
4485 [5] = { /* .symtab */
4486 .sh_type = SHT_SYMTAB,
4487 .sh_offset = offsetof(struct ElfImage, sym),
4488 .sh_size = sizeof(img->sym),
4489 .sh_info = 1,
4490 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4491 .sh_entsize = sizeof(ElfW(Sym)),
4492 },
4493 [6] = { /* .strtab */
4494 .sh_type = SHT_STRTAB,
4495 .sh_offset = offsetof(struct ElfImage, str),
4496 .sh_size = sizeof(img->str),
4497 }
4498 },
4499 .sym = {
4500 [1] = { /* code_gen_buffer */
4501 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4502 .st_shndx = 1,
4503 }
4504 },
4505 .di = {
4506 .len = sizeof(struct DebugInfo) - 4,
4507 .version = 2,
4508 .ptr_size = sizeof(void *),
4509 .cu_die = 1,
4510 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4511 .fn_die = 2,
4512 .fn_name = "code_gen_buffer"
4513 },
4514 .da = {
4515 1, /* abbrev number (the cu) */
4516 0x11, 1, /* DW_TAG_compile_unit, has children */
4517 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4518 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4519 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4520 0, 0, /* end of abbrev */
4521 2, /* abbrev number (the fn) */
4522 0x2e, 0, /* DW_TAG_subprogram, no children */
4523 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4524 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4525 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4526 0, 0, /* end of abbrev */
4527 0 /* no more abbrev */
4528 },
4529 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4530 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 4531 };
4532
4533 /* We only need a single jit entry; statically allocate it. */
4534 static struct jit_code_entry one_entry;
4535
5872bbf2 4536 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 4537 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 4538 DebugFrameHeader *dfh;
813da627 4539
5872bbf2
RH		 4540 img = g_malloc(img_size);
4541 *img = img_template;
813da627 4542
5872bbf2
RH		 4543 img->phdr.p_vaddr = buf;
4544 img->phdr.p_paddr = buf;
4545 img->phdr.p_memsz = buf_size;
813da627 4546
813da627 4547 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 4548 img->shdr[1].sh_addr = buf;
813da627
RH		 4549 img->shdr[1].sh_size = buf_size;
4550
5872bbf2
RH		 4551 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4552 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4553
4554 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4555 img->shdr[4].sh_size = debug_frame_size;
4556
4557 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4558 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4559
4560 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4561 img->sym[1].st_value = buf;
4562 img->sym[1].st_size = buf_size;
813da627 4563
5872bbf2 4564 img->di.cu_low_pc = buf;
45aba097 4565 img->di.cu_high_pc = buf + buf_size;
5872bbf2 4566 img->di.fn_low_pc = buf;
45aba097 4567 img->di.fn_high_pc = buf + buf_size;
813da627 4568
2c90784a
RH		 4569 dfh = (DebugFrameHeader *)(img + 1);
4570 memcpy(dfh, debug_frame, debug_frame_size);
4571 dfh->fde.func_start = buf;
4572 dfh->fde.func_len = buf_size;
4573
813da627
RH		 4574#ifdef DEBUG_JIT
4575 /* Enable this block to be able to debug the ELF image file creation.
4576 One can use readelf, objdump, or other inspection utilities. */
4577 {
eb6b2edf
BM		 4578 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
4579 FILE *f = fopen(jit, "w+b");
813da627 4580 if (f) {
5872bbf2 4581 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 4582 /* Avoid stupid unused return value warning for fwrite. */
4583 }
4584 fclose(f);
4585 }
4586 }
4587#endif
4588
4589 one_entry.symfile_addr = img;
4590 one_entry.symfile_size = img_size;
4591
4592 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4593 __jit_debug_descriptor.relevant_entry = &one_entry;
4594 __jit_debug_descriptor.first_entry = &one_entry;
4595 __jit_debug_register_code();
4596}
4597#else
5872bbf2
RH		 4598/* No support for the feature. Provide the entry point expected by exec.c,
4599 and implement the internal function we declared earlier. */
813da627 4600
755bf9e5 4601static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 4602 const void *debug_frame,
4603 size_t debug_frame_size)
813da627
RH		 4604{
4605}
4606
755bf9e5 4607void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 4608{
4609}
4610#endif /* ELF_HOST_MACHINE */
db432672
RH		 4611
4612#if !TCG_TARGET_MAYBE_vec
4613void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4614{
4615 g_assert_not_reached();
4616}
4617#endif
QEMU mirror