]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
accel/tcg: Add debuginfo support
[mirror_qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
1de7afc9 37#include "qemu/timer.h"
084cfca1 38#include "qemu/cacheflush.h"
ad768e6f 39#include "qemu/cacheinfo.h"
c896fe29 40
c5d3c498 41/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 42 CPU definitions. Currently they are used for qemu_ld/st
43 instructions */
44#define NO_CPU_IO_DEFS
c896fe29 45
63c91552 46#include "exec/exec-all.h"
dcb32f1d 47#include "tcg/tcg-op.h"
813da627 48
edee2579 49#if UINTPTR_MAX == UINT32_MAX
813da627 50# define ELF_CLASS ELFCLASS32
edee2579
RH		 51#else
52# define ELF_CLASS ELFCLASS64
813da627 53#endif
e03b5686 54#if HOST_BIG_ENDIAN
813da627
RH		 55# define ELF_DATA ELFDATA2MSB
56#else
57# define ELF_DATA ELFDATA2LSB
58#endif
59
c896fe29 60#include "elf.h"
508127e2 61#include "exec/log.h"
d2ba8026 62#include "tcg/tcg-ldst.h"
5ff7258c 63#include "tcg-internal.h"
c896fe29 64
139c1837 65/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 66 used here. */
e4d58b41
RH		 67static void tcg_target_init(TCGContext *s);
68static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 69static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 70 intptr_t value, intptr_t addend);
c896fe29 71
497a22eb
RH		 72/* The CIE and FDE header definitions will be common to all hosts. */
73typedef struct {
74 uint32_t len __attribute__((aligned((sizeof(void *)))));
75 uint32_t id;
76 uint8_t version;
77 char augmentation[1];
78 uint8_t code_align;
79 uint8_t data_align;
80 uint8_t return_column;
81} DebugFrameCIE;
82
83typedef struct QEMU_PACKED {
84 uint32_t len __attribute__((aligned((sizeof(void *)))));
85 uint32_t cie_offset;
edee2579
RH		 86 uintptr_t func_start;
87 uintptr_t func_len;
497a22eb
RH		 88} DebugFrameFDEHeader;
89
2c90784a
RH		 90typedef struct QEMU_PACKED {
91 DebugFrameCIE cie;
92 DebugFrameFDEHeader fde;
93} DebugFrameHeader;
94
755bf9e5 95static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 96 const void *debug_frame,
97 size_t debug_frame_size)
813da627
RH		 98 __attribute__((unused));
99
139c1837 100/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 101static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 102 intptr_t arg2);
78113e83 103static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 104static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 105 TCGReg ret, tcg_target_long arg);
5e8892db
MR		 106static void tcg_out_op(TCGContext *s, TCGOpcode opc,
107 const TCGArg args[TCG_MAX_OP_ARGS],
108 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 109#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 110static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
111 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 112static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
113 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 114static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, int64_t arg);
5e8892db
MR		 116static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
117 unsigned vecl, unsigned vece,
118 const TCGArg args[TCG_MAX_OP_ARGS],
119 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 120#else
e7632cfa
RH		 121static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
122 TCGReg dst, TCGReg src)
123{
124 g_assert_not_reached();
125}
d6ecb4a9
RH		 126static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
127 TCGReg dst, TCGReg base, intptr_t offset)
128{
129 g_assert_not_reached();
130}
4e186175
RH		 131static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
132 TCGReg dst, int64_t arg)
e7632cfa
RH		 133{
134 g_assert_not_reached();
135}
5e8892db
MR		 136static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
137 unsigned vecl, unsigned vece,
138 const TCGArg args[TCG_MAX_OP_ARGS],
139 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 140{
141 g_assert_not_reached();
142}
143#endif
2a534aff 144static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 145 intptr_t arg2);
59d7c14e
RH		 146static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
147 TCGReg base, intptr_t ofs);
7b7d8b2d 148static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
cee44b03 149 const TCGHelperInfo *info);
a4fbbd77 150static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 151#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 152static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 153#endif
c896fe29 154
42eb6dfc
RH		 155TCGContext tcg_init_ctx;
156__thread TCGContext *tcg_ctx;
157
5ff7258c 158TCGContext **tcg_ctxs;
0e2d61cf
RH		 159unsigned int tcg_cur_ctxs;
160unsigned int tcg_max_ctxs;
1c2adb95 161TCGv_env cpu_env = 0;
c8bc1168 162const void *tcg_code_gen_epilogue;
db0c51a3 163uintptr_t tcg_splitwx_diff;
df2cce29 164
b91ccb31
RH		 165#ifndef CONFIG_TCG_INTERPRETER
166tcg_prologue_fn *tcg_qemu_tb_exec;
167#endif
168
d2fd745f 169static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 170static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 171
1813e175 172#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 173static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 174{
175 *s->code_ptr++ = v;
176}
177
4196dca6
PM		 178static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
179 uint8_t v)
5c53bb81 180{
1813e175 181 *p = v;
5c53bb81 182}
1813e175 183#endif
5c53bb81 184
1813e175 185#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 186static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 187{
1813e175
RH		 188 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
189 *s->code_ptr++ = v;
190 } else {
191 tcg_insn_unit *p = s->code_ptr;
192 memcpy(p, &v, sizeof(v));
193 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
194 }
c896fe29
FB		 195}
196
4196dca6
PM		 197static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
198 uint16_t v)
5c53bb81 199{
1813e175
RH		 200 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
201 *p = v;
202 } else {
203 memcpy(p, &v, sizeof(v));
204 }
5c53bb81 205}
1813e175 206#endif
5c53bb81 207
1813e175 208#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 209static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 210{
1813e175
RH		 211 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
212 *s->code_ptr++ = v;
213 } else {
214 tcg_insn_unit *p = s->code_ptr;
215 memcpy(p, &v, sizeof(v));
216 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
217 }
c896fe29
FB		 218}
219
4196dca6
PM		 220static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
221 uint32_t v)
5c53bb81 222{
1813e175
RH		 223 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
224 *p = v;
225 } else {
226 memcpy(p, &v, sizeof(v));
227 }
5c53bb81 228}
1813e175 229#endif
5c53bb81 230
1813e175 231#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 232static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 233{
1813e175
RH		 234 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
235 *s->code_ptr++ = v;
236 } else {
237 tcg_insn_unit *p = s->code_ptr;
238 memcpy(p, &v, sizeof(v));
239 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
240 }
ac26eb69
RH		 241}
242
4196dca6
PM		 243static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
244 uint64_t v)
5c53bb81 245{
1813e175
RH		 246 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
247 *p = v;
248 } else {
249 memcpy(p, &v, sizeof(v));
250 }
5c53bb81 251}
1813e175 252#endif
5c53bb81 253
c896fe29
FB		 254/* label relocation processing */
255
1813e175 256static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 257 TCGLabel *l, intptr_t addend)
c896fe29 258{
7ecd02a0 259 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 260
7ecd02a0
RH		 261 r->type = type;
262 r->ptr = code_ptr;
263 r->addend = addend;
264 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 265}
266
92ab8e7d 267static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 268{
eabb7b91 269 tcg_debug_assert(!l->has_value);
c896fe29 270 l->has_value = 1;
92ab8e7d 271 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 272}
273
42a268c2 274TCGLabel *gen_new_label(void)
c896fe29 275{
b1311c4a 276 TCGContext *s = tcg_ctx;
51e3972c 277 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 278
7ecd02a0
RH		 279 memset(l, 0, sizeof(TCGLabel));
280 l->id = s->nb_labels++;
281 QSIMPLEQ_INIT(&l->relocs);
282
bef16ab4 283 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 284
285 return l;
c896fe29
FB		 286}
287
7ecd02a0
RH		 288static bool tcg_resolve_relocs(TCGContext *s)
289{
290 TCGLabel *l;
291
292 QSIMPLEQ_FOREACH(l, &s->labels, next) {
293 TCGRelocation *r;
294 uintptr_t value = l->u.value;
295
296 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
297 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
298 return false;
299 }
300 }
301 }
302 return true;
303}
304
9f754620
RH		 305static void set_jmp_reset_offset(TCGContext *s, int which)
306{
f14bed3f
RH		 307 /*
308 * We will check for overflow at the end of the opcode loop in
309 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
310 */
311 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 312}
313
db6b7d0c 314/* Signal overflow, starting over with fewer guest insns. */
8905770b
MAL		 315static G_NORETURN
316void tcg_raise_tb_overflow(TCGContext *s)
db6b7d0c
RH		 317{
318 siglongjmp(s->jmp_trans, -2);
319}
320
4c22e840
RH		 321#define C_PFX1(P, A) P##A
322#define C_PFX2(P, A, B) P##A##_##B
323#define C_PFX3(P, A, B, C) P##A##_##B##_##C
324#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
325#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
326#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
327
328/* Define an enumeration for the various combinations. */
329
330#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
331#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
332#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
333#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
334
335#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
336#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
337#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
338#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
339
340#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
341
342#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
343#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
344#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
345#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
346
347typedef enum {
348#include "tcg-target-con-set.h"
349} TCGConstraintSetIndex;
350
351static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
352
353#undef C_O0_I1
354#undef C_O0_I2
355#undef C_O0_I3
356#undef C_O0_I4
357#undef C_O1_I1
358#undef C_O1_I2
359#undef C_O1_I3
360#undef C_O1_I4
361#undef C_N1_I2
362#undef C_O2_I1
363#undef C_O2_I2
364#undef C_O2_I3
365#undef C_O2_I4
366
367/* Put all of the constraint sets into an array, indexed by the enum. */
368
369#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
370#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
371#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
372#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
373
374#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
375#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
376#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
377#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
378
379#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
380
381#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
382#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
383#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
384#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
385
386static const TCGTargetOpDef constraint_sets[] = {
387#include "tcg-target-con-set.h"
388};
389
390
391#undef C_O0_I1
392#undef C_O0_I2
393#undef C_O0_I3
394#undef C_O0_I4
395#undef C_O1_I1
396#undef C_O1_I2
397#undef C_O1_I3
398#undef C_O1_I4
399#undef C_N1_I2
400#undef C_O2_I1
401#undef C_O2_I2
402#undef C_O2_I3
403#undef C_O2_I4
404
405/* Expand the enumerator to be returned from tcg_target_op_def(). */
406
407#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
408#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
409#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
410#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
411
412#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
413#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
414#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
415#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
416
417#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
418
419#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
420#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
421#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
422#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
423
139c1837 424#include "tcg-target.c.inc"
c896fe29 425
38b47b19
EC		 426static void alloc_tcg_plugin_context(TCGContext *s)
427{
428#ifdef CONFIG_PLUGIN
429 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
430 s->plugin_tb->insns =
431 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
432#endif
433}
434
3468b59e
EC		 435/*
436 * All TCG threads except the parent (i.e. the one that called tcg_context_init
437 * and registered the target's TCG globals) must register with this function
438 * before initiating translation.
439 *
440 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
441 * of tcg_region_init() for the reasoning behind this.
442 *
443 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
444 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
445 * is not used anymore for translation once this function is called.
446 *
447 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
448 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
449 */
450#ifdef CONFIG_USER_ONLY
451void tcg_register_thread(void)
452{
453 tcg_ctx = &tcg_init_ctx;
454}
455#else
456void tcg_register_thread(void)
457{
458 TCGContext *s = g_malloc(sizeof(*s));
459 unsigned int i, n;
3468b59e
EC		 460
461 *s = tcg_init_ctx;
462
463 /* Relink mem_base. */
464 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
465 if (tcg_init_ctx.temps[i].mem_base) {
466 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
467 tcg_debug_assert(b >= 0 && b < n);
468 s->temps[i].mem_base = &s->temps[b];
469 }
470 }
471
472 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 473 n = qatomic_fetch_inc(&tcg_cur_ctxs);
474 g_assert(n < tcg_max_ctxs);
d73415a3 475 qatomic_set(&tcg_ctxs[n], s);
3468b59e 476
38b47b19
EC		 477 if (n > 0) {
478 alloc_tcg_plugin_context(s);
bf042e8e 479 tcg_region_initial_alloc(s);
38b47b19
EC		 480 }
481
3468b59e 482 tcg_ctx = s;
e8feb96f 483}
3468b59e 484#endif /* !CONFIG_USER_ONLY */
e8feb96f 485
c896fe29
FB		 486/* pool based memory allocation */
487void *tcg_malloc_internal(TCGContext *s, int size)
488{
489 TCGPool *p;
490 int pool_size;
a813e36f 491
c896fe29
FB		 492 if (size > TCG_POOL_CHUNK_SIZE) {
493 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 494 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 495 p->size = size;
4055299e
KB		 496 p->next = s->pool_first_large;
497 s->pool_first_large = p;
498 return p->data;
c896fe29
FB		 499 } else {
500 p = s->pool_current;
501 if (!p) {
502 p = s->pool_first;
503 if (!p)
504 goto new_pool;
505 } else {
506 if (!p->next) {
507 new_pool:
508 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 509 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 510 p->size = pool_size;
511 p->next = NULL;
a813e36f 512 if (s->pool_current) {
c896fe29 513 s->pool_current->next = p;
a813e36f 514 } else {
c896fe29 515 s->pool_first = p;
a813e36f 516 }
c896fe29
FB		 517 } else {
518 p = p->next;
519 }
520 }
521 }
522 s->pool_current = p;
523 s->pool_cur = p->data + size;
524 s->pool_end = p->data + p->size;
525 return p->data;
526}
527
528void tcg_pool_reset(TCGContext *s)
529{
4055299e
KB		 530 TCGPool *p, *t;
531 for (p = s->pool_first_large; p; p = t) {
532 t = p->next;
533 g_free(p);
534 }
535 s->pool_first_large = NULL;
c896fe29
FB		 536 s->pool_cur = s->pool_end = NULL;
537 s->pool_current = NULL;
538}
539
2ef6175a
RH		 540#include "exec/helper-proto.h"
541
39004a71 542static TCGHelperInfo all_helpers[] = {
2ef6175a 543#include "exec/helper-tcg.h"
100b5e01 544};
619205fd 545static GHashTable *helper_table;
100b5e01 546
22f15579 547#ifdef CONFIG_TCG_INTERPRETER
c6ef8c7b
PMD		 548static ffi_type *typecode_to_ffi(int argmask)
549{
550 switch (argmask) {
551 case dh_typecode_void:
552 return &ffi_type_void;
553 case dh_typecode_i32:
554 return &ffi_type_uint32;
555 case dh_typecode_s32:
556 return &ffi_type_sint32;
557 case dh_typecode_i64:
558 return &ffi_type_uint64;
559 case dh_typecode_s64:
560 return &ffi_type_sint64;
561 case dh_typecode_ptr:
562 return &ffi_type_pointer;
563 }
564 g_assert_not_reached();
565}
0c22e176
PMD		 566
567static void init_ffi_layouts(void)
568{
569 /* g_direct_hash/equal for direct comparisons on uint32_t. */
f9c4bb80
RH		 570 GHashTable *ffi_table = g_hash_table_new(NULL, NULL);
571
0c22e176 572 for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
f9c4bb80
RH		 573 TCGHelperInfo *info = &all_helpers[i];
574 unsigned typemask = info->typemask;
0c22e176
PMD		 575 gpointer hash = (gpointer)(uintptr_t)typemask;
576 struct {
577 ffi_cif cif;
578 ffi_type *args[];
579 } *ca;
580 ffi_status status;
581 int nargs;
f9c4bb80 582 ffi_cif *cif;
0c22e176 583
f9c4bb80
RH		 584 cif = g_hash_table_lookup(ffi_table, hash);
585 if (cif) {
586 info->cif = cif;
0c22e176
PMD		 587 continue;
588 }
589
590 /* Ignoring the return type, find the last non-zero field. */
591 nargs = 32 - clz32(typemask >> 3);
592 nargs = DIV_ROUND_UP(nargs, 3);
593
594 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
595 ca->cif.rtype = typecode_to_ffi(typemask & 7);
596 ca->cif.nargs = nargs;
597
598 if (nargs != 0) {
599 ca->cif.arg_types = ca->args;
600 for (int j = 0; j < nargs; ++j) {
601 int typecode = extract32(typemask, (j + 1) * 3, 3);
602 ca->args[j] = typecode_to_ffi(typecode);
603 }
604 }
605
606 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
607 ca->cif.rtype, ca->cif.arg_types);
608 assert(status == FFI_OK);
609
f9c4bb80
RH		 610 cif = &ca->cif;
611 info->cif = cif;
612 g_hash_table_insert(ffi_table, hash, (gpointer)cif);
0c22e176 613 }
f9c4bb80
RH		 614
615 g_hash_table_destroy(ffi_table);
0c22e176
PMD		 616}
617#endif /* CONFIG_TCG_INTERPRETER */
22f15579 618
39004a71
RH		 619typedef struct TCGCumulativeArgs {
620 int arg_idx; /* tcg_gen_callN args[] */
621 int info_in_idx; /* TCGHelperInfo in[] */
622 int arg_slot; /* regs+stack slot */
623 int ref_slot; /* stack slots for references */
624} TCGCumulativeArgs;
625
626static void layout_arg_even(TCGCumulativeArgs *cum)
627{
628 cum->arg_slot += cum->arg_slot & 1;
629}
630
631static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
632 TCGCallArgumentKind kind)
633{
634 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
635
636 *loc = (TCGCallArgumentLoc){
637 .kind = kind,
638 .arg_idx = cum->arg_idx,
639 .arg_slot = cum->arg_slot,
640 };
641 cum->info_in_idx++;
642 cum->arg_slot++;
643}
644
645static void layout_arg_normal_n(TCGCumulativeArgs *cum,
646 TCGHelperInfo *info, int n)
647{
648 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
649
650 for (int i = 0; i < n; ++i) {
651 /* Layout all using the same arg_idx, adjusting the subindex. */
652 loc[i] = (TCGCallArgumentLoc){
653 .kind = TCG_CALL_ARG_NORMAL,
654 .arg_idx = cum->arg_idx,
655 .tmp_subindex = i,
656 .arg_slot = cum->arg_slot + i,
657 };
658 }
659 cum->info_in_idx += n;
660 cum->arg_slot += n;
661}
662
663static void init_call_layout(TCGHelperInfo *info)
664{
665 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
666 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
667 unsigned typemask = info->typemask;
668 unsigned typecode;
669 TCGCumulativeArgs cum = { };
670
671 /*
672 * Parse and place any function return value.
673 */
674 typecode = typemask & 7;
675 switch (typecode) {
676 case dh_typecode_void:
677 info->nr_out = 0;
678 break;
679 case dh_typecode_i32:
680 case dh_typecode_s32:
681 case dh_typecode_ptr:
682 info->nr_out = 1;
683 info->out_kind = TCG_CALL_RET_NORMAL;
684 break;
685 case dh_typecode_i64:
686 case dh_typecode_s64:
687 info->nr_out = 64 / TCG_TARGET_REG_BITS;
688 info->out_kind = TCG_CALL_RET_NORMAL;
689 break;
690 default:
691 g_assert_not_reached();
692 }
693 assert(info->nr_out <= ARRAY_SIZE(tcg_target_call_oarg_regs));
694
695 /*
696 * Parse and place function arguments.
697 */
698 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
699 TCGCallArgumentKind kind;
700 TCGType type;
701
702 typecode = typemask & 7;
703 switch (typecode) {
704 case dh_typecode_i32:
705 case dh_typecode_s32:
706 type = TCG_TYPE_I32;
707 break;
708 case dh_typecode_i64:
709 case dh_typecode_s64:
710 type = TCG_TYPE_I64;
711 break;
712 case dh_typecode_ptr:
713 type = TCG_TYPE_PTR;
714 break;
715 default:
716 g_assert_not_reached();
717 }
718
719 switch (type) {
720 case TCG_TYPE_I32:
721 switch (TCG_TARGET_CALL_ARG_I32) {
722 case TCG_CALL_ARG_EVEN:
723 layout_arg_even(&cum);
724 /* fall through */
725 case TCG_CALL_ARG_NORMAL:
726 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
727 break;
728 case TCG_CALL_ARG_EXTEND:
729 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
730 layout_arg_1(&cum, info, kind);
731 break;
732 default:
733 qemu_build_not_reached();
734 }
735 break;
736
737 case TCG_TYPE_I64:
738 switch (TCG_TARGET_CALL_ARG_I64) {
739 case TCG_CALL_ARG_EVEN:
740 layout_arg_even(&cum);
741 /* fall through */
742 case TCG_CALL_ARG_NORMAL:
743 if (TCG_TARGET_REG_BITS == 32) {
744 layout_arg_normal_n(&cum, info, 2);
745 } else {
746 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
747 }
748 break;
749 default:
750 qemu_build_not_reached();
751 }
752 break;
753
754 default:
755 g_assert_not_reached();
756 }
757 }
758 info->nr_in = cum.info_in_idx;
759
760 /* Validate that we didn't overrun the input array. */
761 assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
762 /* Validate the backend has enough argument space. */
763 assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
764 assert(cum.ref_slot <= max_stk_slots);
765}
766
91478cef 767static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 768static void process_op_defs(TCGContext *s);
1c2adb95
RH		 769static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
770 TCGReg reg, const char *name);
91478cef 771
43b972b7 772static void tcg_context_init(unsigned max_cpus)
c896fe29 773{
a76aabd3 774 TCGContext *s = &tcg_init_ctx;
100b5e01 775 int op, total_args, n, i;
c896fe29
FB		 776 TCGOpDef *def;
777 TCGArgConstraint *args_ct;
1c2adb95 778 TCGTemp *ts;
c896fe29
FB		 779
780 memset(s, 0, sizeof(*s));
c896fe29 781 s->nb_globals = 0;
c70fbf0a 782
c896fe29
FB		 783 /* Count total number of arguments and allocate the corresponding
784 space */
785 total_args = 0;
786 for(op = 0; op < NB_OPS; op++) {
787 def = &tcg_op_defs[op];
788 n = def->nb_iargs + def->nb_oargs;
789 total_args += n;
790 }
791
bc2b17e6 792 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 793
794 for(op = 0; op < NB_OPS; op++) {
795 def = &tcg_op_defs[op];
796 def->args_ct = args_ct;
c896fe29 797 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 798 args_ct += n;
799 }
5cd8f621
RH		 800
801 /* Register helpers. */
84fd9dd3 802 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 803 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 804
100b5e01 805 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
39004a71 806 init_call_layout(&all_helpers[i]);
84fd9dd3 807 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 808 (gpointer)&all_helpers[i]);
100b5e01 809 }
5cd8f621 810
22f15579 811#ifdef CONFIG_TCG_INTERPRETER
0c22e176 812 init_ffi_layouts();
22f15579
RH		 813#endif
814
c896fe29 815 tcg_target_init(s);
f69d277e 816 process_op_defs(s);
91478cef
RH		 817
818 /* Reverse the order of the saved registers, assuming they're all at
819 the start of tcg_target_reg_alloc_order. */
820 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
821 int r = tcg_target_reg_alloc_order[n];
822 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
823 break;
824 }
825 }
826 for (i = 0; i < n; ++i) {
827 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
828 }
829 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
830 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
831 }
b1311c4a 832
38b47b19
EC		 833 alloc_tcg_plugin_context(s);
834
b1311c4a 835 tcg_ctx = s;
3468b59e
EC		 836 /*
837 * In user-mode we simply share the init context among threads, since we
838 * use a single region. See the documentation tcg_region_init() for the
839 * reasoning behind this.
840 * In softmmu we will have at most max_cpus TCG threads.
841 */
842#ifdef CONFIG_USER_ONLY
df2cce29 843 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 844 tcg_cur_ctxs = 1;
845 tcg_max_ctxs = 1;
3468b59e 846#else
0e2d61cf
RH		 847 tcg_max_ctxs = max_cpus;
848 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 849#endif
1c2adb95
RH		 850
851 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
852 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
853 cpu_env = temp_tcgv_ptr(ts);
9002ec79 854}
b03cce8e 855
43b972b7 856void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 857{
43b972b7
RH		 858 tcg_context_init(max_cpus);
859 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 860}
861
6e3b2bfd
EC		 862/*
863 * Allocate TBs right before their corresponding translated code, making
864 * sure that TBs and code are on different cache lines.
865 */
866TranslationBlock *tcg_tb_alloc(TCGContext *s)
867{
868 uintptr_t align = qemu_icache_linesize;
869 TranslationBlock *tb;
870 void *next;
871
e8feb96f 872 retry:
6e3b2bfd
EC		 873 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
874 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
875
876 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 877 if (tcg_region_alloc(s)) {
878 return NULL;
879 }
880 goto retry;
6e3b2bfd 881 }
d73415a3 882 qatomic_set(&s->code_gen_ptr, next);
57a26946 883 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 884 return tb;
885}
886
9002ec79
RH		 887void tcg_prologue_init(TCGContext *s)
888{
b0a0794a 889 size_t prologue_size;
8163b749 890
b0a0794a
RH		 891 s->code_ptr = s->code_gen_ptr;
892 s->code_buf = s->code_gen_ptr;
5b38ee31 893 s->data_gen_ptr = NULL;
b91ccb31
RH		 894
895#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 896 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 897#endif
8163b749 898
5b38ee31
RH		 899#ifdef TCG_TARGET_NEED_POOL_LABELS
900 s->pool_labels = NULL;
901#endif
902
653b87eb 903 qemu_thread_jit_write();
8163b749 904 /* Generate the prologue. */
b03cce8e 905 tcg_target_qemu_prologue(s);
5b38ee31
RH		 906
907#ifdef TCG_TARGET_NEED_POOL_LABELS
908 /* Allow the prologue to put e.g. guest_base into a pool entry. */
909 {
1768987b
RH		 910 int result = tcg_out_pool_finalize(s);
911 tcg_debug_assert(result == 0);
5b38ee31
RH		 912 }
913#endif
914
b0a0794a
RH		 915 prologue_size = tcg_current_code_size(s);
916
df5d2b16 917#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 918 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
919 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 920#endif
8163b749 921
d6b64b2b
RH		 922#ifdef DEBUG_DISAS
923 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
c60f599b 924 FILE *logfile = qemu_log_trylock();
78b54858
RH		 925 if (logfile) {
926 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
927 if (s->data_gen_ptr) {
928 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
929 size_t data_size = prologue_size - code_size;
930 size_t i;
931
932 disas(logfile, s->code_gen_ptr, code_size);
933
934 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
935 if (sizeof(tcg_target_ulong) == 8) {
936 fprintf(logfile,
937 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
938 (uintptr_t)s->data_gen_ptr + i,
939 *(uint64_t *)(s->data_gen_ptr + i));
940 } else {
941 fprintf(logfile,
942 "0x%08" PRIxPTR ": .long 0x%08x\n",
943 (uintptr_t)s->data_gen_ptr + i,
944 *(uint32_t *)(s->data_gen_ptr + i));
945 }
5b38ee31 946 }
78b54858
RH		 947 } else {
948 disas(logfile, s->code_gen_ptr, prologue_size);
5b38ee31 949 }
78b54858 950 fprintf(logfile, "\n");
78b54858 951 qemu_log_unlock(logfile);
5b38ee31 952 }
d6b64b2b
RH		 953 }
954#endif
cedbcb01 955
6eea0434
RH		 956#ifndef CONFIG_TCG_INTERPRETER
957 /*
958 * Assert that goto_ptr is implemented completely, setting an epilogue.
959 * For tci, we use NULL as the signal to return from the interpreter,
960 * so skip this check.
961 */
f4e01e30 962 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 963#endif
d1c74ab3
RH		 964
965 tcg_region_prologue_set(s);
c896fe29
FB		 966}
967
c896fe29
FB		 968void tcg_func_start(TCGContext *s)
969{
970 tcg_pool_reset(s);
971 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 972
973 /* No temps have been previously allocated for size or locality. */
974 memset(s->free_temps, 0, sizeof(s->free_temps));
975
c0522136
RH		 976 /* No constant temps have been previously allocated. */
977 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
978 if (s->const_table[i]) {
979 g_hash_table_remove_all(s->const_table[i]);
980 }
981 }
982
abebf925 983 s->nb_ops = 0;
c896fe29
FB		 984 s->nb_labels = 0;
985 s->current_frame_offset = s->frame_start;
986
0a209d4b
RH		 987#ifdef CONFIG_DEBUG_TCG
988 s->goto_tb_issue_mask = 0;
989#endif
990
15fa08f8
RH		 991 QTAILQ_INIT(&s->ops);
992 QTAILQ_INIT(&s->free_ops);
bef16ab4 993 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 994}
995
ae30e866 996static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 997{
998 int n = s->nb_temps++;
ae30e866
RH		 999
1000 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 1001 tcg_raise_tb_overflow(s);
ae30e866 1002 }
7ca4b752
RH		 1003 return memset(&s->temps[n], 0, sizeof(TCGTemp));
1004}
1005
ae30e866 1006static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 1007{
fa477d25
RH		 1008 TCGTemp *ts;
1009
7ca4b752 1010 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 1011 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 1012 s->nb_globals++;
fa477d25 1013 ts = tcg_temp_alloc(s);
ee17db83 1014 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 1015
1016 return ts;
c896fe29
FB		 1017}
1018
085272b3
RH		 1019static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1020 TCGReg reg, const char *name)
c896fe29 1021{
c896fe29 1022 TCGTemp *ts;
c896fe29 1023
b3a62939 1024 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 1025 tcg_abort();
b3a62939 1026 }
7ca4b752
RH		 1027
1028 ts = tcg_global_alloc(s);
c896fe29
FB		 1029 ts->base_type = type;
1030 ts->type = type;
ee17db83 1031 ts->kind = TEMP_FIXED;
c896fe29 1032 ts->reg = reg;
c896fe29 1033 ts->name = name;
c896fe29 1034 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 1035
085272b3 1036 return ts;
a7812ae4
PB		 1037}
1038
b6638662 1039void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 1040{
b3a62939
RH		 1041 s->frame_start = start;
1042 s->frame_end = start + size;
085272b3
RH		 1043 s->frame_temp
1044 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 1045}
1046
085272b3
RH		 1047TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
1048 intptr_t offset, const char *name)
c896fe29 1049{
b1311c4a 1050 TCGContext *s = tcg_ctx;
dc41aa7d 1051 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 1052 TCGTemp *ts = tcg_global_alloc(s);
aef85402 1053 int indirect_reg = 0;
c896fe29 1054
c0522136
RH		 1055 switch (base_ts->kind) {
1056 case TEMP_FIXED:
1057 break;
1058 case TEMP_GLOBAL:
5a18407f
RH		 1059 /* We do not support double-indirect registers. */
1060 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 1061 base_ts->indirect_base = 1;
5a18407f
RH		 1062 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1063 ? 2 : 1);
1064 indirect_reg = 1;
c0522136
RH		 1065 break;
1066 default:
1067 g_assert_not_reached();
b3915dbb
RH		 1068 }
1069
7ca4b752
RH		 1070 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1071 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 1072 char buf[64];
7ca4b752
RH		 1073
1074 ts->base_type = TCG_TYPE_I64;
c896fe29 1075 ts->type = TCG_TYPE_I32;
b3915dbb 1076 ts->indirect_reg = indirect_reg;
c896fe29 1077 ts->mem_allocated = 1;
b3a62939 1078 ts->mem_base = base_ts;
aef85402 1079 ts->mem_offset = offset;
c896fe29
FB		 1080 pstrcpy(buf, sizeof(buf), name);
1081 pstrcat(buf, sizeof(buf), "_0");
1082 ts->name = strdup(buf);
c896fe29 1083
7ca4b752
RH		 1084 tcg_debug_assert(ts2 == ts + 1);
1085 ts2->base_type = TCG_TYPE_I64;
1086 ts2->type = TCG_TYPE_I32;
b3915dbb 1087 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 1088 ts2->mem_allocated = 1;
1089 ts2->mem_base = base_ts;
aef85402 1090 ts2->mem_offset = offset + 4;
fac87bd2 1091 ts2->temp_subindex = 1;
c896fe29
FB		 1092 pstrcpy(buf, sizeof(buf), name);
1093 pstrcat(buf, sizeof(buf), "_1");
120c1084 1094 ts2->name = strdup(buf);
7ca4b752 1095 } else {
c896fe29
FB		 1096 ts->base_type = type;
1097 ts->type = type;
b3915dbb 1098 ts->indirect_reg = indirect_reg;
c896fe29 1099 ts->mem_allocated = 1;
b3a62939 1100 ts->mem_base = base_ts;
c896fe29 1101 ts->mem_offset = offset;
c896fe29 1102 ts->name = name;
c896fe29 1103 }
085272b3 1104 return ts;
a7812ae4
PB		 1105}
1106
5bfa8034 1107TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
c896fe29 1108{
b1311c4a 1109 TCGContext *s = tcg_ctx;
ee17db83 1110 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
c896fe29 1111 TCGTemp *ts;
641d5fbe 1112 int idx, k;
c896fe29 1113
0ec9eabc
RH		 1114 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
1115 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
1116 if (idx < TCG_MAX_TEMPS) {
1117 /* There is already an available temp with the right type. */
1118 clear_bit(idx, s->free_temps[k].l);
1119
e8996ee0 1120 ts = &s->temps[idx];
e8996ee0 1121 ts->temp_allocated = 1;
7ca4b752 1122 tcg_debug_assert(ts->base_type == type);
ee17db83 1123 tcg_debug_assert(ts->kind == kind);
e8996ee0 1124 } else {
7ca4b752
RH		 1125 ts = tcg_temp_alloc(s);
1126 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1127 TCGTemp *ts2 = tcg_temp_alloc(s);
1128
f6aa2f7d 1129 ts->base_type = type;
e8996ee0
FB		 1130 ts->type = TCG_TYPE_I32;
1131 ts->temp_allocated = 1;
ee17db83 1132 ts->kind = kind;
7ca4b752
RH		 1133
1134 tcg_debug_assert(ts2 == ts + 1);
1135 ts2->base_type = TCG_TYPE_I64;
1136 ts2->type = TCG_TYPE_I32;
1137 ts2->temp_allocated = 1;
fac87bd2 1138 ts2->temp_subindex = 1;
ee17db83 1139 ts2->kind = kind;
7ca4b752 1140 } else {
e8996ee0
FB		 1141 ts->base_type = type;
1142 ts->type = type;
1143 ts->temp_allocated = 1;
ee17db83 1144 ts->kind = kind;
e8996ee0 1145 }
c896fe29 1146 }
27bfd83c
PM		 1147
1148#if defined(CONFIG_DEBUG_TCG)
1149 s->temps_in_use++;
1150#endif
085272b3 1151 return ts;
c896fe29
FB		 1152}
1153
d2fd745f
RH		 1154TCGv_vec tcg_temp_new_vec(TCGType type)
1155{
1156 TCGTemp *t;
1157
1158#ifdef CONFIG_DEBUG_TCG
1159 switch (type) {
1160 case TCG_TYPE_V64:
1161 assert(TCG_TARGET_HAS_v64);
1162 break;
1163 case TCG_TYPE_V128:
1164 assert(TCG_TARGET_HAS_v128);
1165 break;
1166 case TCG_TYPE_V256:
1167 assert(TCG_TARGET_HAS_v256);
1168 break;
1169 default:
1170 g_assert_not_reached();
1171 }
1172#endif
1173
1174 t = tcg_temp_new_internal(type, 0);
1175 return temp_tcgv_vec(t);
1176}
1177
1178/* Create a new temp of the same type as an existing temp. */
1179TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1180{
1181 TCGTemp *t = tcgv_vec_temp(match);
1182
1183 tcg_debug_assert(t->temp_allocated != 0);
1184
1185 t = tcg_temp_new_internal(t->base_type, 0);
1186 return temp_tcgv_vec(t);
1187}
1188
5bfa8034 1189void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1190{
b1311c4a 1191 TCGContext *s = tcg_ctx;
085272b3 1192 int k, idx;
c896fe29 1193
c7482438
RH		 1194 switch (ts->kind) {
1195 case TEMP_CONST:
1196 /*
1197 * In order to simplify users of tcg_constant_*,
1198 * silently ignore free.
1199 */
c0522136 1200 return;
c7482438
RH		 1201 case TEMP_NORMAL:
1202 case TEMP_LOCAL:
1203 break;
1204 default:
1205 g_assert_not_reached();
c0522136
RH		 1206 }
1207
27bfd83c
PM		 1208#if defined(CONFIG_DEBUG_TCG)
1209 s->temps_in_use--;
1210 if (s->temps_in_use < 0) {
1211 fprintf(stderr, "More temporaries freed than allocated!\n");
1212 }
1213#endif
1214
eabb7b91 1215 tcg_debug_assert(ts->temp_allocated != 0);
e8996ee0 1216 ts->temp_allocated = 0;
0ec9eabc 1217
085272b3 1218 idx = temp_idx(ts);
ee17db83 1219 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1220 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1221}
1222
c0522136
RH		 1223TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1224{
1225 TCGContext *s = tcg_ctx;
1226 GHashTable *h = s->const_table[type];
1227 TCGTemp *ts;
1228
1229 if (h == NULL) {
1230 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1231 s->const_table[type] = h;
1232 }
1233
1234 ts = g_hash_table_lookup(h, &val);
1235 if (ts == NULL) {
aef85402
RH		 1236 int64_t *val_ptr;
1237
c0522136
RH		 1238 ts = tcg_temp_alloc(s);
1239
1240 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1241 TCGTemp *ts2 = tcg_temp_alloc(s);
1242
aef85402
RH		 1243 tcg_debug_assert(ts2 == ts + 1);
1244
c0522136
RH		 1245 ts->base_type = TCG_TYPE_I64;
1246 ts->type = TCG_TYPE_I32;
1247 ts->kind = TEMP_CONST;
1248 ts->temp_allocated = 1;
c0522136 1249
c0522136
RH		 1250 ts2->base_type = TCG_TYPE_I64;
1251 ts2->type = TCG_TYPE_I32;
1252 ts2->kind = TEMP_CONST;
1253 ts2->temp_allocated = 1;
fac87bd2 1254 ts2->temp_subindex = 1;
aef85402
RH		 1255
1256 /*
1257 * Retain the full value of the 64-bit constant in the low
1258 * part, so that the hash table works. Actual uses will
1259 * truncate the value to the low part.
1260 */
1261 ts[HOST_BIG_ENDIAN].val = val;
1262 ts[!HOST_BIG_ENDIAN].val = val >> 32;
1263 val_ptr = &ts[HOST_BIG_ENDIAN].val;
c0522136
RH		 1264 } else {
1265 ts->base_type = type;
1266 ts->type = type;
1267 ts->kind = TEMP_CONST;
1268 ts->temp_allocated = 1;
1269 ts->val = val;
aef85402 1270 val_ptr = &ts->val;
c0522136 1271 }
aef85402 1272 g_hash_table_insert(h, val_ptr, ts);
c0522136
RH		 1273 }
1274
1275 return ts;
1276}
1277
1278TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1279{
1280 val = dup_const(vece, val);
1281 return temp_tcgv_vec(tcg_constant_internal(type, val));
1282}
1283
88d4005b
RH		 1284TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1285{
1286 TCGTemp *t = tcgv_vec_temp(match);
1287
1288 tcg_debug_assert(t->temp_allocated != 0);
1289 return tcg_constant_vec(t->base_type, vece, val);
1290}
1291
a7812ae4 1292TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1293{
a7812ae4
PB		 1294 TCGv_i32 t0;
1295 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1296 tcg_gen_movi_i32(t0, val);
1297 return t0;
1298}
c896fe29 1299
a7812ae4 1300TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1301{
a7812ae4
PB		 1302 TCGv_i64 t0;
1303 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1304 tcg_gen_movi_i64(t0, val);
1305 return t0;
c896fe29
FB		 1306}
1307
a7812ae4 1308TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1309{
a7812ae4
PB		 1310 TCGv_i32 t0;
1311 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1312 tcg_gen_movi_i32(t0, val);
1313 return t0;
1314}
1315
a7812ae4 1316TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1317{
a7812ae4
PB		 1318 TCGv_i64 t0;
1319 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1320 tcg_gen_movi_i64(t0, val);
1321 return t0;
1322}
1323
27bfd83c
PM		 1324#if defined(CONFIG_DEBUG_TCG)
1325void tcg_clear_temp_count(void)
1326{
b1311c4a 1327 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1328 s->temps_in_use = 0;
1329}
1330
1331int tcg_check_temp_count(void)
1332{
b1311c4a 1333 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1334 if (s->temps_in_use) {
1335 /* Clear the count so that we don't give another
1336 * warning immediately next time around.
1337 */
1338 s->temps_in_use = 0;
1339 return 1;
1340 }
1341 return 0;
1342}
1343#endif
1344
be0f34b5
RH		 1345/* Return true if OP may appear in the opcode stream.
1346 Test the runtime variable that controls each opcode. */
1347bool tcg_op_supported(TCGOpcode op)
1348{
d2fd745f
RH		 1349 const bool have_vec
1350 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1351
be0f34b5
RH		 1352 switch (op) {
1353 case INDEX_op_discard:
1354 case INDEX_op_set_label:
1355 case INDEX_op_call:
1356 case INDEX_op_br:
1357 case INDEX_op_mb:
1358 case INDEX_op_insn_start:
1359 case INDEX_op_exit_tb:
1360 case INDEX_op_goto_tb:
f4e01e30 1361 case INDEX_op_goto_ptr:
be0f34b5
RH		 1362 case INDEX_op_qemu_ld_i32:
1363 case INDEX_op_qemu_st_i32:
1364 case INDEX_op_qemu_ld_i64:
1365 case INDEX_op_qemu_st_i64:
1366 return true;
1367
07ce0b05
RH		 1368 case INDEX_op_qemu_st8_i32:
1369 return TCG_TARGET_HAS_qemu_st8_i32;
1370
be0f34b5 1371 case INDEX_op_mov_i32:
be0f34b5
RH		 1372 case INDEX_op_setcond_i32:
1373 case INDEX_op_brcond_i32:
1374 case INDEX_op_ld8u_i32:
1375 case INDEX_op_ld8s_i32:
1376 case INDEX_op_ld16u_i32:
1377 case INDEX_op_ld16s_i32:
1378 case INDEX_op_ld_i32:
1379 case INDEX_op_st8_i32:
1380 case INDEX_op_st16_i32:
1381 case INDEX_op_st_i32:
1382 case INDEX_op_add_i32:
1383 case INDEX_op_sub_i32:
1384 case INDEX_op_mul_i32:
1385 case INDEX_op_and_i32:
1386 case INDEX_op_or_i32:
1387 case INDEX_op_xor_i32:
1388 case INDEX_op_shl_i32:
1389 case INDEX_op_shr_i32:
1390 case INDEX_op_sar_i32:
1391 return true;
1392
1393 case INDEX_op_movcond_i32:
1394 return TCG_TARGET_HAS_movcond_i32;
1395 case INDEX_op_div_i32:
1396 case INDEX_op_divu_i32:
1397 return TCG_TARGET_HAS_div_i32;
1398 case INDEX_op_rem_i32:
1399 case INDEX_op_remu_i32:
1400 return TCG_TARGET_HAS_rem_i32;
1401 case INDEX_op_div2_i32:
1402 case INDEX_op_divu2_i32:
1403 return TCG_TARGET_HAS_div2_i32;
1404 case INDEX_op_rotl_i32:
1405 case INDEX_op_rotr_i32:
1406 return TCG_TARGET_HAS_rot_i32;
1407 case INDEX_op_deposit_i32:
1408 return TCG_TARGET_HAS_deposit_i32;
1409 case INDEX_op_extract_i32:
1410 return TCG_TARGET_HAS_extract_i32;
1411 case INDEX_op_sextract_i32:
1412 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1413 case INDEX_op_extract2_i32:
1414 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1415 case INDEX_op_add2_i32:
1416 return TCG_TARGET_HAS_add2_i32;
1417 case INDEX_op_sub2_i32:
1418 return TCG_TARGET_HAS_sub2_i32;
1419 case INDEX_op_mulu2_i32:
1420 return TCG_TARGET_HAS_mulu2_i32;
1421 case INDEX_op_muls2_i32:
1422 return TCG_TARGET_HAS_muls2_i32;
1423 case INDEX_op_muluh_i32:
1424 return TCG_TARGET_HAS_muluh_i32;
1425 case INDEX_op_mulsh_i32:
1426 return TCG_TARGET_HAS_mulsh_i32;
1427 case INDEX_op_ext8s_i32:
1428 return TCG_TARGET_HAS_ext8s_i32;
1429 case INDEX_op_ext16s_i32:
1430 return TCG_TARGET_HAS_ext16s_i32;
1431 case INDEX_op_ext8u_i32:
1432 return TCG_TARGET_HAS_ext8u_i32;
1433 case INDEX_op_ext16u_i32:
1434 return TCG_TARGET_HAS_ext16u_i32;
1435 case INDEX_op_bswap16_i32:
1436 return TCG_TARGET_HAS_bswap16_i32;
1437 case INDEX_op_bswap32_i32:
1438 return TCG_TARGET_HAS_bswap32_i32;
1439 case INDEX_op_not_i32:
1440 return TCG_TARGET_HAS_not_i32;
1441 case INDEX_op_neg_i32:
1442 return TCG_TARGET_HAS_neg_i32;
1443 case INDEX_op_andc_i32:
1444 return TCG_TARGET_HAS_andc_i32;
1445 case INDEX_op_orc_i32:
1446 return TCG_TARGET_HAS_orc_i32;
1447 case INDEX_op_eqv_i32:
1448 return TCG_TARGET_HAS_eqv_i32;
1449 case INDEX_op_nand_i32:
1450 return TCG_TARGET_HAS_nand_i32;
1451 case INDEX_op_nor_i32:
1452 return TCG_TARGET_HAS_nor_i32;
1453 case INDEX_op_clz_i32:
1454 return TCG_TARGET_HAS_clz_i32;
1455 case INDEX_op_ctz_i32:
1456 return TCG_TARGET_HAS_ctz_i32;
1457 case INDEX_op_ctpop_i32:
1458 return TCG_TARGET_HAS_ctpop_i32;
1459
1460 case INDEX_op_brcond2_i32:
1461 case INDEX_op_setcond2_i32:
1462 return TCG_TARGET_REG_BITS == 32;
1463
1464 case INDEX_op_mov_i64:
be0f34b5
RH		 1465 case INDEX_op_setcond_i64:
1466 case INDEX_op_brcond_i64:
1467 case INDEX_op_ld8u_i64:
1468 case INDEX_op_ld8s_i64:
1469 case INDEX_op_ld16u_i64:
1470 case INDEX_op_ld16s_i64:
1471 case INDEX_op_ld32u_i64:
1472 case INDEX_op_ld32s_i64:
1473 case INDEX_op_ld_i64:
1474 case INDEX_op_st8_i64:
1475 case INDEX_op_st16_i64:
1476 case INDEX_op_st32_i64:
1477 case INDEX_op_st_i64:
1478 case INDEX_op_add_i64:
1479 case INDEX_op_sub_i64:
1480 case INDEX_op_mul_i64:
1481 case INDEX_op_and_i64:
1482 case INDEX_op_or_i64:
1483 case INDEX_op_xor_i64:
1484 case INDEX_op_shl_i64:
1485 case INDEX_op_shr_i64:
1486 case INDEX_op_sar_i64:
1487 case INDEX_op_ext_i32_i64:
1488 case INDEX_op_extu_i32_i64:
1489 return TCG_TARGET_REG_BITS == 64;
1490
1491 case INDEX_op_movcond_i64:
1492 return TCG_TARGET_HAS_movcond_i64;
1493 case INDEX_op_div_i64:
1494 case INDEX_op_divu_i64:
1495 return TCG_TARGET_HAS_div_i64;
1496 case INDEX_op_rem_i64:
1497 case INDEX_op_remu_i64:
1498 return TCG_TARGET_HAS_rem_i64;
1499 case INDEX_op_div2_i64:
1500 case INDEX_op_divu2_i64:
1501 return TCG_TARGET_HAS_div2_i64;
1502 case INDEX_op_rotl_i64:
1503 case INDEX_op_rotr_i64:
1504 return TCG_TARGET_HAS_rot_i64;
1505 case INDEX_op_deposit_i64:
1506 return TCG_TARGET_HAS_deposit_i64;
1507 case INDEX_op_extract_i64:
1508 return TCG_TARGET_HAS_extract_i64;
1509 case INDEX_op_sextract_i64:
1510 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1511 case INDEX_op_extract2_i64:
1512 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1513 case INDEX_op_extrl_i64_i32:
1514 return TCG_TARGET_HAS_extrl_i64_i32;
1515 case INDEX_op_extrh_i64_i32:
1516 return TCG_TARGET_HAS_extrh_i64_i32;
1517 case INDEX_op_ext8s_i64:
1518 return TCG_TARGET_HAS_ext8s_i64;
1519 case INDEX_op_ext16s_i64:
1520 return TCG_TARGET_HAS_ext16s_i64;
1521 case INDEX_op_ext32s_i64:
1522 return TCG_TARGET_HAS_ext32s_i64;
1523 case INDEX_op_ext8u_i64:
1524 return TCG_TARGET_HAS_ext8u_i64;
1525 case INDEX_op_ext16u_i64:
1526 return TCG_TARGET_HAS_ext16u_i64;
1527 case INDEX_op_ext32u_i64:
1528 return TCG_TARGET_HAS_ext32u_i64;
1529 case INDEX_op_bswap16_i64:
1530 return TCG_TARGET_HAS_bswap16_i64;
1531 case INDEX_op_bswap32_i64:
1532 return TCG_TARGET_HAS_bswap32_i64;
1533 case INDEX_op_bswap64_i64:
1534 return TCG_TARGET_HAS_bswap64_i64;
1535 case INDEX_op_not_i64:
1536 return TCG_TARGET_HAS_not_i64;
1537 case INDEX_op_neg_i64:
1538 return TCG_TARGET_HAS_neg_i64;
1539 case INDEX_op_andc_i64:
1540 return TCG_TARGET_HAS_andc_i64;
1541 case INDEX_op_orc_i64:
1542 return TCG_TARGET_HAS_orc_i64;
1543 case INDEX_op_eqv_i64:
1544 return TCG_TARGET_HAS_eqv_i64;
1545 case INDEX_op_nand_i64:
1546 return TCG_TARGET_HAS_nand_i64;
1547 case INDEX_op_nor_i64:
1548 return TCG_TARGET_HAS_nor_i64;
1549 case INDEX_op_clz_i64:
1550 return TCG_TARGET_HAS_clz_i64;
1551 case INDEX_op_ctz_i64:
1552 return TCG_TARGET_HAS_ctz_i64;
1553 case INDEX_op_ctpop_i64:
1554 return TCG_TARGET_HAS_ctpop_i64;
1555 case INDEX_op_add2_i64:
1556 return TCG_TARGET_HAS_add2_i64;
1557 case INDEX_op_sub2_i64:
1558 return TCG_TARGET_HAS_sub2_i64;
1559 case INDEX_op_mulu2_i64:
1560 return TCG_TARGET_HAS_mulu2_i64;
1561 case INDEX_op_muls2_i64:
1562 return TCG_TARGET_HAS_muls2_i64;
1563 case INDEX_op_muluh_i64:
1564 return TCG_TARGET_HAS_muluh_i64;
1565 case INDEX_op_mulsh_i64:
1566 return TCG_TARGET_HAS_mulsh_i64;
1567
d2fd745f
RH		 1568 case INDEX_op_mov_vec:
1569 case INDEX_op_dup_vec:
37ee55a0 1570 case INDEX_op_dupm_vec:
d2fd745f
RH		 1571 case INDEX_op_ld_vec:
1572 case INDEX_op_st_vec:
1573 case INDEX_op_add_vec:
1574 case INDEX_op_sub_vec:
1575 case INDEX_op_and_vec:
1576 case INDEX_op_or_vec:
1577 case INDEX_op_xor_vec:
212be173 1578 case INDEX_op_cmp_vec:
d2fd745f
RH		 1579 return have_vec;
1580 case INDEX_op_dup2_vec:
1581 return have_vec && TCG_TARGET_REG_BITS == 32;
1582 case INDEX_op_not_vec:
1583 return have_vec && TCG_TARGET_HAS_not_vec;
1584 case INDEX_op_neg_vec:
1585 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1586 case INDEX_op_abs_vec:
1587 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1588 case INDEX_op_andc_vec:
1589 return have_vec && TCG_TARGET_HAS_andc_vec;
1590 case INDEX_op_orc_vec:
1591 return have_vec && TCG_TARGET_HAS_orc_vec;
ed523473
RH		 1592 case INDEX_op_nand_vec:
1593 return have_vec && TCG_TARGET_HAS_nand_vec;
1594 case INDEX_op_nor_vec:
1595 return have_vec && TCG_TARGET_HAS_nor_vec;
1596 case INDEX_op_eqv_vec:
1597 return have_vec && TCG_TARGET_HAS_eqv_vec;
3774030a
RH		 1598 case INDEX_op_mul_vec:
1599 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1600 case INDEX_op_shli_vec:
1601 case INDEX_op_shri_vec:
1602 case INDEX_op_sari_vec:
1603 return have_vec && TCG_TARGET_HAS_shi_vec;
1604 case INDEX_op_shls_vec:
1605 case INDEX_op_shrs_vec:
1606 case INDEX_op_sars_vec:
1607 return have_vec && TCG_TARGET_HAS_shs_vec;
1608 case INDEX_op_shlv_vec:
1609 case INDEX_op_shrv_vec:
1610 case INDEX_op_sarv_vec:
1611 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1612 case INDEX_op_rotli_vec:
1613 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1614 case INDEX_op_rotls_vec:
1615 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1616 case INDEX_op_rotlv_vec:
1617 case INDEX_op_rotrv_vec:
1618 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1619 case INDEX_op_ssadd_vec:
1620 case INDEX_op_usadd_vec:
1621 case INDEX_op_sssub_vec:
1622 case INDEX_op_ussub_vec:
1623 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1624 case INDEX_op_smin_vec:
1625 case INDEX_op_umin_vec:
1626 case INDEX_op_smax_vec:
1627 case INDEX_op_umax_vec:
1628 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1629 case INDEX_op_bitsel_vec:
1630 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1631 case INDEX_op_cmpsel_vec:
1632 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1633
db432672
RH		 1634 default:
1635 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1636 return true;
be0f34b5 1637 }
be0f34b5
RH		 1638}
1639
39004a71
RH		 1640static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
1641
ae8b75dc 1642void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1643{
3e92aa34 1644 const TCGHelperInfo *info;
39004a71
RH		 1645 TCGv_i64 extend_free[MAX_CALL_IARGS];
1646 int n_extend = 0;
75e8b9b7 1647 TCGOp *op;
39004a71 1648 int i, n, pi = 0, total_args;
afb49896 1649
619205fd 1650 info = g_hash_table_lookup(helper_table, (gpointer)func);
39004a71
RH		 1651 total_args = info->nr_out + info->nr_in + 2;
1652 op = tcg_op_alloc(INDEX_op_call, total_args);
2bece2c8 1653
38b47b19
EC		 1654#ifdef CONFIG_PLUGIN
1655 /* detect non-plugin helpers */
1656 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1657 tcg_ctx->plugin_insn->calls_helpers = true;
1658 }
1659#endif
1660
39004a71
RH		 1661 TCGOP_CALLO(op) = n = info->nr_out;
1662 switch (n) {
1663 case 0:
1664 tcg_debug_assert(ret == NULL);
1665 break;
1666 case 1:
1667 tcg_debug_assert(ret != NULL);
1668 op->args[pi++] = temp_arg(ret);
1669 break;
1670 case 2:
1671 tcg_debug_assert(ret != NULL);
1672 tcg_debug_assert(ret->base_type == ret->type + 1);
1673 tcg_debug_assert(ret->temp_subindex == 0);
1674 op->args[pi++] = temp_arg(ret);
1675 op->args[pi++] = temp_arg(ret + 1);
1676 break;
1677 default:
1678 g_assert_not_reached();
1679 }
1680
1681 TCGOP_CALLI(op) = n = info->nr_in;
1682 for (i = 0; i < n; i++) {
1683 const TCGCallArgumentLoc *loc = &info->in[i];
1684 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
1685
1686 switch (loc->kind) {
1687 case TCG_CALL_ARG_NORMAL:
1688 op->args[pi++] = temp_arg(ts);
1689 break;
eb8b0224 1690
39004a71
RH		 1691 case TCG_CALL_ARG_EXTEND_U:
1692 case TCG_CALL_ARG_EXTEND_S:
1693 {
eb8b0224 1694 TCGv_i64 temp = tcg_temp_new_i64();
39004a71
RH		 1695 TCGv_i32 orig = temp_tcgv_i32(ts);
1696
1697 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
eb8b0224
RH		 1698 tcg_gen_ext_i32_i64(temp, orig);
1699 } else {
1700 tcg_gen_extu_i32_i64(temp, orig);
1701 }
39004a71
RH		 1702 op->args[pi++] = tcgv_i64_arg(temp);
1703 extend_free[n_extend++] = temp;
2bece2c8 1704 }
e2a9dd6b 1705 break;
7b7d8b2d 1706
e2a9dd6b
RH		 1707 default:
1708 g_assert_not_reached();
c896fe29
FB		 1709 }
1710 }
75e8b9b7 1711 op->args[pi++] = (uintptr_t)func;
3e92aa34 1712 op->args[pi++] = (uintptr_t)info;
39004a71 1713 tcg_debug_assert(pi == total_args);
a7812ae4 1714
39004a71 1715 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
7319d83a 1716
39004a71
RH		 1717 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
1718 for (i = 0; i < n_extend; ++i) {
1719 tcg_temp_free_i64(extend_free[i]);
2bece2c8 1720 }
c896fe29 1721}
c896fe29 1722
8fcd3692 1723static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1724{
ac3b8891 1725 int i, n;
ac3b8891 1726
ee17db83
RH		 1727 for (i = 0, n = s->nb_temps; i < n; i++) {
1728 TCGTemp *ts = &s->temps[i];
1729 TCGTempVal val = TEMP_VAL_MEM;
1730
1731 switch (ts->kind) {
c0522136
RH		 1732 case TEMP_CONST:
1733 val = TEMP_VAL_CONST;
1734 break;
ee17db83
RH		 1735 case TEMP_FIXED:
1736 val = TEMP_VAL_REG;
1737 break;
1738 case TEMP_GLOBAL:
1739 break;
1740 case TEMP_NORMAL:
c7482438 1741 case TEMP_EBB:
ee17db83
RH		 1742 val = TEMP_VAL_DEAD;
1743 /* fall through */
1744 case TEMP_LOCAL:
1745 ts->mem_allocated = 0;
1746 break;
1747 default:
1748 g_assert_not_reached();
1749 }
1750 ts->val_type = val;
e8996ee0 1751 }
f8b2f202
RH		 1752
1753 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1754}
1755
f8b2f202
RH		 1756static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1757 TCGTemp *ts)
c896fe29 1758{
1807f4c4 1759 int idx = temp_idx(ts);
ac56dd48 1760
ee17db83
RH		 1761 switch (ts->kind) {
1762 case TEMP_FIXED:
1763 case TEMP_GLOBAL:
ac56dd48 1764 pstrcpy(buf, buf_size, ts->name);
ee17db83
RH		 1765 break;
1766 case TEMP_LOCAL:
f8b2f202 1767 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83 1768 break;
c7482438
RH		 1769 case TEMP_EBB:
1770 snprintf(buf, buf_size, "ebb%d", idx - s->nb_globals);
1771 break;
ee17db83 1772 case TEMP_NORMAL:
f8b2f202 1773 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1774 break;
c0522136
RH		 1775 case TEMP_CONST:
1776 switch (ts->type) {
1777 case TCG_TYPE_I32:
1778 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1779 break;
1780#if TCG_TARGET_REG_BITS > 32
1781 case TCG_TYPE_I64:
1782 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1783 break;
1784#endif
1785 case TCG_TYPE_V64:
1786 case TCG_TYPE_V128:
1787 case TCG_TYPE_V256:
1788 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1789 64 << (ts->type - TCG_TYPE_V64), ts->val);
1790 break;
1791 default:
1792 g_assert_not_reached();
1793 }
1794 break;
c896fe29
FB		 1795 }
1796 return buf;
1797}
1798
43439139
RH		 1799static char *tcg_get_arg_str(TCGContext *s, char *buf,
1800 int buf_size, TCGArg arg)
f8b2f202 1801{
43439139 1802 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1803}
1804
f48f3ede
BS		 1805static const char * const cond_name[] =
1806{
0aed257f
RH		 1807 [TCG_COND_NEVER] = "never",
1808 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1809 [TCG_COND_EQ] = "eq",
1810 [TCG_COND_NE] = "ne",
1811 [TCG_COND_LT] = "lt",
1812 [TCG_COND_GE] = "ge",
1813 [TCG_COND_LE] = "le",
1814 [TCG_COND_GT] = "gt",
1815 [TCG_COND_LTU] = "ltu",
1816 [TCG_COND_GEU] = "geu",
1817 [TCG_COND_LEU] = "leu",
1818 [TCG_COND_GTU] = "gtu"
1819};
1820
f713d6ad
RH		 1821static const char * const ldst_name[] =
1822{
1823 [MO_UB] = "ub",
1824 [MO_SB] = "sb",
1825 [MO_LEUW] = "leuw",
1826 [MO_LESW] = "lesw",
1827 [MO_LEUL] = "leul",
1828 [MO_LESL] = "lesl",
fc313c64 1829 [MO_LEUQ] = "leq",
f713d6ad
RH		 1830 [MO_BEUW] = "beuw",
1831 [MO_BESW] = "besw",
1832 [MO_BEUL] = "beul",
1833 [MO_BESL] = "besl",
fc313c64 1834 [MO_BEUQ] = "beq",
f713d6ad
RH		 1835};
1836
1f00b27f 1837static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 1838#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 1839 [MO_UNALN >> MO_ASHIFT] = "un+",
1840 [MO_ALIGN >> MO_ASHIFT] = "",
1841#else
1842 [MO_UNALN >> MO_ASHIFT] = "",
1843 [MO_ALIGN >> MO_ASHIFT] = "al+",
1844#endif
1845 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1846 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1847 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1848 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1849 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1850 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1851};
1852
587195bd
RH		 1853static const char bswap_flag_name[][6] = {
1854 [TCG_BSWAP_IZ] = "iz",
1855 [TCG_BSWAP_OZ] = "oz",
1856 [TCG_BSWAP_OS] = "os",
1857 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1858 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1859};
1860
b016486e
RH		 1861static inline bool tcg_regset_single(TCGRegSet d)
1862{
1863 return (d & (d - 1)) == 0;
1864}
1865
1866static inline TCGReg tcg_regset_first(TCGRegSet d)
1867{
1868 if (TCG_TARGET_NB_REGS <= 32) {
1869 return ctz32(d);
1870 } else {
1871 return ctz64(d);
1872 }
1873}
1874
b7a83ff8
RH		 1875/* Return only the number of characters output -- no error return. */
1876#define ne_fprintf(...) \
1877 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
1878
1879static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
c896fe29 1880{
c896fe29 1881 char buf[128];
c45cb8bb 1882 TCGOp *op;
c45cb8bb 1883
15fa08f8 1884 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 1885 int i, k, nb_oargs, nb_iargs, nb_cargs;
1886 const TCGOpDef *def;
c45cb8bb 1887 TCGOpcode c;
bdfb460e 1888 int col = 0;
c896fe29 1889
c45cb8bb 1890 c = op->opc;
c896fe29 1891 def = &tcg_op_defs[c];
c45cb8bb 1892
765b842a 1893 if (c == INDEX_op_insn_start) {
b016486e 1894 nb_oargs = 0;
b7a83ff8 1895 col += ne_fprintf(f, "\n ----");
9aef40ed
RH		 1896
1897 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1898 target_ulong a;
7e4597d7 1899#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 1900 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 1901#else
efee3746 1902 a = op->args[i];
7e4597d7 1903#endif
b7a83ff8 1904 col += ne_fprintf(f, " " TARGET_FMT_lx, a);
eeacee4d 1905 }
7e4597d7 1906 } else if (c == INDEX_op_call) {
3e92aa34 1907 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 1908 void *func = tcg_call_func(op);
3e92aa34 1909
c896fe29 1910 /* variable number of arguments */
cd9090aa
RH		 1911 nb_oargs = TCGOP_CALLO(op);
1912 nb_iargs = TCGOP_CALLI(op);
c896fe29 1913 nb_cargs = def->nb_cargs;
c896fe29 1914
b7a83ff8 1915 col += ne_fprintf(f, " %s ", def->name);
3e92aa34
RH		 1916
1917 /*
1918 * Print the function name from TCGHelperInfo, if available.
1919 * Note that plugins have a template function for the info,
1920 * but the actual function pointer comes from the plugin.
1921 */
3e92aa34 1922 if (func == info->func) {
b7a83ff8 1923 col += ne_fprintf(f, "%s", info->name);
3e92aa34 1924 } else {
b7a83ff8 1925 col += ne_fprintf(f, "plugin(%p)", func);
3e92aa34
RH		 1926 }
1927
b7a83ff8 1928 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
cf066674 1929 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1930 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1931 op->args[i]));
b03cce8e 1932 }
cf066674 1933 for (i = 0; i < nb_iargs; i++) {
efee3746 1934 TCGArg arg = op->args[nb_oargs + i];
39004a71 1935 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
b7a83ff8 1936 col += ne_fprintf(f, ",%s", t);
e8996ee0 1937 }
b03cce8e 1938 } else {
b7a83ff8 1939 col += ne_fprintf(f, " %s ", def->name);
c45cb8bb
RH		 1940
1941 nb_oargs = def->nb_oargs;
1942 nb_iargs = def->nb_iargs;
1943 nb_cargs = def->nb_cargs;
1944
d2fd745f 1945 if (def->flags & TCG_OPF_VECTOR) {
b7a83ff8
RH		 1946 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
1947 8 << TCGOP_VECE(op));
d2fd745f
RH		 1948 }
1949
b03cce8e 1950 k = 0;
c45cb8bb 1951 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1952 const char *sep = k ? "," : "";
1953 col += ne_fprintf(f, "%s%s", sep,
1954 tcg_get_arg_str(s, buf, sizeof(buf),
1955 op->args[k++]));
b03cce8e 1956 }
c45cb8bb 1957 for (i = 0; i < nb_iargs; i++) {
b7a83ff8
RH		 1958 const char *sep = k ? "," : "";
1959 col += ne_fprintf(f, "%s%s", sep,
1960 tcg_get_arg_str(s, buf, sizeof(buf),
1961 op->args[k++]));
b03cce8e 1962 }
be210acb
RH		 1963 switch (c) {
1964 case INDEX_op_brcond_i32:
be210acb 1965 case INDEX_op_setcond_i32:
ffc5ea09 1966 case INDEX_op_movcond_i32:
ffc5ea09 1967 case INDEX_op_brcond2_i32:
be210acb 1968 case INDEX_op_setcond2_i32:
ffc5ea09 1969 case INDEX_op_brcond_i64:
be210acb 1970 case INDEX_op_setcond_i64:
ffc5ea09 1971 case INDEX_op_movcond_i64:
212be173 1972 case INDEX_op_cmp_vec:
f75da298 1973 case INDEX_op_cmpsel_vec:
efee3746
RH		 1974 if (op->args[k] < ARRAY_SIZE(cond_name)
1975 && cond_name[op->args[k]]) {
b7a83ff8 1976 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
eeacee4d 1977 } else {
b7a83ff8 1978 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 1979 }
f48f3ede 1980 i = 1;
be210acb 1981 break;
f713d6ad
RH		 1982 case INDEX_op_qemu_ld_i32:
1983 case INDEX_op_qemu_st_i32:
07ce0b05 1984 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 1985 case INDEX_op_qemu_ld_i64:
1986 case INDEX_op_qemu_st_i64:
59227d5d 1987 {
9002ffcb 1988 MemOpIdx oi = op->args[k++];
14776ab5 1989 MemOp op = get_memop(oi);
59227d5d
RH		 1990 unsigned ix = get_mmuidx(oi);
1991
59c4b7e8 1992 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
b7a83ff8 1993 col += ne_fprintf(f, ",$0x%x,%u", op, ix);
59c4b7e8 1994 } else {
1f00b27f
SS		 1995 const char *s_al, *s_op;
1996 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 1997 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
b7a83ff8 1998 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 1999 }
2000 i = 1;
f713d6ad 2001 }
f713d6ad 2002 break;
587195bd
RH		 2003 case INDEX_op_bswap16_i32:
2004 case INDEX_op_bswap16_i64:
2005 case INDEX_op_bswap32_i32:
2006 case INDEX_op_bswap32_i64:
2007 case INDEX_op_bswap64_i64:
2008 {
2009 TCGArg flags = op->args[k];
2010 const char *name = NULL;
2011
2012 if (flags < ARRAY_SIZE(bswap_flag_name)) {
2013 name = bswap_flag_name[flags];
2014 }
2015 if (name) {
b7a83ff8 2016 col += ne_fprintf(f, ",%s", name);
587195bd 2017 } else {
b7a83ff8 2018 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
587195bd
RH		 2019 }
2020 i = k = 1;
2021 }
2022 break;
be210acb 2023 default:
f48f3ede 2024 i = 0;
be210acb
RH		 2025 break;
2026 }
51e3972c
RH		 2027 switch (c) {
2028 case INDEX_op_set_label:
2029 case INDEX_op_br:
2030 case INDEX_op_brcond_i32:
2031 case INDEX_op_brcond_i64:
2032 case INDEX_op_brcond2_i32:
b7a83ff8
RH		 2033 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
2034 arg_label(op->args[k])->id);
51e3972c
RH		 2035 i++, k++;
2036 break;
2037 default:
2038 break;
2039 }
2040 for (; i < nb_cargs; i++, k++) {
b7a83ff8
RH		 2041 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
2042 op->args[k]);
bdfb460e
RH		 2043 }
2044 }
bdfb460e 2045
1894f69a 2046 if (have_prefs || op->life) {
b7a83ff8
RH		 2047 for (; col < 40; ++col) {
2048 putc(' ', f);
bdfb460e 2049 }
1894f69a
RH		 2050 }
2051
2052 if (op->life) {
2053 unsigned life = op->life;
bdfb460e
RH		 2054
2055 if (life & (SYNC_ARG * 3)) {
b7a83ff8 2056 ne_fprintf(f, " sync:");
bdfb460e
RH		 2057 for (i = 0; i < 2; ++i) {
2058 if (life & (SYNC_ARG << i)) {
b7a83ff8 2059 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2060 }
2061 }
2062 }
2063 life /= DEAD_ARG;
2064 if (life) {
b7a83ff8 2065 ne_fprintf(f, " dead:");
bdfb460e
RH		 2066 for (i = 0; life; ++i, life >>= 1) {
2067 if (life & 1) {
b7a83ff8 2068 ne_fprintf(f, " %d", i);
bdfb460e
RH		 2069 }
2070 }
b03cce8e 2071 }
c896fe29 2072 }
1894f69a
RH		 2073
2074 if (have_prefs) {
2075 for (i = 0; i < nb_oargs; ++i) {
31fd884b 2076 TCGRegSet set = output_pref(op, i);
1894f69a
RH		 2077
2078 if (i == 0) {
b7a83ff8 2079 ne_fprintf(f, " pref=");
1894f69a 2080 } else {
b7a83ff8 2081 ne_fprintf(f, ",");
1894f69a
RH		 2082 }
2083 if (set == 0) {
b7a83ff8 2084 ne_fprintf(f, "none");
1894f69a 2085 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
b7a83ff8 2086 ne_fprintf(f, "all");
1894f69a
RH		 2087#ifdef CONFIG_DEBUG_TCG
2088 } else if (tcg_regset_single(set)) {
2089 TCGReg reg = tcg_regset_first(set);
b7a83ff8 2090 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
1894f69a
RH		 2091#endif
2092 } else if (TCG_TARGET_NB_REGS <= 32) {
b7a83ff8 2093 ne_fprintf(f, "0x%x", (uint32_t)set);
1894f69a 2094 } else {
b7a83ff8 2095 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
1894f69a
RH		 2096 }
2097 }
2098 }
2099
b7a83ff8 2100 putc('\n', f);
c896fe29
FB		 2101 }
2102}
2103
2104/* we give more priority to constraints with less registers */
2105static int get_constraint_priority(const TCGOpDef *def, int k)
2106{
74a11790 2107 const TCGArgConstraint *arg_ct = &def->args_ct[k];
29f5e925 2108 int n = ctpop64(arg_ct->regs);
c896fe29 2109
29f5e925
RH		 2110 /*
2111 * Sort constraints of a single register first, which includes output
2112 * aliases (which must exactly match the input already allocated).
2113 */
2114 if (n == 1 || arg_ct->oalias) {
2115 return INT_MAX;
2116 }
2117
2118 /*
2119 * Sort register pairs next, first then second immediately after.
2120 * Arbitrarily sort multiple pairs by the index of the first reg;
2121 * there shouldn't be many pairs.
2122 */
2123 switch (arg_ct->pair) {
2124 case 1:
2125 case 3:
2126 return (k + 1) * 2;
2127 case 2:
2128 return (arg_ct->pair_index + 1) * 2 - 1;
c896fe29 2129 }
29f5e925
RH		 2130
2131 /* Finally, sort by decreasing register count. */
2132 assert(n > 1);
2133 return -n;
c896fe29
FB		 2134}
2135
2136/* sort from highest priority to lowest */
2137static void sort_constraints(TCGOpDef *def, int start, int n)
2138{
66792f90
RH		 2139 int i, j;
2140 TCGArgConstraint *a = def->args_ct;
c896fe29 2141
66792f90
RH		 2142 for (i = 0; i < n; i++) {
2143 a[start + i].sort_index = start + i;
2144 }
2145 if (n <= 1) {
c896fe29 2146 return;
66792f90
RH		 2147 }
2148 for (i = 0; i < n - 1; i++) {
2149 for (j = i + 1; j < n; j++) {
2150 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2151 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2152 if (p1 < p2) {
66792f90
RH		 2153 int tmp = a[start + i].sort_index;
2154 a[start + i].sort_index = a[start + j].sort_index;
2155 a[start + j].sort_index = tmp;
c896fe29
FB		 2156 }
2157 }
2158 }
2159}
2160
f69d277e 2161static void process_op_defs(TCGContext *s)
c896fe29 2162{
a9751609 2163 TCGOpcode op;
c896fe29 2164
f69d277e
RH		 2165 for (op = 0; op < NB_OPS; op++) {
2166 TCGOpDef *def = &tcg_op_defs[op];
2167 const TCGTargetOpDef *tdefs;
29f5e925
RH		 2168 bool saw_alias_pair = false;
2169 int i, o, i2, o2, nb_args;
f69d277e
RH		 2170
2171 if (def->flags & TCG_OPF_NOT_PRESENT) {
2172 continue;
2173 }
2174
c896fe29 2175 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2176 if (nb_args == 0) {
2177 continue;
2178 }
2179
4c22e840
RH		 2180 /*
2181 * Macro magic should make it impossible, but double-check that
2182 * the array index is in range. Since the signness of an enum
2183 * is implementation defined, force the result to unsigned.
2184 */
2185 unsigned con_set = tcg_target_op_def(op);
2186 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2187 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2188
2189 for (i = 0; i < nb_args; i++) {
2190 const char *ct_str = tdefs->args_ct_str[i];
8940ea0d
PMD		 2191 bool input_p = i >= def->nb_oargs;
2192
f69d277e 2193 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2194 tcg_debug_assert(ct_str != NULL);
f69d277e 2195
8940ea0d
PMD		 2196 switch (*ct_str) {
2197 case '0' ... '9':
2198 o = *ct_str - '0';
2199 tcg_debug_assert(input_p);
2200 tcg_debug_assert(o < def->nb_oargs);
2201 tcg_debug_assert(def->args_ct[o].regs != 0);
2202 tcg_debug_assert(!def->args_ct[o].oalias);
2203 def->args_ct[i] = def->args_ct[o];
2204 /* The output sets oalias. */
2205 def->args_ct[o].oalias = 1;
2206 def->args_ct[o].alias_index = i;
2207 /* The input sets ialias. */
2208 def->args_ct[i].ialias = 1;
2209 def->args_ct[i].alias_index = o;
29f5e925
RH		 2210 if (def->args_ct[i].pair) {
2211 saw_alias_pair = true;
2212 }
8940ea0d
PMD		 2213 tcg_debug_assert(ct_str[1] == '\0');
2214 continue;
2215
2216 case '&':
2217 tcg_debug_assert(!input_p);
2218 def->args_ct[i].newreg = true;
2219 ct_str++;
2220 break;
29f5e925
RH		 2221
2222 case 'p': /* plus */
2223 /* Allocate to the register after the previous. */
2224 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2225 o = i - 1;
2226 tcg_debug_assert(!def->args_ct[o].pair);
2227 tcg_debug_assert(!def->args_ct[o].ct);
2228 def->args_ct[i] = (TCGArgConstraint){
2229 .pair = 2,
2230 .pair_index = o,
2231 .regs = def->args_ct[o].regs << 1,
2232 };
2233 def->args_ct[o].pair = 1;
2234 def->args_ct[o].pair_index = i;
2235 tcg_debug_assert(ct_str[1] == '\0');
2236 continue;
2237
2238 case 'm': /* minus */
2239 /* Allocate to the register before the previous. */
2240 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2241 o = i - 1;
2242 tcg_debug_assert(!def->args_ct[o].pair);
2243 tcg_debug_assert(!def->args_ct[o].ct);
2244 def->args_ct[i] = (TCGArgConstraint){
2245 .pair = 1,
2246 .pair_index = o,
2247 .regs = def->args_ct[o].regs >> 1,
2248 };
2249 def->args_ct[o].pair = 2;
2250 def->args_ct[o].pair_index = i;
2251 tcg_debug_assert(ct_str[1] == '\0');
2252 continue;
8940ea0d
PMD		 2253 }
2254
2255 do {
2256 switch (*ct_str) {
17280ff4
RH		 2257 case 'i':
2258 def->args_ct[i].ct |= TCG_CT_CONST;
17280ff4 2259 break;
358b4923 2260
358b4923
RH		 2261 /* Include all of the target-specific constraints. */
2262
2263#undef CONST
2264#define CONST(CASE, MASK) \
8940ea0d 2265 case CASE: def->args_ct[i].ct |= MASK; break;
358b4923 2266#define REGS(CASE, MASK) \
8940ea0d 2267 case CASE: def->args_ct[i].regs |= MASK; break;
358b4923
RH		 2268
2269#include "tcg-target-con-str.h"
2270
2271#undef REGS
2272#undef CONST
17280ff4 2273 default:
8940ea0d
PMD		 2274 case '0' ... '9':
2275 case '&':
29f5e925
RH		 2276 case 'p':
2277 case 'm':
17280ff4 2278 /* Typo in TCGTargetOpDef constraint. */
358b4923 2279 g_assert_not_reached();
c896fe29 2280 }
8940ea0d 2281 } while (*++ct_str != '\0');
c896fe29
FB		 2282 }
2283
c68aaa18 2284 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2285 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2286
29f5e925
RH		 2287 /*
2288 * Fix up output pairs that are aliased with inputs.
2289 * When we created the alias, we copied pair from the output.
2290 * There are three cases:
2291 * (1a) Pairs of inputs alias pairs of outputs.
2292 * (1b) One input aliases the first of a pair of outputs.
2293 * (2) One input aliases the second of a pair of outputs.
2294 *
2295 * Case 1a is handled by making sure that the pair_index'es are
2296 * properly updated so that they appear the same as a pair of inputs.
2297 *
2298 * Case 1b is handled by setting the pair_index of the input to
2299 * itself, simply so it doesn't point to an unrelated argument.
2300 * Since we don't encounter the "second" during the input allocation
2301 * phase, nothing happens with the second half of the input pair.
2302 *
2303 * Case 2 is handled by setting the second input to pair=3, the
2304 * first output to pair=3, and the pair_index'es to match.
2305 */
2306 if (saw_alias_pair) {
2307 for (i = def->nb_oargs; i < nb_args; i++) {
2308 /*
2309 * Since [0-9pm] must be alone in the constraint string,
2310 * the only way they can both be set is if the pair comes
2311 * from the output alias.
2312 */
2313 if (!def->args_ct[i].ialias) {
2314 continue;
2315 }
2316 switch (def->args_ct[i].pair) {
2317 case 0:
2318 break;
2319 case 1:
2320 o = def->args_ct[i].alias_index;
2321 o2 = def->args_ct[o].pair_index;
2322 tcg_debug_assert(def->args_ct[o].pair == 1);
2323 tcg_debug_assert(def->args_ct[o2].pair == 2);
2324 if (def->args_ct[o2].oalias) {
2325 /* Case 1a */
2326 i2 = def->args_ct[o2].alias_index;
2327 tcg_debug_assert(def->args_ct[i2].pair == 2);
2328 def->args_ct[i2].pair_index = i;
2329 def->args_ct[i].pair_index = i2;
2330 } else {
2331 /* Case 1b */
2332 def->args_ct[i].pair_index = i;
2333 }
2334 break;
2335 case 2:
2336 o = def->args_ct[i].alias_index;
2337 o2 = def->args_ct[o].pair_index;
2338 tcg_debug_assert(def->args_ct[o].pair == 2);
2339 tcg_debug_assert(def->args_ct[o2].pair == 1);
2340 if (def->args_ct[o2].oalias) {
2341 /* Case 1a */
2342 i2 = def->args_ct[o2].alias_index;
2343 tcg_debug_assert(def->args_ct[i2].pair == 1);
2344 def->args_ct[i2].pair_index = i;
2345 def->args_ct[i].pair_index = i2;
2346 } else {
2347 /* Case 2 */
2348 def->args_ct[i].pair = 3;
2349 def->args_ct[o2].pair = 3;
2350 def->args_ct[i].pair_index = o2;
2351 def->args_ct[o2].pair_index = i;
2352 }
2353 break;
2354 default:
2355 g_assert_not_reached();
2356 }
2357 }
2358 }
2359
c896fe29
FB		 2360 /* sort the constraints (XXX: this is just an heuristic) */
2361 sort_constraints(def, 0, def->nb_oargs);
2362 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2363 }
c896fe29
FB		 2364}
2365
0c627cdc
RH		 2366void tcg_op_remove(TCGContext *s, TCGOp *op)
2367{
d88a117e
RH		 2368 TCGLabel *label;
2369
2370 switch (op->opc) {
2371 case INDEX_op_br:
2372 label = arg_label(op->args[0]);
2373 label->refs--;
2374 break;
2375 case INDEX_op_brcond_i32:
2376 case INDEX_op_brcond_i64:
2377 label = arg_label(op->args[3]);
2378 label->refs--;
2379 break;
2380 case INDEX_op_brcond2_i32:
2381 label = arg_label(op->args[5]);
2382 label->refs--;
2383 break;
2384 default:
2385 break;
2386 }
2387
15fa08f8
RH		 2388 QTAILQ_REMOVE(&s->ops, op, link);
2389 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2390 s->nb_ops--;
0c627cdc
RH		 2391
2392#ifdef CONFIG_PROFILER
d73415a3 2393 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2394#endif
2395}
2396
a80cdd31
RH		 2397void tcg_remove_ops_after(TCGOp *op)
2398{
2399 TCGContext *s = tcg_ctx;
2400
2401 while (true) {
2402 TCGOp *last = tcg_last_op();
2403 if (last == op) {
2404 return;
2405 }
2406 tcg_op_remove(s, last);
2407 }
2408}
2409
d4478943 2410static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
5a18407f 2411{
15fa08f8 2412 TCGContext *s = tcg_ctx;
cb10bc63
RH		 2413 TCGOp *op = NULL;
2414
2415 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
2416 QTAILQ_FOREACH(op, &s->free_ops, link) {
2417 if (nargs <= op->nargs) {
2418 QTAILQ_REMOVE(&s->free_ops, op, link);
2419 nargs = op->nargs;
2420 goto found;
2421 }
2422 }
15fa08f8 2423 }
cb10bc63
RH		 2424
2425 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
2426 nargs = MAX(4, nargs);
2427 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
2428
2429 found:
15fa08f8
RH		 2430 memset(op, 0, offsetof(TCGOp, link));
2431 op->opc = opc;
cb10bc63
RH		 2432 op->nargs = nargs;
2433
2434 /* Check for bitfield overflow. */
2435 tcg_debug_assert(op->nargs == nargs);
5a18407f 2436
cb10bc63 2437 s->nb_ops++;
15fa08f8
RH		 2438 return op;
2439}
2440
d4478943 2441TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
15fa08f8 2442{
d4478943 2443 TCGOp *op = tcg_op_alloc(opc, nargs);
15fa08f8
RH		 2444 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2445 return op;
2446}
5a18407f 2447
d4478943
PMD		 2448TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
2449 TCGOpcode opc, unsigned nargs)
15fa08f8 2450{
d4478943 2451 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 2452 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2453 return new_op;
2454}
2455
d4478943
PMD		 2456TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
2457 TCGOpcode opc, unsigned nargs)
5a18407f 2458{
d4478943 2459 TCGOp *new_op = tcg_op_alloc(opc, nargs);
15fa08f8 2460 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2461 return new_op;
2462}
2463
b4fc67c7
RH		 2464/* Reachable analysis : remove unreachable code. */
2465static void reachable_code_pass(TCGContext *s)
2466{
2467 TCGOp *op, *op_next;
2468 bool dead = false;
2469
2470 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2471 bool remove = dead;
2472 TCGLabel *label;
b4fc67c7
RH		 2473
2474 switch (op->opc) {
2475 case INDEX_op_set_label:
2476 label = arg_label(op->args[0]);
2477 if (label->refs == 0) {
2478 /*
2479 * While there is an occasional backward branch, virtually
2480 * all branches generated by the translators are forward.
2481 * Which means that generally we will have already removed
2482 * all references to the label that will be, and there is
2483 * little to be gained by iterating.
2484 */
2485 remove = true;
2486 } else {
2487 /* Once we see a label, insns become live again. */
2488 dead = false;
2489 remove = false;
2490
2491 /*
2492 * Optimization can fold conditional branches to unconditional.
2493 * If we find a label with one reference which is preceded by
2494 * an unconditional branch to it, remove both. This needed to
2495 * wait until the dead code in between them was removed.
2496 */
2497 if (label->refs == 1) {
eae3eb3e 2498 TCGOp *op_prev = QTAILQ_PREV(op, link);
b4fc67c7
RH		 2499 if (op_prev->opc == INDEX_op_br &&
2500 label == arg_label(op_prev->args[0])) {
2501 tcg_op_remove(s, op_prev);
2502 remove = true;
2503 }
2504 }
2505 }
2506 break;
2507
2508 case INDEX_op_br:
2509 case INDEX_op_exit_tb:
2510 case INDEX_op_goto_ptr:
2511 /* Unconditional branches; everything following is dead. */
2512 dead = true;
2513 break;
2514
2515 case INDEX_op_call:
2516 /* Notice noreturn helper calls, raising exceptions. */
90163900 2517 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2518 dead = true;
2519 }
2520 break;
2521
2522 case INDEX_op_insn_start:
2523 /* Never remove -- we need to keep these for unwind. */
2524 remove = false;
2525 break;
2526
2527 default:
2528 break;
2529 }
2530
2531 if (remove) {
2532 tcg_op_remove(s, op);
2533 }
2534 }
2535}
2536
c70fbf0a
RH		 2537#define TS_DEAD 1
2538#define TS_MEM 2
2539
5a18407f
RH		 2540#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2541#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2542
25f49c5f
RH		 2543/* For liveness_pass_1, the register preferences for a given temp. */
2544static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2545{
2546 return ts->state_ptr;
2547}
2548
2549/* For liveness_pass_1, reset the preferences for a given temp to the
2550 * maximal regset for its type.
2551 */
2552static inline void la_reset_pref(TCGTemp *ts)
2553{
2554 *la_temp_pref(ts)
2555 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2556}
2557
9c43b68d
AJ		 2558/* liveness analysis: end of function: all temps are dead, and globals
2559 should be in memory. */
2616c808 2560static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2561{
b83eabea
RH		 2562 int i;
2563
2564 for (i = 0; i < ng; ++i) {
2565 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2566 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2567 }
2568 for (i = ng; i < nt; ++i) {
2569 s->temps[i].state = TS_DEAD;
25f49c5f 2570 la_reset_pref(&s->temps[i]);
b83eabea 2571 }
c896fe29
FB		 2572}
2573
9c43b68d
AJ		 2574/* liveness analysis: end of basic block: all temps are dead, globals
2575 and local temps should be in memory. */
2616c808 2576static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2577{
b83eabea 2578 int i;
641d5fbe 2579
ee17db83
RH		 2580 for (i = 0; i < nt; ++i) {
2581 TCGTemp *ts = &s->temps[i];
2582 int state;
2583
2584 switch (ts->kind) {
2585 case TEMP_FIXED:
2586 case TEMP_GLOBAL:
2587 case TEMP_LOCAL:
2588 state = TS_DEAD | TS_MEM;
2589 break;
2590 case TEMP_NORMAL:
c7482438 2591 case TEMP_EBB:
c0522136 2592 case TEMP_CONST:
ee17db83
RH		 2593 state = TS_DEAD;
2594 break;
2595 default:
2596 g_assert_not_reached();
2597 }
2598 ts->state = state;
2599 la_reset_pref(ts);
641d5fbe
FB		 2600 }
2601}
2602
f65a061c
RH		 2603/* liveness analysis: sync globals back to memory. */
2604static void la_global_sync(TCGContext *s, int ng)
2605{
2606 int i;
2607
2608 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2609 int state = s->temps[i].state;
2610 s->temps[i].state = state | TS_MEM;
2611 if (state == TS_DEAD) {
2612 /* If the global was previously dead, reset prefs. */
2613 la_reset_pref(&s->temps[i]);
2614 }
f65a061c
RH		 2615 }
2616}
2617
b4cb76e6 2618/*
c7482438
RH		 2619 * liveness analysis: conditional branch: all temps are dead unless
2620 * explicitly live-across-conditional-branch, globals and local temps
2621 * should be synced.
b4cb76e6
RH		 2622 */
2623static void la_bb_sync(TCGContext *s, int ng, int nt)
2624{
2625 la_global_sync(s, ng);
2626
2627 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2628 TCGTemp *ts = &s->temps[i];
2629 int state;
2630
2631 switch (ts->kind) {
2632 case TEMP_LOCAL:
2633 state = ts->state;
2634 ts->state = state | TS_MEM;
b4cb76e6
RH		 2635 if (state != TS_DEAD) {
2636 continue;
2637 }
c0522136
RH		 2638 break;
2639 case TEMP_NORMAL:
b4cb76e6 2640 s->temps[i].state = TS_DEAD;
c0522136 2641 break;
c7482438 2642 case TEMP_EBB:
c0522136
RH		 2643 case TEMP_CONST:
2644 continue;
2645 default:
2646 g_assert_not_reached();
b4cb76e6
RH		 2647 }
2648 la_reset_pref(&s->temps[i]);
2649 }
2650}
2651
f65a061c
RH		 2652/* liveness analysis: sync globals back to memory and kill. */
2653static void la_global_kill(TCGContext *s, int ng)
2654{
2655 int i;
2656
2657 for (i = 0; i < ng; i++) {
2658 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2659 la_reset_pref(&s->temps[i]);
2660 }
2661}
2662
2663/* liveness analysis: note live globals crossing calls. */
2664static void la_cross_call(TCGContext *s, int nt)
2665{
2666 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2667 int i;
2668
2669 for (i = 0; i < nt; i++) {
2670 TCGTemp *ts = &s->temps[i];
2671 if (!(ts->state & TS_DEAD)) {
2672 TCGRegSet *pset = la_temp_pref(ts);
2673 TCGRegSet set = *pset;
2674
2675 set &= mask;
2676 /* If the combination is not possible, restart. */
2677 if (set == 0) {
2678 set = tcg_target_available_regs[ts->type] & mask;
2679 }
2680 *pset = set;
2681 }
f65a061c
RH		 2682 }
2683}
2684
a1b3c48d 2685/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2686 given input arguments is dead. Instructions updating dead
2687 temporaries are removed. */
b83eabea 2688static void liveness_pass_1(TCGContext *s)
c896fe29 2689{
c70fbf0a 2690 int nb_globals = s->nb_globals;
2616c808 2691 int nb_temps = s->nb_temps;
15fa08f8 2692 TCGOp *op, *op_prev;
25f49c5f
RH		 2693 TCGRegSet *prefs;
2694 int i;
2695
2696 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2697 for (i = 0; i < nb_temps; ++i) {
2698 s->temps[i].state_ptr = prefs + i;
2699 }
a1b3c48d 2700
ae36a246 2701 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2702 la_func_end(s, nb_globals, nb_temps);
c896fe29 2703
eae3eb3e 2704 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2705 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2706 TCGOpcode opc_new, opc_new2;
2707 bool have_opc_new2;
a1b3c48d 2708 TCGLifeData arg_life = 0;
25f49c5f 2709 TCGTemp *ts;
c45cb8bb
RH		 2710 TCGOpcode opc = op->opc;
2711 const TCGOpDef *def = &tcg_op_defs[opc];
2712
c45cb8bb 2713 switch (opc) {
c896fe29 2714 case INDEX_op_call:
c6e113f5 2715 {
39004a71
RH		 2716 const TCGHelperInfo *info = tcg_call_info(op);
2717 int call_flags = tcg_call_flags(op);
c896fe29 2718
cd9090aa
RH		 2719 nb_oargs = TCGOP_CALLO(op);
2720 nb_iargs = TCGOP_CALLI(op);
c6e113f5 2721
c45cb8bb 2722 /* pure functions can be removed if their result is unused */
78505279 2723 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2724 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2725 ts = arg_temp(op->args[i]);
2726 if (ts->state != TS_DEAD) {
c6e113f5 2727 goto do_not_remove_call;
9c43b68d 2728 }
c6e113f5 2729 }
c45cb8bb 2730 goto do_remove;
152c35aa
RH		 2731 }
2732 do_not_remove_call:
c896fe29 2733
25f49c5f 2734 /* Output args are dead. */
152c35aa 2735 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2736 ts = arg_temp(op->args[i]);
2737 if (ts->state & TS_DEAD) {
152c35aa
RH		 2738 arg_life |= DEAD_ARG << i;
2739 }
25f49c5f 2740 if (ts->state & TS_MEM) {
152c35aa 2741 arg_life |= SYNC_ARG << i;
c6e113f5 2742 }
25f49c5f
RH		 2743 ts->state = TS_DEAD;
2744 la_reset_pref(ts);
152c35aa 2745 }
78505279 2746
31fd884b
RH		 2747 /* Not used -- it will be tcg_target_call_oarg_reg(). */
2748 memset(op->output_pref, 0, sizeof(op->output_pref));
2749
152c35aa
RH		 2750 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2751 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2752 la_global_kill(s, nb_globals);
152c35aa 2753 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2754 la_global_sync(s, nb_globals);
152c35aa 2755 }
b9c18f56 2756
25f49c5f 2757 /* Record arguments that die in this helper. */
152c35aa 2758 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f 2759 ts = arg_temp(op->args[i]);
39004a71 2760 if (ts->state & TS_DEAD) {
152c35aa 2761 arg_life |= DEAD_ARG << i;
c6e113f5 2762 }
152c35aa 2763 }
25f49c5f
RH		 2764
2765 /* For all live registers, remove call-clobbered prefs. */
2766 la_cross_call(s, nb_temps);
2767
39004a71
RH		 2768 /*
2769 * Input arguments are live for preceding opcodes.
2770 *
2771 * For those arguments that die, and will be allocated in
2772 * registers, clear the register set for that arg, to be
2773 * filled in below. For args that will be on the stack,
2774 * reset to any available reg. Process arguments in reverse
2775 * order so that if a temp is used more than once, the stack
2776 * reset to max happens before the register reset to 0.
2777 */
2778 for (i = nb_iargs - 1; i >= 0; i--) {
2779 const TCGCallArgumentLoc *loc = &info->in[i];
2780 ts = arg_temp(op->args[nb_oargs + i]);
25f49c5f 2781
39004a71
RH		 2782 if (ts->state & TS_DEAD) {
2783 switch (loc->kind) {
2784 case TCG_CALL_ARG_NORMAL:
2785 case TCG_CALL_ARG_EXTEND_U:
2786 case TCG_CALL_ARG_EXTEND_S:
2787 if (REG_P(loc)) {
2788 *la_temp_pref(ts) = 0;
2789 break;
2790 }
2791 /* fall through */
2792 default:
2793 *la_temp_pref(ts) =
2794 tcg_target_available_regs[ts->type];
2795 break;
2796 }
25f49c5f
RH		 2797 ts->state &= ~TS_DEAD;
2798 }
2799 }
2800
39004a71
RH		 2801 /*
2802 * For each input argument, add its input register to prefs.
2803 * If a temp is used once, this produces a single set bit;
2804 * if a temp is used multiple times, this produces a set.
2805 */
2806 for (i = 0; i < nb_iargs; i++) {
2807 const TCGCallArgumentLoc *loc = &info->in[i];
2808 ts = arg_temp(op->args[nb_oargs + i]);
2809
2810 switch (loc->kind) {
2811 case TCG_CALL_ARG_NORMAL:
2812 case TCG_CALL_ARG_EXTEND_U:
2813 case TCG_CALL_ARG_EXTEND_S:
2814 if (REG_P(loc)) {
2815 tcg_regset_set_reg(*la_temp_pref(ts),
2816 tcg_target_call_iarg_regs[loc->arg_slot]);
2817 }
2818 break;
2819 default:
2820 break;
c19f47bf 2821 }
c896fe29 2822 }
c896fe29 2823 }
c896fe29 2824 break;
765b842a 2825 case INDEX_op_insn_start:
c896fe29 2826 break;
5ff9d6a4 2827 case INDEX_op_discard:
5ff9d6a4 2828 /* mark the temporary as dead */
25f49c5f
RH		 2829 ts = arg_temp(op->args[0]);
2830 ts->state = TS_DEAD;
2831 la_reset_pref(ts);
5ff9d6a4 2832 break;
1305c451
RH		 2833
2834 case INDEX_op_add2_i32:
c45cb8bb 2835 opc_new = INDEX_op_add_i32;
f1fae40c 2836 goto do_addsub2;
1305c451 2837 case INDEX_op_sub2_i32:
c45cb8bb 2838 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 2839 goto do_addsub2;
2840 case INDEX_op_add2_i64:
c45cb8bb 2841 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 2842 goto do_addsub2;
2843 case INDEX_op_sub2_i64:
c45cb8bb 2844 opc_new = INDEX_op_sub_i64;
f1fae40c 2845 do_addsub2:
1305c451
RH		 2846 nb_iargs = 4;
2847 nb_oargs = 2;
2848 /* Test if the high part of the operation is dead, but not
2849 the low part. The result can be optimized to a simple
2850 add or sub. This happens often for x86_64 guest when the
2851 cpu mode is set to 32 bit. */
b83eabea
RH		 2852 if (arg_temp(op->args[1])->state == TS_DEAD) {
2853 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 2854 goto do_remove;
2855 }
c45cb8bb
RH		 2856 /* Replace the opcode and adjust the args in place,
2857 leaving 3 unused args at the end. */
2858 op->opc = opc = opc_new;
efee3746
RH		 2859 op->args[1] = op->args[2];
2860 op->args[2] = op->args[4];
1305c451
RH		 2861 /* Fall through and mark the single-word operation live. */
2862 nb_iargs = 2;
2863 nb_oargs = 1;
2864 }
2865 goto do_not_remove;
2866
1414968a 2867 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 2868 opc_new = INDEX_op_mul_i32;
2869 opc_new2 = INDEX_op_muluh_i32;
2870 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 2871 goto do_mul2;
f1fae40c 2872 case INDEX_op_muls2_i32:
c45cb8bb
RH		 2873 opc_new = INDEX_op_mul_i32;
2874 opc_new2 = INDEX_op_mulsh_i32;
2875 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 2876 goto do_mul2;
2877 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 2878 opc_new = INDEX_op_mul_i64;
2879 opc_new2 = INDEX_op_muluh_i64;
2880 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 2881 goto do_mul2;
f1fae40c 2882 case INDEX_op_muls2_i64:
c45cb8bb
RH		 2883 opc_new = INDEX_op_mul_i64;
2884 opc_new2 = INDEX_op_mulsh_i64;
2885 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 2886 goto do_mul2;
f1fae40c 2887 do_mul2:
1414968a
RH		 2888 nb_iargs = 2;
2889 nb_oargs = 2;
b83eabea
RH		 2890 if (arg_temp(op->args[1])->state == TS_DEAD) {
2891 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 2892 /* Both parts of the operation are dead. */
1414968a
RH		 2893 goto do_remove;
2894 }
03271524 2895 /* The high part of the operation is dead; generate the low. */
c45cb8bb 2896 op->opc = opc = opc_new;
efee3746
RH		 2897 op->args[1] = op->args[2];
2898 op->args[2] = op->args[3];
b83eabea 2899 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 2900 /* The low part of the operation is dead; generate the high. */
2901 op->opc = opc = opc_new2;
efee3746
RH		 2902 op->args[0] = op->args[1];
2903 op->args[1] = op->args[2];
2904 op->args[2] = op->args[3];
03271524
RH		 2905 } else {
2906 goto do_not_remove;
1414968a 2907 }
03271524
RH		 2908 /* Mark the single-word operation live. */
2909 nb_oargs = 1;
1414968a
RH		 2910 goto do_not_remove;
2911
c896fe29 2912 default:
1305c451 2913 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 2914 nb_iargs = def->nb_iargs;
2915 nb_oargs = def->nb_oargs;
c896fe29 2916
49516bc0
AJ		 2917 /* Test if the operation can be removed because all
2918 its outputs are dead. We assume that nb_oargs == 0
2919 implies side effects */
2920 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 2921 for (i = 0; i < nb_oargs; i++) {
b83eabea 2922 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 2923 goto do_not_remove;
9c43b68d 2924 }
49516bc0 2925 }
152c35aa
RH		 2926 goto do_remove;
2927 }
2928 goto do_not_remove;
49516bc0 2929
152c35aa
RH		 2930 do_remove:
2931 tcg_op_remove(s, op);
2932 break;
2933
2934 do_not_remove:
152c35aa 2935 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2936 ts = arg_temp(op->args[i]);
2937
2938 /* Remember the preference of the uses that followed. */
31fd884b
RH		 2939 if (i < ARRAY_SIZE(op->output_pref)) {
2940 op->output_pref[i] = *la_temp_pref(ts);
2941 }
25f49c5f
RH		 2942
2943 /* Output args are dead. */
2944 if (ts->state & TS_DEAD) {
152c35aa 2945 arg_life |= DEAD_ARG << i;
49516bc0 2946 }
25f49c5f 2947 if (ts->state & TS_MEM) {
152c35aa
RH		 2948 arg_life |= SYNC_ARG << i;
2949 }
25f49c5f
RH		 2950 ts->state = TS_DEAD;
2951 la_reset_pref(ts);
152c35aa 2952 }
49516bc0 2953
25f49c5f 2954 /* If end of basic block, update. */
ae36a246
RH		 2955 if (def->flags & TCG_OPF_BB_EXIT) {
2956 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 2957 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2958 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 2959 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 2960 la_bb_end(s, nb_globals, nb_temps);
152c35aa 2961 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 2962 la_global_sync(s, nb_globals);
25f49c5f
RH		 2963 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2964 la_cross_call(s, nb_temps);
2965 }
152c35aa
RH		 2966 }
2967
25f49c5f 2968 /* Record arguments that die in this opcode. */
152c35aa 2969 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2970 ts = arg_temp(op->args[i]);
2971 if (ts->state & TS_DEAD) {
152c35aa 2972 arg_life |= DEAD_ARG << i;
c896fe29 2973 }
c896fe29 2974 }
25f49c5f
RH		 2975
2976 /* Input arguments are live for preceding opcodes. */
152c35aa 2977 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2978 ts = arg_temp(op->args[i]);
2979 if (ts->state & TS_DEAD) {
2980 /* For operands that were dead, initially allow
2981 all regs for the type. */
2982 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2983 ts->state &= ~TS_DEAD;
2984 }
2985 }
2986
2987 /* Incorporate constraints for this operand. */
2988 switch (opc) {
2989 case INDEX_op_mov_i32:
2990 case INDEX_op_mov_i64:
2991 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2992 have proper constraints. That said, special case
2993 moves to propagate preferences backward. */
2994 if (IS_DEAD_ARG(1)) {
2995 *la_temp_pref(arg_temp(op->args[0]))
2996 = *la_temp_pref(arg_temp(op->args[1]));
2997 }
2998 break;
2999
3000 default:
3001 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3002 const TCGArgConstraint *ct = &def->args_ct[i];
3003 TCGRegSet set, *pset;
3004
3005 ts = arg_temp(op->args[i]);
3006 pset = la_temp_pref(ts);
3007 set = *pset;
3008
9be0d080 3009 set &= ct->regs;
bc2b17e6 3010 if (ct->ialias) {
31fd884b 3011 set &= output_pref(op, ct->alias_index);
25f49c5f
RH		 3012 }
3013 /* If the combination is not possible, restart. */
3014 if (set == 0) {
9be0d080 3015 set = ct->regs;
25f49c5f
RH		 3016 }
3017 *pset = set;
3018 }
3019 break;
152c35aa 3020 }
c896fe29
FB		 3021 break;
3022 }
bee158cb 3023 op->life = arg_life;
1ff0a2c5 3024 }
c896fe29 3025}
c896fe29 3026
5a18407f 3027/* Liveness analysis: Convert indirect regs to direct temporaries. */
b83eabea 3028static bool liveness_pass_2(TCGContext *s)
5a18407f
RH		 3029{
3030 int nb_globals = s->nb_globals;
15fa08f8 3031 int nb_temps, i;
5a18407f 3032 bool changes = false;
15fa08f8 3033 TCGOp *op, *op_next;
5a18407f 3034
5a18407f
RH		 3035 /* Create a temporary for each indirect global. */
3036 for (i = 0; i < nb_globals; ++i) {
3037 TCGTemp *its = &s->temps[i];
3038 if (its->indirect_reg) {
3039 TCGTemp *dts = tcg_temp_alloc(s);
3040 dts->type = its->type;
3041 dts->base_type = its->base_type;
c7482438 3042 dts->kind = TEMP_EBB;
b83eabea
RH		 3043 its->state_ptr = dts;
3044 } else {
3045 its->state_ptr = NULL;
5a18407f 3046 }
b83eabea
RH		 3047 /* All globals begin dead. */
3048 its->state = TS_DEAD;
3049 }
3050 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3051 TCGTemp *its = &s->temps[i];
3052 its->state_ptr = NULL;
3053 its->state = TS_DEAD;
5a18407f 3054 }
5a18407f 3055
15fa08f8 3056 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 3057 TCGOpcode opc = op->opc;
3058 const TCGOpDef *def = &tcg_op_defs[opc];
3059 TCGLifeData arg_life = op->life;
3060 int nb_iargs, nb_oargs, call_flags;
b83eabea 3061 TCGTemp *arg_ts, *dir_ts;
5a18407f 3062
5a18407f 3063 if (opc == INDEX_op_call) {
cd9090aa
RH		 3064 nb_oargs = TCGOP_CALLO(op);
3065 nb_iargs = TCGOP_CALLI(op);
90163900 3066 call_flags = tcg_call_flags(op);
5a18407f
RH		 3067 } else {
3068 nb_iargs = def->nb_iargs;
3069 nb_oargs = def->nb_oargs;
3070
3071 /* Set flags similar to how calls require. */
b4cb76e6
RH		 3072 if (def->flags & TCG_OPF_COND_BRANCH) {
3073 /* Like reading globals: sync_globals */
3074 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3075 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 3076 /* Like writing globals: save_globals */
3077 call_flags = 0;
3078 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3079 /* Like reading globals: sync_globals */
3080 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3081 } else {
3082 /* No effect on globals. */
3083 call_flags = (TCG_CALL_NO_READ_GLOBALS |
3084 TCG_CALL_NO_WRITE_GLOBALS);
3085 }
3086 }
3087
3088 /* Make sure that input arguments are available. */
3089 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3090 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3091 dir_ts = arg_ts->state_ptr;
3092 if (dir_ts && arg_ts->state == TS_DEAD) {
3093 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
3094 ? INDEX_op_ld_i32
3095 : INDEX_op_ld_i64);
3096 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
3097
3098 lop->args[0] = temp_arg(dir_ts);
3099 lop->args[1] = temp_arg(arg_ts->mem_base);
3100 lop->args[2] = arg_ts->mem_offset;
3101
3102 /* Loaded, but synced with memory. */
3103 arg_ts->state = TS_MEM;
5a18407f
RH		 3104 }
3105 }
3106
3107 /* Perform input replacement, and mark inputs that became dead.
3108 No action is required except keeping temp_state up to date
3109 so that we reload when needed. */
3110 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea 3111 arg_ts = arg_temp(op->args[i]);
39004a71
RH		 3112 dir_ts = arg_ts->state_ptr;
3113 if (dir_ts) {
3114 op->args[i] = temp_arg(dir_ts);
3115 changes = true;
3116 if (IS_DEAD_ARG(i)) {
3117 arg_ts->state = TS_DEAD;
5a18407f
RH		 3118 }
3119 }
3120 }
3121
3122 /* Liveness analysis should ensure that the following are
3123 all correct, for call sites and basic block end points. */
3124 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
3125 /* Nothing to do */
3126 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
3127 for (i = 0; i < nb_globals; ++i) {
3128 /* Liveness should see that globals are synced back,
3129 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 3130 arg_ts = &s->temps[i];
3131 tcg_debug_assert(arg_ts->state_ptr == 0
3132 || arg_ts->state != 0);
5a18407f
RH		 3133 }
3134 } else {
3135 for (i = 0; i < nb_globals; ++i) {
3136 /* Liveness should see that globals are saved back,
3137 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 3138 arg_ts = &s->temps[i];
3139 tcg_debug_assert(arg_ts->state_ptr == 0
3140 || arg_ts->state == TS_DEAD);
5a18407f
RH		 3141 }
3142 }
3143
3144 /* Outputs become available. */
61f15c48
RH		 3145 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
3146 arg_ts = arg_temp(op->args[0]);
b83eabea 3147 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 3148 if (dir_ts) {
3149 op->args[0] = temp_arg(dir_ts);
3150 changes = true;
3151
3152 /* The output is now live and modified. */
3153 arg_ts->state = 0;
3154
3155 if (NEED_SYNC_ARG(0)) {
3156 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3157 ? INDEX_op_st_i32
3158 : INDEX_op_st_i64);
d4478943 3159 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
61f15c48
RH		 3160 TCGTemp *out_ts = dir_ts;
3161
3162 if (IS_DEAD_ARG(0)) {
3163 out_ts = arg_temp(op->args[1]);
3164 arg_ts->state = TS_DEAD;
3165 tcg_op_remove(s, op);
3166 } else {
3167 arg_ts->state = TS_MEM;
3168 }
3169
3170 sop->args[0] = temp_arg(out_ts);
3171 sop->args[1] = temp_arg(arg_ts->mem_base);
3172 sop->args[2] = arg_ts->mem_offset;
3173 } else {
3174 tcg_debug_assert(!IS_DEAD_ARG(0));
3175 }
5a18407f 3176 }
61f15c48
RH		 3177 } else {
3178 for (i = 0; i < nb_oargs; i++) {
3179 arg_ts = arg_temp(op->args[i]);
3180 dir_ts = arg_ts->state_ptr;
3181 if (!dir_ts) {
3182 continue;
3183 }
3184 op->args[i] = temp_arg(dir_ts);
3185 changes = true;
5a18407f 3186
61f15c48
RH		 3187 /* The output is now live and modified. */
3188 arg_ts->state = 0;
5a18407f 3189
61f15c48
RH		 3190 /* Sync outputs upon their last write. */
3191 if (NEED_SYNC_ARG(i)) {
3192 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3193 ? INDEX_op_st_i32
3194 : INDEX_op_st_i64);
d4478943 3195 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
5a18407f 3196
61f15c48
RH		 3197 sop->args[0] = temp_arg(dir_ts);
3198 sop->args[1] = temp_arg(arg_ts->mem_base);
3199 sop->args[2] = arg_ts->mem_offset;
5a18407f 3200
61f15c48
RH		 3201 arg_ts->state = TS_MEM;
3202 }
3203 /* Drop outputs that are dead. */
3204 if (IS_DEAD_ARG(i)) {
3205 arg_ts->state = TS_DEAD;
3206 }
5a18407f
RH		 3207 }
3208 }
3209 }
3210
3211 return changes;
3212}
3213
2272e4a7 3214static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 3215{
31c96417
RH		 3216 int size = tcg_type_size(ts->type);
3217 int align;
3218 intptr_t off;
c1c09194
RH		 3219
3220 switch (ts->type) {
3221 case TCG_TYPE_I32:
31c96417 3222 align = 4;
c1c09194
RH		 3223 break;
3224 case TCG_TYPE_I64:
3225 case TCG_TYPE_V64:
31c96417 3226 align = 8;
c1c09194
RH		 3227 break;
3228 case TCG_TYPE_V128:
c1c09194
RH		 3229 case TCG_TYPE_V256:
3230 /* Note that we do not require aligned storage for V256. */
31c96417 3231 align = 16;
c1c09194
RH		 3232 break;
3233 default:
3234 g_assert_not_reached();
b591dc59 3235 }
c1c09194 3236
b9537d59
RH		 3237 /*
3238 * Assume the stack is sufficiently aligned.
3239 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3240 * and do not require 16 byte vector alignment. This seems slightly
3241 * easier than fully parameterizing the above switch statement.
3242 */
3243 align = MIN(TCG_TARGET_STACK_ALIGN, align);
c1c09194 3244 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 3245
3246 /* If we've exhausted the stack frame, restart with a smaller TB. */
3247 if (off + size > s->frame_end) {
3248 tcg_raise_tb_overflow(s);
3249 }
c1c09194
RH		 3250 s->current_frame_offset = off + size;
3251
3252 ts->mem_offset = off;
9defd1bd
RH		 3253#if defined(__sparc__)
3254 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3255#endif
b3a62939 3256 ts->mem_base = s->frame_temp;
c896fe29 3257 ts->mem_allocated = 1;
c896fe29
FB		 3258}
3259
098859f1
RH		 3260/* Assign @reg to @ts, and update reg_to_temp[]. */
3261static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
3262{
3263 if (ts->val_type == TEMP_VAL_REG) {
3264 TCGReg old = ts->reg;
3265 tcg_debug_assert(s->reg_to_temp[old] == ts);
3266 if (old == reg) {
3267 return;
3268 }
3269 s->reg_to_temp[old] = NULL;
3270 }
3271 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
3272 s->reg_to_temp[reg] = ts;
3273 ts->val_type = TEMP_VAL_REG;
3274 ts->reg = reg;
3275}
3276
3277/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
3278static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
3279{
3280 tcg_debug_assert(type != TEMP_VAL_REG);
3281 if (ts->val_type == TEMP_VAL_REG) {
3282 TCGReg reg = ts->reg;
3283 tcg_debug_assert(s->reg_to_temp[reg] == ts);
3284 s->reg_to_temp[reg] = NULL;
3285 }
3286 ts->val_type = type;
3287}
3288
b722452a 3289static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3290
59d7c14e
RH		 3291/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3292 mark it free; otherwise mark it dead. */
3293static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3294{
c0522136
RH		 3295 TCGTempVal new_type;
3296
3297 switch (ts->kind) {
3298 case TEMP_FIXED:
59d7c14e 3299 return;
c0522136
RH		 3300 case TEMP_GLOBAL:
3301 case TEMP_LOCAL:
3302 new_type = TEMP_VAL_MEM;
3303 break;
3304 case TEMP_NORMAL:
c7482438 3305 case TEMP_EBB:
c0522136
RH		 3306 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3307 break;
3308 case TEMP_CONST:
3309 new_type = TEMP_VAL_CONST;
3310 break;
3311 default:
3312 g_assert_not_reached();
59d7c14e 3313 }
098859f1 3314 set_temp_val_nonreg(s, ts, new_type);
59d7c14e 3315}
7f6ceedf 3316
59d7c14e
RH		 3317/* Mark a temporary as dead. */
3318static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3319{
3320 temp_free_or_dead(s, ts, 1);
3321}
3322
3323/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3324 registers needs to be allocated to store a constant. If 'free_or_dead'
3325 is non-zero, subsequently release the temporary; if it is positive, the
3326 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3327static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3328 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3329{
c0522136 3330 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3331 if (!ts->mem_allocated) {
2272e4a7 3332 temp_allocate_frame(s, ts);
59d7c14e 3333 }
59d7c14e
RH		 3334 switch (ts->val_type) {
3335 case TEMP_VAL_CONST:
3336 /* If we're going to free the temp immediately, then we won't
3337 require it later in a register, so attempt to store the
3338 constant to memory directly. */
3339 if (free_or_dead
3340 && tcg_out_sti(s, ts->type, ts->val,
3341 ts->mem_base->reg, ts->mem_offset)) {
3342 break;
3343 }
3344 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3345 allocated_regs, preferred_regs);
59d7c14e
RH		 3346 /* fallthrough */
3347
3348 case TEMP_VAL_REG:
3349 tcg_out_st(s, ts->type, ts->reg,
3350 ts->mem_base->reg, ts->mem_offset);
3351 break;
3352
3353 case TEMP_VAL_MEM:
3354 break;
3355
3356 case TEMP_VAL_DEAD:
3357 default:
3358 tcg_abort();
3359 }
3360 ts->mem_coherent = 1;
3361 }
3362 if (free_or_dead) {
3363 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3364 }
7f6ceedf
AJ		 3365}
3366
c896fe29 3367/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3368static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3369{
f8b2f202 3370 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3371 if (ts != NULL) {
98b4e186 3372 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3373 }
3374}
3375
b016486e
RH		 3376/**
3377 * tcg_reg_alloc:
3378 * @required_regs: Set of registers in which we must allocate.
3379 * @allocated_regs: Set of registers which must be avoided.
3380 * @preferred_regs: Set of registers we should prefer.
3381 * @rev: True if we search the registers in "indirect" order.
3382 *
3383 * The allocated register must be in @required_regs & ~@allocated_regs,
3384 * but if we can put it in @preferred_regs we may save a move later.
3385 */
3386static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3387 TCGRegSet allocated_regs,
3388 TCGRegSet preferred_regs, bool rev)
c896fe29 3389{
b016486e
RH		 3390 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3391 TCGRegSet reg_ct[2];
91478cef 3392 const int *order;
c896fe29 3393
b016486e
RH		 3394 reg_ct[1] = required_regs & ~allocated_regs;
3395 tcg_debug_assert(reg_ct[1] != 0);
3396 reg_ct[0] = reg_ct[1] & preferred_regs;
3397
3398 /* Skip the preferred_regs option if it cannot be satisfied,
3399 or if the preference made no difference. */
3400 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3401
91478cef 3402 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3403
b016486e
RH		 3404 /* Try free registers, preferences first. */
3405 for (j = f; j < 2; j++) {
3406 TCGRegSet set = reg_ct[j];
3407
3408 if (tcg_regset_single(set)) {
3409 /* One register in the set. */
3410 TCGReg reg = tcg_regset_first(set);
3411 if (s->reg_to_temp[reg] == NULL) {
3412 return reg;
3413 }
3414 } else {
3415 for (i = 0; i < n; i++) {
3416 TCGReg reg = order[i];
3417 if (s->reg_to_temp[reg] == NULL &&
3418 tcg_regset_test_reg(set, reg)) {
3419 return reg;
3420 }
3421 }
3422 }
c896fe29
FB		 3423 }
3424
b016486e
RH		 3425 /* We must spill something. */
3426 for (j = f; j < 2; j++) {
3427 TCGRegSet set = reg_ct[j];
3428
3429 if (tcg_regset_single(set)) {
3430 /* One register in the set. */
3431 TCGReg reg = tcg_regset_first(set);
b3915dbb 3432 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3433 return reg;
b016486e
RH		 3434 } else {
3435 for (i = 0; i < n; i++) {
3436 TCGReg reg = order[i];
3437 if (tcg_regset_test_reg(set, reg)) {
3438 tcg_reg_free(s, reg, allocated_regs);
3439 return reg;
3440 }
3441 }
c896fe29
FB		 3442 }
3443 }
3444
3445 tcg_abort();
3446}
3447
29f5e925
RH		 3448static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
3449 TCGRegSet allocated_regs,
3450 TCGRegSet preferred_regs, bool rev)
3451{
3452 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3453 TCGRegSet reg_ct[2];
3454 const int *order;
3455
3456 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
3457 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
3458 tcg_debug_assert(reg_ct[1] != 0);
3459 reg_ct[0] = reg_ct[1] & preferred_regs;
3460
3461 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3462
3463 /*
3464 * Skip the preferred_regs option if it cannot be satisfied,
3465 * or if the preference made no difference.
3466 */
3467 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3468
3469 /*
3470 * Minimize the number of flushes by looking for 2 free registers first,
3471 * then a single flush, then two flushes.
3472 */
3473 for (fmin = 2; fmin >= 0; fmin--) {
3474 for (j = k; j < 2; j++) {
3475 TCGRegSet set = reg_ct[j];
3476
3477 for (i = 0; i < n; i++) {
3478 TCGReg reg = order[i];
3479
3480 if (tcg_regset_test_reg(set, reg)) {
3481 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
3482 if (f >= fmin) {
3483 tcg_reg_free(s, reg, allocated_regs);
3484 tcg_reg_free(s, reg + 1, allocated_regs);
3485 return reg;
3486 }
3487 }
3488 }
3489 }
3490 }
3491 tcg_abort();
3492}
3493
40ae5c62
RH		 3494/* Make sure the temporary is in a register. If needed, allocate the register
3495 from DESIRED while avoiding ALLOCATED. */
3496static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3497 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3498{
3499 TCGReg reg;
3500
3501 switch (ts->val_type) {
3502 case TEMP_VAL_REG:
3503 return;
3504 case TEMP_VAL_CONST:
b016486e 3505 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3506 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3507 if (ts->type <= TCG_TYPE_I64) {
3508 tcg_out_movi(s, ts->type, reg, ts->val);
3509 } else {
4e186175
RH		 3510 uint64_t val = ts->val;
3511 MemOp vece = MO_64;
3512
3513 /*
3514 * Find the minimal vector element that matches the constant.
3515 * The targets will, in general, have to do this search anyway,
3516 * do this generically.
3517 */
4e186175
RH		 3518 if (val == dup_const(MO_8, val)) {
3519 vece = MO_8;
3520 } else if (val == dup_const(MO_16, val)) {
3521 vece = MO_16;
0b4286dd 3522 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3523 vece = MO_32;
3524 }
3525
3526 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3527 }
40ae5c62
RH		 3528 ts->mem_coherent = 0;
3529 break;
3530 case TEMP_VAL_MEM:
b016486e 3531 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3532 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3533 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3534 ts->mem_coherent = 1;
3535 break;
3536 case TEMP_VAL_DEAD:
3537 default:
3538 tcg_abort();
3539 }
098859f1 3540 set_temp_val_reg(s, ts, reg);
40ae5c62
RH		 3541}
3542
59d7c14e
RH		 3543/* Save a temporary to memory. 'allocated_regs' is used in case a
3544 temporary registers needs to be allocated to store a constant. */
3545static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3546{
5a18407f
RH		 3547 /* The liveness analysis already ensures that globals are back
3548 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3549 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3550}
3551
9814dd27 3552/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3553 modified be the following code. 'allocated_regs' is used in case a
3554 temporary registers needs to be allocated to store a constant. */
3555static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3556{
ac3b8891 3557 int i, n;
c896fe29 3558
ac3b8891 3559 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3560 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3561 }
e5097dc8
FB		 3562}
3563
3d5c5f87
AJ		 3564/* sync globals to their canonical location and assume they can be
3565 read by the following code. 'allocated_regs' is used in case a
3566 temporary registers needs to be allocated to store a constant. */
3567static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3568{
ac3b8891 3569 int i, n;
3d5c5f87 3570
ac3b8891 3571 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3572 TCGTemp *ts = &s->temps[i];
5a18407f 3573 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3574 || ts->kind == TEMP_FIXED
5a18407f 3575 || ts->mem_coherent);
3d5c5f87
AJ		 3576 }
3577}
3578
e5097dc8 3579/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3580 all globals are stored at their canonical location. */
3581static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3582{
e5097dc8
FB		 3583 int i;
3584
b13eb728
RH		 3585 for (i = s->nb_globals; i < s->nb_temps; i++) {
3586 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3587
3588 switch (ts->kind) {
3589 case TEMP_LOCAL:
b13eb728 3590 temp_save(s, ts, allocated_regs);
c0522136
RH		 3591 break;
3592 case TEMP_NORMAL:
c7482438 3593 case TEMP_EBB:
5a18407f
RH		 3594 /* The liveness analysis already ensures that temps are dead.
3595 Keep an tcg_debug_assert for safety. */
3596 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3597 break;
3598 case TEMP_CONST:
3599 /* Similarly, we should have freed any allocated register. */
3600 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3601 break;
3602 default:
3603 g_assert_not_reached();
c896fe29
FB		 3604 }
3605 }
e8996ee0
FB		 3606
3607 save_globals(s, allocated_regs);
c896fe29
FB		 3608}
3609
b4cb76e6 3610/*
c7482438
RH		 3611 * At a conditional branch, we assume all temporaries are dead unless
3612 * explicitly live-across-conditional-branch; all globals and local
3613 * temps are synced to their location.
b4cb76e6
RH		 3614 */
3615static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3616{
3617 sync_globals(s, allocated_regs);
3618
3619 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3620 TCGTemp *ts = &s->temps[i];
3621 /*
3622 * The liveness analysis already ensures that temps are dead.
3623 * Keep tcg_debug_asserts for safety.
3624 */
c0522136
RH		 3625 switch (ts->kind) {
3626 case TEMP_LOCAL:
b4cb76e6 3627 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136
RH		 3628 break;
3629 case TEMP_NORMAL:
b4cb76e6 3630 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136 3631 break;
c7482438 3632 case TEMP_EBB:
c0522136
RH		 3633 case TEMP_CONST:
3634 break;
3635 default:
3636 g_assert_not_reached();
b4cb76e6
RH		 3637 }
3638 }
3639}
3640
bab1671f 3641/*
c58f4c97 3642 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3643 */
0fe4fca4 3644static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3645 tcg_target_ulong val, TCGLifeData arg_life,
3646 TCGRegSet preferred_regs)
e8996ee0 3647{
d63e3b6e 3648 /* ENV should not be modified. */
e01fa97d 3649 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3650
3651 /* The movi is not explicitly generated here. */
098859f1 3652 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
59d7c14e
RH		 3653 ots->val = val;
3654 ots->mem_coherent = 0;
3655 if (NEED_SYNC_ARG(0)) {
ba87719c 3656 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3657 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3658 temp_dead(s, ots);
4c4e1ab2 3659 }
e8996ee0
FB		 3660}
3661
bab1671f
RH		 3662/*
3663 * Specialized code generation for INDEX_op_mov_*.
3664 */
dd186292 3665static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3666{
dd186292 3667 const TCGLifeData arg_life = op->life;
69e3706d 3668 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3669 TCGTemp *ts, *ots;
450445d5 3670 TCGType otype, itype;
098859f1 3671 TCGReg oreg, ireg;
c896fe29 3672
d21369f5 3673 allocated_regs = s->reserved_regs;
31fd884b 3674 preferred_regs = output_pref(op, 0);
43439139
RH		 3675 ots = arg_temp(op->args[0]);
3676 ts = arg_temp(op->args[1]);
450445d5 3677
d63e3b6e 3678 /* ENV should not be modified. */
e01fa97d 3679 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3680
450445d5
RH		 3681 /* Note that otype != itype for no-op truncation. */
3682 otype = ots->type;
3683 itype = ts->type;
c29c1d7e 3684
0fe4fca4
PB		 3685 if (ts->val_type == TEMP_VAL_CONST) {
3686 /* propagate constant or generate sti */
3687 tcg_target_ulong val = ts->val;
3688 if (IS_DEAD_ARG(1)) {
3689 temp_dead(s, ts);
3690 }
69e3706d 3691 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3692 return;
3693 }
3694
3695 /* If the source value is in memory we're going to be forced
3696 to have it in a register in order to perform the copy. Copy
3697 the SOURCE value into its own register first, that way we
3698 don't have to reload SOURCE the next time it is used. */
3699 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3700 temp_load(s, ts, tcg_target_available_regs[itype],
3701 allocated_regs, preferred_regs);
c29c1d7e 3702 }
0fe4fca4 3703 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
098859f1
RH		 3704 ireg = ts->reg;
3705
d63e3b6e 3706 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3707 /* mov to a non-saved dead register makes no sense (even with
3708 liveness analysis disabled). */
eabb7b91 3709 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3710 if (!ots->mem_allocated) {
2272e4a7 3711 temp_allocate_frame(s, ots);
c29c1d7e 3712 }
098859f1 3713 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3714 if (IS_DEAD_ARG(1)) {
f8bf00f1 3715 temp_dead(s, ts);
c29c1d7e 3716 }
f8bf00f1 3717 temp_dead(s, ots);
098859f1
RH		 3718 return;
3719 }
3720
3721 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
3722 /*
3723 * The mov can be suppressed. Kill input first, so that it
3724 * is unlinked from reg_to_temp, then set the output to the
3725 * reg that we saved from the input.
3726 */
3727 temp_dead(s, ts);
3728 oreg = ireg;
c29c1d7e 3729 } else {
098859f1
RH		 3730 if (ots->val_type == TEMP_VAL_REG) {
3731 oreg = ots->reg;
c896fe29 3732 } else {
098859f1
RH		 3733 /* Make sure to not spill the input register during allocation. */
3734 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
3735 allocated_regs | ((TCGRegSet)1 << ireg),
3736 preferred_regs, ots->indirect_base);
c896fe29 3737 }
098859f1
RH		 3738 if (!tcg_out_mov(s, otype, oreg, ireg)) {
3739 /*
3740 * Cross register class move not supported.
3741 * Store the source register into the destination slot
3742 * and leave the destination temp as TEMP_VAL_MEM.
3743 */
3744 assert(!temp_readonly(ots));
3745 if (!ts->mem_allocated) {
3746 temp_allocate_frame(s, ots);
3747 }
3748 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
3749 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
3750 ots->mem_coherent = 1;
3751 return;
c896fe29 3752 }
ec7a869d 3753 }
098859f1
RH		 3754 set_temp_val_reg(s, ots, oreg);
3755 ots->mem_coherent = 0;
3756
3757 if (NEED_SYNC_ARG(0)) {
3758 temp_sync(s, ots, allocated_regs, 0, 0);
3759 }
c896fe29
FB		 3760}
3761
bab1671f
RH		 3762/*
3763 * Specialized code generation for INDEX_op_dup_vec.
3764 */
3765static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3766{
3767 const TCGLifeData arg_life = op->life;
3768 TCGRegSet dup_out_regs, dup_in_regs;
3769 TCGTemp *its, *ots;
3770 TCGType itype, vtype;
3771 unsigned vece;
31c96417 3772 int lowpart_ofs;
bab1671f
RH		 3773 bool ok;
3774
3775 ots = arg_temp(op->args[0]);
3776 its = arg_temp(op->args[1]);
3777
3778 /* ENV should not be modified. */
e01fa97d 3779 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 3780
3781 itype = its->type;
3782 vece = TCGOP_VECE(op);
3783 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3784
3785 if (its->val_type == TEMP_VAL_CONST) {
3786 /* Propagate constant via movi -> dupi. */
3787 tcg_target_ulong val = its->val;
3788 if (IS_DEAD_ARG(1)) {
3789 temp_dead(s, its);
3790 }
31fd884b 3791 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
bab1671f
RH		 3792 return;
3793 }
3794
9be0d080
RH		 3795 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3796 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 3797
3798 /* Allocate the output register now. */
3799 if (ots->val_type != TEMP_VAL_REG) {
3800 TCGRegSet allocated_regs = s->reserved_regs;
098859f1 3801 TCGReg oreg;
bab1671f
RH		 3802
3803 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3804 /* Make sure to not spill the input register. */
3805 tcg_regset_set_reg(allocated_regs, its->reg);
3806 }
098859f1 3807 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 3808 output_pref(op, 0), ots->indirect_base);
098859f1 3809 set_temp_val_reg(s, ots, oreg);
bab1671f
RH		 3810 }
3811
3812 switch (its->val_type) {
3813 case TEMP_VAL_REG:
3814 /*
3815 * The dup constriaints must be broad, covering all possible VECE.
3816 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3817 * to fail, indicating that extra moves are required for that case.
3818 */
3819 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3820 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3821 goto done;
3822 }
3823 /* Try again from memory or a vector input register. */
3824 }
3825 if (!its->mem_coherent) {
3826 /*
3827 * The input register is not synced, and so an extra store
3828 * would be required to use memory. Attempt an integer-vector
3829 * register move first. We do not have a TCGRegSet for this.
3830 */
3831 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3832 break;
3833 }
3834 /* Sync the temp back to its slot and load from there. */
3835 temp_sync(s, its, s->reserved_regs, 0, 0);
3836 }
3837 /* fall through */
3838
3839 case TEMP_VAL_MEM:
31c96417
RH		 3840 lowpart_ofs = 0;
3841 if (HOST_BIG_ENDIAN) {
3842 lowpart_ofs = tcg_type_size(itype) - (1 << vece);
3843 }
d6ecb4a9 3844 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
31c96417 3845 its->mem_offset + lowpart_ofs)) {
d6ecb4a9
RH		 3846 goto done;
3847 }
098859f1 3848 /* Load the input into the destination vector register. */
bab1671f
RH		 3849 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3850 break;
3851
3852 default:
3853 g_assert_not_reached();
3854 }
3855
3856 /* We now have a vector input register, so dup must succeed. */
3857 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3858 tcg_debug_assert(ok);
3859
3860 done:
36f5539c 3861 ots->mem_coherent = 0;
bab1671f
RH		 3862 if (IS_DEAD_ARG(1)) {
3863 temp_dead(s, its);
3864 }
3865 if (NEED_SYNC_ARG(0)) {
3866 temp_sync(s, ots, s->reserved_regs, 0, 0);
3867 }
3868 if (IS_DEAD_ARG(0)) {
3869 temp_dead(s, ots);
3870 }
3871}
3872
dd186292 3873static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 3874{
dd186292
RH		 3875 const TCGLifeData arg_life = op->life;
3876 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 3877 TCGRegSet i_allocated_regs;
3878 TCGRegSet o_allocated_regs;
b6638662
RH		 3879 int i, k, nb_iargs, nb_oargs;
3880 TCGReg reg;
c896fe29
FB		 3881 TCGArg arg;
3882 const TCGArgConstraint *arg_ct;
3883 TCGTemp *ts;
3884 TCGArg new_args[TCG_MAX_OP_ARGS];
3885 int const_args[TCG_MAX_OP_ARGS];
3886
3887 nb_oargs = def->nb_oargs;
3888 nb_iargs = def->nb_iargs;
3889
3890 /* copy constants */
a813e36f 3891 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 3892 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 3893 sizeof(TCGArg) * def->nb_cargs);
3894
d21369f5
RH		 3895 i_allocated_regs = s->reserved_regs;
3896 o_allocated_regs = s->reserved_regs;
82790a87 3897
a813e36f 3898 /* satisfy input constraints */
dd186292 3899 for (k = 0; k < nb_iargs; k++) {
29f5e925
RH		 3900 TCGRegSet i_preferred_regs, i_required_regs;
3901 bool allocate_new_reg, copyto_new_reg;
3902 TCGTemp *ts2;
3903 int i1, i2;
d62816f2 3904
66792f90 3905 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 3906 arg = op->args[i];
c896fe29 3907 arg_ct = &def->args_ct[i];
43439139 3908 ts = arg_temp(arg);
40ae5c62
RH		 3909
3910 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 3911 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 3912 /* constant is OK for instruction */
3913 const_args[i] = 1;
3914 new_args[i] = ts->val;
d62816f2 3915 continue;
c896fe29 3916 }
40ae5c62 3917
1c1824dc
RH		 3918 reg = ts->reg;
3919 i_preferred_regs = 0;
29f5e925 3920 i_required_regs = arg_ct->regs;
1c1824dc 3921 allocate_new_reg = false;
29f5e925
RH		 3922 copyto_new_reg = false;
3923
3924 switch (arg_ct->pair) {
3925 case 0: /* not paired */
3926 if (arg_ct->ialias) {
31fd884b 3927 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 3928
3929 /*
3930 * If the input is readonly, then it cannot also be an
3931 * output and aliased to itself. If the input is not
3932 * dead after the instruction, we must allocate a new
3933 * register and move it.
3934 */
3935 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3936 allocate_new_reg = true;
3937 } else if (ts->val_type == TEMP_VAL_REG) {
3938 /*
3939 * Check if the current register has already been
3940 * allocated for another input.
3941 */
3942 allocate_new_reg =
3943 tcg_regset_test_reg(i_allocated_regs, reg);
3944 }
3945 }
3946 if (!allocate_new_reg) {
3947 temp_load(s, ts, i_required_regs, i_allocated_regs,
3948 i_preferred_regs);
3949 reg = ts->reg;
3950 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
3951 }
3952 if (allocate_new_reg) {
3953 /*
3954 * Allocate a new register matching the constraint
3955 * and move the temporary register into it.
3956 */
3957 temp_load(s, ts, tcg_target_available_regs[ts->type],
3958 i_allocated_regs, 0);
3959 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
3960 i_preferred_regs, ts->indirect_base);
3961 copyto_new_reg = true;
3962 }
3963 break;
3964
3965 case 1:
3966 /* First of an input pair; if i1 == i2, the second is an output. */
3967 i1 = i;
3968 i2 = arg_ct->pair_index;
3969 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
3970
3971 /*
3972 * It is easier to default to allocating a new pair
3973 * and to identify a few cases where it's not required.
3974 */
3975 if (arg_ct->ialias) {
31fd884b 3976 i_preferred_regs = output_pref(op, arg_ct->alias_index);
29f5e925
RH		 3977 if (IS_DEAD_ARG(i1) &&
3978 IS_DEAD_ARG(i2) &&
3979 !temp_readonly(ts) &&
3980 ts->val_type == TEMP_VAL_REG &&
3981 ts->reg < TCG_TARGET_NB_REGS - 1 &&
3982 tcg_regset_test_reg(i_required_regs, reg) &&
3983 !tcg_regset_test_reg(i_allocated_regs, reg) &&
3984 !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
3985 (ts2
3986 ? ts2->val_type == TEMP_VAL_REG &&
3987 ts2->reg == reg + 1 &&
3988 !temp_readonly(ts2)
3989 : s->reg_to_temp[reg + 1] == NULL)) {
3990 break;
3991 }
3992 } else {
3993 /* Without aliasing, the pair must also be an input. */
3994 tcg_debug_assert(ts2);
3995 if (ts->val_type == TEMP_VAL_REG &&
3996 ts2->val_type == TEMP_VAL_REG &&
3997 ts2->reg == reg + 1 &&
3998 tcg_regset_test_reg(i_required_regs, reg)) {
3999 break;
4000 }
4001 }
4002 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
4003 0, ts->indirect_base);
4004 goto do_pair;
4005
4006 case 2: /* pair second */
4007 reg = new_args[arg_ct->pair_index] + 1;
4008 goto do_pair;
1c1824dc 4009
29f5e925
RH		 4010 case 3: /* ialias with second output, no first input */
4011 tcg_debug_assert(arg_ct->ialias);
31fd884b 4012 i_preferred_regs = output_pref(op, arg_ct->alias_index);
d62816f2 4013
29f5e925
RH		 4014 if (IS_DEAD_ARG(i) &&
4015 !temp_readonly(ts) &&
4016 ts->val_type == TEMP_VAL_REG &&
4017 reg > 0 &&
4018 s->reg_to_temp[reg - 1] == NULL &&
4019 tcg_regset_test_reg(i_required_regs, reg) &&
4020 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4021 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
4022 tcg_regset_set_reg(i_allocated_regs, reg - 1);
4023 break;
4024 }
4025 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
4026 i_allocated_regs, 0,
4027 ts->indirect_base);
4028 tcg_regset_set_reg(i_allocated_regs, reg);
4029 reg += 1;
4030 goto do_pair;
4031
4032 do_pair:
c0522136 4033 /*
29f5e925
RH		 4034 * If an aliased input is not dead after the instruction,
4035 * we must allocate a new register and move it.
c0522136 4036 */
29f5e925
RH		 4037 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
4038 TCGRegSet t_allocated_regs = i_allocated_regs;
4039
1c1824dc 4040 /*
29f5e925
RH		 4041 * Because of the alias, and the continued life, make sure
4042 * that the temp is somewhere *other* than the reg pair,
4043 * and we get a copy in reg.
1c1824dc 4044 */
29f5e925
RH		 4045 tcg_regset_set_reg(t_allocated_regs, reg);
4046 tcg_regset_set_reg(t_allocated_regs, reg + 1);
4047 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
4048 /* If ts was already in reg, copy it somewhere else. */
4049 TCGReg nr;
4050 bool ok;
4051
4052 tcg_debug_assert(ts->kind != TEMP_FIXED);
4053 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
4054 t_allocated_regs, 0, ts->indirect_base);
4055 ok = tcg_out_mov(s, ts->type, nr, reg);
4056 tcg_debug_assert(ok);
4057
4058 set_temp_val_reg(s, ts, nr);
4059 } else {
4060 temp_load(s, ts, tcg_target_available_regs[ts->type],
4061 t_allocated_regs, 0);
4062 copyto_new_reg = true;
4063 }
4064 } else {
4065 /* Preferably allocate to reg, otherwise copy. */
4066 i_required_regs = (TCGRegSet)1 << reg;
4067 temp_load(s, ts, i_required_regs, i_allocated_regs,
4068 i_preferred_regs);
4069 copyto_new_reg = ts->reg != reg;
5ff9d6a4 4070 }
29f5e925 4071 break;
d62816f2 4072
29f5e925
RH		 4073 default:
4074 g_assert_not_reached();
1c1824dc 4075 }
d62816f2 4076
29f5e925 4077 if (copyto_new_reg) {
78113e83 4078 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 4079 /*
4080 * Cross register class move not supported. Sync the
4081 * temp back to its slot and load from there.
4082 */
4083 temp_sync(s, ts, i_allocated_regs, 0, 0);
4084 tcg_out_ld(s, ts->type, reg,
4085 ts->mem_base->reg, ts->mem_offset);
78113e83 4086 }
c896fe29 4087 }
c896fe29
FB		 4088 new_args[i] = reg;
4089 const_args[i] = 0;
82790a87 4090 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29 4091 }
a813e36f 4092
a52ad07e
AJ		 4093 /* mark dead temporaries and free the associated registers */
4094 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4095 if (IS_DEAD_ARG(i)) {
43439139 4096 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 4097 }
4098 }
4099
b4cb76e6
RH		 4100 if (def->flags & TCG_OPF_COND_BRANCH) {
4101 tcg_reg_alloc_cbranch(s, i_allocated_regs);
4102 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 4103 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 4104 } else {
e8996ee0 4105 if (def->flags & TCG_OPF_CALL_CLOBBER) {
a813e36f 4106 /* XXX: permit generic clobber register list ? */
c8074023
RH		 4107 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4108 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 4109 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 4110 }
c896fe29 4111 }
3d5c5f87
AJ		 4112 }
4113 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4114 /* sync globals if the op has side effects and might trigger
4115 an exception. */
82790a87 4116 sync_globals(s, i_allocated_regs);
c896fe29 4117 }
a813e36f 4118
e8996ee0 4119 /* satisfy the output constraints */
e8996ee0 4120 for(k = 0; k < nb_oargs; k++) {
66792f90 4121 i = def->args_ct[k].sort_index;
dd186292 4122 arg = op->args[i];
e8996ee0 4123 arg_ct = &def->args_ct[i];
43439139 4124 ts = arg_temp(arg);
d63e3b6e
RH		 4125
4126 /* ENV should not be modified. */
e01fa97d 4127 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4128
29f5e925
RH		 4129 switch (arg_ct->pair) {
4130 case 0: /* not paired */
4131 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
4132 reg = new_args[arg_ct->alias_index];
4133 } else if (arg_ct->newreg) {
4134 reg = tcg_reg_alloc(s, arg_ct->regs,
4135 i_allocated_regs | o_allocated_regs,
31fd884b 4136 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4137 } else {
4138 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
31fd884b 4139 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4140 }
4141 break;
4142
4143 case 1: /* first of pair */
4144 tcg_debug_assert(!arg_ct->newreg);
4145 if (arg_ct->oalias) {
4146 reg = new_args[arg_ct->alias_index];
4147 break;
4148 }
4149 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
31fd884b 4150 output_pref(op, k), ts->indirect_base);
29f5e925
RH		 4151 break;
4152
4153 case 2: /* second of pair */
4154 tcg_debug_assert(!arg_ct->newreg);
4155 if (arg_ct->oalias) {
4156 reg = new_args[arg_ct->alias_index];
4157 } else {
4158 reg = new_args[arg_ct->pair_index] + 1;
4159 }
4160 break;
4161
4162 case 3: /* first of pair, aliasing with a second input */
4163 tcg_debug_assert(!arg_ct->newreg);
4164 reg = new_args[arg_ct->pair_index] - 1;
4165 break;
4166
4167 default:
4168 g_assert_not_reached();
c896fe29 4169 }
82790a87 4170 tcg_regset_set_reg(o_allocated_regs, reg);
098859f1 4171 set_temp_val_reg(s, ts, reg);
d63e3b6e 4172 ts->mem_coherent = 0;
e8996ee0 4173 new_args[i] = reg;
c896fe29 4174 }
c896fe29
FB		 4175 }
4176
c896fe29 4177 /* emit instruction */
d2fd745f
RH		 4178 if (def->flags & TCG_OPF_VECTOR) {
4179 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
4180 new_args, const_args);
4181 } else {
4182 tcg_out_op(s, op->opc, new_args, const_args);
4183 }
4184
c896fe29
FB		 4185 /* move the outputs in the correct register if needed */
4186 for(i = 0; i < nb_oargs; i++) {
43439139 4187 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 4188
4189 /* ENV should not be modified. */
e01fa97d 4190 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4191
ec7a869d 4192 if (NEED_SYNC_ARG(i)) {
98b4e186 4193 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 4194 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 4195 temp_dead(s, ts);
ec7a869d 4196 }
c896fe29
FB		 4197 }
4198}
4199
efe86b21
RH		 4200static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
4201{
4202 const TCGLifeData arg_life = op->life;
4203 TCGTemp *ots, *itsl, *itsh;
4204 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4205
4206 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
4207 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
4208 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
4209
4210 ots = arg_temp(op->args[0]);
4211 itsl = arg_temp(op->args[1]);
4212 itsh = arg_temp(op->args[2]);
4213
4214 /* ENV should not be modified. */
4215 tcg_debug_assert(!temp_readonly(ots));
4216
4217 /* Allocate the output register now. */
4218 if (ots->val_type != TEMP_VAL_REG) {
4219 TCGRegSet allocated_regs = s->reserved_regs;
4220 TCGRegSet dup_out_regs =
4221 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
098859f1 4222 TCGReg oreg;
efe86b21
RH		 4223
4224 /* Make sure to not spill the input registers. */
4225 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
4226 tcg_regset_set_reg(allocated_regs, itsl->reg);
4227 }
4228 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
4229 tcg_regset_set_reg(allocated_regs, itsh->reg);
4230 }
4231
098859f1 4232 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
31fd884b 4233 output_pref(op, 0), ots->indirect_base);
098859f1 4234 set_temp_val_reg(s, ots, oreg);
efe86b21
RH		 4235 }
4236
4237 /* Promote dup2 of immediates to dupi_vec. */
4238 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
4239 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
4240 MemOp vece = MO_64;
4241
4242 if (val == dup_const(MO_8, val)) {
4243 vece = MO_8;
4244 } else if (val == dup_const(MO_16, val)) {
4245 vece = MO_16;
4246 } else if (val == dup_const(MO_32, val)) {
4247 vece = MO_32;
4248 }
4249
4250 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
4251 goto done;
4252 }
4253
4254 /* If the two inputs form one 64-bit value, try dupm_vec. */
aef85402
RH		 4255 if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
4256 itsh->temp_subindex == !HOST_BIG_ENDIAN &&
4257 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
4258 TCGTemp *its = itsl - HOST_BIG_ENDIAN;
4259
4260 temp_sync(s, its + 0, s->reserved_regs, 0, 0);
4261 temp_sync(s, its + 1, s->reserved_regs, 0, 0);
4262
efe86b21
RH		 4263 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
4264 its->mem_base->reg, its->mem_offset)) {
4265 goto done;
4266 }
4267 }
4268
4269 /* Fall back to generic expansion. */
4270 return false;
4271
4272 done:
36f5539c 4273 ots->mem_coherent = 0;
efe86b21
RH		 4274 if (IS_DEAD_ARG(1)) {
4275 temp_dead(s, itsl);
4276 }
4277 if (IS_DEAD_ARG(2)) {
4278 temp_dead(s, itsh);
4279 }
4280 if (NEED_SYNC_ARG(0)) {
4281 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
4282 } else if (IS_DEAD_ARG(0)) {
4283 temp_dead(s, ots);
4284 }
4285 return true;
4286}
4287
39004a71
RH		 4288static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
4289 TCGRegSet allocated_regs)
c896fe29 4290{
39004a71
RH		 4291 if (ts->val_type == TEMP_VAL_REG) {
4292 if (ts->reg != reg) {
4293 tcg_reg_free(s, reg, allocated_regs);
4294 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4295 /*
4296 * Cross register class move not supported. Sync the
4297 * temp back to its slot and load from there.
4298 */
4299 temp_sync(s, ts, allocated_regs, 0, 0);
4300 tcg_out_ld(s, ts->type, reg,
4301 ts->mem_base->reg, ts->mem_offset);
4302 }
4303 }
4304 } else {
4305 TCGRegSet arg_set = 0;
c896fe29 4306
39004a71
RH		 4307 tcg_reg_free(s, reg, allocated_regs);
4308 tcg_regset_set_reg(arg_set, reg);
4309 temp_load(s, ts, arg_set, allocated_regs, 0);
b03cce8e 4310 }
39004a71 4311}
39cf05d3 4312
39004a71
RH		 4313static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts,
4314 TCGRegSet allocated_regs)
4315{
4316 /*
4317 * When the destination is on the stack, load up the temp and store.
4318 * If there are many call-saved registers, the temp might live to
4319 * see another use; otherwise it'll be discarded.
4320 */
4321 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
4322 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
4323 TCG_TARGET_CALL_STACK_OFFSET +
4324 stk_slot * sizeof(tcg_target_long));
4325}
a813e36f 4326
39004a71
RH		 4327static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
4328 TCGTemp *ts, TCGRegSet *allocated_regs)
4329{
4330 if (REG_P(l)) {
4331 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
4332 load_arg_reg(s, reg, ts, *allocated_regs);
4333 tcg_regset_set_reg(*allocated_regs, reg);
4334 } else {
4335 load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs),
4336 ts, *allocated_regs);
4337 }
4338}
40ae5c62 4339
39004a71
RH		 4340static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
4341{
4342 const int nb_oargs = TCGOP_CALLO(op);
4343 const int nb_iargs = TCGOP_CALLI(op);
4344 const TCGLifeData arg_life = op->life;
4345 const TCGHelperInfo *info = tcg_call_info(op);
4346 TCGRegSet allocated_regs = s->reserved_regs;
4347 int i;
40ae5c62 4348
39004a71
RH		 4349 /*
4350 * Move inputs into place in reverse order,
4351 * so that we place stacked arguments first.
4352 */
4353 for (i = nb_iargs - 1; i >= 0; --i) {
4354 const TCGCallArgumentLoc *loc = &info->in[i];
4355 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
40ae5c62 4356
39004a71
RH		 4357 switch (loc->kind) {
4358 case TCG_CALL_ARG_NORMAL:
4359 case TCG_CALL_ARG_EXTEND_U:
4360 case TCG_CALL_ARG_EXTEND_S:
4361 load_arg_normal(s, loc, ts, &allocated_regs);
4362 break;
4363 default:
4364 g_assert_not_reached();
c896fe29 4365 }
c896fe29 4366 }
a813e36f 4367
39004a71 4368 /* Mark dead temporaries and free the associated registers. */
dd186292 4369 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 4370 if (IS_DEAD_ARG(i)) {
43439139 4371 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 4372 }
4373 }
a813e36f 4374
39004a71 4375 /* Clobber call registers. */
c8074023
RH		 4376 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4377 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 4378 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 4379 }
4380 }
78505279 4381
39004a71
RH		 4382 /*
4383 * Save globals if they might be written by the helper,
4384 * sync them if they might be read.
4385 */
4386 if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
78505279 4387 /* Nothing to do */
39004a71 4388 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
78505279
AJ		 4389 sync_globals(s, allocated_regs);
4390 } else {
b9c18f56
AJ		 4391 save_globals(s, allocated_regs);
4392 }
c896fe29 4393
cee44b03 4394 tcg_out_call(s, tcg_call_func(op), info);
c896fe29 4395
39004a71
RH		 4396 /* Assign output registers and emit moves if needed. */
4397 switch (info->out_kind) {
4398 case TCG_CALL_RET_NORMAL:
4399 for (i = 0; i < nb_oargs; i++) {
4400 TCGTemp *ts = arg_temp(op->args[i]);
4401 TCGReg reg = tcg_target_call_oarg_regs[i];
d63e3b6e 4402
39004a71
RH		 4403 /* ENV should not be modified. */
4404 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4405
39004a71
RH		 4406 set_temp_val_reg(s, ts, reg);
4407 ts->mem_coherent = 0;
4408 }
4409 break;
4410 default:
4411 g_assert_not_reached();
4412 }
4413
4414 /* Flush or discard output registers as needed. */
4415 for (i = 0; i < nb_oargs; i++) {
4416 TCGTemp *ts = arg_temp(op->args[i]);
d63e3b6e 4417 if (NEED_SYNC_ARG(i)) {
39004a71 4418 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
d63e3b6e
RH		 4419 } else if (IS_DEAD_ARG(i)) {
4420 temp_dead(s, ts);
c896fe29
FB		 4421 }
4422 }
c896fe29
FB		 4423}
4424
4425#ifdef CONFIG_PROFILER
4426
c3fac113
EC		 4427/* avoid copy/paste errors */
4428#define PROF_ADD(to, from, field) \
4429 do { \
d73415a3 4430 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 4431 } while (0)
4432
4433#define PROF_MAX(to, from, field) \
4434 do { \
d73415a3 4435 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 4436 if (val__ > (to)->field) { \
4437 (to)->field = val__; \
4438 } \
4439 } while (0)
4440
4441/* Pass in a zero'ed @prof */
4442static inline
4443void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4444{
0e2d61cf 4445 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 4446 unsigned int i;
4447
3468b59e 4448 for (i = 0; i < n_ctxs; i++) {
d73415a3 4449 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 4450 const TCGProfile *orig = &s->prof;
c3fac113
EC		 4451
4452 if (counters) {
72fd2efb 4453 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 4454 PROF_ADD(prof, orig, tb_count1);
4455 PROF_ADD(prof, orig, tb_count);
4456 PROF_ADD(prof, orig, op_count);
4457 PROF_MAX(prof, orig, op_count_max);
4458 PROF_ADD(prof, orig, temp_count);
4459 PROF_MAX(prof, orig, temp_count_max);
4460 PROF_ADD(prof, orig, del_op_count);
4461 PROF_ADD(prof, orig, code_in_len);
4462 PROF_ADD(prof, orig, code_out_len);
4463 PROF_ADD(prof, orig, search_out_len);
4464 PROF_ADD(prof, orig, interm_time);
4465 PROF_ADD(prof, orig, code_time);
4466 PROF_ADD(prof, orig, la_time);
4467 PROF_ADD(prof, orig, opt_time);
4468 PROF_ADD(prof, orig, restore_count);
4469 PROF_ADD(prof, orig, restore_time);
4470 }
4471 if (table) {
4472 int i;
4473
4474 for (i = 0; i < NB_OPS; i++) {
4475 PROF_ADD(prof, orig, table_op_count[i]);
4476 }
4477 }
4478 }
4479}
4480
4481#undef PROF_ADD
4482#undef PROF_MAX
4483
4484static void tcg_profile_snapshot_counters(TCGProfile *prof)
4485{
4486 tcg_profile_snapshot(prof, true, false);
4487}
4488
4489static void tcg_profile_snapshot_table(TCGProfile *prof)
4490{
4491 tcg_profile_snapshot(prof, false, true);
4492}
c896fe29 4493
b6a7f3e0 4494void tcg_dump_op_count(GString *buf)
c896fe29 4495{
c3fac113 4496 TCGProfile prof = {};
c896fe29 4497 int i;
d70724ce 4498
c3fac113 4499 tcg_profile_snapshot_table(&prof);
15fc7daa 4500 for (i = 0; i < NB_OPS; i++) {
b6a7f3e0
DB		 4501 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
4502 prof.table_op_count[i]);
c896fe29 4503 }
c896fe29 4504}
72fd2efb
EC		 4505
4506int64_t tcg_cpu_exec_time(void)
4507{
0e2d61cf 4508 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4509 unsigned int i;
4510 int64_t ret = 0;
4511
4512 for (i = 0; i < n_ctxs; i++) {
d73415a3 4513 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4514 const TCGProfile *prof = &s->prof;
4515
d73415a3 4516 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4517 }
4518 return ret;
4519}
246ae24d 4520#else
b6a7f3e0 4521void tcg_dump_op_count(GString *buf)
246ae24d 4522{
b6a7f3e0 4523 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
246ae24d 4524}
72fd2efb
EC		 4525
4526int64_t tcg_cpu_exec_time(void)
4527{
4528 error_report("%s: TCG profiler not compiled", __func__);
4529 exit(EXIT_FAILURE);
4530}
c896fe29
FB		 4531#endif
4532
4533
fbf59aad 4534int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
c896fe29 4535{
c3fac113
EC		 4536#ifdef CONFIG_PROFILER
4537 TCGProfile *prof = &s->prof;
4538#endif
15fa08f8
RH		 4539 int i, num_insns;
4540 TCGOp *op;
c896fe29 4541
04fe6400
RH		 4542#ifdef CONFIG_PROFILER
4543 {
c1f543b7 4544 int n = 0;
04fe6400 4545
15fa08f8
RH		 4546 QTAILQ_FOREACH(op, &s->ops, link) {
4547 n++;
4548 }
d73415a3 4549 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4550 if (n > prof->op_count_max) {
d73415a3 4551 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4552 }
4553
4554 n = s->nb_temps;
d73415a3 4555 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4556 if (n > prof->temp_count_max) {
d73415a3 4557 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4558 }
4559 }
4560#endif
4561
c896fe29 4562#ifdef DEBUG_DISAS
d977e1c2 4563 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
fbf59aad 4564 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4565 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4566 if (logfile) {
4567 fprintf(logfile, "OP:\n");
b7a83ff8 4568 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4569 fprintf(logfile, "\n");
4570 qemu_log_unlock(logfile);
4571 }
c896fe29
FB		 4572 }
4573#endif
4574
bef16ab4
RH		 4575#ifdef CONFIG_DEBUG_TCG
4576 /* Ensure all labels referenced have been emitted. */
4577 {
4578 TCGLabel *l;
4579 bool error = false;
4580
4581 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4582 if (unlikely(!l->present) && l->refs) {
4583 qemu_log_mask(CPU_LOG_TB_OP,
4584 "$L%d referenced but not present.\n", l->id);
4585 error = true;
4586 }
4587 }
4588 assert(!error);
4589 }
4590#endif
4591
c5cc28ff 4592#ifdef CONFIG_PROFILER
d73415a3 4593 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4594#endif
4595
8f2e8c07 4596#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4597 tcg_optimize(s);
8f2e8c07
KB		 4598#endif
4599
a23a9ec6 4600#ifdef CONFIG_PROFILER
d73415a3
SH		 4601 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4602 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4603#endif
c5cc28ff 4604
b4fc67c7 4605 reachable_code_pass(s);
b83eabea 4606 liveness_pass_1(s);
5a18407f 4607
b83eabea 4608 if (s->nb_indirects > 0) {
5a18407f 4609#ifdef DEBUG_DISAS
b83eabea 4610 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
fbf59aad 4611 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4612 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4613 if (logfile) {
4614 fprintf(logfile, "OP before indirect lowering:\n");
b7a83ff8 4615 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4616 fprintf(logfile, "\n");
4617 qemu_log_unlock(logfile);
4618 }
b83eabea 4619 }
5a18407f 4620#endif
b83eabea
RH		 4621 /* Replace indirect temps with direct temps. */
4622 if (liveness_pass_2(s)) {
4623 /* If changes were made, re-run liveness. */
4624 liveness_pass_1(s);
5a18407f
RH		 4625 }
4626 }
c5cc28ff 4627
a23a9ec6 4628#ifdef CONFIG_PROFILER
d73415a3 4629 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4630#endif
c896fe29
FB		 4631
4632#ifdef DEBUG_DISAS
d977e1c2 4633 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
fbf59aad 4634 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4635 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4636 if (logfile) {
4637 fprintf(logfile, "OP after optimization and liveness analysis:\n");
b7a83ff8 4638 tcg_dump_ops(s, logfile, true);
78b54858
RH		 4639 fprintf(logfile, "\n");
4640 qemu_log_unlock(logfile);
4641 }
c896fe29
FB		 4642 }
4643#endif
4644
35abb009
RH		 4645 /* Initialize goto_tb jump offsets. */
4646 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
4647 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
4648 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
4649 if (TCG_TARGET_HAS_direct_jump) {
4650 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
4651 tcg_ctx->tb_jmp_target_addr = NULL;
4652 } else {
4653 tcg_ctx->tb_jmp_insn_offset = NULL;
4654 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
4655 }
4656
c896fe29
FB		 4657 tcg_reg_alloc_start(s);
4658
db0c51a3
RH		 4659 /*
4660 * Reset the buffer pointers when restarting after overflow.
4661 * TODO: Move this into translate-all.c with the rest of the
4662 * buffer management. Having only this done here is confusing.
4663 */
4664 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4665 s->code_ptr = s->code_buf;
c896fe29 4666
659ef5cb 4667#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4668 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4669#endif
57a26946
RH		 4670#ifdef TCG_TARGET_NEED_POOL_LABELS
4671 s->pool_labels = NULL;
4672#endif
9ecefc84 4673
fca8a500 4674 num_insns = -1;
15fa08f8 4675 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4676 TCGOpcode opc = op->opc;
b3db8758 4677
c896fe29 4678#ifdef CONFIG_PROFILER
d73415a3 4679 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 4680#endif
c45cb8bb
RH		 4681
4682 switch (opc) {
c896fe29 4683 case INDEX_op_mov_i32:
c896fe29 4684 case INDEX_op_mov_i64:
d2fd745f 4685 case INDEX_op_mov_vec:
dd186292 4686 tcg_reg_alloc_mov(s, op);
c896fe29 4687 break;
bab1671f
RH		 4688 case INDEX_op_dup_vec:
4689 tcg_reg_alloc_dup(s, op);
4690 break;
765b842a 4691 case INDEX_op_insn_start:
fca8a500 4692 if (num_insns >= 0) {
9f754620
RH		 4693 size_t off = tcg_current_code_size(s);
4694 s->gen_insn_end_off[num_insns] = off;
4695 /* Assert that we do not overflow our stored offset. */
4696 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 4697 }
4698 num_insns++;
bad729e2
RH		 4699 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4700 target_ulong a;
4701#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 4702 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 4703#else
efee3746 4704 a = op->args[i];
bad729e2 4705#endif
fca8a500 4706 s->gen_insn_data[num_insns][i] = a;
bad729e2 4707 }
c896fe29 4708 break;
5ff9d6a4 4709 case INDEX_op_discard:
43439139 4710 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 4711 break;
c896fe29 4712 case INDEX_op_set_label:
e8996ee0 4713 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 4714 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 4715 break;
4716 case INDEX_op_call:
dd186292 4717 tcg_reg_alloc_call(s, op);
c45cb8bb 4718 break;
efe86b21
RH		 4719 case INDEX_op_dup2_vec:
4720 if (tcg_reg_alloc_dup2(s, op)) {
4721 break;
4722 }
4723 /* fall through */
c896fe29 4724 default:
25c4d9cc 4725 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 4726 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 4727 /* Note: in order to speed up the code, it would be much
4728 faster to have specialized register allocator functions for
4729 some common argument patterns */
dd186292 4730 tcg_reg_alloc_op(s, op);
c896fe29
FB		 4731 break;
4732 }
b125f9dc
RH		 4733 /* Test for (pending) buffer overflow. The assumption is that any
4734 one operation beginning below the high water mark cannot overrun
4735 the buffer completely. Thus we can test for overflow after
4736 generating code without having to check during generation. */
644da9b3 4737 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 4738 return -1;
4739 }
6e6c4efe
RH		 4740 /* Test for TB overflow, as seen by gen_insn_end_off. */
4741 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4742 return -2;
4743 }
c896fe29 4744 }
fca8a500
RH		 4745 tcg_debug_assert(num_insns >= 0);
4746 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 4747
b76f0d8c 4748 /* Generate TB finalization at the end of block */
659ef5cb 4749#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 4750 i = tcg_out_ldst_finalize(s);
4751 if (i < 0) {
4752 return i;
23dceda6 4753 }
659ef5cb 4754#endif
57a26946 4755#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 4756 i = tcg_out_pool_finalize(s);
4757 if (i < 0) {
4758 return i;
57a26946
RH		 4759 }
4760#endif
7ecd02a0
RH		 4761 if (!tcg_resolve_relocs(s)) {
4762 return -2;
4763 }
c896fe29 4764
df5d2b16 4765#ifndef CONFIG_TCG_INTERPRETER
c896fe29 4766 /* flush instruction cache */
db0c51a3
RH		 4767 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4768 (uintptr_t)s->code_buf,
1da8de39 4769 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 4770#endif
2aeabc08 4771
1813e175 4772 return tcg_current_code_size(s);
c896fe29
FB		 4773}
4774
a23a9ec6 4775#ifdef CONFIG_PROFILER
3a841ab5 4776void tcg_dump_info(GString *buf)
a23a9ec6 4777{
c3fac113
EC		 4778 TCGProfile prof = {};
4779 const TCGProfile *s;
4780 int64_t tb_count;
4781 int64_t tb_div_count;
4782 int64_t tot;
4783
4784 tcg_profile_snapshot_counters(&prof);
4785 s = &prof;
4786 tb_count = s->tb_count;
4787 tb_div_count = tb_count ? tb_count : 1;
4788 tot = s->interm_time + s->code_time;
a23a9ec6 4789
3a841ab5
DB		 4790 g_string_append_printf(buf, "JIT cycles %" PRId64
4791 " (%0.3f s at 2.4 GHz)\n",
4792 tot, tot / 2.4e9);
4793 g_string_append_printf(buf, "translated TBs %" PRId64
4794 " (aborted=%" PRId64 " %0.1f%%)\n",
4795 tb_count, s->tb_count1 - tb_count,
4796 (double)(s->tb_count1 - s->tb_count)
4797 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4798 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
4799 (double)s->op_count / tb_div_count, s->op_count_max);
4800 g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
4801 (double)s->del_op_count / tb_div_count);
4802 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
4803 (double)s->temp_count / tb_div_count,
4804 s->temp_count_max);
4805 g_string_append_printf(buf, "avg host code/TB %0.1f\n",
4806 (double)s->code_out_len / tb_div_count);
4807 g_string_append_printf(buf, "avg search data/TB %0.1f\n",
4808 (double)s->search_out_len / tb_div_count);
a813e36f 4809
3a841ab5
DB		 4810 g_string_append_printf(buf, "cycles/op %0.1f\n",
4811 s->op_count ? (double)tot / s->op_count : 0);
4812 g_string_append_printf(buf, "cycles/in byte %0.1f\n",
4813 s->code_in_len ? (double)tot / s->code_in_len : 0);
4814 g_string_append_printf(buf, "cycles/out byte %0.1f\n",
4815 s->code_out_len ? (double)tot / s->code_out_len : 0);
4816 g_string_append_printf(buf, "cycles/search byte %0.1f\n",
4817 s->search_out_len ?
4818 (double)tot / s->search_out_len : 0);
fca8a500 4819 if (tot == 0) {
a23a9ec6 4820 tot = 1;
fca8a500 4821 }
3a841ab5
DB		 4822 g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
4823 (double)s->interm_time / tot * 100.0);
4824 g_string_append_printf(buf, " gen_code time %0.1f%%\n",
4825 (double)s->code_time / tot * 100.0);
4826 g_string_append_printf(buf, "optim./code time %0.1f%%\n",
4827 (double)s->opt_time / (s->code_time ?
4828 s->code_time : 1)
4829 * 100.0);
4830 g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
4831 (double)s->la_time / (s->code_time ?
4832 s->code_time : 1) * 100.0);
4833 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
4834 s->restore_count);
4835 g_string_append_printf(buf, " avg cycles %0.1f\n",
4836 s->restore_count ?
4837 (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 4838}
4839#else
3a841ab5 4840void tcg_dump_info(GString *buf)
a23a9ec6 4841{
3a841ab5 4842 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
a23a9ec6
FB		 4843}
4844#endif
813da627
RH		 4845
4846#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 4847/* In order to use this feature, the backend needs to do three things:
4848
4849 (1) Define ELF_HOST_MACHINE to indicate both what value to
4850 put into the ELF image and to indicate support for the feature.
4851
4852 (2) Define tcg_register_jit. This should create a buffer containing
4853 the contents of a .debug_frame section that describes the post-
4854 prologue unwind info for the tcg machine.
4855
4856 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4857*/
813da627
RH		 4858
4859/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4860typedef enum {
4861 JIT_NOACTION = 0,
4862 JIT_REGISTER_FN,
4863 JIT_UNREGISTER_FN
4864} jit_actions_t;
4865
4866struct jit_code_entry {
4867 struct jit_code_entry *next_entry;
4868 struct jit_code_entry *prev_entry;
4869 const void *symfile_addr;
4870 uint64_t symfile_size;
4871};
4872
4873struct jit_descriptor {
4874 uint32_t version;
4875 uint32_t action_flag;
4876 struct jit_code_entry *relevant_entry;
4877 struct jit_code_entry *first_entry;
4878};
4879
4880void __jit_debug_register_code(void) __attribute__((noinline));
4881void __jit_debug_register_code(void)
4882{
4883 asm("");
4884}
4885
4886/* Must statically initialize the version, because GDB may check
4887 the version before we can set it. */
4888struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4889
4890/* End GDB interface. */
4891
4892static int find_string(const char *strtab, const char *str)
4893{
4894 const char *p = strtab + 1;
4895
4896 while (1) {
4897 if (strcmp(p, str) == 0) {
4898 return p - strtab;
4899 }
4900 p += strlen(p) + 1;
4901 }
4902}
4903
755bf9e5 4904static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 4905 const void *debug_frame,
4906 size_t debug_frame_size)
813da627 4907{
5872bbf2
RH		 4908 struct __attribute__((packed)) DebugInfo {
4909 uint32_t len;
4910 uint16_t version;
4911 uint32_t abbrev;
4912 uint8_t ptr_size;
4913 uint8_t cu_die;
4914 uint16_t cu_lang;
4915 uintptr_t cu_low_pc;
4916 uintptr_t cu_high_pc;
4917 uint8_t fn_die;
4918 char fn_name[16];
4919 uintptr_t fn_low_pc;
4920 uintptr_t fn_high_pc;
4921 uint8_t cu_eoc;
4922 };
813da627
RH		 4923
4924 struct ElfImage {
4925 ElfW(Ehdr) ehdr;
4926 ElfW(Phdr) phdr;
5872bbf2
RH		 4927 ElfW(Shdr) shdr[7];
4928 ElfW(Sym) sym[2];
4929 struct DebugInfo di;
4930 uint8_t da[24];
4931 char str[80];
4932 };
4933
4934 struct ElfImage *img;
4935
4936 static const struct ElfImage img_template = {
4937 .ehdr = {
4938 .e_ident[EI_MAG0] = ELFMAG0,
4939 .e_ident[EI_MAG1] = ELFMAG1,
4940 .e_ident[EI_MAG2] = ELFMAG2,
4941 .e_ident[EI_MAG3] = ELFMAG3,
4942 .e_ident[EI_CLASS] = ELF_CLASS,
4943 .e_ident[EI_DATA] = ELF_DATA,
4944 .e_ident[EI_VERSION] = EV_CURRENT,
4945 .e_type = ET_EXEC,
4946 .e_machine = ELF_HOST_MACHINE,
4947 .e_version = EV_CURRENT,
4948 .e_phoff = offsetof(struct ElfImage, phdr),
4949 .e_shoff = offsetof(struct ElfImage, shdr),
4950 .e_ehsize = sizeof(ElfW(Shdr)),
4951 .e_phentsize = sizeof(ElfW(Phdr)),
4952 .e_phnum = 1,
4953 .e_shentsize = sizeof(ElfW(Shdr)),
4954 .e_shnum = ARRAY_SIZE(img->shdr),
4955 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 4956#ifdef ELF_HOST_FLAGS
4957 .e_flags = ELF_HOST_FLAGS,
4958#endif
4959#ifdef ELF_OSABI
4960 .e_ident[EI_OSABI] = ELF_OSABI,
4961#endif
5872bbf2
RH		 4962 },
4963 .phdr = {
4964 .p_type = PT_LOAD,
4965 .p_flags = PF_X,
4966 },
4967 .shdr = {
4968 [0] = { .sh_type = SHT_NULL },
4969 /* Trick: The contents of code_gen_buffer are not present in
4970 this fake ELF file; that got allocated elsewhere. Therefore
4971 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4972 will not look for contents. We can record any address. */
4973 [1] = { /* .text */
4974 .sh_type = SHT_NOBITS,
4975 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4976 },
4977 [2] = { /* .debug_info */
4978 .sh_type = SHT_PROGBITS,
4979 .sh_offset = offsetof(struct ElfImage, di),
4980 .sh_size = sizeof(struct DebugInfo),
4981 },
4982 [3] = { /* .debug_abbrev */
4983 .sh_type = SHT_PROGBITS,
4984 .sh_offset = offsetof(struct ElfImage, da),
4985 .sh_size = sizeof(img->da),
4986 },
4987 [4] = { /* .debug_frame */
4988 .sh_type = SHT_PROGBITS,
4989 .sh_offset = sizeof(struct ElfImage),
4990 },
4991 [5] = { /* .symtab */
4992 .sh_type = SHT_SYMTAB,
4993 .sh_offset = offsetof(struct ElfImage, sym),
4994 .sh_size = sizeof(img->sym),
4995 .sh_info = 1,
4996 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4997 .sh_entsize = sizeof(ElfW(Sym)),
4998 },
4999 [6] = { /* .strtab */
5000 .sh_type = SHT_STRTAB,
5001 .sh_offset = offsetof(struct ElfImage, str),
5002 .sh_size = sizeof(img->str),
5003 }
5004 },
5005 .sym = {
5006 [1] = { /* code_gen_buffer */
5007 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
5008 .st_shndx = 1,
5009 }
5010 },
5011 .di = {
5012 .len = sizeof(struct DebugInfo) - 4,
5013 .version = 2,
5014 .ptr_size = sizeof(void *),
5015 .cu_die = 1,
5016 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
5017 .fn_die = 2,
5018 .fn_name = "code_gen_buffer"
5019 },
5020 .da = {
5021 1, /* abbrev number (the cu) */
5022 0x11, 1, /* DW_TAG_compile_unit, has children */
5023 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
5024 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5025 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5026 0, 0, /* end of abbrev */
5027 2, /* abbrev number (the fn) */
5028 0x2e, 0, /* DW_TAG_subprogram, no children */
5029 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
5030 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5031 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5032 0, 0, /* end of abbrev */
5033 0 /* no more abbrev */
5034 },
5035 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
5036 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 5037 };
5038
5039 /* We only need a single jit entry; statically allocate it. */
5040 static struct jit_code_entry one_entry;
5041
5872bbf2 5042 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 5043 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 5044 DebugFrameHeader *dfh;
813da627 5045
5872bbf2
RH		 5046 img = g_malloc(img_size);
5047 *img = img_template;
813da627 5048
5872bbf2
RH		 5049 img->phdr.p_vaddr = buf;
5050 img->phdr.p_paddr = buf;
5051 img->phdr.p_memsz = buf_size;
813da627 5052
813da627 5053 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 5054 img->shdr[1].sh_addr = buf;
813da627
RH		 5055 img->shdr[1].sh_size = buf_size;
5056
5872bbf2
RH		 5057 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
5058 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
5059
5060 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
5061 img->shdr[4].sh_size = debug_frame_size;
5062
5063 img->shdr[5].sh_name = find_string(img->str, ".symtab");
5064 img->shdr[6].sh_name = find_string(img->str, ".strtab");
5065
5066 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
5067 img->sym[1].st_value = buf;
5068 img->sym[1].st_size = buf_size;
813da627 5069
5872bbf2 5070 img->di.cu_low_pc = buf;
45aba097 5071 img->di.cu_high_pc = buf + buf_size;
5872bbf2 5072 img->di.fn_low_pc = buf;
45aba097 5073 img->di.fn_high_pc = buf + buf_size;
813da627 5074
2c90784a
RH		 5075 dfh = (DebugFrameHeader *)(img + 1);
5076 memcpy(dfh, debug_frame, debug_frame_size);
5077 dfh->fde.func_start = buf;
5078 dfh->fde.func_len = buf_size;
5079
813da627
RH		 5080#ifdef DEBUG_JIT
5081 /* Enable this block to be able to debug the ELF image file creation.
5082 One can use readelf, objdump, or other inspection utilities. */
5083 {
eb6b2edf
BM		 5084 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
5085 FILE *f = fopen(jit, "w+b");
813da627 5086 if (f) {
5872bbf2 5087 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 5088 /* Avoid stupid unused return value warning for fwrite. */
5089 }
5090 fclose(f);
5091 }
5092 }
5093#endif
5094
5095 one_entry.symfile_addr = img;
5096 one_entry.symfile_size = img_size;
5097
5098 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
5099 __jit_debug_descriptor.relevant_entry = &one_entry;
5100 __jit_debug_descriptor.first_entry = &one_entry;
5101 __jit_debug_register_code();
5102}
5103#else
5872bbf2
RH		 5104/* No support for the feature. Provide the entry point expected by exec.c,
5105 and implement the internal function we declared earlier. */
813da627 5106
755bf9e5 5107static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 5108 const void *debug_frame,
5109 size_t debug_frame_size)
813da627
RH		 5110{
5111}
5112
755bf9e5 5113void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 5114{
5115}
5116#endif /* ELF_HOST_MACHINE */
db432672
RH		 5117
5118#if !TCG_TARGET_MAYBE_vec
5119void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
5120{
5121 g_assert_not_reached();
5122}
5123#endif
QEMU mirror