]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/ppc/tcg-target.c.inc
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg: Replace TCG_TARGET_CALL_ALIGN_ARGS with TCG_TARGET_CALL_ARG_I64
[mirror_qemu.git] / tcg / ppc / tcg-target.c.inc
CommitLineData
810260a8 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
53c89efd 25#include "elf.h"
139c1837 26#include "../tcg-pool.c.inc"
8605cbcd 27#include "../tcg-ldst.c.inc"
53c89efd 28
d2168985
RH		 29/*
30 * Standardize on the _CALL_FOO symbols used by GCC:
31 * Apple XCode does not define _CALL_DARWIN.
32 * Clang defines _CALL_ELF (64-bit) but not _CALL_SYSV (32-bit).
33 */
34#if !defined(_CALL_SYSV) && \
35 !defined(_CALL_DARWIN) && \
36 !defined(_CALL_AIX) && \
37 !defined(_CALL_ELF)
38# if defined(__APPLE__)
39# define _CALL_DARWIN
40# elif defined(__ELF__) && TCG_TARGET_REG_BITS == 32
41# define _CALL_SYSV
42# else
43# error "Unknown ABI"
44# endif
a813e36f 45#endif
d2168985 46
7f25c469 47#ifdef _CALL_SYSV
c8eef960
RH		 48# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN
49#else
50# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL
7f25c469 51#endif
ffcfbece 52
dfca1778
RH		 53/* For some memory operations, we need a scratch that isn't R0. For the AIX
54 calling convention, we can re-use the TOC register since we'll be reloading
55 it at every call. Otherwise R12 will do nicely as neither a call-saved
56 register nor a parameter register. */
57#ifdef _CALL_AIX
58# define TCG_REG_TMP1 TCG_REG_R2
59#else
60# define TCG_REG_TMP1 TCG_REG_R12
61#endif
62
42281ec6
RH		 63#define TCG_VEC_TMP1 TCG_REG_V0
64#define TCG_VEC_TMP2 TCG_REG_V1
65
5964fca8
RH		 66#define TCG_REG_TB TCG_REG_R31
67#define USE_REG_TB (TCG_TARGET_REG_BITS == 64)
a84ac4cb 68
de3d636d
RH		 69/* Shorthand for size of a pointer. Avoid promotion to unsigned. */
70#define SZP ((int)sizeof(void *))
71
4c3831a0
RH		 72/* Shorthand for size of a register. */
73#define SZR (TCG_TARGET_REG_BITS / 8)
74
3d582c61
RH		 75#define TCG_CT_CONST_S16 0x100
76#define TCG_CT_CONST_U16 0x200
77#define TCG_CT_CONST_S32 0x400
78#define TCG_CT_CONST_U32 0x800
79#define TCG_CT_CONST_ZERO 0x1000
6c858762 80#define TCG_CT_CONST_MONE 0x2000
d0b07481 81#define TCG_CT_CONST_WSZ 0x4000
fe6f943f 82
85d251d7
RH		 83#define ALL_GENERAL_REGS 0xffffffffu
84#define ALL_VECTOR_REGS 0xffffffff00000000ull
85
86#ifdef CONFIG_SOFTMMU
87#define ALL_QLOAD_REGS \
88 (ALL_GENERAL_REGS & \
89 ~((1 << TCG_REG_R3) | (1 << TCG_REG_R4) | (1 << TCG_REG_R5)))
90#define ALL_QSTORE_REGS \
91 (ALL_GENERAL_REGS & ~((1 << TCG_REG_R3) | (1 << TCG_REG_R4) | \
92 (1 << TCG_REG_R5) | (1 << TCG_REG_R6)))
93#else
94#define ALL_QLOAD_REGS (ALL_GENERAL_REGS & ~(1 << TCG_REG_R3))
95#define ALL_QSTORE_REGS ALL_QLOAD_REGS
96#endif
97
7d9dae0a 98TCGPowerISA have_isa;
63922f46 99static bool have_isel;
4b06c216 100bool have_altivec;
47c906ae 101bool have_vsx;
49d9870a 102
4cbea598 103#ifndef CONFIG_SOFTMMU
f6548c0a 104#define TCG_GUEST_BASE_REG 30
f6548c0a 105#endif
106
8d8fdbae 107#ifdef CONFIG_DEBUG_TCG
42281ec6
RH		 108static const char tcg_target_reg_names[TCG_TARGET_NB_REGS][4] = {
109 "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7",
110 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
111 "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
112 "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
113 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
114 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
115 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
116 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
810260a8 117};
d4a9eb1f 118#endif
810260a8 119
120static const int tcg_target_reg_alloc_order[] = {
5e1702b0 121 TCG_REG_R14, /* call saved registers */
810260a8 122 TCG_REG_R15,
123 TCG_REG_R16,
124 TCG_REG_R17,
125 TCG_REG_R18,
126 TCG_REG_R19,
127 TCG_REG_R20,
128 TCG_REG_R21,
129 TCG_REG_R22,
130 TCG_REG_R23,
5e1702b0
RH		 131 TCG_REG_R24,
132 TCG_REG_R25,
133 TCG_REG_R26,
134 TCG_REG_R27,
810260a8 135 TCG_REG_R28,
136 TCG_REG_R29,
137 TCG_REG_R30,
138 TCG_REG_R31,
5e1702b0
RH		 139 TCG_REG_R12, /* call clobbered, non-arguments */
140 TCG_REG_R11,
dfca1778
RH		 141 TCG_REG_R2,
142 TCG_REG_R13,
5e1702b0 143 TCG_REG_R10, /* call clobbered, arguments */
810260a8 144 TCG_REG_R9,
5e1702b0
RH		 145 TCG_REG_R8,
146 TCG_REG_R7,
147 TCG_REG_R6,
148 TCG_REG_R5,
149 TCG_REG_R4,
150 TCG_REG_R3,
42281ec6
RH		 151
152 /* V0 and V1 reserved as temporaries; V20 - V31 are call-saved */
153 TCG_REG_V2, /* call clobbered, vectors */
154 TCG_REG_V3,
155 TCG_REG_V4,
156 TCG_REG_V5,
157 TCG_REG_V6,
158 TCG_REG_V7,
159 TCG_REG_V8,
160 TCG_REG_V9,
161 TCG_REG_V10,
162 TCG_REG_V11,
163 TCG_REG_V12,
164 TCG_REG_V13,
165 TCG_REG_V14,
166 TCG_REG_V15,
167 TCG_REG_V16,
168 TCG_REG_V17,
169 TCG_REG_V18,
170 TCG_REG_V19,
810260a8 171};
172
173static const int tcg_target_call_iarg_regs[] = {
174 TCG_REG_R3,
175 TCG_REG_R4,
176 TCG_REG_R5,
177 TCG_REG_R6,
178 TCG_REG_R7,
179 TCG_REG_R8,
180 TCG_REG_R9,
181 TCG_REG_R10
182};
183
be9c4183 184static const int tcg_target_call_oarg_regs[] = {
dfca1778
RH		 185 TCG_REG_R3,
186 TCG_REG_R4
810260a8 187};
188
189static const int tcg_target_callee_save_regs[] = {
2fa169ba 190#ifdef _CALL_DARWIN
5d7ff5bb
AF		 191 TCG_REG_R11,
192#endif
810260a8 193 TCG_REG_R14,
194 TCG_REG_R15,
195 TCG_REG_R16,
196 TCG_REG_R17,
197 TCG_REG_R18,
198 TCG_REG_R19,
199 TCG_REG_R20,
200 TCG_REG_R21,
201 TCG_REG_R22,
202 TCG_REG_R23,
095271d4 203 TCG_REG_R24,
204 TCG_REG_R25,
205 TCG_REG_R26,
cea5f9a2 206 TCG_REG_R27, /* currently used for the global env */
810260a8 207 TCG_REG_R28,
208 TCG_REG_R29,
209 TCG_REG_R30,
210 TCG_REG_R31
211};
212
b0940da0
RH		 213static inline bool in_range_b(tcg_target_long target)
214{
215 return target == sextract64(target, 0, 26);
216}
217
d54401df
RH		 218static uint32_t reloc_pc24_val(const tcg_insn_unit *pc,
219 const tcg_insn_unit *target)
810260a8 220{
e083c4a2 221 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
eabb7b91 222 tcg_debug_assert(in_range_b(disp));
810260a8 223 return disp & 0x3fffffc;
224}
225
d54401df 226static bool reloc_pc24(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
810260a8 227{
d54401df
RH		 228 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
229 ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx);
230
d5132903 231 if (in_range_b(disp)) {
d54401df 232 *src_rw = (*src_rw & ~0x3fffffc) | (disp & 0x3fffffc);
d5132903
RH		 233 return true;
234 }
235 return false;
810260a8 236}
237
d54401df
RH		 238static uint16_t reloc_pc14_val(const tcg_insn_unit *pc,
239 const tcg_insn_unit *target)
810260a8 240{
e083c4a2 241 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
eabb7b91 242 tcg_debug_assert(disp == (int16_t) disp);
810260a8 243 return disp & 0xfffc;
244}
245
d54401df 246static bool reloc_pc14(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
810260a8 247{
d54401df
RH		 248 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
249 ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx);
250
d5132903 251 if (disp == (int16_t) disp) {
d54401df 252 *src_rw = (*src_rw & ~0xfffc) | (disp & 0xfffc);
d5132903
RH		 253 return true;
254 }
255 return false;
810260a8 256}
257
810260a8 258/* test if a constant matches the constraint */
a4fbbd77 259static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
810260a8 260{
3d582c61
RH		 261 if (ct & TCG_CT_CONST) {
262 return 1;
1194dcba
RH		 263 }
264
265 /* The only 32-bit constraint we use aside from
266 TCG_CT_CONST is TCG_CT_CONST_S16. */
267 if (type == TCG_TYPE_I32) {
268 val = (int32_t)val;
269 }
270
271 if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) {
3d582c61
RH		 272 return 1;
273 } else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val) {
810260a8 274 return 1;
3d582c61 275 } else if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
fe6f943f 276 return 1;
3d582c61
RH		 277 } else if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
278 return 1;
279 } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
280 return 1;
6c858762
RH		 281 } else if ((ct & TCG_CT_CONST_MONE) && val == -1) {
282 return 1;
d0b07481
RH		 283 } else if ((ct & TCG_CT_CONST_WSZ)
284 && val == (type == TCG_TYPE_I32 ? 32 : 64)) {
285 return 1;
3d582c61 286 }
810260a8 287 return 0;
288}
289
290#define OPCD(opc) ((opc)<<26)
291#define XO19(opc) (OPCD(19)|((opc)<<1))
8a94cfb0
AB		 292#define MD30(opc) (OPCD(30)|((opc)<<2))
293#define MDS30(opc) (OPCD(30)|((opc)<<1))
810260a8 294#define XO31(opc) (OPCD(31)|((opc)<<1))
295#define XO58(opc) (OPCD(58)|(opc))
296#define XO62(opc) (OPCD(62)|(opc))
1838905e 297#define VX4(opc) (OPCD(4)|(opc))
810260a8 298
299#define B OPCD( 18)
300#define BC OPCD( 16)
301#define LBZ OPCD( 34)
302#define LHZ OPCD( 40)
303#define LHA OPCD( 42)
304#define LWZ OPCD( 32)
644f591a 305#define LWZUX XO31( 55)
810260a8 306#define STB OPCD( 38)
307#define STH OPCD( 44)
308#define STW OPCD( 36)
309
310#define STD XO62( 0)
311#define STDU XO62( 1)
312#define STDX XO31(149)
313
314#define LD XO58( 0)
315#define LDX XO31( 21)
316#define LDU XO58( 1)
644f591a 317#define LDUX XO31( 53)
301f6d90 318#define LWA XO58( 2)
810260a8 319#define LWAX XO31(341)
320
1cd62ae9 321#define ADDIC OPCD( 12)
810260a8 322#define ADDI OPCD( 14)
323#define ADDIS OPCD( 15)
324#define ORI OPCD( 24)
325#define ORIS OPCD( 25)
326#define XORI OPCD( 26)
327#define XORIS OPCD( 27)
328#define ANDI OPCD( 28)
329#define ANDIS OPCD( 29)
330#define MULLI OPCD( 7)
331#define CMPLI OPCD( 10)
332#define CMPI OPCD( 11)
148bdd23 333#define SUBFIC OPCD( 8)
810260a8 334
335#define LWZU OPCD( 33)
336#define STWU OPCD( 37)
337
313d91c7 338#define RLWIMI OPCD( 20)
810260a8 339#define RLWINM OPCD( 21)
313d91c7 340#define RLWNM OPCD( 23)
810260a8 341
8a94cfb0
AB		 342#define RLDICL MD30( 0)
343#define RLDICR MD30( 1)
344#define RLDIMI MD30( 3)
345#define RLDCL MDS30( 8)
810260a8 346
347#define BCLR XO19( 16)
348#define BCCTR XO19(528)
349#define CRAND XO19(257)
350#define CRANDC XO19(129)
351#define CRNAND XO19(225)
352#define CROR XO19(449)
1cd62ae9 353#define CRNOR XO19( 33)
810260a8 354
355#define EXTSB XO31(954)
356#define EXTSH XO31(922)
357#define EXTSW XO31(986)
358#define ADD XO31(266)
359#define ADDE XO31(138)
6c858762
RH		 360#define ADDME XO31(234)
361#define ADDZE XO31(202)
810260a8 362#define ADDC XO31( 10)
363#define AND XO31( 28)
364#define SUBF XO31( 40)
365#define SUBFC XO31( 8)
366#define SUBFE XO31(136)
6c858762
RH		 367#define SUBFME XO31(232)
368#define SUBFZE XO31(200)
810260a8 369#define OR XO31(444)
370#define XOR XO31(316)
371#define MULLW XO31(235)
8fa391a0 372#define MULHW XO31( 75)
810260a8 373#define MULHWU XO31( 11)
374#define DIVW XO31(491)
375#define DIVWU XO31(459)
4d573822
MKF		 376#define MODSW XO31(779)
377#define MODUW XO31(267)
810260a8 378#define CMP XO31( 0)
379#define CMPL XO31( 32)
380#define LHBRX XO31(790)
381#define LWBRX XO31(534)
49d9870a 382#define LDBRX XO31(532)
810260a8 383#define STHBRX XO31(918)
384#define STWBRX XO31(662)
49d9870a 385#define STDBRX XO31(660)
810260a8 386#define MFSPR XO31(339)
387#define MTSPR XO31(467)
388#define SRAWI XO31(824)
389#define NEG XO31(104)
1cd62ae9 390#define MFCR XO31( 19)
6995a4a0 391#define MFOCRF (MFCR | (1u << 20))
157f2662 392#define NOR XO31(124)
1cd62ae9 393#define CNTLZW XO31( 26)
394#define CNTLZD XO31( 58)
d0b07481
RH		 395#define CNTTZW XO31(538)
396#define CNTTZD XO31(570)
33e75fb9
RH		 397#define CNTPOPW XO31(378)
398#define CNTPOPD XO31(506)
ce1010d6
RH		 399#define ANDC XO31( 60)
400#define ORC XO31(412)
401#define EQV XO31(284)
402#define NAND XO31(476)
70fac59a 403#define ISEL XO31( 15)
810260a8 404
405#define MULLD XO31(233)
406#define MULHD XO31( 73)
407#define MULHDU XO31( 9)
408#define DIVD XO31(489)
409#define DIVDU XO31(457)
4d573822
MKF		 410#define MODSD XO31(777)
411#define MODUD XO31(265)
810260a8 412
413#define LBZX XO31( 87)
4f4a67ae 414#define LHZX XO31(279)
810260a8 415#define LHAX XO31(343)
416#define LWZX XO31( 23)
417#define STBX XO31(215)
418#define STHX XO31(407)
419#define STWX XO31(151)
420
7b4af5ee
PK		 421#define EIEIO XO31(854)
422#define HWSYNC XO31(598)
423#define LWSYNC (HWSYNC | (1u << 21))
424
541dd4ce 425#define SPR(a, b) ((((a)<<5)|(b))<<11)
810260a8 426#define LR SPR(8, 0)
427#define CTR SPR(9, 0)
428
429#define SLW XO31( 24)
430#define SRW XO31(536)
431#define SRAW XO31(792)
432
433#define SLD XO31( 27)
434#define SRD XO31(539)
435#define SRAD XO31(794)
fe6f943f 436#define SRADI XO31(413<<1)
810260a8 437
780b573f
RH		 438#define BRH XO31(219)
439#define BRW XO31(155)
440#define BRD XO31(187)
441
810260a8 442#define TW XO31( 4)
541dd4ce 443#define TRAP (TW | TO(31))
810260a8 444
a84ac4cb
RH		 445#define NOP ORI /* ori 0,0,0 */
446
6ef14d7e
RH		 447#define LVX XO31(103)
448#define LVEBX XO31(7)
449#define LVEHX XO31(39)
450#define LVEWX XO31(71)
47c906ae
RH		 451#define LXSDX (XO31(588) | 1) /* v2.06, force tx=1 */
452#define LXVDSX (XO31(332) | 1) /* v2.06, force tx=1 */
b2dda640 453#define LXSIWZX (XO31(12) | 1) /* v2.07, force tx=1 */
6e11cde1
RH		 454#define LXV (OPCD(61) | 8 | 1) /* v3.00, force tx=1 */
455#define LXSD (OPCD(57) | 2) /* v3.00 */
456#define LXVWSX (XO31(364) | 1) /* v3.00, force tx=1 */
6ef14d7e
RH		 457
458#define STVX XO31(231)
459#define STVEWX XO31(199)
47c906ae 460#define STXSDX (XO31(716) | 1) /* v2.06, force sx=1 */
b2dda640 461#define STXSIWX (XO31(140) | 1) /* v2.07, force sx=1 */
6e11cde1
RH		 462#define STXV (OPCD(61) | 8 | 5) /* v3.00, force sx=1 */
463#define STXSD (OPCD(61) | 2) /* v3.00 */
6ef14d7e 464
e9d1a53a
RH		 465#define VADDSBS VX4(768)
466#define VADDUBS VX4(512)
d6750811 467#define VADDUBM VX4(0)
e9d1a53a
RH		 468#define VADDSHS VX4(832)
469#define VADDUHS VX4(576)
d6750811 470#define VADDUHM VX4(64)
e9d1a53a
RH		 471#define VADDSWS VX4(896)
472#define VADDUWS VX4(640)
d6750811 473#define VADDUWM VX4(128)
64ff1c6d 474#define VADDUDM VX4(192) /* v2.07 */
d6750811 475
e9d1a53a
RH		 476#define VSUBSBS VX4(1792)
477#define VSUBUBS VX4(1536)
d6750811 478#define VSUBUBM VX4(1024)
e9d1a53a
RH		 479#define VSUBSHS VX4(1856)
480#define VSUBUHS VX4(1600)
d6750811 481#define VSUBUHM VX4(1088)
e9d1a53a
RH		 482#define VSUBSWS VX4(1920)
483#define VSUBUWS VX4(1664)
d6750811 484#define VSUBUWM VX4(1152)
64ff1c6d 485#define VSUBUDM VX4(1216) /* v2.07 */
d6750811 486
d7cd6a2f
RH		 487#define VNEGW (VX4(1538) | (6 << 16)) /* v3.00 */
488#define VNEGD (VX4(1538) | (7 << 16)) /* v3.00 */
489
e2382972
RH		 490#define VMAXSB VX4(258)
491#define VMAXSH VX4(322)
492#define VMAXSW VX4(386)
64ff1c6d 493#define VMAXSD VX4(450) /* v2.07 */
e2382972
RH		 494#define VMAXUB VX4(2)
495#define VMAXUH VX4(66)
496#define VMAXUW VX4(130)
64ff1c6d 497#define VMAXUD VX4(194) /* v2.07 */
e2382972
RH		 498#define VMINSB VX4(770)
499#define VMINSH VX4(834)
500#define VMINSW VX4(898)
64ff1c6d 501#define VMINSD VX4(962) /* v2.07 */
e2382972
RH		 502#define VMINUB VX4(514)
503#define VMINUH VX4(578)
504#define VMINUW VX4(642)
64ff1c6d 505#define VMINUD VX4(706) /* v2.07 */
e2382972 506
6ef14d7e
RH		 507#define VCMPEQUB VX4(6)
508#define VCMPEQUH VX4(70)
509#define VCMPEQUW VX4(134)
64ff1c6d 510#define VCMPEQUD VX4(199) /* v2.07 */
6ef14d7e
RH		 511#define VCMPGTSB VX4(774)
512#define VCMPGTSH VX4(838)
513#define VCMPGTSW VX4(902)
64ff1c6d 514#define VCMPGTSD VX4(967) /* v2.07 */
6ef14d7e
RH		 515#define VCMPGTUB VX4(518)
516#define VCMPGTUH VX4(582)
517#define VCMPGTUW VX4(646)
64ff1c6d 518#define VCMPGTUD VX4(711) /* v2.07 */
d7cd6a2f
RH		 519#define VCMPNEB VX4(7) /* v3.00 */
520#define VCMPNEH VX4(71) /* v3.00 */
521#define VCMPNEW VX4(135) /* v3.00 */
6ef14d7e 522
dabae097
RH		 523#define VSLB VX4(260)
524#define VSLH VX4(324)
525#define VSLW VX4(388)
64ff1c6d 526#define VSLD VX4(1476) /* v2.07 */
dabae097
RH		 527#define VSRB VX4(516)
528#define VSRH VX4(580)
529#define VSRW VX4(644)
64ff1c6d 530#define VSRD VX4(1732) /* v2.07 */
dabae097
RH		 531#define VSRAB VX4(772)
532#define VSRAH VX4(836)
533#define VSRAW VX4(900)
64ff1c6d 534#define VSRAD VX4(964) /* v2.07 */
d9897efa
RH		 535#define VRLB VX4(4)
536#define VRLH VX4(68)
537#define VRLW VX4(132)
64ff1c6d 538#define VRLD VX4(196) /* v2.07 */
d9897efa
RH		 539
540#define VMULEUB VX4(520)
541#define VMULEUH VX4(584)
64ff1c6d 542#define VMULEUW VX4(648) /* v2.07 */
d9897efa
RH		 543#define VMULOUB VX4(8)
544#define VMULOUH VX4(72)
64ff1c6d
RH		 545#define VMULOUW VX4(136) /* v2.07 */
546#define VMULUWM VX4(137) /* v2.07 */
73ebe95e 547#define VMULLD VX4(457) /* v3.10 */
d9897efa
RH		 548#define VMSUMUHM VX4(38)
549
550#define VMRGHB VX4(12)
551#define VMRGHH VX4(76)
552#define VMRGHW VX4(140)
553#define VMRGLB VX4(268)
554#define VMRGLH VX4(332)
555#define VMRGLW VX4(396)
556
557#define VPKUHUM VX4(14)
558#define VPKUWUM VX4(78)
dabae097 559
6ef14d7e
RH		 560#define VAND VX4(1028)
561#define VANDC VX4(1092)
562#define VNOR VX4(1284)
563#define VOR VX4(1156)
564#define VXOR VX4(1220)
64ff1c6d
RH		 565#define VEQV VX4(1668) /* v2.07 */
566#define VNAND VX4(1412) /* v2.07 */
567#define VORC VX4(1348) /* v2.07 */
6ef14d7e
RH		 568
569#define VSPLTB VX4(524)
570#define VSPLTH VX4(588)
571#define VSPLTW VX4(652)
572#define VSPLTISB VX4(780)
573#define VSPLTISH VX4(844)
574#define VSPLTISW VX4(908)
575
576#define VSLDOI VX4(44)
577
47c906ae
RH		 578#define XXPERMDI (OPCD(60) | (10 << 3) | 7) /* v2.06, force ax=bx=tx=1 */
579#define XXSEL (OPCD(60) | (3 << 4) | 0xf) /* v2.06, force ax=bx=cx=tx=1 */
b7ce3cff 580#define XXSPLTIB (OPCD(60) | (360 << 1) | 1) /* v3.00, force tx=1 */
47c906ae 581
7097312d
RH		 582#define MFVSRD (XO31(51) | 1) /* v2.07, force sx=1 */
583#define MFVSRWZ (XO31(115) | 1) /* v2.07, force sx=1 */
584#define MTVSRD (XO31(179) | 1) /* v2.07, force tx=1 */
585#define MTVSRWZ (XO31(243) | 1) /* v2.07, force tx=1 */
b7ce3cff
RH		 586#define MTVSRDD (XO31(435) | 1) /* v3.00, force tx=1 */
587#define MTVSRWS (XO31(403) | 1) /* v3.00, force tx=1 */
7097312d 588
810260a8 589#define RT(r) ((r)<<21)
590#define RS(r) ((r)<<21)
591#define RA(r) ((r)<<16)
592#define RB(r) ((r)<<11)
593#define TO(t) ((t)<<21)
594#define SH(s) ((s)<<11)
595#define MB(b) ((b)<<6)
596#define ME(e) ((e)<<1)
597#define BO(o) ((o)<<21)
598#define MB64(b) ((b)<<5)
6995a4a0 599#define FXM(b) (1 << (19 - (b)))
810260a8 600
b82f769c
RH		 601#define VRT(r) (((r) & 31) << 21)
602#define VRA(r) (((r) & 31) << 16)
603#define VRB(r) (((r) & 31) << 11)
604#define VRC(r) (((r) & 31) << 6)
605
810260a8 606#define LK 1
607
2fd8eddc
RH		 608#define TAB(t, a, b) (RT(t) | RA(a) | RB(b))
609#define SAB(s, a, b) (RS(s) | RA(a) | RB(b))
610#define TAI(s, a, i) (RT(s) | RA(a) | ((i) & 0xffff))
611#define SAI(s, a, i) (RS(s) | RA(a) | ((i) & 0xffff))
810260a8 612
613#define BF(n) ((n)<<23)
614#define BI(n, c) (((c)+((n)*4))<<16)
615#define BT(n, c) (((c)+((n)*4))<<21)
616#define BA(n, c) (((c)+((n)*4))<<16)
617#define BB(n, c) (((c)+((n)*4))<<11)
70fac59a 618#define BC_(n, c) (((c)+((n)*4))<<6)
810260a8 619
541dd4ce
RH		 620#define BO_COND_TRUE BO(12)
621#define BO_COND_FALSE BO( 4)
622#define BO_ALWAYS BO(20)
810260a8 623
624enum {
625 CR_LT,
626 CR_GT,
627 CR_EQ,
628 CR_SO
629};
630
0aed257f 631static const uint32_t tcg_to_bc[] = {
541dd4ce
RH		 632 [TCG_COND_EQ] = BC | BI(7, CR_EQ) | BO_COND_TRUE,
633 [TCG_COND_NE] = BC | BI(7, CR_EQ) | BO_COND_FALSE,
634 [TCG_COND_LT] = BC | BI(7, CR_LT) | BO_COND_TRUE,
635 [TCG_COND_GE] = BC | BI(7, CR_LT) | BO_COND_FALSE,
636 [TCG_COND_LE] = BC | BI(7, CR_GT) | BO_COND_FALSE,
637 [TCG_COND_GT] = BC | BI(7, CR_GT) | BO_COND_TRUE,
638 [TCG_COND_LTU] = BC | BI(7, CR_LT) | BO_COND_TRUE,
639 [TCG_COND_GEU] = BC | BI(7, CR_LT) | BO_COND_FALSE,
640 [TCG_COND_LEU] = BC | BI(7, CR_GT) | BO_COND_FALSE,
641 [TCG_COND_GTU] = BC | BI(7, CR_GT) | BO_COND_TRUE,
810260a8 642};
643
70fac59a
RH		 644/* The low bit here is set if the RA and RB fields must be inverted. */
645static const uint32_t tcg_to_isel[] = {
646 [TCG_COND_EQ] = ISEL | BC_(7, CR_EQ),
647 [TCG_COND_NE] = ISEL | BC_(7, CR_EQ) | 1,
648 [TCG_COND_LT] = ISEL | BC_(7, CR_LT),
649 [TCG_COND_GE] = ISEL | BC_(7, CR_LT) | 1,
650 [TCG_COND_LE] = ISEL | BC_(7, CR_GT) | 1,
651 [TCG_COND_GT] = ISEL | BC_(7, CR_GT),
652 [TCG_COND_LTU] = ISEL | BC_(7, CR_LT),
653 [TCG_COND_GEU] = ISEL | BC_(7, CR_LT) | 1,
654 [TCG_COND_LEU] = ISEL | BC_(7, CR_GT) | 1,
655 [TCG_COND_GTU] = ISEL | BC_(7, CR_GT),
656};
657
6ac17786 658static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
030ffe39
RH		 659 intptr_t value, intptr_t addend)
660{
d54401df 661 const tcg_insn_unit *target;
6ef14d7e
RH		 662 int16_t lo;
663 int32_t hi;
030ffe39
RH		 664
665 value += addend;
d54401df 666 target = (const tcg_insn_unit *)value;
030ffe39
RH		 667
668 switch (type) {
669 case R_PPC_REL14:
d5132903 670 return reloc_pc14(code_ptr, target);
030ffe39 671 case R_PPC_REL24:
d5132903 672 return reloc_pc24(code_ptr, target);
030ffe39 673 case R_PPC_ADDR16:
a7cdaf71
RH		 674 /*
675 * We are (slightly) abusing this relocation type. In particular,
676 * assert that the low 2 bits are zero, and do not modify them.
677 * That way we can use this with LD et al that have opcode bits
678 * in the low 2 bits of the insn.
679 */
680 if ((value & 3) || value != (int16_t)value) {
681 return false;
030ffe39 682 }
a7cdaf71 683 *code_ptr = (*code_ptr & ~0xfffc) | (value & 0xfffc);
030ffe39 684 break;
6ef14d7e
RH		 685 case R_PPC_ADDR32:
686 /*
687 * We are abusing this relocation type. Again, this points to
688 * a pair of insns, lis + load. This is an absolute address
689 * relocation for PPC32 so the lis cannot be removed.
690 */
691 lo = value;
692 hi = value - lo;
693 if (hi + lo != value) {
694 return false;
695 }
696 code_ptr[0] = deposit32(code_ptr[0], 0, 16, hi >> 16);
697 code_ptr[1] = deposit32(code_ptr[1], 0, 16, lo);
698 break;
030ffe39
RH		 699 default:
700 g_assert_not_reached();
701 }
6ac17786 702 return true;
030ffe39
RH		 703}
704
a84ac4cb
RH		 705static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
706 TCGReg base, tcg_target_long offset);
707
78113e83 708static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
810260a8 709{
6ef14d7e
RH		 710 if (ret == arg) {
711 return true;
712 }
713 switch (type) {
714 case TCG_TYPE_I64:
715 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
716 /* fallthru */
717 case TCG_TYPE_I32:
7097312d
RH		 718 if (ret < TCG_REG_V0) {
719 if (arg < TCG_REG_V0) {
720 tcg_out32(s, OR | SAB(arg, ret, arg));
721 break;
722 } else if (have_isa_2_07) {
723 tcg_out32(s, (type == TCG_TYPE_I32 ? MFVSRWZ : MFVSRD)
724 | VRT(arg) | RA(ret));
725 break;
726 } else {
727 /* Altivec does not support vector->integer moves. */
728 return false;
729 }
730 } else if (arg < TCG_REG_V0) {
731 if (have_isa_2_07) {
732 tcg_out32(s, (type == TCG_TYPE_I32 ? MTVSRWZ : MTVSRD)
733 | VRT(ret) | RA(arg));
734 break;
735 } else {
736 /* Altivec does not support integer->vector moves. */
737 return false;
738 }
6ef14d7e
RH		 739 }
740 /* fallthru */
741 case TCG_TYPE_V64:
742 case TCG_TYPE_V128:
743 tcg_debug_assert(ret >= TCG_REG_V0 && arg >= TCG_REG_V0);
744 tcg_out32(s, VOR | VRT(ret) | VRA(arg) | VRB(arg));
745 break;
746 default:
747 g_assert_not_reached();
f8b84129 748 }
78113e83 749 return true;
810260a8 750}
751
aceac8d6
RH		 752static inline void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs,
753 int sh, int mb)
810260a8 754{
eabb7b91 755 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
541dd4ce
RH		 756 sh = SH(sh & 0x1f) | (((sh >> 5) & 1) << 1);
757 mb = MB64((mb >> 5) | ((mb << 1) & 0x3f));
758 tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb);
810260a8 759}
760
9e555b73
RH		 761static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs,
762 int sh, int mb, int me)
763{
764 tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me));
765}
766
f4bf14f4
RH		 767static inline void tcg_out_ext8s(TCGContext *s, TCGReg dst, TCGReg src)
768{
769 tcg_out32(s, EXTSB | RA(dst) | RS(src));
770}
771
772static inline void tcg_out_ext16s(TCGContext *s, TCGReg dst, TCGReg src)
773{
774 tcg_out32(s, EXTSH | RA(dst) | RS(src));
775}
776
780b573f
RH		 777static inline void tcg_out_ext16u(TCGContext *s, TCGReg dst, TCGReg src)
778{
779 tcg_out32(s, ANDI | SAI(src, dst, 0xffff));
780}
781
f4bf14f4
RH		 782static inline void tcg_out_ext32s(TCGContext *s, TCGReg dst, TCGReg src)
783{
784 tcg_out32(s, EXTSW | RA(dst) | RS(src));
785}
786
6e5e0602
RH		 787static inline void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
788{
789 tcg_out_rld(s, RLDICL, dst, src, 0, 32);
790}
791
a757e1ee
RH		 792static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
793{
794 tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
795}
796
0a9564b9
RH		 797static inline void tcg_out_shli64(TCGContext *s, TCGReg dst, TCGReg src, int c)
798{
799 tcg_out_rld(s, RLDICR, dst, src, c, 63 - c);
800}
801
05dd01fa
RH		 802static inline void tcg_out_sari32(TCGContext *s, TCGReg dst, TCGReg src, int c)
803{
804 /* Limit immediate shift count lest we create an illegal insn. */
805 tcg_out32(s, SRAWI | RA(dst) | RS(src) | SH(c & 31));
806}
807
a757e1ee
RH		 808static inline void tcg_out_shri32(TCGContext *s, TCGReg dst, TCGReg src, int c)
809{
810 tcg_out_rlw(s, RLWINM, dst, src, 32 - c, c, 31);
811}
812
5e916c28
RH		 813static inline void tcg_out_shri64(TCGContext *s, TCGReg dst, TCGReg src, int c)
814{
815 tcg_out_rld(s, RLDICL, dst, src, 64 - c, c);
816}
817
05dd01fa
RH		 818static inline void tcg_out_sari64(TCGContext *s, TCGReg dst, TCGReg src, int c)
819{
820 tcg_out32(s, SRADI | RA(dst) | RS(src) | SH(c & 0x1f) | ((c >> 4) & 2));
821}
822
26ce7005 823static void tcg_out_bswap16(TCGContext *s, TCGReg dst, TCGReg src, int flags)
783d3ecd
RH		 824{
825 TCGReg tmp = dst == src ? TCG_REG_R0 : dst;
826
780b573f
RH		 827 if (have_isa_3_10) {
828 tcg_out32(s, BRH | RA(dst) | RS(src));
829 if (flags & TCG_BSWAP_OS) {
830 tcg_out_ext16s(s, dst, dst);
831 } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
832 tcg_out_ext16u(s, dst, dst);
833 }
834 return;
835 }
836
783d3ecd
RH		 837 /*
838 * In the following,
839 * dep(a, b, m) -> (a & ~m) | (b & m)
840 *
841 * Begin with: src = xxxxabcd
842 */
843 /* tmp = rol32(src, 24) & 0x000000ff = 0000000c */
844 tcg_out_rlw(s, RLWINM, tmp, src, 24, 24, 31);
845 /* tmp = dep(tmp, rol32(src, 8), 0x0000ff00) = 000000dc */
846 tcg_out_rlw(s, RLWIMI, tmp, src, 8, 16, 23);
847
26ce7005
RH		 848 if (flags & TCG_BSWAP_OS) {
849 tcg_out_ext16s(s, dst, tmp);
850 } else {
851 tcg_out_mov(s, TCG_TYPE_REG, dst, tmp);
852 }
783d3ecd
RH		 853}
854
26ce7005 855static void tcg_out_bswap32(TCGContext *s, TCGReg dst, TCGReg src, int flags)
8a611d86
RH		 856{
857 TCGReg tmp = dst == src ? TCG_REG_R0 : dst;
858
780b573f
RH		 859 if (have_isa_3_10) {
860 tcg_out32(s, BRW | RA(dst) | RS(src));
861 if (flags & TCG_BSWAP_OS) {
862 tcg_out_ext32s(s, dst, dst);
863 } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
864 tcg_out_ext32u(s, dst, dst);
865 }
866 return;
867 }
868
8a611d86
RH		 869 /*
870 * Stolen from gcc's builtin_bswap32.
871 * In the following,
872 * dep(a, b, m) -> (a & ~m) | (b & m)
873 *
874 * Begin with: src = xxxxabcd
875 */
876 /* tmp = rol32(src, 8) & 0xffffffff = 0000bcda */
877 tcg_out_rlw(s, RLWINM, tmp, src, 8, 0, 31);
878 /* tmp = dep(tmp, rol32(src, 24), 0xff000000) = 0000dcda */
879 tcg_out_rlw(s, RLWIMI, tmp, src, 24, 0, 7);
880 /* tmp = dep(tmp, rol32(src, 24), 0x0000ff00) = 0000dcba */
881 tcg_out_rlw(s, RLWIMI, tmp, src, 24, 16, 23);
882
26ce7005
RH		 883 if (flags & TCG_BSWAP_OS) {
884 tcg_out_ext32s(s, dst, tmp);
885 } else {
886 tcg_out_mov(s, TCG_TYPE_REG, dst, tmp);
887 }
8a611d86
RH		 888}
889
674ba588
RH		 890static void tcg_out_bswap64(TCGContext *s, TCGReg dst, TCGReg src)
891{
892 TCGReg t0 = dst == src ? TCG_REG_R0 : dst;
893 TCGReg t1 = dst == src ? dst : TCG_REG_R0;
894
780b573f
RH		 895 if (have_isa_3_10) {
896 tcg_out32(s, BRD | RA(dst) | RS(src));
897 return;
898 }
899
674ba588
RH		 900 /*
901 * In the following,
902 * dep(a, b, m) -> (a & ~m) | (b & m)
903 *
904 * Begin with: src = abcdefgh
905 */
906 /* t0 = rol32(src, 8) & 0xffffffff = 0000fghe */
907 tcg_out_rlw(s, RLWINM, t0, src, 8, 0, 31);
908 /* t0 = dep(t0, rol32(src, 24), 0xff000000) = 0000hghe */
909 tcg_out_rlw(s, RLWIMI, t0, src, 24, 0, 7);
910 /* t0 = dep(t0, rol32(src, 24), 0x0000ff00) = 0000hgfe */
911 tcg_out_rlw(s, RLWIMI, t0, src, 24, 16, 23);
912
913 /* t0 = rol64(t0, 32) = hgfe0000 */
914 tcg_out_rld(s, RLDICL, t0, t0, 32, 0);
915 /* t1 = rol64(src, 32) = efghabcd */
916 tcg_out_rld(s, RLDICL, t1, src, 32, 0);
917
918 /* t0 = dep(t0, rol32(t1, 24), 0xffffffff) = hgfebcda */
919 tcg_out_rlw(s, RLWIMI, t0, t1, 8, 0, 31);
920 /* t0 = dep(t0, rol32(t1, 24), 0xff000000) = hgfedcda */
921 tcg_out_rlw(s, RLWIMI, t0, t1, 24, 0, 7);
922 /* t0 = dep(t0, rol32(t1, 24), 0x0000ff00) = hgfedcba */
923 tcg_out_rlw(s, RLWIMI, t0, t1, 24, 16, 23);
924
925 tcg_out_mov(s, TCG_TYPE_REG, dst, t0);
926}
927
77bfc7c0
RH		 928/* Emit a move into ret of arg, if it can be done in one insn. */
929static bool tcg_out_movi_one(TCGContext *s, TCGReg ret, tcg_target_long arg)
930{
931 if (arg == (int16_t)arg) {
932 tcg_out32(s, ADDI | TAI(ret, 0, arg));
933 return true;
934 }
935 if (arg == (int32_t)arg && (arg & 0xffff) == 0) {
936 tcg_out32(s, ADDIS | TAI(ret, 0, arg >> 16));
937 return true;
938 }
939 return false;
940}
941
5964fca8
RH		 942static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
943 tcg_target_long arg, bool in_prologue)
810260a8 944{
5964fca8 945 intptr_t tb_diff;
77bfc7c0
RH		 946 tcg_target_long tmp;
947 int shift;
5964fca8
RH		 948
949 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
950
951 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
952 arg = (int32_t)arg;
953 }
954
955 /* Load 16-bit immediates with one insn. */
77bfc7c0 956 if (tcg_out_movi_one(s, ret, arg)) {
5964fca8
RH		 957 return;
958 }
959
960 /* Load addresses within the TB with one insn. */
e6dc7f81 961 tb_diff = tcg_tbrel_diff(s, (void *)arg);
5964fca8
RH		 962 if (!in_prologue && USE_REG_TB && tb_diff == (int16_t)tb_diff) {
963 tcg_out32(s, ADDI | TAI(ret, TCG_REG_TB, tb_diff));
964 return;
965 }
966
77bfc7c0
RH		 967 /* Load 32-bit immediates with two insns. Note that we've already
968 eliminated bare ADDIS, so we know both insns are required. */
5964fca8 969 if (TCG_TARGET_REG_BITS == 32 || arg == (int32_t)arg) {
2fd8eddc 970 tcg_out32(s, ADDIS | TAI(ret, 0, arg >> 16));
77bfc7c0 971 tcg_out32(s, ORI | SAI(ret, ret, arg));
5964fca8 972 return;
810260a8 973 }
5964fca8 974 if (arg == (uint32_t)arg && !(arg & 0x8000)) {
421233a1
RH		 975 tcg_out32(s, ADDI | TAI(ret, 0, arg));
976 tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
5964fca8
RH		 977 return;
978 }
a84ac4cb 979
77bfc7c0
RH		 980 /* Load masked 16-bit value. */
981 if (arg > 0 && (arg & 0x8000)) {
982 tmp = arg | 0x7fff;
983 if ((tmp & (tmp + 1)) == 0) {
984 int mb = clz64(tmp + 1) + 1;
985 tcg_out32(s, ADDI | TAI(ret, 0, arg));
986 tcg_out_rld(s, RLDICL, ret, ret, 0, mb);
987 return;
988 }
989 }
990
991 /* Load common masks with 2 insns. */
992 shift = ctz64(arg);
993 tmp = arg >> shift;
994 if (tmp == (int16_t)tmp) {
995 tcg_out32(s, ADDI | TAI(ret, 0, tmp));
996 tcg_out_shli64(s, ret, ret, shift);
997 return;
998 }
999 shift = clz64(arg);
1000 if (tcg_out_movi_one(s, ret, arg << shift)) {
1001 tcg_out_shri64(s, ret, ret, shift);
1002 return;
1003 }
1004
5964fca8
RH		 1005 /* Load addresses within 2GB of TB with 2 (or rarely 3) insns. */
1006 if (!in_prologue && USE_REG_TB && tb_diff == (int32_t)tb_diff) {
1007 tcg_out_mem_long(s, ADDI, ADD, ret, TCG_REG_TB, tb_diff);
1008 return;
1009 }
a84ac4cb 1010
53c89efd
RH		 1011 /* Use the constant pool, if possible. */
1012 if (!in_prologue && USE_REG_TB) {
1013 new_pool_label(s, arg, R_PPC_ADDR16, s->code_ptr,
e6dc7f81 1014 tcg_tbrel_diff(s, NULL));
a7cdaf71 1015 tcg_out32(s, LD | TAI(ret, TCG_REG_TB, 0));
53c89efd
RH		 1016 return;
1017 }
1018
77bfc7c0
RH		 1019 tmp = arg >> 31 >> 1;
1020 tcg_out_movi(s, TCG_TYPE_I32, ret, tmp);
1021 if (tmp) {
5964fca8 1022 tcg_out_shli64(s, ret, ret, 32);
810260a8 1023 }
5964fca8
RH		 1024 if (arg & 0xffff0000) {
1025 tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
1026 }
1027 if (arg & 0xffff) {
1028 tcg_out32(s, ORI | SAI(ret, ret, arg));
1029 }
1030}
1031
4e186175
RH		 1032static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
1033 TCGReg ret, int64_t val)
5964fca8 1034{
6ef14d7e
RH		 1035 uint32_t load_insn;
1036 int rel, low;
1037 intptr_t add;
1038
4e186175
RH		 1039 switch (vece) {
1040 case MO_8:
1041 low = (int8_t)val;
1042 if (low >= -16 && low < 16) {
6ef14d7e
RH		 1043 tcg_out32(s, VSPLTISB | VRT(ret) | ((val & 31) << 16));
1044 return;
1045 }
4e186175
RH		 1046 if (have_isa_3_00) {
1047 tcg_out32(s, XXSPLTIB | VRT(ret) | ((val & 0xff) << 11));
1048 return;
1049 }
1050 break;
1051
1052 case MO_16:
1053 low = (int16_t)val;
1054 if (low >= -16 && low < 16) {
6ef14d7e
RH		 1055 tcg_out32(s, VSPLTISH | VRT(ret) | ((val & 31) << 16));
1056 return;
1057 }
4e186175
RH		 1058 break;
1059
1060 case MO_32:
1061 low = (int32_t)val;
1062 if (low >= -16 && low < 16) {
6ef14d7e
RH		 1063 tcg_out32(s, VSPLTISW | VRT(ret) | ((val & 31) << 16));
1064 return;
1065 }
4e186175 1066 break;
b7ce3cff 1067 }
6ef14d7e
RH		 1068
1069 /*
1070 * Otherwise we must load the value from the constant pool.
1071 */
1072 if (USE_REG_TB) {
1073 rel = R_PPC_ADDR16;
e6dc7f81 1074 add = tcg_tbrel_diff(s, NULL);
6ef14d7e
RH		 1075 } else {
1076 rel = R_PPC_ADDR32;
1077 add = 0;
1078 }
1079
47c906ae
RH		 1080 if (have_vsx) {
1081 load_insn = type == TCG_TYPE_V64 ? LXSDX : LXVDSX;
1082 load_insn |= VRT(ret) | RB(TCG_REG_TMP1);
1083 if (TCG_TARGET_REG_BITS == 64) {
1084 new_pool_label(s, val, rel, s->code_ptr, add);
1085 } else {
4e186175 1086 new_pool_l2(s, rel, s->code_ptr, add, val >> 32, val);
47c906ae 1087 }
6ef14d7e 1088 } else {
47c906ae
RH		 1089 load_insn = LVX | VRT(ret) | RB(TCG_REG_TMP1);
1090 if (TCG_TARGET_REG_BITS == 64) {
1091 new_pool_l2(s, rel, s->code_ptr, add, val, val);
1092 } else {
4e186175
RH		 1093 new_pool_l4(s, rel, s->code_ptr, add,
1094 val >> 32, val, val >> 32, val);
47c906ae 1095 }
6ef14d7e
RH		 1096 }
1097
1098 if (USE_REG_TB) {
1099 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, 0, 0));
1100 load_insn |= RA(TCG_REG_TB);
1101 } else {
1102 tcg_out32(s, ADDIS | TAI(TCG_REG_TMP1, 0, 0));
1103 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, TCG_REG_TMP1, 0));
1104 }
1105 tcg_out32(s, load_insn);
4b06c216
RH		 1106}
1107
1108static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret,
1109 tcg_target_long arg)
1110{
1111 switch (type) {
1112 case TCG_TYPE_I32:
1113 case TCG_TYPE_I64:
1114 tcg_debug_assert(ret < TCG_REG_V0);
1115 tcg_out_movi_int(s, type, ret, arg, false);
1116 break;
1117
4b06c216
RH		 1118 default:
1119 g_assert_not_reached();
1120 }
810260a8 1121}
1122
637af30c 1123static bool mask_operand(uint32_t c, int *mb, int *me)
a9249dff
RH		 1124{
1125 uint32_t lsb, test;
1126
1127 /* Accept a bit pattern like:
1128 0....01....1
1129 1....10....0
1130 0..01..10..0
1131 Keep track of the transitions. */
1132 if (c == 0 || c == -1) {
1133 return false;
1134 }
1135 test = c;
1136 lsb = test & -test;
1137 test += lsb;
1138 if (test & (test - 1)) {
1139 return false;
1140 }
1141
1142 *me = clz32(lsb);
1143 *mb = test ? clz32(test & -test) + 1 : 0;
1144 return true;
1145}
1146
637af30c
RH		 1147static bool mask64_operand(uint64_t c, int *mb, int *me)
1148{
1149 uint64_t lsb;
1150
1151 if (c == 0) {
1152 return false;
1153 }
1154
1155 lsb = c & -c;
1156 /* Accept 1..10..0. */
1157 if (c == -lsb) {
1158 *mb = 0;
1159 *me = clz64(lsb);
1160 return true;
1161 }
1162 /* Accept 0..01..1. */
1163 if (lsb == 1 && (c & (c + 1)) == 0) {
1164 *mb = clz64(c + 1) + 1;
1165 *me = 63;
1166 return true;
1167 }
1168 return false;
1169}
1170
a9249dff
RH		 1171static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
1172{
1173 int mb, me;
1174
1e1df962
RH		 1175 if (mask_operand(c, &mb, &me)) {
1176 tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
1177 } else if ((c & 0xffff) == c) {
a9249dff
RH		 1178 tcg_out32(s, ANDI | SAI(src, dst, c));
1179 return;
1180 } else if ((c & 0xffff0000) == c) {
1181 tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
1182 return;
a9249dff 1183 } else {
8327a470
RH		 1184 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c);
1185 tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
a9249dff
RH		 1186 }
1187}
1188
637af30c
RH		 1189static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c)
1190{
1191 int mb, me;
1192
eabb7b91 1193 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
1e1df962 1194 if (mask64_operand(c, &mb, &me)) {
637af30c
RH		 1195 if (mb == 0) {
1196 tcg_out_rld(s, RLDICR, dst, src, 0, me);
1197 } else {
1198 tcg_out_rld(s, RLDICL, dst, src, 0, mb);
1199 }
1e1df962
RH		 1200 } else if ((c & 0xffff) == c) {
1201 tcg_out32(s, ANDI | SAI(src, dst, c));
1202 return;
1203 } else if ((c & 0xffff0000) == c) {
1204 tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
1205 return;
637af30c 1206 } else {
8327a470
RH		 1207 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c);
1208 tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
637af30c
RH		 1209 }
1210}
1211
dce74c57
RH		 1212static void tcg_out_zori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c,
1213 int op_lo, int op_hi)
1214{
1215 if (c >> 16) {
1216 tcg_out32(s, op_hi | SAI(src, dst, c >> 16));
1217 src = dst;
1218 }
1219 if (c & 0xffff) {
1220 tcg_out32(s, op_lo | SAI(src, dst, c));
1221 src = dst;
1222 }
1223}
1224
1225static void tcg_out_ori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
1226{
1227 tcg_out_zori32(s, dst, src, c, ORI, ORIS);
1228}
1229
1230static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
1231{
1232 tcg_out_zori32(s, dst, src, c, XORI, XORIS);
1233}
1234
2be7d76b 1235static void tcg_out_b(TCGContext *s, int mask, const tcg_insn_unit *target)
5d7ff5bb 1236{
e083c4a2 1237 ptrdiff_t disp = tcg_pcrel_diff(s, target);
b0940da0 1238 if (in_range_b(disp)) {
541dd4ce
RH		 1239 tcg_out32(s, B | (disp & 0x3fffffc) | mask);
1240 } else {
de3d636d 1241 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, (uintptr_t)target);
8327a470 1242 tcg_out32(s, MTSPR | RS(TCG_REG_R0) | CTR);
541dd4ce 1243 tcg_out32(s, BCCTR | BO_ALWAYS | mask);
5d7ff5bb
AF		 1244 }
1245}
1246
b18d5d2b
RH		 1247static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
1248 TCGReg base, tcg_target_long offset)
810260a8 1249{
b18d5d2b 1250 tcg_target_long orig = offset, l0, l1, extra = 0, align = 0;
6e11cde1 1251 bool is_int_store = false;
dfca1778 1252 TCGReg rs = TCG_REG_TMP1;
b18d5d2b 1253
b18d5d2b
RH		 1254 switch (opi) {
1255 case LD: case LWA:
1256 align = 3;
1257 /* FALLTHRU */
1258 default:
6ef14d7e 1259 if (rt > TCG_REG_R0 && rt < TCG_REG_V0) {
b18d5d2b 1260 rs = rt;
de7761a3 1261 break;
b18d5d2b
RH		 1262 }
1263 break;
6e11cde1
RH		 1264 case LXSD:
1265 case STXSD:
1266 align = 3;
1267 break;
1268 case LXV:
1269 case STXV:
1270 align = 15;
1271 break;
b18d5d2b
RH		 1272 case STD:
1273 align = 3;
de7761a3 1274 /* FALLTHRU */
b18d5d2b 1275 case STB: case STH: case STW:
6e11cde1 1276 is_int_store = true;
b18d5d2b 1277 break;
810260a8 1278 }
810260a8 1279
b18d5d2b 1280 /* For unaligned, or very large offsets, use the indexed form. */
6ef14d7e 1281 if (offset & align || offset != (int32_t)offset || opi == 0) {
d4cba13b
RH		 1282 if (rs == base) {
1283 rs = TCG_REG_R0;
1284 }
6e11cde1 1285 tcg_debug_assert(!is_int_store || rs != rt);
de7761a3 1286 tcg_out_movi(s, TCG_TYPE_PTR, rs, orig);
6ef14d7e 1287 tcg_out32(s, opx | TAB(rt & 31, base, rs));
b18d5d2b
RH		 1288 return;
1289 }
1290
1291 l0 = (int16_t)offset;
1292 offset = (offset - l0) >> 16;
1293 l1 = (int16_t)offset;
1294
1295 if (l1 < 0 && orig >= 0) {
1296 extra = 0x4000;
1297 l1 = (int16_t)(offset - 0x4000);
1298 }
1299 if (l1) {
1300 tcg_out32(s, ADDIS | TAI(rs, base, l1));
1301 base = rs;
1302 }
1303 if (extra) {
1304 tcg_out32(s, ADDIS | TAI(rs, base, extra));
1305 base = rs;
1306 }
1307 if (opi != ADDI || base != rt || l0 != 0) {
6ef14d7e 1308 tcg_out32(s, opi | TAI(rt & 31, base, l0));
828808f5 1309 }
1310}
1311
6ef14d7e
RH		 1312static void tcg_out_vsldoi(TCGContext *s, TCGReg ret,
1313 TCGReg va, TCGReg vb, int shb)
d604f1a9 1314{
6ef14d7e
RH		 1315 tcg_out32(s, VSLDOI | VRT(ret) | VRA(va) | VRB(vb) | (shb << 6));
1316}
810260a8 1317
6ef14d7e
RH		 1318static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
1319 TCGReg base, intptr_t offset)
1320{
1321 int shift;
1322
1323 switch (type) {
1324 case TCG_TYPE_I32:
1325 if (ret < TCG_REG_V0) {
1326 tcg_out_mem_long(s, LWZ, LWZX, ret, base, offset);
1327 break;
1328 }
b2dda640
RH		 1329 if (have_isa_2_07 && have_vsx) {
1330 tcg_out_mem_long(s, 0, LXSIWZX, ret, base, offset);
1331 break;
1332 }
6ef14d7e
RH		 1333 tcg_debug_assert((offset & 3) == 0);
1334 tcg_out_mem_long(s, 0, LVEWX, ret, base, offset);
1335 shift = (offset - 4) & 0xc;
1336 if (shift) {
1337 tcg_out_vsldoi(s, ret, ret, ret, shift);
1338 }
1339 break;
1340 case TCG_TYPE_I64:
1341 if (ret < TCG_REG_V0) {
1342 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
1343 tcg_out_mem_long(s, LD, LDX, ret, base, offset);
1344 break;
1345 }
1346 /* fallthru */
1347 case TCG_TYPE_V64:
1348 tcg_debug_assert(ret >= TCG_REG_V0);
47c906ae 1349 if (have_vsx) {
6e11cde1
RH		 1350 tcg_out_mem_long(s, have_isa_3_00 ? LXSD : 0, LXSDX,
1351 ret, base, offset);
47c906ae
RH		 1352 break;
1353 }
6ef14d7e
RH		 1354 tcg_debug_assert((offset & 7) == 0);
1355 tcg_out_mem_long(s, 0, LVX, ret, base, offset & -16);
1356 if (offset & 8) {
1357 tcg_out_vsldoi(s, ret, ret, ret, 8);
1358 }
1359 break;
1360 case TCG_TYPE_V128:
1361 tcg_debug_assert(ret >= TCG_REG_V0);
1362 tcg_debug_assert((offset & 15) == 0);
6e11cde1
RH		 1363 tcg_out_mem_long(s, have_isa_3_00 ? LXV : 0,
1364 LVX, ret, base, offset);
6ef14d7e
RH		 1365 break;
1366 default:
1367 g_assert_not_reached();
d604f1a9 1368 }
d604f1a9 1369}
fedee3e7 1370
6ef14d7e
RH		 1371static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
1372 TCGReg base, intptr_t offset)
810260a8 1373{
6ef14d7e 1374 int shift;
fedee3e7 1375
6ef14d7e
RH		 1376 switch (type) {
1377 case TCG_TYPE_I32:
1378 if (arg < TCG_REG_V0) {
1379 tcg_out_mem_long(s, STW, STWX, arg, base, offset);
1380 break;
1381 }
b2dda640
RH		 1382 if (have_isa_2_07 && have_vsx) {
1383 tcg_out_mem_long(s, 0, STXSIWX, arg, base, offset);
1384 break;
1385 }
1386 assert((offset & 3) == 0);
6ef14d7e
RH		 1387 tcg_debug_assert((offset & 3) == 0);
1388 shift = (offset - 4) & 0xc;
1389 if (shift) {
1390 tcg_out_vsldoi(s, TCG_VEC_TMP1, arg, arg, shift);
1391 arg = TCG_VEC_TMP1;
1392 }
1393 tcg_out_mem_long(s, 0, STVEWX, arg, base, offset);
1394 break;
1395 case TCG_TYPE_I64:
1396 if (arg < TCG_REG_V0) {
1397 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
1398 tcg_out_mem_long(s, STD, STDX, arg, base, offset);
1399 break;
1400 }
1401 /* fallthru */
1402 case TCG_TYPE_V64:
1403 tcg_debug_assert(arg >= TCG_REG_V0);
47c906ae 1404 if (have_vsx) {
6e11cde1
RH		 1405 tcg_out_mem_long(s, have_isa_3_00 ? STXSD : 0,
1406 STXSDX, arg, base, offset);
47c906ae
RH		 1407 break;
1408 }
6ef14d7e
RH		 1409 tcg_debug_assert((offset & 7) == 0);
1410 if (offset & 8) {
1411 tcg_out_vsldoi(s, TCG_VEC_TMP1, arg, arg, 8);
1412 arg = TCG_VEC_TMP1;
1413 }
1414 tcg_out_mem_long(s, 0, STVEWX, arg, base, offset);
1415 tcg_out_mem_long(s, 0, STVEWX, arg, base, offset + 4);
1416 break;
1417 case TCG_TYPE_V128:
1418 tcg_debug_assert(arg >= TCG_REG_V0);
6e11cde1
RH		 1419 tcg_out_mem_long(s, have_isa_3_00 ? STXV : 0,
1420 STVX, arg, base, offset);
6ef14d7e
RH		 1421 break;
1422 default:
1423 g_assert_not_reached();
fedee3e7 1424 }
d604f1a9 1425}
810260a8 1426
59d7c14e
RH		 1427static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1428 TCGReg base, intptr_t ofs)
1429{
1430 return false;
1431}
1432
d604f1a9
RH		 1433static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
1434 int const_arg2, int cr, TCGType type)
1435{
1436 int imm;
1437 uint32_t op;
810260a8 1438
abcf61c4
RH		 1439 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
1440
d604f1a9
RH		 1441 /* Simplify the comparisons below wrt CMPI. */
1442 if (type == TCG_TYPE_I32) {
1443 arg2 = (int32_t)arg2;
4a40e231 1444 }
fedee3e7 1445
d604f1a9
RH		 1446 switch (cond) {
1447 case TCG_COND_EQ:
1448 case TCG_COND_NE:
1449 if (const_arg2) {
1450 if ((int16_t) arg2 == arg2) {
1451 op = CMPI;
1452 imm = 1;
1453 break;
1454 } else if ((uint16_t) arg2 == arg2) {
1455 op = CMPLI;
1456 imm = 1;
1457 break;
1458 }
1459 }
1460 op = CMPL;
1461 imm = 0;
1462 break;
fedee3e7 1463
d604f1a9
RH		 1464 case TCG_COND_LT:
1465 case TCG_COND_GE:
1466 case TCG_COND_LE:
1467 case TCG_COND_GT:
1468 if (const_arg2) {
1469 if ((int16_t) arg2 == arg2) {
1470 op = CMPI;
1471 imm = 1;
1472 break;
1473 }
1474 }
1475 op = CMP;
1476 imm = 0;
1477 break;
fedee3e7 1478
d604f1a9
RH		 1479 case TCG_COND_LTU:
1480 case TCG_COND_GEU:
1481 case TCG_COND_LEU:
1482 case TCG_COND_GTU:
1483 if (const_arg2) {
1484 if ((uint16_t) arg2 == arg2) {
1485 op = CMPLI;
1486 imm = 1;
1487 break;
1488 }
1489 }
1490 op = CMPL;
1491 imm = 0;
1492 break;
fedee3e7 1493
d604f1a9
RH		 1494 default:
1495 tcg_abort();
fedee3e7 1496 }
d604f1a9 1497 op |= BF(cr) | ((type == TCG_TYPE_I64) << 21);
fedee3e7 1498
d604f1a9
RH		 1499 if (imm) {
1500 tcg_out32(s, op | RA(arg1) | (arg2 & 0xffff));
1501 } else {
1502 if (const_arg2) {
1503 tcg_out_movi(s, type, TCG_REG_R0, arg2);
1504 arg2 = TCG_REG_R0;
1505 }
1506 tcg_out32(s, op | RA(arg1) | RB(arg2));
1507 }
810260a8 1508}
1509
d604f1a9
RH		 1510static void tcg_out_setcond_eq0(TCGContext *s, TCGType type,
1511 TCGReg dst, TCGReg src)
7f12d649 1512{
a757e1ee
RH		 1513 if (type == TCG_TYPE_I32) {
1514 tcg_out32(s, CNTLZW | RS(src) | RA(dst));
1515 tcg_out_shri32(s, dst, dst, 5);
1516 } else {
1517 tcg_out32(s, CNTLZD | RS(src) | RA(dst));
1518 tcg_out_shri64(s, dst, dst, 6);
1519 }
7f12d649
RH		 1520}
1521
d604f1a9 1522static void tcg_out_setcond_ne0(TCGContext *s, TCGReg dst, TCGReg src)
7f12d649 1523{
d604f1a9
RH		 1524 /* X != 0 implies X + -1 generates a carry. Extra addition
1525 trickery means: R = X-1 + ~X + C = X-1 + (-X+1) + C = C. */
1526 if (dst != src) {
1527 tcg_out32(s, ADDIC | TAI(dst, src, -1));
1528 tcg_out32(s, SUBFE | TAB(dst, dst, src));
7f12d649 1529 } else {
d604f1a9
RH		 1530 tcg_out32(s, ADDIC | TAI(TCG_REG_R0, src, -1));
1531 tcg_out32(s, SUBFE | TAB(dst, TCG_REG_R0, src));
7f12d649 1532 }
d604f1a9 1533}
7f12d649 1534
d604f1a9
RH		 1535static TCGReg tcg_gen_setcond_xor(TCGContext *s, TCGReg arg1, TCGArg arg2,
1536 bool const_arg2)
1537{
1538 if (const_arg2) {
1539 if ((uint32_t)arg2 == arg2) {
1540 tcg_out_xori32(s, TCG_REG_R0, arg1, arg2);
1541 } else {
1542 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, arg2);
1543 tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, TCG_REG_R0));
1544 }
1545 } else {
1546 tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, arg2));
1547 }
1548 return TCG_REG_R0;
7f12d649
RH		 1549}
1550
d604f1a9
RH		 1551static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
1552 TCGArg arg0, TCGArg arg1, TCGArg arg2,
1553 int const_arg2)
7f12d649 1554{
d604f1a9 1555 int crop, sh;
7f12d649 1556
eabb7b91 1557 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
a757e1ee 1558
d604f1a9
RH		 1559 /* Ignore high bits of a potential constant arg2. */
1560 if (type == TCG_TYPE_I32) {
1561 arg2 = (uint32_t)arg2;
1562 }
7f12d649 1563
d604f1a9
RH		 1564 /* Handle common and trivial cases before handling anything else. */
1565 if (arg2 == 0) {
1566 switch (cond) {
1567 case TCG_COND_EQ:
1568 tcg_out_setcond_eq0(s, type, arg0, arg1);
1569 return;
1570 case TCG_COND_NE:
a757e1ee 1571 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
d604f1a9
RH		 1572 tcg_out_ext32u(s, TCG_REG_R0, arg1);
1573 arg1 = TCG_REG_R0;
1574 }
1575 tcg_out_setcond_ne0(s, arg0, arg1);
1576 return;
1577 case TCG_COND_GE:
1578 tcg_out32(s, NOR | SAB(arg1, arg0, arg1));
1579 arg1 = arg0;
1580 /* FALLTHRU */
1581 case TCG_COND_LT:
1582 /* Extract the sign bit. */
a757e1ee
RH		 1583 if (type == TCG_TYPE_I32) {
1584 tcg_out_shri32(s, arg0, arg1, 31);
1585 } else {
1586 tcg_out_shri64(s, arg0, arg1, 63);
1587 }
d604f1a9
RH		 1588 return;
1589 default:
1590 break;
1591 }
1592 }
7f12d649 1593
d604f1a9
RH		 1594 /* If we have ISEL, we can implement everything with 3 or 4 insns.
1595 All other cases below are also at least 3 insns, so speed up the
1596 code generator by not considering them and always using ISEL. */
63922f46 1597 if (have_isel) {
d604f1a9 1598 int isel, tab;
7f12d649 1599
d604f1a9 1600 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
7f12d649 1601
d604f1a9 1602 isel = tcg_to_isel[cond];
7f12d649 1603
d604f1a9
RH		 1604 tcg_out_movi(s, type, arg0, 1);
1605 if (isel & 1) {
1606 /* arg0 = (bc ? 0 : 1) */
1607 tab = TAB(arg0, 0, arg0);
1608 isel &= ~1;
1609 } else {
1610 /* arg0 = (bc ? 1 : 0) */
1611 tcg_out_movi(s, type, TCG_REG_R0, 0);
1612 tab = TAB(arg0, arg0, TCG_REG_R0);
1613 }
1614 tcg_out32(s, isel | tab);
1615 return;
1616 }
49d9870a 1617
d604f1a9
RH		 1618 switch (cond) {
1619 case TCG_COND_EQ:
1620 arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
1621 tcg_out_setcond_eq0(s, type, arg0, arg1);
1622 return;
810260a8 1623
d604f1a9
RH		 1624 case TCG_COND_NE:
1625 arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
1626 /* Discard the high bits only once, rather than both inputs. */
a757e1ee 1627 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
d604f1a9
RH		 1628 tcg_out_ext32u(s, TCG_REG_R0, arg1);
1629 arg1 = TCG_REG_R0;
1630 }
1631 tcg_out_setcond_ne0(s, arg0, arg1);
1632 return;
810260a8 1633
d604f1a9
RH		 1634 case TCG_COND_GT:
1635 case TCG_COND_GTU:
1636 sh = 30;
1637 crop = 0;
1638 goto crtest;
810260a8 1639
d604f1a9
RH		 1640 case TCG_COND_LT:
1641 case TCG_COND_LTU:
1642 sh = 29;
1643 crop = 0;
1644 goto crtest;
810260a8 1645
d604f1a9
RH		 1646 case TCG_COND_GE:
1647 case TCG_COND_GEU:
1648 sh = 31;
1649 crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_LT) | BB(7, CR_LT);
1650 goto crtest;
810260a8 1651
d604f1a9
RH		 1652 case TCG_COND_LE:
1653 case TCG_COND_LEU:
1654 sh = 31;
1655 crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_GT) | BB(7, CR_GT);
1656 crtest:
1657 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
1658 if (crop) {
1659 tcg_out32(s, crop);
1660 }
1661 tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
1662 tcg_out_rlw(s, RLWINM, arg0, TCG_REG_R0, sh, 31, 31);
1663 break;
1664
1665 default:
1666 tcg_abort();
1667 }
810260a8 1668}
1669
bec16311 1670static void tcg_out_bc(TCGContext *s, int bc, TCGLabel *l)
810260a8 1671{
d604f1a9 1672 if (l->has_value) {
d54401df 1673 bc |= reloc_pc14_val(tcg_splitwx_to_rx(s->code_ptr), l->u.value_ptr);
49d9870a 1674 } else {
bec16311 1675 tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, l, 0);
810260a8 1676 }
f9c7246f 1677 tcg_out32(s, bc);
810260a8 1678}
1679
d604f1a9
RH		 1680static void tcg_out_brcond(TCGContext *s, TCGCond cond,
1681 TCGArg arg1, TCGArg arg2, int const_arg2,
bec16311 1682 TCGLabel *l, TCGType type)
810260a8 1683{
d604f1a9 1684 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
bec16311 1685 tcg_out_bc(s, tcg_to_bc[cond], l);
d604f1a9 1686}
fa94c3be 1687
d604f1a9
RH		 1688static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond,
1689 TCGArg dest, TCGArg c1, TCGArg c2, TCGArg v1,
1690 TCGArg v2, bool const_c2)
1691{
1692 /* If for some reason both inputs are zero, don't produce bad code. */
1693 if (v1 == 0 && v2 == 0) {
1694 tcg_out_movi(s, type, dest, 0);
1695 return;
b9e946c7 1696 }
f6548c0a 1697
d604f1a9 1698 tcg_out_cmp(s, cond, c1, c2, const_c2, 7, type);
a69abbe0 1699
63922f46 1700 if (have_isel) {
d604f1a9 1701 int isel = tcg_to_isel[cond];
810260a8 1702
d604f1a9
RH		 1703 /* Swap the V operands if the operation indicates inversion. */
1704 if (isel & 1) {
1705 int t = v1;
1706 v1 = v2;
1707 v2 = t;
1708 isel &= ~1;
1709 }
1710 /* V1 == 0 is handled by isel; V2 == 0 must be handled by hand. */
1711 if (v2 == 0) {
1712 tcg_out_movi(s, type, TCG_REG_R0, 0);
1713 }
1714 tcg_out32(s, isel | TAB(dest, v1, v2));
1715 } else {
1716 if (dest == v2) {
1717 cond = tcg_invert_cond(cond);
1718 v2 = v1;
1719 } else if (dest != v1) {
1720 if (v1 == 0) {
1721 tcg_out_movi(s, type, dest, 0);
1722 } else {
1723 tcg_out_mov(s, type, dest, v1);
1724 }
1725 }
1726 /* Branch forward over one insn */
1727 tcg_out32(s, tcg_to_bc[cond] | 8);
1728 if (v2 == 0) {
1729 tcg_out_movi(s, type, dest, 0);
1730 } else {
1731 tcg_out_mov(s, type, dest, v2);
1732 }
29b69198 1733 }
810260a8 1734}
1735
d0b07481
RH		 1736static void tcg_out_cntxz(TCGContext *s, TCGType type, uint32_t opc,
1737 TCGArg a0, TCGArg a1, TCGArg a2, bool const_a2)
1738{
1739 if (const_a2 && a2 == (type == TCG_TYPE_I32 ? 32 : 64)) {
1740 tcg_out32(s, opc | RA(a0) | RS(a1));
1741 } else {
1742 tcg_out_cmp(s, TCG_COND_EQ, a1, 0, 1, 7, type);
1743 /* Note that the only other valid constant for a2 is 0. */
63922f46 1744 if (have_isel) {
d0b07481
RH		 1745 tcg_out32(s, opc | RA(TCG_REG_R0) | RS(a1));
1746 tcg_out32(s, tcg_to_isel[TCG_COND_EQ] | TAB(a0, a2, TCG_REG_R0));
1747 } else if (!const_a2 && a0 == a2) {
1748 tcg_out32(s, tcg_to_bc[TCG_COND_EQ] | 8);
1749 tcg_out32(s, opc | RA(a0) | RS(a1));
1750 } else {
1751 tcg_out32(s, opc | RA(a0) | RS(a1));
1752 tcg_out32(s, tcg_to_bc[TCG_COND_NE] | 8);
1753 if (const_a2) {
1754 tcg_out_movi(s, type, a0, 0);
1755 } else {
1756 tcg_out_mov(s, type, a0, a2);
1757 }
1758 }
1759 }
1760}
1761
abcf61c4
RH		 1762static void tcg_out_cmp2(TCGContext *s, const TCGArg *args,
1763 const int *const_args)
1764{
1765 static const struct { uint8_t bit1, bit2; } bits[] = {
1766 [TCG_COND_LT ] = { CR_LT, CR_LT },
1767 [TCG_COND_LE ] = { CR_LT, CR_GT },
1768 [TCG_COND_GT ] = { CR_GT, CR_GT },
1769 [TCG_COND_GE ] = { CR_GT, CR_LT },
1770 [TCG_COND_LTU] = { CR_LT, CR_LT },
1771 [TCG_COND_LEU] = { CR_LT, CR_GT },
1772 [TCG_COND_GTU] = { CR_GT, CR_GT },
1773 [TCG_COND_GEU] = { CR_GT, CR_LT },
1774 };
1775
1776 TCGCond cond = args[4], cond2;
1777 TCGArg al, ah, bl, bh;
1778 int blconst, bhconst;
1779 int op, bit1, bit2;
1780
1781 al = args[0];
1782 ah = args[1];
1783 bl = args[2];
1784 bh = args[3];
1785 blconst = const_args[2];
1786 bhconst = const_args[3];
1787
1788 switch (cond) {
1789 case TCG_COND_EQ:
1790 op = CRAND;
1791 goto do_equality;
1792 case TCG_COND_NE:
1793 op = CRNAND;
1794 do_equality:
1795 tcg_out_cmp(s, cond, al, bl, blconst, 6, TCG_TYPE_I32);
1796 tcg_out_cmp(s, cond, ah, bh, bhconst, 7, TCG_TYPE_I32);
1797 tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
1798 break;
1799
1800 case TCG_COND_LT:
1801 case TCG_COND_LE:
1802 case TCG_COND_GT:
1803 case TCG_COND_GE:
1804 case TCG_COND_LTU:
1805 case TCG_COND_LEU:
1806 case TCG_COND_GTU:
1807 case TCG_COND_GEU:
1808 bit1 = bits[cond].bit1;
1809 bit2 = bits[cond].bit2;
1810 op = (bit1 != bit2 ? CRANDC : CRAND);
1811 cond2 = tcg_unsigned_cond(cond);
1812
1813 tcg_out_cmp(s, cond, ah, bh, bhconst, 6, TCG_TYPE_I32);
1814 tcg_out_cmp(s, cond2, al, bl, blconst, 7, TCG_TYPE_I32);
1815 tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, bit2));
1816 tcg_out32(s, CROR | BT(7, CR_EQ) | BA(6, bit1) | BB(7, CR_EQ));
1817 break;
1818
1819 default:
1820 tcg_abort();
1821 }
1822}
1823
1824static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
1825 const int *const_args)
1826{
1827 tcg_out_cmp2(s, args + 1, const_args + 1);
1828 tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
1829 tcg_out_rlw(s, RLWINM, args[0], TCG_REG_R0, 31, 31, 31);
1830}
1831
1832static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args,
1833 const int *const_args)
1834{
1835 tcg_out_cmp2(s, args, const_args);
bec16311 1836 tcg_out_bc(s, BC | BI(7, CR_EQ) | BO_COND_TRUE, arg_label(args[5]));
abcf61c4
RH		 1837}
1838
7b4af5ee
PK		 1839static void tcg_out_mb(TCGContext *s, TCGArg a0)
1840{
fc879703
NP		 1841 uint32_t insn;
1842
1843 if (a0 & TCG_MO_ST_LD) {
1844 insn = HWSYNC;
1845 } else {
7b4af5ee 1846 insn = LWSYNC;
7b4af5ee 1847 }
fc879703 1848
7b4af5ee
PK		 1849 tcg_out32(s, insn);
1850}
1851
20add588 1852static inline uint64_t make_pair(tcg_insn_unit i1, tcg_insn_unit i2)
810260a8 1853{
20add588
LL		 1854 if (HOST_BIG_ENDIAN) {
1855 return (uint64_t)i1 << 32 | i2;
1856 }
1857 return (uint64_t)i2 << 32 | i1;
1858}
1859
1860static inline void ppc64_replace2(uintptr_t rx, uintptr_t rw,
1861 tcg_insn_unit i0, tcg_insn_unit i1)
1862{
1863#if TCG_TARGET_REG_BITS == 64
1864 qatomic_set((uint64_t *)rw, make_pair(i0, i1));
1865 flush_idcache_range(rx, rw, 8);
5bfd75a3 1866#else
20add588 1867 qemu_build_not_reached();
5bfd75a3 1868#endif
20add588 1869}
5bfd75a3 1870
20add588
LL		 1871static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw,
1872 tcg_insn_unit i0, tcg_insn_unit i1,
1873 tcg_insn_unit i2, tcg_insn_unit i3)
1874{
1875 uint64_t p[2];
1876
1877 p[!HOST_BIG_ENDIAN] = make_pair(i0, i1);
1878 p[HOST_BIG_ENDIAN] = make_pair(i2, i3);
1879
1880 /*
1881 * There's no convenient way to get the compiler to allocate a pair
1882 * of registers at an even index, so copy into r6/r7 and clobber.
1883 */
1884 asm("mr %%r6, %1\n\t"
1885 "mr %%r7, %2\n\t"
1886 "stq %%r6, %0"
1887 : "=Q"(*(__int128 *)rw) : "r"(p[0]), "r"(p[1]) : "r6", "r7");
1888 flush_idcache_range(rx, rw, 16);
1889}
1890
1891void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
1892 uintptr_t jmp_rw, uintptr_t addr)
1893{
1894 tcg_insn_unit i0, i1, i2, i3;
1895 intptr_t tb_diff = addr - tc_ptr;
1896 intptr_t br_diff = addr - (jmp_rx + 4);
1897 intptr_t lo, hi;
1898
1899 if (TCG_TARGET_REG_BITS == 32) {
1acbad0f 1900 intptr_t diff = addr - jmp_rx;
5964fca8 1901 tcg_debug_assert(in_range_b(diff));
1acbad0f
RH		 1902 qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc));
1903 flush_idcache_range(jmp_rx, jmp_rw, 4);
20add588
LL		 1904 return;
1905 }
1906
1907 /*
1908 * For 16-bit displacements, we can use a single add + branch.
1909 * This happens quite often.
1910 */
1911 if (tb_diff == (int16_t)tb_diff) {
1912 i0 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff);
1913 i1 = B | (br_diff & 0x3fffffc);
1914 ppc64_replace2(jmp_rx, jmp_rw, i0, i1);
1915 return;
1916 }
1917
1918 lo = (int16_t)tb_diff;
1919 hi = (int32_t)(tb_diff - lo);
1920 assert(tb_diff == hi + lo);
1921 i0 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16);
1922 i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo);
1923
1924 /*
1925 * Without stq from 2.07, we can only update two insns,
1926 * and those must be the ones that load the target address.
1927 */
1928 if (!have_isa_2_07) {
1929 ppc64_replace2(jmp_rx, jmp_rw, i0, i1);
1930 return;
1931 }
1932
1933 /*
1934 * For 26-bit displacements, we can use a direct branch.
1935 * Otherwise we still need the indirect branch, which we
1936 * must restore after a potential direct branch write.
1937 */
1938 br_diff -= 4;
1939 if (in_range_b(br_diff)) {
1940 i2 = B | (br_diff & 0x3fffffc);
1941 i3 = NOP;
1942 } else {
1943 i2 = MTSPR | RS(TCG_REG_TB) | CTR;
1944 i3 = BCCTR | BO_ALWAYS;
5964fca8 1945 }
20add588 1946 ppc64_replace4(jmp_rx, jmp_rw, i0, i1, i2, i3);
399f1648 1947}
810260a8 1948
8605cbcd
RH		 1949static void tcg_out_call_int(TCGContext *s, int lk,
1950 const tcg_insn_unit *target)
810260a8 1951{
eaf7d1cf 1952#ifdef _CALL_AIX
d604f1a9
RH		 1953 /* Look through the descriptor. If the branch is in range, and we
1954 don't have to spend too much effort on building the toc. */
2be7d76b
RH		 1955 const void *tgt = ((const void * const *)target)[0];
1956 uintptr_t toc = ((const uintptr_t *)target)[1];
d604f1a9 1957 intptr_t diff = tcg_pcrel_diff(s, tgt);
b18d5d2b 1958
d604f1a9 1959 if (in_range_b(diff) && toc == (uint32_t)toc) {
dfca1778 1960 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, toc);
8605cbcd 1961 tcg_out_b(s, lk, tgt);
541dd4ce 1962 } else {
d604f1a9
RH		 1963 /* Fold the low bits of the constant into the addresses below. */
1964 intptr_t arg = (intptr_t)target;
1965 int ofs = (int16_t)arg;
1966
1967 if (ofs + 8 < 0x8000) {
1968 arg -= ofs;
1969 } else {
1970 ofs = 0;
1971 }
dfca1778
RH		 1972 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, arg);
1973 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_TMP1, ofs);
d604f1a9 1974 tcg_out32(s, MTSPR | RA(TCG_REG_R0) | CTR);
dfca1778 1975 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_REG_TMP1, ofs + SZP);
8605cbcd 1976 tcg_out32(s, BCCTR | BO_ALWAYS | lk);
541dd4ce 1977 }
77e58d0d
UW		 1978#elif defined(_CALL_ELF) && _CALL_ELF == 2
1979 intptr_t diff;
1980
1981 /* In the ELFv2 ABI, we have to set up r12 to contain the destination
1982 address, which the callee uses to compute its TOC address. */
1983 /* FIXME: when the branch is in range, we could avoid r12 load if we
1984 knew that the destination uses the same TOC, and what its local
1985 entry point offset is. */
1986 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R12, (intptr_t)target);
1987
1988 diff = tcg_pcrel_diff(s, target);
1989 if (in_range_b(diff)) {
8605cbcd 1990 tcg_out_b(s, lk, target);
77e58d0d
UW		 1991 } else {
1992 tcg_out32(s, MTSPR | RS(TCG_REG_R12) | CTR);
8605cbcd 1993 tcg_out32(s, BCCTR | BO_ALWAYS | lk);
77e58d0d 1994 }
eaf7d1cf 1995#else
8605cbcd 1996 tcg_out_b(s, lk, target);
d604f1a9 1997#endif
810260a8 1998}
1999
8605cbcd
RH		 2000static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)
2001{
2002 tcg_out_call_int(s, LK, target);
2003}
2004
4b473e0c 2005static const uint32_t qemu_ldx_opc[(MO_SSIZE + MO_BSWAP) + 1] = {
d604f1a9
RH		 2006 [MO_UB] = LBZX,
2007 [MO_UW] = LHZX,
2008 [MO_UL] = LWZX,
fc313c64 2009 [MO_UQ] = LDX,
d604f1a9
RH		 2010 [MO_SW] = LHAX,
2011 [MO_SL] = LWAX,
2012 [MO_BSWAP | MO_UB] = LBZX,
2013 [MO_BSWAP | MO_UW] = LHBRX,
2014 [MO_BSWAP | MO_UL] = LWBRX,
fc313c64 2015 [MO_BSWAP | MO_UQ] = LDBRX,
d604f1a9 2016};
810260a8 2017
4b473e0c 2018static const uint32_t qemu_stx_opc[(MO_SIZE + MO_BSWAP) + 1] = {
d604f1a9
RH		 2019 [MO_UB] = STBX,
2020 [MO_UW] = STHX,
2021 [MO_UL] = STWX,
fc313c64 2022 [MO_UQ] = STDX,
d604f1a9
RH		 2023 [MO_BSWAP | MO_UB] = STBX,
2024 [MO_BSWAP | MO_UW] = STHBRX,
2025 [MO_BSWAP | MO_UL] = STWBRX,
fc313c64 2026 [MO_BSWAP | MO_UQ] = STDBRX,
d604f1a9 2027};
991041a4 2028
d604f1a9
RH		 2029static const uint32_t qemu_exts_opc[4] = {
2030 EXTSB, EXTSH, EXTSW, 0
2031};
810260a8 2032
d604f1a9
RH		 2033#if defined (CONFIG_SOFTMMU)
2034/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
2035 * int mmu_idx, uintptr_t ra)
2036 */
4b473e0c 2037static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
d604f1a9
RH		 2038 [MO_UB] = helper_ret_ldub_mmu,
2039 [MO_LEUW] = helper_le_lduw_mmu,
2040 [MO_LEUL] = helper_le_ldul_mmu,
fc313c64 2041 [MO_LEUQ] = helper_le_ldq_mmu,
d604f1a9
RH		 2042 [MO_BEUW] = helper_be_lduw_mmu,
2043 [MO_BEUL] = helper_be_ldul_mmu,
fc313c64 2044 [MO_BEUQ] = helper_be_ldq_mmu,
d604f1a9 2045};
810260a8 2046
d604f1a9
RH		 2047/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
2048 * uintxx_t val, int mmu_idx, uintptr_t ra)
2049 */
4b473e0c 2050static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
d604f1a9
RH		 2051 [MO_UB] = helper_ret_stb_mmu,
2052 [MO_LEUW] = helper_le_stw_mmu,
2053 [MO_LEUL] = helper_le_stl_mmu,
fc313c64 2054 [MO_LEUQ] = helper_le_stq_mmu,
d604f1a9
RH		 2055 [MO_BEUW] = helper_be_stw_mmu,
2056 [MO_BEUL] = helper_be_stl_mmu,
fc313c64 2057 [MO_BEUQ] = helper_be_stq_mmu,
d604f1a9 2058};
810260a8 2059
269bd5d8
RH		 2060/* We expect to use a 16-bit negative offset from ENV. */
2061QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
2062QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -32768);
2063
d604f1a9
RH		 2064/* Perform the TLB load and compare. Places the result of the comparison
2065 in CR7, loads the addend of the TLB into R3, and returns the register
2066 containing the guest address (zero-extended into R4). Clobbers R0 and R2. */
2067
14776ab5 2068static TCGReg tcg_out_tlb_read(TCGContext *s, MemOp opc,
7f25c469 2069 TCGReg addrlo, TCGReg addrhi,
d604f1a9
RH		 2070 int mem_index, bool is_read)
2071{
2072 int cmp_off
2073 = (is_read
644f591a
RH		 2074 ? offsetof(CPUTLBEntry, addr_read)
2075 : offsetof(CPUTLBEntry, addr_write));
269bd5d8
RH		 2076 int fast_off = TLB_MASK_TABLE_OFS(mem_index);
2077 int mask_off = fast_off + offsetof(CPUTLBDescFast, mask);
2078 int table_off = fast_off + offsetof(CPUTLBDescFast, table);
85aa8081
RH		 2079 unsigned s_bits = opc & MO_SIZE;
2080 unsigned a_bits = get_alignment_bits(opc);
d604f1a9 2081
644f591a 2082 /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */
269bd5d8
RH		 2083 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_AREG0, mask_off);
2084 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R4, TCG_AREG0, table_off);
644f591a
RH		 2085
2086 /* Extract the page index, shifted into place for tlb index. */
2087 if (TCG_TARGET_REG_BITS == 32) {
2088 tcg_out_shri32(s, TCG_REG_TMP1, addrlo,
2089 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
4c314da6 2090 } else {
644f591a
RH		 2091 tcg_out_shri64(s, TCG_REG_TMP1, addrlo,
2092 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
810260a8 2093 }
644f591a 2094 tcg_out32(s, AND | SAB(TCG_REG_R3, TCG_REG_R3, TCG_REG_TMP1));
810260a8 2095
644f591a
RH		 2096 /* Load the TLB comparator. */
2097 if (cmp_off == 0 && TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) {
2098 uint32_t lxu = (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32
2099 ? LWZUX : LDUX);
2100 tcg_out32(s, lxu | TAB(TCG_REG_TMP1, TCG_REG_R3, TCG_REG_R4));
7f25c469 2101 } else {
644f591a
RH		 2102 tcg_out32(s, ADD | TAB(TCG_REG_R3, TCG_REG_R3, TCG_REG_R4));
2103 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
2104 tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP1, TCG_REG_R3, cmp_off + 4);
2105 tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_R4, TCG_REG_R3, cmp_off);
2106 } else {
2107 tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP1, TCG_REG_R3, cmp_off);
2108 }
7f25c469 2109 }
d604f1a9
RH		 2110
2111 /* Load the TLB addend for use on the fast path. Do this asap
2112 to minimize any load use delay. */
644f591a
RH		 2113 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_REG_R3,
2114 offsetof(CPUTLBEntry, addend));
d604f1a9 2115
68d45bb6 2116 /* Clear the non-page, non-alignment bits from the address */
85aa8081
RH		 2117 if (TCG_TARGET_REG_BITS == 32) {
2118 /* We don't support unaligned accesses on 32-bits.
2119 * Preserve the bottom bits and thus trigger a comparison
2120 * failure on unaligned accesses.
68d45bb6 2121 */
85aa8081 2122 if (a_bits < s_bits) {
1f00b27f
SS		 2123 a_bits = s_bits;
2124 }
7f25c469 2125 tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0,
1f00b27f 2126 (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS);
85aa8081
RH		 2127 } else {
2128 TCGReg t = addrlo;
2129
2130 /* If the access is unaligned, we need to make sure we fail if we
2131 * cross a page boundary. The trick is to add the access size-1
2132 * to the address before masking the low bits. That will make the
2133 * address overflow to the next page if we cross a page boundary,
2134 * which will then force a mismatch of the TLB compare.
2135 */
2136 if (a_bits < s_bits) {
2137 unsigned a_mask = (1 << a_bits) - 1;
2138 unsigned s_mask = (1 << s_bits) - 1;
2139 tcg_out32(s, ADDI | TAI(TCG_REG_R0, t, s_mask - a_mask));
2140 t = TCG_REG_R0;
2141 }
2142
2143 /* Mask the address for the requested alignment. */
2144 if (TARGET_LONG_BITS == 32) {
2145 tcg_out_rlw(s, RLWINM, TCG_REG_R0, t, 0,
2146 (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS);
644f591a
RH		 2147 /* Zero-extend the address for use in the final address. */
2148 tcg_out_ext32u(s, TCG_REG_R4, addrlo);
2149 addrlo = TCG_REG_R4;
85aa8081
RH		 2150 } else if (a_bits == 0) {
2151 tcg_out_rld(s, RLDICR, TCG_REG_R0, t, 0, 63 - TARGET_PAGE_BITS);
2152 } else {
2153 tcg_out_rld(s, RLDICL, TCG_REG_R0, t,
1f00b27f 2154 64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - a_bits);
68d45bb6 2155 tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
68d45bb6 2156 }
70fac59a 2157 }
d604f1a9 2158
7f25c469 2159 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
dfca1778
RH		 2160 tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
2161 0, 7, TCG_TYPE_I32);
7f25c469
RH		 2162 tcg_out_cmp(s, TCG_COND_EQ, addrhi, TCG_REG_R4, 0, 6, TCG_TYPE_I32);
2163 tcg_out32(s, CRAND | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
2164 } else {
dfca1778
RH		 2165 tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
2166 0, 7, TCG_TYPE_TL);
7f25c469 2167 }
d604f1a9 2168
7f25c469 2169 return addrlo;
70fac59a 2170}
1cd62ae9 2171
d604f1a9
RH		 2172/* Record the context of a call to the out of line helper code for the slow
2173 path for a load or store, so that we can later generate the correct
2174 helper code. */
9002ffcb 2175static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi,
7f25c469
RH		 2176 TCGReg datalo_reg, TCGReg datahi_reg,
2177 TCGReg addrlo_reg, TCGReg addrhi_reg,
3972ef6f 2178 tcg_insn_unit *raddr, tcg_insn_unit *lptr)
70fac59a 2179{
d604f1a9
RH		 2180 TCGLabelQemuLdst *label = new_ldst_label(s);
2181
2182 label->is_ld = is_ld;
3972ef6f 2183 label->oi = oi;
7f25c469
RH		 2184 label->datalo_reg = datalo_reg;
2185 label->datahi_reg = datahi_reg;
2186 label->addrlo_reg = addrlo_reg;
2187 label->addrhi_reg = addrhi_reg;
e5e2e4c7 2188 label->raddr = tcg_splitwx_to_rx(raddr);
7f25c469 2189 label->label_ptr[0] = lptr;
70fac59a 2190}
1cd62ae9 2191
aeee05f5 2192static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
70fac59a 2193{
9002ffcb 2194 MemOpIdx oi = lb->oi;
14776ab5 2195 MemOp opc = get_memop(oi);
7f25c469 2196 TCGReg hi, lo, arg = TCG_REG_R3;
70fac59a 2197
d54401df 2198 if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
aeee05f5
RH		 2199 return false;
2200 }
70fac59a 2201
7f25c469 2202 tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
1cd62ae9 2203
7f25c469
RH		 2204 lo = lb->addrlo_reg;
2205 hi = lb->addrhi_reg;
2206 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
c8eef960 2207 arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN);
7f25c469
RH		 2208 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
2209 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
2210 } else {
2211 /* If the address needed to be zero-extended, we'll have already
2212 placed it in R4. The only remaining case is 64-bit guest. */
2213 tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
2214 }
752c1fdb 2215
3972ef6f 2216 tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
7f25c469 2217 tcg_out32(s, MFSPR | RT(arg) | LR);
70fac59a 2218
2b7ec66f 2219 tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
70fac59a 2220
7f25c469
RH		 2221 lo = lb->datalo_reg;
2222 hi = lb->datahi_reg;
2223 if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) {
2224 tcg_out_mov(s, TCG_TYPE_I32, lo, TCG_REG_R4);
2225 tcg_out_mov(s, TCG_TYPE_I32, hi, TCG_REG_R3);
2226 } else if (opc & MO_SIGN) {
d604f1a9 2227 uint32_t insn = qemu_exts_opc[opc & MO_SIZE];
7f25c469 2228 tcg_out32(s, insn | RA(lo) | RS(TCG_REG_R3));
d604f1a9 2229 } else {
7f25c469 2230 tcg_out_mov(s, TCG_TYPE_REG, lo, TCG_REG_R3);
70fac59a
RH		 2231 }
2232
d604f1a9 2233 tcg_out_b(s, 0, lb->raddr);
aeee05f5 2234 return true;
d604f1a9 2235}
70fac59a 2236
aeee05f5 2237static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
d604f1a9 2238{
9002ffcb 2239 MemOpIdx oi = lb->oi;
14776ab5
TN		 2240 MemOp opc = get_memop(oi);
2241 MemOp s_bits = opc & MO_SIZE;
7f25c469 2242 TCGReg hi, lo, arg = TCG_REG_R3;
1cd62ae9 2243
d54401df 2244 if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
aeee05f5
RH		 2245 return false;
2246 }
1cd62ae9 2247
7f25c469
RH		 2248 tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
2249
2250 lo = lb->addrlo_reg;
2251 hi = lb->addrhi_reg;
2252 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
c8eef960 2253 arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN);
7f25c469
RH		 2254 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
2255 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
2256 } else {
2257 /* If the address needed to be zero-extended, we'll have already
2258 placed it in R4. The only remaining case is 64-bit guest. */
2259 tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
2260 }
1cd62ae9 2261
7f25c469
RH		 2262 lo = lb->datalo_reg;
2263 hi = lb->datahi_reg;
2264 if (TCG_TARGET_REG_BITS == 32) {
2265 switch (s_bits) {
2266 case MO_64:
c8eef960 2267 arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN);
7f25c469
RH		 2268 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
2269 /* FALLTHRU */
2270 case MO_32:
2271 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
2272 break;
2273 default:
2274 tcg_out_rlw(s, RLWINM, arg++, lo, 0, 32 - (8 << s_bits), 31);
2275 break;
2276 }
2277 } else {
2278 if (s_bits == MO_64) {
2279 tcg_out_mov(s, TCG_TYPE_I64, arg++, lo);
2280 } else {
2281 tcg_out_rld(s, RLDICL, arg++, lo, 0, 64 - (8 << s_bits));
2282 }
2283 }
1cd62ae9 2284
3972ef6f 2285 tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
7f25c469 2286 tcg_out32(s, MFSPR | RT(arg) | LR);
1cd62ae9 2287
2b7ec66f 2288 tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
d604f1a9
RH		 2289
2290 tcg_out_b(s, 0, lb->raddr);
aeee05f5 2291 return true;
1cd62ae9 2292}
8605cbcd
RH		 2293#else
2294
2295static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo,
2296 TCGReg addrhi, unsigned a_bits)
2297{
2298 unsigned a_mask = (1 << a_bits) - 1;
2299 TCGLabelQemuLdst *label = new_ldst_label(s);
2300
2301 label->is_ld = is_ld;
2302 label->addrlo_reg = addrlo;
2303 label->addrhi_reg = addrhi;
2304
2305 /* We are expecting a_bits to max out at 7, much lower than ANDI. */
2306 tcg_debug_assert(a_bits < 16);
2307 tcg_out32(s, ANDI | SAI(addrlo, TCG_REG_R0, a_mask));
2308
2309 label->label_ptr[0] = s->code_ptr;
2310 tcg_out32(s, BC | BI(0, CR_EQ) | BO_COND_FALSE | LK);
2311
2312 label->raddr = tcg_splitwx_to_rx(s->code_ptr);
2313}
2314
2315static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
2316{
2317 if (!reloc_pc14(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
2318 return false;
2319 }
2320
2321 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
2322 TCGReg arg = TCG_REG_R4;
c8eef960
RH		 2323
2324 arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN);
8605cbcd
RH		 2325 if (l->addrlo_reg != arg) {
2326 tcg_out_mov(s, TCG_TYPE_I32, arg, l->addrhi_reg);
2327 tcg_out_mov(s, TCG_TYPE_I32, arg + 1, l->addrlo_reg);
2328 } else if (l->addrhi_reg != arg + 1) {
2329 tcg_out_mov(s, TCG_TYPE_I32, arg + 1, l->addrlo_reg);
2330 tcg_out_mov(s, TCG_TYPE_I32, arg, l->addrhi_reg);
2331 } else {
2332 tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R0, arg);
2333 tcg_out_mov(s, TCG_TYPE_I32, arg, arg + 1);
2334 tcg_out_mov(s, TCG_TYPE_I32, arg + 1, TCG_REG_R0);
2335 }
2336 } else {
2337 tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R4, l->addrlo_reg);
2338 }
2339 tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R3, TCG_AREG0);
2340
2341 /* "Tail call" to the helper, with the return address back inline. */
2342 tcg_out_call_int(s, 0, (const void *)(l->is_ld ? helper_unaligned_ld
2343 : helper_unaligned_st));
2344 return true;
2345}
2346
2347static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
2348{
2349 return tcg_out_fail_alignment(s, l);
2350}
2351
2352static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
2353{
2354 return tcg_out_fail_alignment(s, l);
2355}
2356
d604f1a9 2357#endif /* SOFTMMU */
1cd62ae9 2358
7f25c469 2359static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
810260a8 2360{
7f25c469
RH		 2361 TCGReg datalo, datahi, addrlo, rbase;
2362 TCGReg addrhi __attribute__((unused));
9002ffcb 2363 MemOpIdx oi;
14776ab5 2364 MemOp opc, s_bits;
d604f1a9 2365#ifdef CONFIG_SOFTMMU
7f25c469 2366 int mem_index;
d604f1a9 2367 tcg_insn_unit *label_ptr;
8605cbcd
RH		 2368#else
2369 unsigned a_bits;
d604f1a9 2370#endif
810260a8 2371
7f25c469
RH		 2372 datalo = *args++;
2373 datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
2374 addrlo = *args++;
2375 addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
59227d5d
RH		 2376 oi = *args++;
2377 opc = get_memop(oi);
7f25c469
RH		 2378 s_bits = opc & MO_SIZE;
2379
d604f1a9 2380#ifdef CONFIG_SOFTMMU
59227d5d 2381 mem_index = get_mmuidx(oi);
68d45bb6 2382 addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true);
d604f1a9
RH		 2383
2384 /* Load a pointer into the current opcode w/conditional branch-link. */
2385 label_ptr = s->code_ptr;
f9c7246f 2386 tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
d604f1a9
RH		 2387
2388 rbase = TCG_REG_R3;
2389#else /* !CONFIG_SOFTMMU */
8605cbcd
RH		 2390 a_bits = get_alignment_bits(opc);
2391 if (a_bits) {
2392 tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits);
2393 }
b76f21a7 2394 rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
7f25c469 2395 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
dfca1778
RH		 2396 tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
2397 addrlo = TCG_REG_TMP1;
d604f1a9
RH		 2398 }
2399#endif
2400
7f25c469
RH		 2401 if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
2402 if (opc & MO_BSWAP) {
2403 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2404 tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
2405 tcg_out32(s, LWBRX | TAB(datahi, rbase, TCG_REG_R0));
2406 } else if (rbase != 0) {
2407 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2408 tcg_out32(s, LWZX | TAB(datahi, rbase, addrlo));
2409 tcg_out32(s, LWZX | TAB(datalo, rbase, TCG_REG_R0));
2410 } else if (addrlo == datahi) {
2411 tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
2412 tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
2413 } else {
2414 tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
2415 tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
2416 }
541dd4ce 2417 } else {
2b7ec66f 2418 uint32_t insn = qemu_ldx_opc[opc & (MO_BSWAP | MO_SSIZE)];
4e33fe01 2419 if (!have_isa_2_06 && insn == LDBRX) {
7f25c469
RH		 2420 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2421 tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
2422 tcg_out32(s, LWBRX | TAB(TCG_REG_R0, rbase, TCG_REG_R0));
2423 tcg_out_rld(s, RLDIMI, datalo, TCG_REG_R0, 32, 0);
2424 } else if (insn) {
2425 tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
2426 } else {
2427 insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)];
2428 tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
2429 insn = qemu_exts_opc[s_bits];
2430 tcg_out32(s, insn | RA(datalo) | RS(datalo));
2431 }
810260a8 2432 }
810260a8 2433
d604f1a9 2434#ifdef CONFIG_SOFTMMU
3972ef6f
RH		 2435 add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
2436 s->code_ptr, label_ptr);
d604f1a9 2437#endif
810260a8 2438}
2439
7f25c469 2440static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
027ffea9 2441{
7f25c469
RH		 2442 TCGReg datalo, datahi, addrlo, rbase;
2443 TCGReg addrhi __attribute__((unused));
9002ffcb 2444 MemOpIdx oi;
14776ab5 2445 MemOp opc, s_bits;
d604f1a9 2446#ifdef CONFIG_SOFTMMU
7f25c469 2447 int mem_index;
d604f1a9 2448 tcg_insn_unit *label_ptr;
8605cbcd
RH		 2449#else
2450 unsigned a_bits;
d604f1a9 2451#endif
027ffea9 2452
7f25c469
RH		 2453 datalo = *args++;
2454 datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
2455 addrlo = *args++;
2456 addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
59227d5d
RH		 2457 oi = *args++;
2458 opc = get_memop(oi);
7f25c469
RH		 2459 s_bits = opc & MO_SIZE;
2460
d604f1a9 2461#ifdef CONFIG_SOFTMMU
59227d5d 2462 mem_index = get_mmuidx(oi);
68d45bb6 2463 addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false);
027ffea9 2464
d604f1a9
RH		 2465 /* Load a pointer into the current opcode w/conditional branch-link. */
2466 label_ptr = s->code_ptr;
f9c7246f 2467 tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
027ffea9 2468
d604f1a9
RH		 2469 rbase = TCG_REG_R3;
2470#else /* !CONFIG_SOFTMMU */
8605cbcd
RH		 2471 a_bits = get_alignment_bits(opc);
2472 if (a_bits) {
2473 tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits);
2474 }
b76f21a7 2475 rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
7f25c469 2476 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
dfca1778
RH		 2477 tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
2478 addrlo = TCG_REG_TMP1;
d604f1a9
RH		 2479 }
2480#endif
2481
7f25c469
RH		 2482 if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
2483 if (opc & MO_BSWAP) {
2484 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2485 tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
2486 tcg_out32(s, STWBRX | SAB(datahi, rbase, TCG_REG_R0));
2487 } else if (rbase != 0) {
2488 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2489 tcg_out32(s, STWX | SAB(datahi, rbase, addrlo));
2490 tcg_out32(s, STWX | SAB(datalo, rbase, TCG_REG_R0));
2491 } else {
2492 tcg_out32(s, STW | TAI(datahi, addrlo, 0));
2493 tcg_out32(s, STW | TAI(datalo, addrlo, 4));
2494 }
027ffea9 2495 } else {
2b7ec66f 2496 uint32_t insn = qemu_stx_opc[opc & (MO_BSWAP | MO_SIZE)];
4e33fe01 2497 if (!have_isa_2_06 && insn == STDBRX) {
7f25c469 2498 tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
dfca1778 2499 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, addrlo, 4));
7f25c469 2500 tcg_out_shri64(s, TCG_REG_R0, datalo, 32);
dfca1778 2501 tcg_out32(s, STWBRX | SAB(TCG_REG_R0, rbase, TCG_REG_TMP1));
7f25c469
RH		 2502 } else {
2503 tcg_out32(s, insn | SAB(datalo, rbase, addrlo));
2504 }
027ffea9 2505 }
d604f1a9
RH		 2506
2507#ifdef CONFIG_SOFTMMU
3972ef6f
RH		 2508 add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
2509 s->code_ptr, label_ptr);
d604f1a9 2510#endif
027ffea9
RH		 2511}
2512
53c89efd
RH		 2513static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
2514{
2515 int i;
2516 for (i = 0; i < count; ++i) {
2517 p[i] = NOP;
2518 }
2519}
2520
a921fddc
RH		 2521/* Parameters for function call generation, used in tcg.c. */
2522#define TCG_TARGET_STACK_ALIGN 16
a921fddc
RH		 2523#define TCG_TARGET_EXTEND_ARGS 1
2524
802ca56e
RH		 2525#ifdef _CALL_AIX
2526# define LINK_AREA_SIZE (6 * SZR)
2527# define LR_OFFSET (1 * SZR)
2528# define TCG_TARGET_CALL_STACK_OFFSET (LINK_AREA_SIZE + 8 * SZR)
2fa169ba 2529#elif defined(_CALL_DARWIN)
1045fc04
PM		 2530# define LINK_AREA_SIZE (6 * SZR)
2531# define LR_OFFSET (2 * SZR)
ffcfbece
RH		 2532#elif TCG_TARGET_REG_BITS == 64
2533# if defined(_CALL_ELF) && _CALL_ELF == 2
2534# define LINK_AREA_SIZE (4 * SZR)
2535# define LR_OFFSET (1 * SZR)
2536# endif
2537#else /* TCG_TARGET_REG_BITS == 32 */
2538# if defined(_CALL_SYSV)
ffcfbece
RH		 2539# define LINK_AREA_SIZE (2 * SZR)
2540# define LR_OFFSET (1 * SZR)
ffcfbece
RH		 2541# endif
2542#endif
2543#ifndef LR_OFFSET
2544# error "Unhandled abi"
2545#endif
2546#ifndef TCG_TARGET_CALL_STACK_OFFSET
a2a98f80 2547# define TCG_TARGET_CALL_STACK_OFFSET LINK_AREA_SIZE
802ca56e
RH		 2548#endif
2549
2550#define CPU_TEMP_BUF_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long))
2551#define REG_SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * SZR)
d604f1a9 2552
802ca56e
RH		 2553#define FRAME_SIZE ((TCG_TARGET_CALL_STACK_OFFSET \
2554 + TCG_STATIC_CALL_ARGS_SIZE \
2555 + CPU_TEMP_BUF_SIZE \
2556 + REG_SAVE_SIZE \
2557 + TCG_TARGET_STACK_ALIGN - 1) \
2558 & -TCG_TARGET_STACK_ALIGN)
2559
2560#define REG_SAVE_BOT (FRAME_SIZE - REG_SAVE_SIZE)
d604f1a9
RH		 2561
2562static void tcg_target_qemu_prologue(TCGContext *s)
810260a8 2563{
d604f1a9 2564 int i;
810260a8 2565
802ca56e 2566#ifdef _CALL_AIX
d54401df
RH		 2567 const void **desc = (const void **)s->code_ptr;
2568 desc[0] = tcg_splitwx_to_rx(desc + 2); /* entry point */
2569 desc[1] = 0; /* environment pointer */
2570 s->code_ptr = (void *)(desc + 2); /* skip over descriptor */
d604f1a9
RH		 2571#endif
2572
a84ac4cb
RH		 2573 tcg_set_frame(s, TCG_REG_CALL_STACK, REG_SAVE_BOT - CPU_TEMP_BUF_SIZE,
2574 CPU_TEMP_BUF_SIZE);
2575
d604f1a9
RH		 2576 /* Prologue */
2577 tcg_out32(s, MFSPR | RT(TCG_REG_R0) | LR);
ffcfbece
RH		 2578 tcg_out32(s, (SZR == 8 ? STDU : STWU)
2579 | SAI(TCG_REG_R1, TCG_REG_R1, -FRAME_SIZE));
802ca56e 2580
d604f1a9 2581 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
4c3831a0
RH		 2582 tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
2583 TCG_REG_R1, REG_SAVE_BOT + i * SZR);
d604f1a9 2584 }
802ca56e 2585 tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
d604f1a9 2586
4cbea598 2587#ifndef CONFIG_SOFTMMU
b76f21a7 2588 if (guest_base) {
5964fca8 2589 tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base, true);
d604f1a9
RH		 2590 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
2591 }
2592#endif
2593
2594 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
2595 tcg_out32(s, MTSPR | RS(tcg_target_call_iarg_regs[1]) | CTR);
5964fca8
RH		 2596 if (USE_REG_TB) {
2597 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, tcg_target_call_iarg_regs[1]);
a84ac4cb 2598 }
5964fca8 2599 tcg_out32(s, BCCTR | BO_ALWAYS);
d604f1a9
RH		 2600
2601 /* Epilogue */
c8bc1168 2602 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
d604f1a9 2603
802ca56e 2604 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
d604f1a9 2605 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
4c3831a0
RH		 2606 tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
2607 TCG_REG_R1, REG_SAVE_BOT + i * SZR);
d604f1a9 2608 }
d604f1a9
RH		 2609 tcg_out32(s, MTSPR | RS(TCG_REG_R0) | LR);
2610 tcg_out32(s, ADDI | TAI(TCG_REG_R1, TCG_REG_R1, FRAME_SIZE));
2611 tcg_out32(s, BCLR | BO_ALWAYS);
810260a8 2612}
2613
5e8892db
MR		 2614static void tcg_out_op(TCGContext *s, TCGOpcode opc,
2615 const TCGArg args[TCG_MAX_OP_ARGS],
2616 const int const_args[TCG_MAX_OP_ARGS])
810260a8 2617{
ee924fa6 2618 TCGArg a0, a1, a2;
e46b9681 2619
810260a8 2620 switch (opc) {
2621 case INDEX_op_exit_tb:
de3d636d 2622 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
d54401df 2623 tcg_out_b(s, 0, tcg_code_gen_epilogue);
810260a8 2624 break;
2625 case INDEX_op_goto_tb:
5964fca8
RH		 2626 if (s->tb_jmp_insn_offset) {
2627 /* Direct jump. */
2628 if (TCG_TARGET_REG_BITS == 64) {
20add588
LL		 2629 /* Ensure the next insns are 8 or 16-byte aligned. */
2630 while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
5964fca8
RH		 2631 tcg_out32(s, NOP);
2632 }
2633 s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
2634 tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
2635 tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
2636 } else {
2637 s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
2638 tcg_out32(s, B);
2639 s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
2640 break;
2641 }
2642 } else {
2643 /* Indirect jump. */
2644 tcg_debug_assert(s->tb_jmp_insn_offset == NULL);
2645 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, 0,
2646 (intptr_t)(s->tb_jmp_insn_offset + args[0]));
2647 }
2648 tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
5bfd75a3 2649 tcg_out32(s, BCCTR | BO_ALWAYS);
9f754620 2650 set_jmp_reset_offset(s, args[0]);
5964fca8
RH		 2651 if (USE_REG_TB) {
2652 /* For the unlinked case, need to reset TCG_REG_TB. */
2d6f38eb
RH		 2653 tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
2654 -tcg_current_code_size(s));
5964fca8 2655 }
810260a8 2656 break;
0c240785
RH		 2657 case INDEX_op_goto_ptr:
2658 tcg_out32(s, MTSPR | RS(args[0]) | CTR);
5964fca8
RH		 2659 if (USE_REG_TB) {
2660 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, args[0]);
2661 }
2662 tcg_out32(s, ADDI | TAI(TCG_REG_R3, 0, 0));
0c240785
RH		 2663 tcg_out32(s, BCCTR | BO_ALWAYS);
2664 break;
810260a8 2665 case INDEX_op_br:
2666 {
bec16311 2667 TCGLabel *l = arg_label(args[0]);
f9c7246f 2668 uint32_t insn = B;
810260a8 2669
2670 if (l->has_value) {
d54401df
RH		 2671 insn |= reloc_pc24_val(tcg_splitwx_to_rx(s->code_ptr),
2672 l->u.value_ptr);
541dd4ce 2673 } else {
bec16311 2674 tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0);
810260a8 2675 }
f9c7246f 2676 tcg_out32(s, insn);
810260a8 2677 }
2678 break;
810260a8 2679 case INDEX_op_ld8u_i32:
2680 case INDEX_op_ld8u_i64:
b18d5d2b 2681 tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
810260a8 2682 break;
2683 case INDEX_op_ld8s_i32:
2684 case INDEX_op_ld8s_i64:
b18d5d2b 2685 tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
f4bf14f4 2686 tcg_out_ext8s(s, args[0], args[0]);
810260a8 2687 break;
2688 case INDEX_op_ld16u_i32:
2689 case INDEX_op_ld16u_i64:
b18d5d2b 2690 tcg_out_mem_long(s, LHZ, LHZX, args[0], args[1], args[2]);
810260a8 2691 break;
2692 case INDEX_op_ld16s_i32:
2693 case INDEX_op_ld16s_i64:
b18d5d2b 2694 tcg_out_mem_long(s, LHA, LHAX, args[0], args[1], args[2]);
810260a8 2695 break;
2696 case INDEX_op_ld_i32:
2697 case INDEX_op_ld32u_i64:
b18d5d2b 2698 tcg_out_mem_long(s, LWZ, LWZX, args[0], args[1], args[2]);
810260a8 2699 break;
2700 case INDEX_op_ld32s_i64:
b18d5d2b 2701 tcg_out_mem_long(s, LWA, LWAX, args[0], args[1], args[2]);
810260a8 2702 break;
2703 case INDEX_op_ld_i64:
b18d5d2b 2704 tcg_out_mem_long(s, LD, LDX, args[0], args[1], args[2]);
810260a8 2705 break;
2706 case INDEX_op_st8_i32:
2707 case INDEX_op_st8_i64:
b18d5d2b 2708 tcg_out_mem_long(s, STB, STBX, args[0], args[1], args[2]);
810260a8 2709 break;
2710 case INDEX_op_st16_i32:
2711 case INDEX_op_st16_i64:
b18d5d2b 2712 tcg_out_mem_long(s, STH, STHX, args[0], args[1], args[2]);
810260a8 2713 break;
2714 case INDEX_op_st_i32:
2715 case INDEX_op_st32_i64:
b18d5d2b 2716 tcg_out_mem_long(s, STW, STWX, args[0], args[1], args[2]);
810260a8 2717 break;
2718 case INDEX_op_st_i64:
b18d5d2b 2719 tcg_out_mem_long(s, STD, STDX, args[0], args[1], args[2]);
810260a8 2720 break;
2721
2722 case INDEX_op_add_i32:
ee924fa6
RH		 2723 a0 = args[0], a1 = args[1], a2 = args[2];
2724 if (const_args[2]) {
ee924fa6 2725 do_addi_32:
b18d5d2b 2726 tcg_out_mem_long(s, ADDI, ADD, a0, a1, (int32_t)a2);
ee924fa6
RH		 2727 } else {
2728 tcg_out32(s, ADD | TAB(a0, a1, a2));
2729 }
810260a8 2730 break;
2731 case INDEX_op_sub_i32:
ee924fa6 2732 a0 = args[0], a1 = args[1], a2 = args[2];
148bdd23
RH		 2733 if (const_args[1]) {
2734 if (const_args[2]) {
2735 tcg_out_movi(s, TCG_TYPE_I32, a0, a1 - a2);
2736 } else {
2737 tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
2738 }
2739 } else if (const_args[2]) {
ee924fa6
RH		 2740 a2 = -a2;
2741 goto do_addi_32;
2742 } else {
2743 tcg_out32(s, SUBF | TAB(a0, a2, a1));
2744 }
810260a8 2745 break;
2746
2747 case INDEX_op_and_i32:
37251b98 2748 a0 = args[0], a1 = args[1], a2 = args[2];
a9249dff 2749 if (const_args[2]) {
37251b98 2750 tcg_out_andi32(s, a0, a1, a2);
a9249dff 2751 } else {
37251b98 2752 tcg_out32(s, AND | SAB(a1, a0, a2));
a9249dff
RH		 2753 }
2754 break;
2755 case INDEX_op_and_i64:
37251b98 2756 a0 = args[0], a1 = args[1], a2 = args[2];
810260a8 2757 if (const_args[2]) {
37251b98 2758 tcg_out_andi64(s, a0, a1, a2);
637af30c 2759 } else {
37251b98 2760 tcg_out32(s, AND | SAB(a1, a0, a2));
810260a8 2761 }
810260a8 2762 break;
fe6f943f 2763 case INDEX_op_or_i64:
810260a8 2764 case INDEX_op_or_i32:
dce74c57 2765 a0 = args[0], a1 = args[1], a2 = args[2];
810260a8 2766 if (const_args[2]) {
dce74c57
RH		 2767 tcg_out_ori32(s, a0, a1, a2);
2768 } else {
2769 tcg_out32(s, OR | SAB(a1, a0, a2));
810260a8 2770 }
810260a8 2771 break;
fe6f943f 2772 case INDEX_op_xor_i64:
810260a8 2773 case INDEX_op_xor_i32:
dce74c57 2774 a0 = args[0], a1 = args[1], a2 = args[2];
810260a8 2775 if (const_args[2]) {
dce74c57
RH		 2776 tcg_out_xori32(s, a0, a1, a2);
2777 } else {
2778 tcg_out32(s, XOR | SAB(a1, a0, a2));
810260a8 2779 }
810260a8 2780 break;
ce1010d6 2781 case INDEX_op_andc_i32:
37251b98
RH		 2782 a0 = args[0], a1 = args[1], a2 = args[2];
2783 if (const_args[2]) {
2784 tcg_out_andi32(s, a0, a1, ~a2);
2785 } else {
2786 tcg_out32(s, ANDC | SAB(a1, a0, a2));
2787 }
2788 break;
ce1010d6 2789 case INDEX_op_andc_i64:
37251b98
RH		 2790 a0 = args[0], a1 = args[1], a2 = args[2];
2791 if (const_args[2]) {
2792 tcg_out_andi64(s, a0, a1, ~a2);
2793 } else {
2794 tcg_out32(s, ANDC | SAB(a1, a0, a2));
2795 }
ce1010d6
RH		 2796 break;
2797 case INDEX_op_orc_i32:
37251b98
RH		 2798 if (const_args[2]) {
2799 tcg_out_ori32(s, args[0], args[1], ~args[2]);
2800 break;
2801 }
2802 /* FALLTHRU */
ce1010d6
RH		 2803 case INDEX_op_orc_i64:
2804 tcg_out32(s, ORC | SAB(args[1], args[0], args[2]));
2805 break;
2806 case INDEX_op_eqv_i32:
37251b98
RH		 2807 if (const_args[2]) {
2808 tcg_out_xori32(s, args[0], args[1], ~args[2]);
2809 break;
2810 }
2811 /* FALLTHRU */
ce1010d6
RH		 2812 case INDEX_op_eqv_i64:
2813 tcg_out32(s, EQV | SAB(args[1], args[0], args[2]));
2814 break;
2815 case INDEX_op_nand_i32:
2816 case INDEX_op_nand_i64:
2817 tcg_out32(s, NAND | SAB(args[1], args[0], args[2]));
2818 break;
2819 case INDEX_op_nor_i32:
2820 case INDEX_op_nor_i64:
2821 tcg_out32(s, NOR | SAB(args[1], args[0], args[2]));
2822 break;
810260a8 2823
d0b07481
RH		 2824 case INDEX_op_clz_i32:
2825 tcg_out_cntxz(s, TCG_TYPE_I32, CNTLZW, args[0], args[1],
2826 args[2], const_args[2]);
2827 break;
2828 case INDEX_op_ctz_i32:
2829 tcg_out_cntxz(s, TCG_TYPE_I32, CNTTZW, args[0], args[1],
2830 args[2], const_args[2]);
2831 break;
33e75fb9
RH		 2832 case INDEX_op_ctpop_i32:
2833 tcg_out32(s, CNTPOPW | SAB(args[1], args[0], 0));
2834 break;
d0b07481
RH		 2835
2836 case INDEX_op_clz_i64:
2837 tcg_out_cntxz(s, TCG_TYPE_I64, CNTLZD, args[0], args[1],
2838 args[2], const_args[2]);
2839 break;
2840 case INDEX_op_ctz_i64:
2841 tcg_out_cntxz(s, TCG_TYPE_I64, CNTTZD, args[0], args[1],
2842 args[2], const_args[2]);
2843 break;
33e75fb9
RH		 2844 case INDEX_op_ctpop_i64:
2845 tcg_out32(s, CNTPOPD | SAB(args[1], args[0], 0));
2846 break;
d0b07481 2847
810260a8 2848 case INDEX_op_mul_i32:
ef809300 2849 a0 = args[0], a1 = args[1], a2 = args[2];
810260a8 2850 if (const_args[2]) {
ef809300
RH		 2851 tcg_out32(s, MULLI | TAI(a0, a1, a2));
2852 } else {
2853 tcg_out32(s, MULLW | TAB(a0, a1, a2));
810260a8 2854 }
810260a8 2855 break;
2856
2857 case INDEX_op_div_i32:
541dd4ce 2858 tcg_out32(s, DIVW | TAB(args[0], args[1], args[2]));
810260a8 2859 break;
2860
2861 case INDEX_op_divu_i32:
541dd4ce 2862 tcg_out32(s, DIVWU | TAB(args[0], args[1], args[2]));
810260a8 2863 break;
2864
4d573822
MKF		 2865 case INDEX_op_rem_i32:
2866 tcg_out32(s, MODSW | TAB(args[0], args[1], args[2]));
2867 break;
2868
2869 case INDEX_op_remu_i32:
2870 tcg_out32(s, MODUW | TAB(args[0], args[1], args[2]));
2871 break;
2872
810260a8 2873 case INDEX_op_shl_i32:
2874 if (const_args[2]) {
94248cfc
CF		 2875 /* Limit immediate shift count lest we create an illegal insn. */
2876 tcg_out_shli32(s, args[0], args[1], args[2] & 31);
9e555b73 2877 } else {
541dd4ce 2878 tcg_out32(s, SLW | SAB(args[1], args[0], args[2]));
9e555b73 2879 }
810260a8 2880 break;
2881 case INDEX_op_shr_i32:
2882 if (const_args[2]) {
94248cfc
CF		 2883 /* Limit immediate shift count lest we create an illegal insn. */
2884 tcg_out_shri32(s, args[0], args[1], args[2] & 31);
9e555b73 2885 } else {
541dd4ce 2886 tcg_out32(s, SRW | SAB(args[1], args[0], args[2]));
9e555b73 2887 }
810260a8 2888 break;
2889 case INDEX_op_sar_i32:
541dd4ce 2890 if (const_args[2]) {
05dd01fa 2891 tcg_out_sari32(s, args[0], args[1], args[2]);
541dd4ce
RH		 2892 } else {
2893 tcg_out32(s, SRAW | SAB(args[1], args[0], args[2]));
2894 }
810260a8 2895 break;
313d91c7
RH		 2896 case INDEX_op_rotl_i32:
2897 if (const_args[2]) {
2898 tcg_out_rlw(s, RLWINM, args[0], args[1], args[2], 0, 31);
2899 } else {
2900 tcg_out32(s, RLWNM | SAB(args[1], args[0], args[2])
2901 | MB(0) | ME(31));
2902 }
2903 break;
2904 case INDEX_op_rotr_i32:
2905 if (const_args[2]) {
2906 tcg_out_rlw(s, RLWINM, args[0], args[1], 32 - args[2], 0, 31);
2907 } else {
8327a470
RH		 2908 tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 32));
2909 tcg_out32(s, RLWNM | SAB(args[1], args[0], TCG_REG_R0)
313d91c7
RH		 2910 | MB(0) | ME(31));
2911 }
2912 break;
810260a8 2913
2914 case INDEX_op_brcond_i32:
4c314da6 2915 tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
bec16311 2916 arg_label(args[3]), TCG_TYPE_I32);
e924bbec 2917 break;
810260a8 2918 case INDEX_op_brcond_i64:
4c314da6 2919 tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
bec16311 2920 arg_label(args[3]), TCG_TYPE_I64);
810260a8 2921 break;
abcf61c4
RH		 2922 case INDEX_op_brcond2_i32:
2923 tcg_out_brcond2(s, args, const_args);
2924 break;
810260a8 2925
2926 case INDEX_op_neg_i32:
810260a8 2927 case INDEX_op_neg_i64:
541dd4ce 2928 tcg_out32(s, NEG | RT(args[0]) | RA(args[1]));
810260a8 2929 break;
2930
157f2662 2931 case INDEX_op_not_i32:
2932 case INDEX_op_not_i64:
541dd4ce 2933 tcg_out32(s, NOR | SAB(args[1], args[0], args[1]));
157f2662 2934 break;
2935
810260a8 2936 case INDEX_op_add_i64:
ee924fa6
RH		 2937 a0 = args[0], a1 = args[1], a2 = args[2];
2938 if (const_args[2]) {
ee924fa6 2939 do_addi_64:
b18d5d2b 2940 tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2);
ee924fa6
RH		 2941 } else {
2942 tcg_out32(s, ADD | TAB(a0, a1, a2));
2943 }
810260a8 2944 break;
2945 case INDEX_op_sub_i64:
ee924fa6 2946 a0 = args[0], a1 = args[1], a2 = args[2];
148bdd23
RH		 2947 if (const_args[1]) {
2948 if (const_args[2]) {
2949 tcg_out_movi(s, TCG_TYPE_I64, a0, a1 - a2);
2950 } else {
2951 tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
2952 }
2953 } else if (const_args[2]) {
ee924fa6
RH		 2954 a2 = -a2;
2955 goto do_addi_64;
2956 } else {
2957 tcg_out32(s, SUBF | TAB(a0, a2, a1));
2958 }
810260a8 2959 break;
2960
2961 case INDEX_op_shl_i64:
541dd4ce 2962 if (const_args[2]) {
94248cfc
CF		 2963 /* Limit immediate shift count lest we create an illegal insn. */
2964 tcg_out_shli64(s, args[0], args[1], args[2] & 63);
541dd4ce
RH		 2965 } else {
2966 tcg_out32(s, SLD | SAB(args[1], args[0], args[2]));
2967 }
810260a8 2968 break;
2969 case INDEX_op_shr_i64:
541dd4ce 2970 if (const_args[2]) {
94248cfc
CF		 2971 /* Limit immediate shift count lest we create an illegal insn. */
2972 tcg_out_shri64(s, args[0], args[1], args[2] & 63);
541dd4ce
RH		 2973 } else {
2974 tcg_out32(s, SRD | SAB(args[1], args[0], args[2]));
2975 }
810260a8 2976 break;
2977 case INDEX_op_sar_i64:
fe6f943f 2978 if (const_args[2]) {
05dd01fa 2979 tcg_out_sari64(s, args[0], args[1], args[2]);
541dd4ce
RH		 2980 } else {
2981 tcg_out32(s, SRAD | SAB(args[1], args[0], args[2]));
fe6f943f 2982 }
810260a8 2983 break;
313d91c7
RH		 2984 case INDEX_op_rotl_i64:
2985 if (const_args[2]) {
2986 tcg_out_rld(s, RLDICL, args[0], args[1], args[2], 0);
2987 } else {
2988 tcg_out32(s, RLDCL | SAB(args[1], args[0], args[2]) | MB64(0));
2989 }
2990 break;
2991 case INDEX_op_rotr_i64:
2992 if (const_args[2]) {
2993 tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 0);
2994 } else {
8327a470
RH		 2995 tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 64));
2996 tcg_out32(s, RLDCL | SAB(args[1], args[0], TCG_REG_R0) | MB64(0));
313d91c7
RH		 2997 }
2998 break;
810260a8 2999
3000 case INDEX_op_mul_i64:
ef809300
RH		 3001 a0 = args[0], a1 = args[1], a2 = args[2];
3002 if (const_args[2]) {
3003 tcg_out32(s, MULLI | TAI(a0, a1, a2));
3004 } else {
3005 tcg_out32(s, MULLD | TAB(a0, a1, a2));
3006 }
810260a8 3007 break;
3008 case INDEX_op_div_i64:
541dd4ce 3009 tcg_out32(s, DIVD | TAB(args[0], args[1], args[2]));
810260a8 3010 break;
3011 case INDEX_op_divu_i64:
541dd4ce 3012 tcg_out32(s, DIVDU | TAB(args[0], args[1], args[2]));
810260a8 3013 break;
4d573822
MKF		 3014 case INDEX_op_rem_i64:
3015 tcg_out32(s, MODSD | TAB(args[0], args[1], args[2]));
3016 break;
3017 case INDEX_op_remu_i64:
3018 tcg_out32(s, MODUD | TAB(args[0], args[1], args[2]));
3019 break;
810260a8 3020
1768ec06 3021 case INDEX_op_qemu_ld_i32:
7f25c469
RH		 3022 tcg_out_qemu_ld(s, args, false);
3023 break;
1768ec06 3024 case INDEX_op_qemu_ld_i64:
7f25c469 3025 tcg_out_qemu_ld(s, args, true);
810260a8 3026 break;
1768ec06 3027 case INDEX_op_qemu_st_i32:
7f25c469
RH		 3028 tcg_out_qemu_st(s, args, false);
3029 break;
1768ec06 3030 case INDEX_op_qemu_st_i64:
7f25c469 3031 tcg_out_qemu_st(s, args, true);
810260a8 3032 break;
3033
e46b9681 3034 case INDEX_op_ext8s_i32:
3035 case INDEX_op_ext8s_i64:
f4bf14f4
RH		 3036 tcg_out_ext8s(s, args[0], args[1]);
3037 break;
e46b9681 3038 case INDEX_op_ext16s_i32:
3039 case INDEX_op_ext16s_i64:
f4bf14f4
RH		 3040 tcg_out_ext16s(s, args[0], args[1]);
3041 break;
4f2331e5 3042 case INDEX_op_ext_i32_i64:
e46b9681 3043 case INDEX_op_ext32s_i64:
f4bf14f4 3044 tcg_out_ext32s(s, args[0], args[1]);
e46b9681 3045 break;
4f2331e5
AJ		 3046 case INDEX_op_extu_i32_i64:
3047 tcg_out_ext32u(s, args[0], args[1]);
3048 break;
e46b9681 3049
1cd62ae9 3050 case INDEX_op_setcond_i32:
541dd4ce
RH		 3051 tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
3052 const_args[2]);
1cd62ae9 3053 break;
3054 case INDEX_op_setcond_i64:
541dd4ce
RH		 3055 tcg_out_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], args[2],
3056 const_args[2]);
1cd62ae9 3057 break;
abcf61c4
RH		 3058 case INDEX_op_setcond2_i32:
3059 tcg_out_setcond2(s, args, const_args);
3060 break;
1cd62ae9 3061
5d221582
RH		 3062 case INDEX_op_bswap16_i32:
3063 case INDEX_op_bswap16_i64:
26ce7005 3064 tcg_out_bswap16(s, args[0], args[1], args[2]);
5d221582 3065 break;
5d221582 3066 case INDEX_op_bswap32_i32:
26ce7005
RH		 3067 tcg_out_bswap32(s, args[0], args[1], 0);
3068 break;
5d221582 3069 case INDEX_op_bswap32_i64:
26ce7005 3070 tcg_out_bswap32(s, args[0], args[1], args[2]);
5d221582 3071 break;
68aebd45 3072 case INDEX_op_bswap64_i64:
674ba588 3073 tcg_out_bswap64(s, args[0], args[1]);
68aebd45
RH		 3074 break;
3075
33de9ed2 3076 case INDEX_op_deposit_i32:
39dc85b9
RH		 3077 if (const_args[2]) {
3078 uint32_t mask = ((2u << (args[4] - 1)) - 1) << args[3];
3079 tcg_out_andi32(s, args[0], args[0], ~mask);
3080 } else {
3081 tcg_out_rlw(s, RLWIMI, args[0], args[2], args[3],
3082 32 - args[3] - args[4], 31 - args[3]);
3083 }
33de9ed2
RH		 3084 break;
3085 case INDEX_op_deposit_i64:
39dc85b9
RH		 3086 if (const_args[2]) {
3087 uint64_t mask = ((2ull << (args[4] - 1)) - 1) << args[3];
3088 tcg_out_andi64(s, args[0], args[0], ~mask);
3089 } else {
3090 tcg_out_rld(s, RLDIMI, args[0], args[2], args[3],
3091 64 - args[3] - args[4]);
3092 }
33de9ed2
RH		 3093 break;
3094
c05021c3
RH		 3095 case INDEX_op_extract_i32:
3096 tcg_out_rlw(s, RLWINM, args[0], args[1],
3097 32 - args[2], 32 - args[3], 31);
3098 break;
3099 case INDEX_op_extract_i64:
3100 tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 64 - args[3]);
3101 break;
3102
027ffea9
RH		 3103 case INDEX_op_movcond_i32:
3104 tcg_out_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1], args[2],
3105 args[3], args[4], const_args[2]);
3106 break;
3107 case INDEX_op_movcond_i64:
3108 tcg_out_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1], args[2],
3109 args[3], args[4], const_args[2]);
3110 break;
3111
796f1a68 3112#if TCG_TARGET_REG_BITS == 64
6c858762 3113 case INDEX_op_add2_i64:
796f1a68
RH		 3114#else
3115 case INDEX_op_add2_i32:
3116#endif
6c858762
RH		 3117 /* Note that the CA bit is defined based on the word size of the
3118 environment. So in 64-bit mode it's always carry-out of bit 63.
3119 The fallback code using deposit works just as well for 32-bit. */
3120 a0 = args[0], a1 = args[1];
84247357 3121 if (a0 == args[3] || (!const_args[5] && a0 == args[5])) {
6c858762
RH		 3122 a0 = TCG_REG_R0;
3123 }
84247357
AB		 3124 if (const_args[4]) {
3125 tcg_out32(s, ADDIC | TAI(a0, args[2], args[4]));
6c858762 3126 } else {
84247357 3127 tcg_out32(s, ADDC | TAB(a0, args[2], args[4]));
6c858762
RH		 3128 }
3129 if (const_args[5]) {
84247357 3130 tcg_out32(s, (args[5] ? ADDME : ADDZE) | RT(a1) | RA(args[3]));
6c858762 3131 } else {
84247357 3132 tcg_out32(s, ADDE | TAB(a1, args[3], args[5]));
6c858762
RH		 3133 }
3134 if (a0 != args[0]) {
de3d636d 3135 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
6c858762
RH		 3136 }
3137 break;
3138
796f1a68 3139#if TCG_TARGET_REG_BITS == 64
6c858762 3140 case INDEX_op_sub2_i64:
796f1a68
RH		 3141#else
3142 case INDEX_op_sub2_i32:
3143#endif
6c858762 3144 a0 = args[0], a1 = args[1];
b31284ce 3145 if (a0 == args[5] || (!const_args[3] && a0 == args[3])) {
6c858762
RH		 3146 a0 = TCG_REG_R0;
3147 }
3148 if (const_args[2]) {
b31284ce 3149 tcg_out32(s, SUBFIC | TAI(a0, args[4], args[2]));
6c858762 3150 } else {
b31284ce 3151 tcg_out32(s, SUBFC | TAB(a0, args[4], args[2]));
6c858762 3152 }
b31284ce
RH		 3153 if (const_args[3]) {
3154 tcg_out32(s, (args[3] ? SUBFME : SUBFZE) | RT(a1) | RA(args[5]));
6c858762 3155 } else {
b31284ce 3156 tcg_out32(s, SUBFE | TAB(a1, args[5], args[3]));
6c858762
RH		 3157 }
3158 if (a0 != args[0]) {
de3d636d 3159 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
6c858762
RH		 3160 }
3161 break;
3162
abcf61c4
RH		 3163 case INDEX_op_muluh_i32:
3164 tcg_out32(s, MULHWU | TAB(args[0], args[1], args[2]));
3165 break;
8fa391a0
RH		 3166 case INDEX_op_mulsh_i32:
3167 tcg_out32(s, MULHW | TAB(args[0], args[1], args[2]));
3168 break;
32f5717f
RH		 3169 case INDEX_op_muluh_i64:
3170 tcg_out32(s, MULHDU | TAB(args[0], args[1], args[2]));
3171 break;
3172 case INDEX_op_mulsh_i64:
3173 tcg_out32(s, MULHD | TAB(args[0], args[1], args[2]));
6645c147
RH		 3174 break;
3175
7b4af5ee
PK		 3176 case INDEX_op_mb:
3177 tcg_out_mb(s, args[0]);
3178 break;
3179
96d0ee7f
RH		 3180 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
3181 case INDEX_op_mov_i64:
96d0ee7f 3182 case INDEX_op_call: /* Always emitted via tcg_out_call. */
810260a8 3183 default:
541dd4ce 3184 tcg_abort();
810260a8 3185 }
3186}
3187
4b06c216
RH		 3188int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
3189{
6ef14d7e
RH		 3190 switch (opc) {
3191 case INDEX_op_and_vec:
3192 case INDEX_op_or_vec:
3193 case INDEX_op_xor_vec:
3194 case INDEX_op_andc_vec:
3195 case INDEX_op_not_vec:
fa8e90d6
RH		 3196 case INDEX_op_nor_vec:
3197 case INDEX_op_eqv_vec:
3198 case INDEX_op_nand_vec:
6ef14d7e 3199 return 1;
64ff1c6d
RH		 3200 case INDEX_op_orc_vec:
3201 return have_isa_2_07;
d6750811
RH		 3202 case INDEX_op_add_vec:
3203 case INDEX_op_sub_vec:
e2382972
RH		 3204 case INDEX_op_smax_vec:
3205 case INDEX_op_smin_vec:
3206 case INDEX_op_umax_vec:
3207 case INDEX_op_umin_vec:
64ff1c6d
RH		 3208 case INDEX_op_shlv_vec:
3209 case INDEX_op_shrv_vec:
3210 case INDEX_op_sarv_vec:
ab87a66f 3211 case INDEX_op_rotlv_vec:
64ff1c6d 3212 return vece <= MO_32 || have_isa_2_07;
e9d1a53a
RH		 3213 case INDEX_op_ssadd_vec:
3214 case INDEX_op_sssub_vec:
3215 case INDEX_op_usadd_vec:
3216 case INDEX_op_ussub_vec:
e2382972 3217 return vece <= MO_32;
6ef14d7e 3218 case INDEX_op_cmp_vec:
dabae097
RH		 3219 case INDEX_op_shli_vec:
3220 case INDEX_op_shri_vec:
3221 case INDEX_op_sari_vec:
ab87a66f 3222 case INDEX_op_rotli_vec:
64ff1c6d 3223 return vece <= MO_32 || have_isa_2_07 ? -1 : 0;
d7cd6a2f
RH		 3224 case INDEX_op_neg_vec:
3225 return vece >= MO_32 && have_isa_3_00;
64ff1c6d
RH		 3226 case INDEX_op_mul_vec:
3227 switch (vece) {
3228 case MO_8:
3229 case MO_16:
3230 return -1;
3231 case MO_32:
3232 return have_isa_2_07 ? 1 : -1;
73ebe95e
LP		 3233 case MO_64:
3234 return have_isa_3_10;
64ff1c6d
RH		 3235 }
3236 return 0;
47c906ae
RH		 3237 case INDEX_op_bitsel_vec:
3238 return have_vsx;
ab87a66f
RH		 3239 case INDEX_op_rotrv_vec:
3240 return -1;
6ef14d7e
RH		 3241 default:
3242 return 0;
3243 }
4b06c216
RH		 3244}
3245
3246static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
3247 TCGReg dst, TCGReg src)
3248{
6ef14d7e 3249 tcg_debug_assert(dst >= TCG_REG_V0);
b7ce3cff
RH		 3250
3251 /* Splat from integer reg allowed via constraints for v3.00. */
3252 if (src < TCG_REG_V0) {
3253 tcg_debug_assert(have_isa_3_00);
3254 switch (vece) {
3255 case MO_64:
3256 tcg_out32(s, MTVSRDD | VRT(dst) | RA(src) | RB(src));
3257 return true;
3258 case MO_32:
3259 tcg_out32(s, MTVSRWS | VRT(dst) | RA(src));
3260 return true;
3261 default:
3262 /* Fail, so that we fall back on either dupm or mov+dup. */
3263 return false;
3264 }
3265 }
6ef14d7e
RH		 3266
3267 /*
3268 * Recall we use (or emulate) VSX integer loads, so the integer is
3269 * right justified within the left (zero-index) double-word.
3270 */
3271 switch (vece) {
3272 case MO_8:
3273 tcg_out32(s, VSPLTB | VRT(dst) | VRB(src) | (7 << 16));
3274 break;
3275 case MO_16:
3276 tcg_out32(s, VSPLTH | VRT(dst) | VRB(src) | (3 << 16));
3277 break;
3278 case MO_32:
3279 tcg_out32(s, VSPLTW | VRT(dst) | VRB(src) | (1 << 16));
3280 break;
3281 case MO_64:
47c906ae
RH		 3282 if (have_vsx) {
3283 tcg_out32(s, XXPERMDI | VRT(dst) | VRA(src) | VRB(src));
3284 break;
3285 }
6ef14d7e
RH		 3286 tcg_out_vsldoi(s, TCG_VEC_TMP1, src, src, 8);
3287 tcg_out_vsldoi(s, dst, TCG_VEC_TMP1, src, 8);
3288 break;
3289 default:
3290 g_assert_not_reached();
3291 }
3292 return true;
4b06c216
RH		 3293}
3294
3295static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
3296 TCGReg out, TCGReg base, intptr_t offset)
3297{
6ef14d7e
RH		 3298 int elt;
3299
3300 tcg_debug_assert(out >= TCG_REG_V0);
3301 switch (vece) {
3302 case MO_8:
6e11cde1
RH		 3303 if (have_isa_3_00) {
3304 tcg_out_mem_long(s, LXV, LVX, out, base, offset & -16);
3305 } else {
3306 tcg_out_mem_long(s, 0, LVEBX, out, base, offset);
3307 }
6ef14d7e 3308 elt = extract32(offset, 0, 4);
e03b5686 3309#if !HOST_BIG_ENDIAN
6ef14d7e
RH		 3310 elt ^= 15;
3311#endif
3312 tcg_out32(s, VSPLTB | VRT(out) | VRB(out) | (elt << 16));
3313 break;
3314 case MO_16:
3315 tcg_debug_assert((offset & 1) == 0);
6e11cde1
RH		 3316 if (have_isa_3_00) {
3317 tcg_out_mem_long(s, LXV | 8, LVX, out, base, offset & -16);
3318 } else {
3319 tcg_out_mem_long(s, 0, LVEHX, out, base, offset);
3320 }
6ef14d7e 3321 elt = extract32(offset, 1, 3);
e03b5686 3322#if !HOST_BIG_ENDIAN
6ef14d7e
RH		 3323 elt ^= 7;
3324#endif
3325 tcg_out32(s, VSPLTH | VRT(out) | VRB(out) | (elt << 16));
3326 break;
3327 case MO_32:
6e11cde1
RH		 3328 if (have_isa_3_00) {
3329 tcg_out_mem_long(s, 0, LXVWSX, out, base, offset);
3330 break;
3331 }
6ef14d7e
RH		 3332 tcg_debug_assert((offset & 3) == 0);
3333 tcg_out_mem_long(s, 0, LVEWX, out, base, offset);
3334 elt = extract32(offset, 2, 2);
e03b5686 3335#if !HOST_BIG_ENDIAN
6ef14d7e
RH		 3336 elt ^= 3;
3337#endif
3338 tcg_out32(s, VSPLTW | VRT(out) | VRB(out) | (elt << 16));
3339 break;
3340 case MO_64:
47c906ae
RH		 3341 if (have_vsx) {
3342 tcg_out_mem_long(s, 0, LXVDSX, out, base, offset);
3343 break;
3344 }
6ef14d7e
RH		 3345 tcg_debug_assert((offset & 7) == 0);
3346 tcg_out_mem_long(s, 0, LVX, out, base, offset & -16);
3347 tcg_out_vsldoi(s, TCG_VEC_TMP1, out, out, 8);
3348 elt = extract32(offset, 3, 1);
e03b5686 3349#if !HOST_BIG_ENDIAN
6ef14d7e
RH		 3350 elt = !elt;
3351#endif
3352 if (elt) {
3353 tcg_out_vsldoi(s, out, out, TCG_VEC_TMP1, 8);
3354 } else {
3355 tcg_out_vsldoi(s, out, TCG_VEC_TMP1, out, 8);
3356 }
3357 break;
3358 default:
3359 g_assert_not_reached();
3360 }
3361 return true;
4b06c216
RH		 3362}
3363
3364static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
3365 unsigned vecl, unsigned vece,
5e8892db
MR		 3366 const TCGArg args[TCG_MAX_OP_ARGS],
3367 const int const_args[TCG_MAX_OP_ARGS])
4b06c216 3368{
6ef14d7e 3369 static const uint32_t
64ff1c6d
RH		 3370 add_op[4] = { VADDUBM, VADDUHM, VADDUWM, VADDUDM },
3371 sub_op[4] = { VSUBUBM, VSUBUHM, VSUBUWM, VSUBUDM },
73ebe95e 3372 mul_op[4] = { 0, 0, VMULUWM, VMULLD },
d7cd6a2f 3373 neg_op[4] = { 0, 0, VNEGW, VNEGD },
64ff1c6d 3374 eq_op[4] = { VCMPEQUB, VCMPEQUH, VCMPEQUW, VCMPEQUD },
d7cd6a2f 3375 ne_op[4] = { VCMPNEB, VCMPNEH, VCMPNEW, 0 },
64ff1c6d
RH		 3376 gts_op[4] = { VCMPGTSB, VCMPGTSH, VCMPGTSW, VCMPGTSD },
3377 gtu_op[4] = { VCMPGTUB, VCMPGTUH, VCMPGTUW, VCMPGTUD },
e9d1a53a
RH		 3378 ssadd_op[4] = { VADDSBS, VADDSHS, VADDSWS, 0 },
3379 usadd_op[4] = { VADDUBS, VADDUHS, VADDUWS, 0 },
3380 sssub_op[4] = { VSUBSBS, VSUBSHS, VSUBSWS, 0 },
3381 ussub_op[4] = { VSUBUBS, VSUBUHS, VSUBUWS, 0 },
64ff1c6d
RH		 3382 umin_op[4] = { VMINUB, VMINUH, VMINUW, VMINUD },
3383 smin_op[4] = { VMINSB, VMINSH, VMINSW, VMINSD },
3384 umax_op[4] = { VMAXUB, VMAXUH, VMAXUW, VMAXUD },
3385 smax_op[4] = { VMAXSB, VMAXSH, VMAXSW, VMAXSD },
3386 shlv_op[4] = { VSLB, VSLH, VSLW, VSLD },
3387 shrv_op[4] = { VSRB, VSRH, VSRW, VSRD },
3388 sarv_op[4] = { VSRAB, VSRAH, VSRAW, VSRAD },
d9897efa
RH		 3389 mrgh_op[4] = { VMRGHB, VMRGHH, VMRGHW, 0 },
3390 mrgl_op[4] = { VMRGLB, VMRGLH, VMRGLW, 0 },
64ff1c6d
RH		 3391 muleu_op[4] = { VMULEUB, VMULEUH, VMULEUW, 0 },
3392 mulou_op[4] = { VMULOUB, VMULOUH, VMULOUW, 0 },
d9897efa 3393 pkum_op[4] = { VPKUHUM, VPKUWUM, 0, 0 },
64ff1c6d 3394 rotl_op[4] = { VRLB, VRLH, VRLW, VRLD };
6ef14d7e
RH		 3395
3396 TCGType type = vecl + TCG_TYPE_V64;
3397 TCGArg a0 = args[0], a1 = args[1], a2 = args[2];
3398 uint32_t insn;
3399
3400 switch (opc) {
3401 case INDEX_op_ld_vec:
3402 tcg_out_ld(s, type, a0, a1, a2);
3403 return;
3404 case INDEX_op_st_vec:
3405 tcg_out_st(s, type, a0, a1, a2);
3406 return;
3407 case INDEX_op_dupm_vec:
3408 tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
3409 return;
3410
d6750811
RH		 3411 case INDEX_op_add_vec:
3412 insn = add_op[vece];
3413 break;
3414 case INDEX_op_sub_vec:
3415 insn = sub_op[vece];
3416 break;
d7cd6a2f
RH		 3417 case INDEX_op_neg_vec:
3418 insn = neg_op[vece];
3419 a2 = a1;
3420 a1 = 0;
3421 break;
64ff1c6d 3422 case INDEX_op_mul_vec:
73ebe95e 3423 insn = mul_op[vece];
64ff1c6d 3424 break;
e9d1a53a
RH		 3425 case INDEX_op_ssadd_vec:
3426 insn = ssadd_op[vece];
3427 break;
3428 case INDEX_op_sssub_vec:
3429 insn = sssub_op[vece];
3430 break;
3431 case INDEX_op_usadd_vec:
3432 insn = usadd_op[vece];
3433 break;
3434 case INDEX_op_ussub_vec:
3435 insn = ussub_op[vece];
3436 break;
e2382972
RH		 3437 case INDEX_op_smin_vec:
3438 insn = smin_op[vece];
3439 break;
3440 case INDEX_op_umin_vec:
3441 insn = umin_op[vece];
3442 break;
3443 case INDEX_op_smax_vec:
3444 insn = smax_op[vece];
3445 break;
3446 case INDEX_op_umax_vec:
3447 insn = umax_op[vece];
3448 break;
dabae097
RH		 3449 case INDEX_op_shlv_vec:
3450 insn = shlv_op[vece];
3451 break;
3452 case INDEX_op_shrv_vec:
3453 insn = shrv_op[vece];
3454 break;
3455 case INDEX_op_sarv_vec:
3456 insn = sarv_op[vece];
3457 break;
6ef14d7e
RH		 3458 case INDEX_op_and_vec:
3459 insn = VAND;
3460 break;
3461 case INDEX_op_or_vec:
3462 insn = VOR;
3463 break;
3464 case INDEX_op_xor_vec:
3465 insn = VXOR;
3466 break;
3467 case INDEX_op_andc_vec:
3468 insn = VANDC;
3469 break;
3470 case INDEX_op_not_vec:
3471 insn = VNOR;
3472 a2 = a1;
3473 break;
64ff1c6d
RH		 3474 case INDEX_op_orc_vec:
3475 insn = VORC;
3476 break;
fa8e90d6
RH		 3477 case INDEX_op_nand_vec:
3478 insn = VNAND;
3479 break;
3480 case INDEX_op_nor_vec:
3481 insn = VNOR;
3482 break;
3483 case INDEX_op_eqv_vec:
3484 insn = VEQV;
3485 break;
6ef14d7e
RH		 3486
3487 case INDEX_op_cmp_vec:
3488 switch (args[3]) {
3489 case TCG_COND_EQ:
3490 insn = eq_op[vece];
3491 break;
d7cd6a2f
RH		 3492 case TCG_COND_NE:
3493 insn = ne_op[vece];
3494 break;
6ef14d7e
RH		 3495 case TCG_COND_GT:
3496 insn = gts_op[vece];
3497 break;
3498 case TCG_COND_GTU:
3499 insn = gtu_op[vece];
3500 break;
3501 default:
3502 g_assert_not_reached();
3503 }
3504 break;
3505
47c906ae
RH		 3506 case INDEX_op_bitsel_vec:
3507 tcg_out32(s, XXSEL | VRT(a0) | VRC(a1) | VRB(a2) | VRA(args[3]));
3508 return;
3509
597cf978
RH		 3510 case INDEX_op_dup2_vec:
3511 assert(TCG_TARGET_REG_BITS == 32);
3512 /* With inputs a1 = xLxx, a2 = xHxx */
3513 tcg_out32(s, VMRGHW | VRT(a0) | VRA(a2) | VRB(a1)); /* a0 = xxHL */
3514 tcg_out_vsldoi(s, TCG_VEC_TMP1, a0, a0, 8); /* tmp = HLxx */
3515 tcg_out_vsldoi(s, a0, a0, TCG_VEC_TMP1, 8); /* a0 = HLHL */
3516 return;
3517
d9897efa
RH		 3518 case INDEX_op_ppc_mrgh_vec:
3519 insn = mrgh_op[vece];
3520 break;
3521 case INDEX_op_ppc_mrgl_vec:
3522 insn = mrgl_op[vece];
3523 break;
3524 case INDEX_op_ppc_muleu_vec:
3525 insn = muleu_op[vece];
3526 break;
3527 case INDEX_op_ppc_mulou_vec:
3528 insn = mulou_op[vece];
3529 break;
3530 case INDEX_op_ppc_pkum_vec:
3531 insn = pkum_op[vece];
3532 break;
ab87a66f 3533 case INDEX_op_rotlv_vec:
d9897efa
RH		 3534 insn = rotl_op[vece];
3535 break;
3536 case INDEX_op_ppc_msum_vec:
3537 tcg_debug_assert(vece == MO_16);
3538 tcg_out32(s, VMSUMUHM | VRT(a0) | VRA(a1) | VRB(a2) | VRC(args[3]));
3539 return;
3540
6ef14d7e 3541 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
6ef14d7e
RH		 3542 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
3543 default:
3544 g_assert_not_reached();
3545 }
3546
3547 tcg_debug_assert(insn != 0);
3548 tcg_out32(s, insn | VRT(a0) | VRA(a1) | VRB(a2));
3549}
3550
dabae097
RH		 3551static void expand_vec_shi(TCGType type, unsigned vece, TCGv_vec v0,
3552 TCGv_vec v1, TCGArg imm, TCGOpcode opci)
3553{
44aa59a0
RH		 3554 TCGv_vec t1;
3555
3556 if (vece == MO_32) {
3557 /*
3558 * Only 5 bits are significant, and VSPLTISB can represent -16..15.
3559 * So using negative numbers gets us the 4th bit easily.
3560 */
3561 imm = sextract32(imm, 0, 5);
3562 } else {
3563 imm &= (8 << vece) - 1;
3564 }
dabae097 3565
44aa59a0
RH		 3566 /* Splat w/bytes for xxspltib when 2.07 allows MO_64. */
3567 t1 = tcg_constant_vec(type, MO_8, imm);
dabae097
RH		 3568 vec_gen_3(opci, type, vece, tcgv_vec_arg(v0),
3569 tcgv_vec_arg(v1), tcgv_vec_arg(t1));
dabae097
RH		 3570}
3571
6ef14d7e
RH		 3572static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0,
3573 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
3574{
3575 bool need_swap = false, need_inv = false;
3576
64ff1c6d 3577 tcg_debug_assert(vece <= MO_32 || have_isa_2_07);
6ef14d7e
RH		 3578
3579 switch (cond) {
3580 case TCG_COND_EQ:
3581 case TCG_COND_GT:
3582 case TCG_COND_GTU:
3583 break;
3584 case TCG_COND_NE:
d7cd6a2f
RH		 3585 if (have_isa_3_00 && vece <= MO_32) {
3586 break;
3587 }
3588 /* fall through */
6ef14d7e
RH		 3589 case TCG_COND_LE:
3590 case TCG_COND_LEU:
3591 need_inv = true;
3592 break;
3593 case TCG_COND_LT:
3594 case TCG_COND_LTU:
3595 need_swap = true;
3596 break;
3597 case TCG_COND_GE:
3598 case TCG_COND_GEU:
3599 need_swap = need_inv = true;
3600 break;
3601 default:
3602 g_assert_not_reached();
3603 }
3604
3605 if (need_inv) {
3606 cond = tcg_invert_cond(cond);
3607 }
3608 if (need_swap) {
3609 TCGv_vec t1;
3610 t1 = v1, v1 = v2, v2 = t1;
3611 cond = tcg_swap_cond(cond);
3612 }
3613
3614 vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0),
3615 tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond);
3616
3617 if (need_inv) {
3618 tcg_gen_not_vec(vece, v0, v0);
3619 }
4b06c216
RH		 3620}
3621
d9897efa
RH		 3622static void expand_vec_mul(TCGType type, unsigned vece, TCGv_vec v0,
3623 TCGv_vec v1, TCGv_vec v2)
3624{
3625 TCGv_vec t1 = tcg_temp_new_vec(type);
3626 TCGv_vec t2 = tcg_temp_new_vec(type);
44aa59a0 3627 TCGv_vec c0, c16;
d9897efa
RH		 3628
3629 switch (vece) {
3630 case MO_8:
3631 case MO_16:
3632 vec_gen_3(INDEX_op_ppc_muleu_vec, type, vece, tcgv_vec_arg(t1),
3633 tcgv_vec_arg(v1), tcgv_vec_arg(v2));
3634 vec_gen_3(INDEX_op_ppc_mulou_vec, type, vece, tcgv_vec_arg(t2),
3635 tcgv_vec_arg(v1), tcgv_vec_arg(v2));
3636 vec_gen_3(INDEX_op_ppc_mrgh_vec, type, vece + 1, tcgv_vec_arg(v0),
3637 tcgv_vec_arg(t1), tcgv_vec_arg(t2));
3638 vec_gen_3(INDEX_op_ppc_mrgl_vec, type, vece + 1, tcgv_vec_arg(t1),
3639 tcgv_vec_arg(t1), tcgv_vec_arg(t2));
3640 vec_gen_3(INDEX_op_ppc_pkum_vec, type, vece, tcgv_vec_arg(v0),
3641 tcgv_vec_arg(v0), tcgv_vec_arg(t1));
3642 break;
3643
3644 case MO_32:
64ff1c6d 3645 tcg_debug_assert(!have_isa_2_07);
44aa59a0
RH		 3646 /*
3647 * Only 5 bits are significant, and VSPLTISB can represent -16..15.
3648 * So using -16 is a quick way to represent 16.
3649 */
3650 c16 = tcg_constant_vec(type, MO_8, -16);
3651 c0 = tcg_constant_vec(type, MO_8, 0);
3652
ab87a66f 3653 vec_gen_3(INDEX_op_rotlv_vec, type, MO_32, tcgv_vec_arg(t1),
44aa59a0 3654 tcgv_vec_arg(v2), tcgv_vec_arg(c16));
d9897efa
RH		 3655 vec_gen_3(INDEX_op_ppc_mulou_vec, type, MO_16, tcgv_vec_arg(t2),
3656 tcgv_vec_arg(v1), tcgv_vec_arg(v2));
44aa59a0
RH		 3657 vec_gen_4(INDEX_op_ppc_msum_vec, type, MO_16, tcgv_vec_arg(t1),
3658 tcgv_vec_arg(v1), tcgv_vec_arg(t1), tcgv_vec_arg(c0));
3659 vec_gen_3(INDEX_op_shlv_vec, type, MO_32, tcgv_vec_arg(t1),
3660 tcgv_vec_arg(t1), tcgv_vec_arg(c16));
3661 tcg_gen_add_vec(MO_32, v0, t1, t2);
d9897efa
RH		 3662 break;
3663
3664 default:
3665 g_assert_not_reached();
3666 }
3667 tcg_temp_free_vec(t1);
3668 tcg_temp_free_vec(t2);
3669}
3670
4b06c216
RH		 3671void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
3672 TCGArg a0, ...)
3673{
6ef14d7e 3674 va_list va;
ab87a66f 3675 TCGv_vec v0, v1, v2, t0;
dabae097 3676 TCGArg a2;
6ef14d7e
RH		 3677
3678 va_start(va, a0);
3679 v0 = temp_tcgv_vec(arg_temp(a0));
3680 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
dabae097 3681 a2 = va_arg(va, TCGArg);
6ef14d7e
RH		 3682
3683 switch (opc) {
dabae097
RH		 3684 case INDEX_op_shli_vec:
3685 expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_shlv_vec);
3686 break;
3687 case INDEX_op_shri_vec:
3688 expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_shrv_vec);
3689 break;
3690 case INDEX_op_sari_vec:
3691 expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_sarv_vec);
3692 break;
ab87a66f
RH		 3693 case INDEX_op_rotli_vec:
3694 expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_rotlv_vec);
3695 break;
6ef14d7e 3696 case INDEX_op_cmp_vec:
dabae097 3697 v2 = temp_tcgv_vec(arg_temp(a2));
6ef14d7e
RH		 3698 expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg));
3699 break;
d9897efa
RH		 3700 case INDEX_op_mul_vec:
3701 v2 = temp_tcgv_vec(arg_temp(a2));
3702 expand_vec_mul(type, vece, v0, v1, v2);
3703 break;
ab87a66f
RH		 3704 case INDEX_op_rotlv_vec:
3705 v2 = temp_tcgv_vec(arg_temp(a2));
3706 t0 = tcg_temp_new_vec(type);
3707 tcg_gen_neg_vec(vece, t0, v2);
3708 tcg_gen_rotlv_vec(vece, v0, v1, t0);
3709 tcg_temp_free_vec(t0);
3710 break;
6ef14d7e
RH		 3711 default:
3712 g_assert_not_reached();
3713 }
3714 va_end(va);
4b06c216
RH		 3715}
3716
6893016b
RH		 3717static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op)
3718{
6cb3658a
RH		 3719 switch (op) {
3720 case INDEX_op_goto_ptr:
6893016b 3721 return C_O0_I1(r);
796f1a68 3722
6cb3658a
RH		 3723 case INDEX_op_ld8u_i32:
3724 case INDEX_op_ld8s_i32:
3725 case INDEX_op_ld16u_i32:
3726 case INDEX_op_ld16s_i32:
3727 case INDEX_op_ld_i32:
6cb3658a
RH		 3728 case INDEX_op_ctpop_i32:
3729 case INDEX_op_neg_i32:
3730 case INDEX_op_not_i32:
3731 case INDEX_op_ext8s_i32:
3732 case INDEX_op_ext16s_i32:
3733 case INDEX_op_bswap16_i32:
3734 case INDEX_op_bswap32_i32:
3735 case INDEX_op_extract_i32:
3736 case INDEX_op_ld8u_i64:
3737 case INDEX_op_ld8s_i64:
3738 case INDEX_op_ld16u_i64:
3739 case INDEX_op_ld16s_i64:
3740 case INDEX_op_ld32u_i64:
3741 case INDEX_op_ld32s_i64:
3742 case INDEX_op_ld_i64:
6cb3658a
RH		 3743 case INDEX_op_ctpop_i64:
3744 case INDEX_op_neg_i64:
3745 case INDEX_op_not_i64:
3746 case INDEX_op_ext8s_i64:
3747 case INDEX_op_ext16s_i64:
3748 case INDEX_op_ext32s_i64:
3749 case INDEX_op_ext_i32_i64:
3750 case INDEX_op_extu_i32_i64:
3751 case INDEX_op_bswap16_i64:
3752 case INDEX_op_bswap32_i64:
3753 case INDEX_op_bswap64_i64:
3754 case INDEX_op_extract_i64:
6893016b
RH		 3755 return C_O1_I1(r, r);
3756
3757 case INDEX_op_st8_i32:
3758 case INDEX_op_st16_i32:
3759 case INDEX_op_st_i32:
3760 case INDEX_op_st8_i64:
3761 case INDEX_op_st16_i64:
3762 case INDEX_op_st32_i64:
3763 case INDEX_op_st_i64:
3764 return C_O0_I2(r, r);
abcf61c4 3765
6cb3658a
RH		 3766 case INDEX_op_add_i32:
3767 case INDEX_op_and_i32:
3768 case INDEX_op_or_i32:
3769 case INDEX_op_xor_i32:
3770 case INDEX_op_andc_i32:
3771 case INDEX_op_orc_i32:
3772 case INDEX_op_eqv_i32:
3773 case INDEX_op_shl_i32:
3774 case INDEX_op_shr_i32:
3775 case INDEX_op_sar_i32:
3776 case INDEX_op_rotl_i32:
3777 case INDEX_op_rotr_i32:
3778 case INDEX_op_setcond_i32:
3779 case INDEX_op_and_i64:
3780 case INDEX_op_andc_i64:
3781 case INDEX_op_shl_i64:
3782 case INDEX_op_shr_i64:
3783 case INDEX_op_sar_i64:
3784 case INDEX_op_rotl_i64:
3785 case INDEX_op_rotr_i64:
3786 case INDEX_op_setcond_i64:
6893016b
RH		 3787 return C_O1_I2(r, r, ri);
3788
6cb3658a
RH		 3789 case INDEX_op_mul_i32:
3790 case INDEX_op_mul_i64:
6893016b
RH		 3791 return C_O1_I2(r, r, rI);
3792
6cb3658a
RH		 3793 case INDEX_op_div_i32:
3794 case INDEX_op_divu_i32:
4d573822
MKF		 3795 case INDEX_op_rem_i32:
3796 case INDEX_op_remu_i32:
6cb3658a
RH		 3797 case INDEX_op_nand_i32:
3798 case INDEX_op_nor_i32:
3799 case INDEX_op_muluh_i32:
3800 case INDEX_op_mulsh_i32:
3801 case INDEX_op_orc_i64:
3802 case INDEX_op_eqv_i64:
3803 case INDEX_op_nand_i64:
3804 case INDEX_op_nor_i64:
3805 case INDEX_op_div_i64:
3806 case INDEX_op_divu_i64:
4d573822
MKF		 3807 case INDEX_op_rem_i64:
3808 case INDEX_op_remu_i64:
6cb3658a
RH		 3809 case INDEX_op_mulsh_i64:
3810 case INDEX_op_muluh_i64:
6893016b
RH		 3811 return C_O1_I2(r, r, r);
3812
6cb3658a 3813 case INDEX_op_sub_i32:
6893016b 3814 return C_O1_I2(r, rI, ri);
6cb3658a 3815 case INDEX_op_add_i64:
6893016b 3816 return C_O1_I2(r, r, rT);
6cb3658a
RH		 3817 case INDEX_op_or_i64:
3818 case INDEX_op_xor_i64:
6893016b 3819 return C_O1_I2(r, r, rU);
6cb3658a 3820 case INDEX_op_sub_i64:
6893016b 3821 return C_O1_I2(r, rI, rT);
6cb3658a
RH		 3822 case INDEX_op_clz_i32:
3823 case INDEX_op_ctz_i32:
3824 case INDEX_op_clz_i64:
3825 case INDEX_op_ctz_i64:
6893016b 3826 return C_O1_I2(r, r, rZW);
796f1a68 3827
6cb3658a
RH		 3828 case INDEX_op_brcond_i32:
3829 case INDEX_op_brcond_i64:
6893016b 3830 return C_O0_I2(r, ri);
6c858762 3831
6cb3658a
RH		 3832 case INDEX_op_movcond_i32:
3833 case INDEX_op_movcond_i64:
6893016b 3834 return C_O1_I4(r, r, ri, rZ, rZ);
6cb3658a
RH		 3835 case INDEX_op_deposit_i32:
3836 case INDEX_op_deposit_i64:
6893016b 3837 return C_O1_I2(r, 0, rZ);
6cb3658a 3838 case INDEX_op_brcond2_i32:
6893016b 3839 return C_O0_I4(r, r, ri, ri);
6cb3658a 3840 case INDEX_op_setcond2_i32:
6893016b 3841 return C_O1_I4(r, r, r, ri, ri);
6cb3658a
RH		 3842 case INDEX_op_add2_i64:
3843 case INDEX_op_add2_i32:
6893016b 3844 return C_O2_I4(r, r, r, r, rI, rZM);
6cb3658a
RH		 3845 case INDEX_op_sub2_i64:
3846 case INDEX_op_sub2_i32:
6893016b 3847 return C_O2_I4(r, r, rI, rZM, r, r);
810260a8 3848
6cb3658a
RH		 3849 case INDEX_op_qemu_ld_i32:
3850 return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32
6893016b
RH		 3851 ? C_O1_I1(r, L)
3852 : C_O1_I2(r, L, L));
3853
6cb3658a
RH		 3854 case INDEX_op_qemu_st_i32:
3855 return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32
6893016b
RH		 3856 ? C_O0_I2(S, S)
3857 : C_O0_I3(S, S, S));
3858
6cb3658a 3859 case INDEX_op_qemu_ld_i64:
6893016b
RH		 3860 return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L)
3861 : TARGET_LONG_BITS == 32 ? C_O2_I1(L, L, L)
3862 : C_O2_I2(L, L, L, L));
3863
6cb3658a 3864 case INDEX_op_qemu_st_i64:
6893016b
RH		 3865 return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(S, S)
3866 : TARGET_LONG_BITS == 32 ? C_O0_I3(S, S, S)
3867 : C_O0_I4(S, S, S, S));
f69d277e 3868
d6750811
RH		 3869 case INDEX_op_add_vec:
3870 case INDEX_op_sub_vec:
d9897efa 3871 case INDEX_op_mul_vec:
6ef14d7e
RH		 3872 case INDEX_op_and_vec:
3873 case INDEX_op_or_vec:
3874 case INDEX_op_xor_vec:
3875 case INDEX_op_andc_vec:
3876 case INDEX_op_orc_vec:
fa8e90d6
RH		 3877 case INDEX_op_nor_vec:
3878 case INDEX_op_eqv_vec:
3879 case INDEX_op_nand_vec:
6ef14d7e 3880 case INDEX_op_cmp_vec:
e9d1a53a
RH		 3881 case INDEX_op_ssadd_vec:
3882 case INDEX_op_sssub_vec:
3883 case INDEX_op_usadd_vec:
3884 case INDEX_op_ussub_vec:
e2382972
RH		 3885 case INDEX_op_smax_vec:
3886 case INDEX_op_smin_vec:
3887 case INDEX_op_umax_vec:
3888 case INDEX_op_umin_vec:
dabae097
RH		 3889 case INDEX_op_shlv_vec:
3890 case INDEX_op_shrv_vec:
3891 case INDEX_op_sarv_vec:
ab87a66f
RH		 3892 case INDEX_op_rotlv_vec:
3893 case INDEX_op_rotrv_vec:
d9897efa
RH		 3894 case INDEX_op_ppc_mrgh_vec:
3895 case INDEX_op_ppc_mrgl_vec:
3896 case INDEX_op_ppc_muleu_vec:
3897 case INDEX_op_ppc_mulou_vec:
3898 case INDEX_op_ppc_pkum_vec:
597cf978 3899 case INDEX_op_dup2_vec:
6893016b
RH		 3900 return C_O1_I2(v, v, v);
3901
6ef14d7e 3902 case INDEX_op_not_vec:
d7cd6a2f 3903 case INDEX_op_neg_vec:
6893016b
RH		 3904 return C_O1_I1(v, v);
3905
b7ce3cff 3906 case INDEX_op_dup_vec:
6893016b
RH		 3907 return have_isa_3_00 ? C_O1_I1(v, vr) : C_O1_I1(v, v);
3908
6ef14d7e 3909 case INDEX_op_ld_vec:
6ef14d7e 3910 case INDEX_op_dupm_vec:
6893016b
RH		 3911 return C_O1_I1(v, r);
3912
3913 case INDEX_op_st_vec:
3914 return C_O0_I2(v, r);
3915
47c906ae 3916 case INDEX_op_bitsel_vec:
d9897efa 3917 case INDEX_op_ppc_msum_vec:
6893016b 3918 return C_O1_I3(v, v, v, v);
6ef14d7e 3919
6cb3658a 3920 default:
6893016b 3921 g_assert_not_reached();
f69d277e 3922 }
f69d277e
RH		 3923}
3924
541dd4ce 3925static void tcg_target_init(TCGContext *s)
810260a8 3926{
cd629de1 3927 unsigned long hwcap = qemu_getauxval(AT_HWCAP);
d0b07481
RH		 3928 unsigned long hwcap2 = qemu_getauxval(AT_HWCAP2);
3929
7d9dae0a 3930 have_isa = tcg_isa_base;
1e6e9aca 3931 if (hwcap & PPC_FEATURE_ARCH_2_06) {
7d9dae0a 3932 have_isa = tcg_isa_2_06;
1e6e9aca 3933 }
64ff1c6d
RH		 3934#ifdef PPC_FEATURE2_ARCH_2_07
3935 if (hwcap2 & PPC_FEATURE2_ARCH_2_07) {
3936 have_isa = tcg_isa_2_07;
3937 }
3938#endif
d0b07481
RH		 3939#ifdef PPC_FEATURE2_ARCH_3_00
3940 if (hwcap2 & PPC_FEATURE2_ARCH_3_00) {
7d9dae0a 3941 have_isa = tcg_isa_3_00;
d0b07481
RH		 3942 }
3943#endif
73ebe95e
LP		 3944#ifdef PPC_FEATURE2_ARCH_3_10
3945 if (hwcap2 & PPC_FEATURE2_ARCH_3_10) {
3946 have_isa = tcg_isa_3_10;
3947 }
3948#endif
1e6e9aca 3949
63922f46
RH		 3950#ifdef PPC_FEATURE2_HAS_ISEL
3951 /* Prefer explicit instruction from the kernel. */
3952 have_isel = (hwcap2 & PPC_FEATURE2_HAS_ISEL) != 0;
3953#else
3954 /* Fall back to knowing Power7 (2.06) has ISEL. */
3955 have_isel = have_isa_2_06;
3956#endif
3957
68f340d4
RH		 3958 if (hwcap & PPC_FEATURE_HAS_ALTIVEC) {
3959 have_altivec = true;
47c906ae
RH		 3960 /* We only care about the portion of VSX that overlaps Altivec. */
3961 if (hwcap & PPC_FEATURE_HAS_VSX) {
3962 have_vsx = true;
3963 }
68f340d4
RH		 3964 }
3965
f46934df
RH		 3966 tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff;
3967 tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff;
4b06c216
RH		 3968 if (have_altivec) {
3969 tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
3970 tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
3971 }
f46934df
RH		 3972
3973 tcg_target_call_clobber_regs = 0;
3974 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
3975 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
3976 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
3977 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
3978 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
3979 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
3980 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R7);
3981 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
3982 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
3983 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
3984 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
3985 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R12);
810260a8 3986
42281ec6
RH		 3987 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V0);
3988 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V1);
3989 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V2);
3990 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V3);
3991 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V4);
3992 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V5);
3993 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V6);
3994 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V7);
3995 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V8);
3996 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V9);
3997 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V10);
3998 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V11);
3999 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V12);
4000 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V13);
4001 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V14);
4002 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V15);
4003 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V16);
4004 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V17);
4005 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V18);
4006 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V19);
4007
ccb1bb66 4008 s->reserved_regs = 0;
5e1702b0
RH		 4009 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0); /* tcg temp */
4010 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1); /* stack pointer */
dfca1778
RH		 4011#if defined(_CALL_SYSV)
4012 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2); /* toc pointer */
5d7ff5bb 4013#endif
dfca1778 4014#if defined(_CALL_SYSV) || TCG_TARGET_REG_BITS == 64
5e1702b0 4015 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13); /* thread pointer */
dfca1778
RH		 4016#endif
4017 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1); /* mem temp */
42281ec6
RH		 4018 tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP1);
4019 tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP2);
5964fca8
RH		 4020 if (USE_REG_TB) {
4021 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB); /* tb->tc_ptr */
a84ac4cb 4022 }
810260a8 4023}
fa94c3be 4024
ffcfbece 4025#ifdef __ELF__
fa94c3be
RH		 4026typedef struct {
4027 DebugFrameCIE cie;
4028 DebugFrameFDEHeader fde;
4029 uint8_t fde_def_cfa[4];
4030 uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2 + 3];
4031} DebugFrame;
4032
4033/* We're expecting a 2 byte uleb128 encoded value. */
4034QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
4035
ffcfbece
RH		 4036#if TCG_TARGET_REG_BITS == 64
4037# define ELF_HOST_MACHINE EM_PPC64
4038#else
4039# define ELF_HOST_MACHINE EM_PPC
4040#endif
fa94c3be
RH		 4041
4042static DebugFrame debug_frame = {
4043 .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
4044 .cie.id = -1,
4045 .cie.version = 1,
4046 .cie.code_align = 1,
802ca56e 4047 .cie.data_align = (-SZR & 0x7f), /* sleb128 -SZR */
fa94c3be
RH		 4048 .cie.return_column = 65,
4049
4050 /* Total FDE size does not include the "len" member. */
4051 .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
4052
4053 .fde_def_cfa = {
802ca56e 4054 12, TCG_REG_R1, /* DW_CFA_def_cfa r1, ... */
fa94c3be
RH		 4055 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
4056 (FRAME_SIZE >> 7)
4057 },
4058 .fde_reg_ofs = {
802ca56e
RH		 4059 /* DW_CFA_offset_extended_sf, lr, LR_OFFSET */
4060 0x11, 65, (LR_OFFSET / -SZR) & 0x7f,
fa94c3be
RH		 4061 }
4062};
4063
755bf9e5 4064void tcg_register_jit(const void *buf, size_t buf_size)
fa94c3be
RH		 4065{
4066 uint8_t *p = &debug_frame.fde_reg_ofs[3];
4067 int i;
4068
4069 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i, p += 2) {
4070 p[0] = 0x80 + tcg_target_callee_save_regs[i];
802ca56e 4071 p[1] = (FRAME_SIZE - (REG_SAVE_BOT + i * SZR)) / SZR;
fa94c3be
RH		 4072 }
4073
802ca56e 4074 debug_frame.fde.func_start = (uintptr_t)buf;
fa94c3be
RH		 4075 debug_frame.fde.func_len = buf_size;
4076
4077 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
4078}
ffcfbece 4079#endif /* __ELF__ */
80eca687
LMC		 4080#undef VMULEUB
4081#undef VMULEUH
4082#undef VMULEUW
4083#undef VMULOUB
4084#undef VMULOUH
4085#undef VMULOUW
89a5a1ae 4086#undef VMSUMUHM
QEMU mirror