]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/s390x/tcg-target.c.inc
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg/s390x: Check for general-instruction-extension facility at startup
[mirror_qemu.git] / tcg / s390x / tcg-target.c.inc
CommitLineData
2827822e
AG		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2009 Ulrich Hecht <uli@suse.de>
48bb3750
RH		 5 * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
6 * Copyright (c) 2010 Richard Henderson <rth@twiddle.net>
2827822e
AG		 7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a copy
9 * of this software and associated documentation files (the "Software"), to deal
10 * in the Software without restriction, including without limitation the rights
11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 * copies of the Software, and to permit persons to whom the Software is
13 * furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 * THE SOFTWARE.
25 */
26
a01fc30d
RH		 27/* We only support generating code for 64-bit mode. */
28#if TCG_TARGET_REG_BITS != 64
29#error "unsupported code generation mode"
30#endif
31
1cd49868 32#include "../tcg-ldst.c.inc"
139c1837 33#include "../tcg-pool.c.inc"
c9baa30f
RH		 34#include "elf.h"
35
a8f0269e
RH		 36#define TCG_CT_CONST_S16 0x100
37#define TCG_CT_CONST_S32 0x200
a534bb15
RH		 38#define TCG_CT_CONST_S33 0x400
39#define TCG_CT_CONST_ZERO 0x800
48bb3750 40
c947deb1 41#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16)
34ef7676
RH		 42#define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32)
43
c947deb1
RH		 44/*
45 * For softmmu, we need to avoid conflicts with the first 3
46 * argument registers to perform the tlb lookup, and to call
47 * the helper function.
48 */
49#ifdef CONFIG_SOFTMMU
50#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_R2, 3)
51#else
52#define SOFTMMU_RESERVE_REGS 0
53#endif
54
55
48bb3750
RH		 56/* Several places within the instruction set 0 means "no register"
57 rather than TCG_REG_R0. */
58#define TCG_REG_NONE 0
59
60/* A scratch register that may be be used throughout the backend. */
ce411066 61#define TCG_TMP0 TCG_REG_R1
48bb3750 62
4cbea598 63#ifndef CONFIG_SOFTMMU
48bb3750 64#define TCG_GUEST_BASE_REG TCG_REG_R13
48bb3750
RH		 65#endif
66
48bb3750
RH		 67/* All of the following instructions are prefixed with their instruction
68 format, and are defined as 8- or 16-bit quantities, even when the two
69 halves of the 16-bit quantity may appear 32 bits apart in the insn.
70 This makes it easy to copy the values from the tables in Appendix B. */
71typedef enum S390Opcode {
72 RIL_AFI = 0xc209,
73 RIL_AGFI = 0xc208,
3790b918 74 RIL_ALFI = 0xc20b,
48bb3750
RH		 75 RIL_ALGFI = 0xc20a,
76 RIL_BRASL = 0xc005,
77 RIL_BRCL = 0xc004,
78 RIL_CFI = 0xc20d,
79 RIL_CGFI = 0xc20c,
80 RIL_CLFI = 0xc20f,
81 RIL_CLGFI = 0xc20e,
a534bb15
RH		 82 RIL_CLRL = 0xc60f,
83 RIL_CLGRL = 0xc60a,
84 RIL_CRL = 0xc60d,
85 RIL_CGRL = 0xc608,
48bb3750
RH		 86 RIL_IIHF = 0xc008,
87 RIL_IILF = 0xc009,
88 RIL_LARL = 0xc000,
89 RIL_LGFI = 0xc001,
90 RIL_LGRL = 0xc408,
91 RIL_LLIHF = 0xc00e,
92 RIL_LLILF = 0xc00f,
93 RIL_LRL = 0xc40d,
94 RIL_MSFI = 0xc201,
95 RIL_MSGFI = 0xc200,
96 RIL_NIHF = 0xc00a,
97 RIL_NILF = 0xc00b,
98 RIL_OIHF = 0xc00c,
99 RIL_OILF = 0xc00d,
3790b918 100 RIL_SLFI = 0xc205,
0db921e6 101 RIL_SLGFI = 0xc204,
48bb3750
RH		 102 RIL_XIHF = 0xc006,
103 RIL_XILF = 0xc007,
104
105 RI_AGHI = 0xa70b,
106 RI_AHI = 0xa70a,
107 RI_BRC = 0xa704,
a534bb15
RH		 108 RI_CHI = 0xa70e,
109 RI_CGHI = 0xa70f,
48bb3750
RH		 110 RI_IIHH = 0xa500,
111 RI_IIHL = 0xa501,
112 RI_IILH = 0xa502,
113 RI_IILL = 0xa503,
114 RI_LGHI = 0xa709,
115 RI_LLIHH = 0xa50c,
116 RI_LLIHL = 0xa50d,
117 RI_LLILH = 0xa50e,
118 RI_LLILL = 0xa50f,
119 RI_MGHI = 0xa70d,
120 RI_MHI = 0xa70c,
121 RI_NIHH = 0xa504,
122 RI_NIHL = 0xa505,
123 RI_NILH = 0xa506,
124 RI_NILL = 0xa507,
125 RI_OIHH = 0xa508,
126 RI_OIHL = 0xa509,
127 RI_OILH = 0xa50a,
128 RI_OILL = 0xa50b,
1cd49868 129 RI_TMLL = 0xa701,
48bb3750
RH		 130
131 RIE_CGIJ = 0xec7c,
132 RIE_CGRJ = 0xec64,
133 RIE_CIJ = 0xec7e,
134 RIE_CLGRJ = 0xec65,
135 RIE_CLIJ = 0xec7f,
136 RIE_CLGIJ = 0xec7d,
137 RIE_CLRJ = 0xec77,
138 RIE_CRJ = 0xec76,
7af525af 139 RIE_LOCGHI = 0xec46,
d5690ea4 140 RIE_RISBG = 0xec55,
48bb3750
RH		 141
142 RRE_AGR = 0xb908,
3790b918
RH		 143 RRE_ALGR = 0xb90a,
144 RRE_ALCR = 0xb998,
145 RRE_ALCGR = 0xb988,
48bb3750
RH		 146 RRE_CGR = 0xb920,
147 RRE_CLGR = 0xb921,
148 RRE_DLGR = 0xb987,
149 RRE_DLR = 0xb997,
150 RRE_DSGFR = 0xb91d,
151 RRE_DSGR = 0xb90d,
ce411066 152 RRE_FLOGR = 0xb983,
48bb3750
RH		 153 RRE_LGBR = 0xb906,
154 RRE_LCGR = 0xb903,
155 RRE_LGFR = 0xb914,
156 RRE_LGHR = 0xb907,
157 RRE_LGR = 0xb904,
158 RRE_LLGCR = 0xb984,
159 RRE_LLGFR = 0xb916,
160 RRE_LLGHR = 0xb985,
161 RRE_LRVR = 0xb91f,
162 RRE_LRVGR = 0xb90f,
163 RRE_LTGR = 0xb902,
36017dc6 164 RRE_MLGR = 0xb986,
48bb3750
RH		 165 RRE_MSGR = 0xb90c,
166 RRE_MSR = 0xb252,
167 RRE_NGR = 0xb980,
168 RRE_OGR = 0xb981,
169 RRE_SGR = 0xb909,
3790b918
RH		 170 RRE_SLGR = 0xb90b,
171 RRE_SLBR = 0xb999,
172 RRE_SLBGR = 0xb989,
48bb3750
RH		 173 RRE_XGR = 0xb982,
174
96a9f093
RH		 175 RRF_LOCR = 0xb9f2,
176 RRF_LOCGR = 0xb9e2,
c2097136
RH		 177 RRF_NRK = 0xb9f4,
178 RRF_NGRK = 0xb9e4,
179 RRF_ORK = 0xb9f6,
180 RRF_OGRK = 0xb9e6,
181 RRF_SRK = 0xb9f9,
182 RRF_SGRK = 0xb9e9,
183 RRF_SLRK = 0xb9fb,
184 RRF_SLGRK = 0xb9eb,
185 RRF_XRK = 0xb9f7,
186 RRF_XGRK = 0xb9e7,
96a9f093 187
48bb3750 188 RR_AR = 0x1a,
3790b918 189 RR_ALR = 0x1e,
48bb3750
RH		 190 RR_BASR = 0x0d,
191 RR_BCR = 0x07,
192 RR_CLR = 0x15,
193 RR_CR = 0x19,
194 RR_DR = 0x1d,
195 RR_LCR = 0x13,
196 RR_LR = 0x18,
197 RR_LTR = 0x12,
198 RR_NR = 0x14,
199 RR_OR = 0x16,
200 RR_SR = 0x1b,
3790b918 201 RR_SLR = 0x1f,
48bb3750
RH		 202 RR_XR = 0x17,
203
204 RSY_RLL = 0xeb1d,
205 RSY_RLLG = 0xeb1c,
206 RSY_SLLG = 0xeb0d,
c2097136 207 RSY_SLLK = 0xebdf,
48bb3750 208 RSY_SRAG = 0xeb0a,
c2097136 209 RSY_SRAK = 0xebdc,
48bb3750 210 RSY_SRLG = 0xeb0c,
c2097136 211 RSY_SRLK = 0xebde,
48bb3750
RH		 212
213 RS_SLL = 0x89,
214 RS_SRA = 0x8a,
215 RS_SRL = 0x88,
216
217 RXY_AG = 0xe308,
218 RXY_AY = 0xe35a,
219 RXY_CG = 0xe320,
a534bb15
RH		 220 RXY_CLG = 0xe321,
221 RXY_CLY = 0xe355,
48bb3750 222 RXY_CY = 0xe359,
0db921e6 223 RXY_LAY = 0xe371,
48bb3750
RH		 224 RXY_LB = 0xe376,
225 RXY_LG = 0xe304,
226 RXY_LGB = 0xe377,
227 RXY_LGF = 0xe314,
228 RXY_LGH = 0xe315,
229 RXY_LHY = 0xe378,
230 RXY_LLGC = 0xe390,
231 RXY_LLGF = 0xe316,
232 RXY_LLGH = 0xe391,
233 RXY_LMG = 0xeb04,
234 RXY_LRV = 0xe31e,
235 RXY_LRVG = 0xe30f,
236 RXY_LRVH = 0xe31f,
237 RXY_LY = 0xe358,
bdcd5d19 238 RXY_NG = 0xe380,
4046d9ca 239 RXY_OG = 0xe381,
48bb3750
RH		 240 RXY_STCY = 0xe372,
241 RXY_STG = 0xe324,
242 RXY_STHY = 0xe370,
243 RXY_STMG = 0xeb24,
244 RXY_STRV = 0xe33e,
245 RXY_STRVG = 0xe32f,
246 RXY_STRVH = 0xe33f,
247 RXY_STY = 0xe350,
5bf67a92 248 RXY_XG = 0xe382,
48bb3750
RH		 249
250 RX_A = 0x5a,
251 RX_C = 0x59,
252 RX_L = 0x58,
0db921e6 253 RX_LA = 0x41,
48bb3750
RH		 254 RX_LH = 0x48,
255 RX_ST = 0x50,
256 RX_STC = 0x42,
257 RX_STH = 0x40,
ed3d51ec 258
79cada86
RH		 259 VRIa_VGBM = 0xe744,
260 VRIa_VREPI = 0xe745,
261 VRIb_VGM = 0xe746,
262 VRIc_VREP = 0xe74d,
263
ae77bbe5
RH		 264 VRRa_VLC = 0xe7de,
265 VRRa_VLP = 0xe7df,
b33ce725 266 VRRa_VLR = 0xe756,
a429ee29
RH		 267 VRRc_VA = 0xe7f3,
268 VRRc_VCEQ = 0xe7f8, /* we leave the m5 cs field 0 */
269 VRRc_VCH = 0xe7fb, /* " */
270 VRRc_VCHL = 0xe7f9, /* " */
22cb37b4
RH		 271 VRRc_VERLLV = 0xe773,
272 VRRc_VESLV = 0xe770,
273 VRRc_VESRAV = 0xe77a,
274 VRRc_VESRLV = 0xe778,
479b61cb 275 VRRc_VML = 0xe7a2,
220db7a6
RH		 276 VRRc_VMN = 0xe7fe,
277 VRRc_VMNL = 0xe7fc,
278 VRRc_VMX = 0xe7ff,
279 VRRc_VMXL = 0xe7fd,
a429ee29 280 VRRc_VN = 0xe768,
ae77bbe5 281 VRRc_VNC = 0xe769,
21eab5bf 282 VRRc_VNN = 0xe76e,
ae77bbe5 283 VRRc_VNO = 0xe76b,
21eab5bf 284 VRRc_VNX = 0xe76c,
a429ee29 285 VRRc_VO = 0xe76a,
ae77bbe5 286 VRRc_VOC = 0xe76f,
4223c9c1 287 VRRc_VPKS = 0xe797, /* we leave the m5 cs field 0 */
a429ee29 288 VRRc_VS = 0xe7f7,
4223c9c1
RH		 289 VRRa_VUPH = 0xe7d7,
290 VRRa_VUPL = 0xe7d6,
a429ee29 291 VRRc_VX = 0xe76d,
9bca986d 292 VRRe_VSEL = 0xe78d,
79cada86 293 VRRf_VLVGP = 0xe762,
b33ce725 294
22cb37b4
RH		 295 VRSa_VERLL = 0xe733,
296 VRSa_VESL = 0xe730,
297 VRSa_VESRA = 0xe73a,
298 VRSa_VESRL = 0xe738,
b33ce725
RH		 299 VRSb_VLVG = 0xe722,
300 VRSc_VLGV = 0xe721,
301
2dabf742
RH		 302 VRX_VL = 0xe706,
303 VRX_VLLEZ = 0xe704,
79cada86 304 VRX_VLREP = 0xe705,
2dabf742
RH		 305 VRX_VST = 0xe70e,
306 VRX_VSTEF = 0xe70b,
307 VRX_VSTEG = 0xe70a,
308
ed3d51ec 309 NOP = 0x0707,
48bb3750
RH		 310} S390Opcode;
311
8d8fdbae 312#ifdef CONFIG_DEBUG_TCG
48bb3750 313static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
34ef7676
RH		 314 "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7",
315 "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
316 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
317 "%v0", "%v1", "%v2", "%v3", "%v4", "%v5", "%v6", "%v7",
318 "%v8", "%v9", "%v10", "%v11", "%v12", "%v13", "%v14", "%v15",
319 "%v16", "%v17", "%v18", "%v19", "%v20", "%v21", "%v22", "%v23",
320 "%v24", "%v25", "%v26", "%v27", "%v28", "%v29", "%v30", "%v31",
48bb3750
RH		 321};
322#endif
323
324/* Since R6 is a potential argument register, choose it last of the
325 call-saved registers. Likewise prefer the call-clobbered registers
326 in reverse order to maximize the chance of avoiding the arguments. */
2827822e 327static const int tcg_target_reg_alloc_order[] = {
f24efee4 328 /* Call saved registers. */
48bb3750
RH		 329 TCG_REG_R13,
330 TCG_REG_R12,
331 TCG_REG_R11,
332 TCG_REG_R10,
333 TCG_REG_R9,
334 TCG_REG_R8,
335 TCG_REG_R7,
336 TCG_REG_R6,
f24efee4 337 /* Call clobbered registers. */
48bb3750
RH		 338 TCG_REG_R14,
339 TCG_REG_R0,
340 TCG_REG_R1,
f24efee4 341 /* Argument registers, in reverse order of allocation. */
48bb3750
RH		 342 TCG_REG_R5,
343 TCG_REG_R4,
344 TCG_REG_R3,
345 TCG_REG_R2,
34ef7676
RH		 346
347 /* V8-V15 are call saved, and omitted. */
348 TCG_REG_V0,
349 TCG_REG_V1,
350 TCG_REG_V2,
351 TCG_REG_V3,
352 TCG_REG_V4,
353 TCG_REG_V5,
354 TCG_REG_V6,
355 TCG_REG_V7,
356 TCG_REG_V16,
357 TCG_REG_V17,
358 TCG_REG_V18,
359 TCG_REG_V19,
360 TCG_REG_V20,
361 TCG_REG_V21,
362 TCG_REG_V22,
363 TCG_REG_V23,
364 TCG_REG_V24,
365 TCG_REG_V25,
366 TCG_REG_V26,
367 TCG_REG_V27,
368 TCG_REG_V28,
369 TCG_REG_V29,
370 TCG_REG_V30,
371 TCG_REG_V31,
2827822e
AG		 372};
373
374static const int tcg_target_call_iarg_regs[] = {
48bb3750
RH		 375 TCG_REG_R2,
376 TCG_REG_R3,
377 TCG_REG_R4,
378 TCG_REG_R5,
379 TCG_REG_R6,
2827822e
AG		 380};
381
382static const int tcg_target_call_oarg_regs[] = {
48bb3750 383 TCG_REG_R2,
48bb3750
RH		 384};
385
386#define S390_CC_EQ 8
387#define S390_CC_LT 4
388#define S390_CC_GT 2
389#define S390_CC_OV 1
390#define S390_CC_NE (S390_CC_LT | S390_CC_GT)
391#define S390_CC_LE (S390_CC_LT | S390_CC_EQ)
392#define S390_CC_GE (S390_CC_GT | S390_CC_EQ)
393#define S390_CC_NEVER 0
394#define S390_CC_ALWAYS 15
395
396/* Condition codes that result from a COMPARE and COMPARE LOGICAL. */
0aed257f 397static const uint8_t tcg_cond_to_s390_cond[] = {
48bb3750
RH		 398 [TCG_COND_EQ] = S390_CC_EQ,
399 [TCG_COND_NE] = S390_CC_NE,
400 [TCG_COND_LT] = S390_CC_LT,
401 [TCG_COND_LE] = S390_CC_LE,
402 [TCG_COND_GT] = S390_CC_GT,
403 [TCG_COND_GE] = S390_CC_GE,
404 [TCG_COND_LTU] = S390_CC_LT,
405 [TCG_COND_LEU] = S390_CC_LE,
406 [TCG_COND_GTU] = S390_CC_GT,
407 [TCG_COND_GEU] = S390_CC_GE,
408};
409
410/* Condition codes that result from a LOAD AND TEST. Here, we have no
411 unsigned instruction variation, however since the test is vs zero we
412 can re-map the outcomes appropriately. */
0aed257f 413static const uint8_t tcg_cond_to_ltr_cond[] = {
48bb3750
RH		 414 [TCG_COND_EQ] = S390_CC_EQ,
415 [TCG_COND_NE] = S390_CC_NE,
416 [TCG_COND_LT] = S390_CC_LT,
417 [TCG_COND_LE] = S390_CC_LE,
418 [TCG_COND_GT] = S390_CC_GT,
419 [TCG_COND_GE] = S390_CC_GE,
420 [TCG_COND_LTU] = S390_CC_NEVER,
421 [TCG_COND_LEU] = S390_CC_EQ,
422 [TCG_COND_GTU] = S390_CC_NE,
423 [TCG_COND_GEU] = S390_CC_ALWAYS,
424};
425
426#ifdef CONFIG_SOFTMMU
4b473e0c 427static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = {
f24efee4
RH		 428 [MO_UB] = helper_ret_ldub_mmu,
429 [MO_SB] = helper_ret_ldsb_mmu,
430 [MO_LEUW] = helper_le_lduw_mmu,
431 [MO_LESW] = helper_le_ldsw_mmu,
432 [MO_LEUL] = helper_le_ldul_mmu,
433 [MO_LESL] = helper_le_ldsl_mmu,
fc313c64 434 [MO_LEUQ] = helper_le_ldq_mmu,
f24efee4
RH		 435 [MO_BEUW] = helper_be_lduw_mmu,
436 [MO_BESW] = helper_be_ldsw_mmu,
437 [MO_BEUL] = helper_be_ldul_mmu,
438 [MO_BESL] = helper_be_ldsl_mmu,
fc313c64 439 [MO_BEUQ] = helper_be_ldq_mmu,
e141ab52
BS		 440};
441
4b473e0c 442static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
f24efee4
RH		 443 [MO_UB] = helper_ret_stb_mmu,
444 [MO_LEUW] = helper_le_stw_mmu,
445 [MO_LEUL] = helper_le_stl_mmu,
fc313c64 446 [MO_LEUQ] = helper_le_stq_mmu,
f24efee4
RH		 447 [MO_BEUW] = helper_be_stw_mmu,
448 [MO_BEUL] = helper_be_stl_mmu,
fc313c64 449 [MO_BEUQ] = helper_be_stq_mmu,
e141ab52 450};
e141ab52 451#endif
48bb3750 452
79dae4dd 453static const tcg_insn_unit *tb_ret_addr;
34ef7676 454uint64_t s390_facilities[3];
2827822e 455
2dabf742
RH		 456static inline bool is_general_reg(TCGReg r)
457{
458 return r <= TCG_REG_R15;
459}
460
461static inline bool is_vector_reg(TCGReg r)
462{
463 return r >= TCG_REG_V0 && r <= TCG_REG_V31;
464}
465
79dae4dd 466static bool patch_reloc(tcg_insn_unit *src_rw, int type,
2ba7fae2 467 intptr_t value, intptr_t addend)
2827822e 468{
79dae4dd 469 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
e692a349 470 intptr_t pcrel2;
28eef8aa 471 uint32_t old;
e692a349
RH		 472
473 value += addend;
79dae4dd 474 pcrel2 = (tcg_insn_unit *)value - src_rx;
48bb3750
RH		 475
476 switch (type) {
477 case R_390_PC16DBL:
55dfd8fe 478 if (pcrel2 == (int16_t)pcrel2) {
79dae4dd 479 tcg_patch16(src_rw, pcrel2);
55dfd8fe
RH		 480 return true;
481 }
48bb3750
RH		 482 break;
483 case R_390_PC32DBL:
55dfd8fe 484 if (pcrel2 == (int32_t)pcrel2) {
79dae4dd 485 tcg_patch32(src_rw, pcrel2);
55dfd8fe
RH		 486 return true;
487 }
48bb3750 488 break;
28eef8aa 489 case R_390_20:
55dfd8fe 490 if (value == sextract64(value, 0, 20)) {
79dae4dd 491 old = *(uint32_t *)src_rw & 0xf00000ff;
55dfd8fe 492 old |= ((value & 0xfff) << 16) | ((value & 0xff000) >> 4);
79dae4dd 493 tcg_patch32(src_rw, old);
55dfd8fe
RH		 494 return true;
495 }
28eef8aa 496 break;
48bb3750 497 default:
e692a349 498 g_assert_not_reached();
48bb3750 499 }
55dfd8fe 500 return false;
2827822e
AG		 501}
502
2827822e 503/* Test if a constant matches the constraint. */
a4fbbd77 504static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
2827822e 505{
48bb3750
RH		 506 if (ct & TCG_CT_CONST) {
507 return 1;
508 }
509
671c835b 510 if (type == TCG_TYPE_I32) {
48bb3750
RH		 511 val = (int32_t)val;
512 }
513
514 /* The following are mutually exclusive. */
a8f0269e
RH		 515 if (ct & TCG_CT_CONST_S16) {
516 return val == (int16_t)val;
517 } else if (ct & TCG_CT_CONST_S32) {
518 return val == (int32_t)val;
ba18b07d
RH		 519 } else if (ct & TCG_CT_CONST_S33) {
520 return val >= -0xffffffffll && val <= 0xffffffffll;
752b1be9
RH		 521 } else if (ct & TCG_CT_CONST_ZERO) {
522 return val == 0;
48bb3750
RH		 523 }
524
2827822e
AG		 525 return 0;
526}
527
48bb3750
RH		 528/* Emit instructions according to the given instruction format. */
529
530static void tcg_out_insn_RR(TCGContext *s, S390Opcode op, TCGReg r1, TCGReg r2)
531{
532 tcg_out16(s, (op << 8) | (r1 << 4) | r2);
533}
534
535static void tcg_out_insn_RRE(TCGContext *s, S390Opcode op,
536 TCGReg r1, TCGReg r2)
537{
538 tcg_out32(s, (op << 16) | (r1 << 4) | r2);
539}
540
96a9f093
RH		 541static void tcg_out_insn_RRF(TCGContext *s, S390Opcode op,
542 TCGReg r1, TCGReg r2, int m3)
543{
544 tcg_out32(s, (op << 16) | (m3 << 12) | (r1 << 4) | r2);
545}
546
48bb3750
RH		 547static void tcg_out_insn_RI(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
548{
549 tcg_out32(s, (op << 16) | (r1 << 20) | (i2 & 0xffff));
550}
551
7af525af
RH		 552static void tcg_out_insn_RIE(TCGContext *s, S390Opcode op, TCGReg r1,
553 int i2, int m3)
554{
555 tcg_out16(s, (op & 0xff00) | (r1 << 4) | m3);
556 tcg_out32(s, (i2 << 16) | (op & 0xff));
557}
558
48bb3750
RH		 559static void tcg_out_insn_RIL(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
560{
561 tcg_out16(s, op | (r1 << 4));
562 tcg_out32(s, i2);
563}
564
565static void tcg_out_insn_RS(TCGContext *s, S390Opcode op, TCGReg r1,
566 TCGReg b2, TCGReg r3, int disp)
567{
568 tcg_out32(s, (op << 24) | (r1 << 20) | (r3 << 16) | (b2 << 12)
569 | (disp & 0xfff));
570}
571
572static void tcg_out_insn_RSY(TCGContext *s, S390Opcode op, TCGReg r1,
573 TCGReg b2, TCGReg r3, int disp)
574{
575 tcg_out16(s, (op & 0xff00) | (r1 << 4) | r3);
576 tcg_out32(s, (op & 0xff) | (b2 << 28)
577 | ((disp & 0xfff) << 16) | ((disp & 0xff000) >> 4));
578}
579
580#define tcg_out_insn_RX tcg_out_insn_RS
581#define tcg_out_insn_RXY tcg_out_insn_RSY
582
2dabf742
RH		 583static int RXB(TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4)
584{
585 /*
586 * Shift bit 4 of each regno to its corresponding bit of RXB.
587 * RXB itself begins at bit 8 of the instruction so 8 - 4 = 4
588 * is the left-shift of the 4th operand.
589 */
590 return ((v1 & 0x10) << (4 + 3))
591 | ((v2 & 0x10) << (4 + 2))
592 | ((v3 & 0x10) << (4 + 1))
593 | ((v4 & 0x10) << (4 + 0));
594}
595
79cada86
RH		 596static void tcg_out_insn_VRIa(TCGContext *s, S390Opcode op,
597 TCGReg v1, uint16_t i2, int m3)
598{
599 tcg_debug_assert(is_vector_reg(v1));
600 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4));
601 tcg_out16(s, i2);
602 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m3 << 12));
603}
604
605static void tcg_out_insn_VRIb(TCGContext *s, S390Opcode op,
606 TCGReg v1, uint8_t i2, uint8_t i3, int m4)
607{
608 tcg_debug_assert(is_vector_reg(v1));
609 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4));
610 tcg_out16(s, (i2 << 8) | (i3 & 0xff));
611 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m4 << 12));
612}
613
614static void tcg_out_insn_VRIc(TCGContext *s, S390Opcode op,
615 TCGReg v1, uint16_t i2, TCGReg v3, int m4)
616{
617 tcg_debug_assert(is_vector_reg(v1));
618 tcg_debug_assert(is_vector_reg(v3));
619 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v3 & 0xf));
620 tcg_out16(s, i2);
621 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, v3, 0) | (m4 << 12));
622}
623
b33ce725
RH		 624static void tcg_out_insn_VRRa(TCGContext *s, S390Opcode op,
625 TCGReg v1, TCGReg v2, int m3)
626{
627 tcg_debug_assert(is_vector_reg(v1));
628 tcg_debug_assert(is_vector_reg(v2));
629 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf));
630 tcg_out32(s, (op & 0x00ff) | RXB(v1, v2, 0, 0) | (m3 << 12));
631}
632
a429ee29
RH		 633static void tcg_out_insn_VRRc(TCGContext *s, S390Opcode op,
634 TCGReg v1, TCGReg v2, TCGReg v3, int m4)
635{
636 tcg_debug_assert(is_vector_reg(v1));
637 tcg_debug_assert(is_vector_reg(v2));
638 tcg_debug_assert(is_vector_reg(v3));
639 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf));
640 tcg_out16(s, v3 << 12);
641 tcg_out16(s, (op & 0x00ff) | RXB(v1, v2, v3, 0) | (m4 << 12));
642}
643
9bca986d
RH		 644static void tcg_out_insn_VRRe(TCGContext *s, S390Opcode op,
645 TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4)
646{
647 tcg_debug_assert(is_vector_reg(v1));
648 tcg_debug_assert(is_vector_reg(v2));
649 tcg_debug_assert(is_vector_reg(v3));
650 tcg_debug_assert(is_vector_reg(v4));
651 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf));
652 tcg_out16(s, v3 << 12);
653 tcg_out16(s, (op & 0x00ff) | RXB(v1, v2, v3, v4) | (v4 << 12));
654}
655
79cada86
RH		 656static void tcg_out_insn_VRRf(TCGContext *s, S390Opcode op,
657 TCGReg v1, TCGReg r2, TCGReg r3)
658{
659 tcg_debug_assert(is_vector_reg(v1));
660 tcg_debug_assert(is_general_reg(r2));
661 tcg_debug_assert(is_general_reg(r3));
662 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | r2);
663 tcg_out16(s, r3 << 12);
664 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0));
665}
666
22cb37b4
RH		 667static void tcg_out_insn_VRSa(TCGContext *s, S390Opcode op, TCGReg v1,
668 intptr_t d2, TCGReg b2, TCGReg v3, int m4)
669{
670 tcg_debug_assert(is_vector_reg(v1));
671 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
672 tcg_debug_assert(is_general_reg(b2));
673 tcg_debug_assert(is_vector_reg(v3));
674 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v3 & 0xf));
675 tcg_out16(s, b2 << 12 | d2);
676 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, v3, 0) | (m4 << 12));
677}
678
b33ce725
RH		 679static void tcg_out_insn_VRSb(TCGContext *s, S390Opcode op, TCGReg v1,
680 intptr_t d2, TCGReg b2, TCGReg r3, int m4)
681{
682 tcg_debug_assert(is_vector_reg(v1));
683 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
684 tcg_debug_assert(is_general_reg(b2));
685 tcg_debug_assert(is_general_reg(r3));
686 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | r3);
687 tcg_out16(s, b2 << 12 | d2);
688 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m4 << 12));
689}
690
691static void tcg_out_insn_VRSc(TCGContext *s, S390Opcode op, TCGReg r1,
692 intptr_t d2, TCGReg b2, TCGReg v3, int m4)
693{
694 tcg_debug_assert(is_general_reg(r1));
695 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
696 tcg_debug_assert(is_general_reg(b2));
697 tcg_debug_assert(is_vector_reg(v3));
698 tcg_out16(s, (op & 0xff00) | (r1 << 4) | (v3 & 0xf));
699 tcg_out16(s, b2 << 12 | d2);
700 tcg_out16(s, (op & 0x00ff) | RXB(0, 0, v3, 0) | (m4 << 12));
701}
702
2dabf742
RH		 703static void tcg_out_insn_VRX(TCGContext *s, S390Opcode op, TCGReg v1,
704 TCGReg b2, TCGReg x2, intptr_t d2, int m3)
705{
706 tcg_debug_assert(is_vector_reg(v1));
707 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
708 tcg_debug_assert(is_general_reg(x2));
709 tcg_debug_assert(is_general_reg(b2));
710 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | x2);
711 tcg_out16(s, (b2 << 12) | d2);
712 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m3 << 12));
713}
714
48bb3750
RH		 715/* Emit an opcode with "type-checking" of the format. */
716#define tcg_out_insn(S, FMT, OP, ...) \
717 glue(tcg_out_insn_,FMT)(S, glue(glue(FMT,_),OP), ## __VA_ARGS__)
718
719
720/* emit 64-bit shifts */
721static void tcg_out_sh64(TCGContext* s, S390Opcode op, TCGReg dest,
722 TCGReg src, TCGReg sh_reg, int sh_imm)
723{
724 tcg_out_insn_RSY(s, op, dest, sh_reg, src, sh_imm);
725}
726
727/* emit 32-bit shifts */
728static void tcg_out_sh32(TCGContext* s, S390Opcode op, TCGReg dest,
729 TCGReg sh_reg, int sh_imm)
730{
731 tcg_out_insn_RS(s, op, dest, sh_reg, 0, sh_imm);
732}
733
78113e83 734static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
48bb3750 735{
b33ce725
RH		 736 if (src == dst) {
737 return true;
738 }
739 switch (type) {
740 case TCG_TYPE_I32:
741 if (likely(is_general_reg(dst) && is_general_reg(src))) {
48bb3750 742 tcg_out_insn(s, RR, LR, dst, src);
b33ce725 743 break;
48bb3750 744 }
b33ce725
RH		 745 /* fallthru */
746
747 case TCG_TYPE_I64:
748 if (likely(is_general_reg(dst))) {
749 if (likely(is_general_reg(src))) {
750 tcg_out_insn(s, RRE, LGR, dst, src);
751 } else {
752 tcg_out_insn(s, VRSc, VLGV, dst, 0, 0, src, 3);
753 }
754 break;
755 } else if (is_general_reg(src)) {
756 tcg_out_insn(s, VRSb, VLVG, dst, 0, 0, src, 3);
757 break;
758 }
759 /* fallthru */
760
761 case TCG_TYPE_V64:
762 case TCG_TYPE_V128:
763 tcg_out_insn(s, VRRa, VLR, dst, src, 0);
764 break;
765
766 default:
767 g_assert_not_reached();
48bb3750 768 }
78113e83 769 return true;
48bb3750
RH		 770}
771
28eef8aa
RH		 772static const S390Opcode lli_insns[4] = {
773 RI_LLILL, RI_LLILH, RI_LLIHL, RI_LLIHH
774};
48bb3750 775
28eef8aa
RH		 776static bool maybe_out_small_movi(TCGContext *s, TCGType type,
777 TCGReg ret, tcg_target_long sval)
778{
48bb3750
RH		 779 tcg_target_ulong uval = sval;
780 int i;
781
782 if (type == TCG_TYPE_I32) {
783 uval = (uint32_t)sval;
784 sval = (int32_t)sval;
785 }
786
787 /* Try all 32-bit insns that can load it in one go. */
788 if (sval >= -0x8000 && sval < 0x8000) {
789 tcg_out_insn(s, RI, LGHI, ret, sval);
28eef8aa 790 return true;
48bb3750
RH		 791 }
792
793 for (i = 0; i < 4; i++) {
dba6a9eb 794 tcg_target_long mask = 0xffffull << i * 16;
48bb3750 795 if ((uval & mask) == uval) {
dba6a9eb 796 tcg_out_insn_RI(s, lli_insns[i], ret, uval >> i * 16);
28eef8aa 797 return true;
48bb3750
RH		 798 }
799 }
800
28eef8aa
RH		 801 return false;
802}
803
804/* load a register with an immediate value */
ccbecb44
RH		 805static void tcg_out_movi(TCGContext *s, TCGType type,
806 TCGReg ret, tcg_target_long sval)
28eef8aa
RH		 807{
808 tcg_target_ulong uval;
809
810 /* Try all 32-bit insns that can load it in one go. */
811 if (maybe_out_small_movi(s, type, ret, sval)) {
812 return;
813 }
814
815 uval = sval;
816 if (type == TCG_TYPE_I32) {
817 uval = (uint32_t)sval;
818 sval = (int32_t)sval;
819 }
820
48bb3750 821 /* Try all 48-bit insns that can load it in one go. */
3e25f7da
RH		 822 if (sval == (int32_t)sval) {
823 tcg_out_insn(s, RIL, LGFI, ret, sval);
824 return;
825 }
826 if (uval <= 0xffffffff) {
827 tcg_out_insn(s, RIL, LLILF, ret, uval);
828 return;
829 }
830 if ((uval & 0xffffffff) == 0) {
831 tcg_out_insn(s, RIL, LLIHF, ret, uval >> 32);
832 return;
48bb3750
RH		 833 }
834
829e1376
RH		 835 /* Try for PC-relative address load. For odd addresses,
836 attempt to use an offset from the start of the TB. */
48bb3750 837 if ((sval & 1) == 0) {
8c081b18 838 ptrdiff_t off = tcg_pcrel_diff(s, (void *)sval) >> 1;
48bb3750
RH		 839 if (off == (int32_t)off) {
840 tcg_out_insn(s, RIL, LARL, ret, off);
841 return;
842 }
843 }
844
ba2c7479 845 /* Otherwise, stuff it in the constant pool. */
9c3bfb79
RH		 846 tcg_out_insn(s, RIL, LGRL, ret, 0);
847 new_pool_label(s, sval, R_390_PC32DBL, s->code_ptr - 2, 2);
48bb3750
RH		 848}
849
48bb3750
RH		 850/* Emit a load/store type instruction. Inputs are:
851 DATA: The register to be loaded or stored.
852 BASE+OFS: The effective address.
853 OPC_RX: If the operation has an RX format opcode (e.g. STC), otherwise 0.
854 OPC_RXY: The RXY format opcode for the operation (e.g. STCY). */
855
856static void tcg_out_mem(TCGContext *s, S390Opcode opc_rx, S390Opcode opc_rxy,
857 TCGReg data, TCGReg base, TCGReg index,
858 tcg_target_long ofs)
859{
860 if (ofs < -0x80000 || ofs >= 0x80000) {
78c9f7c5
RH		 861 /* Combine the low 20 bits of the offset with the actual load insn;
862 the high 44 bits must come from an immediate load. */
863 tcg_target_long low = ((ofs & 0xfffff) ^ 0x80000) - 0x80000;
864 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - low);
865 ofs = low;
48bb3750
RH		 866
867 /* If we were already given an index register, add it in. */
868 if (index != TCG_REG_NONE) {
869 tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
870 }
871 index = TCG_TMP0;
872 }
873
874 if (opc_rx && ofs >= 0 && ofs < 0x1000) {
875 tcg_out_insn_RX(s, opc_rx, data, base, index, ofs);
876 } else {
877 tcg_out_insn_RXY(s, opc_rxy, data, base, index, ofs);
878 }
2827822e
AG		 879}
880
2dabf742
RH		 881static void tcg_out_vrx_mem(TCGContext *s, S390Opcode opc_vrx,
882 TCGReg data, TCGReg base, TCGReg index,
883 tcg_target_long ofs, int m3)
884{
885 if (ofs < 0 || ofs >= 0x1000) {
886 if (ofs >= -0x80000 && ofs < 0x80000) {
887 tcg_out_insn(s, RXY, LAY, TCG_TMP0, base, index, ofs);
888 base = TCG_TMP0;
889 index = TCG_REG_NONE;
890 ofs = 0;
891 } else {
892 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs);
893 if (index != TCG_REG_NONE) {
894 tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
895 }
896 index = TCG_TMP0;
897 ofs = 0;
898 }
899 }
900 tcg_out_insn_VRX(s, opc_vrx, data, base, index, ofs, m3);
901}
48bb3750 902
2827822e 903/* load data without address translation or endianness conversion */
2dabf742
RH		 904static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data,
905 TCGReg base, intptr_t ofs)
2827822e 906{
2dabf742
RH		 907 switch (type) {
908 case TCG_TYPE_I32:
909 if (likely(is_general_reg(data))) {
910 tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs);
911 break;
912 }
913 tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_32);
914 break;
915
916 case TCG_TYPE_I64:
917 if (likely(is_general_reg(data))) {
918 tcg_out_mem(s, 0, RXY_LG, data, base, TCG_REG_NONE, ofs);
919 break;
920 }
921 /* fallthru */
922
923 case TCG_TYPE_V64:
924 tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_64);
925 break;
926
927 case TCG_TYPE_V128:
928 /* Hint quadword aligned. */
929 tcg_out_vrx_mem(s, VRX_VL, data, base, TCG_REG_NONE, ofs, 4);
930 break;
931
932 default:
933 g_assert_not_reached();
48bb3750 934 }
2827822e
AG		 935}
936
2dabf742
RH		 937static void tcg_out_st(TCGContext *s, TCGType type, TCGReg data,
938 TCGReg base, intptr_t ofs)
2827822e 939{
2dabf742
RH		 940 switch (type) {
941 case TCG_TYPE_I32:
942 if (likely(is_general_reg(data))) {
943 tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs);
944 } else {
945 tcg_out_vrx_mem(s, VRX_VSTEF, data, base, TCG_REG_NONE, ofs, 1);
946 }
947 break;
948
949 case TCG_TYPE_I64:
950 if (likely(is_general_reg(data))) {
951 tcg_out_mem(s, 0, RXY_STG, data, base, TCG_REG_NONE, ofs);
952 break;
953 }
954 /* fallthru */
955
956 case TCG_TYPE_V64:
957 tcg_out_vrx_mem(s, VRX_VSTEG, data, base, TCG_REG_NONE, ofs, 0);
958 break;
959
960 case TCG_TYPE_V128:
961 /* Hint quadword aligned. */
962 tcg_out_vrx_mem(s, VRX_VST, data, base, TCG_REG_NONE, ofs, 4);
963 break;
964
965 default:
966 g_assert_not_reached();
48bb3750
RH		 967 }
968}
969
59d7c14e
RH		 970static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
971 TCGReg base, intptr_t ofs)
972{
973 return false;
974}
975
f0bffc27
RH		 976static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src,
977 int msb, int lsb, int ofs, int z)
978{
979 /* Format RIE-f */
980 tcg_out16(s, (RIE_RISBG & 0xff00) | (dest << 4) | src);
981 tcg_out16(s, (msb << 8) | (z << 7) | lsb);
982 tcg_out16(s, (ofs << 8) | (RIE_RISBG & 0xff));
983}
984
48bb3750
RH		 985static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
986{
3e25f7da 987 tcg_out_insn(s, RRE, LGBR, dest, src);
48bb3750
RH		 988}
989
990static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
991{
3e25f7da 992 tcg_out_insn(s, RRE, LLGCR, dest, src);
48bb3750
RH		 993}
994
995static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
996{
3e25f7da 997 tcg_out_insn(s, RRE, LGHR, dest, src);
48bb3750
RH		 998}
999
1000static void tgen_ext16u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1001{
3e25f7da 1002 tcg_out_insn(s, RRE, LLGHR, dest, src);
48bb3750
RH		 1003}
1004
1005static inline void tgen_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
1006{
1007 tcg_out_insn(s, RRE, LGFR, dest, src);
1008}
1009
1010static inline void tgen_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
1011{
1012 tcg_out_insn(s, RRE, LLGFR, dest, src);
1013}
1014
f0bffc27
RH		 1015/* Accept bit patterns like these:
1016 0....01....1
1017 1....10....0
1018 1..10..01..1
1019 0..01..10..0
1020 Copied from gcc sources. */
1021static inline bool risbg_mask(uint64_t c)
1022{
1023 uint64_t lsb;
1024 /* We don't change the number of transitions by inverting,
1025 so make sure we start with the LSB zero. */
1026 if (c & 1) {
1027 c = ~c;
1028 }
1029 /* Reject all zeros or all ones. */
1030 if (c == 0) {
1031 return false;
1032 }
1033 /* Find the first transition. */
1034 lsb = c & -c;
1035 /* Invert to look for a second transition. */
1036 c = ~c;
1037 /* Erase the first transition. */
1038 c &= -lsb;
1039 /* Find the second transition, if any. */
1040 lsb = c & -c;
1041 /* Match if all the bits are 1's, or if c is zero. */
1042 return c == -lsb;
1043}
1044
547ec121
RH		 1045static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
1046{
1047 int msb, lsb;
1048 if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
1049 /* Achieve wraparound by swapping msb and lsb. */
1050 msb = 64 - ctz64(~val);
1051 lsb = clz64(~val) - 1;
1052 } else {
1053 msb = clz64(val);
1054 lsb = 63 - ctz64(val);
1055 }
1056 tcg_out_risbg(s, out, in, msb, lsb, 0, 1);
1057}
1058
07ff7983 1059static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
48bb3750
RH		 1060{
1061 static const S390Opcode ni_insns[4] = {
1062 RI_NILL, RI_NILH, RI_NIHL, RI_NIHH
1063 };
1064 static const S390Opcode nif_insns[2] = {
1065 RIL_NILF, RIL_NIHF
1066 };
07ff7983 1067 uint64_t valid = (type == TCG_TYPE_I32 ? 0xffffffffull : -1ull);
48bb3750
RH		 1068 int i;
1069
48bb3750 1070 /* Look for the zero-extensions. */
07ff7983 1071 if ((val & valid) == 0xffffffff) {
48bb3750
RH		 1072 tgen_ext32u(s, dest, dest);
1073 return;
1074 }
3e25f7da
RH		 1075 if ((val & valid) == 0xff) {
1076 tgen_ext8u(s, TCG_TYPE_I64, dest, dest);
1077 return;
1078 }
1079 if ((val & valid) == 0xffff) {
1080 tgen_ext16u(s, TCG_TYPE_I64, dest, dest);
1081 return;
07ff7983 1082 }
48bb3750 1083
07ff7983
RH		 1084 /* Try all 32-bit insns that can perform it in one go. */
1085 for (i = 0; i < 4; i++) {
dba6a9eb 1086 tcg_target_ulong mask = ~(0xffffull << i * 16);
07ff7983 1087 if (((val | ~valid) & mask) == mask) {
dba6a9eb 1088 tcg_out_insn_RI(s, ni_insns[i], dest, val >> i * 16);
07ff7983 1089 return;
48bb3750 1090 }
07ff7983 1091 }
48bb3750 1092
07ff7983 1093 /* Try all 48-bit insns that can perform it in one go. */
3e25f7da
RH		 1094 for (i = 0; i < 2; i++) {
1095 tcg_target_ulong mask = ~(0xffffffffull << i * 32);
1096 if (((val | ~valid) & mask) == mask) {
1097 tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i * 32);
1098 return;
48bb3750 1099 }
07ff7983 1100 }
9c3bfb79 1101 if (risbg_mask(val)) {
547ec121 1102 tgen_andi_risbg(s, dest, dest, val);
f0bffc27
RH		 1103 return;
1104 }
48bb3750 1105
ccbecb44 1106 tcg_out_movi(s, type, TCG_TMP0, val);
07ff7983
RH		 1107 if (type == TCG_TYPE_I32) {
1108 tcg_out_insn(s, RR, NR, dest, TCG_TMP0);
48bb3750 1109 } else {
07ff7983 1110 tcg_out_insn(s, RRE, NGR, dest, TCG_TMP0);
48bb3750
RH		 1111 }
1112}
1113
4046d9ca 1114static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
48bb3750
RH		 1115{
1116 static const S390Opcode oi_insns[4] = {
1117 RI_OILL, RI_OILH, RI_OIHL, RI_OIHH
1118 };
4046d9ca 1119 static const S390Opcode oif_insns[2] = {
48bb3750
RH		 1120 RIL_OILF, RIL_OIHF
1121 };
1122
1123 int i;
1124
1125 /* Look for no-op. */
4046d9ca 1126 if (unlikely(val == 0)) {
48bb3750
RH		 1127 return;
1128 }
1129
4046d9ca
RH		 1130 /* Try all 32-bit insns that can perform it in one go. */
1131 for (i = 0; i < 4; i++) {
dba6a9eb 1132 tcg_target_ulong mask = (0xffffull << i * 16);
4046d9ca 1133 if ((val & mask) != 0 && (val & ~mask) == 0) {
dba6a9eb 1134 tcg_out_insn_RI(s, oi_insns[i], dest, val >> i * 16);
4046d9ca 1135 return;
48bb3750 1136 }
4046d9ca 1137 }
48bb3750 1138
4046d9ca 1139 /* Try all 48-bit insns that can perform it in one go. */
3e25f7da
RH		 1140 for (i = 0; i < 2; i++) {
1141 tcg_target_ulong mask = (0xffffffffull << i * 32);
1142 if ((val & mask) != 0 && (val & ~mask) == 0) {
1143 tcg_out_insn_RIL(s, oif_insns[i], dest, val >> i * 32);
1144 return;
48bb3750 1145 }
4046d9ca 1146 }
48bb3750 1147
4046d9ca
RH		 1148 if (maybe_out_small_movi(s, type, TCG_TMP0, val)) {
1149 if (type == TCG_TYPE_I32) {
1150 tcg_out_insn(s, RR, OR, dest, TCG_TMP0);
1151 } else {
1152 tcg_out_insn(s, RRE, OGR, dest, TCG_TMP0);
1153 }
4046d9ca 1154 } else {
48bb3750
RH		 1155 /* Perform the OR via sequential modifications to the high and
1156 low parts. Do this via recursion to handle 16-bit vs 32-bit
1157 masks in each half. */
4046d9ca
RH		 1158 tgen_ori(s, type, dest, val & 0x00000000ffffffffull);
1159 tgen_ori(s, type, dest, val & 0xffffffff00000000ull);
48bb3750
RH		 1160 }
1161}
1162
5bf67a92 1163static void tgen_xori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
48bb3750 1164{
5bf67a92 1165 /* Try all 48-bit insns that can perform it in one go. */
3e25f7da
RH		 1166 if ((val & 0xffffffff00000000ull) == 0) {
1167 tcg_out_insn(s, RIL, XILF, dest, val);
1168 return;
1169 }
1170 if ((val & 0x00000000ffffffffull) == 0) {
1171 tcg_out_insn(s, RIL, XIHF, dest, val >> 32);
1172 return;
48bb3750 1173 }
5bf67a92 1174
5bf67a92
RH		 1175 if (maybe_out_small_movi(s, type, TCG_TMP0, val)) {
1176 if (type == TCG_TYPE_I32) {
1177 tcg_out_insn(s, RR, XR, dest, TCG_TMP0);
1178 } else {
1179 tcg_out_insn(s, RRE, XGR, dest, TCG_TMP0);
1180 }
5bf67a92
RH		 1181 } else {
1182 /* Perform the xor by parts. */
5bf67a92
RH		 1183 if (val & 0xffffffff) {
1184 tcg_out_insn(s, RIL, XILF, dest, val);
1185 }
1186 if (val > 0xffffffff) {
1187 tcg_out_insn(s, RIL, XIHF, dest, val >> 32);
1188 }
48bb3750
RH		 1189 }
1190}
1191
1192static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
65839b56 1193 TCGArg c2, bool c2const, bool need_carry)
48bb3750 1194{
bcc66562 1195 bool is_unsigned = is_unsigned_cond(c);
a534bb15
RH		 1196 S390Opcode op;
1197
48bb3750
RH		 1198 if (c2const) {
1199 if (c2 == 0) {
65839b56
RH		 1200 if (!(is_unsigned && need_carry)) {
1201 if (type == TCG_TYPE_I32) {
1202 tcg_out_insn(s, RR, LTR, r1, r1);
1203 } else {
1204 tcg_out_insn(s, RRE, LTGR, r1, r1);
1205 }
1206 return tcg_cond_to_ltr_cond[c];
1207 }
65839b56 1208 }
a534bb15
RH		 1209
1210 if (!is_unsigned && c2 == (int16_t)c2) {
1211 op = (type == TCG_TYPE_I32 ? RI_CHI : RI_CGHI);
1212 tcg_out_insn_RI(s, op, r1, c2);
1213 goto exit;
1214 }
1215
3e25f7da
RH		 1216 if (type == TCG_TYPE_I32) {
1217 op = (is_unsigned ? RIL_CLFI : RIL_CFI);
1218 tcg_out_insn_RIL(s, op, r1, c2);
1219 goto exit;
1220 }
1221 if (c2 == (is_unsigned ? (TCGArg)(uint32_t)c2 : (TCGArg)(int32_t)c2)) {
1222 op = (is_unsigned ? RIL_CLGFI : RIL_CGFI);
1223 tcg_out_insn_RIL(s, op, r1, c2);
1224 goto exit;
48bb3750 1225 }
a534bb15
RH		 1226
1227 /* Use the constant pool, but not for small constants. */
1228 if (maybe_out_small_movi(s, type, TCG_TMP0, c2)) {
1229 c2 = TCG_TMP0;
1230 /* fall through to reg-reg */
48bb3750 1231 } else {
3e25f7da
RH		 1232 op = (is_unsigned ? RIL_CLGRL : RIL_CGRL);
1233 tcg_out_insn_RIL(s, op, r1, 0);
1234 new_pool_label(s, c2, R_390_PC32DBL, s->code_ptr - 2, 2);
a534bb15 1235 goto exit;
48bb3750
RH		 1236 }
1237 }
a534bb15
RH		 1238
1239 if (type == TCG_TYPE_I32) {
1240 op = (is_unsigned ? RR_CLR : RR_CR);
1241 tcg_out_insn_RR(s, op, r1, c2);
1242 } else {
1243 op = (is_unsigned ? RRE_CLGR : RRE_CGR);
1244 tcg_out_insn_RRE(s, op, r1, c2);
1245 }
1246
1247 exit:
48bb3750
RH		 1248 return tcg_cond_to_s390_cond[c];
1249}
1250
7b7066b1 1251static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
96a9f093 1252 TCGReg dest, TCGReg c1, TCGArg c2, int c2const)
48bb3750 1253{
7b7066b1 1254 int cc;
7af525af 1255 bool have_loc;
7b7066b1 1256
7af525af 1257 /* With LOC2, we can always emit the minimum 3 insns. */
748b7f3e 1258 if (HAVE_FACILITY(LOAD_ON_COND2)) {
7af525af
RH		 1259 /* Emit: d = 0, d = (cc ? 1 : d). */
1260 cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
1261 tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
1262 tcg_out_insn(s, RIE, LOCGHI, dest, 1, cc);
1263 return;
1264 }
1265
748b7f3e 1266 have_loc = HAVE_FACILITY(LOAD_ON_COND);
7af525af 1267
4609190b
RH		 1268 /* For HAVE_LOC, only the paths through GTU/GT/LEU/LE are smaller. */
1269 restart:
7b7066b1 1270 switch (cond) {
4609190b
RH		 1271 case TCG_COND_NE:
1272 /* X != 0 is X > 0. */
1273 if (c2const && c2 == 0) {
1274 cond = TCG_COND_GTU;
1275 } else {
1276 break;
1277 }
1278 /* fallthru */
1279
7b7066b1
RH		 1280 case TCG_COND_GTU:
1281 case TCG_COND_GT:
7b7066b1
RH		 1282 /* The result of a compare has CC=2 for GT and CC=3 unused.
1283 ADD LOGICAL WITH CARRY considers (CC & 2) the carry bit. */
65839b56 1284 tgen_cmp(s, type, cond, c1, c2, c2const, true);
7b7066b1
RH		 1285 tcg_out_movi(s, type, dest, 0);
1286 tcg_out_insn(s, RRE, ALCGR, dest, dest);
1287 return;
1288
4609190b
RH		 1289 case TCG_COND_EQ:
1290 /* X == 0 is X <= 0. */
1291 if (c2const && c2 == 0) {
1292 cond = TCG_COND_LEU;
7b7066b1 1293 } else {
4609190b 1294 break;
7b7066b1 1295 }
4609190b 1296 /* fallthru */
7b7066b1
RH		 1297
1298 case TCG_COND_LEU:
4609190b
RH		 1299 case TCG_COND_LE:
1300 /* As above, but we're looking for borrow, or !carry.
1301 The second insn computes d - d - borrow, or -1 for true
1302 and 0 for false. So we must mask to 1 bit afterward. */
1303 tgen_cmp(s, type, cond, c1, c2, c2const, true);
1304 tcg_out_insn(s, RRE, SLBGR, dest, dest);
1305 tgen_andi(s, type, dest, 1);
1306 return;
1307
1308 case TCG_COND_GEU:
7b7066b1
RH		 1309 case TCG_COND_LTU:
1310 case TCG_COND_LT:
4609190b
RH		 1311 case TCG_COND_GE:
1312 /* Swap operands so that we can use LEU/GTU/GT/LE. */
7b7066b1 1313 if (c2const) {
7af525af 1314 if (have_loc) {
4609190b 1315 break;
7af525af 1316 }
7b7066b1
RH		 1317 tcg_out_movi(s, type, TCG_TMP0, c2);
1318 c2 = c1;
1319 c2const = 0;
1320 c1 = TCG_TMP0;
1321 } else {
1322 TCGReg t = c1;
1323 c1 = c2;
1324 c2 = t;
1325 }
7b7066b1 1326 cond = tcg_swap_cond(cond);
4609190b 1327 goto restart;
48bb3750 1328
7b7066b1 1329 default:
4609190b 1330 g_assert_not_reached();
7b7066b1
RH		 1331 }
1332
65839b56 1333 cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
4609190b
RH		 1334 if (have_loc) {
1335 /* Emit: d = 0, t = 1, d = (cc ? t : d). */
1336 tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
1337 tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 1);
1338 tcg_out_insn(s, RRF, LOCGR, dest, TCG_TMP0, cc);
1339 } else {
1340 /* Emit: d = 1; if (cc) goto over; d = 0; over: */
1341 tcg_out_movi(s, type, dest, 1);
1342 tcg_out_insn(s, RI, BRC, cc, (4 + 4) >> 1);
1343 tcg_out_movi(s, type, dest, 0);
1344 }
48bb3750
RH		 1345}
1346
96a9f093 1347static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, TCGReg dest,
7af525af
RH		 1348 TCGReg c1, TCGArg c2, int c2const,
1349 TCGArg v3, int v3const)
96a9f093
RH		 1350{
1351 int cc;
748b7f3e 1352 if (HAVE_FACILITY(LOAD_ON_COND)) {
65839b56 1353 cc = tgen_cmp(s, type, c, c1, c2, c2const, false);
7af525af
RH		 1354 if (v3const) {
1355 tcg_out_insn(s, RIE, LOCGHI, dest, v3, cc);
1356 } else {
1357 tcg_out_insn(s, RRF, LOCGR, dest, v3, cc);
1358 }
96a9f093
RH		 1359 } else {
1360 c = tcg_invert_cond(c);
65839b56 1361 cc = tgen_cmp(s, type, c, c1, c2, c2const, false);
96a9f093
RH		 1362
1363 /* Emit: if (cc) goto over; dest = r3; over: */
1364 tcg_out_insn(s, RI, BRC, cc, (4 + 4) >> 1);
7af525af 1365 tcg_out_insn(s, RRE, LGR, dest, v3);
96a9f093
RH		 1366 }
1367}
1368
ce411066
RH		 1369static void tgen_clz(TCGContext *s, TCGReg dest, TCGReg a1,
1370 TCGArg a2, int a2const)
1371{
1372 /* Since this sets both R and R+1, we have no choice but to store the
1373 result into R0, allowing R1 == TCG_TMP0 to be clobbered as well. */
1374 QEMU_BUILD_BUG_ON(TCG_TMP0 != TCG_REG_R1);
1375 tcg_out_insn(s, RRE, FLOGR, TCG_REG_R0, a1);
1376
1377 if (a2const && a2 == 64) {
1378 tcg_out_mov(s, TCG_TYPE_I64, dest, TCG_REG_R0);
1379 } else {
1380 if (a2const) {
1381 tcg_out_movi(s, TCG_TYPE_I64, dest, a2);
1382 } else {
1383 tcg_out_mov(s, TCG_TYPE_I64, dest, a2);
1384 }
748b7f3e 1385 if (HAVE_FACILITY(LOAD_ON_COND)) {
ce411066
RH		 1386 /* Emit: if (one bit found) dest = r0. */
1387 tcg_out_insn(s, RRF, LOCGR, dest, TCG_REG_R0, 2);
1388 } else {
1389 /* Emit: if (no one bit found) goto over; dest = r0; over: */
1390 tcg_out_insn(s, RI, BRC, 8, (4 + 4) >> 1);
1391 tcg_out_insn(s, RRE, LGR, dest, TCG_REG_R0);
1392 }
1393 }
1394}
1395
d5690ea4 1396static void tgen_deposit(TCGContext *s, TCGReg dest, TCGReg src,
752b1be9 1397 int ofs, int len, int z)
d5690ea4
RH		 1398{
1399 int lsb = (63 - ofs);
1400 int msb = lsb - (len - 1);
752b1be9 1401 tcg_out_risbg(s, dest, src, msb, lsb, ofs, z);
d5690ea4
RH		 1402}
1403
b0bf5fe8
RH		 1404static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src,
1405 int ofs, int len)
1406{
1407 tcg_out_risbg(s, dest, src, 64 - len, 63, 64 - ofs, 1);
1408}
1409
ffd0e507 1410static void tgen_gotoi(TCGContext *s, int cc, const tcg_insn_unit *dest)
48bb3750 1411{
79dae4dd 1412 ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
8c081b18 1413 if (off == (int16_t)off) {
48bb3750
RH		 1414 tcg_out_insn(s, RI, BRC, cc, off);
1415 } else if (off == (int32_t)off) {
1416 tcg_out_insn(s, RIL, BRCL, cc, off);
1417 } else {
8c081b18 1418 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
48bb3750
RH		 1419 tcg_out_insn(s, RR, BCR, cc, TCG_TMP0);
1420 }
1421}
1422
bec16311 1423static void tgen_branch(TCGContext *s, int cc, TCGLabel *l)
48bb3750 1424{
48bb3750 1425 if (l->has_value) {
8c081b18 1426 tgen_gotoi(s, cc, l->u.value_ptr);
48bb3750
RH		 1427 } else {
1428 tcg_out16(s, RI_BRC | (cc << 4));
e692a349 1429 tcg_out_reloc(s, s->code_ptr, R_390_PC16DBL, l, 2);
8c081b18 1430 s->code_ptr += 1;
48bb3750
RH		 1431 }
1432}
1433
1434static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc,
bec16311 1435 TCGReg r1, TCGReg r2, TCGLabel *l)
48bb3750 1436{
79dae4dd 1437 tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
48bb3750 1438 tcg_out16(s, (opc & 0xff00) | (r1 << 4) | r2);
79dae4dd 1439 tcg_out16(s, 0);
48bb3750
RH		 1440 tcg_out16(s, cc << 12 | (opc & 0xff));
1441}
1442
1443static void tgen_compare_imm_branch(TCGContext *s, S390Opcode opc, int cc,
bec16311 1444 TCGReg r1, int i2, TCGLabel *l)
48bb3750 1445{
79dae4dd 1446 tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
48bb3750 1447 tcg_out16(s, (opc & 0xff00) | (r1 << 4) | cc);
79dae4dd 1448 tcg_out16(s, 0);
48bb3750
RH		 1449 tcg_out16(s, (i2 << 8) | (opc & 0xff));
1450}
1451
1452static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
bec16311 1453 TCGReg r1, TCGArg c2, int c2const, TCGLabel *l)
48bb3750
RH		 1454{
1455 int cc;
9c3bfb79
RH		 1456 bool is_unsigned = is_unsigned_cond(c);
1457 bool in_range;
1458 S390Opcode opc;
48bb3750 1459
9c3bfb79 1460 cc = tcg_cond_to_s390_cond[c];
48bb3750 1461
9c3bfb79
RH		 1462 if (!c2const) {
1463 opc = (type == TCG_TYPE_I32
1464 ? (is_unsigned ? RIE_CLRJ : RIE_CRJ)
1465 : (is_unsigned ? RIE_CLGRJ : RIE_CGRJ));
1466 tgen_compare_branch(s, opc, cc, r1, c2, l);
1467 return;
1468 }
48bb3750 1469
9c3bfb79
RH		 1470 /*
1471 * COMPARE IMMEDIATE AND BRANCH RELATIVE has an 8-bit immediate field.
1472 * If the immediate we've been given does not fit that range, we'll
1473 * fall back to separate compare and branch instructions using the
1474 * larger comparison range afforded by COMPARE IMMEDIATE.
1475 */
1476 if (type == TCG_TYPE_I32) {
1477 if (is_unsigned) {
1478 opc = RIE_CLIJ;
1479 in_range = (uint32_t)c2 == (uint8_t)c2;
48bb3750 1480 } else {
9c3bfb79
RH		 1481 opc = RIE_CIJ;
1482 in_range = (int32_t)c2 == (int8_t)c2;
48bb3750 1483 }
9c3bfb79
RH		 1484 } else {
1485 if (is_unsigned) {
1486 opc = RIE_CLGIJ;
1487 in_range = (uint64_t)c2 == (uint8_t)c2;
1488 } else {
1489 opc = RIE_CGIJ;
1490 in_range = (int64_t)c2 == (int8_t)c2;
48bb3750
RH		 1491 }
1492 }
9c3bfb79
RH		 1493 if (in_range) {
1494 tgen_compare_imm_branch(s, opc, cc, r1, c2, l);
1495 return;
1496 }
48bb3750 1497
65839b56 1498 cc = tgen_cmp(s, type, c, r1, c2, c2const, false);
bec16311 1499 tgen_branch(s, cc, l);
48bb3750
RH		 1500}
1501
cee44b03 1502static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *dest)
48bb3750 1503{
79dae4dd 1504 ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
48bb3750
RH		 1505 if (off == (int32_t)off) {
1506 tcg_out_insn(s, RIL, BRASL, TCG_REG_R14, off);
1507 } else {
8c081b18 1508 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
48bb3750
RH		 1509 tcg_out_insn(s, RR, BASR, TCG_REG_R14, TCG_TMP0);
1510 }
1511}
1512
cee44b03
RH		 1513static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest,
1514 const TCGHelperInfo *info)
1515{
1516 tcg_out_call_int(s, dest);
1517}
1518
14776ab5 1519static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
48bb3750
RH		 1520 TCGReg base, TCGReg index, int disp)
1521{
3c8691f5 1522 switch (opc & (MO_SSIZE | MO_BSWAP)) {
a5a04f28 1523 case MO_UB:
48bb3750
RH		 1524 tcg_out_insn(s, RXY, LLGC, data, base, index, disp);
1525 break;
a5a04f28 1526 case MO_SB:
48bb3750
RH		 1527 tcg_out_insn(s, RXY, LGB, data, base, index, disp);
1528 break;
b8dd88b8
RH		 1529
1530 case MO_UW | MO_BSWAP:
1531 /* swapped unsigned halfword load with upper bits zeroed */
1532 tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
1533 tgen_ext16u(s, TCG_TYPE_I64, data, data);
1534 break;
a5a04f28 1535 case MO_UW:
b8dd88b8
RH		 1536 tcg_out_insn(s, RXY, LLGH, data, base, index, disp);
1537 break;
1538
1539 case MO_SW | MO_BSWAP:
1540 /* swapped sign-extended halfword load */
1541 tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
1542 tgen_ext16s(s, TCG_TYPE_I64, data, data);
48bb3750 1543 break;
a5a04f28 1544 case MO_SW:
b8dd88b8
RH		 1545 tcg_out_insn(s, RXY, LGH, data, base, index, disp);
1546 break;
1547
1548 case MO_UL | MO_BSWAP:
1549 /* swapped unsigned int load with upper bits zeroed */
1550 tcg_out_insn(s, RXY, LRV, data, base, index, disp);
1551 tgen_ext32u(s, data, data);
48bb3750 1552 break;
a5a04f28 1553 case MO_UL:
b8dd88b8
RH		 1554 tcg_out_insn(s, RXY, LLGF, data, base, index, disp);
1555 break;
1556
1557 case MO_SL | MO_BSWAP:
1558 /* swapped sign-extended int load */
1559 tcg_out_insn(s, RXY, LRV, data, base, index, disp);
1560 tgen_ext32s(s, data, data);
48bb3750 1561 break;
a5a04f28 1562 case MO_SL:
b8dd88b8
RH		 1563 tcg_out_insn(s, RXY, LGF, data, base, index, disp);
1564 break;
1565
fc313c64 1566 case MO_UQ | MO_BSWAP:
b8dd88b8 1567 tcg_out_insn(s, RXY, LRVG, data, base, index, disp);
48bb3750 1568 break;
fc313c64 1569 case MO_UQ:
b8dd88b8 1570 tcg_out_insn(s, RXY, LG, data, base, index, disp);
48bb3750 1571 break;
b8dd88b8 1572
48bb3750
RH		 1573 default:
1574 tcg_abort();
1575 }
1576}
1577
14776ab5 1578static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data,
48bb3750
RH		 1579 TCGReg base, TCGReg index, int disp)
1580{
3c8691f5 1581 switch (opc & (MO_SIZE | MO_BSWAP)) {
a5a04f28 1582 case MO_UB:
48bb3750
RH		 1583 if (disp >= 0 && disp < 0x1000) {
1584 tcg_out_insn(s, RX, STC, data, base, index, disp);
1585 } else {
1586 tcg_out_insn(s, RXY, STCY, data, base, index, disp);
1587 }
1588 break;
b8dd88b8
RH		 1589
1590 case MO_UW | MO_BSWAP:
1591 tcg_out_insn(s, RXY, STRVH, data, base, index, disp);
1592 break;
a5a04f28 1593 case MO_UW:
b8dd88b8 1594 if (disp >= 0 && disp < 0x1000) {
48bb3750
RH		 1595 tcg_out_insn(s, RX, STH, data, base, index, disp);
1596 } else {
1597 tcg_out_insn(s, RXY, STHY, data, base, index, disp);
1598 }
1599 break;
b8dd88b8
RH		 1600
1601 case MO_UL | MO_BSWAP:
1602 tcg_out_insn(s, RXY, STRV, data, base, index, disp);
1603 break;
a5a04f28 1604 case MO_UL:
b8dd88b8 1605 if (disp >= 0 && disp < 0x1000) {
48bb3750
RH		 1606 tcg_out_insn(s, RX, ST, data, base, index, disp);
1607 } else {
1608 tcg_out_insn(s, RXY, STY, data, base, index, disp);
1609 }
1610 break;
b8dd88b8 1611
fc313c64 1612 case MO_UQ | MO_BSWAP:
b8dd88b8
RH		 1613 tcg_out_insn(s, RXY, STRVG, data, base, index, disp);
1614 break;
fc313c64 1615 case MO_UQ:
b8dd88b8 1616 tcg_out_insn(s, RXY, STG, data, base, index, disp);
48bb3750 1617 break;
b8dd88b8 1618
48bb3750
RH		 1619 default:
1620 tcg_abort();
1621 }
1622}
1623
1624#if defined(CONFIG_SOFTMMU)
269bd5d8
RH		 1625/* We're expecting to use a 20-bit negative offset on the tlb memory ops. */
1626QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
1627QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 19));
fb596415
RH		 1628
1629/* Load and compare a TLB entry, leaving the flags set. Loads the TLB
1630 addend into R2. Returns a register with the santitized guest address. */
14776ab5 1631static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
fb596415 1632 int mem_index, bool is_ld)
48bb3750 1633{
85aa8081
RH		 1634 unsigned s_bits = opc & MO_SIZE;
1635 unsigned a_bits = get_alignment_bits(opc);
1636 unsigned s_mask = (1 << s_bits) - 1;
1637 unsigned a_mask = (1 << a_bits) - 1;
269bd5d8
RH		 1638 int fast_off = TLB_MASK_TABLE_OFS(mem_index);
1639 int mask_off = fast_off + offsetof(CPUTLBDescFast, mask);
1640 int table_off = fast_off + offsetof(CPUTLBDescFast, table);
a5e39810
RH		 1641 int ofs, a_off;
1642 uint64_t tlb_mask;
1643
4f47e338
RH		 1644 tcg_out_sh64(s, RSY_SRLG, TCG_REG_R2, addr_reg, TCG_REG_NONE,
1645 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1646 tcg_out_insn(s, RXY, NG, TCG_REG_R2, TCG_AREG0, TCG_REG_NONE, mask_off);
1647 tcg_out_insn(s, RXY, AG, TCG_REG_R2, TCG_AREG0, TCG_REG_NONE, table_off);
1648
a5e39810
RH		 1649 /* For aligned accesses, we check the first byte and include the alignment
1650 bits within the address. For unaligned access, we check that we don't
1651 cross pages using the address of the last byte of the access. */
85aa8081
RH		 1652 a_off = (a_bits >= s_bits ? 0 : s_mask - a_mask);
1653 tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
9c3bfb79 1654 if (a_off == 0) {
4f47e338 1655 tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
48bb3750 1656 } else {
a5e39810 1657 tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
547ec121 1658 tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
48bb3750
RH		 1659 }
1660
fb596415 1661 if (is_ld) {
4f47e338 1662 ofs = offsetof(CPUTLBEntry, addr_read);
fb596415 1663 } else {
4f47e338 1664 ofs = offsetof(CPUTLBEntry, addr_write);
48bb3750 1665 }
48bb3750 1666 if (TARGET_LONG_BITS == 32) {
4f47e338 1667 tcg_out_insn(s, RX, C, TCG_REG_R3, TCG_REG_R2, TCG_REG_NONE, ofs);
48bb3750 1668 } else {
4f47e338 1669 tcg_out_insn(s, RXY, CG, TCG_REG_R3, TCG_REG_R2, TCG_REG_NONE, ofs);
48bb3750
RH		 1670 }
1671
4f47e338
RH		 1672 tcg_out_insn(s, RXY, LG, TCG_REG_R2, TCG_REG_R2, TCG_REG_NONE,
1673 offsetof(CPUTLBEntry, addend));
fb596415 1674
48bb3750 1675 if (TARGET_LONG_BITS == 32) {
fb596415
RH		 1676 tgen_ext32u(s, TCG_REG_R3, addr_reg);
1677 return TCG_REG_R3;
48bb3750 1678 }
fb596415
RH		 1679 return addr_reg;
1680}
48bb3750 1681
9002ffcb 1682static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi,
3972ef6f 1683 TCGReg data, TCGReg addr,
fb596415
RH		 1684 tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
1685{
1686 TCGLabelQemuLdst *label = new_ldst_label(s);
1687
1688 label->is_ld = is_ld;
3972ef6f 1689 label->oi = oi;
fb596415
RH		 1690 label->datalo_reg = data;
1691 label->addrlo_reg = addr;
e5e2e4c7 1692 label->raddr = tcg_splitwx_to_rx(raddr);
fb596415
RH		 1693 label->label_ptr[0] = label_ptr;
1694}
48bb3750 1695
aeee05f5 1696static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
fb596415
RH		 1697{
1698 TCGReg addr_reg = lb->addrlo_reg;
1699 TCGReg data_reg = lb->datalo_reg;
9002ffcb 1700 MemOpIdx oi = lb->oi;
14776ab5 1701 MemOp opc = get_memop(oi);
48bb3750 1702
aeee05f5 1703 if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
79dae4dd 1704 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
aeee05f5
RH		 1705 return false;
1706 }
48bb3750 1707
fb596415
RH		 1708 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1709 if (TARGET_LONG_BITS == 64) {
1710 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
1711 }
3972ef6f 1712 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R4, oi);
fb596415 1713 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R5, (uintptr_t)lb->raddr);
cee44b03 1714 tcg_out_call_int(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)]);
fb596415 1715 tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_R2);
65a62a75 1716
fb596415 1717 tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
aeee05f5 1718 return true;
48bb3750
RH		 1719}
1720
aeee05f5 1721static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
48bb3750 1722{
fb596415
RH		 1723 TCGReg addr_reg = lb->addrlo_reg;
1724 TCGReg data_reg = lb->datalo_reg;
9002ffcb 1725 MemOpIdx oi = lb->oi;
14776ab5 1726 MemOp opc = get_memop(oi);
fb596415 1727
aeee05f5 1728 if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
79dae4dd 1729 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
aeee05f5
RH		 1730 return false;
1731 }
fb596415
RH		 1732
1733 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1734 if (TARGET_LONG_BITS == 64) {
1735 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
1736 }
1737 switch (opc & MO_SIZE) {
1738 case MO_UB:
1739 tgen_ext8u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1740 break;
1741 case MO_UW:
1742 tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1743 break;
1744 case MO_UL:
1745 tgen_ext32u(s, TCG_REG_R4, data_reg);
1746 break;
fc313c64 1747 case MO_UQ:
fb596415
RH		 1748 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1749 break;
1750 default:
1751 tcg_abort();
1752 }
3972ef6f 1753 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
fb596415 1754 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
cee44b03 1755 tcg_out_call_int(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
fb596415
RH		 1756
1757 tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
aeee05f5 1758 return true;
48bb3750
RH		 1759}
1760#else
1cd49868
RH		 1761static void tcg_out_test_alignment(TCGContext *s, bool is_ld,
1762 TCGReg addrlo, unsigned a_bits)
1763{
1764 unsigned a_mask = (1 << a_bits) - 1;
1765 TCGLabelQemuLdst *l = new_ldst_label(s);
1766
1767 l->is_ld = is_ld;
1768 l->addrlo_reg = addrlo;
1769
1770 /* We are expecting a_bits to max out at 7, much lower than TMLL. */
1771 tcg_debug_assert(a_bits < 16);
1772 tcg_out_insn(s, RI, TMLL, addrlo, a_mask);
1773
1774 tcg_out16(s, RI_BRC | (7 << 4)); /* CC in {1,2,3} */
1775 l->label_ptr[0] = s->code_ptr;
1776 s->code_ptr += 1;
1777
1778 l->raddr = tcg_splitwx_to_rx(s->code_ptr);
1779}
1780
1781static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
1782{
1783 if (!patch_reloc(l->label_ptr[0], R_390_PC16DBL,
1784 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
1785 return false;
1786 }
1787
1788 tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R3, l->addrlo_reg);
1789 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1790
1791 /* "Tail call" to the helper, with the return address back inline. */
1792 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R14, (uintptr_t)l->raddr);
1793 tgen_gotoi(s, S390_CC_ALWAYS, (const void *)(l->is_ld ? helper_unaligned_ld
1794 : helper_unaligned_st));
1795 return true;
1796}
1797
1798static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
1799{
1800 return tcg_out_fail_alignment(s, l);
1801}
1802
1803static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
1804{
1805 return tcg_out_fail_alignment(s, l);
1806}
1807
48bb3750
RH		 1808static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
1809 TCGReg *index_reg, tcg_target_long *disp)
1810{
1811 if (TARGET_LONG_BITS == 32) {
1812 tgen_ext32u(s, TCG_TMP0, *addr_reg);
1813 *addr_reg = TCG_TMP0;
1814 }
b76f21a7 1815 if (guest_base < 0x80000) {
48bb3750 1816 *index_reg = TCG_REG_NONE;
b76f21a7 1817 *disp = guest_base;
48bb3750
RH		 1818 } else {
1819 *index_reg = TCG_GUEST_BASE_REG;
1820 *disp = 0;
1821 }
1822}
1823#endif /* CONFIG_SOFTMMU */
1824
f24efee4 1825static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
9002ffcb 1826 MemOpIdx oi)
48bb3750 1827{
14776ab5 1828 MemOp opc = get_memop(oi);
fb596415 1829#ifdef CONFIG_SOFTMMU
59227d5d 1830 unsigned mem_index = get_mmuidx(oi);
fb596415
RH		 1831 tcg_insn_unit *label_ptr;
1832 TCGReg base_reg;
1833
1834 base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1);
1835
cd3b29b7
AJ		 1836 tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
1837 label_ptr = s->code_ptr;
1838 s->code_ptr += 1;
fb596415
RH		 1839
1840 tcg_out_qemu_ld_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
48bb3750 1841
3972ef6f 1842 add_qemu_ldst_label(s, 1, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
48bb3750 1843#else
f24efee4
RH		 1844 TCGReg index_reg;
1845 tcg_target_long disp;
1cd49868 1846 unsigned a_bits = get_alignment_bits(opc);
f24efee4 1847
1cd49868
RH		 1848 if (a_bits) {
1849 tcg_out_test_alignment(s, true, addr_reg, a_bits);
1850 }
48bb3750
RH		 1851 tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
1852 tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, index_reg, disp);
1853#endif
1854}
1855
f24efee4 1856static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
9002ffcb 1857 MemOpIdx oi)
48bb3750 1858{
14776ab5 1859 MemOp opc = get_memop(oi);
fb596415 1860#ifdef CONFIG_SOFTMMU
59227d5d 1861 unsigned mem_index = get_mmuidx(oi);
fb596415
RH		 1862 tcg_insn_unit *label_ptr;
1863 TCGReg base_reg;
1864
1865 base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0);
1866
cd3b29b7
AJ		 1867 tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
1868 label_ptr = s->code_ptr;
1869 s->code_ptr += 1;
fb596415
RH		 1870
1871 tcg_out_qemu_st_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
48bb3750 1872
3972ef6f 1873 add_qemu_ldst_label(s, 0, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
48bb3750 1874#else
f24efee4
RH		 1875 TCGReg index_reg;
1876 tcg_target_long disp;
1cd49868 1877 unsigned a_bits = get_alignment_bits(opc);
f24efee4 1878
1cd49868
RH		 1879 if (a_bits) {
1880 tcg_out_test_alignment(s, false, addr_reg, a_bits);
1881 }
48bb3750
RH		 1882 tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
1883 tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, index_reg, disp);
1884#endif
2827822e
AG		 1885}
1886
48bb3750
RH		 1887# define OP_32_64(x) \
1888 case glue(glue(INDEX_op_,x),_i32): \
1889 case glue(glue(INDEX_op_,x),_i64)
48bb3750 1890
a9751609 1891static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
5e8892db
MR		 1892 const TCGArg args[TCG_MAX_OP_ARGS],
1893 const int const_args[TCG_MAX_OP_ARGS])
2827822e 1894{
c2097136 1895 S390Opcode op, op2;
0db921e6 1896 TCGArg a0, a1, a2;
48bb3750
RH		 1897
1898 switch (opc) {
1899 case INDEX_op_exit_tb:
46644483
RH		 1900 /* Reuse the zeroing that exists for goto_ptr. */
1901 a0 = args[0];
1902 if (a0 == 0) {
8b5c2b62 1903 tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
46644483
RH		 1904 } else {
1905 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
1906 tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
1907 }
48bb3750
RH		 1908 break;
1909
1910 case INDEX_op_goto_tb:
829e1376 1911 a0 = args[0];
6bd739ed
RH		 1912 /*
1913 * branch displacement must be aligned for atomic patching;
1914 * see if we need to add extra nop before branch
1915 */
1916 if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
1917 tcg_out16(s, NOP);
829e1376 1918 }
6bd739ed
RH		 1919 tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
1920 s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
1921 s->code_ptr += 2;
9f754620 1922 set_jmp_reset_offset(s, a0);
48bb3750
RH		 1923 break;
1924
46644483 1925 case INDEX_op_goto_ptr:
829e1376 1926 a0 = args[0];
829e1376 1927 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0);
46644483
RH		 1928 break;
1929
48bb3750
RH		 1930 OP_32_64(ld8u):
1931 /* ??? LLC (RXY format) is only present with the extended-immediate
1932 facility, whereas LLGC is always present. */
1933 tcg_out_mem(s, 0, RXY_LLGC, args[0], args[1], TCG_REG_NONE, args[2]);
1934 break;
1935
1936 OP_32_64(ld8s):
1937 /* ??? LB is no smaller than LGB, so no point to using it. */
1938 tcg_out_mem(s, 0, RXY_LGB, args[0], args[1], TCG_REG_NONE, args[2]);
1939 break;
1940
1941 OP_32_64(ld16u):
1942 /* ??? LLH (RXY format) is only present with the extended-immediate
1943 facility, whereas LLGH is always present. */
1944 tcg_out_mem(s, 0, RXY_LLGH, args[0], args[1], TCG_REG_NONE, args[2]);
1945 break;
1946
1947 case INDEX_op_ld16s_i32:
1948 tcg_out_mem(s, RX_LH, RXY_LHY, args[0], args[1], TCG_REG_NONE, args[2]);
1949 break;
1950
1951 case INDEX_op_ld_i32:
1952 tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]);
1953 break;
1954
1955 OP_32_64(st8):
1956 tcg_out_mem(s, RX_STC, RXY_STCY, args[0], args[1],
1957 TCG_REG_NONE, args[2]);
1958 break;
1959
1960 OP_32_64(st16):
1961 tcg_out_mem(s, RX_STH, RXY_STHY, args[0], args[1],
1962 TCG_REG_NONE, args[2]);
1963 break;
1964
1965 case INDEX_op_st_i32:
1966 tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
1967 break;
1968
1969 case INDEX_op_add_i32:
0db921e6 1970 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
48bb3750 1971 if (const_args[2]) {
0db921e6
RH		 1972 do_addi_32:
1973 if (a0 == a1) {
1974 if (a2 == (int16_t)a2) {
1975 tcg_out_insn(s, RI, AHI, a0, a2);
1976 break;
1977 }
3e25f7da
RH		 1978 tcg_out_insn(s, RIL, AFI, a0, a2);
1979 break;
0db921e6
RH		 1980 }
1981 tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
1982 } else if (a0 == a1) {
1983 tcg_out_insn(s, RR, AR, a0, a2);
48bb3750 1984 } else {
0db921e6 1985 tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
48bb3750
RH		 1986 }
1987 break;
1988 case INDEX_op_sub_i32:
0db921e6 1989 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
48bb3750 1990 if (const_args[2]) {
0db921e6
RH		 1991 a2 = -a2;
1992 goto do_addi_32;
c2097136
RH		 1993 } else if (a0 == a1) {
1994 tcg_out_insn(s, RR, SR, a0, a2);
1995 } else {
1996 tcg_out_insn(s, RRF, SRK, a0, a1, a2);
48bb3750
RH		 1997 }
1998 break;
1999
2000 case INDEX_op_and_i32:
c2097136 2001 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
48bb3750 2002 if (const_args[2]) {
c2097136
RH		 2003 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
2004 tgen_andi(s, TCG_TYPE_I32, a0, a2);
2005 } else if (a0 == a1) {
2006 tcg_out_insn(s, RR, NR, a0, a2);
48bb3750 2007 } else {
c2097136 2008 tcg_out_insn(s, RRF, NRK, a0, a1, a2);
48bb3750
RH		 2009 }
2010 break;
2011 case INDEX_op_or_i32:
c2097136 2012 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
48bb3750 2013 if (const_args[2]) {
c2097136 2014 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
4046d9ca 2015 tgen_ori(s, TCG_TYPE_I32, a0, a2);
c2097136
RH		 2016 } else if (a0 == a1) {
2017 tcg_out_insn(s, RR, OR, a0, a2);
48bb3750 2018 } else {
c2097136 2019 tcg_out_insn(s, RRF, ORK, a0, a1, a2);
48bb3750
RH		 2020 }
2021 break;
2022 case INDEX_op_xor_i32:
c2097136 2023 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
48bb3750 2024 if (const_args[2]) {
c2097136 2025 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
5bf67a92 2026 tgen_xori(s, TCG_TYPE_I32, a0, a2);
c2097136 2027 } else if (a0 == a1) {
48bb3750 2028 tcg_out_insn(s, RR, XR, args[0], args[2]);
c2097136
RH		 2029 } else {
2030 tcg_out_insn(s, RRF, XRK, a0, a1, a2);
48bb3750
RH		 2031 }
2032 break;
2033
2034 case INDEX_op_neg_i32:
2035 tcg_out_insn(s, RR, LCR, args[0], args[1]);
2036 break;
2037
2038 case INDEX_op_mul_i32:
2039 if (const_args[2]) {
2040 if ((int32_t)args[2] == (int16_t)args[2]) {
2041 tcg_out_insn(s, RI, MHI, args[0], args[2]);
2042 } else {
2043 tcg_out_insn(s, RIL, MSFI, args[0], args[2]);
2044 }
2045 } else {
2046 tcg_out_insn(s, RRE, MSR, args[0], args[2]);
2047 }
2048 break;
2049
2050 case INDEX_op_div2_i32:
4143f78d
RH		 2051 tcg_debug_assert(args[0] == args[2]);
2052 tcg_debug_assert(args[1] == args[3]);
2053 tcg_debug_assert((args[1] & 1) == 0);
2054 tcg_debug_assert(args[0] == args[1] + 1);
2055 tcg_out_insn(s, RR, DR, args[1], args[4]);
48bb3750
RH		 2056 break;
2057 case INDEX_op_divu2_i32:
4143f78d
RH		 2058 tcg_debug_assert(args[0] == args[2]);
2059 tcg_debug_assert(args[1] == args[3]);
2060 tcg_debug_assert((args[1] & 1) == 0);
2061 tcg_debug_assert(args[0] == args[1] + 1);
2062 tcg_out_insn(s, RRE, DLR, args[1], args[4]);
48bb3750
RH		 2063 break;
2064
2065 case INDEX_op_shl_i32:
2066 op = RS_SLL;
c2097136 2067 op2 = RSY_SLLK;
48bb3750 2068 do_shift32:
c2097136
RH		 2069 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
2070 if (a0 == a1) {
2071 if (const_args[2]) {
2072 tcg_out_sh32(s, op, a0, TCG_REG_NONE, a2);
2073 } else {
2074 tcg_out_sh32(s, op, a0, a2, 0);
2075 }
48bb3750 2076 } else {
c2097136
RH		 2077 /* Using tcg_out_sh64 here for the format; it is a 32-bit shift. */
2078 if (const_args[2]) {
2079 tcg_out_sh64(s, op2, a0, a1, TCG_REG_NONE, a2);
2080 } else {
2081 tcg_out_sh64(s, op2, a0, a1, a2, 0);
2082 }
48bb3750
RH		 2083 }
2084 break;
2085 case INDEX_op_shr_i32:
2086 op = RS_SRL;
c2097136 2087 op2 = RSY_SRLK;
48bb3750
RH		 2088 goto do_shift32;
2089 case INDEX_op_sar_i32:
2090 op = RS_SRA;
c2097136 2091 op2 = RSY_SRAK;
48bb3750
RH		 2092 goto do_shift32;
2093
2094 case INDEX_op_rotl_i32:
2095 /* ??? Using tcg_out_sh64 here for the format; it is a 32-bit rol. */
2096 if (const_args[2]) {
2097 tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_REG_NONE, args[2]);
2098 } else {
2099 tcg_out_sh64(s, RSY_RLL, args[0], args[1], args[2], 0);
2100 }
2101 break;
2102 case INDEX_op_rotr_i32:
2103 if (const_args[2]) {
2104 tcg_out_sh64(s, RSY_RLL, args[0], args[1],
2105 TCG_REG_NONE, (32 - args[2]) & 31);
2106 } else {
2107 tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
2108 tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_TMP0, 0);
2109 }
2110 break;
2111
2112 case INDEX_op_ext8s_i32:
2113 tgen_ext8s(s, TCG_TYPE_I32, args[0], args[1]);
2114 break;
2115 case INDEX_op_ext16s_i32:
2116 tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
2117 break;
2118 case INDEX_op_ext8u_i32:
2119 tgen_ext8u(s, TCG_TYPE_I32, args[0], args[1]);
2120 break;
2121 case INDEX_op_ext16u_i32:
2122 tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
2123 break;
2124
1619ee9e
RH		 2125 case INDEX_op_bswap16_i32:
2126 a0 = args[0], a1 = args[1], a2 = args[2];
2127 tcg_out_insn(s, RRE, LRVR, a0, a1);
2128 if (a2 & TCG_BSWAP_OS) {
2129 tcg_out_sh32(s, RS_SRA, a0, TCG_REG_NONE, 16);
2130 } else {
2131 tcg_out_sh32(s, RS_SRL, a0, TCG_REG_NONE, 16);
2132 }
48bb3750 2133 break;
1619ee9e
RH		 2134 case INDEX_op_bswap16_i64:
2135 a0 = args[0], a1 = args[1], a2 = args[2];
2136 tcg_out_insn(s, RRE, LRVGR, a0, a1);
2137 if (a2 & TCG_BSWAP_OS) {
2138 tcg_out_sh64(s, RSY_SRAG, a0, a0, TCG_REG_NONE, 48);
2139 } else {
2140 tcg_out_sh64(s, RSY_SRLG, a0, a0, TCG_REG_NONE, 48);
2141 }
2142 break;
2143
2144 case INDEX_op_bswap32_i32:
48bb3750
RH		 2145 tcg_out_insn(s, RRE, LRVR, args[0], args[1]);
2146 break;
1619ee9e
RH		 2147 case INDEX_op_bswap32_i64:
2148 a0 = args[0], a1 = args[1], a2 = args[2];
2149 tcg_out_insn(s, RRE, LRVR, a0, a1);
2150 if (a2 & TCG_BSWAP_OS) {
2151 tgen_ext32s(s, a0, a0);
2152 } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
2153 tgen_ext32u(s, a0, a0);
2154 }
2155 break;
48bb3750 2156
3790b918 2157 case INDEX_op_add2_i32:
ad19b358
RH		 2158 if (const_args[4]) {
2159 tcg_out_insn(s, RIL, ALFI, args[0], args[4]);
2160 } else {
2161 tcg_out_insn(s, RR, ALR, args[0], args[4]);
2162 }
3790b918
RH		 2163 tcg_out_insn(s, RRE, ALCR, args[1], args[5]);
2164 break;
2165 case INDEX_op_sub2_i32:
ad19b358
RH		 2166 if (const_args[4]) {
2167 tcg_out_insn(s, RIL, SLFI, args[0], args[4]);
2168 } else {
2169 tcg_out_insn(s, RR, SLR, args[0], args[4]);
2170 }
3790b918
RH		 2171 tcg_out_insn(s, RRE, SLBR, args[1], args[5]);
2172 break;
2173
48bb3750 2174 case INDEX_op_br:
bec16311 2175 tgen_branch(s, S390_CC_ALWAYS, arg_label(args[0]));
48bb3750
RH		 2176 break;
2177
2178 case INDEX_op_brcond_i32:
2179 tgen_brcond(s, TCG_TYPE_I32, args[2], args[0],
bec16311 2180 args[1], const_args[1], arg_label(args[3]));
48bb3750
RH		 2181 break;
2182 case INDEX_op_setcond_i32:
2183 tgen_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1],
2184 args[2], const_args[2]);
2185 break;
96a9f093
RH		 2186 case INDEX_op_movcond_i32:
2187 tgen_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1],
7af525af 2188 args[2], const_args[2], args[3], const_args[3]);
96a9f093 2189 break;
48bb3750 2190
f24efee4 2191 case INDEX_op_qemu_ld_i32:
48bb3750 2192 /* ??? Technically we can use a non-extending instruction. */
f24efee4 2193 case INDEX_op_qemu_ld_i64:
59227d5d 2194 tcg_out_qemu_ld(s, args[0], args[1], args[2]);
48bb3750 2195 break;
f24efee4
RH		 2196 case INDEX_op_qemu_st_i32:
2197 case INDEX_op_qemu_st_i64:
59227d5d 2198 tcg_out_qemu_st(s, args[0], args[1], args[2]);
48bb3750
RH		 2199 break;
2200
48bb3750
RH		 2201 case INDEX_op_ld16s_i64:
2202 tcg_out_mem(s, 0, RXY_LGH, args[0], args[1], TCG_REG_NONE, args[2]);
2203 break;
2204 case INDEX_op_ld32u_i64:
2205 tcg_out_mem(s, 0, RXY_LLGF, args[0], args[1], TCG_REG_NONE, args[2]);
2206 break;
2207 case INDEX_op_ld32s_i64:
2208 tcg_out_mem(s, 0, RXY_LGF, args[0], args[1], TCG_REG_NONE, args[2]);
2209 break;
2210 case INDEX_op_ld_i64:
2211 tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
2212 break;
2213
2214 case INDEX_op_st32_i64:
2215 tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
2216 break;
2217 case INDEX_op_st_i64:
2218 tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
2219 break;
2220
2221 case INDEX_op_add_i64:
0db921e6 2222 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2223 if (const_args[2]) {
0db921e6
RH		 2224 do_addi_64:
2225 if (a0 == a1) {
2226 if (a2 == (int16_t)a2) {
2227 tcg_out_insn(s, RI, AGHI, a0, a2);
2228 break;
2229 }
3e25f7da
RH		 2230 if (a2 == (int32_t)a2) {
2231 tcg_out_insn(s, RIL, AGFI, a0, a2);
2232 break;
2233 }
2234 if (a2 == (uint32_t)a2) {
2235 tcg_out_insn(s, RIL, ALGFI, a0, a2);
2236 break;
2237 }
2238 if (-a2 == (uint32_t)-a2) {
2239 tcg_out_insn(s, RIL, SLGFI, a0, -a2);
2240 break;
0db921e6
RH		 2241 }
2242 }
2243 tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
2244 } else if (a0 == a1) {
2245 tcg_out_insn(s, RRE, AGR, a0, a2);
48bb3750 2246 } else {
0db921e6 2247 tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
48bb3750
RH		 2248 }
2249 break;
2250 case INDEX_op_sub_i64:
0db921e6 2251 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2252 if (const_args[2]) {
0db921e6
RH		 2253 a2 = -a2;
2254 goto do_addi_64;
c2097136
RH		 2255 } else if (a0 == a1) {
2256 tcg_out_insn(s, RRE, SGR, a0, a2);
48bb3750 2257 } else {
c2097136 2258 tcg_out_insn(s, RRF, SGRK, a0, a1, a2);
48bb3750
RH		 2259 }
2260 break;
2261
2262 case INDEX_op_and_i64:
c2097136 2263 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2264 if (const_args[2]) {
c2097136 2265 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
07ff7983 2266 tgen_andi(s, TCG_TYPE_I64, args[0], args[2]);
c2097136 2267 } else if (a0 == a1) {
48bb3750 2268 tcg_out_insn(s, RRE, NGR, args[0], args[2]);
c2097136
RH		 2269 } else {
2270 tcg_out_insn(s, RRF, NGRK, a0, a1, a2);
48bb3750
RH		 2271 }
2272 break;
2273 case INDEX_op_or_i64:
c2097136 2274 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2275 if (const_args[2]) {
c2097136 2276 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
4046d9ca 2277 tgen_ori(s, TCG_TYPE_I64, a0, a2);
c2097136
RH		 2278 } else if (a0 == a1) {
2279 tcg_out_insn(s, RRE, OGR, a0, a2);
48bb3750 2280 } else {
c2097136 2281 tcg_out_insn(s, RRF, OGRK, a0, a1, a2);
48bb3750
RH		 2282 }
2283 break;
2284 case INDEX_op_xor_i64:
c2097136 2285 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2286 if (const_args[2]) {
c2097136 2287 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
5bf67a92 2288 tgen_xori(s, TCG_TYPE_I64, a0, a2);
c2097136
RH		 2289 } else if (a0 == a1) {
2290 tcg_out_insn(s, RRE, XGR, a0, a2);
48bb3750 2291 } else {
c2097136 2292 tcg_out_insn(s, RRF, XGRK, a0, a1, a2);
48bb3750
RH		 2293 }
2294 break;
2295
2296 case INDEX_op_neg_i64:
2297 tcg_out_insn(s, RRE, LCGR, args[0], args[1]);
2298 break;
2299 case INDEX_op_bswap64_i64:
2300 tcg_out_insn(s, RRE, LRVGR, args[0], args[1]);
2301 break;
2302
2303 case INDEX_op_mul_i64:
2304 if (const_args[2]) {
2305 if (args[2] == (int16_t)args[2]) {
2306 tcg_out_insn(s, RI, MGHI, args[0], args[2]);
2307 } else {
2308 tcg_out_insn(s, RIL, MSGFI, args[0], args[2]);
2309 }
2310 } else {
2311 tcg_out_insn(s, RRE, MSGR, args[0], args[2]);
2312 }
2313 break;
2314
2315 case INDEX_op_div2_i64:
4143f78d
RH		 2316 /*
2317 * ??? We get an unnecessary sign-extension of the dividend
2318 * into op0 with this definition, but as we do in fact always
2319 * produce both quotient and remainder using INDEX_op_div_i64
2320 * instead requires jumping through even more hoops.
2321 */
2322 tcg_debug_assert(args[0] == args[2]);
2323 tcg_debug_assert(args[1] == args[3]);
2324 tcg_debug_assert((args[1] & 1) == 0);
2325 tcg_debug_assert(args[0] == args[1] + 1);
2326 tcg_out_insn(s, RRE, DSGR, args[1], args[4]);
48bb3750
RH		 2327 break;
2328 case INDEX_op_divu2_i64:
4143f78d
RH		 2329 tcg_debug_assert(args[0] == args[2]);
2330 tcg_debug_assert(args[1] == args[3]);
2331 tcg_debug_assert((args[1] & 1) == 0);
2332 tcg_debug_assert(args[0] == args[1] + 1);
2333 tcg_out_insn(s, RRE, DLGR, args[1], args[4]);
48bb3750 2334 break;
36017dc6 2335 case INDEX_op_mulu2_i64:
4143f78d
RH		 2336 tcg_debug_assert(args[0] == args[2]);
2337 tcg_debug_assert((args[1] & 1) == 0);
2338 tcg_debug_assert(args[0] == args[1] + 1);
2339 tcg_out_insn(s, RRE, MLGR, args[1], args[3]);
36017dc6 2340 break;
48bb3750
RH		 2341
2342 case INDEX_op_shl_i64:
2343 op = RSY_SLLG;
2344 do_shift64:
2345 if (const_args[2]) {
2346 tcg_out_sh64(s, op, args[0], args[1], TCG_REG_NONE, args[2]);
2347 } else {
2348 tcg_out_sh64(s, op, args[0], args[1], args[2], 0);
2349 }
2350 break;
2351 case INDEX_op_shr_i64:
2352 op = RSY_SRLG;
2353 goto do_shift64;
2354 case INDEX_op_sar_i64:
2355 op = RSY_SRAG;
2356 goto do_shift64;
2357
2358 case INDEX_op_rotl_i64:
2359 if (const_args[2]) {
2360 tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
2361 TCG_REG_NONE, args[2]);
2362 } else {
2363 tcg_out_sh64(s, RSY_RLLG, args[0], args[1], args[2], 0);
2364 }
2365 break;
2366 case INDEX_op_rotr_i64:
2367 if (const_args[2]) {
2368 tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
2369 TCG_REG_NONE, (64 - args[2]) & 63);
2370 } else {
2371 /* We can use the smaller 32-bit negate because only the
2372 low 6 bits are examined for the rotate. */
2373 tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
2374 tcg_out_sh64(s, RSY_RLLG, args[0], args[1], TCG_TMP0, 0);
2375 }
2376 break;
2377
2378 case INDEX_op_ext8s_i64:
2379 tgen_ext8s(s, TCG_TYPE_I64, args[0], args[1]);
2380 break;
2381 case INDEX_op_ext16s_i64:
2382 tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
2383 break;
4f2331e5 2384 case INDEX_op_ext_i32_i64:
48bb3750
RH		 2385 case INDEX_op_ext32s_i64:
2386 tgen_ext32s(s, args[0], args[1]);
2387 break;
2388 case INDEX_op_ext8u_i64:
2389 tgen_ext8u(s, TCG_TYPE_I64, args[0], args[1]);
2390 break;
2391 case INDEX_op_ext16u_i64:
2392 tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
2393 break;
4f2331e5 2394 case INDEX_op_extu_i32_i64:
48bb3750
RH		 2395 case INDEX_op_ext32u_i64:
2396 tgen_ext32u(s, args[0], args[1]);
2397 break;
2398
3790b918 2399 case INDEX_op_add2_i64:
ad19b358
RH		 2400 if (const_args[4]) {
2401 if ((int64_t)args[4] >= 0) {
2402 tcg_out_insn(s, RIL, ALGFI, args[0], args[4]);
2403 } else {
2404 tcg_out_insn(s, RIL, SLGFI, args[0], -args[4]);
2405 }
2406 } else {
2407 tcg_out_insn(s, RRE, ALGR, args[0], args[4]);
2408 }
3790b918
RH		 2409 tcg_out_insn(s, RRE, ALCGR, args[1], args[5]);
2410 break;
2411 case INDEX_op_sub2_i64:
ad19b358
RH		 2412 if (const_args[4]) {
2413 if ((int64_t)args[4] >= 0) {
2414 tcg_out_insn(s, RIL, SLGFI, args[0], args[4]);
2415 } else {
2416 tcg_out_insn(s, RIL, ALGFI, args[0], -args[4]);
2417 }
2418 } else {
2419 tcg_out_insn(s, RRE, SLGR, args[0], args[4]);
2420 }
3790b918
RH		 2421 tcg_out_insn(s, RRE, SLBGR, args[1], args[5]);
2422 break;
2423
48bb3750
RH		 2424 case INDEX_op_brcond_i64:
2425 tgen_brcond(s, TCG_TYPE_I64, args[2], args[0],
bec16311 2426 args[1], const_args[1], arg_label(args[3]));
48bb3750
RH		 2427 break;
2428 case INDEX_op_setcond_i64:
2429 tgen_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1],
2430 args[2], const_args[2]);
2431 break;
96a9f093
RH		 2432 case INDEX_op_movcond_i64:
2433 tgen_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1],
7af525af 2434 args[2], const_args[2], args[3], const_args[3]);
96a9f093 2435 break;
48bb3750 2436
d5690ea4 2437 OP_32_64(deposit):
752b1be9
RH		 2438 a0 = args[0], a1 = args[1], a2 = args[2];
2439 if (const_args[1]) {
2440 tgen_deposit(s, a0, a2, args[3], args[4], 1);
2441 } else {
2442 /* Since we can't support "0Z" as a constraint, we allow a1 in
2443 any register. Fix things up as if a matching constraint. */
2444 if (a0 != a1) {
2445 TCGType type = (opc == INDEX_op_deposit_i64);
2446 if (a0 == a2) {
2447 tcg_out_mov(s, type, TCG_TMP0, a2);
2448 a2 = TCG_TMP0;
2449 }
2450 tcg_out_mov(s, type, a0, a1);
2451 }
2452 tgen_deposit(s, a0, a2, args[3], args[4], 0);
2453 }
d5690ea4 2454 break;
752b1be9 2455
b0bf5fe8
RH		 2456 OP_32_64(extract):
2457 tgen_extract(s, args[0], args[1], args[2], args[3]);
2458 break;
d5690ea4 2459
ce411066
RH		 2460 case INDEX_op_clz_i64:
2461 tgen_clz(s, args[0], args[1], args[2], const_args[2]);
2462 break;
2463
c9314d61
PK		 2464 case INDEX_op_mb:
2465 /* The host memory model is quite strong, we simply need to
2466 serialize the instruction stream. */
2467 if (args[0] & TCG_MO_ST_LD) {
748b7f3e 2468 tcg_out_insn(s, RR, BCR, HAVE_FACILITY(FAST_BCR_SER) ? 14 : 15, 0);
c9314d61
PK		 2469 }
2470 break;
2471
96d0ee7f
RH		 2472 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
2473 case INDEX_op_mov_i64:
96d0ee7f 2474 case INDEX_op_call: /* Always emitted via tcg_out_call. */
48bb3750 2475 default:
48bb3750
RH		 2476 tcg_abort();
2477 }
2827822e
AG		 2478}
2479
34ef7676
RH		 2480static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
2481 TCGReg dst, TCGReg src)
2482{
79cada86
RH		 2483 if (is_general_reg(src)) {
2484 /* Replicate general register into two MO_64. */
2485 tcg_out_insn(s, VRRf, VLVGP, dst, src, src);
2486 if (vece == MO_64) {
2487 return true;
2488 }
6e591a85 2489 src = dst;
79cada86
RH		 2490 }
2491
2492 /*
2493 * Recall that the "standard" integer, within a vector, is the
2494 * rightmost element of the leftmost doubleword, a-la VLLEZ.
2495 */
2496 tcg_out_insn(s, VRIc, VREP, dst, (8 >> vece) - 1, src, vece);
2497 return true;
34ef7676
RH		 2498}
2499
2500static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
2501 TCGReg dst, TCGReg base, intptr_t offset)
2502{
79cada86
RH		 2503 tcg_out_vrx_mem(s, VRX_VLREP, dst, base, TCG_REG_NONE, offset, vece);
2504 return true;
34ef7676
RH		 2505}
2506
2507static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
2508 TCGReg dst, int64_t val)
2509{
79cada86
RH		 2510 int i, mask, msb, lsb;
2511
2512 /* Look for int16_t elements. */
2513 if (vece <= MO_16 ||
2514 (vece == MO_32 ? (int32_t)val : val) == (int16_t)val) {
2515 tcg_out_insn(s, VRIa, VREPI, dst, val, vece);
2516 return;
2517 }
2518
2519 /* Look for bit masks. */
2520 if (vece == MO_32) {
2521 if (risbg_mask((int32_t)val)) {
2522 /* Handle wraparound by swapping msb and lsb. */
2523 if ((val & 0x80000001u) == 0x80000001u) {
2524 msb = 32 - ctz32(~val);
2525 lsb = clz32(~val) - 1;
2526 } else {
2527 msb = clz32(val);
2528 lsb = 31 - ctz32(val);
2529 }
a66669c9 2530 tcg_out_insn(s, VRIb, VGM, dst, msb, lsb, MO_32);
79cada86
RH		 2531 return;
2532 }
2533 } else {
2534 if (risbg_mask(val)) {
2535 /* Handle wraparound by swapping msb and lsb. */
2536 if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
2537 /* Handle wraparound by swapping msb and lsb. */
2538 msb = 64 - ctz64(~val);
2539 lsb = clz64(~val) - 1;
2540 } else {
2541 msb = clz64(val);
2542 lsb = 63 - ctz64(val);
2543 }
a66669c9 2544 tcg_out_insn(s, VRIb, VGM, dst, msb, lsb, MO_64);
79cada86
RH		 2545 return;
2546 }
2547 }
2548
2549 /* Look for all bytes 0x00 or 0xff. */
2550 for (i = mask = 0; i < 8; i++) {
2551 uint8_t byte = val >> (i * 8);
2552 if (byte == 0xff) {
2553 mask |= 1 << i;
2554 } else if (byte != 0) {
2555 break;
2556 }
2557 }
2558 if (i == 8) {
2559 tcg_out_insn(s, VRIa, VGBM, dst, mask * 0x0101, 0);
2560 return;
2561 }
2562
2563 /* Otherwise, stuff it in the constant pool. */
2564 tcg_out_insn(s, RIL, LARL, TCG_TMP0, 0);
2565 new_pool_label(s, val, R_390_PC32DBL, s->code_ptr - 2, 2);
2566 tcg_out_insn(s, VRX, VLREP, dst, TCG_TMP0, TCG_REG_NONE, 0, MO_64);
34ef7676
RH		 2567}
2568
2569static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2570 unsigned vecl, unsigned vece,
d58f0173
MR		 2571 const TCGArg args[TCG_MAX_OP_ARGS],
2572 const int const_args[TCG_MAX_OP_ARGS])
34ef7676 2573{
a429ee29
RH		 2574 TCGType type = vecl + TCG_TYPE_V64;
2575 TCGArg a0 = args[0], a1 = args[1], a2 = args[2];
2576
2577 switch (opc) {
2578 case INDEX_op_ld_vec:
2579 tcg_out_ld(s, type, a0, a1, a2);
2580 break;
2581 case INDEX_op_st_vec:
2582 tcg_out_st(s, type, a0, a1, a2);
2583 break;
2584 case INDEX_op_dupm_vec:
2585 tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
2586 break;
2587
ae77bbe5
RH		 2588 case INDEX_op_abs_vec:
2589 tcg_out_insn(s, VRRa, VLP, a0, a1, vece);
2590 break;
2591 case INDEX_op_neg_vec:
2592 tcg_out_insn(s, VRRa, VLC, a0, a1, vece);
2593 break;
2594 case INDEX_op_not_vec:
2595 tcg_out_insn(s, VRRc, VNO, a0, a1, a1, 0);
2596 break;
2597
a429ee29
RH		 2598 case INDEX_op_add_vec:
2599 tcg_out_insn(s, VRRc, VA, a0, a1, a2, vece);
2600 break;
2601 case INDEX_op_sub_vec:
2602 tcg_out_insn(s, VRRc, VS, a0, a1, a2, vece);
2603 break;
2604 case INDEX_op_and_vec:
2605 tcg_out_insn(s, VRRc, VN, a0, a1, a2, 0);
2606 break;
ae77bbe5
RH		 2607 case INDEX_op_andc_vec:
2608 tcg_out_insn(s, VRRc, VNC, a0, a1, a2, 0);
2609 break;
479b61cb
RH		 2610 case INDEX_op_mul_vec:
2611 tcg_out_insn(s, VRRc, VML, a0, a1, a2, vece);
2612 break;
a429ee29
RH		 2613 case INDEX_op_or_vec:
2614 tcg_out_insn(s, VRRc, VO, a0, a1, a2, 0);
2615 break;
ae77bbe5
RH		 2616 case INDEX_op_orc_vec:
2617 tcg_out_insn(s, VRRc, VOC, a0, a1, a2, 0);
2618 break;
a429ee29
RH		 2619 case INDEX_op_xor_vec:
2620 tcg_out_insn(s, VRRc, VX, a0, a1, a2, 0);
2621 break;
21eab5bf
RH		 2622 case INDEX_op_nand_vec:
2623 tcg_out_insn(s, VRRc, VNN, a0, a1, a2, 0);
2624 break;
2625 case INDEX_op_nor_vec:
2626 tcg_out_insn(s, VRRc, VNO, a0, a1, a2, 0);
2627 break;
2628 case INDEX_op_eqv_vec:
2629 tcg_out_insn(s, VRRc, VNX, a0, a1, a2, 0);
2630 break;
a429ee29 2631
22cb37b4
RH		 2632 case INDEX_op_shli_vec:
2633 tcg_out_insn(s, VRSa, VESL, a0, a2, TCG_REG_NONE, a1, vece);
2634 break;
2635 case INDEX_op_shri_vec:
2636 tcg_out_insn(s, VRSa, VESRL, a0, a2, TCG_REG_NONE, a1, vece);
2637 break;
2638 case INDEX_op_sari_vec:
2639 tcg_out_insn(s, VRSa, VESRA, a0, a2, TCG_REG_NONE, a1, vece);
2640 break;
2641 case INDEX_op_rotli_vec:
2642 tcg_out_insn(s, VRSa, VERLL, a0, a2, TCG_REG_NONE, a1, vece);
2643 break;
2644 case INDEX_op_shls_vec:
2645 tcg_out_insn(s, VRSa, VESL, a0, 0, a2, a1, vece);
2646 break;
2647 case INDEX_op_shrs_vec:
2648 tcg_out_insn(s, VRSa, VESRL, a0, 0, a2, a1, vece);
2649 break;
2650 case INDEX_op_sars_vec:
2651 tcg_out_insn(s, VRSa, VESRA, a0, 0, a2, a1, vece);
2652 break;
2653 case INDEX_op_rotls_vec:
2654 tcg_out_insn(s, VRSa, VERLL, a0, 0, a2, a1, vece);
2655 break;
2656 case INDEX_op_shlv_vec:
2657 tcg_out_insn(s, VRRc, VESLV, a0, a1, a2, vece);
2658 break;
2659 case INDEX_op_shrv_vec:
2660 tcg_out_insn(s, VRRc, VESRLV, a0, a1, a2, vece);
2661 break;
2662 case INDEX_op_sarv_vec:
2663 tcg_out_insn(s, VRRc, VESRAV, a0, a1, a2, vece);
2664 break;
2665 case INDEX_op_rotlv_vec:
2666 tcg_out_insn(s, VRRc, VERLLV, a0, a1, a2, vece);
2667 break;
2668
220db7a6
RH		 2669 case INDEX_op_smin_vec:
2670 tcg_out_insn(s, VRRc, VMN, a0, a1, a2, vece);
2671 break;
2672 case INDEX_op_smax_vec:
2673 tcg_out_insn(s, VRRc, VMX, a0, a1, a2, vece);
2674 break;
2675 case INDEX_op_umin_vec:
2676 tcg_out_insn(s, VRRc, VMNL, a0, a1, a2, vece);
2677 break;
2678 case INDEX_op_umax_vec:
2679 tcg_out_insn(s, VRRc, VMXL, a0, a1, a2, vece);
2680 break;
2681
9bca986d 2682 case INDEX_op_bitsel_vec:
6e5f9fb7 2683 tcg_out_insn(s, VRRe, VSEL, a0, a2, args[3], a1);
9bca986d
RH		 2684 break;
2685
a429ee29
RH		 2686 case INDEX_op_cmp_vec:
2687 switch ((TCGCond)args[3]) {
2688 case TCG_COND_EQ:
2689 tcg_out_insn(s, VRRc, VCEQ, a0, a1, a2, vece);
2690 break;
2691 case TCG_COND_GT:
2692 tcg_out_insn(s, VRRc, VCH, a0, a1, a2, vece);
2693 break;
2694 case TCG_COND_GTU:
2695 tcg_out_insn(s, VRRc, VCHL, a0, a1, a2, vece);
2696 break;
2697 default:
2698 g_assert_not_reached();
2699 }
2700 break;
2701
4223c9c1
RH		 2702 case INDEX_op_s390_vuph_vec:
2703 tcg_out_insn(s, VRRa, VUPH, a0, a1, vece);
2704 break;
2705 case INDEX_op_s390_vupl_vec:
2706 tcg_out_insn(s, VRRa, VUPL, a0, a1, vece);
2707 break;
2708 case INDEX_op_s390_vpks_vec:
2709 tcg_out_insn(s, VRRc, VPKS, a0, a1, a2, vece);
2710 break;
2711
a429ee29
RH		 2712 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
2713 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
2714 default:
2715 g_assert_not_reached();
2716 }
34ef7676
RH		 2717}
2718
2719int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
2720{
a429ee29 2721 switch (opc) {
ae77bbe5 2722 case INDEX_op_abs_vec:
a429ee29
RH		 2723 case INDEX_op_add_vec:
2724 case INDEX_op_and_vec:
ae77bbe5 2725 case INDEX_op_andc_vec:
9bca986d 2726 case INDEX_op_bitsel_vec:
21eab5bf
RH		 2727 case INDEX_op_eqv_vec:
2728 case INDEX_op_nand_vec:
ae77bbe5 2729 case INDEX_op_neg_vec:
21eab5bf 2730 case INDEX_op_nor_vec:
ae77bbe5 2731 case INDEX_op_not_vec:
a429ee29 2732 case INDEX_op_or_vec:
ae77bbe5 2733 case INDEX_op_orc_vec:
22cb37b4
RH		 2734 case INDEX_op_rotli_vec:
2735 case INDEX_op_rotls_vec:
2736 case INDEX_op_rotlv_vec:
2737 case INDEX_op_sari_vec:
2738 case INDEX_op_sars_vec:
2739 case INDEX_op_sarv_vec:
2740 case INDEX_op_shli_vec:
2741 case INDEX_op_shls_vec:
2742 case INDEX_op_shlv_vec:
2743 case INDEX_op_shri_vec:
2744 case INDEX_op_shrs_vec:
2745 case INDEX_op_shrv_vec:
220db7a6
RH		 2746 case INDEX_op_smax_vec:
2747 case INDEX_op_smin_vec:
a429ee29 2748 case INDEX_op_sub_vec:
220db7a6
RH		 2749 case INDEX_op_umax_vec:
2750 case INDEX_op_umin_vec:
a429ee29
RH		 2751 case INDEX_op_xor_vec:
2752 return 1;
2753 case INDEX_op_cmp_vec:
ea3f2af8 2754 case INDEX_op_cmpsel_vec:
22cb37b4 2755 case INDEX_op_rotrv_vec:
a429ee29 2756 return -1;
479b61cb
RH		 2757 case INDEX_op_mul_vec:
2758 return vece < MO_64;
4223c9c1
RH		 2759 case INDEX_op_ssadd_vec:
2760 case INDEX_op_sssub_vec:
2761 return vece < MO_64 ? -1 : 0;
a429ee29
RH		 2762 default:
2763 return 0;
2764 }
2765}
2766
2767static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0,
2768 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
2769{
2770 bool need_swap = false, need_inv = false;
2771
2772 switch (cond) {
2773 case TCG_COND_EQ:
2774 case TCG_COND_GT:
2775 case TCG_COND_GTU:
2776 break;
2777 case TCG_COND_NE:
2778 case TCG_COND_LE:
2779 case TCG_COND_LEU:
2780 need_inv = true;
2781 break;
2782 case TCG_COND_LT:
2783 case TCG_COND_LTU:
2784 need_swap = true;
2785 break;
2786 case TCG_COND_GE:
2787 case TCG_COND_GEU:
2788 need_swap = need_inv = true;
2789 break;
2790 default:
2791 g_assert_not_reached();
2792 }
2793
2794 if (need_inv) {
2795 cond = tcg_invert_cond(cond);
2796 }
2797 if (need_swap) {
2798 TCGv_vec t1;
2799 t1 = v1, v1 = v2, v2 = t1;
2800 cond = tcg_swap_cond(cond);
2801 }
2802
2803 vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0),
2804 tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond);
2805
2806 return need_inv;
2807}
2808
2809static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0,
2810 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
2811{
2812 if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) {
2813 tcg_gen_not_vec(vece, v0, v0);
2814 }
34ef7676
RH		 2815}
2816
ea3f2af8
RH		 2817static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0,
2818 TCGv_vec c1, TCGv_vec c2,
2819 TCGv_vec v3, TCGv_vec v4, TCGCond cond)
2820{
2821 TCGv_vec t = tcg_temp_new_vec(type);
2822
2823 if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) {
2824 /* Invert the sense of the compare by swapping arguments. */
2825 tcg_gen_bitsel_vec(vece, v0, t, v4, v3);
2826 } else {
2827 tcg_gen_bitsel_vec(vece, v0, t, v3, v4);
2828 }
2829 tcg_temp_free_vec(t);
2830}
2831
4223c9c1
RH		 2832static void expand_vec_sat(TCGType type, unsigned vece, TCGv_vec v0,
2833 TCGv_vec v1, TCGv_vec v2, TCGOpcode add_sub_opc)
2834{
2835 TCGv_vec h1 = tcg_temp_new_vec(type);
2836 TCGv_vec h2 = tcg_temp_new_vec(type);
2837 TCGv_vec l1 = tcg_temp_new_vec(type);
2838 TCGv_vec l2 = tcg_temp_new_vec(type);
2839
2840 tcg_debug_assert (vece < MO_64);
2841
2842 /* Unpack with sign-extension. */
2843 vec_gen_2(INDEX_op_s390_vuph_vec, type, vece,
2844 tcgv_vec_arg(h1), tcgv_vec_arg(v1));
2845 vec_gen_2(INDEX_op_s390_vuph_vec, type, vece,
2846 tcgv_vec_arg(h2), tcgv_vec_arg(v2));
2847
2848 vec_gen_2(INDEX_op_s390_vupl_vec, type, vece,
2849 tcgv_vec_arg(l1), tcgv_vec_arg(v1));
2850 vec_gen_2(INDEX_op_s390_vupl_vec, type, vece,
2851 tcgv_vec_arg(l2), tcgv_vec_arg(v2));
2852
2853 /* Arithmetic on a wider element size. */
2854 vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(h1),
2855 tcgv_vec_arg(h1), tcgv_vec_arg(h2));
2856 vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(l1),
2857 tcgv_vec_arg(l1), tcgv_vec_arg(l2));
2858
2859 /* Pack with saturation. */
2860 vec_gen_3(INDEX_op_s390_vpks_vec, type, vece + 1,
2861 tcgv_vec_arg(v0), tcgv_vec_arg(h1), tcgv_vec_arg(l1));
2862
2863 tcg_temp_free_vec(h1);
2864 tcg_temp_free_vec(h2);
2865 tcg_temp_free_vec(l1);
2866 tcg_temp_free_vec(l2);
2867}
2868
34ef7676
RH		 2869void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
2870 TCGArg a0, ...)
2871{
a429ee29 2872 va_list va;
ea3f2af8 2873 TCGv_vec v0, v1, v2, v3, v4, t0;
a429ee29
RH		 2874
2875 va_start(va, a0);
2876 v0 = temp_tcgv_vec(arg_temp(a0));
2877 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2878 v2 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2879
2880 switch (opc) {
2881 case INDEX_op_cmp_vec:
2882 expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg));
2883 break;
2884
ea3f2af8
RH		 2885 case INDEX_op_cmpsel_vec:
2886 v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2887 v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2888 expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg));
2889 break;
2890
22cb37b4
RH		 2891 case INDEX_op_rotrv_vec:
2892 t0 = tcg_temp_new_vec(type);
2893 tcg_gen_neg_vec(vece, t0, v2);
2894 tcg_gen_rotlv_vec(vece, v0, v1, t0);
2895 tcg_temp_free_vec(t0);
2896 break;
2897
4223c9c1
RH		 2898 case INDEX_op_ssadd_vec:
2899 expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_add_vec);
2900 break;
2901 case INDEX_op_sssub_vec:
2902 expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_sub_vec);
2903 break;
2904
a429ee29
RH		 2905 default:
2906 g_assert_not_reached();
2907 }
2908 va_end(va);
34ef7676
RH		 2909}
2910
d1c36a90 2911static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op)
f69d277e 2912{
9b5500b6
RH		 2913 switch (op) {
2914 case INDEX_op_goto_ptr:
d1c36a90 2915 return C_O0_I1(r);
9b5500b6
RH		 2916
2917 case INDEX_op_ld8u_i32:
2918 case INDEX_op_ld8u_i64:
2919 case INDEX_op_ld8s_i32:
2920 case INDEX_op_ld8s_i64:
2921 case INDEX_op_ld16u_i32:
2922 case INDEX_op_ld16u_i64:
2923 case INDEX_op_ld16s_i32:
2924 case INDEX_op_ld16s_i64:
2925 case INDEX_op_ld_i32:
2926 case INDEX_op_ld32u_i64:
2927 case INDEX_op_ld32s_i64:
2928 case INDEX_op_ld_i64:
d1c36a90
RH		 2929 return C_O1_I1(r, r);
2930
9b5500b6
RH		 2931 case INDEX_op_st8_i32:
2932 case INDEX_op_st8_i64:
2933 case INDEX_op_st16_i32:
2934 case INDEX_op_st16_i64:
2935 case INDEX_op_st_i32:
2936 case INDEX_op_st32_i64:
2937 case INDEX_op_st_i64:
d1c36a90 2938 return C_O0_I2(r, r);
9b5500b6
RH		 2939
2940 case INDEX_op_add_i32:
2941 case INDEX_op_add_i64:
d1c36a90
RH		 2942 case INDEX_op_shl_i64:
2943 case INDEX_op_shr_i64:
2944 case INDEX_op_sar_i64:
2945 case INDEX_op_rotl_i32:
2946 case INDEX_op_rotl_i64:
2947 case INDEX_op_rotr_i32:
2948 case INDEX_op_rotr_i64:
2949 case INDEX_op_clz_i64:
2950 case INDEX_op_setcond_i32:
2951 case INDEX_op_setcond_i64:
2952 return C_O1_I2(r, r, ri);
2953
9b5500b6
RH		 2954 case INDEX_op_sub_i32:
2955 case INDEX_op_sub_i64:
bdcd5d19
RH		 2956 case INDEX_op_and_i32:
2957 case INDEX_op_and_i64:
4046d9ca
RH		 2958 case INDEX_op_or_i32:
2959 case INDEX_op_or_i64:
5bf67a92
RH		 2960 case INDEX_op_xor_i32:
2961 case INDEX_op_xor_i64:
748b7f3e 2962 return (HAVE_FACILITY(DISTINCT_OPS)
d1c36a90
RH		 2963 ? C_O1_I2(r, r, ri)
2964 : C_O1_I2(r, 0, ri));
a8f0269e 2965
9b5500b6 2966 case INDEX_op_mul_i32:
9c3bfb79 2967 return C_O1_I2(r, 0, ri);
9b5500b6 2968 case INDEX_op_mul_i64:
9c3bfb79 2969 return C_O1_I2(r, 0, rJ);
a8f0269e 2970
9b5500b6
RH		 2971 case INDEX_op_shl_i32:
2972 case INDEX_op_shr_i32:
2973 case INDEX_op_sar_i32:
748b7f3e 2974 return (HAVE_FACILITY(DISTINCT_OPS)
d1c36a90
RH		 2975 ? C_O1_I2(r, r, ri)
2976 : C_O1_I2(r, 0, ri));
9b5500b6
RH		 2977
2978 case INDEX_op_brcond_i32:
2979 case INDEX_op_brcond_i64:
d1c36a90 2980 return C_O0_I2(r, ri);
9b5500b6
RH		 2981
2982 case INDEX_op_bswap16_i32:
2983 case INDEX_op_bswap16_i64:
2984 case INDEX_op_bswap32_i32:
2985 case INDEX_op_bswap32_i64:
2986 case INDEX_op_bswap64_i64:
2987 case INDEX_op_neg_i32:
2988 case INDEX_op_neg_i64:
2989 case INDEX_op_ext8s_i32:
2990 case INDEX_op_ext8s_i64:
2991 case INDEX_op_ext8u_i32:
2992 case INDEX_op_ext8u_i64:
2993 case INDEX_op_ext16s_i32:
2994 case INDEX_op_ext16s_i64:
2995 case INDEX_op_ext16u_i32:
2996 case INDEX_op_ext16u_i64:
2997 case INDEX_op_ext32s_i64:
2998 case INDEX_op_ext32u_i64:
2999 case INDEX_op_ext_i32_i64:
3000 case INDEX_op_extu_i32_i64:
3001 case INDEX_op_extract_i32:
3002 case INDEX_op_extract_i64:
d1c36a90 3003 return C_O1_I1(r, r);
9b5500b6
RH		 3004
3005 case INDEX_op_qemu_ld_i32:
3006 case INDEX_op_qemu_ld_i64:
d1c36a90 3007 return C_O1_I1(r, L);
9b5500b6
RH		 3008 case INDEX_op_qemu_st_i64:
3009 case INDEX_op_qemu_st_i32:
d1c36a90 3010 return C_O0_I2(L, L);
f69d277e 3011
9b5500b6
RH		 3012 case INDEX_op_deposit_i32:
3013 case INDEX_op_deposit_i64:
d1c36a90
RH		 3014 return C_O1_I2(r, rZ, r);
3015
9b5500b6
RH		 3016 case INDEX_op_movcond_i32:
3017 case INDEX_op_movcond_i64:
748b7f3e 3018 return (HAVE_FACILITY(LOAD_ON_COND2)
d1c36a90
RH		 3019 ? C_O1_I4(r, r, ri, rI, 0)
3020 : C_O1_I4(r, r, ri, r, 0));
3021
9b5500b6
RH		 3022 case INDEX_op_div2_i32:
3023 case INDEX_op_div2_i64:
3024 case INDEX_op_divu2_i32:
3025 case INDEX_op_divu2_i64:
4143f78d 3026 return C_O2_I3(o, m, 0, 1, r);
d1c36a90 3027
9b5500b6 3028 case INDEX_op_mulu2_i64:
4143f78d 3029 return C_O2_I2(o, m, 0, r);
ba18b07d 3030
9b5500b6 3031 case INDEX_op_add2_i32:
9b5500b6 3032 case INDEX_op_sub2_i32:
3e25f7da 3033 return C_O2_I4(r, r, 0, 1, ri, r);
d1c36a90 3034
ba18b07d 3035 case INDEX_op_add2_i64:
9b5500b6 3036 case INDEX_op_sub2_i64:
3e25f7da 3037 return C_O2_I4(r, r, 0, 1, rA, r);
9b5500b6 3038
34ef7676
RH		 3039 case INDEX_op_st_vec:
3040 return C_O0_I2(v, r);
3041 case INDEX_op_ld_vec:
3042 case INDEX_op_dupm_vec:
3043 return C_O1_I1(v, r);
3044 case INDEX_op_dup_vec:
3045 return C_O1_I1(v, vr);
ae77bbe5
RH		 3046 case INDEX_op_abs_vec:
3047 case INDEX_op_neg_vec:
3048 case INDEX_op_not_vec:
22cb37b4
RH		 3049 case INDEX_op_rotli_vec:
3050 case INDEX_op_sari_vec:
3051 case INDEX_op_shli_vec:
3052 case INDEX_op_shri_vec:
4223c9c1
RH		 3053 case INDEX_op_s390_vuph_vec:
3054 case INDEX_op_s390_vupl_vec:
ae77bbe5 3055 return C_O1_I1(v, v);
34ef7676
RH		 3056 case INDEX_op_add_vec:
3057 case INDEX_op_sub_vec:
3058 case INDEX_op_and_vec:
ae77bbe5 3059 case INDEX_op_andc_vec:
34ef7676 3060 case INDEX_op_or_vec:
ae77bbe5 3061 case INDEX_op_orc_vec:
34ef7676 3062 case INDEX_op_xor_vec:
21eab5bf
RH		 3063 case INDEX_op_nand_vec:
3064 case INDEX_op_nor_vec:
3065 case INDEX_op_eqv_vec:
34ef7676 3066 case INDEX_op_cmp_vec:
479b61cb 3067 case INDEX_op_mul_vec:
22cb37b4
RH		 3068 case INDEX_op_rotlv_vec:
3069 case INDEX_op_rotrv_vec:
3070 case INDEX_op_shlv_vec:
3071 case INDEX_op_shrv_vec:
3072 case INDEX_op_sarv_vec:
220db7a6
RH		 3073 case INDEX_op_smax_vec:
3074 case INDEX_op_smin_vec:
3075 case INDEX_op_umax_vec:
3076 case INDEX_op_umin_vec:
4223c9c1 3077 case INDEX_op_s390_vpks_vec:
34ef7676 3078 return C_O1_I2(v, v, v);
22cb37b4
RH		 3079 case INDEX_op_rotls_vec:
3080 case INDEX_op_shls_vec:
3081 case INDEX_op_shrs_vec:
3082 case INDEX_op_sars_vec:
3083 return C_O1_I2(v, v, r);
9bca986d
RH		 3084 case INDEX_op_bitsel_vec:
3085 return C_O1_I3(v, v, v, v);
34ef7676 3086
9b5500b6 3087 default:
d1c36a90 3088 g_assert_not_reached();
f69d277e 3089 }
f69d277e
RH		 3090}
3091
34ef7676
RH		 3092/*
3093 * Mainline glibc added HWCAP_S390_VX before it was kernel abi.
3094 * Some distros have fixed this up locally, others have not.
3095 */
3096#ifndef HWCAP_S390_VXRS
3097#define HWCAP_S390_VXRS 2048
3098#endif
3099
b2c98d9d 3100static void query_s390_facilities(void)
48bb3750 3101{
c9baa30f 3102 unsigned long hwcap = qemu_getauxval(AT_HWCAP);
761ea522 3103 const char *which;
48bb3750 3104
c9baa30f
RH		 3105 /* Is STORE FACILITY LIST EXTENDED available? Honestly, I believe this
3106 is present on all 64-bit systems, but let's check for it anyway. */
3107 if (hwcap & HWCAP_S390_STFLE) {
748b7f3e
RH		 3108 register int r0 __asm__("0") = ARRAY_SIZE(s390_facilities) - 1;
3109 register void *r1 __asm__("1") = s390_facilities;
48bb3750 3110
c9baa30f 3111 /* stfle 0(%r1) */
c9baa30f 3112 asm volatile(".word 0xb2b0,0x1000"
748b7f3e 3113 : "=r"(r0) : "r"(r0), "r"(r1) : "memory", "cc");
48bb3750 3114 }
34ef7676
RH		 3115
3116 /*
3117 * Use of vector registers requires os support beyond the facility bit.
3118 * If the kernel does not advertise support, disable the facility bits.
3119 * There is nothing else we currently care about in the 3rd word, so
3120 * disable VECTOR with one store.
3121 */
a429ee29 3122 if (!(hwcap & HWCAP_S390_VXRS)) {
34ef7676
RH		 3123 s390_facilities[2] = 0;
3124 }
761ea522
RH		 3125
3126 /*
3127 * Check for all required facilities.
3128 * ZARCH_ACTIVE is done via preprocessor check for 64-bit.
3129 */
3130 if (!HAVE_FACILITY(LONG_DISP)) {
3131 which = "long-displacement";
3132 goto fail;
3133 }
3e25f7da
RH		 3134 if (!HAVE_FACILITY(EXT_IMM)) {
3135 which = "extended-immediate";
3136 goto fail;
3137 }
9c3bfb79
RH		 3138 if (!HAVE_FACILITY(GEN_INST_EXT)) {
3139 which = "general-instructions-extension";
3140 goto fail;
3141 }
761ea522
RH		 3142 return;
3143
3144 fail:
3145 error_report("%s: missing required facility %s", __func__, which);
3146 exit(EXIT_FAILURE);
48bb3750
RH		 3147}
3148
3149static void tcg_target_init(TCGContext *s)
2827822e 3150{
b2c98d9d 3151 query_s390_facilities();
48bb3750 3152
f46934df
RH		 3153 tcg_target_available_regs[TCG_TYPE_I32] = 0xffff;
3154 tcg_target_available_regs[TCG_TYPE_I64] = 0xffff;
34ef7676
RH		 3155 if (HAVE_FACILITY(VECTOR)) {
3156 tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
3157 tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
3158 }
48bb3750 3159
ccb1bb66 3160 tcg_target_call_clobber_regs = 0;
48bb3750
RH		 3161 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
3162 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R1);
3163 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
3164 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
3165 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
3166 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
f24efee4
RH		 3167 /* The r6 register is technically call-saved, but it's also a parameter
3168 register, so it can get killed by setup for the qemu_st helper. */
3169 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
48bb3750
RH		 3170 /* The return register can be considered call-clobbered. */
3171 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14);
3172
34ef7676
RH		 3173 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V0);
3174 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V1);
3175 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V2);
3176 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V3);
3177 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V4);
3178 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V5);
3179 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V6);
3180 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V7);
3181 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V16);
3182 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V17);
3183 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V18);
3184 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V19);
3185 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V20);
3186 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V21);
3187 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V22);
3188 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V23);
3189 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V24);
3190 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V25);
3191 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V26);
3192 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V27);
3193 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V28);
3194 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V29);
3195 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V30);
3196 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V31);
3197
ccb1bb66 3198 s->reserved_regs = 0;
48bb3750
RH		 3199 tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);
3200 /* XXX many insns can't be used with R0, so we better avoid it for now */
3201 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);
3202 tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
2827822e
AG		 3203}
3204
f167dc37
RH		 3205#define FRAME_SIZE ((int)(TCG_TARGET_CALL_STACK_OFFSET \
3206 + TCG_STATIC_CALL_ARGS_SIZE \
3207 + CPU_TEMP_BUF_NLONGS * sizeof(long)))
3208
48bb3750 3209static void tcg_target_qemu_prologue(TCGContext *s)
2827822e 3210{
48bb3750
RH		 3211 /* stmg %r6,%r15,48(%r15) (save registers) */
3212 tcg_out_insn(s, RXY, STMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15, 48);
3213
a4924e8b 3214 /* aghi %r15,-frame_size */
f167dc37 3215 tcg_out_insn(s, RI, AGHI, TCG_REG_R15, -FRAME_SIZE);
a4924e8b
RH		 3216
3217 tcg_set_frame(s, TCG_REG_CALL_STACK,
3218 TCG_STATIC_CALL_ARGS_SIZE + TCG_TARGET_CALL_STACK_OFFSET,
3219 CPU_TEMP_BUF_NLONGS * sizeof(long));
48bb3750 3220
090d0bfd 3221#ifndef CONFIG_SOFTMMU
b76f21a7 3222 if (guest_base >= 0x80000) {
ccbecb44 3223 tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
48bb3750
RH		 3224 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
3225 }
090d0bfd 3226#endif
48bb3750 3227
cea5f9a2 3228 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
829e1376 3229
cea5f9a2
BS		 3230 /* br %r3 (go to TB) */
3231 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, tcg_target_call_iarg_regs[1]);
48bb3750 3232
46644483
RH		 3233 /*
3234 * Return path for goto_ptr. Set return value to 0, a-la exit_tb,
3235 * and fall through to the rest of the epilogue.
3236 */
c8bc1168 3237 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
46644483
RH		 3238 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, 0);
3239
3240 /* TB epilogue */
79dae4dd 3241 tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
48bb3750 3242
a4924e8b
RH		 3243 /* lmg %r6,%r15,fs+48(%r15) (restore registers) */
3244 tcg_out_insn(s, RXY, LMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15,
f167dc37 3245 FRAME_SIZE + 48);
48bb3750
RH		 3246
3247 /* br %r14 (return) */
3248 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_R14);
2827822e 3249}
f167dc37 3250
28eef8aa
RH		 3251static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
3252{
3253 memset(p, 0x07, count * sizeof(tcg_insn_unit));
3254}
3255
f167dc37 3256typedef struct {
d2e16f2c 3257 DebugFrameHeader h;
f167dc37
RH		 3258 uint8_t fde_def_cfa[4];
3259 uint8_t fde_reg_ofs[18];
3260} DebugFrame;
3261
3262/* We're expecting a 2 byte uleb128 encoded value. */
3263QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
3264
3265#define ELF_HOST_MACHINE EM_S390
3266
d2e16f2c
RH		 3267static const DebugFrame debug_frame = {
3268 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
3269 .h.cie.id = -1,
3270 .h.cie.version = 1,
3271 .h.cie.code_align = 1,
3272 .h.cie.data_align = 8, /* sleb128 8 */
3273 .h.cie.return_column = TCG_REG_R14,
f167dc37
RH		 3274
3275 /* Total FDE size does not include the "len" member. */
d2e16f2c 3276 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
f167dc37
RH		 3277
3278 .fde_def_cfa = {
3279 12, TCG_REG_CALL_STACK, /* DW_CFA_def_cfa %r15, ... */
3280 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
3281 (FRAME_SIZE >> 7)
3282 },
3283 .fde_reg_ofs = {
3284 0x86, 6, /* DW_CFA_offset, %r6, 48 */
3285 0x87, 7, /* DW_CFA_offset, %r7, 56 */
3286 0x88, 8, /* DW_CFA_offset, %r8, 64 */
3287 0x89, 9, /* DW_CFA_offset, %r92, 72 */
3288 0x8a, 10, /* DW_CFA_offset, %r10, 80 */
3289 0x8b, 11, /* DW_CFA_offset, %r11, 88 */
3290 0x8c, 12, /* DW_CFA_offset, %r12, 96 */
3291 0x8d, 13, /* DW_CFA_offset, %r13, 104 */
3292 0x8e, 14, /* DW_CFA_offset, %r14, 112 */
3293 }
3294};
3295
755bf9e5 3296void tcg_register_jit(const void *buf, size_t buf_size)
f167dc37 3297{
f167dc37
RH		 3298 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
3299}
QEMU mirror