]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/s390x/tcg-target.c.inc
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg/s390x: Check for load-on-condition facility at startup
[mirror_qemu.git] / tcg / s390x / tcg-target.c.inc
CommitLineData
2827822e
AG		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2009 Ulrich Hecht <uli@suse.de>
48bb3750
RH		 5 * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
6 * Copyright (c) 2010 Richard Henderson <rth@twiddle.net>
2827822e
AG		 7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a copy
9 * of this software and associated documentation files (the "Software"), to deal
10 * in the Software without restriction, including without limitation the rights
11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 * copies of the Software, and to permit persons to whom the Software is
13 * furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 * THE SOFTWARE.
25 */
26
a01fc30d
RH		 27/* We only support generating code for 64-bit mode. */
28#if TCG_TARGET_REG_BITS != 64
29#error "unsupported code generation mode"
30#endif
31
1cd49868 32#include "../tcg-ldst.c.inc"
139c1837 33#include "../tcg-pool.c.inc"
c9baa30f
RH		 34#include "elf.h"
35
a8f0269e
RH		 36#define TCG_CT_CONST_S16 0x100
37#define TCG_CT_CONST_S32 0x200
a534bb15
RH		 38#define TCG_CT_CONST_S33 0x400
39#define TCG_CT_CONST_ZERO 0x800
48bb3750 40
c947deb1 41#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16)
34ef7676
RH		 42#define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32)
43
c947deb1
RH		 44/*
45 * For softmmu, we need to avoid conflicts with the first 3
46 * argument registers to perform the tlb lookup, and to call
47 * the helper function.
48 */
49#ifdef CONFIG_SOFTMMU
50#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_R2, 3)
51#else
52#define SOFTMMU_RESERVE_REGS 0
53#endif
54
55
48bb3750
RH		 56/* Several places within the instruction set 0 means "no register"
57 rather than TCG_REG_R0. */
58#define TCG_REG_NONE 0
59
60/* A scratch register that may be be used throughout the backend. */
ce411066 61#define TCG_TMP0 TCG_REG_R1
48bb3750 62
4cbea598 63#ifndef CONFIG_SOFTMMU
48bb3750 64#define TCG_GUEST_BASE_REG TCG_REG_R13
48bb3750
RH		 65#endif
66
48bb3750
RH		 67/* All of the following instructions are prefixed with their instruction
68 format, and are defined as 8- or 16-bit quantities, even when the two
69 halves of the 16-bit quantity may appear 32 bits apart in the insn.
70 This makes it easy to copy the values from the tables in Appendix B. */
71typedef enum S390Opcode {
72 RIL_AFI = 0xc209,
73 RIL_AGFI = 0xc208,
3790b918 74 RIL_ALFI = 0xc20b,
48bb3750
RH		 75 RIL_ALGFI = 0xc20a,
76 RIL_BRASL = 0xc005,
77 RIL_BRCL = 0xc004,
78 RIL_CFI = 0xc20d,
79 RIL_CGFI = 0xc20c,
80 RIL_CLFI = 0xc20f,
81 RIL_CLGFI = 0xc20e,
a534bb15
RH		 82 RIL_CLRL = 0xc60f,
83 RIL_CLGRL = 0xc60a,
84 RIL_CRL = 0xc60d,
85 RIL_CGRL = 0xc608,
48bb3750
RH		 86 RIL_IIHF = 0xc008,
87 RIL_IILF = 0xc009,
88 RIL_LARL = 0xc000,
89 RIL_LGFI = 0xc001,
90 RIL_LGRL = 0xc408,
91 RIL_LLIHF = 0xc00e,
92 RIL_LLILF = 0xc00f,
93 RIL_LRL = 0xc40d,
94 RIL_MSFI = 0xc201,
95 RIL_MSGFI = 0xc200,
96 RIL_NIHF = 0xc00a,
97 RIL_NILF = 0xc00b,
98 RIL_OIHF = 0xc00c,
99 RIL_OILF = 0xc00d,
3790b918 100 RIL_SLFI = 0xc205,
0db921e6 101 RIL_SLGFI = 0xc204,
48bb3750
RH		 102 RIL_XIHF = 0xc006,
103 RIL_XILF = 0xc007,
104
105 RI_AGHI = 0xa70b,
106 RI_AHI = 0xa70a,
107 RI_BRC = 0xa704,
a534bb15
RH		 108 RI_CHI = 0xa70e,
109 RI_CGHI = 0xa70f,
48bb3750
RH		 110 RI_IIHH = 0xa500,
111 RI_IIHL = 0xa501,
112 RI_IILH = 0xa502,
113 RI_IILL = 0xa503,
114 RI_LGHI = 0xa709,
115 RI_LLIHH = 0xa50c,
116 RI_LLIHL = 0xa50d,
117 RI_LLILH = 0xa50e,
118 RI_LLILL = 0xa50f,
119 RI_MGHI = 0xa70d,
120 RI_MHI = 0xa70c,
121 RI_NIHH = 0xa504,
122 RI_NIHL = 0xa505,
123 RI_NILH = 0xa506,
124 RI_NILL = 0xa507,
125 RI_OIHH = 0xa508,
126 RI_OIHL = 0xa509,
127 RI_OILH = 0xa50a,
128 RI_OILL = 0xa50b,
1cd49868 129 RI_TMLL = 0xa701,
48bb3750
RH		 130
131 RIE_CGIJ = 0xec7c,
132 RIE_CGRJ = 0xec64,
133 RIE_CIJ = 0xec7e,
134 RIE_CLGRJ = 0xec65,
135 RIE_CLIJ = 0xec7f,
136 RIE_CLGIJ = 0xec7d,
137 RIE_CLRJ = 0xec77,
138 RIE_CRJ = 0xec76,
7af525af 139 RIE_LOCGHI = 0xec46,
d5690ea4 140 RIE_RISBG = 0xec55,
48bb3750
RH		 141
142 RRE_AGR = 0xb908,
3790b918
RH		 143 RRE_ALGR = 0xb90a,
144 RRE_ALCR = 0xb998,
145 RRE_ALCGR = 0xb988,
48bb3750
RH		 146 RRE_CGR = 0xb920,
147 RRE_CLGR = 0xb921,
148 RRE_DLGR = 0xb987,
149 RRE_DLR = 0xb997,
150 RRE_DSGFR = 0xb91d,
151 RRE_DSGR = 0xb90d,
ce411066 152 RRE_FLOGR = 0xb983,
48bb3750
RH		 153 RRE_LGBR = 0xb906,
154 RRE_LCGR = 0xb903,
155 RRE_LGFR = 0xb914,
156 RRE_LGHR = 0xb907,
157 RRE_LGR = 0xb904,
158 RRE_LLGCR = 0xb984,
159 RRE_LLGFR = 0xb916,
160 RRE_LLGHR = 0xb985,
161 RRE_LRVR = 0xb91f,
162 RRE_LRVGR = 0xb90f,
163 RRE_LTGR = 0xb902,
36017dc6 164 RRE_MLGR = 0xb986,
48bb3750
RH		 165 RRE_MSGR = 0xb90c,
166 RRE_MSR = 0xb252,
167 RRE_NGR = 0xb980,
168 RRE_OGR = 0xb981,
169 RRE_SGR = 0xb909,
3790b918
RH		 170 RRE_SLGR = 0xb90b,
171 RRE_SLBR = 0xb999,
172 RRE_SLBGR = 0xb989,
48bb3750
RH		 173 RRE_XGR = 0xb982,
174
96a9f093
RH		 175 RRF_LOCR = 0xb9f2,
176 RRF_LOCGR = 0xb9e2,
c2097136
RH		 177 RRF_NRK = 0xb9f4,
178 RRF_NGRK = 0xb9e4,
179 RRF_ORK = 0xb9f6,
180 RRF_OGRK = 0xb9e6,
181 RRF_SRK = 0xb9f9,
182 RRF_SGRK = 0xb9e9,
183 RRF_SLRK = 0xb9fb,
184 RRF_SLGRK = 0xb9eb,
185 RRF_XRK = 0xb9f7,
186 RRF_XGRK = 0xb9e7,
96a9f093 187
48bb3750 188 RR_AR = 0x1a,
3790b918 189 RR_ALR = 0x1e,
48bb3750
RH		 190 RR_BASR = 0x0d,
191 RR_BCR = 0x07,
192 RR_CLR = 0x15,
193 RR_CR = 0x19,
194 RR_DR = 0x1d,
195 RR_LCR = 0x13,
196 RR_LR = 0x18,
197 RR_LTR = 0x12,
198 RR_NR = 0x14,
199 RR_OR = 0x16,
200 RR_SR = 0x1b,
3790b918 201 RR_SLR = 0x1f,
48bb3750
RH		 202 RR_XR = 0x17,
203
204 RSY_RLL = 0xeb1d,
205 RSY_RLLG = 0xeb1c,
206 RSY_SLLG = 0xeb0d,
c2097136 207 RSY_SLLK = 0xebdf,
48bb3750 208 RSY_SRAG = 0xeb0a,
c2097136 209 RSY_SRAK = 0xebdc,
48bb3750 210 RSY_SRLG = 0xeb0c,
c2097136 211 RSY_SRLK = 0xebde,
48bb3750
RH		 212
213 RS_SLL = 0x89,
214 RS_SRA = 0x8a,
215 RS_SRL = 0x88,
216
217 RXY_AG = 0xe308,
218 RXY_AY = 0xe35a,
219 RXY_CG = 0xe320,
a534bb15
RH		 220 RXY_CLG = 0xe321,
221 RXY_CLY = 0xe355,
48bb3750 222 RXY_CY = 0xe359,
0db921e6 223 RXY_LAY = 0xe371,
48bb3750
RH		 224 RXY_LB = 0xe376,
225 RXY_LG = 0xe304,
226 RXY_LGB = 0xe377,
227 RXY_LGF = 0xe314,
228 RXY_LGH = 0xe315,
229 RXY_LHY = 0xe378,
230 RXY_LLGC = 0xe390,
231 RXY_LLGF = 0xe316,
232 RXY_LLGH = 0xe391,
233 RXY_LMG = 0xeb04,
234 RXY_LRV = 0xe31e,
235 RXY_LRVG = 0xe30f,
236 RXY_LRVH = 0xe31f,
237 RXY_LY = 0xe358,
bdcd5d19 238 RXY_NG = 0xe380,
4046d9ca 239 RXY_OG = 0xe381,
48bb3750
RH		 240 RXY_STCY = 0xe372,
241 RXY_STG = 0xe324,
242 RXY_STHY = 0xe370,
243 RXY_STMG = 0xeb24,
244 RXY_STRV = 0xe33e,
245 RXY_STRVG = 0xe32f,
246 RXY_STRVH = 0xe33f,
247 RXY_STY = 0xe350,
5bf67a92 248 RXY_XG = 0xe382,
48bb3750
RH		 249
250 RX_A = 0x5a,
251 RX_C = 0x59,
252 RX_L = 0x58,
0db921e6 253 RX_LA = 0x41,
48bb3750
RH		 254 RX_LH = 0x48,
255 RX_ST = 0x50,
256 RX_STC = 0x42,
257 RX_STH = 0x40,
ed3d51ec 258
79cada86
RH		 259 VRIa_VGBM = 0xe744,
260 VRIa_VREPI = 0xe745,
261 VRIb_VGM = 0xe746,
262 VRIc_VREP = 0xe74d,
263
ae77bbe5
RH		 264 VRRa_VLC = 0xe7de,
265 VRRa_VLP = 0xe7df,
b33ce725 266 VRRa_VLR = 0xe756,
a429ee29
RH		 267 VRRc_VA = 0xe7f3,
268 VRRc_VCEQ = 0xe7f8, /* we leave the m5 cs field 0 */
269 VRRc_VCH = 0xe7fb, /* " */
270 VRRc_VCHL = 0xe7f9, /* " */
22cb37b4
RH		 271 VRRc_VERLLV = 0xe773,
272 VRRc_VESLV = 0xe770,
273 VRRc_VESRAV = 0xe77a,
274 VRRc_VESRLV = 0xe778,
479b61cb 275 VRRc_VML = 0xe7a2,
220db7a6
RH		 276 VRRc_VMN = 0xe7fe,
277 VRRc_VMNL = 0xe7fc,
278 VRRc_VMX = 0xe7ff,
279 VRRc_VMXL = 0xe7fd,
a429ee29 280 VRRc_VN = 0xe768,
ae77bbe5 281 VRRc_VNC = 0xe769,
21eab5bf 282 VRRc_VNN = 0xe76e,
ae77bbe5 283 VRRc_VNO = 0xe76b,
21eab5bf 284 VRRc_VNX = 0xe76c,
a429ee29 285 VRRc_VO = 0xe76a,
ae77bbe5 286 VRRc_VOC = 0xe76f,
4223c9c1 287 VRRc_VPKS = 0xe797, /* we leave the m5 cs field 0 */
a429ee29 288 VRRc_VS = 0xe7f7,
4223c9c1
RH		 289 VRRa_VUPH = 0xe7d7,
290 VRRa_VUPL = 0xe7d6,
a429ee29 291 VRRc_VX = 0xe76d,
9bca986d 292 VRRe_VSEL = 0xe78d,
79cada86 293 VRRf_VLVGP = 0xe762,
b33ce725 294
22cb37b4
RH		 295 VRSa_VERLL = 0xe733,
296 VRSa_VESL = 0xe730,
297 VRSa_VESRA = 0xe73a,
298 VRSa_VESRL = 0xe738,
b33ce725
RH		 299 VRSb_VLVG = 0xe722,
300 VRSc_VLGV = 0xe721,
301
2dabf742
RH		 302 VRX_VL = 0xe706,
303 VRX_VLLEZ = 0xe704,
79cada86 304 VRX_VLREP = 0xe705,
2dabf742
RH		 305 VRX_VST = 0xe70e,
306 VRX_VSTEF = 0xe70b,
307 VRX_VSTEG = 0xe70a,
308
ed3d51ec 309 NOP = 0x0707,
48bb3750
RH		 310} S390Opcode;
311
8d8fdbae 312#ifdef CONFIG_DEBUG_TCG
48bb3750 313static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
34ef7676
RH		 314 "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7",
315 "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
316 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
317 "%v0", "%v1", "%v2", "%v3", "%v4", "%v5", "%v6", "%v7",
318 "%v8", "%v9", "%v10", "%v11", "%v12", "%v13", "%v14", "%v15",
319 "%v16", "%v17", "%v18", "%v19", "%v20", "%v21", "%v22", "%v23",
320 "%v24", "%v25", "%v26", "%v27", "%v28", "%v29", "%v30", "%v31",
48bb3750
RH		 321};
322#endif
323
324/* Since R6 is a potential argument register, choose it last of the
325 call-saved registers. Likewise prefer the call-clobbered registers
326 in reverse order to maximize the chance of avoiding the arguments. */
2827822e 327static const int tcg_target_reg_alloc_order[] = {
f24efee4 328 /* Call saved registers. */
48bb3750
RH		 329 TCG_REG_R13,
330 TCG_REG_R12,
331 TCG_REG_R11,
332 TCG_REG_R10,
333 TCG_REG_R9,
334 TCG_REG_R8,
335 TCG_REG_R7,
336 TCG_REG_R6,
f24efee4 337 /* Call clobbered registers. */
48bb3750
RH		 338 TCG_REG_R14,
339 TCG_REG_R0,
340 TCG_REG_R1,
f24efee4 341 /* Argument registers, in reverse order of allocation. */
48bb3750
RH		 342 TCG_REG_R5,
343 TCG_REG_R4,
344 TCG_REG_R3,
345 TCG_REG_R2,
34ef7676
RH		 346
347 /* V8-V15 are call saved, and omitted. */
348 TCG_REG_V0,
349 TCG_REG_V1,
350 TCG_REG_V2,
351 TCG_REG_V3,
352 TCG_REG_V4,
353 TCG_REG_V5,
354 TCG_REG_V6,
355 TCG_REG_V7,
356 TCG_REG_V16,
357 TCG_REG_V17,
358 TCG_REG_V18,
359 TCG_REG_V19,
360 TCG_REG_V20,
361 TCG_REG_V21,
362 TCG_REG_V22,
363 TCG_REG_V23,
364 TCG_REG_V24,
365 TCG_REG_V25,
366 TCG_REG_V26,
367 TCG_REG_V27,
368 TCG_REG_V28,
369 TCG_REG_V29,
370 TCG_REG_V30,
371 TCG_REG_V31,
2827822e
AG		 372};
373
374static const int tcg_target_call_iarg_regs[] = {
48bb3750
RH		 375 TCG_REG_R2,
376 TCG_REG_R3,
377 TCG_REG_R4,
378 TCG_REG_R5,
379 TCG_REG_R6,
2827822e
AG		 380};
381
382static const int tcg_target_call_oarg_regs[] = {
48bb3750 383 TCG_REG_R2,
48bb3750
RH		 384};
385
386#define S390_CC_EQ 8
387#define S390_CC_LT 4
388#define S390_CC_GT 2
389#define S390_CC_OV 1
390#define S390_CC_NE (S390_CC_LT | S390_CC_GT)
391#define S390_CC_LE (S390_CC_LT | S390_CC_EQ)
392#define S390_CC_GE (S390_CC_GT | S390_CC_EQ)
393#define S390_CC_NEVER 0
394#define S390_CC_ALWAYS 15
395
396/* Condition codes that result from a COMPARE and COMPARE LOGICAL. */
0aed257f 397static const uint8_t tcg_cond_to_s390_cond[] = {
48bb3750
RH		 398 [TCG_COND_EQ] = S390_CC_EQ,
399 [TCG_COND_NE] = S390_CC_NE,
400 [TCG_COND_LT] = S390_CC_LT,
401 [TCG_COND_LE] = S390_CC_LE,
402 [TCG_COND_GT] = S390_CC_GT,
403 [TCG_COND_GE] = S390_CC_GE,
404 [TCG_COND_LTU] = S390_CC_LT,
405 [TCG_COND_LEU] = S390_CC_LE,
406 [TCG_COND_GTU] = S390_CC_GT,
407 [TCG_COND_GEU] = S390_CC_GE,
408};
409
410/* Condition codes that result from a LOAD AND TEST. Here, we have no
411 unsigned instruction variation, however since the test is vs zero we
412 can re-map the outcomes appropriately. */
0aed257f 413static const uint8_t tcg_cond_to_ltr_cond[] = {
48bb3750
RH		 414 [TCG_COND_EQ] = S390_CC_EQ,
415 [TCG_COND_NE] = S390_CC_NE,
416 [TCG_COND_LT] = S390_CC_LT,
417 [TCG_COND_LE] = S390_CC_LE,
418 [TCG_COND_GT] = S390_CC_GT,
419 [TCG_COND_GE] = S390_CC_GE,
420 [TCG_COND_LTU] = S390_CC_NEVER,
421 [TCG_COND_LEU] = S390_CC_EQ,
422 [TCG_COND_GTU] = S390_CC_NE,
423 [TCG_COND_GEU] = S390_CC_ALWAYS,
424};
425
426#ifdef CONFIG_SOFTMMU
4b473e0c 427static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = {
f24efee4
RH		 428 [MO_UB] = helper_ret_ldub_mmu,
429 [MO_SB] = helper_ret_ldsb_mmu,
430 [MO_LEUW] = helper_le_lduw_mmu,
431 [MO_LESW] = helper_le_ldsw_mmu,
432 [MO_LEUL] = helper_le_ldul_mmu,
433 [MO_LESL] = helper_le_ldsl_mmu,
fc313c64 434 [MO_LEUQ] = helper_le_ldq_mmu,
f24efee4
RH		 435 [MO_BEUW] = helper_be_lduw_mmu,
436 [MO_BESW] = helper_be_ldsw_mmu,
437 [MO_BEUL] = helper_be_ldul_mmu,
438 [MO_BESL] = helper_be_ldsl_mmu,
fc313c64 439 [MO_BEUQ] = helper_be_ldq_mmu,
e141ab52
BS		 440};
441
4b473e0c 442static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
f24efee4
RH		 443 [MO_UB] = helper_ret_stb_mmu,
444 [MO_LEUW] = helper_le_stw_mmu,
445 [MO_LEUL] = helper_le_stl_mmu,
fc313c64 446 [MO_LEUQ] = helper_le_stq_mmu,
f24efee4
RH		 447 [MO_BEUW] = helper_be_stw_mmu,
448 [MO_BEUL] = helper_be_stl_mmu,
fc313c64 449 [MO_BEUQ] = helper_be_stq_mmu,
e141ab52 450};
e141ab52 451#endif
48bb3750 452
79dae4dd 453static const tcg_insn_unit *tb_ret_addr;
34ef7676 454uint64_t s390_facilities[3];
2827822e 455
2dabf742
RH		 456static inline bool is_general_reg(TCGReg r)
457{
458 return r <= TCG_REG_R15;
459}
460
461static inline bool is_vector_reg(TCGReg r)
462{
463 return r >= TCG_REG_V0 && r <= TCG_REG_V31;
464}
465
79dae4dd 466static bool patch_reloc(tcg_insn_unit *src_rw, int type,
2ba7fae2 467 intptr_t value, intptr_t addend)
2827822e 468{
79dae4dd 469 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
e692a349 470 intptr_t pcrel2;
28eef8aa 471 uint32_t old;
e692a349
RH		 472
473 value += addend;
79dae4dd 474 pcrel2 = (tcg_insn_unit *)value - src_rx;
48bb3750
RH		 475
476 switch (type) {
477 case R_390_PC16DBL:
55dfd8fe 478 if (pcrel2 == (int16_t)pcrel2) {
79dae4dd 479 tcg_patch16(src_rw, pcrel2);
55dfd8fe
RH		 480 return true;
481 }
48bb3750
RH		 482 break;
483 case R_390_PC32DBL:
55dfd8fe 484 if (pcrel2 == (int32_t)pcrel2) {
79dae4dd 485 tcg_patch32(src_rw, pcrel2);
55dfd8fe
RH		 486 return true;
487 }
48bb3750 488 break;
28eef8aa 489 case R_390_20:
55dfd8fe 490 if (value == sextract64(value, 0, 20)) {
79dae4dd 491 old = *(uint32_t *)src_rw & 0xf00000ff;
55dfd8fe 492 old |= ((value & 0xfff) << 16) | ((value & 0xff000) >> 4);
79dae4dd 493 tcg_patch32(src_rw, old);
55dfd8fe
RH		 494 return true;
495 }
28eef8aa 496 break;
48bb3750 497 default:
e692a349 498 g_assert_not_reached();
48bb3750 499 }
55dfd8fe 500 return false;
2827822e
AG		 501}
502
2827822e 503/* Test if a constant matches the constraint. */
a4fbbd77 504static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
2827822e 505{
48bb3750
RH		 506 if (ct & TCG_CT_CONST) {
507 return 1;
508 }
509
671c835b 510 if (type == TCG_TYPE_I32) {
48bb3750
RH		 511 val = (int32_t)val;
512 }
513
514 /* The following are mutually exclusive. */
a8f0269e
RH		 515 if (ct & TCG_CT_CONST_S16) {
516 return val == (int16_t)val;
517 } else if (ct & TCG_CT_CONST_S32) {
518 return val == (int32_t)val;
ba18b07d
RH		 519 } else if (ct & TCG_CT_CONST_S33) {
520 return val >= -0xffffffffll && val <= 0xffffffffll;
752b1be9
RH		 521 } else if (ct & TCG_CT_CONST_ZERO) {
522 return val == 0;
48bb3750
RH		 523 }
524
2827822e
AG		 525 return 0;
526}
527
48bb3750
RH		 528/* Emit instructions according to the given instruction format. */
529
530static void tcg_out_insn_RR(TCGContext *s, S390Opcode op, TCGReg r1, TCGReg r2)
531{
532 tcg_out16(s, (op << 8) | (r1 << 4) | r2);
533}
534
535static void tcg_out_insn_RRE(TCGContext *s, S390Opcode op,
536 TCGReg r1, TCGReg r2)
537{
538 tcg_out32(s, (op << 16) | (r1 << 4) | r2);
539}
540
96a9f093
RH		 541static void tcg_out_insn_RRF(TCGContext *s, S390Opcode op,
542 TCGReg r1, TCGReg r2, int m3)
543{
544 tcg_out32(s, (op << 16) | (m3 << 12) | (r1 << 4) | r2);
545}
546
48bb3750
RH		 547static void tcg_out_insn_RI(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
548{
549 tcg_out32(s, (op << 16) | (r1 << 20) | (i2 & 0xffff));
550}
551
7af525af
RH		 552static void tcg_out_insn_RIE(TCGContext *s, S390Opcode op, TCGReg r1,
553 int i2, int m3)
554{
555 tcg_out16(s, (op & 0xff00) | (r1 << 4) | m3);
556 tcg_out32(s, (i2 << 16) | (op & 0xff));
557}
558
48bb3750
RH		 559static void tcg_out_insn_RIL(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
560{
561 tcg_out16(s, op | (r1 << 4));
562 tcg_out32(s, i2);
563}
564
565static void tcg_out_insn_RS(TCGContext *s, S390Opcode op, TCGReg r1,
566 TCGReg b2, TCGReg r3, int disp)
567{
568 tcg_out32(s, (op << 24) | (r1 << 20) | (r3 << 16) | (b2 << 12)
569 | (disp & 0xfff));
570}
571
572static void tcg_out_insn_RSY(TCGContext *s, S390Opcode op, TCGReg r1,
573 TCGReg b2, TCGReg r3, int disp)
574{
575 tcg_out16(s, (op & 0xff00) | (r1 << 4) | r3);
576 tcg_out32(s, (op & 0xff) | (b2 << 28)
577 | ((disp & 0xfff) << 16) | ((disp & 0xff000) >> 4));
578}
579
580#define tcg_out_insn_RX tcg_out_insn_RS
581#define tcg_out_insn_RXY tcg_out_insn_RSY
582
2dabf742
RH		 583static int RXB(TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4)
584{
585 /*
586 * Shift bit 4 of each regno to its corresponding bit of RXB.
587 * RXB itself begins at bit 8 of the instruction so 8 - 4 = 4
588 * is the left-shift of the 4th operand.
589 */
590 return ((v1 & 0x10) << (4 + 3))
591 | ((v2 & 0x10) << (4 + 2))
592 | ((v3 & 0x10) << (4 + 1))
593 | ((v4 & 0x10) << (4 + 0));
594}
595
79cada86
RH		 596static void tcg_out_insn_VRIa(TCGContext *s, S390Opcode op,
597 TCGReg v1, uint16_t i2, int m3)
598{
599 tcg_debug_assert(is_vector_reg(v1));
600 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4));
601 tcg_out16(s, i2);
602 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m3 << 12));
603}
604
605static void tcg_out_insn_VRIb(TCGContext *s, S390Opcode op,
606 TCGReg v1, uint8_t i2, uint8_t i3, int m4)
607{
608 tcg_debug_assert(is_vector_reg(v1));
609 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4));
610 tcg_out16(s, (i2 << 8) | (i3 & 0xff));
611 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m4 << 12));
612}
613
614static void tcg_out_insn_VRIc(TCGContext *s, S390Opcode op,
615 TCGReg v1, uint16_t i2, TCGReg v3, int m4)
616{
617 tcg_debug_assert(is_vector_reg(v1));
618 tcg_debug_assert(is_vector_reg(v3));
619 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v3 & 0xf));
620 tcg_out16(s, i2);
621 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, v3, 0) | (m4 << 12));
622}
623
b33ce725
RH		 624static void tcg_out_insn_VRRa(TCGContext *s, S390Opcode op,
625 TCGReg v1, TCGReg v2, int m3)
626{
627 tcg_debug_assert(is_vector_reg(v1));
628 tcg_debug_assert(is_vector_reg(v2));
629 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf));
630 tcg_out32(s, (op & 0x00ff) | RXB(v1, v2, 0, 0) | (m3 << 12));
631}
632
a429ee29
RH		 633static void tcg_out_insn_VRRc(TCGContext *s, S390Opcode op,
634 TCGReg v1, TCGReg v2, TCGReg v3, int m4)
635{
636 tcg_debug_assert(is_vector_reg(v1));
637 tcg_debug_assert(is_vector_reg(v2));
638 tcg_debug_assert(is_vector_reg(v3));
639 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf));
640 tcg_out16(s, v3 << 12);
641 tcg_out16(s, (op & 0x00ff) | RXB(v1, v2, v3, 0) | (m4 << 12));
642}
643
9bca986d
RH		 644static void tcg_out_insn_VRRe(TCGContext *s, S390Opcode op,
645 TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4)
646{
647 tcg_debug_assert(is_vector_reg(v1));
648 tcg_debug_assert(is_vector_reg(v2));
649 tcg_debug_assert(is_vector_reg(v3));
650 tcg_debug_assert(is_vector_reg(v4));
651 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf));
652 tcg_out16(s, v3 << 12);
653 tcg_out16(s, (op & 0x00ff) | RXB(v1, v2, v3, v4) | (v4 << 12));
654}
655
79cada86
RH		 656static void tcg_out_insn_VRRf(TCGContext *s, S390Opcode op,
657 TCGReg v1, TCGReg r2, TCGReg r3)
658{
659 tcg_debug_assert(is_vector_reg(v1));
660 tcg_debug_assert(is_general_reg(r2));
661 tcg_debug_assert(is_general_reg(r3));
662 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | r2);
663 tcg_out16(s, r3 << 12);
664 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0));
665}
666
22cb37b4
RH		 667static void tcg_out_insn_VRSa(TCGContext *s, S390Opcode op, TCGReg v1,
668 intptr_t d2, TCGReg b2, TCGReg v3, int m4)
669{
670 tcg_debug_assert(is_vector_reg(v1));
671 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
672 tcg_debug_assert(is_general_reg(b2));
673 tcg_debug_assert(is_vector_reg(v3));
674 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v3 & 0xf));
675 tcg_out16(s, b2 << 12 | d2);
676 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, v3, 0) | (m4 << 12));
677}
678
b33ce725
RH		 679static void tcg_out_insn_VRSb(TCGContext *s, S390Opcode op, TCGReg v1,
680 intptr_t d2, TCGReg b2, TCGReg r3, int m4)
681{
682 tcg_debug_assert(is_vector_reg(v1));
683 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
684 tcg_debug_assert(is_general_reg(b2));
685 tcg_debug_assert(is_general_reg(r3));
686 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | r3);
687 tcg_out16(s, b2 << 12 | d2);
688 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m4 << 12));
689}
690
691static void tcg_out_insn_VRSc(TCGContext *s, S390Opcode op, TCGReg r1,
692 intptr_t d2, TCGReg b2, TCGReg v3, int m4)
693{
694 tcg_debug_assert(is_general_reg(r1));
695 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
696 tcg_debug_assert(is_general_reg(b2));
697 tcg_debug_assert(is_vector_reg(v3));
698 tcg_out16(s, (op & 0xff00) | (r1 << 4) | (v3 & 0xf));
699 tcg_out16(s, b2 << 12 | d2);
700 tcg_out16(s, (op & 0x00ff) | RXB(0, 0, v3, 0) | (m4 << 12));
701}
702
2dabf742
RH		 703static void tcg_out_insn_VRX(TCGContext *s, S390Opcode op, TCGReg v1,
704 TCGReg b2, TCGReg x2, intptr_t d2, int m3)
705{
706 tcg_debug_assert(is_vector_reg(v1));
707 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
708 tcg_debug_assert(is_general_reg(x2));
709 tcg_debug_assert(is_general_reg(b2));
710 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | x2);
711 tcg_out16(s, (b2 << 12) | d2);
712 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m3 << 12));
713}
714
48bb3750
RH		 715/* Emit an opcode with "type-checking" of the format. */
716#define tcg_out_insn(S, FMT, OP, ...) \
717 glue(tcg_out_insn_,FMT)(S, glue(glue(FMT,_),OP), ## __VA_ARGS__)
718
719
720/* emit 64-bit shifts */
721static void tcg_out_sh64(TCGContext* s, S390Opcode op, TCGReg dest,
722 TCGReg src, TCGReg sh_reg, int sh_imm)
723{
724 tcg_out_insn_RSY(s, op, dest, sh_reg, src, sh_imm);
725}
726
727/* emit 32-bit shifts */
728static void tcg_out_sh32(TCGContext* s, S390Opcode op, TCGReg dest,
729 TCGReg sh_reg, int sh_imm)
730{
731 tcg_out_insn_RS(s, op, dest, sh_reg, 0, sh_imm);
732}
733
78113e83 734static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
48bb3750 735{
b33ce725
RH		 736 if (src == dst) {
737 return true;
738 }
739 switch (type) {
740 case TCG_TYPE_I32:
741 if (likely(is_general_reg(dst) && is_general_reg(src))) {
48bb3750 742 tcg_out_insn(s, RR, LR, dst, src);
b33ce725 743 break;
48bb3750 744 }
b33ce725
RH		 745 /* fallthru */
746
747 case TCG_TYPE_I64:
748 if (likely(is_general_reg(dst))) {
749 if (likely(is_general_reg(src))) {
750 tcg_out_insn(s, RRE, LGR, dst, src);
751 } else {
752 tcg_out_insn(s, VRSc, VLGV, dst, 0, 0, src, 3);
753 }
754 break;
755 } else if (is_general_reg(src)) {
756 tcg_out_insn(s, VRSb, VLVG, dst, 0, 0, src, 3);
757 break;
758 }
759 /* fallthru */
760
761 case TCG_TYPE_V64:
762 case TCG_TYPE_V128:
763 tcg_out_insn(s, VRRa, VLR, dst, src, 0);
764 break;
765
766 default:
767 g_assert_not_reached();
48bb3750 768 }
78113e83 769 return true;
48bb3750
RH		 770}
771
28eef8aa
RH		 772static const S390Opcode lli_insns[4] = {
773 RI_LLILL, RI_LLILH, RI_LLIHL, RI_LLIHH
774};
48bb3750 775
28eef8aa
RH		 776static bool maybe_out_small_movi(TCGContext *s, TCGType type,
777 TCGReg ret, tcg_target_long sval)
778{
48bb3750
RH		 779 tcg_target_ulong uval = sval;
780 int i;
781
782 if (type == TCG_TYPE_I32) {
783 uval = (uint32_t)sval;
784 sval = (int32_t)sval;
785 }
786
787 /* Try all 32-bit insns that can load it in one go. */
788 if (sval >= -0x8000 && sval < 0x8000) {
789 tcg_out_insn(s, RI, LGHI, ret, sval);
28eef8aa 790 return true;
48bb3750
RH		 791 }
792
793 for (i = 0; i < 4; i++) {
dba6a9eb 794 tcg_target_long mask = 0xffffull << i * 16;
48bb3750 795 if ((uval & mask) == uval) {
dba6a9eb 796 tcg_out_insn_RI(s, lli_insns[i], ret, uval >> i * 16);
28eef8aa 797 return true;
48bb3750
RH		 798 }
799 }
800
28eef8aa
RH		 801 return false;
802}
803
804/* load a register with an immediate value */
ccbecb44
RH		 805static void tcg_out_movi(TCGContext *s, TCGType type,
806 TCGReg ret, tcg_target_long sval)
28eef8aa
RH		 807{
808 tcg_target_ulong uval;
809
810 /* Try all 32-bit insns that can load it in one go. */
811 if (maybe_out_small_movi(s, type, ret, sval)) {
812 return;
813 }
814
815 uval = sval;
816 if (type == TCG_TYPE_I32) {
817 uval = (uint32_t)sval;
818 sval = (int32_t)sval;
819 }
820
48bb3750 821 /* Try all 48-bit insns that can load it in one go. */
3e25f7da
RH		 822 if (sval == (int32_t)sval) {
823 tcg_out_insn(s, RIL, LGFI, ret, sval);
824 return;
825 }
826 if (uval <= 0xffffffff) {
827 tcg_out_insn(s, RIL, LLILF, ret, uval);
828 return;
829 }
830 if ((uval & 0xffffffff) == 0) {
831 tcg_out_insn(s, RIL, LLIHF, ret, uval >> 32);
832 return;
48bb3750
RH		 833 }
834
829e1376
RH		 835 /* Try for PC-relative address load. For odd addresses,
836 attempt to use an offset from the start of the TB. */
48bb3750 837 if ((sval & 1) == 0) {
8c081b18 838 ptrdiff_t off = tcg_pcrel_diff(s, (void *)sval) >> 1;
48bb3750
RH		 839 if (off == (int32_t)off) {
840 tcg_out_insn(s, RIL, LARL, ret, off);
841 return;
842 }
843 }
844
ba2c7479 845 /* Otherwise, stuff it in the constant pool. */
9c3bfb79
RH		 846 tcg_out_insn(s, RIL, LGRL, ret, 0);
847 new_pool_label(s, sval, R_390_PC32DBL, s->code_ptr - 2, 2);
48bb3750
RH		 848}
849
48bb3750
RH		 850/* Emit a load/store type instruction. Inputs are:
851 DATA: The register to be loaded or stored.
852 BASE+OFS: The effective address.
853 OPC_RX: If the operation has an RX format opcode (e.g. STC), otherwise 0.
854 OPC_RXY: The RXY format opcode for the operation (e.g. STCY). */
855
856static void tcg_out_mem(TCGContext *s, S390Opcode opc_rx, S390Opcode opc_rxy,
857 TCGReg data, TCGReg base, TCGReg index,
858 tcg_target_long ofs)
859{
860 if (ofs < -0x80000 || ofs >= 0x80000) {
78c9f7c5
RH		 861 /* Combine the low 20 bits of the offset with the actual load insn;
862 the high 44 bits must come from an immediate load. */
863 tcg_target_long low = ((ofs & 0xfffff) ^ 0x80000) - 0x80000;
864 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - low);
865 ofs = low;
48bb3750
RH		 866
867 /* If we were already given an index register, add it in. */
868 if (index != TCG_REG_NONE) {
869 tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
870 }
871 index = TCG_TMP0;
872 }
873
874 if (opc_rx && ofs >= 0 && ofs < 0x1000) {
875 tcg_out_insn_RX(s, opc_rx, data, base, index, ofs);
876 } else {
877 tcg_out_insn_RXY(s, opc_rxy, data, base, index, ofs);
878 }
2827822e
AG		 879}
880
2dabf742
RH		 881static void tcg_out_vrx_mem(TCGContext *s, S390Opcode opc_vrx,
882 TCGReg data, TCGReg base, TCGReg index,
883 tcg_target_long ofs, int m3)
884{
885 if (ofs < 0 || ofs >= 0x1000) {
886 if (ofs >= -0x80000 && ofs < 0x80000) {
887 tcg_out_insn(s, RXY, LAY, TCG_TMP0, base, index, ofs);
888 base = TCG_TMP0;
889 index = TCG_REG_NONE;
890 ofs = 0;
891 } else {
892 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs);
893 if (index != TCG_REG_NONE) {
894 tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
895 }
896 index = TCG_TMP0;
897 ofs = 0;
898 }
899 }
900 tcg_out_insn_VRX(s, opc_vrx, data, base, index, ofs, m3);
901}
48bb3750 902
2827822e 903/* load data without address translation or endianness conversion */
2dabf742
RH		 904static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data,
905 TCGReg base, intptr_t ofs)
2827822e 906{
2dabf742
RH		 907 switch (type) {
908 case TCG_TYPE_I32:
909 if (likely(is_general_reg(data))) {
910 tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs);
911 break;
912 }
913 tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_32);
914 break;
915
916 case TCG_TYPE_I64:
917 if (likely(is_general_reg(data))) {
918 tcg_out_mem(s, 0, RXY_LG, data, base, TCG_REG_NONE, ofs);
919 break;
920 }
921 /* fallthru */
922
923 case TCG_TYPE_V64:
924 tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_64);
925 break;
926
927 case TCG_TYPE_V128:
928 /* Hint quadword aligned. */
929 tcg_out_vrx_mem(s, VRX_VL, data, base, TCG_REG_NONE, ofs, 4);
930 break;
931
932 default:
933 g_assert_not_reached();
48bb3750 934 }
2827822e
AG		 935}
936
2dabf742
RH		 937static void tcg_out_st(TCGContext *s, TCGType type, TCGReg data,
938 TCGReg base, intptr_t ofs)
2827822e 939{
2dabf742
RH		 940 switch (type) {
941 case TCG_TYPE_I32:
942 if (likely(is_general_reg(data))) {
943 tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs);
944 } else {
945 tcg_out_vrx_mem(s, VRX_VSTEF, data, base, TCG_REG_NONE, ofs, 1);
946 }
947 break;
948
949 case TCG_TYPE_I64:
950 if (likely(is_general_reg(data))) {
951 tcg_out_mem(s, 0, RXY_STG, data, base, TCG_REG_NONE, ofs);
952 break;
953 }
954 /* fallthru */
955
956 case TCG_TYPE_V64:
957 tcg_out_vrx_mem(s, VRX_VSTEG, data, base, TCG_REG_NONE, ofs, 0);
958 break;
959
960 case TCG_TYPE_V128:
961 /* Hint quadword aligned. */
962 tcg_out_vrx_mem(s, VRX_VST, data, base, TCG_REG_NONE, ofs, 4);
963 break;
964
965 default:
966 g_assert_not_reached();
48bb3750
RH		 967 }
968}
969
59d7c14e
RH		 970static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
971 TCGReg base, intptr_t ofs)
972{
973 return false;
974}
975
f0bffc27
RH		 976static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src,
977 int msb, int lsb, int ofs, int z)
978{
979 /* Format RIE-f */
980 tcg_out16(s, (RIE_RISBG & 0xff00) | (dest << 4) | src);
981 tcg_out16(s, (msb << 8) | (z << 7) | lsb);
982 tcg_out16(s, (ofs << 8) | (RIE_RISBG & 0xff));
983}
984
48bb3750
RH		 985static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
986{
3e25f7da 987 tcg_out_insn(s, RRE, LGBR, dest, src);
48bb3750
RH		 988}
989
990static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
991{
3e25f7da 992 tcg_out_insn(s, RRE, LLGCR, dest, src);
48bb3750
RH		 993}
994
995static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
996{
3e25f7da 997 tcg_out_insn(s, RRE, LGHR, dest, src);
48bb3750
RH		 998}
999
1000static void tgen_ext16u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1001{
3e25f7da 1002 tcg_out_insn(s, RRE, LLGHR, dest, src);
48bb3750
RH		 1003}
1004
1005static inline void tgen_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
1006{
1007 tcg_out_insn(s, RRE, LGFR, dest, src);
1008}
1009
1010static inline void tgen_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
1011{
1012 tcg_out_insn(s, RRE, LLGFR, dest, src);
1013}
1014
f0bffc27
RH		 1015/* Accept bit patterns like these:
1016 0....01....1
1017 1....10....0
1018 1..10..01..1
1019 0..01..10..0
1020 Copied from gcc sources. */
1021static inline bool risbg_mask(uint64_t c)
1022{
1023 uint64_t lsb;
1024 /* We don't change the number of transitions by inverting,
1025 so make sure we start with the LSB zero. */
1026 if (c & 1) {
1027 c = ~c;
1028 }
1029 /* Reject all zeros or all ones. */
1030 if (c == 0) {
1031 return false;
1032 }
1033 /* Find the first transition. */
1034 lsb = c & -c;
1035 /* Invert to look for a second transition. */
1036 c = ~c;
1037 /* Erase the first transition. */
1038 c &= -lsb;
1039 /* Find the second transition, if any. */
1040 lsb = c & -c;
1041 /* Match if all the bits are 1's, or if c is zero. */
1042 return c == -lsb;
1043}
1044
547ec121
RH		 1045static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
1046{
1047 int msb, lsb;
1048 if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
1049 /* Achieve wraparound by swapping msb and lsb. */
1050 msb = 64 - ctz64(~val);
1051 lsb = clz64(~val) - 1;
1052 } else {
1053 msb = clz64(val);
1054 lsb = 63 - ctz64(val);
1055 }
1056 tcg_out_risbg(s, out, in, msb, lsb, 0, 1);
1057}
1058
07ff7983 1059static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
48bb3750
RH		 1060{
1061 static const S390Opcode ni_insns[4] = {
1062 RI_NILL, RI_NILH, RI_NIHL, RI_NIHH
1063 };
1064 static const S390Opcode nif_insns[2] = {
1065 RIL_NILF, RIL_NIHF
1066 };
07ff7983 1067 uint64_t valid = (type == TCG_TYPE_I32 ? 0xffffffffull : -1ull);
48bb3750
RH		 1068 int i;
1069
48bb3750 1070 /* Look for the zero-extensions. */
07ff7983 1071 if ((val & valid) == 0xffffffff) {
48bb3750
RH		 1072 tgen_ext32u(s, dest, dest);
1073 return;
1074 }
3e25f7da
RH		 1075 if ((val & valid) == 0xff) {
1076 tgen_ext8u(s, TCG_TYPE_I64, dest, dest);
1077 return;
1078 }
1079 if ((val & valid) == 0xffff) {
1080 tgen_ext16u(s, TCG_TYPE_I64, dest, dest);
1081 return;
07ff7983 1082 }
48bb3750 1083
07ff7983
RH		 1084 /* Try all 32-bit insns that can perform it in one go. */
1085 for (i = 0; i < 4; i++) {
dba6a9eb 1086 tcg_target_ulong mask = ~(0xffffull << i * 16);
07ff7983 1087 if (((val | ~valid) & mask) == mask) {
dba6a9eb 1088 tcg_out_insn_RI(s, ni_insns[i], dest, val >> i * 16);
07ff7983 1089 return;
48bb3750 1090 }
07ff7983 1091 }
48bb3750 1092
07ff7983 1093 /* Try all 48-bit insns that can perform it in one go. */
3e25f7da
RH		 1094 for (i = 0; i < 2; i++) {
1095 tcg_target_ulong mask = ~(0xffffffffull << i * 32);
1096 if (((val | ~valid) & mask) == mask) {
1097 tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i * 32);
1098 return;
48bb3750 1099 }
07ff7983 1100 }
9c3bfb79 1101 if (risbg_mask(val)) {
547ec121 1102 tgen_andi_risbg(s, dest, dest, val);
f0bffc27
RH		 1103 return;
1104 }
48bb3750 1105
ccbecb44 1106 tcg_out_movi(s, type, TCG_TMP0, val);
07ff7983
RH		 1107 if (type == TCG_TYPE_I32) {
1108 tcg_out_insn(s, RR, NR, dest, TCG_TMP0);
48bb3750 1109 } else {
07ff7983 1110 tcg_out_insn(s, RRE, NGR, dest, TCG_TMP0);
48bb3750
RH		 1111 }
1112}
1113
4046d9ca 1114static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
48bb3750
RH		 1115{
1116 static const S390Opcode oi_insns[4] = {
1117 RI_OILL, RI_OILH, RI_OIHL, RI_OIHH
1118 };
4046d9ca 1119 static const S390Opcode oif_insns[2] = {
48bb3750
RH		 1120 RIL_OILF, RIL_OIHF
1121 };
1122
1123 int i;
1124
1125 /* Look for no-op. */
4046d9ca 1126 if (unlikely(val == 0)) {
48bb3750
RH		 1127 return;
1128 }
1129
4046d9ca
RH		 1130 /* Try all 32-bit insns that can perform it in one go. */
1131 for (i = 0; i < 4; i++) {
dba6a9eb 1132 tcg_target_ulong mask = (0xffffull << i * 16);
4046d9ca 1133 if ((val & mask) != 0 && (val & ~mask) == 0) {
dba6a9eb 1134 tcg_out_insn_RI(s, oi_insns[i], dest, val >> i * 16);
4046d9ca 1135 return;
48bb3750 1136 }
4046d9ca 1137 }
48bb3750 1138
4046d9ca 1139 /* Try all 48-bit insns that can perform it in one go. */
3e25f7da
RH		 1140 for (i = 0; i < 2; i++) {
1141 tcg_target_ulong mask = (0xffffffffull << i * 32);
1142 if ((val & mask) != 0 && (val & ~mask) == 0) {
1143 tcg_out_insn_RIL(s, oif_insns[i], dest, val >> i * 32);
1144 return;
48bb3750 1145 }
4046d9ca 1146 }
48bb3750 1147
4046d9ca
RH		 1148 if (maybe_out_small_movi(s, type, TCG_TMP0, val)) {
1149 if (type == TCG_TYPE_I32) {
1150 tcg_out_insn(s, RR, OR, dest, TCG_TMP0);
1151 } else {
1152 tcg_out_insn(s, RRE, OGR, dest, TCG_TMP0);
1153 }
4046d9ca 1154 } else {
48bb3750
RH		 1155 /* Perform the OR via sequential modifications to the high and
1156 low parts. Do this via recursion to handle 16-bit vs 32-bit
1157 masks in each half. */
4046d9ca
RH		 1158 tgen_ori(s, type, dest, val & 0x00000000ffffffffull);
1159 tgen_ori(s, type, dest, val & 0xffffffff00000000ull);
48bb3750
RH		 1160 }
1161}
1162
5bf67a92 1163static void tgen_xori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
48bb3750 1164{
5bf67a92 1165 /* Try all 48-bit insns that can perform it in one go. */
3e25f7da
RH		 1166 if ((val & 0xffffffff00000000ull) == 0) {
1167 tcg_out_insn(s, RIL, XILF, dest, val);
1168 return;
1169 }
1170 if ((val & 0x00000000ffffffffull) == 0) {
1171 tcg_out_insn(s, RIL, XIHF, dest, val >> 32);
1172 return;
48bb3750 1173 }
5bf67a92 1174
5bf67a92
RH		 1175 if (maybe_out_small_movi(s, type, TCG_TMP0, val)) {
1176 if (type == TCG_TYPE_I32) {
1177 tcg_out_insn(s, RR, XR, dest, TCG_TMP0);
1178 } else {
1179 tcg_out_insn(s, RRE, XGR, dest, TCG_TMP0);
1180 }
5bf67a92
RH		 1181 } else {
1182 /* Perform the xor by parts. */
5bf67a92
RH		 1183 if (val & 0xffffffff) {
1184 tcg_out_insn(s, RIL, XILF, dest, val);
1185 }
1186 if (val > 0xffffffff) {
1187 tcg_out_insn(s, RIL, XIHF, dest, val >> 32);
1188 }
48bb3750
RH		 1189 }
1190}
1191
1192static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
65839b56 1193 TCGArg c2, bool c2const, bool need_carry)
48bb3750 1194{
bcc66562 1195 bool is_unsigned = is_unsigned_cond(c);
a534bb15
RH		 1196 S390Opcode op;
1197
48bb3750
RH		 1198 if (c2const) {
1199 if (c2 == 0) {
65839b56
RH		 1200 if (!(is_unsigned && need_carry)) {
1201 if (type == TCG_TYPE_I32) {
1202 tcg_out_insn(s, RR, LTR, r1, r1);
1203 } else {
1204 tcg_out_insn(s, RRE, LTGR, r1, r1);
1205 }
1206 return tcg_cond_to_ltr_cond[c];
1207 }
65839b56 1208 }
a534bb15
RH		 1209
1210 if (!is_unsigned && c2 == (int16_t)c2) {
1211 op = (type == TCG_TYPE_I32 ? RI_CHI : RI_CGHI);
1212 tcg_out_insn_RI(s, op, r1, c2);
1213 goto exit;
1214 }
1215
3e25f7da
RH		 1216 if (type == TCG_TYPE_I32) {
1217 op = (is_unsigned ? RIL_CLFI : RIL_CFI);
1218 tcg_out_insn_RIL(s, op, r1, c2);
1219 goto exit;
1220 }
1221 if (c2 == (is_unsigned ? (TCGArg)(uint32_t)c2 : (TCGArg)(int32_t)c2)) {
1222 op = (is_unsigned ? RIL_CLGFI : RIL_CGFI);
1223 tcg_out_insn_RIL(s, op, r1, c2);
1224 goto exit;
48bb3750 1225 }
a534bb15
RH		 1226
1227 /* Use the constant pool, but not for small constants. */
1228 if (maybe_out_small_movi(s, type, TCG_TMP0, c2)) {
1229 c2 = TCG_TMP0;
1230 /* fall through to reg-reg */
48bb3750 1231 } else {
3e25f7da
RH		 1232 op = (is_unsigned ? RIL_CLGRL : RIL_CGRL);
1233 tcg_out_insn_RIL(s, op, r1, 0);
1234 new_pool_label(s, c2, R_390_PC32DBL, s->code_ptr - 2, 2);
a534bb15 1235 goto exit;
48bb3750
RH		 1236 }
1237 }
a534bb15
RH		 1238
1239 if (type == TCG_TYPE_I32) {
1240 op = (is_unsigned ? RR_CLR : RR_CR);
1241 tcg_out_insn_RR(s, op, r1, c2);
1242 } else {
1243 op = (is_unsigned ? RRE_CLGR : RRE_CGR);
1244 tcg_out_insn_RRE(s, op, r1, c2);
1245 }
1246
1247 exit:
48bb3750
RH		 1248 return tcg_cond_to_s390_cond[c];
1249}
1250
7b7066b1 1251static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
96a9f093 1252 TCGReg dest, TCGReg c1, TCGArg c2, int c2const)
48bb3750 1253{
7b7066b1
RH		 1254 int cc;
1255
7af525af 1256 /* With LOC2, we can always emit the minimum 3 insns. */
748b7f3e 1257 if (HAVE_FACILITY(LOAD_ON_COND2)) {
7af525af
RH		 1258 /* Emit: d = 0, d = (cc ? 1 : d). */
1259 cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
1260 tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
1261 tcg_out_insn(s, RIE, LOCGHI, dest, 1, cc);
1262 return;
1263 }
1264
4609190b 1265 restart:
7b7066b1 1266 switch (cond) {
4609190b
RH		 1267 case TCG_COND_NE:
1268 /* X != 0 is X > 0. */
1269 if (c2const && c2 == 0) {
1270 cond = TCG_COND_GTU;
1271 } else {
1272 break;
1273 }
1274 /* fallthru */
1275
7b7066b1
RH		 1276 case TCG_COND_GTU:
1277 case TCG_COND_GT:
7b7066b1
RH		 1278 /* The result of a compare has CC=2 for GT and CC=3 unused.
1279 ADD LOGICAL WITH CARRY considers (CC & 2) the carry bit. */
65839b56 1280 tgen_cmp(s, type, cond, c1, c2, c2const, true);
7b7066b1
RH		 1281 tcg_out_movi(s, type, dest, 0);
1282 tcg_out_insn(s, RRE, ALCGR, dest, dest);
1283 return;
1284
4609190b
RH		 1285 case TCG_COND_EQ:
1286 /* X == 0 is X <= 0. */
1287 if (c2const && c2 == 0) {
1288 cond = TCG_COND_LEU;
7b7066b1 1289 } else {
4609190b 1290 break;
7b7066b1 1291 }
4609190b 1292 /* fallthru */
7b7066b1
RH		 1293
1294 case TCG_COND_LEU:
4609190b
RH		 1295 case TCG_COND_LE:
1296 /* As above, but we're looking for borrow, or !carry.
1297 The second insn computes d - d - borrow, or -1 for true
1298 and 0 for false. So we must mask to 1 bit afterward. */
1299 tgen_cmp(s, type, cond, c1, c2, c2const, true);
1300 tcg_out_insn(s, RRE, SLBGR, dest, dest);
1301 tgen_andi(s, type, dest, 1);
1302 return;
1303
1304 case TCG_COND_GEU:
7b7066b1
RH		 1305 case TCG_COND_LTU:
1306 case TCG_COND_LT:
4609190b
RH		 1307 case TCG_COND_GE:
1308 /* Swap operands so that we can use LEU/GTU/GT/LE. */
c68d5b7a 1309 if (!c2const) {
7b7066b1
RH		 1310 TCGReg t = c1;
1311 c1 = c2;
1312 c2 = t;
c68d5b7a
RH		 1313 cond = tcg_swap_cond(cond);
1314 goto restart;
7b7066b1 1315 }
c68d5b7a 1316 break;
48bb3750 1317
7b7066b1 1318 default:
4609190b 1319 g_assert_not_reached();
7b7066b1
RH		 1320 }
1321
65839b56 1322 cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
c68d5b7a
RH		 1323 /* Emit: d = 0, t = 1, d = (cc ? t : d). */
1324 tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
1325 tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 1);
1326 tcg_out_insn(s, RRF, LOCGR, dest, TCG_TMP0, cc);
48bb3750
RH		 1327}
1328
96a9f093 1329static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, TCGReg dest,
7af525af
RH		 1330 TCGReg c1, TCGArg c2, int c2const,
1331 TCGArg v3, int v3const)
96a9f093 1332{
c68d5b7a
RH		 1333 int cc = tgen_cmp(s, type, c, c1, c2, c2const, false);
1334 if (v3const) {
1335 tcg_out_insn(s, RIE, LOCGHI, dest, v3, cc);
96a9f093 1336 } else {
c68d5b7a 1337 tcg_out_insn(s, RRF, LOCGR, dest, v3, cc);
96a9f093
RH		 1338 }
1339}
1340
ce411066
RH		 1341static void tgen_clz(TCGContext *s, TCGReg dest, TCGReg a1,
1342 TCGArg a2, int a2const)
1343{
1344 /* Since this sets both R and R+1, we have no choice but to store the
1345 result into R0, allowing R1 == TCG_TMP0 to be clobbered as well. */
1346 QEMU_BUILD_BUG_ON(TCG_TMP0 != TCG_REG_R1);
1347 tcg_out_insn(s, RRE, FLOGR, TCG_REG_R0, a1);
1348
1349 if (a2const && a2 == 64) {
1350 tcg_out_mov(s, TCG_TYPE_I64, dest, TCG_REG_R0);
1351 } else {
1352 if (a2const) {
1353 tcg_out_movi(s, TCG_TYPE_I64, dest, a2);
1354 } else {
1355 tcg_out_mov(s, TCG_TYPE_I64, dest, a2);
1356 }
c68d5b7a
RH		 1357 /* Emit: if (one bit found) dest = r0. */
1358 tcg_out_insn(s, RRF, LOCGR, dest, TCG_REG_R0, 2);
ce411066
RH		 1359 }
1360}
1361
d5690ea4 1362static void tgen_deposit(TCGContext *s, TCGReg dest, TCGReg src,
752b1be9 1363 int ofs, int len, int z)
d5690ea4
RH		 1364{
1365 int lsb = (63 - ofs);
1366 int msb = lsb - (len - 1);
752b1be9 1367 tcg_out_risbg(s, dest, src, msb, lsb, ofs, z);
d5690ea4
RH		 1368}
1369
b0bf5fe8
RH		 1370static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src,
1371 int ofs, int len)
1372{
1373 tcg_out_risbg(s, dest, src, 64 - len, 63, 64 - ofs, 1);
1374}
1375
ffd0e507 1376static void tgen_gotoi(TCGContext *s, int cc, const tcg_insn_unit *dest)
48bb3750 1377{
79dae4dd 1378 ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
8c081b18 1379 if (off == (int16_t)off) {
48bb3750
RH		 1380 tcg_out_insn(s, RI, BRC, cc, off);
1381 } else if (off == (int32_t)off) {
1382 tcg_out_insn(s, RIL, BRCL, cc, off);
1383 } else {
8c081b18 1384 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
48bb3750
RH		 1385 tcg_out_insn(s, RR, BCR, cc, TCG_TMP0);
1386 }
1387}
1388
bec16311 1389static void tgen_branch(TCGContext *s, int cc, TCGLabel *l)
48bb3750 1390{
48bb3750 1391 if (l->has_value) {
8c081b18 1392 tgen_gotoi(s, cc, l->u.value_ptr);
48bb3750
RH		 1393 } else {
1394 tcg_out16(s, RI_BRC | (cc << 4));
e692a349 1395 tcg_out_reloc(s, s->code_ptr, R_390_PC16DBL, l, 2);
8c081b18 1396 s->code_ptr += 1;
48bb3750
RH		 1397 }
1398}
1399
1400static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc,
bec16311 1401 TCGReg r1, TCGReg r2, TCGLabel *l)
48bb3750 1402{
79dae4dd 1403 tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
48bb3750 1404 tcg_out16(s, (opc & 0xff00) | (r1 << 4) | r2);
79dae4dd 1405 tcg_out16(s, 0);
48bb3750
RH		 1406 tcg_out16(s, cc << 12 | (opc & 0xff));
1407}
1408
1409static void tgen_compare_imm_branch(TCGContext *s, S390Opcode opc, int cc,
bec16311 1410 TCGReg r1, int i2, TCGLabel *l)
48bb3750 1411{
79dae4dd 1412 tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
48bb3750 1413 tcg_out16(s, (opc & 0xff00) | (r1 << 4) | cc);
79dae4dd 1414 tcg_out16(s, 0);
48bb3750
RH		 1415 tcg_out16(s, (i2 << 8) | (opc & 0xff));
1416}
1417
1418static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
bec16311 1419 TCGReg r1, TCGArg c2, int c2const, TCGLabel *l)
48bb3750
RH		 1420{
1421 int cc;
9c3bfb79
RH		 1422 bool is_unsigned = is_unsigned_cond(c);
1423 bool in_range;
1424 S390Opcode opc;
48bb3750 1425
9c3bfb79 1426 cc = tcg_cond_to_s390_cond[c];
48bb3750 1427
9c3bfb79
RH		 1428 if (!c2const) {
1429 opc = (type == TCG_TYPE_I32
1430 ? (is_unsigned ? RIE_CLRJ : RIE_CRJ)
1431 : (is_unsigned ? RIE_CLGRJ : RIE_CGRJ));
1432 tgen_compare_branch(s, opc, cc, r1, c2, l);
1433 return;
1434 }
48bb3750 1435
9c3bfb79
RH		 1436 /*
1437 * COMPARE IMMEDIATE AND BRANCH RELATIVE has an 8-bit immediate field.
1438 * If the immediate we've been given does not fit that range, we'll
1439 * fall back to separate compare and branch instructions using the
1440 * larger comparison range afforded by COMPARE IMMEDIATE.
1441 */
1442 if (type == TCG_TYPE_I32) {
1443 if (is_unsigned) {
1444 opc = RIE_CLIJ;
1445 in_range = (uint32_t)c2 == (uint8_t)c2;
48bb3750 1446 } else {
9c3bfb79
RH		 1447 opc = RIE_CIJ;
1448 in_range = (int32_t)c2 == (int8_t)c2;
48bb3750 1449 }
9c3bfb79
RH		 1450 } else {
1451 if (is_unsigned) {
1452 opc = RIE_CLGIJ;
1453 in_range = (uint64_t)c2 == (uint8_t)c2;
1454 } else {
1455 opc = RIE_CGIJ;
1456 in_range = (int64_t)c2 == (int8_t)c2;
48bb3750
RH		 1457 }
1458 }
9c3bfb79
RH		 1459 if (in_range) {
1460 tgen_compare_imm_branch(s, opc, cc, r1, c2, l);
1461 return;
1462 }
48bb3750 1463
65839b56 1464 cc = tgen_cmp(s, type, c, r1, c2, c2const, false);
bec16311 1465 tgen_branch(s, cc, l);
48bb3750
RH		 1466}
1467
cee44b03 1468static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *dest)
48bb3750 1469{
79dae4dd 1470 ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
48bb3750
RH		 1471 if (off == (int32_t)off) {
1472 tcg_out_insn(s, RIL, BRASL, TCG_REG_R14, off);
1473 } else {
8c081b18 1474 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
48bb3750
RH		 1475 tcg_out_insn(s, RR, BASR, TCG_REG_R14, TCG_TMP0);
1476 }
1477}
1478
cee44b03
RH		 1479static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest,
1480 const TCGHelperInfo *info)
1481{
1482 tcg_out_call_int(s, dest);
1483}
1484
14776ab5 1485static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
48bb3750
RH		 1486 TCGReg base, TCGReg index, int disp)
1487{
3c8691f5 1488 switch (opc & (MO_SSIZE | MO_BSWAP)) {
a5a04f28 1489 case MO_UB:
48bb3750
RH		 1490 tcg_out_insn(s, RXY, LLGC, data, base, index, disp);
1491 break;
a5a04f28 1492 case MO_SB:
48bb3750
RH		 1493 tcg_out_insn(s, RXY, LGB, data, base, index, disp);
1494 break;
b8dd88b8
RH		 1495
1496 case MO_UW | MO_BSWAP:
1497 /* swapped unsigned halfword load with upper bits zeroed */
1498 tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
1499 tgen_ext16u(s, TCG_TYPE_I64, data, data);
1500 break;
a5a04f28 1501 case MO_UW:
b8dd88b8
RH		 1502 tcg_out_insn(s, RXY, LLGH, data, base, index, disp);
1503 break;
1504
1505 case MO_SW | MO_BSWAP:
1506 /* swapped sign-extended halfword load */
1507 tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
1508 tgen_ext16s(s, TCG_TYPE_I64, data, data);
48bb3750 1509 break;
a5a04f28 1510 case MO_SW:
b8dd88b8
RH		 1511 tcg_out_insn(s, RXY, LGH, data, base, index, disp);
1512 break;
1513
1514 case MO_UL | MO_BSWAP:
1515 /* swapped unsigned int load with upper bits zeroed */
1516 tcg_out_insn(s, RXY, LRV, data, base, index, disp);
1517 tgen_ext32u(s, data, data);
48bb3750 1518 break;
a5a04f28 1519 case MO_UL:
b8dd88b8
RH		 1520 tcg_out_insn(s, RXY, LLGF, data, base, index, disp);
1521 break;
1522
1523 case MO_SL | MO_BSWAP:
1524 /* swapped sign-extended int load */
1525 tcg_out_insn(s, RXY, LRV, data, base, index, disp);
1526 tgen_ext32s(s, data, data);
48bb3750 1527 break;
a5a04f28 1528 case MO_SL:
b8dd88b8
RH		 1529 tcg_out_insn(s, RXY, LGF, data, base, index, disp);
1530 break;
1531
fc313c64 1532 case MO_UQ | MO_BSWAP:
b8dd88b8 1533 tcg_out_insn(s, RXY, LRVG, data, base, index, disp);
48bb3750 1534 break;
fc313c64 1535 case MO_UQ:
b8dd88b8 1536 tcg_out_insn(s, RXY, LG, data, base, index, disp);
48bb3750 1537 break;
b8dd88b8 1538
48bb3750
RH		 1539 default:
1540 tcg_abort();
1541 }
1542}
1543
14776ab5 1544static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data,
48bb3750
RH		 1545 TCGReg base, TCGReg index, int disp)
1546{
3c8691f5 1547 switch (opc & (MO_SIZE | MO_BSWAP)) {
a5a04f28 1548 case MO_UB:
48bb3750
RH		 1549 if (disp >= 0 && disp < 0x1000) {
1550 tcg_out_insn(s, RX, STC, data, base, index, disp);
1551 } else {
1552 tcg_out_insn(s, RXY, STCY, data, base, index, disp);
1553 }
1554 break;
b8dd88b8
RH		 1555
1556 case MO_UW | MO_BSWAP:
1557 tcg_out_insn(s, RXY, STRVH, data, base, index, disp);
1558 break;
a5a04f28 1559 case MO_UW:
b8dd88b8 1560 if (disp >= 0 && disp < 0x1000) {
48bb3750
RH		 1561 tcg_out_insn(s, RX, STH, data, base, index, disp);
1562 } else {
1563 tcg_out_insn(s, RXY, STHY, data, base, index, disp);
1564 }
1565 break;
b8dd88b8
RH		 1566
1567 case MO_UL | MO_BSWAP:
1568 tcg_out_insn(s, RXY, STRV, data, base, index, disp);
1569 break;
a5a04f28 1570 case MO_UL:
b8dd88b8 1571 if (disp >= 0 && disp < 0x1000) {
48bb3750
RH		 1572 tcg_out_insn(s, RX, ST, data, base, index, disp);
1573 } else {
1574 tcg_out_insn(s, RXY, STY, data, base, index, disp);
1575 }
1576 break;
b8dd88b8 1577
fc313c64 1578 case MO_UQ | MO_BSWAP:
b8dd88b8
RH		 1579 tcg_out_insn(s, RXY, STRVG, data, base, index, disp);
1580 break;
fc313c64 1581 case MO_UQ:
b8dd88b8 1582 tcg_out_insn(s, RXY, STG, data, base, index, disp);
48bb3750 1583 break;
b8dd88b8 1584
48bb3750
RH		 1585 default:
1586 tcg_abort();
1587 }
1588}
1589
1590#if defined(CONFIG_SOFTMMU)
269bd5d8
RH		 1591/* We're expecting to use a 20-bit negative offset on the tlb memory ops. */
1592QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
1593QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 19));
fb596415
RH		 1594
1595/* Load and compare a TLB entry, leaving the flags set. Loads the TLB
1596 addend into R2. Returns a register with the santitized guest address. */
14776ab5 1597static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
fb596415 1598 int mem_index, bool is_ld)
48bb3750 1599{
85aa8081
RH		 1600 unsigned s_bits = opc & MO_SIZE;
1601 unsigned a_bits = get_alignment_bits(opc);
1602 unsigned s_mask = (1 << s_bits) - 1;
1603 unsigned a_mask = (1 << a_bits) - 1;
269bd5d8
RH		 1604 int fast_off = TLB_MASK_TABLE_OFS(mem_index);
1605 int mask_off = fast_off + offsetof(CPUTLBDescFast, mask);
1606 int table_off = fast_off + offsetof(CPUTLBDescFast, table);
a5e39810
RH		 1607 int ofs, a_off;
1608 uint64_t tlb_mask;
1609
4f47e338
RH		 1610 tcg_out_sh64(s, RSY_SRLG, TCG_REG_R2, addr_reg, TCG_REG_NONE,
1611 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1612 tcg_out_insn(s, RXY, NG, TCG_REG_R2, TCG_AREG0, TCG_REG_NONE, mask_off);
1613 tcg_out_insn(s, RXY, AG, TCG_REG_R2, TCG_AREG0, TCG_REG_NONE, table_off);
1614
a5e39810
RH		 1615 /* For aligned accesses, we check the first byte and include the alignment
1616 bits within the address. For unaligned access, we check that we don't
1617 cross pages using the address of the last byte of the access. */
85aa8081
RH		 1618 a_off = (a_bits >= s_bits ? 0 : s_mask - a_mask);
1619 tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
9c3bfb79 1620 if (a_off == 0) {
4f47e338 1621 tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
48bb3750 1622 } else {
a5e39810 1623 tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
547ec121 1624 tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
48bb3750
RH		 1625 }
1626
fb596415 1627 if (is_ld) {
4f47e338 1628 ofs = offsetof(CPUTLBEntry, addr_read);
fb596415 1629 } else {
4f47e338 1630 ofs = offsetof(CPUTLBEntry, addr_write);
48bb3750 1631 }
48bb3750 1632 if (TARGET_LONG_BITS == 32) {
4f47e338 1633 tcg_out_insn(s, RX, C, TCG_REG_R3, TCG_REG_R2, TCG_REG_NONE, ofs);
48bb3750 1634 } else {
4f47e338 1635 tcg_out_insn(s, RXY, CG, TCG_REG_R3, TCG_REG_R2, TCG_REG_NONE, ofs);
48bb3750
RH		 1636 }
1637
4f47e338
RH		 1638 tcg_out_insn(s, RXY, LG, TCG_REG_R2, TCG_REG_R2, TCG_REG_NONE,
1639 offsetof(CPUTLBEntry, addend));
fb596415 1640
48bb3750 1641 if (TARGET_LONG_BITS == 32) {
fb596415
RH		 1642 tgen_ext32u(s, TCG_REG_R3, addr_reg);
1643 return TCG_REG_R3;
48bb3750 1644 }
fb596415
RH		 1645 return addr_reg;
1646}
48bb3750 1647
9002ffcb 1648static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi,
3972ef6f 1649 TCGReg data, TCGReg addr,
fb596415
RH		 1650 tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
1651{
1652 TCGLabelQemuLdst *label = new_ldst_label(s);
1653
1654 label->is_ld = is_ld;
3972ef6f 1655 label->oi = oi;
fb596415
RH		 1656 label->datalo_reg = data;
1657 label->addrlo_reg = addr;
e5e2e4c7 1658 label->raddr = tcg_splitwx_to_rx(raddr);
fb596415
RH		 1659 label->label_ptr[0] = label_ptr;
1660}
48bb3750 1661
aeee05f5 1662static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
fb596415
RH		 1663{
1664 TCGReg addr_reg = lb->addrlo_reg;
1665 TCGReg data_reg = lb->datalo_reg;
9002ffcb 1666 MemOpIdx oi = lb->oi;
14776ab5 1667 MemOp opc = get_memop(oi);
48bb3750 1668
aeee05f5 1669 if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
79dae4dd 1670 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
aeee05f5
RH		 1671 return false;
1672 }
48bb3750 1673
fb596415
RH		 1674 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1675 if (TARGET_LONG_BITS == 64) {
1676 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
1677 }
3972ef6f 1678 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R4, oi);
fb596415 1679 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R5, (uintptr_t)lb->raddr);
cee44b03 1680 tcg_out_call_int(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)]);
fb596415 1681 tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_R2);
65a62a75 1682
fb596415 1683 tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
aeee05f5 1684 return true;
48bb3750
RH		 1685}
1686
aeee05f5 1687static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
48bb3750 1688{
fb596415
RH		 1689 TCGReg addr_reg = lb->addrlo_reg;
1690 TCGReg data_reg = lb->datalo_reg;
9002ffcb 1691 MemOpIdx oi = lb->oi;
14776ab5 1692 MemOp opc = get_memop(oi);
fb596415 1693
aeee05f5 1694 if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
79dae4dd 1695 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
aeee05f5
RH		 1696 return false;
1697 }
fb596415
RH		 1698
1699 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1700 if (TARGET_LONG_BITS == 64) {
1701 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
1702 }
1703 switch (opc & MO_SIZE) {
1704 case MO_UB:
1705 tgen_ext8u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1706 break;
1707 case MO_UW:
1708 tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1709 break;
1710 case MO_UL:
1711 tgen_ext32u(s, TCG_REG_R4, data_reg);
1712 break;
fc313c64 1713 case MO_UQ:
fb596415
RH		 1714 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1715 break;
1716 default:
1717 tcg_abort();
1718 }
3972ef6f 1719 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
fb596415 1720 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
cee44b03 1721 tcg_out_call_int(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
fb596415
RH		 1722
1723 tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
aeee05f5 1724 return true;
48bb3750
RH		 1725}
1726#else
1cd49868
RH		 1727static void tcg_out_test_alignment(TCGContext *s, bool is_ld,
1728 TCGReg addrlo, unsigned a_bits)
1729{
1730 unsigned a_mask = (1 << a_bits) - 1;
1731 TCGLabelQemuLdst *l = new_ldst_label(s);
1732
1733 l->is_ld = is_ld;
1734 l->addrlo_reg = addrlo;
1735
1736 /* We are expecting a_bits to max out at 7, much lower than TMLL. */
1737 tcg_debug_assert(a_bits < 16);
1738 tcg_out_insn(s, RI, TMLL, addrlo, a_mask);
1739
1740 tcg_out16(s, RI_BRC | (7 << 4)); /* CC in {1,2,3} */
1741 l->label_ptr[0] = s->code_ptr;
1742 s->code_ptr += 1;
1743
1744 l->raddr = tcg_splitwx_to_rx(s->code_ptr);
1745}
1746
1747static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
1748{
1749 if (!patch_reloc(l->label_ptr[0], R_390_PC16DBL,
1750 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
1751 return false;
1752 }
1753
1754 tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R3, l->addrlo_reg);
1755 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1756
1757 /* "Tail call" to the helper, with the return address back inline. */
1758 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R14, (uintptr_t)l->raddr);
1759 tgen_gotoi(s, S390_CC_ALWAYS, (const void *)(l->is_ld ? helper_unaligned_ld
1760 : helper_unaligned_st));
1761 return true;
1762}
1763
1764static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
1765{
1766 return tcg_out_fail_alignment(s, l);
1767}
1768
1769static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
1770{
1771 return tcg_out_fail_alignment(s, l);
1772}
1773
48bb3750
RH		 1774static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
1775 TCGReg *index_reg, tcg_target_long *disp)
1776{
1777 if (TARGET_LONG_BITS == 32) {
1778 tgen_ext32u(s, TCG_TMP0, *addr_reg);
1779 *addr_reg = TCG_TMP0;
1780 }
b76f21a7 1781 if (guest_base < 0x80000) {
48bb3750 1782 *index_reg = TCG_REG_NONE;
b76f21a7 1783 *disp = guest_base;
48bb3750
RH		 1784 } else {
1785 *index_reg = TCG_GUEST_BASE_REG;
1786 *disp = 0;
1787 }
1788}
1789#endif /* CONFIG_SOFTMMU */
1790
f24efee4 1791static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
9002ffcb 1792 MemOpIdx oi)
48bb3750 1793{
14776ab5 1794 MemOp opc = get_memop(oi);
fb596415 1795#ifdef CONFIG_SOFTMMU
59227d5d 1796 unsigned mem_index = get_mmuidx(oi);
fb596415
RH		 1797 tcg_insn_unit *label_ptr;
1798 TCGReg base_reg;
1799
1800 base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1);
1801
cd3b29b7
AJ		 1802 tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
1803 label_ptr = s->code_ptr;
1804 s->code_ptr += 1;
fb596415
RH		 1805
1806 tcg_out_qemu_ld_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
48bb3750 1807
3972ef6f 1808 add_qemu_ldst_label(s, 1, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
48bb3750 1809#else
f24efee4
RH		 1810 TCGReg index_reg;
1811 tcg_target_long disp;
1cd49868 1812 unsigned a_bits = get_alignment_bits(opc);
f24efee4 1813
1cd49868
RH		 1814 if (a_bits) {
1815 tcg_out_test_alignment(s, true, addr_reg, a_bits);
1816 }
48bb3750
RH		 1817 tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
1818 tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, index_reg, disp);
1819#endif
1820}
1821
f24efee4 1822static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
9002ffcb 1823 MemOpIdx oi)
48bb3750 1824{
14776ab5 1825 MemOp opc = get_memop(oi);
fb596415 1826#ifdef CONFIG_SOFTMMU
59227d5d 1827 unsigned mem_index = get_mmuidx(oi);
fb596415
RH		 1828 tcg_insn_unit *label_ptr;
1829 TCGReg base_reg;
1830
1831 base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0);
1832
cd3b29b7
AJ		 1833 tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
1834 label_ptr = s->code_ptr;
1835 s->code_ptr += 1;
fb596415
RH		 1836
1837 tcg_out_qemu_st_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
48bb3750 1838
3972ef6f 1839 add_qemu_ldst_label(s, 0, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
48bb3750 1840#else
f24efee4
RH		 1841 TCGReg index_reg;
1842 tcg_target_long disp;
1cd49868 1843 unsigned a_bits = get_alignment_bits(opc);
f24efee4 1844
1cd49868
RH		 1845 if (a_bits) {
1846 tcg_out_test_alignment(s, false, addr_reg, a_bits);
1847 }
48bb3750
RH		 1848 tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
1849 tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, index_reg, disp);
1850#endif
2827822e
AG		 1851}
1852
48bb3750
RH		 1853# define OP_32_64(x) \
1854 case glue(glue(INDEX_op_,x),_i32): \
1855 case glue(glue(INDEX_op_,x),_i64)
48bb3750 1856
a9751609 1857static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
5e8892db
MR		 1858 const TCGArg args[TCG_MAX_OP_ARGS],
1859 const int const_args[TCG_MAX_OP_ARGS])
2827822e 1860{
c2097136 1861 S390Opcode op, op2;
0db921e6 1862 TCGArg a0, a1, a2;
48bb3750
RH		 1863
1864 switch (opc) {
1865 case INDEX_op_exit_tb:
46644483
RH		 1866 /* Reuse the zeroing that exists for goto_ptr. */
1867 a0 = args[0];
1868 if (a0 == 0) {
8b5c2b62 1869 tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
46644483
RH		 1870 } else {
1871 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
1872 tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
1873 }
48bb3750
RH		 1874 break;
1875
1876 case INDEX_op_goto_tb:
829e1376 1877 a0 = args[0];
6bd739ed
RH		 1878 /*
1879 * branch displacement must be aligned for atomic patching;
1880 * see if we need to add extra nop before branch
1881 */
1882 if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
1883 tcg_out16(s, NOP);
829e1376 1884 }
6bd739ed
RH		 1885 tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
1886 s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
1887 s->code_ptr += 2;
9f754620 1888 set_jmp_reset_offset(s, a0);
48bb3750
RH		 1889 break;
1890
46644483 1891 case INDEX_op_goto_ptr:
829e1376 1892 a0 = args[0];
829e1376 1893 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0);
46644483
RH		 1894 break;
1895
48bb3750
RH		 1896 OP_32_64(ld8u):
1897 /* ??? LLC (RXY format) is only present with the extended-immediate
1898 facility, whereas LLGC is always present. */
1899 tcg_out_mem(s, 0, RXY_LLGC, args[0], args[1], TCG_REG_NONE, args[2]);
1900 break;
1901
1902 OP_32_64(ld8s):
1903 /* ??? LB is no smaller than LGB, so no point to using it. */
1904 tcg_out_mem(s, 0, RXY_LGB, args[0], args[1], TCG_REG_NONE, args[2]);
1905 break;
1906
1907 OP_32_64(ld16u):
1908 /* ??? LLH (RXY format) is only present with the extended-immediate
1909 facility, whereas LLGH is always present. */
1910 tcg_out_mem(s, 0, RXY_LLGH, args[0], args[1], TCG_REG_NONE, args[2]);
1911 break;
1912
1913 case INDEX_op_ld16s_i32:
1914 tcg_out_mem(s, RX_LH, RXY_LHY, args[0], args[1], TCG_REG_NONE, args[2]);
1915 break;
1916
1917 case INDEX_op_ld_i32:
1918 tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]);
1919 break;
1920
1921 OP_32_64(st8):
1922 tcg_out_mem(s, RX_STC, RXY_STCY, args[0], args[1],
1923 TCG_REG_NONE, args[2]);
1924 break;
1925
1926 OP_32_64(st16):
1927 tcg_out_mem(s, RX_STH, RXY_STHY, args[0], args[1],
1928 TCG_REG_NONE, args[2]);
1929 break;
1930
1931 case INDEX_op_st_i32:
1932 tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
1933 break;
1934
1935 case INDEX_op_add_i32:
0db921e6 1936 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
48bb3750 1937 if (const_args[2]) {
0db921e6
RH		 1938 do_addi_32:
1939 if (a0 == a1) {
1940 if (a2 == (int16_t)a2) {
1941 tcg_out_insn(s, RI, AHI, a0, a2);
1942 break;
1943 }
3e25f7da
RH		 1944 tcg_out_insn(s, RIL, AFI, a0, a2);
1945 break;
0db921e6
RH		 1946 }
1947 tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
1948 } else if (a0 == a1) {
1949 tcg_out_insn(s, RR, AR, a0, a2);
48bb3750 1950 } else {
0db921e6 1951 tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
48bb3750
RH		 1952 }
1953 break;
1954 case INDEX_op_sub_i32:
0db921e6 1955 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
48bb3750 1956 if (const_args[2]) {
0db921e6
RH		 1957 a2 = -a2;
1958 goto do_addi_32;
c2097136
RH		 1959 } else if (a0 == a1) {
1960 tcg_out_insn(s, RR, SR, a0, a2);
1961 } else {
1962 tcg_out_insn(s, RRF, SRK, a0, a1, a2);
48bb3750
RH		 1963 }
1964 break;
1965
1966 case INDEX_op_and_i32:
c2097136 1967 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
48bb3750 1968 if (const_args[2]) {
c2097136
RH		 1969 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
1970 tgen_andi(s, TCG_TYPE_I32, a0, a2);
1971 } else if (a0 == a1) {
1972 tcg_out_insn(s, RR, NR, a0, a2);
48bb3750 1973 } else {
c2097136 1974 tcg_out_insn(s, RRF, NRK, a0, a1, a2);
48bb3750
RH		 1975 }
1976 break;
1977 case INDEX_op_or_i32:
c2097136 1978 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
48bb3750 1979 if (const_args[2]) {
c2097136 1980 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
4046d9ca 1981 tgen_ori(s, TCG_TYPE_I32, a0, a2);
c2097136
RH		 1982 } else if (a0 == a1) {
1983 tcg_out_insn(s, RR, OR, a0, a2);
48bb3750 1984 } else {
c2097136 1985 tcg_out_insn(s, RRF, ORK, a0, a1, a2);
48bb3750
RH		 1986 }
1987 break;
1988 case INDEX_op_xor_i32:
c2097136 1989 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
48bb3750 1990 if (const_args[2]) {
c2097136 1991 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
5bf67a92 1992 tgen_xori(s, TCG_TYPE_I32, a0, a2);
c2097136 1993 } else if (a0 == a1) {
48bb3750 1994 tcg_out_insn(s, RR, XR, args[0], args[2]);
c2097136
RH		 1995 } else {
1996 tcg_out_insn(s, RRF, XRK, a0, a1, a2);
48bb3750
RH		 1997 }
1998 break;
1999
2000 case INDEX_op_neg_i32:
2001 tcg_out_insn(s, RR, LCR, args[0], args[1]);
2002 break;
2003
2004 case INDEX_op_mul_i32:
2005 if (const_args[2]) {
2006 if ((int32_t)args[2] == (int16_t)args[2]) {
2007 tcg_out_insn(s, RI, MHI, args[0], args[2]);
2008 } else {
2009 tcg_out_insn(s, RIL, MSFI, args[0], args[2]);
2010 }
2011 } else {
2012 tcg_out_insn(s, RRE, MSR, args[0], args[2]);
2013 }
2014 break;
2015
2016 case INDEX_op_div2_i32:
4143f78d
RH		 2017 tcg_debug_assert(args[0] == args[2]);
2018 tcg_debug_assert(args[1] == args[3]);
2019 tcg_debug_assert((args[1] & 1) == 0);
2020 tcg_debug_assert(args[0] == args[1] + 1);
2021 tcg_out_insn(s, RR, DR, args[1], args[4]);
48bb3750
RH		 2022 break;
2023 case INDEX_op_divu2_i32:
4143f78d
RH		 2024 tcg_debug_assert(args[0] == args[2]);
2025 tcg_debug_assert(args[1] == args[3]);
2026 tcg_debug_assert((args[1] & 1) == 0);
2027 tcg_debug_assert(args[0] == args[1] + 1);
2028 tcg_out_insn(s, RRE, DLR, args[1], args[4]);
48bb3750
RH		 2029 break;
2030
2031 case INDEX_op_shl_i32:
2032 op = RS_SLL;
c2097136 2033 op2 = RSY_SLLK;
48bb3750 2034 do_shift32:
c2097136
RH		 2035 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
2036 if (a0 == a1) {
2037 if (const_args[2]) {
2038 tcg_out_sh32(s, op, a0, TCG_REG_NONE, a2);
2039 } else {
2040 tcg_out_sh32(s, op, a0, a2, 0);
2041 }
48bb3750 2042 } else {
c2097136
RH		 2043 /* Using tcg_out_sh64 here for the format; it is a 32-bit shift. */
2044 if (const_args[2]) {
2045 tcg_out_sh64(s, op2, a0, a1, TCG_REG_NONE, a2);
2046 } else {
2047 tcg_out_sh64(s, op2, a0, a1, a2, 0);
2048 }
48bb3750
RH		 2049 }
2050 break;
2051 case INDEX_op_shr_i32:
2052 op = RS_SRL;
c2097136 2053 op2 = RSY_SRLK;
48bb3750
RH		 2054 goto do_shift32;
2055 case INDEX_op_sar_i32:
2056 op = RS_SRA;
c2097136 2057 op2 = RSY_SRAK;
48bb3750
RH		 2058 goto do_shift32;
2059
2060 case INDEX_op_rotl_i32:
2061 /* ??? Using tcg_out_sh64 here for the format; it is a 32-bit rol. */
2062 if (const_args[2]) {
2063 tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_REG_NONE, args[2]);
2064 } else {
2065 tcg_out_sh64(s, RSY_RLL, args[0], args[1], args[2], 0);
2066 }
2067 break;
2068 case INDEX_op_rotr_i32:
2069 if (const_args[2]) {
2070 tcg_out_sh64(s, RSY_RLL, args[0], args[1],
2071 TCG_REG_NONE, (32 - args[2]) & 31);
2072 } else {
2073 tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
2074 tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_TMP0, 0);
2075 }
2076 break;
2077
2078 case INDEX_op_ext8s_i32:
2079 tgen_ext8s(s, TCG_TYPE_I32, args[0], args[1]);
2080 break;
2081 case INDEX_op_ext16s_i32:
2082 tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
2083 break;
2084 case INDEX_op_ext8u_i32:
2085 tgen_ext8u(s, TCG_TYPE_I32, args[0], args[1]);
2086 break;
2087 case INDEX_op_ext16u_i32:
2088 tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
2089 break;
2090
1619ee9e
RH		 2091 case INDEX_op_bswap16_i32:
2092 a0 = args[0], a1 = args[1], a2 = args[2];
2093 tcg_out_insn(s, RRE, LRVR, a0, a1);
2094 if (a2 & TCG_BSWAP_OS) {
2095 tcg_out_sh32(s, RS_SRA, a0, TCG_REG_NONE, 16);
2096 } else {
2097 tcg_out_sh32(s, RS_SRL, a0, TCG_REG_NONE, 16);
2098 }
48bb3750 2099 break;
1619ee9e
RH		 2100 case INDEX_op_bswap16_i64:
2101 a0 = args[0], a1 = args[1], a2 = args[2];
2102 tcg_out_insn(s, RRE, LRVGR, a0, a1);
2103 if (a2 & TCG_BSWAP_OS) {
2104 tcg_out_sh64(s, RSY_SRAG, a0, a0, TCG_REG_NONE, 48);
2105 } else {
2106 tcg_out_sh64(s, RSY_SRLG, a0, a0, TCG_REG_NONE, 48);
2107 }
2108 break;
2109
2110 case INDEX_op_bswap32_i32:
48bb3750
RH		 2111 tcg_out_insn(s, RRE, LRVR, args[0], args[1]);
2112 break;
1619ee9e
RH		 2113 case INDEX_op_bswap32_i64:
2114 a0 = args[0], a1 = args[1], a2 = args[2];
2115 tcg_out_insn(s, RRE, LRVR, a0, a1);
2116 if (a2 & TCG_BSWAP_OS) {
2117 tgen_ext32s(s, a0, a0);
2118 } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
2119 tgen_ext32u(s, a0, a0);
2120 }
2121 break;
48bb3750 2122
3790b918 2123 case INDEX_op_add2_i32:
ad19b358
RH		 2124 if (const_args[4]) {
2125 tcg_out_insn(s, RIL, ALFI, args[0], args[4]);
2126 } else {
2127 tcg_out_insn(s, RR, ALR, args[0], args[4]);
2128 }
3790b918
RH		 2129 tcg_out_insn(s, RRE, ALCR, args[1], args[5]);
2130 break;
2131 case INDEX_op_sub2_i32:
ad19b358
RH		 2132 if (const_args[4]) {
2133 tcg_out_insn(s, RIL, SLFI, args[0], args[4]);
2134 } else {
2135 tcg_out_insn(s, RR, SLR, args[0], args[4]);
2136 }
3790b918
RH		 2137 tcg_out_insn(s, RRE, SLBR, args[1], args[5]);
2138 break;
2139
48bb3750 2140 case INDEX_op_br:
bec16311 2141 tgen_branch(s, S390_CC_ALWAYS, arg_label(args[0]));
48bb3750
RH		 2142 break;
2143
2144 case INDEX_op_brcond_i32:
2145 tgen_brcond(s, TCG_TYPE_I32, args[2], args[0],
bec16311 2146 args[1], const_args[1], arg_label(args[3]));
48bb3750
RH		 2147 break;
2148 case INDEX_op_setcond_i32:
2149 tgen_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1],
2150 args[2], const_args[2]);
2151 break;
96a9f093
RH		 2152 case INDEX_op_movcond_i32:
2153 tgen_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1],
7af525af 2154 args[2], const_args[2], args[3], const_args[3]);
96a9f093 2155 break;
48bb3750 2156
f24efee4 2157 case INDEX_op_qemu_ld_i32:
48bb3750 2158 /* ??? Technically we can use a non-extending instruction. */
f24efee4 2159 case INDEX_op_qemu_ld_i64:
59227d5d 2160 tcg_out_qemu_ld(s, args[0], args[1], args[2]);
48bb3750 2161 break;
f24efee4
RH		 2162 case INDEX_op_qemu_st_i32:
2163 case INDEX_op_qemu_st_i64:
59227d5d 2164 tcg_out_qemu_st(s, args[0], args[1], args[2]);
48bb3750
RH		 2165 break;
2166
48bb3750
RH		 2167 case INDEX_op_ld16s_i64:
2168 tcg_out_mem(s, 0, RXY_LGH, args[0], args[1], TCG_REG_NONE, args[2]);
2169 break;
2170 case INDEX_op_ld32u_i64:
2171 tcg_out_mem(s, 0, RXY_LLGF, args[0], args[1], TCG_REG_NONE, args[2]);
2172 break;
2173 case INDEX_op_ld32s_i64:
2174 tcg_out_mem(s, 0, RXY_LGF, args[0], args[1], TCG_REG_NONE, args[2]);
2175 break;
2176 case INDEX_op_ld_i64:
2177 tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
2178 break;
2179
2180 case INDEX_op_st32_i64:
2181 tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
2182 break;
2183 case INDEX_op_st_i64:
2184 tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
2185 break;
2186
2187 case INDEX_op_add_i64:
0db921e6 2188 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2189 if (const_args[2]) {
0db921e6
RH		 2190 do_addi_64:
2191 if (a0 == a1) {
2192 if (a2 == (int16_t)a2) {
2193 tcg_out_insn(s, RI, AGHI, a0, a2);
2194 break;
2195 }
3e25f7da
RH		 2196 if (a2 == (int32_t)a2) {
2197 tcg_out_insn(s, RIL, AGFI, a0, a2);
2198 break;
2199 }
2200 if (a2 == (uint32_t)a2) {
2201 tcg_out_insn(s, RIL, ALGFI, a0, a2);
2202 break;
2203 }
2204 if (-a2 == (uint32_t)-a2) {
2205 tcg_out_insn(s, RIL, SLGFI, a0, -a2);
2206 break;
0db921e6
RH		 2207 }
2208 }
2209 tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
2210 } else if (a0 == a1) {
2211 tcg_out_insn(s, RRE, AGR, a0, a2);
48bb3750 2212 } else {
0db921e6 2213 tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
48bb3750
RH		 2214 }
2215 break;
2216 case INDEX_op_sub_i64:
0db921e6 2217 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2218 if (const_args[2]) {
0db921e6
RH		 2219 a2 = -a2;
2220 goto do_addi_64;
c2097136
RH		 2221 } else if (a0 == a1) {
2222 tcg_out_insn(s, RRE, SGR, a0, a2);
48bb3750 2223 } else {
c2097136 2224 tcg_out_insn(s, RRF, SGRK, a0, a1, a2);
48bb3750
RH		 2225 }
2226 break;
2227
2228 case INDEX_op_and_i64:
c2097136 2229 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2230 if (const_args[2]) {
c2097136 2231 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
07ff7983 2232 tgen_andi(s, TCG_TYPE_I64, args[0], args[2]);
c2097136 2233 } else if (a0 == a1) {
48bb3750 2234 tcg_out_insn(s, RRE, NGR, args[0], args[2]);
c2097136
RH		 2235 } else {
2236 tcg_out_insn(s, RRF, NGRK, a0, a1, a2);
48bb3750
RH		 2237 }
2238 break;
2239 case INDEX_op_or_i64:
c2097136 2240 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2241 if (const_args[2]) {
c2097136 2242 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
4046d9ca 2243 tgen_ori(s, TCG_TYPE_I64, a0, a2);
c2097136
RH		 2244 } else if (a0 == a1) {
2245 tcg_out_insn(s, RRE, OGR, a0, a2);
48bb3750 2246 } else {
c2097136 2247 tcg_out_insn(s, RRF, OGRK, a0, a1, a2);
48bb3750
RH		 2248 }
2249 break;
2250 case INDEX_op_xor_i64:
c2097136 2251 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2252 if (const_args[2]) {
c2097136 2253 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
5bf67a92 2254 tgen_xori(s, TCG_TYPE_I64, a0, a2);
c2097136
RH		 2255 } else if (a0 == a1) {
2256 tcg_out_insn(s, RRE, XGR, a0, a2);
48bb3750 2257 } else {
c2097136 2258 tcg_out_insn(s, RRF, XGRK, a0, a1, a2);
48bb3750
RH		 2259 }
2260 break;
2261
2262 case INDEX_op_neg_i64:
2263 tcg_out_insn(s, RRE, LCGR, args[0], args[1]);
2264 break;
2265 case INDEX_op_bswap64_i64:
2266 tcg_out_insn(s, RRE, LRVGR, args[0], args[1]);
2267 break;
2268
2269 case INDEX_op_mul_i64:
2270 if (const_args[2]) {
2271 if (args[2] == (int16_t)args[2]) {
2272 tcg_out_insn(s, RI, MGHI, args[0], args[2]);
2273 } else {
2274 tcg_out_insn(s, RIL, MSGFI, args[0], args[2]);
2275 }
2276 } else {
2277 tcg_out_insn(s, RRE, MSGR, args[0], args[2]);
2278 }
2279 break;
2280
2281 case INDEX_op_div2_i64:
4143f78d
RH		 2282 /*
2283 * ??? We get an unnecessary sign-extension of the dividend
2284 * into op0 with this definition, but as we do in fact always
2285 * produce both quotient and remainder using INDEX_op_div_i64
2286 * instead requires jumping through even more hoops.
2287 */
2288 tcg_debug_assert(args[0] == args[2]);
2289 tcg_debug_assert(args[1] == args[3]);
2290 tcg_debug_assert((args[1] & 1) == 0);
2291 tcg_debug_assert(args[0] == args[1] + 1);
2292 tcg_out_insn(s, RRE, DSGR, args[1], args[4]);
48bb3750
RH		 2293 break;
2294 case INDEX_op_divu2_i64:
4143f78d
RH		 2295 tcg_debug_assert(args[0] == args[2]);
2296 tcg_debug_assert(args[1] == args[3]);
2297 tcg_debug_assert((args[1] & 1) == 0);
2298 tcg_debug_assert(args[0] == args[1] + 1);
2299 tcg_out_insn(s, RRE, DLGR, args[1], args[4]);
48bb3750 2300 break;
36017dc6 2301 case INDEX_op_mulu2_i64:
4143f78d
RH		 2302 tcg_debug_assert(args[0] == args[2]);
2303 tcg_debug_assert((args[1] & 1) == 0);
2304 tcg_debug_assert(args[0] == args[1] + 1);
2305 tcg_out_insn(s, RRE, MLGR, args[1], args[3]);
36017dc6 2306 break;
48bb3750
RH		 2307
2308 case INDEX_op_shl_i64:
2309 op = RSY_SLLG;
2310 do_shift64:
2311 if (const_args[2]) {
2312 tcg_out_sh64(s, op, args[0], args[1], TCG_REG_NONE, args[2]);
2313 } else {
2314 tcg_out_sh64(s, op, args[0], args[1], args[2], 0);
2315 }
2316 break;
2317 case INDEX_op_shr_i64:
2318 op = RSY_SRLG;
2319 goto do_shift64;
2320 case INDEX_op_sar_i64:
2321 op = RSY_SRAG;
2322 goto do_shift64;
2323
2324 case INDEX_op_rotl_i64:
2325 if (const_args[2]) {
2326 tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
2327 TCG_REG_NONE, args[2]);
2328 } else {
2329 tcg_out_sh64(s, RSY_RLLG, args[0], args[1], args[2], 0);
2330 }
2331 break;
2332 case INDEX_op_rotr_i64:
2333 if (const_args[2]) {
2334 tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
2335 TCG_REG_NONE, (64 - args[2]) & 63);
2336 } else {
2337 /* We can use the smaller 32-bit negate because only the
2338 low 6 bits are examined for the rotate. */
2339 tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
2340 tcg_out_sh64(s, RSY_RLLG, args[0], args[1], TCG_TMP0, 0);
2341 }
2342 break;
2343
2344 case INDEX_op_ext8s_i64:
2345 tgen_ext8s(s, TCG_TYPE_I64, args[0], args[1]);
2346 break;
2347 case INDEX_op_ext16s_i64:
2348 tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
2349 break;
4f2331e5 2350 case INDEX_op_ext_i32_i64:
48bb3750
RH		 2351 case INDEX_op_ext32s_i64:
2352 tgen_ext32s(s, args[0], args[1]);
2353 break;
2354 case INDEX_op_ext8u_i64:
2355 tgen_ext8u(s, TCG_TYPE_I64, args[0], args[1]);
2356 break;
2357 case INDEX_op_ext16u_i64:
2358 tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
2359 break;
4f2331e5 2360 case INDEX_op_extu_i32_i64:
48bb3750
RH		 2361 case INDEX_op_ext32u_i64:
2362 tgen_ext32u(s, args[0], args[1]);
2363 break;
2364
3790b918 2365 case INDEX_op_add2_i64:
ad19b358
RH		 2366 if (const_args[4]) {
2367 if ((int64_t)args[4] >= 0) {
2368 tcg_out_insn(s, RIL, ALGFI, args[0], args[4]);
2369 } else {
2370 tcg_out_insn(s, RIL, SLGFI, args[0], -args[4]);
2371 }
2372 } else {
2373 tcg_out_insn(s, RRE, ALGR, args[0], args[4]);
2374 }
3790b918
RH		 2375 tcg_out_insn(s, RRE, ALCGR, args[1], args[5]);
2376 break;
2377 case INDEX_op_sub2_i64:
ad19b358
RH		 2378 if (const_args[4]) {
2379 if ((int64_t)args[4] >= 0) {
2380 tcg_out_insn(s, RIL, SLGFI, args[0], args[4]);
2381 } else {
2382 tcg_out_insn(s, RIL, ALGFI, args[0], -args[4]);
2383 }
2384 } else {
2385 tcg_out_insn(s, RRE, SLGR, args[0], args[4]);
2386 }
3790b918
RH		 2387 tcg_out_insn(s, RRE, SLBGR, args[1], args[5]);
2388 break;
2389
48bb3750
RH		 2390 case INDEX_op_brcond_i64:
2391 tgen_brcond(s, TCG_TYPE_I64, args[2], args[0],
bec16311 2392 args[1], const_args[1], arg_label(args[3]));
48bb3750
RH		 2393 break;
2394 case INDEX_op_setcond_i64:
2395 tgen_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1],
2396 args[2], const_args[2]);
2397 break;
96a9f093
RH		 2398 case INDEX_op_movcond_i64:
2399 tgen_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1],
7af525af 2400 args[2], const_args[2], args[3], const_args[3]);
96a9f093 2401 break;
48bb3750 2402
d5690ea4 2403 OP_32_64(deposit):
752b1be9
RH		 2404 a0 = args[0], a1 = args[1], a2 = args[2];
2405 if (const_args[1]) {
2406 tgen_deposit(s, a0, a2, args[3], args[4], 1);
2407 } else {
2408 /* Since we can't support "0Z" as a constraint, we allow a1 in
2409 any register. Fix things up as if a matching constraint. */
2410 if (a0 != a1) {
2411 TCGType type = (opc == INDEX_op_deposit_i64);
2412 if (a0 == a2) {
2413 tcg_out_mov(s, type, TCG_TMP0, a2);
2414 a2 = TCG_TMP0;
2415 }
2416 tcg_out_mov(s, type, a0, a1);
2417 }
2418 tgen_deposit(s, a0, a2, args[3], args[4], 0);
2419 }
d5690ea4 2420 break;
752b1be9 2421
b0bf5fe8
RH		 2422 OP_32_64(extract):
2423 tgen_extract(s, args[0], args[1], args[2], args[3]);
2424 break;
d5690ea4 2425
ce411066
RH		 2426 case INDEX_op_clz_i64:
2427 tgen_clz(s, args[0], args[1], args[2], const_args[2]);
2428 break;
2429
c9314d61
PK		 2430 case INDEX_op_mb:
2431 /* The host memory model is quite strong, we simply need to
2432 serialize the instruction stream. */
2433 if (args[0] & TCG_MO_ST_LD) {
748b7f3e 2434 tcg_out_insn(s, RR, BCR, HAVE_FACILITY(FAST_BCR_SER) ? 14 : 15, 0);
c9314d61
PK		 2435 }
2436 break;
2437
96d0ee7f
RH		 2438 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
2439 case INDEX_op_mov_i64:
96d0ee7f 2440 case INDEX_op_call: /* Always emitted via tcg_out_call. */
48bb3750 2441 default:
48bb3750
RH		 2442 tcg_abort();
2443 }
2827822e
AG		 2444}
2445
34ef7676
RH		 2446static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
2447 TCGReg dst, TCGReg src)
2448{
79cada86
RH		 2449 if (is_general_reg(src)) {
2450 /* Replicate general register into two MO_64. */
2451 tcg_out_insn(s, VRRf, VLVGP, dst, src, src);
2452 if (vece == MO_64) {
2453 return true;
2454 }
6e591a85 2455 src = dst;
79cada86
RH		 2456 }
2457
2458 /*
2459 * Recall that the "standard" integer, within a vector, is the
2460 * rightmost element of the leftmost doubleword, a-la VLLEZ.
2461 */
2462 tcg_out_insn(s, VRIc, VREP, dst, (8 >> vece) - 1, src, vece);
2463 return true;
34ef7676
RH		 2464}
2465
2466static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
2467 TCGReg dst, TCGReg base, intptr_t offset)
2468{
79cada86
RH		 2469 tcg_out_vrx_mem(s, VRX_VLREP, dst, base, TCG_REG_NONE, offset, vece);
2470 return true;
34ef7676
RH		 2471}
2472
2473static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
2474 TCGReg dst, int64_t val)
2475{
79cada86
RH		 2476 int i, mask, msb, lsb;
2477
2478 /* Look for int16_t elements. */
2479 if (vece <= MO_16 ||
2480 (vece == MO_32 ? (int32_t)val : val) == (int16_t)val) {
2481 tcg_out_insn(s, VRIa, VREPI, dst, val, vece);
2482 return;
2483 }
2484
2485 /* Look for bit masks. */
2486 if (vece == MO_32) {
2487 if (risbg_mask((int32_t)val)) {
2488 /* Handle wraparound by swapping msb and lsb. */
2489 if ((val & 0x80000001u) == 0x80000001u) {
2490 msb = 32 - ctz32(~val);
2491 lsb = clz32(~val) - 1;
2492 } else {
2493 msb = clz32(val);
2494 lsb = 31 - ctz32(val);
2495 }
a66669c9 2496 tcg_out_insn(s, VRIb, VGM, dst, msb, lsb, MO_32);
79cada86
RH		 2497 return;
2498 }
2499 } else {
2500 if (risbg_mask(val)) {
2501 /* Handle wraparound by swapping msb and lsb. */
2502 if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
2503 /* Handle wraparound by swapping msb and lsb. */
2504 msb = 64 - ctz64(~val);
2505 lsb = clz64(~val) - 1;
2506 } else {
2507 msb = clz64(val);
2508 lsb = 63 - ctz64(val);
2509 }
a66669c9 2510 tcg_out_insn(s, VRIb, VGM, dst, msb, lsb, MO_64);
79cada86
RH		 2511 return;
2512 }
2513 }
2514
2515 /* Look for all bytes 0x00 or 0xff. */
2516 for (i = mask = 0; i < 8; i++) {
2517 uint8_t byte = val >> (i * 8);
2518 if (byte == 0xff) {
2519 mask |= 1 << i;
2520 } else if (byte != 0) {
2521 break;
2522 }
2523 }
2524 if (i == 8) {
2525 tcg_out_insn(s, VRIa, VGBM, dst, mask * 0x0101, 0);
2526 return;
2527 }
2528
2529 /* Otherwise, stuff it in the constant pool. */
2530 tcg_out_insn(s, RIL, LARL, TCG_TMP0, 0);
2531 new_pool_label(s, val, R_390_PC32DBL, s->code_ptr - 2, 2);
2532 tcg_out_insn(s, VRX, VLREP, dst, TCG_TMP0, TCG_REG_NONE, 0, MO_64);
34ef7676
RH		 2533}
2534
2535static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2536 unsigned vecl, unsigned vece,
d58f0173
MR		 2537 const TCGArg args[TCG_MAX_OP_ARGS],
2538 const int const_args[TCG_MAX_OP_ARGS])
34ef7676 2539{
a429ee29
RH		 2540 TCGType type = vecl + TCG_TYPE_V64;
2541 TCGArg a0 = args[0], a1 = args[1], a2 = args[2];
2542
2543 switch (opc) {
2544 case INDEX_op_ld_vec:
2545 tcg_out_ld(s, type, a0, a1, a2);
2546 break;
2547 case INDEX_op_st_vec:
2548 tcg_out_st(s, type, a0, a1, a2);
2549 break;
2550 case INDEX_op_dupm_vec:
2551 tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
2552 break;
2553
ae77bbe5
RH		 2554 case INDEX_op_abs_vec:
2555 tcg_out_insn(s, VRRa, VLP, a0, a1, vece);
2556 break;
2557 case INDEX_op_neg_vec:
2558 tcg_out_insn(s, VRRa, VLC, a0, a1, vece);
2559 break;
2560 case INDEX_op_not_vec:
2561 tcg_out_insn(s, VRRc, VNO, a0, a1, a1, 0);
2562 break;
2563
a429ee29
RH		 2564 case INDEX_op_add_vec:
2565 tcg_out_insn(s, VRRc, VA, a0, a1, a2, vece);
2566 break;
2567 case INDEX_op_sub_vec:
2568 tcg_out_insn(s, VRRc, VS, a0, a1, a2, vece);
2569 break;
2570 case INDEX_op_and_vec:
2571 tcg_out_insn(s, VRRc, VN, a0, a1, a2, 0);
2572 break;
ae77bbe5
RH		 2573 case INDEX_op_andc_vec:
2574 tcg_out_insn(s, VRRc, VNC, a0, a1, a2, 0);
2575 break;
479b61cb
RH		 2576 case INDEX_op_mul_vec:
2577 tcg_out_insn(s, VRRc, VML, a0, a1, a2, vece);
2578 break;
a429ee29
RH		 2579 case INDEX_op_or_vec:
2580 tcg_out_insn(s, VRRc, VO, a0, a1, a2, 0);
2581 break;
ae77bbe5
RH		 2582 case INDEX_op_orc_vec:
2583 tcg_out_insn(s, VRRc, VOC, a0, a1, a2, 0);
2584 break;
a429ee29
RH		 2585 case INDEX_op_xor_vec:
2586 tcg_out_insn(s, VRRc, VX, a0, a1, a2, 0);
2587 break;
21eab5bf
RH		 2588 case INDEX_op_nand_vec:
2589 tcg_out_insn(s, VRRc, VNN, a0, a1, a2, 0);
2590 break;
2591 case INDEX_op_nor_vec:
2592 tcg_out_insn(s, VRRc, VNO, a0, a1, a2, 0);
2593 break;
2594 case INDEX_op_eqv_vec:
2595 tcg_out_insn(s, VRRc, VNX, a0, a1, a2, 0);
2596 break;
a429ee29 2597
22cb37b4
RH		 2598 case INDEX_op_shli_vec:
2599 tcg_out_insn(s, VRSa, VESL, a0, a2, TCG_REG_NONE, a1, vece);
2600 break;
2601 case INDEX_op_shri_vec:
2602 tcg_out_insn(s, VRSa, VESRL, a0, a2, TCG_REG_NONE, a1, vece);
2603 break;
2604 case INDEX_op_sari_vec:
2605 tcg_out_insn(s, VRSa, VESRA, a0, a2, TCG_REG_NONE, a1, vece);
2606 break;
2607 case INDEX_op_rotli_vec:
2608 tcg_out_insn(s, VRSa, VERLL, a0, a2, TCG_REG_NONE, a1, vece);
2609 break;
2610 case INDEX_op_shls_vec:
2611 tcg_out_insn(s, VRSa, VESL, a0, 0, a2, a1, vece);
2612 break;
2613 case INDEX_op_shrs_vec:
2614 tcg_out_insn(s, VRSa, VESRL, a0, 0, a2, a1, vece);
2615 break;
2616 case INDEX_op_sars_vec:
2617 tcg_out_insn(s, VRSa, VESRA, a0, 0, a2, a1, vece);
2618 break;
2619 case INDEX_op_rotls_vec:
2620 tcg_out_insn(s, VRSa, VERLL, a0, 0, a2, a1, vece);
2621 break;
2622 case INDEX_op_shlv_vec:
2623 tcg_out_insn(s, VRRc, VESLV, a0, a1, a2, vece);
2624 break;
2625 case INDEX_op_shrv_vec:
2626 tcg_out_insn(s, VRRc, VESRLV, a0, a1, a2, vece);
2627 break;
2628 case INDEX_op_sarv_vec:
2629 tcg_out_insn(s, VRRc, VESRAV, a0, a1, a2, vece);
2630 break;
2631 case INDEX_op_rotlv_vec:
2632 tcg_out_insn(s, VRRc, VERLLV, a0, a1, a2, vece);
2633 break;
2634
220db7a6
RH		 2635 case INDEX_op_smin_vec:
2636 tcg_out_insn(s, VRRc, VMN, a0, a1, a2, vece);
2637 break;
2638 case INDEX_op_smax_vec:
2639 tcg_out_insn(s, VRRc, VMX, a0, a1, a2, vece);
2640 break;
2641 case INDEX_op_umin_vec:
2642 tcg_out_insn(s, VRRc, VMNL, a0, a1, a2, vece);
2643 break;
2644 case INDEX_op_umax_vec:
2645 tcg_out_insn(s, VRRc, VMXL, a0, a1, a2, vece);
2646 break;
2647
9bca986d 2648 case INDEX_op_bitsel_vec:
6e5f9fb7 2649 tcg_out_insn(s, VRRe, VSEL, a0, a2, args[3], a1);
9bca986d
RH		 2650 break;
2651
a429ee29
RH		 2652 case INDEX_op_cmp_vec:
2653 switch ((TCGCond)args[3]) {
2654 case TCG_COND_EQ:
2655 tcg_out_insn(s, VRRc, VCEQ, a0, a1, a2, vece);
2656 break;
2657 case TCG_COND_GT:
2658 tcg_out_insn(s, VRRc, VCH, a0, a1, a2, vece);
2659 break;
2660 case TCG_COND_GTU:
2661 tcg_out_insn(s, VRRc, VCHL, a0, a1, a2, vece);
2662 break;
2663 default:
2664 g_assert_not_reached();
2665 }
2666 break;
2667
4223c9c1
RH		 2668 case INDEX_op_s390_vuph_vec:
2669 tcg_out_insn(s, VRRa, VUPH, a0, a1, vece);
2670 break;
2671 case INDEX_op_s390_vupl_vec:
2672 tcg_out_insn(s, VRRa, VUPL, a0, a1, vece);
2673 break;
2674 case INDEX_op_s390_vpks_vec:
2675 tcg_out_insn(s, VRRc, VPKS, a0, a1, a2, vece);
2676 break;
2677
a429ee29
RH		 2678 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
2679 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
2680 default:
2681 g_assert_not_reached();
2682 }
34ef7676
RH		 2683}
2684
2685int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
2686{
a429ee29 2687 switch (opc) {
ae77bbe5 2688 case INDEX_op_abs_vec:
a429ee29
RH		 2689 case INDEX_op_add_vec:
2690 case INDEX_op_and_vec:
ae77bbe5 2691 case INDEX_op_andc_vec:
9bca986d 2692 case INDEX_op_bitsel_vec:
21eab5bf
RH		 2693 case INDEX_op_eqv_vec:
2694 case INDEX_op_nand_vec:
ae77bbe5 2695 case INDEX_op_neg_vec:
21eab5bf 2696 case INDEX_op_nor_vec:
ae77bbe5 2697 case INDEX_op_not_vec:
a429ee29 2698 case INDEX_op_or_vec:
ae77bbe5 2699 case INDEX_op_orc_vec:
22cb37b4
RH		 2700 case INDEX_op_rotli_vec:
2701 case INDEX_op_rotls_vec:
2702 case INDEX_op_rotlv_vec:
2703 case INDEX_op_sari_vec:
2704 case INDEX_op_sars_vec:
2705 case INDEX_op_sarv_vec:
2706 case INDEX_op_shli_vec:
2707 case INDEX_op_shls_vec:
2708 case INDEX_op_shlv_vec:
2709 case INDEX_op_shri_vec:
2710 case INDEX_op_shrs_vec:
2711 case INDEX_op_shrv_vec:
220db7a6
RH		 2712 case INDEX_op_smax_vec:
2713 case INDEX_op_smin_vec:
a429ee29 2714 case INDEX_op_sub_vec:
220db7a6
RH		 2715 case INDEX_op_umax_vec:
2716 case INDEX_op_umin_vec:
a429ee29
RH		 2717 case INDEX_op_xor_vec:
2718 return 1;
2719 case INDEX_op_cmp_vec:
ea3f2af8 2720 case INDEX_op_cmpsel_vec:
22cb37b4 2721 case INDEX_op_rotrv_vec:
a429ee29 2722 return -1;
479b61cb
RH		 2723 case INDEX_op_mul_vec:
2724 return vece < MO_64;
4223c9c1
RH		 2725 case INDEX_op_ssadd_vec:
2726 case INDEX_op_sssub_vec:
2727 return vece < MO_64 ? -1 : 0;
a429ee29
RH		 2728 default:
2729 return 0;
2730 }
2731}
2732
2733static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0,
2734 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
2735{
2736 bool need_swap = false, need_inv = false;
2737
2738 switch (cond) {
2739 case TCG_COND_EQ:
2740 case TCG_COND_GT:
2741 case TCG_COND_GTU:
2742 break;
2743 case TCG_COND_NE:
2744 case TCG_COND_LE:
2745 case TCG_COND_LEU:
2746 need_inv = true;
2747 break;
2748 case TCG_COND_LT:
2749 case TCG_COND_LTU:
2750 need_swap = true;
2751 break;
2752 case TCG_COND_GE:
2753 case TCG_COND_GEU:
2754 need_swap = need_inv = true;
2755 break;
2756 default:
2757 g_assert_not_reached();
2758 }
2759
2760 if (need_inv) {
2761 cond = tcg_invert_cond(cond);
2762 }
2763 if (need_swap) {
2764 TCGv_vec t1;
2765 t1 = v1, v1 = v2, v2 = t1;
2766 cond = tcg_swap_cond(cond);
2767 }
2768
2769 vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0),
2770 tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond);
2771
2772 return need_inv;
2773}
2774
2775static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0,
2776 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
2777{
2778 if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) {
2779 tcg_gen_not_vec(vece, v0, v0);
2780 }
34ef7676
RH		 2781}
2782
ea3f2af8
RH		 2783static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0,
2784 TCGv_vec c1, TCGv_vec c2,
2785 TCGv_vec v3, TCGv_vec v4, TCGCond cond)
2786{
2787 TCGv_vec t = tcg_temp_new_vec(type);
2788
2789 if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) {
2790 /* Invert the sense of the compare by swapping arguments. */
2791 tcg_gen_bitsel_vec(vece, v0, t, v4, v3);
2792 } else {
2793 tcg_gen_bitsel_vec(vece, v0, t, v3, v4);
2794 }
2795 tcg_temp_free_vec(t);
2796}
2797
4223c9c1
RH		 2798static void expand_vec_sat(TCGType type, unsigned vece, TCGv_vec v0,
2799 TCGv_vec v1, TCGv_vec v2, TCGOpcode add_sub_opc)
2800{
2801 TCGv_vec h1 = tcg_temp_new_vec(type);
2802 TCGv_vec h2 = tcg_temp_new_vec(type);
2803 TCGv_vec l1 = tcg_temp_new_vec(type);
2804 TCGv_vec l2 = tcg_temp_new_vec(type);
2805
2806 tcg_debug_assert (vece < MO_64);
2807
2808 /* Unpack with sign-extension. */
2809 vec_gen_2(INDEX_op_s390_vuph_vec, type, vece,
2810 tcgv_vec_arg(h1), tcgv_vec_arg(v1));
2811 vec_gen_2(INDEX_op_s390_vuph_vec, type, vece,
2812 tcgv_vec_arg(h2), tcgv_vec_arg(v2));
2813
2814 vec_gen_2(INDEX_op_s390_vupl_vec, type, vece,
2815 tcgv_vec_arg(l1), tcgv_vec_arg(v1));
2816 vec_gen_2(INDEX_op_s390_vupl_vec, type, vece,
2817 tcgv_vec_arg(l2), tcgv_vec_arg(v2));
2818
2819 /* Arithmetic on a wider element size. */
2820 vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(h1),
2821 tcgv_vec_arg(h1), tcgv_vec_arg(h2));
2822 vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(l1),
2823 tcgv_vec_arg(l1), tcgv_vec_arg(l2));
2824
2825 /* Pack with saturation. */
2826 vec_gen_3(INDEX_op_s390_vpks_vec, type, vece + 1,
2827 tcgv_vec_arg(v0), tcgv_vec_arg(h1), tcgv_vec_arg(l1));
2828
2829 tcg_temp_free_vec(h1);
2830 tcg_temp_free_vec(h2);
2831 tcg_temp_free_vec(l1);
2832 tcg_temp_free_vec(l2);
2833}
2834
34ef7676
RH		 2835void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
2836 TCGArg a0, ...)
2837{
a429ee29 2838 va_list va;
ea3f2af8 2839 TCGv_vec v0, v1, v2, v3, v4, t0;
a429ee29
RH		 2840
2841 va_start(va, a0);
2842 v0 = temp_tcgv_vec(arg_temp(a0));
2843 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2844 v2 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2845
2846 switch (opc) {
2847 case INDEX_op_cmp_vec:
2848 expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg));
2849 break;
2850
ea3f2af8
RH		 2851 case INDEX_op_cmpsel_vec:
2852 v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2853 v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2854 expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg));
2855 break;
2856
22cb37b4
RH		 2857 case INDEX_op_rotrv_vec:
2858 t0 = tcg_temp_new_vec(type);
2859 tcg_gen_neg_vec(vece, t0, v2);
2860 tcg_gen_rotlv_vec(vece, v0, v1, t0);
2861 tcg_temp_free_vec(t0);
2862 break;
2863
4223c9c1
RH		 2864 case INDEX_op_ssadd_vec:
2865 expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_add_vec);
2866 break;
2867 case INDEX_op_sssub_vec:
2868 expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_sub_vec);
2869 break;
2870
a429ee29
RH		 2871 default:
2872 g_assert_not_reached();
2873 }
2874 va_end(va);
34ef7676
RH		 2875}
2876
d1c36a90 2877static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op)
f69d277e 2878{
9b5500b6
RH		 2879 switch (op) {
2880 case INDEX_op_goto_ptr:
d1c36a90 2881 return C_O0_I1(r);
9b5500b6
RH		 2882
2883 case INDEX_op_ld8u_i32:
2884 case INDEX_op_ld8u_i64:
2885 case INDEX_op_ld8s_i32:
2886 case INDEX_op_ld8s_i64:
2887 case INDEX_op_ld16u_i32:
2888 case INDEX_op_ld16u_i64:
2889 case INDEX_op_ld16s_i32:
2890 case INDEX_op_ld16s_i64:
2891 case INDEX_op_ld_i32:
2892 case INDEX_op_ld32u_i64:
2893 case INDEX_op_ld32s_i64:
2894 case INDEX_op_ld_i64:
d1c36a90
RH		 2895 return C_O1_I1(r, r);
2896
9b5500b6
RH		 2897 case INDEX_op_st8_i32:
2898 case INDEX_op_st8_i64:
2899 case INDEX_op_st16_i32:
2900 case INDEX_op_st16_i64:
2901 case INDEX_op_st_i32:
2902 case INDEX_op_st32_i64:
2903 case INDEX_op_st_i64:
d1c36a90 2904 return C_O0_I2(r, r);
9b5500b6
RH		 2905
2906 case INDEX_op_add_i32:
2907 case INDEX_op_add_i64:
d1c36a90
RH		 2908 case INDEX_op_shl_i64:
2909 case INDEX_op_shr_i64:
2910 case INDEX_op_sar_i64:
2911 case INDEX_op_rotl_i32:
2912 case INDEX_op_rotl_i64:
2913 case INDEX_op_rotr_i32:
2914 case INDEX_op_rotr_i64:
2915 case INDEX_op_clz_i64:
2916 case INDEX_op_setcond_i32:
2917 case INDEX_op_setcond_i64:
2918 return C_O1_I2(r, r, ri);
2919
9b5500b6
RH		 2920 case INDEX_op_sub_i32:
2921 case INDEX_op_sub_i64:
bdcd5d19
RH		 2922 case INDEX_op_and_i32:
2923 case INDEX_op_and_i64:
4046d9ca
RH		 2924 case INDEX_op_or_i32:
2925 case INDEX_op_or_i64:
5bf67a92
RH		 2926 case INDEX_op_xor_i32:
2927 case INDEX_op_xor_i64:
748b7f3e 2928 return (HAVE_FACILITY(DISTINCT_OPS)
d1c36a90
RH		 2929 ? C_O1_I2(r, r, ri)
2930 : C_O1_I2(r, 0, ri));
a8f0269e 2931
9b5500b6 2932 case INDEX_op_mul_i32:
9c3bfb79 2933 return C_O1_I2(r, 0, ri);
9b5500b6 2934 case INDEX_op_mul_i64:
9c3bfb79 2935 return C_O1_I2(r, 0, rJ);
a8f0269e 2936
9b5500b6
RH		 2937 case INDEX_op_shl_i32:
2938 case INDEX_op_shr_i32:
2939 case INDEX_op_sar_i32:
748b7f3e 2940 return (HAVE_FACILITY(DISTINCT_OPS)
d1c36a90
RH		 2941 ? C_O1_I2(r, r, ri)
2942 : C_O1_I2(r, 0, ri));
9b5500b6
RH		 2943
2944 case INDEX_op_brcond_i32:
2945 case INDEX_op_brcond_i64:
d1c36a90 2946 return C_O0_I2(r, ri);
9b5500b6
RH		 2947
2948 case INDEX_op_bswap16_i32:
2949 case INDEX_op_bswap16_i64:
2950 case INDEX_op_bswap32_i32:
2951 case INDEX_op_bswap32_i64:
2952 case INDEX_op_bswap64_i64:
2953 case INDEX_op_neg_i32:
2954 case INDEX_op_neg_i64:
2955 case INDEX_op_ext8s_i32:
2956 case INDEX_op_ext8s_i64:
2957 case INDEX_op_ext8u_i32:
2958 case INDEX_op_ext8u_i64:
2959 case INDEX_op_ext16s_i32:
2960 case INDEX_op_ext16s_i64:
2961 case INDEX_op_ext16u_i32:
2962 case INDEX_op_ext16u_i64:
2963 case INDEX_op_ext32s_i64:
2964 case INDEX_op_ext32u_i64:
2965 case INDEX_op_ext_i32_i64:
2966 case INDEX_op_extu_i32_i64:
2967 case INDEX_op_extract_i32:
2968 case INDEX_op_extract_i64:
d1c36a90 2969 return C_O1_I1(r, r);
9b5500b6
RH		 2970
2971 case INDEX_op_qemu_ld_i32:
2972 case INDEX_op_qemu_ld_i64:
d1c36a90 2973 return C_O1_I1(r, L);
9b5500b6
RH		 2974 case INDEX_op_qemu_st_i64:
2975 case INDEX_op_qemu_st_i32:
d1c36a90 2976 return C_O0_I2(L, L);
f69d277e 2977
9b5500b6
RH		 2978 case INDEX_op_deposit_i32:
2979 case INDEX_op_deposit_i64:
d1c36a90
RH		 2980 return C_O1_I2(r, rZ, r);
2981
9b5500b6
RH		 2982 case INDEX_op_movcond_i32:
2983 case INDEX_op_movcond_i64:
748b7f3e 2984 return (HAVE_FACILITY(LOAD_ON_COND2)
d1c36a90
RH		 2985 ? C_O1_I4(r, r, ri, rI, 0)
2986 : C_O1_I4(r, r, ri, r, 0));
2987
9b5500b6
RH		 2988 case INDEX_op_div2_i32:
2989 case INDEX_op_div2_i64:
2990 case INDEX_op_divu2_i32:
2991 case INDEX_op_divu2_i64:
4143f78d 2992 return C_O2_I3(o, m, 0, 1, r);
d1c36a90 2993
9b5500b6 2994 case INDEX_op_mulu2_i64:
4143f78d 2995 return C_O2_I2(o, m, 0, r);
ba18b07d 2996
9b5500b6 2997 case INDEX_op_add2_i32:
9b5500b6 2998 case INDEX_op_sub2_i32:
3e25f7da 2999 return C_O2_I4(r, r, 0, 1, ri, r);
d1c36a90 3000
ba18b07d 3001 case INDEX_op_add2_i64:
9b5500b6 3002 case INDEX_op_sub2_i64:
3e25f7da 3003 return C_O2_I4(r, r, 0, 1, rA, r);
9b5500b6 3004
34ef7676
RH		 3005 case INDEX_op_st_vec:
3006 return C_O0_I2(v, r);
3007 case INDEX_op_ld_vec:
3008 case INDEX_op_dupm_vec:
3009 return C_O1_I1(v, r);
3010 case INDEX_op_dup_vec:
3011 return C_O1_I1(v, vr);
ae77bbe5
RH		 3012 case INDEX_op_abs_vec:
3013 case INDEX_op_neg_vec:
3014 case INDEX_op_not_vec:
22cb37b4
RH		 3015 case INDEX_op_rotli_vec:
3016 case INDEX_op_sari_vec:
3017 case INDEX_op_shli_vec:
3018 case INDEX_op_shri_vec:
4223c9c1
RH		 3019 case INDEX_op_s390_vuph_vec:
3020 case INDEX_op_s390_vupl_vec:
ae77bbe5 3021 return C_O1_I1(v, v);
34ef7676
RH		 3022 case INDEX_op_add_vec:
3023 case INDEX_op_sub_vec:
3024 case INDEX_op_and_vec:
ae77bbe5 3025 case INDEX_op_andc_vec:
34ef7676 3026 case INDEX_op_or_vec:
ae77bbe5 3027 case INDEX_op_orc_vec:
34ef7676 3028 case INDEX_op_xor_vec:
21eab5bf
RH		 3029 case INDEX_op_nand_vec:
3030 case INDEX_op_nor_vec:
3031 case INDEX_op_eqv_vec:
34ef7676 3032 case INDEX_op_cmp_vec:
479b61cb 3033 case INDEX_op_mul_vec:
22cb37b4
RH		 3034 case INDEX_op_rotlv_vec:
3035 case INDEX_op_rotrv_vec:
3036 case INDEX_op_shlv_vec:
3037 case INDEX_op_shrv_vec:
3038 case INDEX_op_sarv_vec:
220db7a6
RH		 3039 case INDEX_op_smax_vec:
3040 case INDEX_op_smin_vec:
3041 case INDEX_op_umax_vec:
3042 case INDEX_op_umin_vec:
4223c9c1 3043 case INDEX_op_s390_vpks_vec:
34ef7676 3044 return C_O1_I2(v, v, v);
22cb37b4
RH		 3045 case INDEX_op_rotls_vec:
3046 case INDEX_op_shls_vec:
3047 case INDEX_op_shrs_vec:
3048 case INDEX_op_sars_vec:
3049 return C_O1_I2(v, v, r);
9bca986d
RH		 3050 case INDEX_op_bitsel_vec:
3051 return C_O1_I3(v, v, v, v);
34ef7676 3052
9b5500b6 3053 default:
d1c36a90 3054 g_assert_not_reached();
f69d277e 3055 }
f69d277e
RH		 3056}
3057
34ef7676
RH		 3058/*
3059 * Mainline glibc added HWCAP_S390_VX before it was kernel abi.
3060 * Some distros have fixed this up locally, others have not.
3061 */
3062#ifndef HWCAP_S390_VXRS
3063#define HWCAP_S390_VXRS 2048
3064#endif
3065
b2c98d9d 3066static void query_s390_facilities(void)
48bb3750 3067{
c9baa30f 3068 unsigned long hwcap = qemu_getauxval(AT_HWCAP);
761ea522 3069 const char *which;
48bb3750 3070
c9baa30f
RH		 3071 /* Is STORE FACILITY LIST EXTENDED available? Honestly, I believe this
3072 is present on all 64-bit systems, but let's check for it anyway. */
3073 if (hwcap & HWCAP_S390_STFLE) {
748b7f3e
RH		 3074 register int r0 __asm__("0") = ARRAY_SIZE(s390_facilities) - 1;
3075 register void *r1 __asm__("1") = s390_facilities;
48bb3750 3076
c9baa30f 3077 /* stfle 0(%r1) */
c9baa30f 3078 asm volatile(".word 0xb2b0,0x1000"
748b7f3e 3079 : "=r"(r0) : "r"(r0), "r"(r1) : "memory", "cc");
48bb3750 3080 }
34ef7676
RH		 3081
3082 /*
3083 * Use of vector registers requires os support beyond the facility bit.
3084 * If the kernel does not advertise support, disable the facility bits.
3085 * There is nothing else we currently care about in the 3rd word, so
3086 * disable VECTOR with one store.
3087 */
a429ee29 3088 if (!(hwcap & HWCAP_S390_VXRS)) {
34ef7676
RH		 3089 s390_facilities[2] = 0;
3090 }
761ea522
RH		 3091
3092 /*
c68d5b7a 3093 * Minimum supported cpu revision is z196.
761ea522
RH		 3094 * Check for all required facilities.
3095 * ZARCH_ACTIVE is done via preprocessor check for 64-bit.
3096 */
3097 if (!HAVE_FACILITY(LONG_DISP)) {
3098 which = "long-displacement";
3099 goto fail;
3100 }
3e25f7da
RH		 3101 if (!HAVE_FACILITY(EXT_IMM)) {
3102 which = "extended-immediate";
3103 goto fail;
3104 }
9c3bfb79
RH		 3105 if (!HAVE_FACILITY(GEN_INST_EXT)) {
3106 which = "general-instructions-extension";
3107 goto fail;
3108 }
c68d5b7a
RH		 3109 /*
3110 * Facility 45 is a big bin that contains: distinct-operands,
3111 * fast-BCR-serialization, high-word, population-count,
3112 * interlocked-access-1, and load/store-on-condition-1
3113 */
3114 if (!HAVE_FACILITY(45)) {
3115 which = "45";
3116 goto fail;
3117 }
761ea522
RH		 3118 return;
3119
3120 fail:
3121 error_report("%s: missing required facility %s", __func__, which);
3122 exit(EXIT_FAILURE);
48bb3750
RH		 3123}
3124
3125static void tcg_target_init(TCGContext *s)
2827822e 3126{
b2c98d9d 3127 query_s390_facilities();
48bb3750 3128
f46934df
RH		 3129 tcg_target_available_regs[TCG_TYPE_I32] = 0xffff;
3130 tcg_target_available_regs[TCG_TYPE_I64] = 0xffff;
34ef7676
RH		 3131 if (HAVE_FACILITY(VECTOR)) {
3132 tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
3133 tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
3134 }
48bb3750 3135
ccb1bb66 3136 tcg_target_call_clobber_regs = 0;
48bb3750
RH		 3137 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
3138 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R1);
3139 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
3140 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
3141 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
3142 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
f24efee4
RH		 3143 /* The r6 register is technically call-saved, but it's also a parameter
3144 register, so it can get killed by setup for the qemu_st helper. */
3145 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
48bb3750
RH		 3146 /* The return register can be considered call-clobbered. */
3147 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14);
3148
34ef7676
RH		 3149 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V0);
3150 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V1);
3151 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V2);
3152 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V3);
3153 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V4);
3154 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V5);
3155 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V6);
3156 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V7);
3157 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V16);
3158 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V17);
3159 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V18);
3160 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V19);
3161 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V20);
3162 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V21);
3163 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V22);
3164 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V23);
3165 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V24);
3166 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V25);
3167 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V26);
3168 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V27);
3169 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V28);
3170 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V29);
3171 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V30);
3172 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V31);
3173
ccb1bb66 3174 s->reserved_regs = 0;
48bb3750
RH		 3175 tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);
3176 /* XXX many insns can't be used with R0, so we better avoid it for now */
3177 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);
3178 tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
2827822e
AG		 3179}
3180
f167dc37
RH		 3181#define FRAME_SIZE ((int)(TCG_TARGET_CALL_STACK_OFFSET \
3182 + TCG_STATIC_CALL_ARGS_SIZE \
3183 + CPU_TEMP_BUF_NLONGS * sizeof(long)))
3184
48bb3750 3185static void tcg_target_qemu_prologue(TCGContext *s)
2827822e 3186{
48bb3750
RH		 3187 /* stmg %r6,%r15,48(%r15) (save registers) */
3188 tcg_out_insn(s, RXY, STMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15, 48);
3189
a4924e8b 3190 /* aghi %r15,-frame_size */
f167dc37 3191 tcg_out_insn(s, RI, AGHI, TCG_REG_R15, -FRAME_SIZE);
a4924e8b
RH		 3192
3193 tcg_set_frame(s, TCG_REG_CALL_STACK,
3194 TCG_STATIC_CALL_ARGS_SIZE + TCG_TARGET_CALL_STACK_OFFSET,
3195 CPU_TEMP_BUF_NLONGS * sizeof(long));
48bb3750 3196
090d0bfd 3197#ifndef CONFIG_SOFTMMU
b76f21a7 3198 if (guest_base >= 0x80000) {
ccbecb44 3199 tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
48bb3750
RH		 3200 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
3201 }
090d0bfd 3202#endif
48bb3750 3203
cea5f9a2 3204 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
829e1376 3205
cea5f9a2
BS		 3206 /* br %r3 (go to TB) */
3207 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, tcg_target_call_iarg_regs[1]);
48bb3750 3208
46644483
RH		 3209 /*
3210 * Return path for goto_ptr. Set return value to 0, a-la exit_tb,
3211 * and fall through to the rest of the epilogue.
3212 */
c8bc1168 3213 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
46644483
RH		 3214 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, 0);
3215
3216 /* TB epilogue */
79dae4dd 3217 tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
48bb3750 3218
a4924e8b
RH		 3219 /* lmg %r6,%r15,fs+48(%r15) (restore registers) */
3220 tcg_out_insn(s, RXY, LMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15,
f167dc37 3221 FRAME_SIZE + 48);
48bb3750
RH		 3222
3223 /* br %r14 (return) */
3224 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_R14);
2827822e 3225}
f167dc37 3226
28eef8aa
RH		 3227static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
3228{
3229 memset(p, 0x07, count * sizeof(tcg_insn_unit));
3230}
3231
f167dc37 3232typedef struct {
d2e16f2c 3233 DebugFrameHeader h;
f167dc37
RH		 3234 uint8_t fde_def_cfa[4];
3235 uint8_t fde_reg_ofs[18];
3236} DebugFrame;
3237
3238/* We're expecting a 2 byte uleb128 encoded value. */
3239QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
3240
3241#define ELF_HOST_MACHINE EM_S390
3242
d2e16f2c
RH		 3243static const DebugFrame debug_frame = {
3244 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
3245 .h.cie.id = -1,
3246 .h.cie.version = 1,
3247 .h.cie.code_align = 1,
3248 .h.cie.data_align = 8, /* sleb128 8 */
3249 .h.cie.return_column = TCG_REG_R14,
f167dc37
RH		 3250
3251 /* Total FDE size does not include the "len" member. */
d2e16f2c 3252 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
f167dc37
RH		 3253
3254 .fde_def_cfa = {
3255 12, TCG_REG_CALL_STACK, /* DW_CFA_def_cfa %r15, ... */
3256 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
3257 (FRAME_SIZE >> 7)
3258 },
3259 .fde_reg_ofs = {
3260 0x86, 6, /* DW_CFA_offset, %r6, 48 */
3261 0x87, 7, /* DW_CFA_offset, %r7, 56 */
3262 0x88, 8, /* DW_CFA_offset, %r8, 64 */
3263 0x89, 9, /* DW_CFA_offset, %r92, 72 */
3264 0x8a, 10, /* DW_CFA_offset, %r10, 80 */
3265 0x8b, 11, /* DW_CFA_offset, %r11, 88 */
3266 0x8c, 12, /* DW_CFA_offset, %r12, 96 */
3267 0x8d, 13, /* DW_CFA_offset, %r13, 104 */
3268 0x8e, 14, /* DW_CFA_offset, %r14, 112 */
3269 }
3270};
3271
755bf9e5 3272void tcg_register_jit(const void *buf, size_t buf_size)
f167dc37 3273{
f167dc37
RH		 3274 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
3275}
QEMU mirror