]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/s390x/tcg-target.c.inc
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg/s390x: Issue XILF directly for xor_i32
[mirror_qemu.git] / tcg / s390x / tcg-target.c.inc
CommitLineData
2827822e
AG		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2009 Ulrich Hecht <uli@suse.de>
48bb3750
RH		 5 * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
6 * Copyright (c) 2010 Richard Henderson <rth@twiddle.net>
2827822e
AG		 7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a copy
9 * of this software and associated documentation files (the "Software"), to deal
10 * in the Software without restriction, including without limitation the rights
11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 * copies of the Software, and to permit persons to whom the Software is
13 * furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 * THE SOFTWARE.
25 */
26
a01fc30d
RH		 27/* We only support generating code for 64-bit mode. */
28#if TCG_TARGET_REG_BITS != 64
29#error "unsupported code generation mode"
30#endif
31
1cd49868 32#include "../tcg-ldst.c.inc"
139c1837 33#include "../tcg-pool.c.inc"
c9baa30f
RH		 34#include "elf.h"
35
a8f0269e
RH		 36#define TCG_CT_CONST_S16 0x100
37#define TCG_CT_CONST_S32 0x200
a534bb15
RH		 38#define TCG_CT_CONST_S33 0x400
39#define TCG_CT_CONST_ZERO 0x800
48bb3750 40
c947deb1 41#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16)
34ef7676
RH		 42#define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32)
43
c947deb1
RH		 44/*
45 * For softmmu, we need to avoid conflicts with the first 3
46 * argument registers to perform the tlb lookup, and to call
47 * the helper function.
48 */
49#ifdef CONFIG_SOFTMMU
50#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_R2, 3)
51#else
52#define SOFTMMU_RESERVE_REGS 0
53#endif
54
55
48bb3750
RH		 56/* Several places within the instruction set 0 means "no register"
57 rather than TCG_REG_R0. */
58#define TCG_REG_NONE 0
59
60/* A scratch register that may be be used throughout the backend. */
ce411066 61#define TCG_TMP0 TCG_REG_R1
48bb3750 62
4cbea598 63#ifndef CONFIG_SOFTMMU
48bb3750 64#define TCG_GUEST_BASE_REG TCG_REG_R13
48bb3750
RH		 65#endif
66
48bb3750
RH		 67/* All of the following instructions are prefixed with their instruction
68 format, and are defined as 8- or 16-bit quantities, even when the two
69 halves of the 16-bit quantity may appear 32 bits apart in the insn.
70 This makes it easy to copy the values from the tables in Appendix B. */
71typedef enum S390Opcode {
72 RIL_AFI = 0xc209,
73 RIL_AGFI = 0xc208,
3790b918 74 RIL_ALFI = 0xc20b,
48bb3750
RH		 75 RIL_ALGFI = 0xc20a,
76 RIL_BRASL = 0xc005,
77 RIL_BRCL = 0xc004,
78 RIL_CFI = 0xc20d,
79 RIL_CGFI = 0xc20c,
80 RIL_CLFI = 0xc20f,
81 RIL_CLGFI = 0xc20e,
a534bb15
RH		 82 RIL_CLRL = 0xc60f,
83 RIL_CLGRL = 0xc60a,
84 RIL_CRL = 0xc60d,
85 RIL_CGRL = 0xc608,
48bb3750
RH		 86 RIL_IIHF = 0xc008,
87 RIL_IILF = 0xc009,
88 RIL_LARL = 0xc000,
89 RIL_LGFI = 0xc001,
90 RIL_LGRL = 0xc408,
91 RIL_LLIHF = 0xc00e,
92 RIL_LLILF = 0xc00f,
93 RIL_LRL = 0xc40d,
94 RIL_MSFI = 0xc201,
95 RIL_MSGFI = 0xc200,
96 RIL_NIHF = 0xc00a,
97 RIL_NILF = 0xc00b,
98 RIL_OIHF = 0xc00c,
99 RIL_OILF = 0xc00d,
3790b918 100 RIL_SLFI = 0xc205,
0db921e6 101 RIL_SLGFI = 0xc204,
48bb3750
RH		 102 RIL_XIHF = 0xc006,
103 RIL_XILF = 0xc007,
104
105 RI_AGHI = 0xa70b,
106 RI_AHI = 0xa70a,
107 RI_BRC = 0xa704,
a534bb15
RH		 108 RI_CHI = 0xa70e,
109 RI_CGHI = 0xa70f,
48bb3750
RH		 110 RI_IIHH = 0xa500,
111 RI_IIHL = 0xa501,
112 RI_IILH = 0xa502,
113 RI_IILL = 0xa503,
114 RI_LGHI = 0xa709,
115 RI_LLIHH = 0xa50c,
116 RI_LLIHL = 0xa50d,
117 RI_LLILH = 0xa50e,
118 RI_LLILL = 0xa50f,
119 RI_MGHI = 0xa70d,
120 RI_MHI = 0xa70c,
121 RI_NIHH = 0xa504,
122 RI_NIHL = 0xa505,
123 RI_NILH = 0xa506,
124 RI_NILL = 0xa507,
125 RI_OIHH = 0xa508,
126 RI_OIHL = 0xa509,
127 RI_OILH = 0xa50a,
128 RI_OILL = 0xa50b,
1cd49868 129 RI_TMLL = 0xa701,
48bb3750 130
d84ca804
RH		 131 RIEb_CGRJ = 0xec64,
132 RIEb_CLGRJ = 0xec65,
133 RIEb_CLRJ = 0xec77,
134 RIEb_CRJ = 0xec76,
135
136 RIEc_CGIJ = 0xec7c,
137 RIEc_CIJ = 0xec7e,
138 RIEc_CLGIJ = 0xec7d,
139 RIEc_CLIJ = 0xec7f,
140
141 RIEf_RISBG = 0xec55,
142
143 RIEg_LOCGHI = 0xec46,
48bb3750
RH		 144
145 RRE_AGR = 0xb908,
3790b918
RH		 146 RRE_ALGR = 0xb90a,
147 RRE_ALCR = 0xb998,
148 RRE_ALCGR = 0xb988,
48bb3750
RH		 149 RRE_CGR = 0xb920,
150 RRE_CLGR = 0xb921,
151 RRE_DLGR = 0xb987,
152 RRE_DLR = 0xb997,
153 RRE_DSGFR = 0xb91d,
154 RRE_DSGR = 0xb90d,
ce411066 155 RRE_FLOGR = 0xb983,
48bb3750
RH		 156 RRE_LGBR = 0xb906,
157 RRE_LCGR = 0xb903,
158 RRE_LGFR = 0xb914,
159 RRE_LGHR = 0xb907,
160 RRE_LGR = 0xb904,
161 RRE_LLGCR = 0xb984,
162 RRE_LLGFR = 0xb916,
163 RRE_LLGHR = 0xb985,
164 RRE_LRVR = 0xb91f,
165 RRE_LRVGR = 0xb90f,
166 RRE_LTGR = 0xb902,
36017dc6 167 RRE_MLGR = 0xb986,
48bb3750
RH		 168 RRE_MSGR = 0xb90c,
169 RRE_MSR = 0xb252,
170 RRE_NGR = 0xb980,
171 RRE_OGR = 0xb981,
172 RRE_SGR = 0xb909,
3790b918
RH		 173 RRE_SLGR = 0xb90b,
174 RRE_SLBR = 0xb999,
175 RRE_SLBGR = 0xb989,
48bb3750
RH		 176 RRE_XGR = 0xb982,
177
668ce343 178 RRFa_MGRK = 0xb9ec,
92c89a07
RH		 179 RRFa_MSRKC = 0xb9fd,
180 RRFa_MSGRKC = 0xb9ed,
1dd06b1a
RH		 181 RRFa_NRK = 0xb9f4,
182 RRFa_NGRK = 0xb9e4,
183 RRFa_ORK = 0xb9f6,
184 RRFa_OGRK = 0xb9e6,
185 RRFa_SRK = 0xb9f9,
186 RRFa_SGRK = 0xb9e9,
187 RRFa_SLRK = 0xb9fb,
188 RRFa_SLGRK = 0xb9eb,
189 RRFa_XRK = 0xb9f7,
190 RRFa_XGRK = 0xb9e7,
191
192 RRFc_LOCR = 0xb9f2,
193 RRFc_LOCGR = 0xb9e2,
96a9f093 194
48bb3750 195 RR_AR = 0x1a,
3790b918 196 RR_ALR = 0x1e,
48bb3750
RH		 197 RR_BASR = 0x0d,
198 RR_BCR = 0x07,
199 RR_CLR = 0x15,
200 RR_CR = 0x19,
201 RR_DR = 0x1d,
202 RR_LCR = 0x13,
203 RR_LR = 0x18,
204 RR_LTR = 0x12,
205 RR_NR = 0x14,
206 RR_OR = 0x16,
207 RR_SR = 0x1b,
3790b918 208 RR_SLR = 0x1f,
48bb3750
RH		 209 RR_XR = 0x17,
210
211 RSY_RLL = 0xeb1d,
212 RSY_RLLG = 0xeb1c,
213 RSY_SLLG = 0xeb0d,
c2097136 214 RSY_SLLK = 0xebdf,
48bb3750 215 RSY_SRAG = 0xeb0a,
c2097136 216 RSY_SRAK = 0xebdc,
48bb3750 217 RSY_SRLG = 0xeb0c,
c2097136 218 RSY_SRLK = 0xebde,
48bb3750
RH		 219
220 RS_SLL = 0x89,
221 RS_SRA = 0x8a,
222 RS_SRL = 0x88,
223
224 RXY_AG = 0xe308,
225 RXY_AY = 0xe35a,
226 RXY_CG = 0xe320,
a534bb15
RH		 227 RXY_CLG = 0xe321,
228 RXY_CLY = 0xe355,
48bb3750 229 RXY_CY = 0xe359,
0db921e6 230 RXY_LAY = 0xe371,
48bb3750
RH		 231 RXY_LB = 0xe376,
232 RXY_LG = 0xe304,
233 RXY_LGB = 0xe377,
234 RXY_LGF = 0xe314,
235 RXY_LGH = 0xe315,
236 RXY_LHY = 0xe378,
237 RXY_LLGC = 0xe390,
238 RXY_LLGF = 0xe316,
239 RXY_LLGH = 0xe391,
240 RXY_LMG = 0xeb04,
241 RXY_LRV = 0xe31e,
242 RXY_LRVG = 0xe30f,
243 RXY_LRVH = 0xe31f,
244 RXY_LY = 0xe358,
bdcd5d19 245 RXY_NG = 0xe380,
4046d9ca 246 RXY_OG = 0xe381,
48bb3750
RH		 247 RXY_STCY = 0xe372,
248 RXY_STG = 0xe324,
249 RXY_STHY = 0xe370,
250 RXY_STMG = 0xeb24,
251 RXY_STRV = 0xe33e,
252 RXY_STRVG = 0xe32f,
253 RXY_STRVH = 0xe33f,
254 RXY_STY = 0xe350,
5bf67a92 255 RXY_XG = 0xe382,
48bb3750
RH		 256
257 RX_A = 0x5a,
258 RX_C = 0x59,
259 RX_L = 0x58,
0db921e6 260 RX_LA = 0x41,
48bb3750
RH		 261 RX_LH = 0x48,
262 RX_ST = 0x50,
263 RX_STC = 0x42,
264 RX_STH = 0x40,
ed3d51ec 265
79cada86
RH		 266 VRIa_VGBM = 0xe744,
267 VRIa_VREPI = 0xe745,
268 VRIb_VGM = 0xe746,
269 VRIc_VREP = 0xe74d,
270
ae77bbe5
RH		 271 VRRa_VLC = 0xe7de,
272 VRRa_VLP = 0xe7df,
b33ce725 273 VRRa_VLR = 0xe756,
a429ee29
RH		 274 VRRc_VA = 0xe7f3,
275 VRRc_VCEQ = 0xe7f8, /* we leave the m5 cs field 0 */
276 VRRc_VCH = 0xe7fb, /* " */
277 VRRc_VCHL = 0xe7f9, /* " */
22cb37b4
RH		 278 VRRc_VERLLV = 0xe773,
279 VRRc_VESLV = 0xe770,
280 VRRc_VESRAV = 0xe77a,
281 VRRc_VESRLV = 0xe778,
479b61cb 282 VRRc_VML = 0xe7a2,
220db7a6
RH		 283 VRRc_VMN = 0xe7fe,
284 VRRc_VMNL = 0xe7fc,
285 VRRc_VMX = 0xe7ff,
286 VRRc_VMXL = 0xe7fd,
a429ee29 287 VRRc_VN = 0xe768,
ae77bbe5 288 VRRc_VNC = 0xe769,
21eab5bf 289 VRRc_VNN = 0xe76e,
ae77bbe5 290 VRRc_VNO = 0xe76b,
21eab5bf 291 VRRc_VNX = 0xe76c,
a429ee29 292 VRRc_VO = 0xe76a,
ae77bbe5 293 VRRc_VOC = 0xe76f,
4223c9c1 294 VRRc_VPKS = 0xe797, /* we leave the m5 cs field 0 */
a429ee29 295 VRRc_VS = 0xe7f7,
4223c9c1
RH		 296 VRRa_VUPH = 0xe7d7,
297 VRRa_VUPL = 0xe7d6,
a429ee29 298 VRRc_VX = 0xe76d,
9bca986d 299 VRRe_VSEL = 0xe78d,
79cada86 300 VRRf_VLVGP = 0xe762,
b33ce725 301
22cb37b4
RH		 302 VRSa_VERLL = 0xe733,
303 VRSa_VESL = 0xe730,
304 VRSa_VESRA = 0xe73a,
305 VRSa_VESRL = 0xe738,
b33ce725
RH		 306 VRSb_VLVG = 0xe722,
307 VRSc_VLGV = 0xe721,
308
2dabf742
RH		 309 VRX_VL = 0xe706,
310 VRX_VLLEZ = 0xe704,
79cada86 311 VRX_VLREP = 0xe705,
2dabf742
RH		 312 VRX_VST = 0xe70e,
313 VRX_VSTEF = 0xe70b,
314 VRX_VSTEG = 0xe70a,
315
ed3d51ec 316 NOP = 0x0707,
48bb3750
RH		 317} S390Opcode;
318
8d8fdbae 319#ifdef CONFIG_DEBUG_TCG
48bb3750 320static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
34ef7676
RH		 321 "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7",
322 "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
323 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
324 "%v0", "%v1", "%v2", "%v3", "%v4", "%v5", "%v6", "%v7",
325 "%v8", "%v9", "%v10", "%v11", "%v12", "%v13", "%v14", "%v15",
326 "%v16", "%v17", "%v18", "%v19", "%v20", "%v21", "%v22", "%v23",
327 "%v24", "%v25", "%v26", "%v27", "%v28", "%v29", "%v30", "%v31",
48bb3750
RH		 328};
329#endif
330
331/* Since R6 is a potential argument register, choose it last of the
332 call-saved registers. Likewise prefer the call-clobbered registers
333 in reverse order to maximize the chance of avoiding the arguments. */
2827822e 334static const int tcg_target_reg_alloc_order[] = {
f24efee4 335 /* Call saved registers. */
48bb3750
RH		 336 TCG_REG_R13,
337 TCG_REG_R12,
338 TCG_REG_R11,
339 TCG_REG_R10,
340 TCG_REG_R9,
341 TCG_REG_R8,
342 TCG_REG_R7,
343 TCG_REG_R6,
f24efee4 344 /* Call clobbered registers. */
48bb3750
RH		 345 TCG_REG_R14,
346 TCG_REG_R0,
347 TCG_REG_R1,
f24efee4 348 /* Argument registers, in reverse order of allocation. */
48bb3750
RH		 349 TCG_REG_R5,
350 TCG_REG_R4,
351 TCG_REG_R3,
352 TCG_REG_R2,
34ef7676
RH		 353
354 /* V8-V15 are call saved, and omitted. */
355 TCG_REG_V0,
356 TCG_REG_V1,
357 TCG_REG_V2,
358 TCG_REG_V3,
359 TCG_REG_V4,
360 TCG_REG_V5,
361 TCG_REG_V6,
362 TCG_REG_V7,
363 TCG_REG_V16,
364 TCG_REG_V17,
365 TCG_REG_V18,
366 TCG_REG_V19,
367 TCG_REG_V20,
368 TCG_REG_V21,
369 TCG_REG_V22,
370 TCG_REG_V23,
371 TCG_REG_V24,
372 TCG_REG_V25,
373 TCG_REG_V26,
374 TCG_REG_V27,
375 TCG_REG_V28,
376 TCG_REG_V29,
377 TCG_REG_V30,
378 TCG_REG_V31,
2827822e
AG		 379};
380
381static const int tcg_target_call_iarg_regs[] = {
48bb3750
RH		 382 TCG_REG_R2,
383 TCG_REG_R3,
384 TCG_REG_R4,
385 TCG_REG_R5,
386 TCG_REG_R6,
2827822e
AG		 387};
388
389static const int tcg_target_call_oarg_regs[] = {
48bb3750 390 TCG_REG_R2,
48bb3750
RH		 391};
392
393#define S390_CC_EQ 8
394#define S390_CC_LT 4
395#define S390_CC_GT 2
396#define S390_CC_OV 1
397#define S390_CC_NE (S390_CC_LT | S390_CC_GT)
398#define S390_CC_LE (S390_CC_LT | S390_CC_EQ)
399#define S390_CC_GE (S390_CC_GT | S390_CC_EQ)
400#define S390_CC_NEVER 0
401#define S390_CC_ALWAYS 15
402
403/* Condition codes that result from a COMPARE and COMPARE LOGICAL. */
0aed257f 404static const uint8_t tcg_cond_to_s390_cond[] = {
48bb3750
RH		 405 [TCG_COND_EQ] = S390_CC_EQ,
406 [TCG_COND_NE] = S390_CC_NE,
407 [TCG_COND_LT] = S390_CC_LT,
408 [TCG_COND_LE] = S390_CC_LE,
409 [TCG_COND_GT] = S390_CC_GT,
410 [TCG_COND_GE] = S390_CC_GE,
411 [TCG_COND_LTU] = S390_CC_LT,
412 [TCG_COND_LEU] = S390_CC_LE,
413 [TCG_COND_GTU] = S390_CC_GT,
414 [TCG_COND_GEU] = S390_CC_GE,
415};
416
417/* Condition codes that result from a LOAD AND TEST. Here, we have no
418 unsigned instruction variation, however since the test is vs zero we
419 can re-map the outcomes appropriately. */
0aed257f 420static const uint8_t tcg_cond_to_ltr_cond[] = {
48bb3750
RH		 421 [TCG_COND_EQ] = S390_CC_EQ,
422 [TCG_COND_NE] = S390_CC_NE,
423 [TCG_COND_LT] = S390_CC_LT,
424 [TCG_COND_LE] = S390_CC_LE,
425 [TCG_COND_GT] = S390_CC_GT,
426 [TCG_COND_GE] = S390_CC_GE,
427 [TCG_COND_LTU] = S390_CC_NEVER,
428 [TCG_COND_LEU] = S390_CC_EQ,
429 [TCG_COND_GTU] = S390_CC_NE,
430 [TCG_COND_GEU] = S390_CC_ALWAYS,
431};
432
433#ifdef CONFIG_SOFTMMU
4b473e0c 434static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = {
f24efee4
RH		 435 [MO_UB] = helper_ret_ldub_mmu,
436 [MO_SB] = helper_ret_ldsb_mmu,
437 [MO_LEUW] = helper_le_lduw_mmu,
438 [MO_LESW] = helper_le_ldsw_mmu,
439 [MO_LEUL] = helper_le_ldul_mmu,
440 [MO_LESL] = helper_le_ldsl_mmu,
fc313c64 441 [MO_LEUQ] = helper_le_ldq_mmu,
f24efee4
RH		 442 [MO_BEUW] = helper_be_lduw_mmu,
443 [MO_BESW] = helper_be_ldsw_mmu,
444 [MO_BEUL] = helper_be_ldul_mmu,
445 [MO_BESL] = helper_be_ldsl_mmu,
fc313c64 446 [MO_BEUQ] = helper_be_ldq_mmu,
e141ab52
BS		 447};
448
4b473e0c 449static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
f24efee4
RH		 450 [MO_UB] = helper_ret_stb_mmu,
451 [MO_LEUW] = helper_le_stw_mmu,
452 [MO_LEUL] = helper_le_stl_mmu,
fc313c64 453 [MO_LEUQ] = helper_le_stq_mmu,
f24efee4
RH		 454 [MO_BEUW] = helper_be_stw_mmu,
455 [MO_BEUL] = helper_be_stl_mmu,
fc313c64 456 [MO_BEUQ] = helper_be_stq_mmu,
e141ab52 457};
e141ab52 458#endif
48bb3750 459
79dae4dd 460static const tcg_insn_unit *tb_ret_addr;
34ef7676 461uint64_t s390_facilities[3];
2827822e 462
2dabf742
RH		 463static inline bool is_general_reg(TCGReg r)
464{
465 return r <= TCG_REG_R15;
466}
467
468static inline bool is_vector_reg(TCGReg r)
469{
470 return r >= TCG_REG_V0 && r <= TCG_REG_V31;
471}
472
79dae4dd 473static bool patch_reloc(tcg_insn_unit *src_rw, int type,
2ba7fae2 474 intptr_t value, intptr_t addend)
2827822e 475{
79dae4dd 476 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
e692a349 477 intptr_t pcrel2;
28eef8aa 478 uint32_t old;
e692a349
RH		 479
480 value += addend;
79dae4dd 481 pcrel2 = (tcg_insn_unit *)value - src_rx;
48bb3750
RH		 482
483 switch (type) {
484 case R_390_PC16DBL:
55dfd8fe 485 if (pcrel2 == (int16_t)pcrel2) {
79dae4dd 486 tcg_patch16(src_rw, pcrel2);
55dfd8fe
RH		 487 return true;
488 }
48bb3750
RH		 489 break;
490 case R_390_PC32DBL:
55dfd8fe 491 if (pcrel2 == (int32_t)pcrel2) {
79dae4dd 492 tcg_patch32(src_rw, pcrel2);
55dfd8fe
RH		 493 return true;
494 }
48bb3750 495 break;
28eef8aa 496 case R_390_20:
55dfd8fe 497 if (value == sextract64(value, 0, 20)) {
79dae4dd 498 old = *(uint32_t *)src_rw & 0xf00000ff;
55dfd8fe 499 old |= ((value & 0xfff) << 16) | ((value & 0xff000) >> 4);
79dae4dd 500 tcg_patch32(src_rw, old);
55dfd8fe
RH		 501 return true;
502 }
28eef8aa 503 break;
48bb3750 504 default:
e692a349 505 g_assert_not_reached();
48bb3750 506 }
55dfd8fe 507 return false;
2827822e
AG		 508}
509
2827822e 510/* Test if a constant matches the constraint. */
a4fbbd77 511static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
2827822e 512{
48bb3750
RH		 513 if (ct & TCG_CT_CONST) {
514 return 1;
515 }
516
671c835b 517 if (type == TCG_TYPE_I32) {
48bb3750
RH		 518 val = (int32_t)val;
519 }
520
521 /* The following are mutually exclusive. */
a8f0269e
RH		 522 if (ct & TCG_CT_CONST_S16) {
523 return val == (int16_t)val;
524 } else if (ct & TCG_CT_CONST_S32) {
525 return val == (int32_t)val;
ba18b07d
RH		 526 } else if (ct & TCG_CT_CONST_S33) {
527 return val >= -0xffffffffll && val <= 0xffffffffll;
752b1be9
RH		 528 } else if (ct & TCG_CT_CONST_ZERO) {
529 return val == 0;
48bb3750
RH		 530 }
531
2827822e
AG		 532 return 0;
533}
534
48bb3750
RH		 535/* Emit instructions according to the given instruction format. */
536
537static void tcg_out_insn_RR(TCGContext *s, S390Opcode op, TCGReg r1, TCGReg r2)
538{
539 tcg_out16(s, (op << 8) | (r1 << 4) | r2);
540}
541
542static void tcg_out_insn_RRE(TCGContext *s, S390Opcode op,
543 TCGReg r1, TCGReg r2)
544{
545 tcg_out32(s, (op << 16) | (r1 << 4) | r2);
546}
547
1dd06b1a
RH		 548static void tcg_out_insn_RRFa(TCGContext *s, S390Opcode op,
549 TCGReg r1, TCGReg r2, TCGReg r3)
550{
551 tcg_out32(s, (op << 16) | (r3 << 12) | (r1 << 4) | r2);
552}
553
554static void tcg_out_insn_RRFc(TCGContext *s, S390Opcode op,
555 TCGReg r1, TCGReg r2, int m3)
96a9f093
RH		 556{
557 tcg_out32(s, (op << 16) | (m3 << 12) | (r1 << 4) | r2);
558}
559
48bb3750
RH		 560static void tcg_out_insn_RI(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
561{
562 tcg_out32(s, (op << 16) | (r1 << 20) | (i2 & 0xffff));
563}
564
d84ca804 565static void tcg_out_insn_RIEg(TCGContext *s, S390Opcode op, TCGReg r1,
7af525af
RH		 566 int i2, int m3)
567{
568 tcg_out16(s, (op & 0xff00) | (r1 << 4) | m3);
569 tcg_out32(s, (i2 << 16) | (op & 0xff));
570}
571
48bb3750
RH		 572static void tcg_out_insn_RIL(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
573{
574 tcg_out16(s, op | (r1 << 4));
575 tcg_out32(s, i2);
576}
577
578static void tcg_out_insn_RS(TCGContext *s, S390Opcode op, TCGReg r1,
579 TCGReg b2, TCGReg r3, int disp)
580{
581 tcg_out32(s, (op << 24) | (r1 << 20) | (r3 << 16) | (b2 << 12)
582 | (disp & 0xfff));
583}
584
585static void tcg_out_insn_RSY(TCGContext *s, S390Opcode op, TCGReg r1,
586 TCGReg b2, TCGReg r3, int disp)
587{
588 tcg_out16(s, (op & 0xff00) | (r1 << 4) | r3);
589 tcg_out32(s, (op & 0xff) | (b2 << 28)
590 | ((disp & 0xfff) << 16) | ((disp & 0xff000) >> 4));
591}
592
593#define tcg_out_insn_RX tcg_out_insn_RS
594#define tcg_out_insn_RXY tcg_out_insn_RSY
595
2dabf742
RH		 596static int RXB(TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4)
597{
598 /*
599 * Shift bit 4 of each regno to its corresponding bit of RXB.
600 * RXB itself begins at bit 8 of the instruction so 8 - 4 = 4
601 * is the left-shift of the 4th operand.
602 */
603 return ((v1 & 0x10) << (4 + 3))
604 | ((v2 & 0x10) << (4 + 2))
605 | ((v3 & 0x10) << (4 + 1))
606 | ((v4 & 0x10) << (4 + 0));
607}
608
79cada86
RH		 609static void tcg_out_insn_VRIa(TCGContext *s, S390Opcode op,
610 TCGReg v1, uint16_t i2, int m3)
611{
612 tcg_debug_assert(is_vector_reg(v1));
613 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4));
614 tcg_out16(s, i2);
615 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m3 << 12));
616}
617
618static void tcg_out_insn_VRIb(TCGContext *s, S390Opcode op,
619 TCGReg v1, uint8_t i2, uint8_t i3, int m4)
620{
621 tcg_debug_assert(is_vector_reg(v1));
622 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4));
623 tcg_out16(s, (i2 << 8) | (i3 & 0xff));
624 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m4 << 12));
625}
626
627static void tcg_out_insn_VRIc(TCGContext *s, S390Opcode op,
628 TCGReg v1, uint16_t i2, TCGReg v3, int m4)
629{
630 tcg_debug_assert(is_vector_reg(v1));
631 tcg_debug_assert(is_vector_reg(v3));
632 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v3 & 0xf));
633 tcg_out16(s, i2);
634 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, v3, 0) | (m4 << 12));
635}
636
b33ce725
RH		 637static void tcg_out_insn_VRRa(TCGContext *s, S390Opcode op,
638 TCGReg v1, TCGReg v2, int m3)
639{
640 tcg_debug_assert(is_vector_reg(v1));
641 tcg_debug_assert(is_vector_reg(v2));
642 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf));
643 tcg_out32(s, (op & 0x00ff) | RXB(v1, v2, 0, 0) | (m3 << 12));
644}
645
a429ee29
RH		 646static void tcg_out_insn_VRRc(TCGContext *s, S390Opcode op,
647 TCGReg v1, TCGReg v2, TCGReg v3, int m4)
648{
649 tcg_debug_assert(is_vector_reg(v1));
650 tcg_debug_assert(is_vector_reg(v2));
651 tcg_debug_assert(is_vector_reg(v3));
652 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf));
653 tcg_out16(s, v3 << 12);
654 tcg_out16(s, (op & 0x00ff) | RXB(v1, v2, v3, 0) | (m4 << 12));
655}
656
9bca986d
RH		 657static void tcg_out_insn_VRRe(TCGContext *s, S390Opcode op,
658 TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4)
659{
660 tcg_debug_assert(is_vector_reg(v1));
661 tcg_debug_assert(is_vector_reg(v2));
662 tcg_debug_assert(is_vector_reg(v3));
663 tcg_debug_assert(is_vector_reg(v4));
664 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf));
665 tcg_out16(s, v3 << 12);
666 tcg_out16(s, (op & 0x00ff) | RXB(v1, v2, v3, v4) | (v4 << 12));
667}
668
79cada86
RH		 669static void tcg_out_insn_VRRf(TCGContext *s, S390Opcode op,
670 TCGReg v1, TCGReg r2, TCGReg r3)
671{
672 tcg_debug_assert(is_vector_reg(v1));
673 tcg_debug_assert(is_general_reg(r2));
674 tcg_debug_assert(is_general_reg(r3));
675 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | r2);
676 tcg_out16(s, r3 << 12);
677 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0));
678}
679
22cb37b4
RH		 680static void tcg_out_insn_VRSa(TCGContext *s, S390Opcode op, TCGReg v1,
681 intptr_t d2, TCGReg b2, TCGReg v3, int m4)
682{
683 tcg_debug_assert(is_vector_reg(v1));
684 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
685 tcg_debug_assert(is_general_reg(b2));
686 tcg_debug_assert(is_vector_reg(v3));
687 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v3 & 0xf));
688 tcg_out16(s, b2 << 12 | d2);
689 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, v3, 0) | (m4 << 12));
690}
691
b33ce725
RH		 692static void tcg_out_insn_VRSb(TCGContext *s, S390Opcode op, TCGReg v1,
693 intptr_t d2, TCGReg b2, TCGReg r3, int m4)
694{
695 tcg_debug_assert(is_vector_reg(v1));
696 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
697 tcg_debug_assert(is_general_reg(b2));
698 tcg_debug_assert(is_general_reg(r3));
699 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | r3);
700 tcg_out16(s, b2 << 12 | d2);
701 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m4 << 12));
702}
703
704static void tcg_out_insn_VRSc(TCGContext *s, S390Opcode op, TCGReg r1,
705 intptr_t d2, TCGReg b2, TCGReg v3, int m4)
706{
707 tcg_debug_assert(is_general_reg(r1));
708 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
709 tcg_debug_assert(is_general_reg(b2));
710 tcg_debug_assert(is_vector_reg(v3));
711 tcg_out16(s, (op & 0xff00) | (r1 << 4) | (v3 & 0xf));
712 tcg_out16(s, b2 << 12 | d2);
713 tcg_out16(s, (op & 0x00ff) | RXB(0, 0, v3, 0) | (m4 << 12));
714}
715
2dabf742
RH		 716static void tcg_out_insn_VRX(TCGContext *s, S390Opcode op, TCGReg v1,
717 TCGReg b2, TCGReg x2, intptr_t d2, int m3)
718{
719 tcg_debug_assert(is_vector_reg(v1));
720 tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
721 tcg_debug_assert(is_general_reg(x2));
722 tcg_debug_assert(is_general_reg(b2));
723 tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | x2);
724 tcg_out16(s, (b2 << 12) | d2);
725 tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m3 << 12));
726}
727
48bb3750
RH		 728/* Emit an opcode with "type-checking" of the format. */
729#define tcg_out_insn(S, FMT, OP, ...) \
730 glue(tcg_out_insn_,FMT)(S, glue(glue(FMT,_),OP), ## __VA_ARGS__)
731
732
733/* emit 64-bit shifts */
734static void tcg_out_sh64(TCGContext* s, S390Opcode op, TCGReg dest,
735 TCGReg src, TCGReg sh_reg, int sh_imm)
736{
737 tcg_out_insn_RSY(s, op, dest, sh_reg, src, sh_imm);
738}
739
740/* emit 32-bit shifts */
741static void tcg_out_sh32(TCGContext* s, S390Opcode op, TCGReg dest,
742 TCGReg sh_reg, int sh_imm)
743{
744 tcg_out_insn_RS(s, op, dest, sh_reg, 0, sh_imm);
745}
746
78113e83 747static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
48bb3750 748{
b33ce725
RH		 749 if (src == dst) {
750 return true;
751 }
752 switch (type) {
753 case TCG_TYPE_I32:
754 if (likely(is_general_reg(dst) && is_general_reg(src))) {
48bb3750 755 tcg_out_insn(s, RR, LR, dst, src);
b33ce725 756 break;
48bb3750 757 }
b33ce725
RH		 758 /* fallthru */
759
760 case TCG_TYPE_I64:
761 if (likely(is_general_reg(dst))) {
762 if (likely(is_general_reg(src))) {
763 tcg_out_insn(s, RRE, LGR, dst, src);
764 } else {
765 tcg_out_insn(s, VRSc, VLGV, dst, 0, 0, src, 3);
766 }
767 break;
768 } else if (is_general_reg(src)) {
769 tcg_out_insn(s, VRSb, VLVG, dst, 0, 0, src, 3);
770 break;
771 }
772 /* fallthru */
773
774 case TCG_TYPE_V64:
775 case TCG_TYPE_V128:
776 tcg_out_insn(s, VRRa, VLR, dst, src, 0);
777 break;
778
779 default:
780 g_assert_not_reached();
48bb3750 781 }
78113e83 782 return true;
48bb3750
RH		 783}
784
28eef8aa
RH		 785static const S390Opcode lli_insns[4] = {
786 RI_LLILL, RI_LLILH, RI_LLIHL, RI_LLIHH
787};
48bb3750 788
28eef8aa
RH		 789static bool maybe_out_small_movi(TCGContext *s, TCGType type,
790 TCGReg ret, tcg_target_long sval)
791{
48bb3750
RH		 792 tcg_target_ulong uval = sval;
793 int i;
794
795 if (type == TCG_TYPE_I32) {
796 uval = (uint32_t)sval;
797 sval = (int32_t)sval;
798 }
799
800 /* Try all 32-bit insns that can load it in one go. */
801 if (sval >= -0x8000 && sval < 0x8000) {
802 tcg_out_insn(s, RI, LGHI, ret, sval);
28eef8aa 803 return true;
48bb3750
RH		 804 }
805
806 for (i = 0; i < 4; i++) {
dba6a9eb 807 tcg_target_long mask = 0xffffull << i * 16;
48bb3750 808 if ((uval & mask) == uval) {
dba6a9eb 809 tcg_out_insn_RI(s, lli_insns[i], ret, uval >> i * 16);
28eef8aa 810 return true;
48bb3750
RH		 811 }
812 }
813
28eef8aa
RH		 814 return false;
815}
816
817/* load a register with an immediate value */
ccbecb44
RH		 818static void tcg_out_movi(TCGContext *s, TCGType type,
819 TCGReg ret, tcg_target_long sval)
28eef8aa
RH		 820{
821 tcg_target_ulong uval;
1b74cf6e 822 ptrdiff_t pc_off;
28eef8aa
RH		 823
824 /* Try all 32-bit insns that can load it in one go. */
825 if (maybe_out_small_movi(s, type, ret, sval)) {
826 return;
827 }
828
829 uval = sval;
830 if (type == TCG_TYPE_I32) {
831 uval = (uint32_t)sval;
832 sval = (int32_t)sval;
833 }
834
48bb3750 835 /* Try all 48-bit insns that can load it in one go. */
3e25f7da
RH		 836 if (sval == (int32_t)sval) {
837 tcg_out_insn(s, RIL, LGFI, ret, sval);
838 return;
839 }
840 if (uval <= 0xffffffff) {
841 tcg_out_insn(s, RIL, LLILF, ret, uval);
842 return;
843 }
844 if ((uval & 0xffffffff) == 0) {
845 tcg_out_insn(s, RIL, LLIHF, ret, uval >> 32);
846 return;
48bb3750
RH		 847 }
848
1b74cf6e
RH		 849 /* Try for PC-relative address load. For odd addresses, add one. */
850 pc_off = tcg_pcrel_diff(s, (void *)sval) >> 1;
851 if (pc_off == (int32_t)pc_off) {
852 tcg_out_insn(s, RIL, LARL, ret, pc_off);
853 if (sval & 1) {
854 tcg_out_insn(s, RI, AGHI, ret, 1);
48bb3750 855 }
1b74cf6e 856 return;
48bb3750
RH		 857 }
858
ba2c7479 859 /* Otherwise, stuff it in the constant pool. */
9c3bfb79
RH		 860 tcg_out_insn(s, RIL, LGRL, ret, 0);
861 new_pool_label(s, sval, R_390_PC32DBL, s->code_ptr - 2, 2);
48bb3750
RH		 862}
863
48bb3750
RH		 864/* Emit a load/store type instruction. Inputs are:
865 DATA: The register to be loaded or stored.
866 BASE+OFS: The effective address.
867 OPC_RX: If the operation has an RX format opcode (e.g. STC), otherwise 0.
868 OPC_RXY: The RXY format opcode for the operation (e.g. STCY). */
869
870static void tcg_out_mem(TCGContext *s, S390Opcode opc_rx, S390Opcode opc_rxy,
871 TCGReg data, TCGReg base, TCGReg index,
872 tcg_target_long ofs)
873{
874 if (ofs < -0x80000 || ofs >= 0x80000) {
78c9f7c5
RH		 875 /* Combine the low 20 bits of the offset with the actual load insn;
876 the high 44 bits must come from an immediate load. */
877 tcg_target_long low = ((ofs & 0xfffff) ^ 0x80000) - 0x80000;
878 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - low);
879 ofs = low;
48bb3750
RH		 880
881 /* If we were already given an index register, add it in. */
882 if (index != TCG_REG_NONE) {
883 tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
884 }
885 index = TCG_TMP0;
886 }
887
888 if (opc_rx && ofs >= 0 && ofs < 0x1000) {
889 tcg_out_insn_RX(s, opc_rx, data, base, index, ofs);
890 } else {
891 tcg_out_insn_RXY(s, opc_rxy, data, base, index, ofs);
892 }
2827822e
AG		 893}
894
2dabf742
RH		 895static void tcg_out_vrx_mem(TCGContext *s, S390Opcode opc_vrx,
896 TCGReg data, TCGReg base, TCGReg index,
897 tcg_target_long ofs, int m3)
898{
899 if (ofs < 0 || ofs >= 0x1000) {
900 if (ofs >= -0x80000 && ofs < 0x80000) {
901 tcg_out_insn(s, RXY, LAY, TCG_TMP0, base, index, ofs);
902 base = TCG_TMP0;
903 index = TCG_REG_NONE;
904 ofs = 0;
905 } else {
906 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs);
907 if (index != TCG_REG_NONE) {
908 tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
909 }
910 index = TCG_TMP0;
911 ofs = 0;
912 }
913 }
914 tcg_out_insn_VRX(s, opc_vrx, data, base, index, ofs, m3);
915}
48bb3750 916
2827822e 917/* load data without address translation or endianness conversion */
2dabf742
RH		 918static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data,
919 TCGReg base, intptr_t ofs)
2827822e 920{
2dabf742
RH		 921 switch (type) {
922 case TCG_TYPE_I32:
923 if (likely(is_general_reg(data))) {
924 tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs);
925 break;
926 }
927 tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_32);
928 break;
929
930 case TCG_TYPE_I64:
931 if (likely(is_general_reg(data))) {
932 tcg_out_mem(s, 0, RXY_LG, data, base, TCG_REG_NONE, ofs);
933 break;
934 }
935 /* fallthru */
936
937 case TCG_TYPE_V64:
938 tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_64);
939 break;
940
941 case TCG_TYPE_V128:
942 /* Hint quadword aligned. */
943 tcg_out_vrx_mem(s, VRX_VL, data, base, TCG_REG_NONE, ofs, 4);
944 break;
945
946 default:
947 g_assert_not_reached();
48bb3750 948 }
2827822e
AG		 949}
950
2dabf742
RH		 951static void tcg_out_st(TCGContext *s, TCGType type, TCGReg data,
952 TCGReg base, intptr_t ofs)
2827822e 953{
2dabf742
RH		 954 switch (type) {
955 case TCG_TYPE_I32:
956 if (likely(is_general_reg(data))) {
957 tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs);
958 } else {
959 tcg_out_vrx_mem(s, VRX_VSTEF, data, base, TCG_REG_NONE, ofs, 1);
960 }
961 break;
962
963 case TCG_TYPE_I64:
964 if (likely(is_general_reg(data))) {
965 tcg_out_mem(s, 0, RXY_STG, data, base, TCG_REG_NONE, ofs);
966 break;
967 }
968 /* fallthru */
969
970 case TCG_TYPE_V64:
971 tcg_out_vrx_mem(s, VRX_VSTEG, data, base, TCG_REG_NONE, ofs, 0);
972 break;
973
974 case TCG_TYPE_V128:
975 /* Hint quadword aligned. */
976 tcg_out_vrx_mem(s, VRX_VST, data, base, TCG_REG_NONE, ofs, 4);
977 break;
978
979 default:
980 g_assert_not_reached();
48bb3750
RH		 981 }
982}
983
59d7c14e
RH		 984static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
985 TCGReg base, intptr_t ofs)
986{
987 return false;
988}
989
f0bffc27
RH		 990static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src,
991 int msb, int lsb, int ofs, int z)
992{
993 /* Format RIE-f */
d84ca804 994 tcg_out16(s, (RIEf_RISBG & 0xff00) | (dest << 4) | src);
f0bffc27 995 tcg_out16(s, (msb << 8) | (z << 7) | lsb);
d84ca804 996 tcg_out16(s, (ofs << 8) | (RIEf_RISBG & 0xff));
f0bffc27
RH		 997}
998
48bb3750
RH		 999static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1000{
3e25f7da 1001 tcg_out_insn(s, RRE, LGBR, dest, src);
48bb3750
RH		 1002}
1003
1004static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1005{
3e25f7da 1006 tcg_out_insn(s, RRE, LLGCR, dest, src);
48bb3750
RH		 1007}
1008
1009static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1010{
3e25f7da 1011 tcg_out_insn(s, RRE, LGHR, dest, src);
48bb3750
RH		 1012}
1013
1014static void tgen_ext16u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1015{
3e25f7da 1016 tcg_out_insn(s, RRE, LLGHR, dest, src);
48bb3750
RH		 1017}
1018
1019static inline void tgen_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
1020{
1021 tcg_out_insn(s, RRE, LGFR, dest, src);
1022}
1023
1024static inline void tgen_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
1025{
1026 tcg_out_insn(s, RRE, LLGFR, dest, src);
1027}
1028
f0bffc27
RH		 1029/* Accept bit patterns like these:
1030 0....01....1
1031 1....10....0
1032 1..10..01..1
1033 0..01..10..0
1034 Copied from gcc sources. */
1035static inline bool risbg_mask(uint64_t c)
1036{
1037 uint64_t lsb;
1038 /* We don't change the number of transitions by inverting,
1039 so make sure we start with the LSB zero. */
1040 if (c & 1) {
1041 c = ~c;
1042 }
1043 /* Reject all zeros or all ones. */
1044 if (c == 0) {
1045 return false;
1046 }
1047 /* Find the first transition. */
1048 lsb = c & -c;
1049 /* Invert to look for a second transition. */
1050 c = ~c;
1051 /* Erase the first transition. */
1052 c &= -lsb;
1053 /* Find the second transition, if any. */
1054 lsb = c & -c;
1055 /* Match if all the bits are 1's, or if c is zero. */
1056 return c == -lsb;
1057}
1058
547ec121
RH		 1059static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
1060{
1061 int msb, lsb;
1062 if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
1063 /* Achieve wraparound by swapping msb and lsb. */
1064 msb = 64 - ctz64(~val);
1065 lsb = clz64(~val) - 1;
1066 } else {
1067 msb = clz64(val);
1068 lsb = 63 - ctz64(val);
1069 }
1070 tcg_out_risbg(s, out, in, msb, lsb, 0, 1);
1071}
1072
07ff7983 1073static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
48bb3750
RH		 1074{
1075 static const S390Opcode ni_insns[4] = {
1076 RI_NILL, RI_NILH, RI_NIHL, RI_NIHH
1077 };
1078 static const S390Opcode nif_insns[2] = {
1079 RIL_NILF, RIL_NIHF
1080 };
07ff7983 1081 uint64_t valid = (type == TCG_TYPE_I32 ? 0xffffffffull : -1ull);
48bb3750
RH		 1082 int i;
1083
48bb3750 1084 /* Look for the zero-extensions. */
07ff7983 1085 if ((val & valid) == 0xffffffff) {
48bb3750
RH		 1086 tgen_ext32u(s, dest, dest);
1087 return;
1088 }
3e25f7da
RH		 1089 if ((val & valid) == 0xff) {
1090 tgen_ext8u(s, TCG_TYPE_I64, dest, dest);
1091 return;
1092 }
1093 if ((val & valid) == 0xffff) {
1094 tgen_ext16u(s, TCG_TYPE_I64, dest, dest);
1095 return;
07ff7983 1096 }
48bb3750 1097
07ff7983
RH		 1098 /* Try all 32-bit insns that can perform it in one go. */
1099 for (i = 0; i < 4; i++) {
dba6a9eb 1100 tcg_target_ulong mask = ~(0xffffull << i * 16);
07ff7983 1101 if (((val | ~valid) & mask) == mask) {
dba6a9eb 1102 tcg_out_insn_RI(s, ni_insns[i], dest, val >> i * 16);
07ff7983 1103 return;
48bb3750 1104 }
07ff7983 1105 }
48bb3750 1106
07ff7983 1107 /* Try all 48-bit insns that can perform it in one go. */
3e25f7da
RH		 1108 for (i = 0; i < 2; i++) {
1109 tcg_target_ulong mask = ~(0xffffffffull << i * 32);
1110 if (((val | ~valid) & mask) == mask) {
1111 tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i * 32);
1112 return;
48bb3750 1113 }
07ff7983 1114 }
9c3bfb79 1115 if (risbg_mask(val)) {
547ec121 1116 tgen_andi_risbg(s, dest, dest, val);
f0bffc27
RH		 1117 return;
1118 }
48bb3750 1119
ccbecb44 1120 tcg_out_movi(s, type, TCG_TMP0, val);
07ff7983
RH		 1121 if (type == TCG_TYPE_I32) {
1122 tcg_out_insn(s, RR, NR, dest, TCG_TMP0);
48bb3750 1123 } else {
07ff7983 1124 tcg_out_insn(s, RRE, NGR, dest, TCG_TMP0);
48bb3750
RH		 1125 }
1126}
1127
4046d9ca 1128static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
48bb3750
RH		 1129{
1130 static const S390Opcode oi_insns[4] = {
1131 RI_OILL, RI_OILH, RI_OIHL, RI_OIHH
1132 };
4046d9ca 1133 static const S390Opcode oif_insns[2] = {
48bb3750
RH		 1134 RIL_OILF, RIL_OIHF
1135 };
1136
1137 int i;
1138
1139 /* Look for no-op. */
4046d9ca 1140 if (unlikely(val == 0)) {
48bb3750
RH		 1141 return;
1142 }
1143
4046d9ca
RH		 1144 /* Try all 32-bit insns that can perform it in one go. */
1145 for (i = 0; i < 4; i++) {
dba6a9eb 1146 tcg_target_ulong mask = (0xffffull << i * 16);
4046d9ca 1147 if ((val & mask) != 0 && (val & ~mask) == 0) {
dba6a9eb 1148 tcg_out_insn_RI(s, oi_insns[i], dest, val >> i * 16);
4046d9ca 1149 return;
48bb3750 1150 }
4046d9ca 1151 }
48bb3750 1152
4046d9ca 1153 /* Try all 48-bit insns that can perform it in one go. */
3e25f7da
RH		 1154 for (i = 0; i < 2; i++) {
1155 tcg_target_ulong mask = (0xffffffffull << i * 32);
1156 if ((val & mask) != 0 && (val & ~mask) == 0) {
1157 tcg_out_insn_RIL(s, oif_insns[i], dest, val >> i * 32);
1158 return;
48bb3750 1159 }
4046d9ca 1160 }
48bb3750 1161
4046d9ca
RH		 1162 if (maybe_out_small_movi(s, type, TCG_TMP0, val)) {
1163 if (type == TCG_TYPE_I32) {
1164 tcg_out_insn(s, RR, OR, dest, TCG_TMP0);
1165 } else {
1166 tcg_out_insn(s, RRE, OGR, dest, TCG_TMP0);
1167 }
4046d9ca 1168 } else {
48bb3750
RH		 1169 /* Perform the OR via sequential modifications to the high and
1170 low parts. Do this via recursion to handle 16-bit vs 32-bit
1171 masks in each half. */
4046d9ca
RH		 1172 tgen_ori(s, type, dest, val & 0x00000000ffffffffull);
1173 tgen_ori(s, type, dest, val & 0xffffffff00000000ull);
48bb3750
RH		 1174 }
1175}
1176
5bf67a92 1177static void tgen_xori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
48bb3750 1178{
5bf67a92 1179 /* Try all 48-bit insns that can perform it in one go. */
3e25f7da
RH		 1180 if ((val & 0xffffffff00000000ull) == 0) {
1181 tcg_out_insn(s, RIL, XILF, dest, val);
1182 return;
1183 }
1184 if ((val & 0x00000000ffffffffull) == 0) {
1185 tcg_out_insn(s, RIL, XIHF, dest, val >> 32);
1186 return;
48bb3750 1187 }
5bf67a92 1188
5bf67a92
RH		 1189 if (maybe_out_small_movi(s, type, TCG_TMP0, val)) {
1190 if (type == TCG_TYPE_I32) {
1191 tcg_out_insn(s, RR, XR, dest, TCG_TMP0);
1192 } else {
1193 tcg_out_insn(s, RRE, XGR, dest, TCG_TMP0);
1194 }
5bf67a92
RH		 1195 } else {
1196 /* Perform the xor by parts. */
5bf67a92
RH		 1197 if (val & 0xffffffff) {
1198 tcg_out_insn(s, RIL, XILF, dest, val);
1199 }
1200 if (val > 0xffffffff) {
1201 tcg_out_insn(s, RIL, XIHF, dest, val >> 32);
1202 }
48bb3750
RH		 1203 }
1204}
1205
1206static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
65839b56 1207 TCGArg c2, bool c2const, bool need_carry)
48bb3750 1208{
bcc66562 1209 bool is_unsigned = is_unsigned_cond(c);
a534bb15
RH		 1210 S390Opcode op;
1211
48bb3750
RH		 1212 if (c2const) {
1213 if (c2 == 0) {
65839b56
RH		 1214 if (!(is_unsigned && need_carry)) {
1215 if (type == TCG_TYPE_I32) {
1216 tcg_out_insn(s, RR, LTR, r1, r1);
1217 } else {
1218 tcg_out_insn(s, RRE, LTGR, r1, r1);
1219 }
1220 return tcg_cond_to_ltr_cond[c];
1221 }
65839b56 1222 }
a534bb15
RH		 1223
1224 if (!is_unsigned && c2 == (int16_t)c2) {
1225 op = (type == TCG_TYPE_I32 ? RI_CHI : RI_CGHI);
1226 tcg_out_insn_RI(s, op, r1, c2);
1227 goto exit;
1228 }
1229
3e25f7da
RH		 1230 if (type == TCG_TYPE_I32) {
1231 op = (is_unsigned ? RIL_CLFI : RIL_CFI);
1232 tcg_out_insn_RIL(s, op, r1, c2);
1233 goto exit;
1234 }
1235 if (c2 == (is_unsigned ? (TCGArg)(uint32_t)c2 : (TCGArg)(int32_t)c2)) {
1236 op = (is_unsigned ? RIL_CLGFI : RIL_CGFI);
1237 tcg_out_insn_RIL(s, op, r1, c2);
1238 goto exit;
48bb3750 1239 }
a534bb15
RH		 1240
1241 /* Use the constant pool, but not for small constants. */
1242 if (maybe_out_small_movi(s, type, TCG_TMP0, c2)) {
1243 c2 = TCG_TMP0;
1244 /* fall through to reg-reg */
48bb3750 1245 } else {
3e25f7da
RH		 1246 op = (is_unsigned ? RIL_CLGRL : RIL_CGRL);
1247 tcg_out_insn_RIL(s, op, r1, 0);
1248 new_pool_label(s, c2, R_390_PC32DBL, s->code_ptr - 2, 2);
a534bb15 1249 goto exit;
48bb3750
RH		 1250 }
1251 }
a534bb15
RH		 1252
1253 if (type == TCG_TYPE_I32) {
1254 op = (is_unsigned ? RR_CLR : RR_CR);
1255 tcg_out_insn_RR(s, op, r1, c2);
1256 } else {
1257 op = (is_unsigned ? RRE_CLGR : RRE_CGR);
1258 tcg_out_insn_RRE(s, op, r1, c2);
1259 }
1260
1261 exit:
48bb3750
RH		 1262 return tcg_cond_to_s390_cond[c];
1263}
1264
7b7066b1 1265static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
96a9f093 1266 TCGReg dest, TCGReg c1, TCGArg c2, int c2const)
48bb3750 1267{
7b7066b1
RH		 1268 int cc;
1269
7af525af 1270 /* With LOC2, we can always emit the minimum 3 insns. */
748b7f3e 1271 if (HAVE_FACILITY(LOAD_ON_COND2)) {
7af525af
RH		 1272 /* Emit: d = 0, d = (cc ? 1 : d). */
1273 cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
1274 tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
d84ca804 1275 tcg_out_insn(s, RIEg, LOCGHI, dest, 1, cc);
7af525af
RH		 1276 return;
1277 }
1278
4609190b 1279 restart:
7b7066b1 1280 switch (cond) {
4609190b
RH		 1281 case TCG_COND_NE:
1282 /* X != 0 is X > 0. */
1283 if (c2const && c2 == 0) {
1284 cond = TCG_COND_GTU;
1285 } else {
1286 break;
1287 }
1288 /* fallthru */
1289
7b7066b1
RH		 1290 case TCG_COND_GTU:
1291 case TCG_COND_GT:
7b7066b1
RH		 1292 /* The result of a compare has CC=2 for GT and CC=3 unused.
1293 ADD LOGICAL WITH CARRY considers (CC & 2) the carry bit. */
65839b56 1294 tgen_cmp(s, type, cond, c1, c2, c2const, true);
7b7066b1
RH		 1295 tcg_out_movi(s, type, dest, 0);
1296 tcg_out_insn(s, RRE, ALCGR, dest, dest);
1297 return;
1298
4609190b
RH		 1299 case TCG_COND_EQ:
1300 /* X == 0 is X <= 0. */
1301 if (c2const && c2 == 0) {
1302 cond = TCG_COND_LEU;
7b7066b1 1303 } else {
4609190b 1304 break;
7b7066b1 1305 }
4609190b 1306 /* fallthru */
7b7066b1
RH		 1307
1308 case TCG_COND_LEU:
4609190b
RH		 1309 case TCG_COND_LE:
1310 /* As above, but we're looking for borrow, or !carry.
1311 The second insn computes d - d - borrow, or -1 for true
1312 and 0 for false. So we must mask to 1 bit afterward. */
1313 tgen_cmp(s, type, cond, c1, c2, c2const, true);
1314 tcg_out_insn(s, RRE, SLBGR, dest, dest);
1315 tgen_andi(s, type, dest, 1);
1316 return;
1317
1318 case TCG_COND_GEU:
7b7066b1
RH		 1319 case TCG_COND_LTU:
1320 case TCG_COND_LT:
4609190b
RH		 1321 case TCG_COND_GE:
1322 /* Swap operands so that we can use LEU/GTU/GT/LE. */
c68d5b7a 1323 if (!c2const) {
7b7066b1
RH		 1324 TCGReg t = c1;
1325 c1 = c2;
1326 c2 = t;
c68d5b7a
RH		 1327 cond = tcg_swap_cond(cond);
1328 goto restart;
7b7066b1 1329 }
c68d5b7a 1330 break;
48bb3750 1331
7b7066b1 1332 default:
4609190b 1333 g_assert_not_reached();
7b7066b1
RH		 1334 }
1335
65839b56 1336 cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
c68d5b7a
RH		 1337 /* Emit: d = 0, t = 1, d = (cc ? t : d). */
1338 tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
1339 tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 1);
1dd06b1a 1340 tcg_out_insn(s, RRFc, LOCGR, dest, TCG_TMP0, cc);
48bb3750
RH		 1341}
1342
96a9f093 1343static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, TCGReg dest,
7af525af
RH		 1344 TCGReg c1, TCGArg c2, int c2const,
1345 TCGArg v3, int v3const)
96a9f093 1346{
c68d5b7a
RH		 1347 int cc = tgen_cmp(s, type, c, c1, c2, c2const, false);
1348 if (v3const) {
d84ca804 1349 tcg_out_insn(s, RIEg, LOCGHI, dest, v3, cc);
96a9f093 1350 } else {
1dd06b1a 1351 tcg_out_insn(s, RRFc, LOCGR, dest, v3, cc);
96a9f093
RH		 1352 }
1353}
1354
ce411066
RH		 1355static void tgen_clz(TCGContext *s, TCGReg dest, TCGReg a1,
1356 TCGArg a2, int a2const)
1357{
1358 /* Since this sets both R and R+1, we have no choice but to store the
1359 result into R0, allowing R1 == TCG_TMP0 to be clobbered as well. */
1360 QEMU_BUILD_BUG_ON(TCG_TMP0 != TCG_REG_R1);
1361 tcg_out_insn(s, RRE, FLOGR, TCG_REG_R0, a1);
1362
1363 if (a2const && a2 == 64) {
1364 tcg_out_mov(s, TCG_TYPE_I64, dest, TCG_REG_R0);
1365 } else {
1366 if (a2const) {
1367 tcg_out_movi(s, TCG_TYPE_I64, dest, a2);
1368 } else {
1369 tcg_out_mov(s, TCG_TYPE_I64, dest, a2);
1370 }
c68d5b7a 1371 /* Emit: if (one bit found) dest = r0. */
1dd06b1a 1372 tcg_out_insn(s, RRFc, LOCGR, dest, TCG_REG_R0, 2);
ce411066
RH		 1373 }
1374}
1375
d5690ea4 1376static void tgen_deposit(TCGContext *s, TCGReg dest, TCGReg src,
752b1be9 1377 int ofs, int len, int z)
d5690ea4
RH		 1378{
1379 int lsb = (63 - ofs);
1380 int msb = lsb - (len - 1);
752b1be9 1381 tcg_out_risbg(s, dest, src, msb, lsb, ofs, z);
d5690ea4
RH		 1382}
1383
b0bf5fe8
RH		 1384static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src,
1385 int ofs, int len)
1386{
1387 tcg_out_risbg(s, dest, src, 64 - len, 63, 64 - ofs, 1);
1388}
1389
ffd0e507 1390static void tgen_gotoi(TCGContext *s, int cc, const tcg_insn_unit *dest)
48bb3750 1391{
79dae4dd 1392 ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
8c081b18 1393 if (off == (int16_t)off) {
48bb3750
RH		 1394 tcg_out_insn(s, RI, BRC, cc, off);
1395 } else if (off == (int32_t)off) {
1396 tcg_out_insn(s, RIL, BRCL, cc, off);
1397 } else {
8c081b18 1398 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
48bb3750
RH		 1399 tcg_out_insn(s, RR, BCR, cc, TCG_TMP0);
1400 }
1401}
1402
bec16311 1403static void tgen_branch(TCGContext *s, int cc, TCGLabel *l)
48bb3750 1404{
48bb3750 1405 if (l->has_value) {
8c081b18 1406 tgen_gotoi(s, cc, l->u.value_ptr);
48bb3750
RH		 1407 } else {
1408 tcg_out16(s, RI_BRC | (cc << 4));
e692a349 1409 tcg_out_reloc(s, s->code_ptr, R_390_PC16DBL, l, 2);
8c081b18 1410 s->code_ptr += 1;
48bb3750
RH		 1411 }
1412}
1413
1414static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc,
bec16311 1415 TCGReg r1, TCGReg r2, TCGLabel *l)
48bb3750 1416{
79dae4dd 1417 tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
d84ca804 1418 /* Format RIE-b */
48bb3750 1419 tcg_out16(s, (opc & 0xff00) | (r1 << 4) | r2);
79dae4dd 1420 tcg_out16(s, 0);
48bb3750
RH		 1421 tcg_out16(s, cc << 12 | (opc & 0xff));
1422}
1423
1424static void tgen_compare_imm_branch(TCGContext *s, S390Opcode opc, int cc,
bec16311 1425 TCGReg r1, int i2, TCGLabel *l)
48bb3750 1426{
79dae4dd 1427 tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
d84ca804 1428 /* Format RIE-c */
48bb3750 1429 tcg_out16(s, (opc & 0xff00) | (r1 << 4) | cc);
79dae4dd 1430 tcg_out16(s, 0);
48bb3750
RH		 1431 tcg_out16(s, (i2 << 8) | (opc & 0xff));
1432}
1433
1434static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
bec16311 1435 TCGReg r1, TCGArg c2, int c2const, TCGLabel *l)
48bb3750
RH		 1436{
1437 int cc;
9c3bfb79
RH		 1438 bool is_unsigned = is_unsigned_cond(c);
1439 bool in_range;
1440 S390Opcode opc;
48bb3750 1441
9c3bfb79 1442 cc = tcg_cond_to_s390_cond[c];
48bb3750 1443
9c3bfb79
RH		 1444 if (!c2const) {
1445 opc = (type == TCG_TYPE_I32
d84ca804
RH		 1446 ? (is_unsigned ? RIEb_CLRJ : RIEb_CRJ)
1447 : (is_unsigned ? RIEb_CLGRJ : RIEb_CGRJ));
9c3bfb79
RH		 1448 tgen_compare_branch(s, opc, cc, r1, c2, l);
1449 return;
1450 }
48bb3750 1451
9c3bfb79
RH		 1452 /*
1453 * COMPARE IMMEDIATE AND BRANCH RELATIVE has an 8-bit immediate field.
1454 * If the immediate we've been given does not fit that range, we'll
1455 * fall back to separate compare and branch instructions using the
1456 * larger comparison range afforded by COMPARE IMMEDIATE.
1457 */
1458 if (type == TCG_TYPE_I32) {
1459 if (is_unsigned) {
d84ca804 1460 opc = RIEc_CLIJ;
9c3bfb79 1461 in_range = (uint32_t)c2 == (uint8_t)c2;
48bb3750 1462 } else {
d84ca804 1463 opc = RIEc_CIJ;
9c3bfb79 1464 in_range = (int32_t)c2 == (int8_t)c2;
48bb3750 1465 }
9c3bfb79
RH		 1466 } else {
1467 if (is_unsigned) {
d84ca804 1468 opc = RIEc_CLGIJ;
9c3bfb79
RH		 1469 in_range = (uint64_t)c2 == (uint8_t)c2;
1470 } else {
d84ca804 1471 opc = RIEc_CGIJ;
9c3bfb79 1472 in_range = (int64_t)c2 == (int8_t)c2;
48bb3750
RH		 1473 }
1474 }
9c3bfb79
RH		 1475 if (in_range) {
1476 tgen_compare_imm_branch(s, opc, cc, r1, c2, l);
1477 return;
1478 }
48bb3750 1479
65839b56 1480 cc = tgen_cmp(s, type, c, r1, c2, c2const, false);
bec16311 1481 tgen_branch(s, cc, l);
48bb3750
RH		 1482}
1483
cee44b03 1484static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *dest)
48bb3750 1485{
79dae4dd 1486 ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
48bb3750
RH		 1487 if (off == (int32_t)off) {
1488 tcg_out_insn(s, RIL, BRASL, TCG_REG_R14, off);
1489 } else {
8c081b18 1490 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
48bb3750
RH		 1491 tcg_out_insn(s, RR, BASR, TCG_REG_R14, TCG_TMP0);
1492 }
1493}
1494
cee44b03
RH		 1495static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest,
1496 const TCGHelperInfo *info)
1497{
1498 tcg_out_call_int(s, dest);
1499}
1500
14776ab5 1501static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
48bb3750
RH		 1502 TCGReg base, TCGReg index, int disp)
1503{
3c8691f5 1504 switch (opc & (MO_SSIZE | MO_BSWAP)) {
a5a04f28 1505 case MO_UB:
48bb3750
RH		 1506 tcg_out_insn(s, RXY, LLGC, data, base, index, disp);
1507 break;
a5a04f28 1508 case MO_SB:
48bb3750
RH		 1509 tcg_out_insn(s, RXY, LGB, data, base, index, disp);
1510 break;
b8dd88b8
RH		 1511
1512 case MO_UW | MO_BSWAP:
1513 /* swapped unsigned halfword load with upper bits zeroed */
1514 tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
1515 tgen_ext16u(s, TCG_TYPE_I64, data, data);
1516 break;
a5a04f28 1517 case MO_UW:
b8dd88b8
RH		 1518 tcg_out_insn(s, RXY, LLGH, data, base, index, disp);
1519 break;
1520
1521 case MO_SW | MO_BSWAP:
1522 /* swapped sign-extended halfword load */
1523 tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
1524 tgen_ext16s(s, TCG_TYPE_I64, data, data);
48bb3750 1525 break;
a5a04f28 1526 case MO_SW:
b8dd88b8
RH		 1527 tcg_out_insn(s, RXY, LGH, data, base, index, disp);
1528 break;
1529
1530 case MO_UL | MO_BSWAP:
1531 /* swapped unsigned int load with upper bits zeroed */
1532 tcg_out_insn(s, RXY, LRV, data, base, index, disp);
1533 tgen_ext32u(s, data, data);
48bb3750 1534 break;
a5a04f28 1535 case MO_UL:
b8dd88b8
RH		 1536 tcg_out_insn(s, RXY, LLGF, data, base, index, disp);
1537 break;
1538
1539 case MO_SL | MO_BSWAP:
1540 /* swapped sign-extended int load */
1541 tcg_out_insn(s, RXY, LRV, data, base, index, disp);
1542 tgen_ext32s(s, data, data);
48bb3750 1543 break;
a5a04f28 1544 case MO_SL:
b8dd88b8
RH		 1545 tcg_out_insn(s, RXY, LGF, data, base, index, disp);
1546 break;
1547
fc313c64 1548 case MO_UQ | MO_BSWAP:
b8dd88b8 1549 tcg_out_insn(s, RXY, LRVG, data, base, index, disp);
48bb3750 1550 break;
fc313c64 1551 case MO_UQ:
b8dd88b8 1552 tcg_out_insn(s, RXY, LG, data, base, index, disp);
48bb3750 1553 break;
b8dd88b8 1554
48bb3750
RH		 1555 default:
1556 tcg_abort();
1557 }
1558}
1559
14776ab5 1560static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data,
48bb3750
RH		 1561 TCGReg base, TCGReg index, int disp)
1562{
3c8691f5 1563 switch (opc & (MO_SIZE | MO_BSWAP)) {
a5a04f28 1564 case MO_UB:
48bb3750
RH		 1565 if (disp >= 0 && disp < 0x1000) {
1566 tcg_out_insn(s, RX, STC, data, base, index, disp);
1567 } else {
1568 tcg_out_insn(s, RXY, STCY, data, base, index, disp);
1569 }
1570 break;
b8dd88b8
RH		 1571
1572 case MO_UW | MO_BSWAP:
1573 tcg_out_insn(s, RXY, STRVH, data, base, index, disp);
1574 break;
a5a04f28 1575 case MO_UW:
b8dd88b8 1576 if (disp >= 0 && disp < 0x1000) {
48bb3750
RH		 1577 tcg_out_insn(s, RX, STH, data, base, index, disp);
1578 } else {
1579 tcg_out_insn(s, RXY, STHY, data, base, index, disp);
1580 }
1581 break;
b8dd88b8
RH		 1582
1583 case MO_UL | MO_BSWAP:
1584 tcg_out_insn(s, RXY, STRV, data, base, index, disp);
1585 break;
a5a04f28 1586 case MO_UL:
b8dd88b8 1587 if (disp >= 0 && disp < 0x1000) {
48bb3750
RH		 1588 tcg_out_insn(s, RX, ST, data, base, index, disp);
1589 } else {
1590 tcg_out_insn(s, RXY, STY, data, base, index, disp);
1591 }
1592 break;
b8dd88b8 1593
fc313c64 1594 case MO_UQ | MO_BSWAP:
b8dd88b8
RH		 1595 tcg_out_insn(s, RXY, STRVG, data, base, index, disp);
1596 break;
fc313c64 1597 case MO_UQ:
b8dd88b8 1598 tcg_out_insn(s, RXY, STG, data, base, index, disp);
48bb3750 1599 break;
b8dd88b8 1600
48bb3750
RH		 1601 default:
1602 tcg_abort();
1603 }
1604}
1605
1606#if defined(CONFIG_SOFTMMU)
269bd5d8
RH		 1607/* We're expecting to use a 20-bit negative offset on the tlb memory ops. */
1608QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
1609QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 19));
fb596415
RH		 1610
1611/* Load and compare a TLB entry, leaving the flags set. Loads the TLB
1612 addend into R2. Returns a register with the santitized guest address. */
14776ab5 1613static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
fb596415 1614 int mem_index, bool is_ld)
48bb3750 1615{
85aa8081
RH		 1616 unsigned s_bits = opc & MO_SIZE;
1617 unsigned a_bits = get_alignment_bits(opc);
1618 unsigned s_mask = (1 << s_bits) - 1;
1619 unsigned a_mask = (1 << a_bits) - 1;
269bd5d8
RH		 1620 int fast_off = TLB_MASK_TABLE_OFS(mem_index);
1621 int mask_off = fast_off + offsetof(CPUTLBDescFast, mask);
1622 int table_off = fast_off + offsetof(CPUTLBDescFast, table);
a5e39810
RH		 1623 int ofs, a_off;
1624 uint64_t tlb_mask;
1625
4f47e338
RH		 1626 tcg_out_sh64(s, RSY_SRLG, TCG_REG_R2, addr_reg, TCG_REG_NONE,
1627 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1628 tcg_out_insn(s, RXY, NG, TCG_REG_R2, TCG_AREG0, TCG_REG_NONE, mask_off);
1629 tcg_out_insn(s, RXY, AG, TCG_REG_R2, TCG_AREG0, TCG_REG_NONE, table_off);
1630
a5e39810
RH		 1631 /* For aligned accesses, we check the first byte and include the alignment
1632 bits within the address. For unaligned access, we check that we don't
1633 cross pages using the address of the last byte of the access. */
85aa8081
RH		 1634 a_off = (a_bits >= s_bits ? 0 : s_mask - a_mask);
1635 tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
9c3bfb79 1636 if (a_off == 0) {
4f47e338 1637 tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
48bb3750 1638 } else {
a5e39810 1639 tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
547ec121 1640 tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
48bb3750
RH		 1641 }
1642
fb596415 1643 if (is_ld) {
4f47e338 1644 ofs = offsetof(CPUTLBEntry, addr_read);
fb596415 1645 } else {
4f47e338 1646 ofs = offsetof(CPUTLBEntry, addr_write);
48bb3750 1647 }
48bb3750 1648 if (TARGET_LONG_BITS == 32) {
4f47e338 1649 tcg_out_insn(s, RX, C, TCG_REG_R3, TCG_REG_R2, TCG_REG_NONE, ofs);
48bb3750 1650 } else {
4f47e338 1651 tcg_out_insn(s, RXY, CG, TCG_REG_R3, TCG_REG_R2, TCG_REG_NONE, ofs);
48bb3750
RH		 1652 }
1653
4f47e338
RH		 1654 tcg_out_insn(s, RXY, LG, TCG_REG_R2, TCG_REG_R2, TCG_REG_NONE,
1655 offsetof(CPUTLBEntry, addend));
fb596415 1656
48bb3750 1657 if (TARGET_LONG_BITS == 32) {
fb596415
RH		 1658 tgen_ext32u(s, TCG_REG_R3, addr_reg);
1659 return TCG_REG_R3;
48bb3750 1660 }
fb596415
RH		 1661 return addr_reg;
1662}
48bb3750 1663
9002ffcb 1664static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi,
3972ef6f 1665 TCGReg data, TCGReg addr,
fb596415
RH		 1666 tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
1667{
1668 TCGLabelQemuLdst *label = new_ldst_label(s);
1669
1670 label->is_ld = is_ld;
3972ef6f 1671 label->oi = oi;
fb596415
RH		 1672 label->datalo_reg = data;
1673 label->addrlo_reg = addr;
e5e2e4c7 1674 label->raddr = tcg_splitwx_to_rx(raddr);
fb596415
RH		 1675 label->label_ptr[0] = label_ptr;
1676}
48bb3750 1677
aeee05f5 1678static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
fb596415
RH		 1679{
1680 TCGReg addr_reg = lb->addrlo_reg;
1681 TCGReg data_reg = lb->datalo_reg;
9002ffcb 1682 MemOpIdx oi = lb->oi;
14776ab5 1683 MemOp opc = get_memop(oi);
48bb3750 1684
aeee05f5 1685 if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
79dae4dd 1686 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
aeee05f5
RH		 1687 return false;
1688 }
48bb3750 1689
fb596415
RH		 1690 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1691 if (TARGET_LONG_BITS == 64) {
1692 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
1693 }
3972ef6f 1694 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R4, oi);
fb596415 1695 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R5, (uintptr_t)lb->raddr);
cee44b03 1696 tcg_out_call_int(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)]);
fb596415 1697 tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_R2);
65a62a75 1698
fb596415 1699 tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
aeee05f5 1700 return true;
48bb3750
RH		 1701}
1702
aeee05f5 1703static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
48bb3750 1704{
fb596415
RH		 1705 TCGReg addr_reg = lb->addrlo_reg;
1706 TCGReg data_reg = lb->datalo_reg;
9002ffcb 1707 MemOpIdx oi = lb->oi;
14776ab5 1708 MemOp opc = get_memop(oi);
fb596415 1709
aeee05f5 1710 if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
79dae4dd 1711 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
aeee05f5
RH		 1712 return false;
1713 }
fb596415
RH		 1714
1715 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1716 if (TARGET_LONG_BITS == 64) {
1717 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
1718 }
1719 switch (opc & MO_SIZE) {
1720 case MO_UB:
1721 tgen_ext8u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1722 break;
1723 case MO_UW:
1724 tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1725 break;
1726 case MO_UL:
1727 tgen_ext32u(s, TCG_REG_R4, data_reg);
1728 break;
fc313c64 1729 case MO_UQ:
fb596415
RH		 1730 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1731 break;
1732 default:
1733 tcg_abort();
1734 }
3972ef6f 1735 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
fb596415 1736 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
cee44b03 1737 tcg_out_call_int(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
fb596415
RH		 1738
1739 tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
aeee05f5 1740 return true;
48bb3750
RH		 1741}
1742#else
1cd49868
RH		 1743static void tcg_out_test_alignment(TCGContext *s, bool is_ld,
1744 TCGReg addrlo, unsigned a_bits)
1745{
1746 unsigned a_mask = (1 << a_bits) - 1;
1747 TCGLabelQemuLdst *l = new_ldst_label(s);
1748
1749 l->is_ld = is_ld;
1750 l->addrlo_reg = addrlo;
1751
1752 /* We are expecting a_bits to max out at 7, much lower than TMLL. */
1753 tcg_debug_assert(a_bits < 16);
1754 tcg_out_insn(s, RI, TMLL, addrlo, a_mask);
1755
1756 tcg_out16(s, RI_BRC | (7 << 4)); /* CC in {1,2,3} */
1757 l->label_ptr[0] = s->code_ptr;
1758 s->code_ptr += 1;
1759
1760 l->raddr = tcg_splitwx_to_rx(s->code_ptr);
1761}
1762
1763static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
1764{
1765 if (!patch_reloc(l->label_ptr[0], R_390_PC16DBL,
1766 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
1767 return false;
1768 }
1769
1770 tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R3, l->addrlo_reg);
1771 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1772
1773 /* "Tail call" to the helper, with the return address back inline. */
1774 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R14, (uintptr_t)l->raddr);
1775 tgen_gotoi(s, S390_CC_ALWAYS, (const void *)(l->is_ld ? helper_unaligned_ld
1776 : helper_unaligned_st));
1777 return true;
1778}
1779
1780static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
1781{
1782 return tcg_out_fail_alignment(s, l);
1783}
1784
1785static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
1786{
1787 return tcg_out_fail_alignment(s, l);
1788}
1789
48bb3750
RH		 1790static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
1791 TCGReg *index_reg, tcg_target_long *disp)
1792{
1793 if (TARGET_LONG_BITS == 32) {
1794 tgen_ext32u(s, TCG_TMP0, *addr_reg);
1795 *addr_reg = TCG_TMP0;
1796 }
b76f21a7 1797 if (guest_base < 0x80000) {
48bb3750 1798 *index_reg = TCG_REG_NONE;
b76f21a7 1799 *disp = guest_base;
48bb3750
RH		 1800 } else {
1801 *index_reg = TCG_GUEST_BASE_REG;
1802 *disp = 0;
1803 }
1804}
1805#endif /* CONFIG_SOFTMMU */
1806
f24efee4 1807static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
9002ffcb 1808 MemOpIdx oi)
48bb3750 1809{
14776ab5 1810 MemOp opc = get_memop(oi);
fb596415 1811#ifdef CONFIG_SOFTMMU
59227d5d 1812 unsigned mem_index = get_mmuidx(oi);
fb596415
RH		 1813 tcg_insn_unit *label_ptr;
1814 TCGReg base_reg;
1815
1816 base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1);
1817
cd3b29b7
AJ		 1818 tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
1819 label_ptr = s->code_ptr;
1820 s->code_ptr += 1;
fb596415
RH		 1821
1822 tcg_out_qemu_ld_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
48bb3750 1823
3972ef6f 1824 add_qemu_ldst_label(s, 1, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
48bb3750 1825#else
f24efee4
RH		 1826 TCGReg index_reg;
1827 tcg_target_long disp;
1cd49868 1828 unsigned a_bits = get_alignment_bits(opc);
f24efee4 1829
1cd49868
RH		 1830 if (a_bits) {
1831 tcg_out_test_alignment(s, true, addr_reg, a_bits);
1832 }
48bb3750
RH		 1833 tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
1834 tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, index_reg, disp);
1835#endif
1836}
1837
f24efee4 1838static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
9002ffcb 1839 MemOpIdx oi)
48bb3750 1840{
14776ab5 1841 MemOp opc = get_memop(oi);
fb596415 1842#ifdef CONFIG_SOFTMMU
59227d5d 1843 unsigned mem_index = get_mmuidx(oi);
fb596415
RH		 1844 tcg_insn_unit *label_ptr;
1845 TCGReg base_reg;
1846
1847 base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0);
1848
cd3b29b7
AJ		 1849 tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
1850 label_ptr = s->code_ptr;
1851 s->code_ptr += 1;
fb596415
RH		 1852
1853 tcg_out_qemu_st_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
48bb3750 1854
3972ef6f 1855 add_qemu_ldst_label(s, 0, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
48bb3750 1856#else
f24efee4
RH		 1857 TCGReg index_reg;
1858 tcg_target_long disp;
1cd49868 1859 unsigned a_bits = get_alignment_bits(opc);
f24efee4 1860
1cd49868
RH		 1861 if (a_bits) {
1862 tcg_out_test_alignment(s, false, addr_reg, a_bits);
1863 }
48bb3750
RH		 1864 tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
1865 tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, index_reg, disp);
1866#endif
2827822e
AG		 1867}
1868
48bb3750
RH		 1869# define OP_32_64(x) \
1870 case glue(glue(INDEX_op_,x),_i32): \
1871 case glue(glue(INDEX_op_,x),_i64)
48bb3750 1872
a9751609 1873static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
5e8892db
MR		 1874 const TCGArg args[TCG_MAX_OP_ARGS],
1875 const int const_args[TCG_MAX_OP_ARGS])
2827822e 1876{
c2097136 1877 S390Opcode op, op2;
0db921e6 1878 TCGArg a0, a1, a2;
48bb3750
RH		 1879
1880 switch (opc) {
1881 case INDEX_op_exit_tb:
46644483
RH		 1882 /* Reuse the zeroing that exists for goto_ptr. */
1883 a0 = args[0];
1884 if (a0 == 0) {
8b5c2b62 1885 tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
46644483
RH		 1886 } else {
1887 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
1888 tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
1889 }
48bb3750
RH		 1890 break;
1891
1892 case INDEX_op_goto_tb:
829e1376 1893 a0 = args[0];
6bd739ed
RH		 1894 /*
1895 * branch displacement must be aligned for atomic patching;
1896 * see if we need to add extra nop before branch
1897 */
1898 if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
1899 tcg_out16(s, NOP);
829e1376 1900 }
6bd739ed
RH		 1901 tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
1902 s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
1903 s->code_ptr += 2;
9f754620 1904 set_jmp_reset_offset(s, a0);
48bb3750
RH		 1905 break;
1906
46644483 1907 case INDEX_op_goto_ptr:
829e1376 1908 a0 = args[0];
829e1376 1909 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0);
46644483
RH		 1910 break;
1911
48bb3750
RH		 1912 OP_32_64(ld8u):
1913 /* ??? LLC (RXY format) is only present with the extended-immediate
1914 facility, whereas LLGC is always present. */
1915 tcg_out_mem(s, 0, RXY_LLGC, args[0], args[1], TCG_REG_NONE, args[2]);
1916 break;
1917
1918 OP_32_64(ld8s):
1919 /* ??? LB is no smaller than LGB, so no point to using it. */
1920 tcg_out_mem(s, 0, RXY_LGB, args[0], args[1], TCG_REG_NONE, args[2]);
1921 break;
1922
1923 OP_32_64(ld16u):
1924 /* ??? LLH (RXY format) is only present with the extended-immediate
1925 facility, whereas LLGH is always present. */
1926 tcg_out_mem(s, 0, RXY_LLGH, args[0], args[1], TCG_REG_NONE, args[2]);
1927 break;
1928
1929 case INDEX_op_ld16s_i32:
1930 tcg_out_mem(s, RX_LH, RXY_LHY, args[0], args[1], TCG_REG_NONE, args[2]);
1931 break;
1932
1933 case INDEX_op_ld_i32:
1934 tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]);
1935 break;
1936
1937 OP_32_64(st8):
1938 tcg_out_mem(s, RX_STC, RXY_STCY, args[0], args[1],
1939 TCG_REG_NONE, args[2]);
1940 break;
1941
1942 OP_32_64(st16):
1943 tcg_out_mem(s, RX_STH, RXY_STHY, args[0], args[1],
1944 TCG_REG_NONE, args[2]);
1945 break;
1946
1947 case INDEX_op_st_i32:
1948 tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
1949 break;
1950
1951 case INDEX_op_add_i32:
0db921e6 1952 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
48bb3750 1953 if (const_args[2]) {
0db921e6
RH		 1954 do_addi_32:
1955 if (a0 == a1) {
1956 if (a2 == (int16_t)a2) {
1957 tcg_out_insn(s, RI, AHI, a0, a2);
1958 break;
1959 }
3e25f7da
RH		 1960 tcg_out_insn(s, RIL, AFI, a0, a2);
1961 break;
0db921e6
RH		 1962 }
1963 tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
1964 } else if (a0 == a1) {
1965 tcg_out_insn(s, RR, AR, a0, a2);
48bb3750 1966 } else {
0db921e6 1967 tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
48bb3750
RH		 1968 }
1969 break;
1970 case INDEX_op_sub_i32:
0db921e6 1971 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
48bb3750 1972 if (const_args[2]) {
0db921e6
RH		 1973 a2 = -a2;
1974 goto do_addi_32;
c2097136
RH		 1975 } else if (a0 == a1) {
1976 tcg_out_insn(s, RR, SR, a0, a2);
1977 } else {
1dd06b1a 1978 tcg_out_insn(s, RRFa, SRK, a0, a1, a2);
48bb3750
RH		 1979 }
1980 break;
1981
1982 case INDEX_op_and_i32:
c2097136 1983 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
48bb3750 1984 if (const_args[2]) {
c2097136
RH		 1985 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
1986 tgen_andi(s, TCG_TYPE_I32, a0, a2);
1987 } else if (a0 == a1) {
1988 tcg_out_insn(s, RR, NR, a0, a2);
48bb3750 1989 } else {
1dd06b1a 1990 tcg_out_insn(s, RRFa, NRK, a0, a1, a2);
48bb3750
RH		 1991 }
1992 break;
1993 case INDEX_op_or_i32:
c2097136 1994 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
48bb3750 1995 if (const_args[2]) {
c2097136 1996 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
4046d9ca 1997 tgen_ori(s, TCG_TYPE_I32, a0, a2);
c2097136
RH		 1998 } else if (a0 == a1) {
1999 tcg_out_insn(s, RR, OR, a0, a2);
48bb3750 2000 } else {
1dd06b1a 2001 tcg_out_insn(s, RRFa, ORK, a0, a1, a2);
48bb3750
RH		 2002 }
2003 break;
2004 case INDEX_op_xor_i32:
c2097136 2005 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
48bb3750 2006 if (const_args[2]) {
c2097136 2007 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
a0332aca 2008 tcg_out_insn(s, RIL, XILF, a0, a2);
c2097136 2009 } else if (a0 == a1) {
48bb3750 2010 tcg_out_insn(s, RR, XR, args[0], args[2]);
c2097136 2011 } else {
1dd06b1a 2012 tcg_out_insn(s, RRFa, XRK, a0, a1, a2);
48bb3750
RH		 2013 }
2014 break;
2015
2016 case INDEX_op_neg_i32:
2017 tcg_out_insn(s, RR, LCR, args[0], args[1]);
2018 break;
2019
2020 case INDEX_op_mul_i32:
92c89a07 2021 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
48bb3750 2022 if (const_args[2]) {
92c89a07
RH		 2023 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
2024 if (a2 == (int16_t)a2) {
2025 tcg_out_insn(s, RI, MHI, a0, a2);
48bb3750 2026 } else {
92c89a07 2027 tcg_out_insn(s, RIL, MSFI, a0, a2);
48bb3750 2028 }
92c89a07
RH		 2029 } else if (a0 == a1) {
2030 tcg_out_insn(s, RRE, MSR, a0, a2);
48bb3750 2031 } else {
92c89a07 2032 tcg_out_insn(s, RRFa, MSRKC, a0, a1, a2);
48bb3750
RH		 2033 }
2034 break;
2035
2036 case INDEX_op_div2_i32:
4143f78d
RH		 2037 tcg_debug_assert(args[0] == args[2]);
2038 tcg_debug_assert(args[1] == args[3]);
2039 tcg_debug_assert((args[1] & 1) == 0);
2040 tcg_debug_assert(args[0] == args[1] + 1);
2041 tcg_out_insn(s, RR, DR, args[1], args[4]);
48bb3750
RH		 2042 break;
2043 case INDEX_op_divu2_i32:
4143f78d
RH		 2044 tcg_debug_assert(args[0] == args[2]);
2045 tcg_debug_assert(args[1] == args[3]);
2046 tcg_debug_assert((args[1] & 1) == 0);
2047 tcg_debug_assert(args[0] == args[1] + 1);
2048 tcg_out_insn(s, RRE, DLR, args[1], args[4]);
48bb3750
RH		 2049 break;
2050
2051 case INDEX_op_shl_i32:
2052 op = RS_SLL;
c2097136 2053 op2 = RSY_SLLK;
48bb3750 2054 do_shift32:
c2097136
RH		 2055 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
2056 if (a0 == a1) {
2057 if (const_args[2]) {
2058 tcg_out_sh32(s, op, a0, TCG_REG_NONE, a2);
2059 } else {
2060 tcg_out_sh32(s, op, a0, a2, 0);
2061 }
48bb3750 2062 } else {
c2097136
RH		 2063 /* Using tcg_out_sh64 here for the format; it is a 32-bit shift. */
2064 if (const_args[2]) {
2065 tcg_out_sh64(s, op2, a0, a1, TCG_REG_NONE, a2);
2066 } else {
2067 tcg_out_sh64(s, op2, a0, a1, a2, 0);
2068 }
48bb3750
RH		 2069 }
2070 break;
2071 case INDEX_op_shr_i32:
2072 op = RS_SRL;
c2097136 2073 op2 = RSY_SRLK;
48bb3750
RH		 2074 goto do_shift32;
2075 case INDEX_op_sar_i32:
2076 op = RS_SRA;
c2097136 2077 op2 = RSY_SRAK;
48bb3750
RH		 2078 goto do_shift32;
2079
2080 case INDEX_op_rotl_i32:
2081 /* ??? Using tcg_out_sh64 here for the format; it is a 32-bit rol. */
2082 if (const_args[2]) {
2083 tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_REG_NONE, args[2]);
2084 } else {
2085 tcg_out_sh64(s, RSY_RLL, args[0], args[1], args[2], 0);
2086 }
2087 break;
2088 case INDEX_op_rotr_i32:
2089 if (const_args[2]) {
2090 tcg_out_sh64(s, RSY_RLL, args[0], args[1],
2091 TCG_REG_NONE, (32 - args[2]) & 31);
2092 } else {
2093 tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
2094 tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_TMP0, 0);
2095 }
2096 break;
2097
2098 case INDEX_op_ext8s_i32:
2099 tgen_ext8s(s, TCG_TYPE_I32, args[0], args[1]);
2100 break;
2101 case INDEX_op_ext16s_i32:
2102 tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
2103 break;
2104 case INDEX_op_ext8u_i32:
2105 tgen_ext8u(s, TCG_TYPE_I32, args[0], args[1]);
2106 break;
2107 case INDEX_op_ext16u_i32:
2108 tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
2109 break;
2110
1619ee9e
RH		 2111 case INDEX_op_bswap16_i32:
2112 a0 = args[0], a1 = args[1], a2 = args[2];
2113 tcg_out_insn(s, RRE, LRVR, a0, a1);
2114 if (a2 & TCG_BSWAP_OS) {
2115 tcg_out_sh32(s, RS_SRA, a0, TCG_REG_NONE, 16);
2116 } else {
2117 tcg_out_sh32(s, RS_SRL, a0, TCG_REG_NONE, 16);
2118 }
48bb3750 2119 break;
1619ee9e
RH		 2120 case INDEX_op_bswap16_i64:
2121 a0 = args[0], a1 = args[1], a2 = args[2];
2122 tcg_out_insn(s, RRE, LRVGR, a0, a1);
2123 if (a2 & TCG_BSWAP_OS) {
2124 tcg_out_sh64(s, RSY_SRAG, a0, a0, TCG_REG_NONE, 48);
2125 } else {
2126 tcg_out_sh64(s, RSY_SRLG, a0, a0, TCG_REG_NONE, 48);
2127 }
2128 break;
2129
2130 case INDEX_op_bswap32_i32:
48bb3750
RH		 2131 tcg_out_insn(s, RRE, LRVR, args[0], args[1]);
2132 break;
1619ee9e
RH		 2133 case INDEX_op_bswap32_i64:
2134 a0 = args[0], a1 = args[1], a2 = args[2];
2135 tcg_out_insn(s, RRE, LRVR, a0, a1);
2136 if (a2 & TCG_BSWAP_OS) {
2137 tgen_ext32s(s, a0, a0);
2138 } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
2139 tgen_ext32u(s, a0, a0);
2140 }
2141 break;
48bb3750 2142
3790b918 2143 case INDEX_op_add2_i32:
ad19b358
RH		 2144 if (const_args[4]) {
2145 tcg_out_insn(s, RIL, ALFI, args[0], args[4]);
2146 } else {
2147 tcg_out_insn(s, RR, ALR, args[0], args[4]);
2148 }
3790b918
RH		 2149 tcg_out_insn(s, RRE, ALCR, args[1], args[5]);
2150 break;
2151 case INDEX_op_sub2_i32:
ad19b358
RH		 2152 if (const_args[4]) {
2153 tcg_out_insn(s, RIL, SLFI, args[0], args[4]);
2154 } else {
2155 tcg_out_insn(s, RR, SLR, args[0], args[4]);
2156 }
3790b918
RH		 2157 tcg_out_insn(s, RRE, SLBR, args[1], args[5]);
2158 break;
2159
48bb3750 2160 case INDEX_op_br:
bec16311 2161 tgen_branch(s, S390_CC_ALWAYS, arg_label(args[0]));
48bb3750
RH		 2162 break;
2163
2164 case INDEX_op_brcond_i32:
2165 tgen_brcond(s, TCG_TYPE_I32, args[2], args[0],
bec16311 2166 args[1], const_args[1], arg_label(args[3]));
48bb3750
RH		 2167 break;
2168 case INDEX_op_setcond_i32:
2169 tgen_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1],
2170 args[2], const_args[2]);
2171 break;
96a9f093
RH		 2172 case INDEX_op_movcond_i32:
2173 tgen_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1],
7af525af 2174 args[2], const_args[2], args[3], const_args[3]);
96a9f093 2175 break;
48bb3750 2176
f24efee4 2177 case INDEX_op_qemu_ld_i32:
48bb3750 2178 /* ??? Technically we can use a non-extending instruction. */
f24efee4 2179 case INDEX_op_qemu_ld_i64:
59227d5d 2180 tcg_out_qemu_ld(s, args[0], args[1], args[2]);
48bb3750 2181 break;
f24efee4
RH		 2182 case INDEX_op_qemu_st_i32:
2183 case INDEX_op_qemu_st_i64:
59227d5d 2184 tcg_out_qemu_st(s, args[0], args[1], args[2]);
48bb3750
RH		 2185 break;
2186
48bb3750
RH		 2187 case INDEX_op_ld16s_i64:
2188 tcg_out_mem(s, 0, RXY_LGH, args[0], args[1], TCG_REG_NONE, args[2]);
2189 break;
2190 case INDEX_op_ld32u_i64:
2191 tcg_out_mem(s, 0, RXY_LLGF, args[0], args[1], TCG_REG_NONE, args[2]);
2192 break;
2193 case INDEX_op_ld32s_i64:
2194 tcg_out_mem(s, 0, RXY_LGF, args[0], args[1], TCG_REG_NONE, args[2]);
2195 break;
2196 case INDEX_op_ld_i64:
2197 tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
2198 break;
2199
2200 case INDEX_op_st32_i64:
2201 tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
2202 break;
2203 case INDEX_op_st_i64:
2204 tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
2205 break;
2206
2207 case INDEX_op_add_i64:
0db921e6 2208 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2209 if (const_args[2]) {
0db921e6
RH		 2210 do_addi_64:
2211 if (a0 == a1) {
2212 if (a2 == (int16_t)a2) {
2213 tcg_out_insn(s, RI, AGHI, a0, a2);
2214 break;
2215 }
3e25f7da
RH		 2216 if (a2 == (int32_t)a2) {
2217 tcg_out_insn(s, RIL, AGFI, a0, a2);
2218 break;
2219 }
2220 if (a2 == (uint32_t)a2) {
2221 tcg_out_insn(s, RIL, ALGFI, a0, a2);
2222 break;
2223 }
2224 if (-a2 == (uint32_t)-a2) {
2225 tcg_out_insn(s, RIL, SLGFI, a0, -a2);
2226 break;
0db921e6
RH		 2227 }
2228 }
2229 tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
2230 } else if (a0 == a1) {
2231 tcg_out_insn(s, RRE, AGR, a0, a2);
48bb3750 2232 } else {
0db921e6 2233 tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
48bb3750
RH		 2234 }
2235 break;
2236 case INDEX_op_sub_i64:
0db921e6 2237 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2238 if (const_args[2]) {
0db921e6
RH		 2239 a2 = -a2;
2240 goto do_addi_64;
48bb3750 2241 } else {
1dd06b1a 2242 tcg_out_insn(s, RRFa, SGRK, a0, a1, a2);
48bb3750
RH		 2243 }
2244 break;
2245
2246 case INDEX_op_and_i64:
c2097136 2247 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2248 if (const_args[2]) {
c2097136 2249 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
07ff7983 2250 tgen_andi(s, TCG_TYPE_I64, args[0], args[2]);
c2097136 2251 } else {
1dd06b1a 2252 tcg_out_insn(s, RRFa, NGRK, a0, a1, a2);
48bb3750
RH		 2253 }
2254 break;
2255 case INDEX_op_or_i64:
c2097136 2256 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2257 if (const_args[2]) {
c2097136 2258 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
4046d9ca 2259 tgen_ori(s, TCG_TYPE_I64, a0, a2);
48bb3750 2260 } else {
1dd06b1a 2261 tcg_out_insn(s, RRFa, OGRK, a0, a1, a2);
48bb3750
RH		 2262 }
2263 break;
2264 case INDEX_op_xor_i64:
c2097136 2265 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2266 if (const_args[2]) {
c2097136 2267 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
5bf67a92 2268 tgen_xori(s, TCG_TYPE_I64, a0, a2);
48bb3750 2269 } else {
1dd06b1a 2270 tcg_out_insn(s, RRFa, XGRK, a0, a1, a2);
48bb3750
RH		 2271 }
2272 break;
2273
2274 case INDEX_op_neg_i64:
2275 tcg_out_insn(s, RRE, LCGR, args[0], args[1]);
2276 break;
2277 case INDEX_op_bswap64_i64:
2278 tcg_out_insn(s, RRE, LRVGR, args[0], args[1]);
2279 break;
2280
2281 case INDEX_op_mul_i64:
92c89a07 2282 a0 = args[0], a1 = args[1], a2 = args[2];
48bb3750 2283 if (const_args[2]) {
92c89a07
RH		 2284 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
2285 if (a2 == (int16_t)a2) {
2286 tcg_out_insn(s, RI, MGHI, a0, a2);
48bb3750 2287 } else {
92c89a07 2288 tcg_out_insn(s, RIL, MSGFI, a0, a2);
48bb3750 2289 }
92c89a07
RH		 2290 } else if (a0 == a1) {
2291 tcg_out_insn(s, RRE, MSGR, a0, a2);
48bb3750 2292 } else {
92c89a07 2293 tcg_out_insn(s, RRFa, MSGRKC, a0, a1, a2);
48bb3750
RH		 2294 }
2295 break;
2296
2297 case INDEX_op_div2_i64:
4143f78d
RH		 2298 /*
2299 * ??? We get an unnecessary sign-extension of the dividend
2300 * into op0 with this definition, but as we do in fact always
2301 * produce both quotient and remainder using INDEX_op_div_i64
2302 * instead requires jumping through even more hoops.
2303 */
2304 tcg_debug_assert(args[0] == args[2]);
2305 tcg_debug_assert(args[1] == args[3]);
2306 tcg_debug_assert((args[1] & 1) == 0);
2307 tcg_debug_assert(args[0] == args[1] + 1);
2308 tcg_out_insn(s, RRE, DSGR, args[1], args[4]);
48bb3750
RH		 2309 break;
2310 case INDEX_op_divu2_i64:
4143f78d
RH		 2311 tcg_debug_assert(args[0] == args[2]);
2312 tcg_debug_assert(args[1] == args[3]);
2313 tcg_debug_assert((args[1] & 1) == 0);
2314 tcg_debug_assert(args[0] == args[1] + 1);
2315 tcg_out_insn(s, RRE, DLGR, args[1], args[4]);
48bb3750 2316 break;
36017dc6 2317 case INDEX_op_mulu2_i64:
4143f78d
RH		 2318 tcg_debug_assert(args[0] == args[2]);
2319 tcg_debug_assert((args[1] & 1) == 0);
2320 tcg_debug_assert(args[0] == args[1] + 1);
2321 tcg_out_insn(s, RRE, MLGR, args[1], args[3]);
36017dc6 2322 break;
668ce343
RH		 2323 case INDEX_op_muls2_i64:
2324 tcg_debug_assert((args[1] & 1) == 0);
2325 tcg_debug_assert(args[0] == args[1] + 1);
2326 tcg_out_insn(s, RRFa, MGRK, args[1], args[2], args[3]);
2327 break;
48bb3750
RH		 2328
2329 case INDEX_op_shl_i64:
2330 op = RSY_SLLG;
2331 do_shift64:
2332 if (const_args[2]) {
2333 tcg_out_sh64(s, op, args[0], args[1], TCG_REG_NONE, args[2]);
2334 } else {
2335 tcg_out_sh64(s, op, args[0], args[1], args[2], 0);
2336 }
2337 break;
2338 case INDEX_op_shr_i64:
2339 op = RSY_SRLG;
2340 goto do_shift64;
2341 case INDEX_op_sar_i64:
2342 op = RSY_SRAG;
2343 goto do_shift64;
2344
2345 case INDEX_op_rotl_i64:
2346 if (const_args[2]) {
2347 tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
2348 TCG_REG_NONE, args[2]);
2349 } else {
2350 tcg_out_sh64(s, RSY_RLLG, args[0], args[1], args[2], 0);
2351 }
2352 break;
2353 case INDEX_op_rotr_i64:
2354 if (const_args[2]) {
2355 tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
2356 TCG_REG_NONE, (64 - args[2]) & 63);
2357 } else {
2358 /* We can use the smaller 32-bit negate because only the
2359 low 6 bits are examined for the rotate. */
2360 tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
2361 tcg_out_sh64(s, RSY_RLLG, args[0], args[1], TCG_TMP0, 0);
2362 }
2363 break;
2364
2365 case INDEX_op_ext8s_i64:
2366 tgen_ext8s(s, TCG_TYPE_I64, args[0], args[1]);
2367 break;
2368 case INDEX_op_ext16s_i64:
2369 tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
2370 break;
4f2331e5 2371 case INDEX_op_ext_i32_i64:
48bb3750
RH		 2372 case INDEX_op_ext32s_i64:
2373 tgen_ext32s(s, args[0], args[1]);
2374 break;
2375 case INDEX_op_ext8u_i64:
2376 tgen_ext8u(s, TCG_TYPE_I64, args[0], args[1]);
2377 break;
2378 case INDEX_op_ext16u_i64:
2379 tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
2380 break;
4f2331e5 2381 case INDEX_op_extu_i32_i64:
48bb3750
RH		 2382 case INDEX_op_ext32u_i64:
2383 tgen_ext32u(s, args[0], args[1]);
2384 break;
2385
3790b918 2386 case INDEX_op_add2_i64:
ad19b358
RH		 2387 if (const_args[4]) {
2388 if ((int64_t)args[4] >= 0) {
2389 tcg_out_insn(s, RIL, ALGFI, args[0], args[4]);
2390 } else {
2391 tcg_out_insn(s, RIL, SLGFI, args[0], -args[4]);
2392 }
2393 } else {
2394 tcg_out_insn(s, RRE, ALGR, args[0], args[4]);
2395 }
3790b918
RH		 2396 tcg_out_insn(s, RRE, ALCGR, args[1], args[5]);
2397 break;
2398 case INDEX_op_sub2_i64:
ad19b358
RH		 2399 if (const_args[4]) {
2400 if ((int64_t)args[4] >= 0) {
2401 tcg_out_insn(s, RIL, SLGFI, args[0], args[4]);
2402 } else {
2403 tcg_out_insn(s, RIL, ALGFI, args[0], -args[4]);
2404 }
2405 } else {
2406 tcg_out_insn(s, RRE, SLGR, args[0], args[4]);
2407 }
3790b918
RH		 2408 tcg_out_insn(s, RRE, SLBGR, args[1], args[5]);
2409 break;
2410
48bb3750
RH		 2411 case INDEX_op_brcond_i64:
2412 tgen_brcond(s, TCG_TYPE_I64, args[2], args[0],
bec16311 2413 args[1], const_args[1], arg_label(args[3]));
48bb3750
RH		 2414 break;
2415 case INDEX_op_setcond_i64:
2416 tgen_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1],
2417 args[2], const_args[2]);
2418 break;
96a9f093
RH		 2419 case INDEX_op_movcond_i64:
2420 tgen_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1],
7af525af 2421 args[2], const_args[2], args[3], const_args[3]);
96a9f093 2422 break;
48bb3750 2423
d5690ea4 2424 OP_32_64(deposit):
752b1be9
RH		 2425 a0 = args[0], a1 = args[1], a2 = args[2];
2426 if (const_args[1]) {
2427 tgen_deposit(s, a0, a2, args[3], args[4], 1);
2428 } else {
2429 /* Since we can't support "0Z" as a constraint, we allow a1 in
2430 any register. Fix things up as if a matching constraint. */
2431 if (a0 != a1) {
2432 TCGType type = (opc == INDEX_op_deposit_i64);
2433 if (a0 == a2) {
2434 tcg_out_mov(s, type, TCG_TMP0, a2);
2435 a2 = TCG_TMP0;
2436 }
2437 tcg_out_mov(s, type, a0, a1);
2438 }
2439 tgen_deposit(s, a0, a2, args[3], args[4], 0);
2440 }
d5690ea4 2441 break;
752b1be9 2442
b0bf5fe8
RH		 2443 OP_32_64(extract):
2444 tgen_extract(s, args[0], args[1], args[2], args[3]);
2445 break;
d5690ea4 2446
ce411066
RH		 2447 case INDEX_op_clz_i64:
2448 tgen_clz(s, args[0], args[1], args[2], const_args[2]);
2449 break;
2450
c9314d61
PK		 2451 case INDEX_op_mb:
2452 /* The host memory model is quite strong, we simply need to
2453 serialize the instruction stream. */
2454 if (args[0] & TCG_MO_ST_LD) {
e62d5752
RH		 2455 /* fast-bcr-serialization facility (45) is present */
2456 tcg_out_insn(s, RR, BCR, 14, 0);
c9314d61
PK		 2457 }
2458 break;
2459
96d0ee7f
RH		 2460 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
2461 case INDEX_op_mov_i64:
96d0ee7f 2462 case INDEX_op_call: /* Always emitted via tcg_out_call. */
48bb3750 2463 default:
48bb3750
RH		 2464 tcg_abort();
2465 }
2827822e
AG		 2466}
2467
34ef7676
RH		 2468static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
2469 TCGReg dst, TCGReg src)
2470{
79cada86
RH		 2471 if (is_general_reg(src)) {
2472 /* Replicate general register into two MO_64. */
2473 tcg_out_insn(s, VRRf, VLVGP, dst, src, src);
2474 if (vece == MO_64) {
2475 return true;
2476 }
6e591a85 2477 src = dst;
79cada86
RH		 2478 }
2479
2480 /*
2481 * Recall that the "standard" integer, within a vector, is the
2482 * rightmost element of the leftmost doubleword, a-la VLLEZ.
2483 */
2484 tcg_out_insn(s, VRIc, VREP, dst, (8 >> vece) - 1, src, vece);
2485 return true;
34ef7676
RH		 2486}
2487
2488static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
2489 TCGReg dst, TCGReg base, intptr_t offset)
2490{
79cada86
RH		 2491 tcg_out_vrx_mem(s, VRX_VLREP, dst, base, TCG_REG_NONE, offset, vece);
2492 return true;
34ef7676
RH		 2493}
2494
2495static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
2496 TCGReg dst, int64_t val)
2497{
79cada86
RH		 2498 int i, mask, msb, lsb;
2499
2500 /* Look for int16_t elements. */
2501 if (vece <= MO_16 ||
2502 (vece == MO_32 ? (int32_t)val : val) == (int16_t)val) {
2503 tcg_out_insn(s, VRIa, VREPI, dst, val, vece);
2504 return;
2505 }
2506
2507 /* Look for bit masks. */
2508 if (vece == MO_32) {
2509 if (risbg_mask((int32_t)val)) {
2510 /* Handle wraparound by swapping msb and lsb. */
2511 if ((val & 0x80000001u) == 0x80000001u) {
2512 msb = 32 - ctz32(~val);
2513 lsb = clz32(~val) - 1;
2514 } else {
2515 msb = clz32(val);
2516 lsb = 31 - ctz32(val);
2517 }
a66669c9 2518 tcg_out_insn(s, VRIb, VGM, dst, msb, lsb, MO_32);
79cada86
RH		 2519 return;
2520 }
2521 } else {
2522 if (risbg_mask(val)) {
2523 /* Handle wraparound by swapping msb and lsb. */
2524 if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
2525 /* Handle wraparound by swapping msb and lsb. */
2526 msb = 64 - ctz64(~val);
2527 lsb = clz64(~val) - 1;
2528 } else {
2529 msb = clz64(val);
2530 lsb = 63 - ctz64(val);
2531 }
a66669c9 2532 tcg_out_insn(s, VRIb, VGM, dst, msb, lsb, MO_64);
79cada86
RH		 2533 return;
2534 }
2535 }
2536
2537 /* Look for all bytes 0x00 or 0xff. */
2538 for (i = mask = 0; i < 8; i++) {
2539 uint8_t byte = val >> (i * 8);
2540 if (byte == 0xff) {
2541 mask |= 1 << i;
2542 } else if (byte != 0) {
2543 break;
2544 }
2545 }
2546 if (i == 8) {
2547 tcg_out_insn(s, VRIa, VGBM, dst, mask * 0x0101, 0);
2548 return;
2549 }
2550
2551 /* Otherwise, stuff it in the constant pool. */
2552 tcg_out_insn(s, RIL, LARL, TCG_TMP0, 0);
2553 new_pool_label(s, val, R_390_PC32DBL, s->code_ptr - 2, 2);
2554 tcg_out_insn(s, VRX, VLREP, dst, TCG_TMP0, TCG_REG_NONE, 0, MO_64);
34ef7676
RH		 2555}
2556
2557static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2558 unsigned vecl, unsigned vece,
d58f0173
MR		 2559 const TCGArg args[TCG_MAX_OP_ARGS],
2560 const int const_args[TCG_MAX_OP_ARGS])
34ef7676 2561{
a429ee29
RH		 2562 TCGType type = vecl + TCG_TYPE_V64;
2563 TCGArg a0 = args[0], a1 = args[1], a2 = args[2];
2564
2565 switch (opc) {
2566 case INDEX_op_ld_vec:
2567 tcg_out_ld(s, type, a0, a1, a2);
2568 break;
2569 case INDEX_op_st_vec:
2570 tcg_out_st(s, type, a0, a1, a2);
2571 break;
2572 case INDEX_op_dupm_vec:
2573 tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
2574 break;
2575
ae77bbe5
RH		 2576 case INDEX_op_abs_vec:
2577 tcg_out_insn(s, VRRa, VLP, a0, a1, vece);
2578 break;
2579 case INDEX_op_neg_vec:
2580 tcg_out_insn(s, VRRa, VLC, a0, a1, vece);
2581 break;
2582 case INDEX_op_not_vec:
2583 tcg_out_insn(s, VRRc, VNO, a0, a1, a1, 0);
2584 break;
2585
a429ee29
RH		 2586 case INDEX_op_add_vec:
2587 tcg_out_insn(s, VRRc, VA, a0, a1, a2, vece);
2588 break;
2589 case INDEX_op_sub_vec:
2590 tcg_out_insn(s, VRRc, VS, a0, a1, a2, vece);
2591 break;
2592 case INDEX_op_and_vec:
2593 tcg_out_insn(s, VRRc, VN, a0, a1, a2, 0);
2594 break;
ae77bbe5
RH		 2595 case INDEX_op_andc_vec:
2596 tcg_out_insn(s, VRRc, VNC, a0, a1, a2, 0);
2597 break;
479b61cb
RH		 2598 case INDEX_op_mul_vec:
2599 tcg_out_insn(s, VRRc, VML, a0, a1, a2, vece);
2600 break;
a429ee29
RH		 2601 case INDEX_op_or_vec:
2602 tcg_out_insn(s, VRRc, VO, a0, a1, a2, 0);
2603 break;
ae77bbe5
RH		 2604 case INDEX_op_orc_vec:
2605 tcg_out_insn(s, VRRc, VOC, a0, a1, a2, 0);
2606 break;
a429ee29
RH		 2607 case INDEX_op_xor_vec:
2608 tcg_out_insn(s, VRRc, VX, a0, a1, a2, 0);
2609 break;
21eab5bf
RH		 2610 case INDEX_op_nand_vec:
2611 tcg_out_insn(s, VRRc, VNN, a0, a1, a2, 0);
2612 break;
2613 case INDEX_op_nor_vec:
2614 tcg_out_insn(s, VRRc, VNO, a0, a1, a2, 0);
2615 break;
2616 case INDEX_op_eqv_vec:
2617 tcg_out_insn(s, VRRc, VNX, a0, a1, a2, 0);
2618 break;
a429ee29 2619
22cb37b4
RH		 2620 case INDEX_op_shli_vec:
2621 tcg_out_insn(s, VRSa, VESL, a0, a2, TCG_REG_NONE, a1, vece);
2622 break;
2623 case INDEX_op_shri_vec:
2624 tcg_out_insn(s, VRSa, VESRL, a0, a2, TCG_REG_NONE, a1, vece);
2625 break;
2626 case INDEX_op_sari_vec:
2627 tcg_out_insn(s, VRSa, VESRA, a0, a2, TCG_REG_NONE, a1, vece);
2628 break;
2629 case INDEX_op_rotli_vec:
2630 tcg_out_insn(s, VRSa, VERLL, a0, a2, TCG_REG_NONE, a1, vece);
2631 break;
2632 case INDEX_op_shls_vec:
2633 tcg_out_insn(s, VRSa, VESL, a0, 0, a2, a1, vece);
2634 break;
2635 case INDEX_op_shrs_vec:
2636 tcg_out_insn(s, VRSa, VESRL, a0, 0, a2, a1, vece);
2637 break;
2638 case INDEX_op_sars_vec:
2639 tcg_out_insn(s, VRSa, VESRA, a0, 0, a2, a1, vece);
2640 break;
2641 case INDEX_op_rotls_vec:
2642 tcg_out_insn(s, VRSa, VERLL, a0, 0, a2, a1, vece);
2643 break;
2644 case INDEX_op_shlv_vec:
2645 tcg_out_insn(s, VRRc, VESLV, a0, a1, a2, vece);
2646 break;
2647 case INDEX_op_shrv_vec:
2648 tcg_out_insn(s, VRRc, VESRLV, a0, a1, a2, vece);
2649 break;
2650 case INDEX_op_sarv_vec:
2651 tcg_out_insn(s, VRRc, VESRAV, a0, a1, a2, vece);
2652 break;
2653 case INDEX_op_rotlv_vec:
2654 tcg_out_insn(s, VRRc, VERLLV, a0, a1, a2, vece);
2655 break;
2656
220db7a6
RH		 2657 case INDEX_op_smin_vec:
2658 tcg_out_insn(s, VRRc, VMN, a0, a1, a2, vece);
2659 break;
2660 case INDEX_op_smax_vec:
2661 tcg_out_insn(s, VRRc, VMX, a0, a1, a2, vece);
2662 break;
2663 case INDEX_op_umin_vec:
2664 tcg_out_insn(s, VRRc, VMNL, a0, a1, a2, vece);
2665 break;
2666 case INDEX_op_umax_vec:
2667 tcg_out_insn(s, VRRc, VMXL, a0, a1, a2, vece);
2668 break;
2669
9bca986d 2670 case INDEX_op_bitsel_vec:
6e5f9fb7 2671 tcg_out_insn(s, VRRe, VSEL, a0, a2, args[3], a1);
9bca986d
RH		 2672 break;
2673
a429ee29
RH		 2674 case INDEX_op_cmp_vec:
2675 switch ((TCGCond)args[3]) {
2676 case TCG_COND_EQ:
2677 tcg_out_insn(s, VRRc, VCEQ, a0, a1, a2, vece);
2678 break;
2679 case TCG_COND_GT:
2680 tcg_out_insn(s, VRRc, VCH, a0, a1, a2, vece);
2681 break;
2682 case TCG_COND_GTU:
2683 tcg_out_insn(s, VRRc, VCHL, a0, a1, a2, vece);
2684 break;
2685 default:
2686 g_assert_not_reached();
2687 }
2688 break;
2689
4223c9c1
RH		 2690 case INDEX_op_s390_vuph_vec:
2691 tcg_out_insn(s, VRRa, VUPH, a0, a1, vece);
2692 break;
2693 case INDEX_op_s390_vupl_vec:
2694 tcg_out_insn(s, VRRa, VUPL, a0, a1, vece);
2695 break;
2696 case INDEX_op_s390_vpks_vec:
2697 tcg_out_insn(s, VRRc, VPKS, a0, a1, a2, vece);
2698 break;
2699
a429ee29
RH		 2700 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
2701 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
2702 default:
2703 g_assert_not_reached();
2704 }
34ef7676
RH		 2705}
2706
2707int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
2708{
a429ee29 2709 switch (opc) {
ae77bbe5 2710 case INDEX_op_abs_vec:
a429ee29
RH		 2711 case INDEX_op_add_vec:
2712 case INDEX_op_and_vec:
ae77bbe5 2713 case INDEX_op_andc_vec:
9bca986d 2714 case INDEX_op_bitsel_vec:
21eab5bf
RH		 2715 case INDEX_op_eqv_vec:
2716 case INDEX_op_nand_vec:
ae77bbe5 2717 case INDEX_op_neg_vec:
21eab5bf 2718 case INDEX_op_nor_vec:
ae77bbe5 2719 case INDEX_op_not_vec:
a429ee29 2720 case INDEX_op_or_vec:
ae77bbe5 2721 case INDEX_op_orc_vec:
22cb37b4
RH		 2722 case INDEX_op_rotli_vec:
2723 case INDEX_op_rotls_vec:
2724 case INDEX_op_rotlv_vec:
2725 case INDEX_op_sari_vec:
2726 case INDEX_op_sars_vec:
2727 case INDEX_op_sarv_vec:
2728 case INDEX_op_shli_vec:
2729 case INDEX_op_shls_vec:
2730 case INDEX_op_shlv_vec:
2731 case INDEX_op_shri_vec:
2732 case INDEX_op_shrs_vec:
2733 case INDEX_op_shrv_vec:
220db7a6
RH		 2734 case INDEX_op_smax_vec:
2735 case INDEX_op_smin_vec:
a429ee29 2736 case INDEX_op_sub_vec:
220db7a6
RH		 2737 case INDEX_op_umax_vec:
2738 case INDEX_op_umin_vec:
a429ee29
RH		 2739 case INDEX_op_xor_vec:
2740 return 1;
2741 case INDEX_op_cmp_vec:
ea3f2af8 2742 case INDEX_op_cmpsel_vec:
22cb37b4 2743 case INDEX_op_rotrv_vec:
a429ee29 2744 return -1;
479b61cb
RH		 2745 case INDEX_op_mul_vec:
2746 return vece < MO_64;
4223c9c1
RH		 2747 case INDEX_op_ssadd_vec:
2748 case INDEX_op_sssub_vec:
2749 return vece < MO_64 ? -1 : 0;
a429ee29
RH		 2750 default:
2751 return 0;
2752 }
2753}
2754
2755static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0,
2756 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
2757{
2758 bool need_swap = false, need_inv = false;
2759
2760 switch (cond) {
2761 case TCG_COND_EQ:
2762 case TCG_COND_GT:
2763 case TCG_COND_GTU:
2764 break;
2765 case TCG_COND_NE:
2766 case TCG_COND_LE:
2767 case TCG_COND_LEU:
2768 need_inv = true;
2769 break;
2770 case TCG_COND_LT:
2771 case TCG_COND_LTU:
2772 need_swap = true;
2773 break;
2774 case TCG_COND_GE:
2775 case TCG_COND_GEU:
2776 need_swap = need_inv = true;
2777 break;
2778 default:
2779 g_assert_not_reached();
2780 }
2781
2782 if (need_inv) {
2783 cond = tcg_invert_cond(cond);
2784 }
2785 if (need_swap) {
2786 TCGv_vec t1;
2787 t1 = v1, v1 = v2, v2 = t1;
2788 cond = tcg_swap_cond(cond);
2789 }
2790
2791 vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0),
2792 tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond);
2793
2794 return need_inv;
2795}
2796
2797static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0,
2798 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
2799{
2800 if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) {
2801 tcg_gen_not_vec(vece, v0, v0);
2802 }
34ef7676
RH		 2803}
2804
ea3f2af8
RH		 2805static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0,
2806 TCGv_vec c1, TCGv_vec c2,
2807 TCGv_vec v3, TCGv_vec v4, TCGCond cond)
2808{
2809 TCGv_vec t = tcg_temp_new_vec(type);
2810
2811 if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) {
2812 /* Invert the sense of the compare by swapping arguments. */
2813 tcg_gen_bitsel_vec(vece, v0, t, v4, v3);
2814 } else {
2815 tcg_gen_bitsel_vec(vece, v0, t, v3, v4);
2816 }
2817 tcg_temp_free_vec(t);
2818}
2819
4223c9c1
RH		 2820static void expand_vec_sat(TCGType type, unsigned vece, TCGv_vec v0,
2821 TCGv_vec v1, TCGv_vec v2, TCGOpcode add_sub_opc)
2822{
2823 TCGv_vec h1 = tcg_temp_new_vec(type);
2824 TCGv_vec h2 = tcg_temp_new_vec(type);
2825 TCGv_vec l1 = tcg_temp_new_vec(type);
2826 TCGv_vec l2 = tcg_temp_new_vec(type);
2827
2828 tcg_debug_assert (vece < MO_64);
2829
2830 /* Unpack with sign-extension. */
2831 vec_gen_2(INDEX_op_s390_vuph_vec, type, vece,
2832 tcgv_vec_arg(h1), tcgv_vec_arg(v1));
2833 vec_gen_2(INDEX_op_s390_vuph_vec, type, vece,
2834 tcgv_vec_arg(h2), tcgv_vec_arg(v2));
2835
2836 vec_gen_2(INDEX_op_s390_vupl_vec, type, vece,
2837 tcgv_vec_arg(l1), tcgv_vec_arg(v1));
2838 vec_gen_2(INDEX_op_s390_vupl_vec, type, vece,
2839 tcgv_vec_arg(l2), tcgv_vec_arg(v2));
2840
2841 /* Arithmetic on a wider element size. */
2842 vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(h1),
2843 tcgv_vec_arg(h1), tcgv_vec_arg(h2));
2844 vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(l1),
2845 tcgv_vec_arg(l1), tcgv_vec_arg(l2));
2846
2847 /* Pack with saturation. */
2848 vec_gen_3(INDEX_op_s390_vpks_vec, type, vece + 1,
2849 tcgv_vec_arg(v0), tcgv_vec_arg(h1), tcgv_vec_arg(l1));
2850
2851 tcg_temp_free_vec(h1);
2852 tcg_temp_free_vec(h2);
2853 tcg_temp_free_vec(l1);
2854 tcg_temp_free_vec(l2);
2855}
2856
34ef7676
RH		 2857void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
2858 TCGArg a0, ...)
2859{
a429ee29 2860 va_list va;
ea3f2af8 2861 TCGv_vec v0, v1, v2, v3, v4, t0;
a429ee29
RH		 2862
2863 va_start(va, a0);
2864 v0 = temp_tcgv_vec(arg_temp(a0));
2865 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2866 v2 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2867
2868 switch (opc) {
2869 case INDEX_op_cmp_vec:
2870 expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg));
2871 break;
2872
ea3f2af8
RH		 2873 case INDEX_op_cmpsel_vec:
2874 v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2875 v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
2876 expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg));
2877 break;
2878
22cb37b4
RH		 2879 case INDEX_op_rotrv_vec:
2880 t0 = tcg_temp_new_vec(type);
2881 tcg_gen_neg_vec(vece, t0, v2);
2882 tcg_gen_rotlv_vec(vece, v0, v1, t0);
2883 tcg_temp_free_vec(t0);
2884 break;
2885
4223c9c1
RH		 2886 case INDEX_op_ssadd_vec:
2887 expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_add_vec);
2888 break;
2889 case INDEX_op_sssub_vec:
2890 expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_sub_vec);
2891 break;
2892
a429ee29
RH		 2893 default:
2894 g_assert_not_reached();
2895 }
2896 va_end(va);
34ef7676
RH		 2897}
2898
d1c36a90 2899static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op)
f69d277e 2900{
9b5500b6
RH		 2901 switch (op) {
2902 case INDEX_op_goto_ptr:
d1c36a90 2903 return C_O0_I1(r);
9b5500b6
RH		 2904
2905 case INDEX_op_ld8u_i32:
2906 case INDEX_op_ld8u_i64:
2907 case INDEX_op_ld8s_i32:
2908 case INDEX_op_ld8s_i64:
2909 case INDEX_op_ld16u_i32:
2910 case INDEX_op_ld16u_i64:
2911 case INDEX_op_ld16s_i32:
2912 case INDEX_op_ld16s_i64:
2913 case INDEX_op_ld_i32:
2914 case INDEX_op_ld32u_i64:
2915 case INDEX_op_ld32s_i64:
2916 case INDEX_op_ld_i64:
d1c36a90
RH		 2917 return C_O1_I1(r, r);
2918
9b5500b6
RH		 2919 case INDEX_op_st8_i32:
2920 case INDEX_op_st8_i64:
2921 case INDEX_op_st16_i32:
2922 case INDEX_op_st16_i64:
2923 case INDEX_op_st_i32:
2924 case INDEX_op_st32_i64:
2925 case INDEX_op_st_i64:
d1c36a90 2926 return C_O0_I2(r, r);
9b5500b6
RH		 2927
2928 case INDEX_op_add_i32:
2929 case INDEX_op_add_i64:
d1c36a90
RH		 2930 case INDEX_op_shl_i64:
2931 case INDEX_op_shr_i64:
2932 case INDEX_op_sar_i64:
2933 case INDEX_op_rotl_i32:
2934 case INDEX_op_rotl_i64:
2935 case INDEX_op_rotr_i32:
2936 case INDEX_op_rotr_i64:
2937 case INDEX_op_clz_i64:
2938 case INDEX_op_setcond_i32:
2939 case INDEX_op_setcond_i64:
2940 return C_O1_I2(r, r, ri);
2941
9b5500b6
RH		 2942 case INDEX_op_sub_i32:
2943 case INDEX_op_sub_i64:
bdcd5d19
RH		 2944 case INDEX_op_and_i32:
2945 case INDEX_op_and_i64:
4046d9ca
RH		 2946 case INDEX_op_or_i32:
2947 case INDEX_op_or_i64:
5bf67a92
RH		 2948 case INDEX_op_xor_i32:
2949 case INDEX_op_xor_i64:
238da1c9 2950 return C_O1_I2(r, r, ri);
a8f0269e 2951
9b5500b6 2952 case INDEX_op_mul_i32:
92c89a07
RH		 2953 return (HAVE_FACILITY(MISC_INSN_EXT2)
2954 ? C_O1_I2(r, r, ri)
2955 : C_O1_I2(r, 0, ri));
9b5500b6 2956 case INDEX_op_mul_i64:
92c89a07
RH		 2957 return (HAVE_FACILITY(MISC_INSN_EXT2)
2958 ? C_O1_I2(r, r, rJ)
2959 : C_O1_I2(r, 0, rJ));
a8f0269e 2960
9b5500b6
RH		 2961 case INDEX_op_shl_i32:
2962 case INDEX_op_shr_i32:
2963 case INDEX_op_sar_i32:
238da1c9 2964 return C_O1_I2(r, r, ri);
9b5500b6
RH		 2965
2966 case INDEX_op_brcond_i32:
2967 case INDEX_op_brcond_i64:
d1c36a90 2968 return C_O0_I2(r, ri);
9b5500b6
RH		 2969
2970 case INDEX_op_bswap16_i32:
2971 case INDEX_op_bswap16_i64:
2972 case INDEX_op_bswap32_i32:
2973 case INDEX_op_bswap32_i64:
2974 case INDEX_op_bswap64_i64:
2975 case INDEX_op_neg_i32:
2976 case INDEX_op_neg_i64:
2977 case INDEX_op_ext8s_i32:
2978 case INDEX_op_ext8s_i64:
2979 case INDEX_op_ext8u_i32:
2980 case INDEX_op_ext8u_i64:
2981 case INDEX_op_ext16s_i32:
2982 case INDEX_op_ext16s_i64:
2983 case INDEX_op_ext16u_i32:
2984 case INDEX_op_ext16u_i64:
2985 case INDEX_op_ext32s_i64:
2986 case INDEX_op_ext32u_i64:
2987 case INDEX_op_ext_i32_i64:
2988 case INDEX_op_extu_i32_i64:
2989 case INDEX_op_extract_i32:
2990 case INDEX_op_extract_i64:
d1c36a90 2991 return C_O1_I1(r, r);
9b5500b6
RH		 2992
2993 case INDEX_op_qemu_ld_i32:
2994 case INDEX_op_qemu_ld_i64:
d1c36a90 2995 return C_O1_I1(r, L);
9b5500b6
RH		 2996 case INDEX_op_qemu_st_i64:
2997 case INDEX_op_qemu_st_i32:
d1c36a90 2998 return C_O0_I2(L, L);
f69d277e 2999
9b5500b6
RH		 3000 case INDEX_op_deposit_i32:
3001 case INDEX_op_deposit_i64:
d1c36a90
RH		 3002 return C_O1_I2(r, rZ, r);
3003
9b5500b6
RH		 3004 case INDEX_op_movcond_i32:
3005 case INDEX_op_movcond_i64:
748b7f3e 3006 return (HAVE_FACILITY(LOAD_ON_COND2)
d1c36a90
RH		 3007 ? C_O1_I4(r, r, ri, rI, 0)
3008 : C_O1_I4(r, r, ri, r, 0));
3009
9b5500b6
RH		 3010 case INDEX_op_div2_i32:
3011 case INDEX_op_div2_i64:
3012 case INDEX_op_divu2_i32:
3013 case INDEX_op_divu2_i64:
4143f78d 3014 return C_O2_I3(o, m, 0, 1, r);
d1c36a90 3015
9b5500b6 3016 case INDEX_op_mulu2_i64:
4143f78d 3017 return C_O2_I2(o, m, 0, r);
668ce343
RH		 3018 case INDEX_op_muls2_i64:
3019 return C_O2_I2(o, m, r, r);
ba18b07d 3020
9b5500b6 3021 case INDEX_op_add2_i32:
9b5500b6 3022 case INDEX_op_sub2_i32:
3e25f7da 3023 return C_O2_I4(r, r, 0, 1, ri, r);
d1c36a90 3024
ba18b07d 3025 case INDEX_op_add2_i64:
9b5500b6 3026 case INDEX_op_sub2_i64:
3e25f7da 3027 return C_O2_I4(r, r, 0, 1, rA, r);
9b5500b6 3028
34ef7676
RH		 3029 case INDEX_op_st_vec:
3030 return C_O0_I2(v, r);
3031 case INDEX_op_ld_vec:
3032 case INDEX_op_dupm_vec:
3033 return C_O1_I1(v, r);
3034 case INDEX_op_dup_vec:
3035 return C_O1_I1(v, vr);
ae77bbe5
RH		 3036 case INDEX_op_abs_vec:
3037 case INDEX_op_neg_vec:
3038 case INDEX_op_not_vec:
22cb37b4
RH		 3039 case INDEX_op_rotli_vec:
3040 case INDEX_op_sari_vec:
3041 case INDEX_op_shli_vec:
3042 case INDEX_op_shri_vec:
4223c9c1
RH		 3043 case INDEX_op_s390_vuph_vec:
3044 case INDEX_op_s390_vupl_vec:
ae77bbe5 3045 return C_O1_I1(v, v);
34ef7676
RH		 3046 case INDEX_op_add_vec:
3047 case INDEX_op_sub_vec:
3048 case INDEX_op_and_vec:
ae77bbe5 3049 case INDEX_op_andc_vec:
34ef7676 3050 case INDEX_op_or_vec:
ae77bbe5 3051 case INDEX_op_orc_vec:
34ef7676 3052 case INDEX_op_xor_vec:
21eab5bf
RH		 3053 case INDEX_op_nand_vec:
3054 case INDEX_op_nor_vec:
3055 case INDEX_op_eqv_vec:
34ef7676 3056 case INDEX_op_cmp_vec:
479b61cb 3057 case INDEX_op_mul_vec:
22cb37b4
RH		 3058 case INDEX_op_rotlv_vec:
3059 case INDEX_op_rotrv_vec:
3060 case INDEX_op_shlv_vec:
3061 case INDEX_op_shrv_vec:
3062 case INDEX_op_sarv_vec:
220db7a6
RH		 3063 case INDEX_op_smax_vec:
3064 case INDEX_op_smin_vec:
3065 case INDEX_op_umax_vec:
3066 case INDEX_op_umin_vec:
4223c9c1 3067 case INDEX_op_s390_vpks_vec:
34ef7676 3068 return C_O1_I2(v, v, v);
22cb37b4
RH		 3069 case INDEX_op_rotls_vec:
3070 case INDEX_op_shls_vec:
3071 case INDEX_op_shrs_vec:
3072 case INDEX_op_sars_vec:
3073 return C_O1_I2(v, v, r);
9bca986d
RH		 3074 case INDEX_op_bitsel_vec:
3075 return C_O1_I3(v, v, v, v);
34ef7676 3076
9b5500b6 3077 default:
d1c36a90 3078 g_assert_not_reached();
f69d277e 3079 }
f69d277e
RH		 3080}
3081
34ef7676
RH		 3082/*
3083 * Mainline glibc added HWCAP_S390_VX before it was kernel abi.
3084 * Some distros have fixed this up locally, others have not.
3085 */
3086#ifndef HWCAP_S390_VXRS
3087#define HWCAP_S390_VXRS 2048
3088#endif
3089
b2c98d9d 3090static void query_s390_facilities(void)
48bb3750 3091{
c9baa30f 3092 unsigned long hwcap = qemu_getauxval(AT_HWCAP);
761ea522 3093 const char *which;
48bb3750 3094
c9baa30f
RH		 3095 /* Is STORE FACILITY LIST EXTENDED available? Honestly, I believe this
3096 is present on all 64-bit systems, but let's check for it anyway. */
3097 if (hwcap & HWCAP_S390_STFLE) {
748b7f3e
RH		 3098 register int r0 __asm__("0") = ARRAY_SIZE(s390_facilities) - 1;
3099 register void *r1 __asm__("1") = s390_facilities;
48bb3750 3100
c9baa30f 3101 /* stfle 0(%r1) */
c9baa30f 3102 asm volatile(".word 0xb2b0,0x1000"
748b7f3e 3103 : "=r"(r0) : "r"(r0), "r"(r1) : "memory", "cc");
48bb3750 3104 }
34ef7676
RH		 3105
3106 /*
3107 * Use of vector registers requires os support beyond the facility bit.
3108 * If the kernel does not advertise support, disable the facility bits.
3109 * There is nothing else we currently care about in the 3rd word, so
3110 * disable VECTOR with one store.
3111 */
a429ee29 3112 if (!(hwcap & HWCAP_S390_VXRS)) {
34ef7676
RH		 3113 s390_facilities[2] = 0;
3114 }
761ea522
RH		 3115
3116 /*
c68d5b7a 3117 * Minimum supported cpu revision is z196.
761ea522
RH		 3118 * Check for all required facilities.
3119 * ZARCH_ACTIVE is done via preprocessor check for 64-bit.
3120 */
3121 if (!HAVE_FACILITY(LONG_DISP)) {
3122 which = "long-displacement";
3123 goto fail;
3124 }
3e25f7da
RH		 3125 if (!HAVE_FACILITY(EXT_IMM)) {
3126 which = "extended-immediate";
3127 goto fail;
3128 }
9c3bfb79
RH		 3129 if (!HAVE_FACILITY(GEN_INST_EXT)) {
3130 which = "general-instructions-extension";
3131 goto fail;
3132 }
c68d5b7a
RH		 3133 /*
3134 * Facility 45 is a big bin that contains: distinct-operands,
3135 * fast-BCR-serialization, high-word, population-count,
3136 * interlocked-access-1, and load/store-on-condition-1
3137 */
3138 if (!HAVE_FACILITY(45)) {
3139 which = "45";
3140 goto fail;
3141 }
761ea522
RH		 3142 return;
3143
3144 fail:
3145 error_report("%s: missing required facility %s", __func__, which);
3146 exit(EXIT_FAILURE);
48bb3750
RH		 3147}
3148
3149static void tcg_target_init(TCGContext *s)
2827822e 3150{
b2c98d9d 3151 query_s390_facilities();
48bb3750 3152
f46934df
RH		 3153 tcg_target_available_regs[TCG_TYPE_I32] = 0xffff;
3154 tcg_target_available_regs[TCG_TYPE_I64] = 0xffff;
34ef7676
RH		 3155 if (HAVE_FACILITY(VECTOR)) {
3156 tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
3157 tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
3158 }
48bb3750 3159
ccb1bb66 3160 tcg_target_call_clobber_regs = 0;
48bb3750
RH		 3161 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
3162 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R1);
3163 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
3164 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
3165 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
3166 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
f24efee4
RH		 3167 /* The r6 register is technically call-saved, but it's also a parameter
3168 register, so it can get killed by setup for the qemu_st helper. */
3169 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
48bb3750
RH		 3170 /* The return register can be considered call-clobbered. */
3171 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14);
3172
34ef7676
RH		 3173 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V0);
3174 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V1);
3175 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V2);
3176 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V3);
3177 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V4);
3178 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V5);
3179 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V6);
3180 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V7);
3181 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V16);
3182 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V17);
3183 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V18);
3184 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V19);
3185 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V20);
3186 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V21);
3187 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V22);
3188 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V23);
3189 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V24);
3190 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V25);
3191 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V26);
3192 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V27);
3193 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V28);
3194 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V29);
3195 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V30);
3196 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V31);
3197
ccb1bb66 3198 s->reserved_regs = 0;
48bb3750
RH		 3199 tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);
3200 /* XXX many insns can't be used with R0, so we better avoid it for now */
3201 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);
3202 tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
2827822e
AG		 3203}
3204
f167dc37
RH		 3205#define FRAME_SIZE ((int)(TCG_TARGET_CALL_STACK_OFFSET \
3206 + TCG_STATIC_CALL_ARGS_SIZE \
3207 + CPU_TEMP_BUF_NLONGS * sizeof(long)))
3208
48bb3750 3209static void tcg_target_qemu_prologue(TCGContext *s)
2827822e 3210{
48bb3750
RH		 3211 /* stmg %r6,%r15,48(%r15) (save registers) */
3212 tcg_out_insn(s, RXY, STMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15, 48);
3213
a4924e8b 3214 /* aghi %r15,-frame_size */
f167dc37 3215 tcg_out_insn(s, RI, AGHI, TCG_REG_R15, -FRAME_SIZE);
a4924e8b
RH		 3216
3217 tcg_set_frame(s, TCG_REG_CALL_STACK,
3218 TCG_STATIC_CALL_ARGS_SIZE + TCG_TARGET_CALL_STACK_OFFSET,
3219 CPU_TEMP_BUF_NLONGS * sizeof(long));
48bb3750 3220
090d0bfd 3221#ifndef CONFIG_SOFTMMU
b76f21a7 3222 if (guest_base >= 0x80000) {
ccbecb44 3223 tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
48bb3750
RH		 3224 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
3225 }
090d0bfd 3226#endif
48bb3750 3227
cea5f9a2 3228 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
829e1376 3229
cea5f9a2
BS		 3230 /* br %r3 (go to TB) */
3231 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, tcg_target_call_iarg_regs[1]);
48bb3750 3232
46644483
RH		 3233 /*
3234 * Return path for goto_ptr. Set return value to 0, a-la exit_tb,
3235 * and fall through to the rest of the epilogue.
3236 */
c8bc1168 3237 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
46644483
RH		 3238 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, 0);
3239
3240 /* TB epilogue */
79dae4dd 3241 tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
48bb3750 3242
a4924e8b
RH		 3243 /* lmg %r6,%r15,fs+48(%r15) (restore registers) */
3244 tcg_out_insn(s, RXY, LMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15,
f167dc37 3245 FRAME_SIZE + 48);
48bb3750
RH		 3246
3247 /* br %r14 (return) */
3248 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_R14);
2827822e 3249}
f167dc37 3250
28eef8aa
RH		 3251static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
3252{
3253 memset(p, 0x07, count * sizeof(tcg_insn_unit));
3254}
3255
f167dc37 3256typedef struct {
d2e16f2c 3257 DebugFrameHeader h;
f167dc37
RH		 3258 uint8_t fde_def_cfa[4];
3259 uint8_t fde_reg_ofs[18];
3260} DebugFrame;
3261
3262/* We're expecting a 2 byte uleb128 encoded value. */
3263QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
3264
3265#define ELF_HOST_MACHINE EM_S390
3266
d2e16f2c
RH		 3267static const DebugFrame debug_frame = {
3268 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
3269 .h.cie.id = -1,
3270 .h.cie.version = 1,
3271 .h.cie.code_align = 1,
3272 .h.cie.data_align = 8, /* sleb128 8 */
3273 .h.cie.return_column = TCG_REG_R14,
f167dc37
RH		 3274
3275 /* Total FDE size does not include the "len" member. */
d2e16f2c 3276 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
f167dc37
RH		 3277
3278 .fde_def_cfa = {
3279 12, TCG_REG_CALL_STACK, /* DW_CFA_def_cfa %r15, ... */
3280 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
3281 (FRAME_SIZE >> 7)
3282 },
3283 .fde_reg_ofs = {
3284 0x86, 6, /* DW_CFA_offset, %r6, 48 */
3285 0x87, 7, /* DW_CFA_offset, %r7, 56 */
3286 0x88, 8, /* DW_CFA_offset, %r8, 64 */
3287 0x89, 9, /* DW_CFA_offset, %r92, 72 */
3288 0x8a, 10, /* DW_CFA_offset, %r10, 80 */
3289 0x8b, 11, /* DW_CFA_offset, %r11, 88 */
3290 0x8c, 12, /* DW_CFA_offset, %r12, 96 */
3291 0x8d, 13, /* DW_CFA_offset, %r13, 104 */
3292 0x8e, 14, /* DW_CFA_offset, %r14, 112 */
3293 }
3294};
3295
755bf9e5 3296void tcg_register_jit(const void *buf, size_t buf_size)
f167dc37 3297{
f167dc37
RH		 3298 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
3299}
QEMU mirror