]> git.proxmox.com Git - mirror_qemu.git/blob - tcg/s390/tcg-target.c.inc
projects / mirror_qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
iotests: Add mirror-ready-cancel-error test
[mirror_qemu.git] / tcg / s390 / tcg-target.c.inc
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2009 Ulrich Hecht <uli@suse.de>
5 * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
6 * Copyright (c) 2010 Richard Henderson <rth@twiddle.net>
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a copy
9 * of this software and associated documentation files (the "Software"), to deal
10 * in the Software without restriction, including without limitation the rights
11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 * copies of the Software, and to permit persons to whom the Software is
13 * furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 * THE SOFTWARE.
25 */
26
27 /* We only support generating code for 64-bit mode. */
28 #if TCG_TARGET_REG_BITS != 64
29 #error "unsupported code generation mode"
30 #endif
31
32 #include "../tcg-pool.c.inc"
33 #include "elf.h"
34
35 /* ??? The translation blocks produced by TCG are generally small enough to
36 be entirely reachable with a 16-bit displacement. Leaving the option for
37 a 32-bit displacement here Just In Case. */
38 #define USE_LONG_BRANCHES 0
39
40 #define TCG_CT_CONST_S16 0x100
41 #define TCG_CT_CONST_S32 0x200
42 #define TCG_CT_CONST_S33 0x400
43 #define TCG_CT_CONST_ZERO 0x800
44
45 #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16)
46 /*
47 * For softmmu, we need to avoid conflicts with the first 3
48 * argument registers to perform the tlb lookup, and to call
49 * the helper function.
50 */
51 #ifdef CONFIG_SOFTMMU
52 #define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_R2, 3)
53 #else
54 #define SOFTMMU_RESERVE_REGS 0
55 #endif
56
57
58 /* Several places within the instruction set 0 means "no register"
59 rather than TCG_REG_R0. */
60 #define TCG_REG_NONE 0
61
62 /* A scratch register that may be be used throughout the backend. */
63 #define TCG_TMP0 TCG_REG_R1
64
65 /* A scratch register that holds a pointer to the beginning of the TB.
66 We don't need this when we have pc-relative loads with the general
67 instructions extension facility. */
68 #define TCG_REG_TB TCG_REG_R12
69 #define USE_REG_TB (!(s390_facilities & FACILITY_GEN_INST_EXT))
70
71 #ifndef CONFIG_SOFTMMU
72 #define TCG_GUEST_BASE_REG TCG_REG_R13
73 #endif
74
75 /* All of the following instructions are prefixed with their instruction
76 format, and are defined as 8- or 16-bit quantities, even when the two
77 halves of the 16-bit quantity may appear 32 bits apart in the insn.
78 This makes it easy to copy the values from the tables in Appendix B. */
79 typedef enum S390Opcode {
80 RIL_AFI = 0xc209,
81 RIL_AGFI = 0xc208,
82 RIL_ALFI = 0xc20b,
83 RIL_ALGFI = 0xc20a,
84 RIL_BRASL = 0xc005,
85 RIL_BRCL = 0xc004,
86 RIL_CFI = 0xc20d,
87 RIL_CGFI = 0xc20c,
88 RIL_CLFI = 0xc20f,
89 RIL_CLGFI = 0xc20e,
90 RIL_CLRL = 0xc60f,
91 RIL_CLGRL = 0xc60a,
92 RIL_CRL = 0xc60d,
93 RIL_CGRL = 0xc608,
94 RIL_IIHF = 0xc008,
95 RIL_IILF = 0xc009,
96 RIL_LARL = 0xc000,
97 RIL_LGFI = 0xc001,
98 RIL_LGRL = 0xc408,
99 RIL_LLIHF = 0xc00e,
100 RIL_LLILF = 0xc00f,
101 RIL_LRL = 0xc40d,
102 RIL_MSFI = 0xc201,
103 RIL_MSGFI = 0xc200,
104 RIL_NIHF = 0xc00a,
105 RIL_NILF = 0xc00b,
106 RIL_OIHF = 0xc00c,
107 RIL_OILF = 0xc00d,
108 RIL_SLFI = 0xc205,
109 RIL_SLGFI = 0xc204,
110 RIL_XIHF = 0xc006,
111 RIL_XILF = 0xc007,
112
113 RI_AGHI = 0xa70b,
114 RI_AHI = 0xa70a,
115 RI_BRC = 0xa704,
116 RI_CHI = 0xa70e,
117 RI_CGHI = 0xa70f,
118 RI_IIHH = 0xa500,
119 RI_IIHL = 0xa501,
120 RI_IILH = 0xa502,
121 RI_IILL = 0xa503,
122 RI_LGHI = 0xa709,
123 RI_LLIHH = 0xa50c,
124 RI_LLIHL = 0xa50d,
125 RI_LLILH = 0xa50e,
126 RI_LLILL = 0xa50f,
127 RI_MGHI = 0xa70d,
128 RI_MHI = 0xa70c,
129 RI_NIHH = 0xa504,
130 RI_NIHL = 0xa505,
131 RI_NILH = 0xa506,
132 RI_NILL = 0xa507,
133 RI_OIHH = 0xa508,
134 RI_OIHL = 0xa509,
135 RI_OILH = 0xa50a,
136 RI_OILL = 0xa50b,
137
138 RIE_CGIJ = 0xec7c,
139 RIE_CGRJ = 0xec64,
140 RIE_CIJ = 0xec7e,
141 RIE_CLGRJ = 0xec65,
142 RIE_CLIJ = 0xec7f,
143 RIE_CLGIJ = 0xec7d,
144 RIE_CLRJ = 0xec77,
145 RIE_CRJ = 0xec76,
146 RIE_LOCGHI = 0xec46,
147 RIE_RISBG = 0xec55,
148
149 RRE_AGR = 0xb908,
150 RRE_ALGR = 0xb90a,
151 RRE_ALCR = 0xb998,
152 RRE_ALCGR = 0xb988,
153 RRE_CGR = 0xb920,
154 RRE_CLGR = 0xb921,
155 RRE_DLGR = 0xb987,
156 RRE_DLR = 0xb997,
157 RRE_DSGFR = 0xb91d,
158 RRE_DSGR = 0xb90d,
159 RRE_FLOGR = 0xb983,
160 RRE_LGBR = 0xb906,
161 RRE_LCGR = 0xb903,
162 RRE_LGFR = 0xb914,
163 RRE_LGHR = 0xb907,
164 RRE_LGR = 0xb904,
165 RRE_LLGCR = 0xb984,
166 RRE_LLGFR = 0xb916,
167 RRE_LLGHR = 0xb985,
168 RRE_LRVR = 0xb91f,
169 RRE_LRVGR = 0xb90f,
170 RRE_LTGR = 0xb902,
171 RRE_MLGR = 0xb986,
172 RRE_MSGR = 0xb90c,
173 RRE_MSR = 0xb252,
174 RRE_NGR = 0xb980,
175 RRE_OGR = 0xb981,
176 RRE_SGR = 0xb909,
177 RRE_SLGR = 0xb90b,
178 RRE_SLBR = 0xb999,
179 RRE_SLBGR = 0xb989,
180 RRE_XGR = 0xb982,
181
182 RRF_LOCR = 0xb9f2,
183 RRF_LOCGR = 0xb9e2,
184 RRF_NRK = 0xb9f4,
185 RRF_NGRK = 0xb9e4,
186 RRF_ORK = 0xb9f6,
187 RRF_OGRK = 0xb9e6,
188 RRF_SRK = 0xb9f9,
189 RRF_SGRK = 0xb9e9,
190 RRF_SLRK = 0xb9fb,
191 RRF_SLGRK = 0xb9eb,
192 RRF_XRK = 0xb9f7,
193 RRF_XGRK = 0xb9e7,
194
195 RR_AR = 0x1a,
196 RR_ALR = 0x1e,
197 RR_BASR = 0x0d,
198 RR_BCR = 0x07,
199 RR_CLR = 0x15,
200 RR_CR = 0x19,
201 RR_DR = 0x1d,
202 RR_LCR = 0x13,
203 RR_LR = 0x18,
204 RR_LTR = 0x12,
205 RR_NR = 0x14,
206 RR_OR = 0x16,
207 RR_SR = 0x1b,
208 RR_SLR = 0x1f,
209 RR_XR = 0x17,
210
211 RSY_RLL = 0xeb1d,
212 RSY_RLLG = 0xeb1c,
213 RSY_SLLG = 0xeb0d,
214 RSY_SLLK = 0xebdf,
215 RSY_SRAG = 0xeb0a,
216 RSY_SRAK = 0xebdc,
217 RSY_SRLG = 0xeb0c,
218 RSY_SRLK = 0xebde,
219
220 RS_SLL = 0x89,
221 RS_SRA = 0x8a,
222 RS_SRL = 0x88,
223
224 RXY_AG = 0xe308,
225 RXY_AY = 0xe35a,
226 RXY_CG = 0xe320,
227 RXY_CLG = 0xe321,
228 RXY_CLY = 0xe355,
229 RXY_CY = 0xe359,
230 RXY_LAY = 0xe371,
231 RXY_LB = 0xe376,
232 RXY_LG = 0xe304,
233 RXY_LGB = 0xe377,
234 RXY_LGF = 0xe314,
235 RXY_LGH = 0xe315,
236 RXY_LHY = 0xe378,
237 RXY_LLGC = 0xe390,
238 RXY_LLGF = 0xe316,
239 RXY_LLGH = 0xe391,
240 RXY_LMG = 0xeb04,
241 RXY_LRV = 0xe31e,
242 RXY_LRVG = 0xe30f,
243 RXY_LRVH = 0xe31f,
244 RXY_LY = 0xe358,
245 RXY_NG = 0xe380,
246 RXY_OG = 0xe381,
247 RXY_STCY = 0xe372,
248 RXY_STG = 0xe324,
249 RXY_STHY = 0xe370,
250 RXY_STMG = 0xeb24,
251 RXY_STRV = 0xe33e,
252 RXY_STRVG = 0xe32f,
253 RXY_STRVH = 0xe33f,
254 RXY_STY = 0xe350,
255 RXY_XG = 0xe382,
256
257 RX_A = 0x5a,
258 RX_C = 0x59,
259 RX_L = 0x58,
260 RX_LA = 0x41,
261 RX_LH = 0x48,
262 RX_ST = 0x50,
263 RX_STC = 0x42,
264 RX_STH = 0x40,
265
266 NOP = 0x0707,
267 } S390Opcode;
268
269 #ifdef CONFIG_DEBUG_TCG
270 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
271 "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7",
272 "%r8", "%r9", "%r10" "%r11" "%r12" "%r13" "%r14" "%r15"
273 };
274 #endif
275
276 /* Since R6 is a potential argument register, choose it last of the
277 call-saved registers. Likewise prefer the call-clobbered registers
278 in reverse order to maximize the chance of avoiding the arguments. */
279 static const int tcg_target_reg_alloc_order[] = {
280 /* Call saved registers. */
281 TCG_REG_R13,
282 TCG_REG_R12,
283 TCG_REG_R11,
284 TCG_REG_R10,
285 TCG_REG_R9,
286 TCG_REG_R8,
287 TCG_REG_R7,
288 TCG_REG_R6,
289 /* Call clobbered registers. */
290 TCG_REG_R14,
291 TCG_REG_R0,
292 TCG_REG_R1,
293 /* Argument registers, in reverse order of allocation. */
294 TCG_REG_R5,
295 TCG_REG_R4,
296 TCG_REG_R3,
297 TCG_REG_R2,
298 };
299
300 static const int tcg_target_call_iarg_regs[] = {
301 TCG_REG_R2,
302 TCG_REG_R3,
303 TCG_REG_R4,
304 TCG_REG_R5,
305 TCG_REG_R6,
306 };
307
308 static const int tcg_target_call_oarg_regs[] = {
309 TCG_REG_R2,
310 };
311
312 #define S390_CC_EQ 8
313 #define S390_CC_LT 4
314 #define S390_CC_GT 2
315 #define S390_CC_OV 1
316 #define S390_CC_NE (S390_CC_LT | S390_CC_GT)
317 #define S390_CC_LE (S390_CC_LT | S390_CC_EQ)
318 #define S390_CC_GE (S390_CC_GT | S390_CC_EQ)
319 #define S390_CC_NEVER 0
320 #define S390_CC_ALWAYS 15
321
322 /* Condition codes that result from a COMPARE and COMPARE LOGICAL. */
323 static const uint8_t tcg_cond_to_s390_cond[] = {
324 [TCG_COND_EQ] = S390_CC_EQ,
325 [TCG_COND_NE] = S390_CC_NE,
326 [TCG_COND_LT] = S390_CC_LT,
327 [TCG_COND_LE] = S390_CC_LE,
328 [TCG_COND_GT] = S390_CC_GT,
329 [TCG_COND_GE] = S390_CC_GE,
330 [TCG_COND_LTU] = S390_CC_LT,
331 [TCG_COND_LEU] = S390_CC_LE,
332 [TCG_COND_GTU] = S390_CC_GT,
333 [TCG_COND_GEU] = S390_CC_GE,
334 };
335
336 /* Condition codes that result from a LOAD AND TEST. Here, we have no
337 unsigned instruction variation, however since the test is vs zero we
338 can re-map the outcomes appropriately. */
339 static const uint8_t tcg_cond_to_ltr_cond[] = {
340 [TCG_COND_EQ] = S390_CC_EQ,
341 [TCG_COND_NE] = S390_CC_NE,
342 [TCG_COND_LT] = S390_CC_LT,
343 [TCG_COND_LE] = S390_CC_LE,
344 [TCG_COND_GT] = S390_CC_GT,
345 [TCG_COND_GE] = S390_CC_GE,
346 [TCG_COND_LTU] = S390_CC_NEVER,
347 [TCG_COND_LEU] = S390_CC_EQ,
348 [TCG_COND_GTU] = S390_CC_NE,
349 [TCG_COND_GEU] = S390_CC_ALWAYS,
350 };
351
352 #ifdef CONFIG_SOFTMMU
353 static void * const qemu_ld_helpers[16] = {
354 [MO_UB] = helper_ret_ldub_mmu,
355 [MO_SB] = helper_ret_ldsb_mmu,
356 [MO_LEUW] = helper_le_lduw_mmu,
357 [MO_LESW] = helper_le_ldsw_mmu,
358 [MO_LEUL] = helper_le_ldul_mmu,
359 [MO_LESL] = helper_le_ldsl_mmu,
360 [MO_LEQ] = helper_le_ldq_mmu,
361 [MO_BEUW] = helper_be_lduw_mmu,
362 [MO_BESW] = helper_be_ldsw_mmu,
363 [MO_BEUL] = helper_be_ldul_mmu,
364 [MO_BESL] = helper_be_ldsl_mmu,
365 [MO_BEQ] = helper_be_ldq_mmu,
366 };
367
368 static void * const qemu_st_helpers[16] = {
369 [MO_UB] = helper_ret_stb_mmu,
370 [MO_LEUW] = helper_le_stw_mmu,
371 [MO_LEUL] = helper_le_stl_mmu,
372 [MO_LEQ] = helper_le_stq_mmu,
373 [MO_BEUW] = helper_be_stw_mmu,
374 [MO_BEUL] = helper_be_stl_mmu,
375 [MO_BEQ] = helper_be_stq_mmu,
376 };
377 #endif
378
379 static const tcg_insn_unit *tb_ret_addr;
380 uint64_t s390_facilities;
381
382 static bool patch_reloc(tcg_insn_unit *src_rw, int type,
383 intptr_t value, intptr_t addend)
384 {
385 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
386 intptr_t pcrel2;
387 uint32_t old;
388
389 value += addend;
390 pcrel2 = (tcg_insn_unit *)value - src_rx;
391
392 switch (type) {
393 case R_390_PC16DBL:
394 if (pcrel2 == (int16_t)pcrel2) {
395 tcg_patch16(src_rw, pcrel2);
396 return true;
397 }
398 break;
399 case R_390_PC32DBL:
400 if (pcrel2 == (int32_t)pcrel2) {
401 tcg_patch32(src_rw, pcrel2);
402 return true;
403 }
404 break;
405 case R_390_20:
406 if (value == sextract64(value, 0, 20)) {
407 old = *(uint32_t *)src_rw & 0xf00000ff;
408 old |= ((value & 0xfff) << 16) | ((value & 0xff000) >> 4);
409 tcg_patch32(src_rw, old);
410 return true;
411 }
412 break;
413 default:
414 g_assert_not_reached();
415 }
416 return false;
417 }
418
419 /* Test if a constant matches the constraint. */
420 static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
421 {
422 if (ct & TCG_CT_CONST) {
423 return 1;
424 }
425
426 if (type == TCG_TYPE_I32) {
427 val = (int32_t)val;
428 }
429
430 /* The following are mutually exclusive. */
431 if (ct & TCG_CT_CONST_S16) {
432 return val == (int16_t)val;
433 } else if (ct & TCG_CT_CONST_S32) {
434 return val == (int32_t)val;
435 } else if (ct & TCG_CT_CONST_S33) {
436 return val >= -0xffffffffll && val <= 0xffffffffll;
437 } else if (ct & TCG_CT_CONST_ZERO) {
438 return val == 0;
439 }
440
441 return 0;
442 }
443
444 /* Emit instructions according to the given instruction format. */
445
446 static void tcg_out_insn_RR(TCGContext *s, S390Opcode op, TCGReg r1, TCGReg r2)
447 {
448 tcg_out16(s, (op << 8) | (r1 << 4) | r2);
449 }
450
451 static void tcg_out_insn_RRE(TCGContext *s, S390Opcode op,
452 TCGReg r1, TCGReg r2)
453 {
454 tcg_out32(s, (op << 16) | (r1 << 4) | r2);
455 }
456
457 static void tcg_out_insn_RRF(TCGContext *s, S390Opcode op,
458 TCGReg r1, TCGReg r2, int m3)
459 {
460 tcg_out32(s, (op << 16) | (m3 << 12) | (r1 << 4) | r2);
461 }
462
463 static void tcg_out_insn_RI(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
464 {
465 tcg_out32(s, (op << 16) | (r1 << 20) | (i2 & 0xffff));
466 }
467
468 static void tcg_out_insn_RIE(TCGContext *s, S390Opcode op, TCGReg r1,
469 int i2, int m3)
470 {
471 tcg_out16(s, (op & 0xff00) | (r1 << 4) | m3);
472 tcg_out32(s, (i2 << 16) | (op & 0xff));
473 }
474
475 static void tcg_out_insn_RIL(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
476 {
477 tcg_out16(s, op | (r1 << 4));
478 tcg_out32(s, i2);
479 }
480
481 static void tcg_out_insn_RS(TCGContext *s, S390Opcode op, TCGReg r1,
482 TCGReg b2, TCGReg r3, int disp)
483 {
484 tcg_out32(s, (op << 24) | (r1 << 20) | (r3 << 16) | (b2 << 12)
485 | (disp & 0xfff));
486 }
487
488 static void tcg_out_insn_RSY(TCGContext *s, S390Opcode op, TCGReg r1,
489 TCGReg b2, TCGReg r3, int disp)
490 {
491 tcg_out16(s, (op & 0xff00) | (r1 << 4) | r3);
492 tcg_out32(s, (op & 0xff) | (b2 << 28)
493 | ((disp & 0xfff) << 16) | ((disp & 0xff000) >> 4));
494 }
495
496 #define tcg_out_insn_RX tcg_out_insn_RS
497 #define tcg_out_insn_RXY tcg_out_insn_RSY
498
499 /* Emit an opcode with "type-checking" of the format. */
500 #define tcg_out_insn(S, FMT, OP, ...) \
501 glue(tcg_out_insn_,FMT)(S, glue(glue(FMT,_),OP), ## __VA_ARGS__)
502
503
504 /* emit 64-bit shifts */
505 static void tcg_out_sh64(TCGContext* s, S390Opcode op, TCGReg dest,
506 TCGReg src, TCGReg sh_reg, int sh_imm)
507 {
508 tcg_out_insn_RSY(s, op, dest, sh_reg, src, sh_imm);
509 }
510
511 /* emit 32-bit shifts */
512 static void tcg_out_sh32(TCGContext* s, S390Opcode op, TCGReg dest,
513 TCGReg sh_reg, int sh_imm)
514 {
515 tcg_out_insn_RS(s, op, dest, sh_reg, 0, sh_imm);
516 }
517
518 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
519 {
520 if (src != dst) {
521 if (type == TCG_TYPE_I32) {
522 tcg_out_insn(s, RR, LR, dst, src);
523 } else {
524 tcg_out_insn(s, RRE, LGR, dst, src);
525 }
526 }
527 return true;
528 }
529
530 static const S390Opcode lli_insns[4] = {
531 RI_LLILL, RI_LLILH, RI_LLIHL, RI_LLIHH
532 };
533
534 static bool maybe_out_small_movi(TCGContext *s, TCGType type,
535 TCGReg ret, tcg_target_long sval)
536 {
537 tcg_target_ulong uval = sval;
538 int i;
539
540 if (type == TCG_TYPE_I32) {
541 uval = (uint32_t)sval;
542 sval = (int32_t)sval;
543 }
544
545 /* Try all 32-bit insns that can load it in one go. */
546 if (sval >= -0x8000 && sval < 0x8000) {
547 tcg_out_insn(s, RI, LGHI, ret, sval);
548 return true;
549 }
550
551 for (i = 0; i < 4; i++) {
552 tcg_target_long mask = 0xffffull << i*16;
553 if ((uval & mask) == uval) {
554 tcg_out_insn_RI(s, lli_insns[i], ret, uval >> i*16);
555 return true;
556 }
557 }
558
559 return false;
560 }
561
562 /* load a register with an immediate value */
563 static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
564 tcg_target_long sval, bool in_prologue)
565 {
566 tcg_target_ulong uval;
567
568 /* Try all 32-bit insns that can load it in one go. */
569 if (maybe_out_small_movi(s, type, ret, sval)) {
570 return;
571 }
572
573 uval = sval;
574 if (type == TCG_TYPE_I32) {
575 uval = (uint32_t)sval;
576 sval = (int32_t)sval;
577 }
578
579 /* Try all 48-bit insns that can load it in one go. */
580 if (s390_facilities & FACILITY_EXT_IMM) {
581 if (sval == (int32_t)sval) {
582 tcg_out_insn(s, RIL, LGFI, ret, sval);
583 return;
584 }
585 if (uval <= 0xffffffff) {
586 tcg_out_insn(s, RIL, LLILF, ret, uval);
587 return;
588 }
589 if ((uval & 0xffffffff) == 0) {
590 tcg_out_insn(s, RIL, LLIHF, ret, uval >> 32);
591 return;
592 }
593 }
594
595 /* Try for PC-relative address load. For odd addresses,
596 attempt to use an offset from the start of the TB. */
597 if ((sval & 1) == 0) {
598 ptrdiff_t off = tcg_pcrel_diff(s, (void *)sval) >> 1;
599 if (off == (int32_t)off) {
600 tcg_out_insn(s, RIL, LARL, ret, off);
601 return;
602 }
603 } else if (USE_REG_TB && !in_prologue) {
604 ptrdiff_t off = tcg_tbrel_diff(s, (void *)sval);
605 if (off == sextract64(off, 0, 20)) {
606 /* This is certain to be an address within TB, and therefore
607 OFF will be negative; don't try RX_LA. */
608 tcg_out_insn(s, RXY, LAY, ret, TCG_REG_TB, TCG_REG_NONE, off);
609 return;
610 }
611 }
612
613 /* A 32-bit unsigned value can be loaded in 2 insns. And given
614 that LLILL, LLIHL, LLILF above did not succeed, we know that
615 both insns are required. */
616 if (uval <= 0xffffffff) {
617 tcg_out_insn(s, RI, LLILL, ret, uval);
618 tcg_out_insn(s, RI, IILH, ret, uval >> 16);
619 return;
620 }
621
622 /* Otherwise, stuff it in the constant pool. */
623 if (s390_facilities & FACILITY_GEN_INST_EXT) {
624 tcg_out_insn(s, RIL, LGRL, ret, 0);
625 new_pool_label(s, sval, R_390_PC32DBL, s->code_ptr - 2, 2);
626 } else if (USE_REG_TB && !in_prologue) {
627 tcg_out_insn(s, RXY, LG, ret, TCG_REG_TB, TCG_REG_NONE, 0);
628 new_pool_label(s, sval, R_390_20, s->code_ptr - 2,
629 tcg_tbrel_diff(s, NULL));
630 } else {
631 TCGReg base = ret ? ret : TCG_TMP0;
632 tcg_out_insn(s, RIL, LARL, base, 0);
633 new_pool_label(s, sval, R_390_PC32DBL, s->code_ptr - 2, 2);
634 tcg_out_insn(s, RXY, LG, ret, base, TCG_REG_NONE, 0);
635 }
636 }
637
638 static void tcg_out_movi(TCGContext *s, TCGType type,
639 TCGReg ret, tcg_target_long sval)
640 {
641 tcg_out_movi_int(s, type, ret, sval, false);
642 }
643
644 /* Emit a load/store type instruction. Inputs are:
645 DATA: The register to be loaded or stored.
646 BASE+OFS: The effective address.
647 OPC_RX: If the operation has an RX format opcode (e.g. STC), otherwise 0.
648 OPC_RXY: The RXY format opcode for the operation (e.g. STCY). */
649
650 static void tcg_out_mem(TCGContext *s, S390Opcode opc_rx, S390Opcode opc_rxy,
651 TCGReg data, TCGReg base, TCGReg index,
652 tcg_target_long ofs)
653 {
654 if (ofs < -0x80000 || ofs >= 0x80000) {
655 /* Combine the low 20 bits of the offset with the actual load insn;
656 the high 44 bits must come from an immediate load. */
657 tcg_target_long low = ((ofs & 0xfffff) ^ 0x80000) - 0x80000;
658 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - low);
659 ofs = low;
660
661 /* If we were already given an index register, add it in. */
662 if (index != TCG_REG_NONE) {
663 tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
664 }
665 index = TCG_TMP0;
666 }
667
668 if (opc_rx && ofs >= 0 && ofs < 0x1000) {
669 tcg_out_insn_RX(s, opc_rx, data, base, index, ofs);
670 } else {
671 tcg_out_insn_RXY(s, opc_rxy, data, base, index, ofs);
672 }
673 }
674
675
676 /* load data without address translation or endianness conversion */
677 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data,
678 TCGReg base, intptr_t ofs)
679 {
680 if (type == TCG_TYPE_I32) {
681 tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs);
682 } else {
683 tcg_out_mem(s, 0, RXY_LG, data, base, TCG_REG_NONE, ofs);
684 }
685 }
686
687 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg data,
688 TCGReg base, intptr_t ofs)
689 {
690 if (type == TCG_TYPE_I32) {
691 tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs);
692 } else {
693 tcg_out_mem(s, 0, RXY_STG, data, base, TCG_REG_NONE, ofs);
694 }
695 }
696
697 static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
698 TCGReg base, intptr_t ofs)
699 {
700 return false;
701 }
702
703 /* load data from an absolute host address */
704 static void tcg_out_ld_abs(TCGContext *s, TCGType type,
705 TCGReg dest, const void *abs)
706 {
707 intptr_t addr = (intptr_t)abs;
708
709 if ((s390_facilities & FACILITY_GEN_INST_EXT) && !(addr & 1)) {
710 ptrdiff_t disp = tcg_pcrel_diff(s, abs) >> 1;
711 if (disp == (int32_t)disp) {
712 if (type == TCG_TYPE_I32) {
713 tcg_out_insn(s, RIL, LRL, dest, disp);
714 } else {
715 tcg_out_insn(s, RIL, LGRL, dest, disp);
716 }
717 return;
718 }
719 }
720 if (USE_REG_TB) {
721 ptrdiff_t disp = tcg_tbrel_diff(s, abs);
722 if (disp == sextract64(disp, 0, 20)) {
723 tcg_out_ld(s, type, dest, TCG_REG_TB, disp);
724 return;
725 }
726 }
727
728 tcg_out_movi(s, TCG_TYPE_PTR, dest, addr & ~0xffff);
729 tcg_out_ld(s, type, dest, dest, addr & 0xffff);
730 }
731
732 static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src,
733 int msb, int lsb, int ofs, int z)
734 {
735 /* Format RIE-f */
736 tcg_out16(s, (RIE_RISBG & 0xff00) | (dest << 4) | src);
737 tcg_out16(s, (msb << 8) | (z << 7) | lsb);
738 tcg_out16(s, (ofs << 8) | (RIE_RISBG & 0xff));
739 }
740
741 static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
742 {
743 if (s390_facilities & FACILITY_EXT_IMM) {
744 tcg_out_insn(s, RRE, LGBR, dest, src);
745 return;
746 }
747
748 if (type == TCG_TYPE_I32) {
749 if (dest == src) {
750 tcg_out_sh32(s, RS_SLL, dest, TCG_REG_NONE, 24);
751 } else {
752 tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 24);
753 }
754 tcg_out_sh32(s, RS_SRA, dest, TCG_REG_NONE, 24);
755 } else {
756 tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 56);
757 tcg_out_sh64(s, RSY_SRAG, dest, dest, TCG_REG_NONE, 56);
758 }
759 }
760
761 static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
762 {
763 if (s390_facilities & FACILITY_EXT_IMM) {
764 tcg_out_insn(s, RRE, LLGCR, dest, src);
765 return;
766 }
767
768 if (dest == src) {
769 tcg_out_movi(s, type, TCG_TMP0, 0xff);
770 src = TCG_TMP0;
771 } else {
772 tcg_out_movi(s, type, dest, 0xff);
773 }
774 if (type == TCG_TYPE_I32) {
775 tcg_out_insn(s, RR, NR, dest, src);
776 } else {
777 tcg_out_insn(s, RRE, NGR, dest, src);
778 }
779 }
780
781 static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
782 {
783 if (s390_facilities & FACILITY_EXT_IMM) {
784 tcg_out_insn(s, RRE, LGHR, dest, src);
785 return;
786 }
787
788 if (type == TCG_TYPE_I32) {
789 if (dest == src) {
790 tcg_out_sh32(s, RS_SLL, dest, TCG_REG_NONE, 16);
791 } else {
792 tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 16);
793 }
794 tcg_out_sh32(s, RS_SRA, dest, TCG_REG_NONE, 16);
795 } else {
796 tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 48);
797 tcg_out_sh64(s, RSY_SRAG, dest, dest, TCG_REG_NONE, 48);
798 }
799 }
800
801 static void tgen_ext16u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
802 {
803 if (s390_facilities & FACILITY_EXT_IMM) {
804 tcg_out_insn(s, RRE, LLGHR, dest, src);
805 return;
806 }
807
808 if (dest == src) {
809 tcg_out_movi(s, type, TCG_TMP0, 0xffff);
810 src = TCG_TMP0;
811 } else {
812 tcg_out_movi(s, type, dest, 0xffff);
813 }
814 if (type == TCG_TYPE_I32) {
815 tcg_out_insn(s, RR, NR, dest, src);
816 } else {
817 tcg_out_insn(s, RRE, NGR, dest, src);
818 }
819 }
820
821 static inline void tgen_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
822 {
823 tcg_out_insn(s, RRE, LGFR, dest, src);
824 }
825
826 static inline void tgen_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
827 {
828 tcg_out_insn(s, RRE, LLGFR, dest, src);
829 }
830
831 /* Accept bit patterns like these:
832 0....01....1
833 1....10....0
834 1..10..01..1
835 0..01..10..0
836 Copied from gcc sources. */
837 static inline bool risbg_mask(uint64_t c)
838 {
839 uint64_t lsb;
840 /* We don't change the number of transitions by inverting,
841 so make sure we start with the LSB zero. */
842 if (c & 1) {
843 c = ~c;
844 }
845 /* Reject all zeros or all ones. */
846 if (c == 0) {
847 return false;
848 }
849 /* Find the first transition. */
850 lsb = c & -c;
851 /* Invert to look for a second transition. */
852 c = ~c;
853 /* Erase the first transition. */
854 c &= -lsb;
855 /* Find the second transition, if any. */
856 lsb = c & -c;
857 /* Match if all the bits are 1's, or if c is zero. */
858 return c == -lsb;
859 }
860
861 static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
862 {
863 int msb, lsb;
864 if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
865 /* Achieve wraparound by swapping msb and lsb. */
866 msb = 64 - ctz64(~val);
867 lsb = clz64(~val) - 1;
868 } else {
869 msb = clz64(val);
870 lsb = 63 - ctz64(val);
871 }
872 tcg_out_risbg(s, out, in, msb, lsb, 0, 1);
873 }
874
875 static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
876 {
877 static const S390Opcode ni_insns[4] = {
878 RI_NILL, RI_NILH, RI_NIHL, RI_NIHH
879 };
880 static const S390Opcode nif_insns[2] = {
881 RIL_NILF, RIL_NIHF
882 };
883 uint64_t valid = (type == TCG_TYPE_I32 ? 0xffffffffull : -1ull);
884 int i;
885
886 /* Look for the zero-extensions. */
887 if ((val & valid) == 0xffffffff) {
888 tgen_ext32u(s, dest, dest);
889 return;
890 }
891 if (s390_facilities & FACILITY_EXT_IMM) {
892 if ((val & valid) == 0xff) {
893 tgen_ext8u(s, TCG_TYPE_I64, dest, dest);
894 return;
895 }
896 if ((val & valid) == 0xffff) {
897 tgen_ext16u(s, TCG_TYPE_I64, dest, dest);
898 return;
899 }
900 }
901
902 /* Try all 32-bit insns that can perform it in one go. */
903 for (i = 0; i < 4; i++) {
904 tcg_target_ulong mask = ~(0xffffull << i*16);
905 if (((val | ~valid) & mask) == mask) {
906 tcg_out_insn_RI(s, ni_insns[i], dest, val >> i*16);
907 return;
908 }
909 }
910
911 /* Try all 48-bit insns that can perform it in one go. */
912 if (s390_facilities & FACILITY_EXT_IMM) {
913 for (i = 0; i < 2; i++) {
914 tcg_target_ulong mask = ~(0xffffffffull << i*32);
915 if (((val | ~valid) & mask) == mask) {
916 tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i*32);
917 return;
918 }
919 }
920 }
921 if ((s390_facilities & FACILITY_GEN_INST_EXT) && risbg_mask(val)) {
922 tgen_andi_risbg(s, dest, dest, val);
923 return;
924 }
925
926 /* Use the constant pool if USE_REG_TB, but not for small constants. */
927 if (USE_REG_TB) {
928 if (!maybe_out_small_movi(s, type, TCG_TMP0, val)) {
929 tcg_out_insn(s, RXY, NG, dest, TCG_REG_TB, TCG_REG_NONE, 0);
930 new_pool_label(s, val & valid, R_390_20, s->code_ptr - 2,
931 tcg_tbrel_diff(s, NULL));
932 return;
933 }
934 } else {
935 tcg_out_movi(s, type, TCG_TMP0, val);
936 }
937 if (type == TCG_TYPE_I32) {
938 tcg_out_insn(s, RR, NR, dest, TCG_TMP0);
939 } else {
940 tcg_out_insn(s, RRE, NGR, dest, TCG_TMP0);
941 }
942 }
943
944 static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
945 {
946 static const S390Opcode oi_insns[4] = {
947 RI_OILL, RI_OILH, RI_OIHL, RI_OIHH
948 };
949 static const S390Opcode oif_insns[2] = {
950 RIL_OILF, RIL_OIHF
951 };
952
953 int i;
954
955 /* Look for no-op. */
956 if (unlikely(val == 0)) {
957 return;
958 }
959
960 /* Try all 32-bit insns that can perform it in one go. */
961 for (i = 0; i < 4; i++) {
962 tcg_target_ulong mask = (0xffffull << i*16);
963 if ((val & mask) != 0 && (val & ~mask) == 0) {
964 tcg_out_insn_RI(s, oi_insns[i], dest, val >> i*16);
965 return;
966 }
967 }
968
969 /* Try all 48-bit insns that can perform it in one go. */
970 if (s390_facilities & FACILITY_EXT_IMM) {
971 for (i = 0; i < 2; i++) {
972 tcg_target_ulong mask = (0xffffffffull << i*32);
973 if ((val & mask) != 0 && (val & ~mask) == 0) {
974 tcg_out_insn_RIL(s, oif_insns[i], dest, val >> i*32);
975 return;
976 }
977 }
978 }
979
980 /* Use the constant pool if USE_REG_TB, but not for small constants. */
981 if (maybe_out_small_movi(s, type, TCG_TMP0, val)) {
982 if (type == TCG_TYPE_I32) {
983 tcg_out_insn(s, RR, OR, dest, TCG_TMP0);
984 } else {
985 tcg_out_insn(s, RRE, OGR, dest, TCG_TMP0);
986 }
987 } else if (USE_REG_TB) {
988 tcg_out_insn(s, RXY, OG, dest, TCG_REG_TB, TCG_REG_NONE, 0);
989 new_pool_label(s, val, R_390_20, s->code_ptr - 2,
990 tcg_tbrel_diff(s, NULL));
991 } else {
992 /* Perform the OR via sequential modifications to the high and
993 low parts. Do this via recursion to handle 16-bit vs 32-bit
994 masks in each half. */
995 tcg_debug_assert(s390_facilities & FACILITY_EXT_IMM);
996 tgen_ori(s, type, dest, val & 0x00000000ffffffffull);
997 tgen_ori(s, type, dest, val & 0xffffffff00000000ull);
998 }
999 }
1000
1001 static void tgen_xori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
1002 {
1003 /* Try all 48-bit insns that can perform it in one go. */
1004 if (s390_facilities & FACILITY_EXT_IMM) {
1005 if ((val & 0xffffffff00000000ull) == 0) {
1006 tcg_out_insn(s, RIL, XILF, dest, val);
1007 return;
1008 }
1009 if ((val & 0x00000000ffffffffull) == 0) {
1010 tcg_out_insn(s, RIL, XIHF, dest, val >> 32);
1011 return;
1012 }
1013 }
1014
1015 /* Use the constant pool if USE_REG_TB, but not for small constants. */
1016 if (maybe_out_small_movi(s, type, TCG_TMP0, val)) {
1017 if (type == TCG_TYPE_I32) {
1018 tcg_out_insn(s, RR, XR, dest, TCG_TMP0);
1019 } else {
1020 tcg_out_insn(s, RRE, XGR, dest, TCG_TMP0);
1021 }
1022 } else if (USE_REG_TB) {
1023 tcg_out_insn(s, RXY, XG, dest, TCG_REG_TB, TCG_REG_NONE, 0);
1024 new_pool_label(s, val, R_390_20, s->code_ptr - 2,
1025 tcg_tbrel_diff(s, NULL));
1026 } else {
1027 /* Perform the xor by parts. */
1028 tcg_debug_assert(s390_facilities & FACILITY_EXT_IMM);
1029 if (val & 0xffffffff) {
1030 tcg_out_insn(s, RIL, XILF, dest, val);
1031 }
1032 if (val > 0xffffffff) {
1033 tcg_out_insn(s, RIL, XIHF, dest, val >> 32);
1034 }
1035 }
1036 }
1037
1038 static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
1039 TCGArg c2, bool c2const, bool need_carry)
1040 {
1041 bool is_unsigned = is_unsigned_cond(c);
1042 S390Opcode op;
1043
1044 if (c2const) {
1045 if (c2 == 0) {
1046 if (!(is_unsigned && need_carry)) {
1047 if (type == TCG_TYPE_I32) {
1048 tcg_out_insn(s, RR, LTR, r1, r1);
1049 } else {
1050 tcg_out_insn(s, RRE, LTGR, r1, r1);
1051 }
1052 return tcg_cond_to_ltr_cond[c];
1053 }
1054 }
1055
1056 if (!is_unsigned && c2 == (int16_t)c2) {
1057 op = (type == TCG_TYPE_I32 ? RI_CHI : RI_CGHI);
1058 tcg_out_insn_RI(s, op, r1, c2);
1059 goto exit;
1060 }
1061
1062 if (s390_facilities & FACILITY_EXT_IMM) {
1063 if (type == TCG_TYPE_I32) {
1064 op = (is_unsigned ? RIL_CLFI : RIL_CFI);
1065 tcg_out_insn_RIL(s, op, r1, c2);
1066 goto exit;
1067 } else if (c2 == (is_unsigned ? (TCGArg)(uint32_t)c2 : (TCGArg)(int32_t)c2)) {
1068 op = (is_unsigned ? RIL_CLGFI : RIL_CGFI);
1069 tcg_out_insn_RIL(s, op, r1, c2);
1070 goto exit;
1071 }
1072 }
1073
1074 /* Use the constant pool, but not for small constants. */
1075 if (maybe_out_small_movi(s, type, TCG_TMP0, c2)) {
1076 c2 = TCG_TMP0;
1077 /* fall through to reg-reg */
1078 } else if (USE_REG_TB) {
1079 if (type == TCG_TYPE_I32) {
1080 op = (is_unsigned ? RXY_CLY : RXY_CY);
1081 tcg_out_insn_RXY(s, op, r1, TCG_REG_TB, TCG_REG_NONE, 0);
1082 new_pool_label(s, (uint32_t)c2, R_390_20, s->code_ptr - 2,
1083 4 - tcg_tbrel_diff(s, NULL));
1084 } else {
1085 op = (is_unsigned ? RXY_CLG : RXY_CG);
1086 tcg_out_insn_RXY(s, op, r1, TCG_REG_TB, TCG_REG_NONE, 0);
1087 new_pool_label(s, c2, R_390_20, s->code_ptr - 2,
1088 tcg_tbrel_diff(s, NULL));
1089 }
1090 goto exit;
1091 } else {
1092 if (type == TCG_TYPE_I32) {
1093 op = (is_unsigned ? RIL_CLRL : RIL_CRL);
1094 tcg_out_insn_RIL(s, op, r1, 0);
1095 new_pool_label(s, (uint32_t)c2, R_390_PC32DBL,
1096 s->code_ptr - 2, 2 + 4);
1097 } else {
1098 op = (is_unsigned ? RIL_CLGRL : RIL_CGRL);
1099 tcg_out_insn_RIL(s, op, r1, 0);
1100 new_pool_label(s, c2, R_390_PC32DBL, s->code_ptr - 2, 2);
1101 }
1102 goto exit;
1103 }
1104 }
1105
1106 if (type == TCG_TYPE_I32) {
1107 op = (is_unsigned ? RR_CLR : RR_CR);
1108 tcg_out_insn_RR(s, op, r1, c2);
1109 } else {
1110 op = (is_unsigned ? RRE_CLGR : RRE_CGR);
1111 tcg_out_insn_RRE(s, op, r1, c2);
1112 }
1113
1114 exit:
1115 return tcg_cond_to_s390_cond[c];
1116 }
1117
1118 static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
1119 TCGReg dest, TCGReg c1, TCGArg c2, int c2const)
1120 {
1121 int cc;
1122 bool have_loc;
1123
1124 /* With LOC2, we can always emit the minimum 3 insns. */
1125 if (s390_facilities & FACILITY_LOAD_ON_COND2) {
1126 /* Emit: d = 0, d = (cc ? 1 : d). */
1127 cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
1128 tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
1129 tcg_out_insn(s, RIE, LOCGHI, dest, 1, cc);
1130 return;
1131 }
1132
1133 have_loc = (s390_facilities & FACILITY_LOAD_ON_COND) != 0;
1134
1135 /* For HAVE_LOC, only the paths through GTU/GT/LEU/LE are smaller. */
1136 restart:
1137 switch (cond) {
1138 case TCG_COND_NE:
1139 /* X != 0 is X > 0. */
1140 if (c2const && c2 == 0) {
1141 cond = TCG_COND_GTU;
1142 } else {
1143 break;
1144 }
1145 /* fallthru */
1146
1147 case TCG_COND_GTU:
1148 case TCG_COND_GT:
1149 /* The result of a compare has CC=2 for GT and CC=3 unused.
1150 ADD LOGICAL WITH CARRY considers (CC & 2) the carry bit. */
1151 tgen_cmp(s, type, cond, c1, c2, c2const, true);
1152 tcg_out_movi(s, type, dest, 0);
1153 tcg_out_insn(s, RRE, ALCGR, dest, dest);
1154 return;
1155
1156 case TCG_COND_EQ:
1157 /* X == 0 is X <= 0. */
1158 if (c2const && c2 == 0) {
1159 cond = TCG_COND_LEU;
1160 } else {
1161 break;
1162 }
1163 /* fallthru */
1164
1165 case TCG_COND_LEU:
1166 case TCG_COND_LE:
1167 /* As above, but we're looking for borrow, or !carry.
1168 The second insn computes d - d - borrow, or -1 for true
1169 and 0 for false. So we must mask to 1 bit afterward. */
1170 tgen_cmp(s, type, cond, c1, c2, c2const, true);
1171 tcg_out_insn(s, RRE, SLBGR, dest, dest);
1172 tgen_andi(s, type, dest, 1);
1173 return;
1174
1175 case TCG_COND_GEU:
1176 case TCG_COND_LTU:
1177 case TCG_COND_LT:
1178 case TCG_COND_GE:
1179 /* Swap operands so that we can use LEU/GTU/GT/LE. */
1180 if (c2const) {
1181 if (have_loc) {
1182 break;
1183 }
1184 tcg_out_movi(s, type, TCG_TMP0, c2);
1185 c2 = c1;
1186 c2const = 0;
1187 c1 = TCG_TMP0;
1188 } else {
1189 TCGReg t = c1;
1190 c1 = c2;
1191 c2 = t;
1192 }
1193 cond = tcg_swap_cond(cond);
1194 goto restart;
1195
1196 default:
1197 g_assert_not_reached();
1198 }
1199
1200 cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
1201 if (have_loc) {
1202 /* Emit: d = 0, t = 1, d = (cc ? t : d). */
1203 tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
1204 tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 1);
1205 tcg_out_insn(s, RRF, LOCGR, dest, TCG_TMP0, cc);
1206 } else {
1207 /* Emit: d = 1; if (cc) goto over; d = 0; over: */
1208 tcg_out_movi(s, type, dest, 1);
1209 tcg_out_insn(s, RI, BRC, cc, (4 + 4) >> 1);
1210 tcg_out_movi(s, type, dest, 0);
1211 }
1212 }
1213
1214 static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, TCGReg dest,
1215 TCGReg c1, TCGArg c2, int c2const,
1216 TCGArg v3, int v3const)
1217 {
1218 int cc;
1219 if (s390_facilities & FACILITY_LOAD_ON_COND) {
1220 cc = tgen_cmp(s, type, c, c1, c2, c2const, false);
1221 if (v3const) {
1222 tcg_out_insn(s, RIE, LOCGHI, dest, v3, cc);
1223 } else {
1224 tcg_out_insn(s, RRF, LOCGR, dest, v3, cc);
1225 }
1226 } else {
1227 c = tcg_invert_cond(c);
1228 cc = tgen_cmp(s, type, c, c1, c2, c2const, false);
1229
1230 /* Emit: if (cc) goto over; dest = r3; over: */
1231 tcg_out_insn(s, RI, BRC, cc, (4 + 4) >> 1);
1232 tcg_out_insn(s, RRE, LGR, dest, v3);
1233 }
1234 }
1235
1236 static void tgen_clz(TCGContext *s, TCGReg dest, TCGReg a1,
1237 TCGArg a2, int a2const)
1238 {
1239 /* Since this sets both R and R+1, we have no choice but to store the
1240 result into R0, allowing R1 == TCG_TMP0 to be clobbered as well. */
1241 QEMU_BUILD_BUG_ON(TCG_TMP0 != TCG_REG_R1);
1242 tcg_out_insn(s, RRE, FLOGR, TCG_REG_R0, a1);
1243
1244 if (a2const && a2 == 64) {
1245 tcg_out_mov(s, TCG_TYPE_I64, dest, TCG_REG_R0);
1246 } else {
1247 if (a2const) {
1248 tcg_out_movi(s, TCG_TYPE_I64, dest, a2);
1249 } else {
1250 tcg_out_mov(s, TCG_TYPE_I64, dest, a2);
1251 }
1252 if (s390_facilities & FACILITY_LOAD_ON_COND) {
1253 /* Emit: if (one bit found) dest = r0. */
1254 tcg_out_insn(s, RRF, LOCGR, dest, TCG_REG_R0, 2);
1255 } else {
1256 /* Emit: if (no one bit found) goto over; dest = r0; over: */
1257 tcg_out_insn(s, RI, BRC, 8, (4 + 4) >> 1);
1258 tcg_out_insn(s, RRE, LGR, dest, TCG_REG_R0);
1259 }
1260 }
1261 }
1262
1263 static void tgen_deposit(TCGContext *s, TCGReg dest, TCGReg src,
1264 int ofs, int len, int z)
1265 {
1266 int lsb = (63 - ofs);
1267 int msb = lsb - (len - 1);
1268 tcg_out_risbg(s, dest, src, msb, lsb, ofs, z);
1269 }
1270
1271 static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src,
1272 int ofs, int len)
1273 {
1274 tcg_out_risbg(s, dest, src, 64 - len, 63, 64 - ofs, 1);
1275 }
1276
1277 static void tgen_gotoi(TCGContext *s, int cc, const tcg_insn_unit *dest)
1278 {
1279 ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
1280 if (off == (int16_t)off) {
1281 tcg_out_insn(s, RI, BRC, cc, off);
1282 } else if (off == (int32_t)off) {
1283 tcg_out_insn(s, RIL, BRCL, cc, off);
1284 } else {
1285 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
1286 tcg_out_insn(s, RR, BCR, cc, TCG_TMP0);
1287 }
1288 }
1289
1290 static void tgen_branch(TCGContext *s, int cc, TCGLabel *l)
1291 {
1292 if (l->has_value) {
1293 tgen_gotoi(s, cc, l->u.value_ptr);
1294 } else if (USE_LONG_BRANCHES) {
1295 tcg_out16(s, RIL_BRCL | (cc << 4));
1296 tcg_out_reloc(s, s->code_ptr, R_390_PC32DBL, l, 2);
1297 s->code_ptr += 2;
1298 } else {
1299 tcg_out16(s, RI_BRC | (cc << 4));
1300 tcg_out_reloc(s, s->code_ptr, R_390_PC16DBL, l, 2);
1301 s->code_ptr += 1;
1302 }
1303 }
1304
1305 static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc,
1306 TCGReg r1, TCGReg r2, TCGLabel *l)
1307 {
1308 tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
1309 tcg_out16(s, (opc & 0xff00) | (r1 << 4) | r2);
1310 tcg_out16(s, 0);
1311 tcg_out16(s, cc << 12 | (opc & 0xff));
1312 }
1313
1314 static void tgen_compare_imm_branch(TCGContext *s, S390Opcode opc, int cc,
1315 TCGReg r1, int i2, TCGLabel *l)
1316 {
1317 tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
1318 tcg_out16(s, (opc & 0xff00) | (r1 << 4) | cc);
1319 tcg_out16(s, 0);
1320 tcg_out16(s, (i2 << 8) | (opc & 0xff));
1321 }
1322
1323 static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
1324 TCGReg r1, TCGArg c2, int c2const, TCGLabel *l)
1325 {
1326 int cc;
1327
1328 if (s390_facilities & FACILITY_GEN_INST_EXT) {
1329 bool is_unsigned = is_unsigned_cond(c);
1330 bool in_range;
1331 S390Opcode opc;
1332
1333 cc = tcg_cond_to_s390_cond[c];
1334
1335 if (!c2const) {
1336 opc = (type == TCG_TYPE_I32
1337 ? (is_unsigned ? RIE_CLRJ : RIE_CRJ)
1338 : (is_unsigned ? RIE_CLGRJ : RIE_CGRJ));
1339 tgen_compare_branch(s, opc, cc, r1, c2, l);
1340 return;
1341 }
1342
1343 /* COMPARE IMMEDIATE AND BRANCH RELATIVE has an 8-bit immediate field.
1344 If the immediate we've been given does not fit that range, we'll
1345 fall back to separate compare and branch instructions using the
1346 larger comparison range afforded by COMPARE IMMEDIATE. */
1347 if (type == TCG_TYPE_I32) {
1348 if (is_unsigned) {
1349 opc = RIE_CLIJ;
1350 in_range = (uint32_t)c2 == (uint8_t)c2;
1351 } else {
1352 opc = RIE_CIJ;
1353 in_range = (int32_t)c2 == (int8_t)c2;
1354 }
1355 } else {
1356 if (is_unsigned) {
1357 opc = RIE_CLGIJ;
1358 in_range = (uint64_t)c2 == (uint8_t)c2;
1359 } else {
1360 opc = RIE_CGIJ;
1361 in_range = (int64_t)c2 == (int8_t)c2;
1362 }
1363 }
1364 if (in_range) {
1365 tgen_compare_imm_branch(s, opc, cc, r1, c2, l);
1366 return;
1367 }
1368 }
1369
1370 cc = tgen_cmp(s, type, c, r1, c2, c2const, false);
1371 tgen_branch(s, cc, l);
1372 }
1373
1374 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest)
1375 {
1376 ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
1377 if (off == (int32_t)off) {
1378 tcg_out_insn(s, RIL, BRASL, TCG_REG_R14, off);
1379 } else {
1380 tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
1381 tcg_out_insn(s, RR, BASR, TCG_REG_R14, TCG_TMP0);
1382 }
1383 }
1384
1385 static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
1386 TCGReg base, TCGReg index, int disp)
1387 {
1388 switch (opc & (MO_SSIZE | MO_BSWAP)) {
1389 case MO_UB:
1390 tcg_out_insn(s, RXY, LLGC, data, base, index, disp);
1391 break;
1392 case MO_SB:
1393 tcg_out_insn(s, RXY, LGB, data, base, index, disp);
1394 break;
1395
1396 case MO_UW | MO_BSWAP:
1397 /* swapped unsigned halfword load with upper bits zeroed */
1398 tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
1399 tgen_ext16u(s, TCG_TYPE_I64, data, data);
1400 break;
1401 case MO_UW:
1402 tcg_out_insn(s, RXY, LLGH, data, base, index, disp);
1403 break;
1404
1405 case MO_SW | MO_BSWAP:
1406 /* swapped sign-extended halfword load */
1407 tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
1408 tgen_ext16s(s, TCG_TYPE_I64, data, data);
1409 break;
1410 case MO_SW:
1411 tcg_out_insn(s, RXY, LGH, data, base, index, disp);
1412 break;
1413
1414 case MO_UL | MO_BSWAP:
1415 /* swapped unsigned int load with upper bits zeroed */
1416 tcg_out_insn(s, RXY, LRV, data, base, index, disp);
1417 tgen_ext32u(s, data, data);
1418 break;
1419 case MO_UL:
1420 tcg_out_insn(s, RXY, LLGF, data, base, index, disp);
1421 break;
1422
1423 case MO_SL | MO_BSWAP:
1424 /* swapped sign-extended int load */
1425 tcg_out_insn(s, RXY, LRV, data, base, index, disp);
1426 tgen_ext32s(s, data, data);
1427 break;
1428 case MO_SL:
1429 tcg_out_insn(s, RXY, LGF, data, base, index, disp);
1430 break;
1431
1432 case MO_Q | MO_BSWAP:
1433 tcg_out_insn(s, RXY, LRVG, data, base, index, disp);
1434 break;
1435 case MO_Q:
1436 tcg_out_insn(s, RXY, LG, data, base, index, disp);
1437 break;
1438
1439 default:
1440 tcg_abort();
1441 }
1442 }
1443
1444 static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data,
1445 TCGReg base, TCGReg index, int disp)
1446 {
1447 switch (opc & (MO_SIZE | MO_BSWAP)) {
1448 case MO_UB:
1449 if (disp >= 0 && disp < 0x1000) {
1450 tcg_out_insn(s, RX, STC, data, base, index, disp);
1451 } else {
1452 tcg_out_insn(s, RXY, STCY, data, base, index, disp);
1453 }
1454 break;
1455
1456 case MO_UW | MO_BSWAP:
1457 tcg_out_insn(s, RXY, STRVH, data, base, index, disp);
1458 break;
1459 case MO_UW:
1460 if (disp >= 0 && disp < 0x1000) {
1461 tcg_out_insn(s, RX, STH, data, base, index, disp);
1462 } else {
1463 tcg_out_insn(s, RXY, STHY, data, base, index, disp);
1464 }
1465 break;
1466
1467 case MO_UL | MO_BSWAP:
1468 tcg_out_insn(s, RXY, STRV, data, base, index, disp);
1469 break;
1470 case MO_UL:
1471 if (disp >= 0 && disp < 0x1000) {
1472 tcg_out_insn(s, RX, ST, data, base, index, disp);
1473 } else {
1474 tcg_out_insn(s, RXY, STY, data, base, index, disp);
1475 }
1476 break;
1477
1478 case MO_Q | MO_BSWAP:
1479 tcg_out_insn(s, RXY, STRVG, data, base, index, disp);
1480 break;
1481 case MO_Q:
1482 tcg_out_insn(s, RXY, STG, data, base, index, disp);
1483 break;
1484
1485 default:
1486 tcg_abort();
1487 }
1488 }
1489
1490 #if defined(CONFIG_SOFTMMU)
1491 #include "../tcg-ldst.c.inc"
1492
1493 /* We're expecting to use a 20-bit negative offset on the tlb memory ops. */
1494 QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
1495 QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 19));
1496
1497 /* Load and compare a TLB entry, leaving the flags set. Loads the TLB
1498 addend into R2. Returns a register with the santitized guest address. */
1499 static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
1500 int mem_index, bool is_ld)
1501 {
1502 unsigned s_bits = opc & MO_SIZE;
1503 unsigned a_bits = get_alignment_bits(opc);
1504 unsigned s_mask = (1 << s_bits) - 1;
1505 unsigned a_mask = (1 << a_bits) - 1;
1506 int fast_off = TLB_MASK_TABLE_OFS(mem_index);
1507 int mask_off = fast_off + offsetof(CPUTLBDescFast, mask);
1508 int table_off = fast_off + offsetof(CPUTLBDescFast, table);
1509 int ofs, a_off;
1510 uint64_t tlb_mask;
1511
1512 tcg_out_sh64(s, RSY_SRLG, TCG_REG_R2, addr_reg, TCG_REG_NONE,
1513 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1514 tcg_out_insn(s, RXY, NG, TCG_REG_R2, TCG_AREG0, TCG_REG_NONE, mask_off);
1515 tcg_out_insn(s, RXY, AG, TCG_REG_R2, TCG_AREG0, TCG_REG_NONE, table_off);
1516
1517 /* For aligned accesses, we check the first byte and include the alignment
1518 bits within the address. For unaligned access, we check that we don't
1519 cross pages using the address of the last byte of the access. */
1520 a_off = (a_bits >= s_bits ? 0 : s_mask - a_mask);
1521 tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
1522 if ((s390_facilities & FACILITY_GEN_INST_EXT) && a_off == 0) {
1523 tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
1524 } else {
1525 tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
1526 tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
1527 }
1528
1529 if (is_ld) {
1530 ofs = offsetof(CPUTLBEntry, addr_read);
1531 } else {
1532 ofs = offsetof(CPUTLBEntry, addr_write);
1533 }
1534 if (TARGET_LONG_BITS == 32) {
1535 tcg_out_insn(s, RX, C, TCG_REG_R3, TCG_REG_R2, TCG_REG_NONE, ofs);
1536 } else {
1537 tcg_out_insn(s, RXY, CG, TCG_REG_R3, TCG_REG_R2, TCG_REG_NONE, ofs);
1538 }
1539
1540 tcg_out_insn(s, RXY, LG, TCG_REG_R2, TCG_REG_R2, TCG_REG_NONE,
1541 offsetof(CPUTLBEntry, addend));
1542
1543 if (TARGET_LONG_BITS == 32) {
1544 tgen_ext32u(s, TCG_REG_R3, addr_reg);
1545 return TCG_REG_R3;
1546 }
1547 return addr_reg;
1548 }
1549
1550 static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
1551 TCGReg data, TCGReg addr,
1552 tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
1553 {
1554 TCGLabelQemuLdst *label = new_ldst_label(s);
1555
1556 label->is_ld = is_ld;
1557 label->oi = oi;
1558 label->datalo_reg = data;
1559 label->addrlo_reg = addr;
1560 label->raddr = tcg_splitwx_to_rx(raddr);
1561 label->label_ptr[0] = label_ptr;
1562 }
1563
1564 static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1565 {
1566 TCGReg addr_reg = lb->addrlo_reg;
1567 TCGReg data_reg = lb->datalo_reg;
1568 TCGMemOpIdx oi = lb->oi;
1569 MemOp opc = get_memop(oi);
1570
1571 if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
1572 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
1573 return false;
1574 }
1575
1576 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1577 if (TARGET_LONG_BITS == 64) {
1578 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
1579 }
1580 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R4, oi);
1581 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R5, (uintptr_t)lb->raddr);
1582 tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)]);
1583 tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_R2);
1584
1585 tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
1586 return true;
1587 }
1588
1589 static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1590 {
1591 TCGReg addr_reg = lb->addrlo_reg;
1592 TCGReg data_reg = lb->datalo_reg;
1593 TCGMemOpIdx oi = lb->oi;
1594 MemOp opc = get_memop(oi);
1595
1596 if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
1597 (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
1598 return false;
1599 }
1600
1601 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
1602 if (TARGET_LONG_BITS == 64) {
1603 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
1604 }
1605 switch (opc & MO_SIZE) {
1606 case MO_UB:
1607 tgen_ext8u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1608 break;
1609 case MO_UW:
1610 tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1611 break;
1612 case MO_UL:
1613 tgen_ext32u(s, TCG_REG_R4, data_reg);
1614 break;
1615 case MO_Q:
1616 tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
1617 break;
1618 default:
1619 tcg_abort();
1620 }
1621 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
1622 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
1623 tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
1624
1625 tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
1626 return true;
1627 }
1628 #else
1629 static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
1630 TCGReg *index_reg, tcg_target_long *disp)
1631 {
1632 if (TARGET_LONG_BITS == 32) {
1633 tgen_ext32u(s, TCG_TMP0, *addr_reg);
1634 *addr_reg = TCG_TMP0;
1635 }
1636 if (guest_base < 0x80000) {
1637 *index_reg = TCG_REG_NONE;
1638 *disp = guest_base;
1639 } else {
1640 *index_reg = TCG_GUEST_BASE_REG;
1641 *disp = 0;
1642 }
1643 }
1644 #endif /* CONFIG_SOFTMMU */
1645
1646 static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
1647 TCGMemOpIdx oi)
1648 {
1649 MemOp opc = get_memop(oi);
1650 #ifdef CONFIG_SOFTMMU
1651 unsigned mem_index = get_mmuidx(oi);
1652 tcg_insn_unit *label_ptr;
1653 TCGReg base_reg;
1654
1655 base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1);
1656
1657 tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
1658 label_ptr = s->code_ptr;
1659 s->code_ptr += 1;
1660
1661 tcg_out_qemu_ld_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
1662
1663 add_qemu_ldst_label(s, 1, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
1664 #else
1665 TCGReg index_reg;
1666 tcg_target_long disp;
1667
1668 tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
1669 tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, index_reg, disp);
1670 #endif
1671 }
1672
1673 static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
1674 TCGMemOpIdx oi)
1675 {
1676 MemOp opc = get_memop(oi);
1677 #ifdef CONFIG_SOFTMMU
1678 unsigned mem_index = get_mmuidx(oi);
1679 tcg_insn_unit *label_ptr;
1680 TCGReg base_reg;
1681
1682 base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0);
1683
1684 tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
1685 label_ptr = s->code_ptr;
1686 s->code_ptr += 1;
1687
1688 tcg_out_qemu_st_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
1689
1690 add_qemu_ldst_label(s, 0, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
1691 #else
1692 TCGReg index_reg;
1693 tcg_target_long disp;
1694
1695 tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
1696 tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, index_reg, disp);
1697 #endif
1698 }
1699
1700 # define OP_32_64(x) \
1701 case glue(glue(INDEX_op_,x),_i32): \
1702 case glue(glue(INDEX_op_,x),_i64)
1703
1704 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
1705 const TCGArg args[TCG_MAX_OP_ARGS],
1706 const int const_args[TCG_MAX_OP_ARGS])
1707 {
1708 S390Opcode op, op2;
1709 TCGArg a0, a1, a2;
1710
1711 switch (opc) {
1712 case INDEX_op_exit_tb:
1713 /* Reuse the zeroing that exists for goto_ptr. */
1714 a0 = args[0];
1715 if (a0 == 0) {
1716 tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
1717 } else {
1718 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
1719 tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
1720 }
1721 break;
1722
1723 case INDEX_op_goto_tb:
1724 a0 = args[0];
1725 if (s->tb_jmp_insn_offset) {
1726 /*
1727 * branch displacement must be aligned for atomic patching;
1728 * see if we need to add extra nop before branch
1729 */
1730 if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
1731 tcg_out16(s, NOP);
1732 }
1733 tcg_debug_assert(!USE_REG_TB);
1734 tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
1735 s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
1736 s->code_ptr += 2;
1737 } else {
1738 /* load address stored at s->tb_jmp_target_addr + a0 */
1739 tcg_out_ld_abs(s, TCG_TYPE_PTR, TCG_REG_TB,
1740 tcg_splitwx_to_rx(s->tb_jmp_target_addr + a0));
1741 /* and go there */
1742 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_TB);
1743 }
1744 set_jmp_reset_offset(s, a0);
1745
1746 /* For the unlinked path of goto_tb, we need to reset
1747 TCG_REG_TB to the beginning of this TB. */
1748 if (USE_REG_TB) {
1749 int ofs = -tcg_current_code_size(s);
1750 /* All TB are restricted to 64KiB by unwind info. */
1751 tcg_debug_assert(ofs == sextract64(ofs, 0, 20));
1752 tcg_out_insn(s, RXY, LAY, TCG_REG_TB,
1753 TCG_REG_TB, TCG_REG_NONE, ofs);
1754 }
1755 break;
1756
1757 case INDEX_op_goto_ptr:
1758 a0 = args[0];
1759 if (USE_REG_TB) {
1760 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, a0);
1761 }
1762 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0);
1763 break;
1764
1765 OP_32_64(ld8u):
1766 /* ??? LLC (RXY format) is only present with the extended-immediate
1767 facility, whereas LLGC is always present. */
1768 tcg_out_mem(s, 0, RXY_LLGC, args[0], args[1], TCG_REG_NONE, args[2]);
1769 break;
1770
1771 OP_32_64(ld8s):
1772 /* ??? LB is no smaller than LGB, so no point to using it. */
1773 tcg_out_mem(s, 0, RXY_LGB, args[0], args[1], TCG_REG_NONE, args[2]);
1774 break;
1775
1776 OP_32_64(ld16u):
1777 /* ??? LLH (RXY format) is only present with the extended-immediate
1778 facility, whereas LLGH is always present. */
1779 tcg_out_mem(s, 0, RXY_LLGH, args[0], args[1], TCG_REG_NONE, args[2]);
1780 break;
1781
1782 case INDEX_op_ld16s_i32:
1783 tcg_out_mem(s, RX_LH, RXY_LHY, args[0], args[1], TCG_REG_NONE, args[2]);
1784 break;
1785
1786 case INDEX_op_ld_i32:
1787 tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]);
1788 break;
1789
1790 OP_32_64(st8):
1791 tcg_out_mem(s, RX_STC, RXY_STCY, args[0], args[1],
1792 TCG_REG_NONE, args[2]);
1793 break;
1794
1795 OP_32_64(st16):
1796 tcg_out_mem(s, RX_STH, RXY_STHY, args[0], args[1],
1797 TCG_REG_NONE, args[2]);
1798 break;
1799
1800 case INDEX_op_st_i32:
1801 tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
1802 break;
1803
1804 case INDEX_op_add_i32:
1805 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
1806 if (const_args[2]) {
1807 do_addi_32:
1808 if (a0 == a1) {
1809 if (a2 == (int16_t)a2) {
1810 tcg_out_insn(s, RI, AHI, a0, a2);
1811 break;
1812 }
1813 if (s390_facilities & FACILITY_EXT_IMM) {
1814 tcg_out_insn(s, RIL, AFI, a0, a2);
1815 break;
1816 }
1817 }
1818 tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
1819 } else if (a0 == a1) {
1820 tcg_out_insn(s, RR, AR, a0, a2);
1821 } else {
1822 tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
1823 }
1824 break;
1825 case INDEX_op_sub_i32:
1826 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
1827 if (const_args[2]) {
1828 a2 = -a2;
1829 goto do_addi_32;
1830 } else if (a0 == a1) {
1831 tcg_out_insn(s, RR, SR, a0, a2);
1832 } else {
1833 tcg_out_insn(s, RRF, SRK, a0, a1, a2);
1834 }
1835 break;
1836
1837 case INDEX_op_and_i32:
1838 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
1839 if (const_args[2]) {
1840 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
1841 tgen_andi(s, TCG_TYPE_I32, a0, a2);
1842 } else if (a0 == a1) {
1843 tcg_out_insn(s, RR, NR, a0, a2);
1844 } else {
1845 tcg_out_insn(s, RRF, NRK, a0, a1, a2);
1846 }
1847 break;
1848 case INDEX_op_or_i32:
1849 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
1850 if (const_args[2]) {
1851 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
1852 tgen_ori(s, TCG_TYPE_I32, a0, a2);
1853 } else if (a0 == a1) {
1854 tcg_out_insn(s, RR, OR, a0, a2);
1855 } else {
1856 tcg_out_insn(s, RRF, ORK, a0, a1, a2);
1857 }
1858 break;
1859 case INDEX_op_xor_i32:
1860 a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2];
1861 if (const_args[2]) {
1862 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
1863 tgen_xori(s, TCG_TYPE_I32, a0, a2);
1864 } else if (a0 == a1) {
1865 tcg_out_insn(s, RR, XR, args[0], args[2]);
1866 } else {
1867 tcg_out_insn(s, RRF, XRK, a0, a1, a2);
1868 }
1869 break;
1870
1871 case INDEX_op_neg_i32:
1872 tcg_out_insn(s, RR, LCR, args[0], args[1]);
1873 break;
1874
1875 case INDEX_op_mul_i32:
1876 if (const_args[2]) {
1877 if ((int32_t)args[2] == (int16_t)args[2]) {
1878 tcg_out_insn(s, RI, MHI, args[0], args[2]);
1879 } else {
1880 tcg_out_insn(s, RIL, MSFI, args[0], args[2]);
1881 }
1882 } else {
1883 tcg_out_insn(s, RRE, MSR, args[0], args[2]);
1884 }
1885 break;
1886
1887 case INDEX_op_div2_i32:
1888 tcg_out_insn(s, RR, DR, TCG_REG_R2, args[4]);
1889 break;
1890 case INDEX_op_divu2_i32:
1891 tcg_out_insn(s, RRE, DLR, TCG_REG_R2, args[4]);
1892 break;
1893
1894 case INDEX_op_shl_i32:
1895 op = RS_SLL;
1896 op2 = RSY_SLLK;
1897 do_shift32:
1898 a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
1899 if (a0 == a1) {
1900 if (const_args[2]) {
1901 tcg_out_sh32(s, op, a0, TCG_REG_NONE, a2);
1902 } else {
1903 tcg_out_sh32(s, op, a0, a2, 0);
1904 }
1905 } else {
1906 /* Using tcg_out_sh64 here for the format; it is a 32-bit shift. */
1907 if (const_args[2]) {
1908 tcg_out_sh64(s, op2, a0, a1, TCG_REG_NONE, a2);
1909 } else {
1910 tcg_out_sh64(s, op2, a0, a1, a2, 0);
1911 }
1912 }
1913 break;
1914 case INDEX_op_shr_i32:
1915 op = RS_SRL;
1916 op2 = RSY_SRLK;
1917 goto do_shift32;
1918 case INDEX_op_sar_i32:
1919 op = RS_SRA;
1920 op2 = RSY_SRAK;
1921 goto do_shift32;
1922
1923 case INDEX_op_rotl_i32:
1924 /* ??? Using tcg_out_sh64 here for the format; it is a 32-bit rol. */
1925 if (const_args[2]) {
1926 tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_REG_NONE, args[2]);
1927 } else {
1928 tcg_out_sh64(s, RSY_RLL, args[0], args[1], args[2], 0);
1929 }
1930 break;
1931 case INDEX_op_rotr_i32:
1932 if (const_args[2]) {
1933 tcg_out_sh64(s, RSY_RLL, args[0], args[1],
1934 TCG_REG_NONE, (32 - args[2]) & 31);
1935 } else {
1936 tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
1937 tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_TMP0, 0);
1938 }
1939 break;
1940
1941 case INDEX_op_ext8s_i32:
1942 tgen_ext8s(s, TCG_TYPE_I32, args[0], args[1]);
1943 break;
1944 case INDEX_op_ext16s_i32:
1945 tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
1946 break;
1947 case INDEX_op_ext8u_i32:
1948 tgen_ext8u(s, TCG_TYPE_I32, args[0], args[1]);
1949 break;
1950 case INDEX_op_ext16u_i32:
1951 tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
1952 break;
1953
1954 case INDEX_op_bswap16_i32:
1955 a0 = args[0], a1 = args[1], a2 = args[2];
1956 tcg_out_insn(s, RRE, LRVR, a0, a1);
1957 if (a2 & TCG_BSWAP_OS) {
1958 tcg_out_sh32(s, RS_SRA, a0, TCG_REG_NONE, 16);
1959 } else {
1960 tcg_out_sh32(s, RS_SRL, a0, TCG_REG_NONE, 16);
1961 }
1962 break;
1963 case INDEX_op_bswap16_i64:
1964 a0 = args[0], a1 = args[1], a2 = args[2];
1965 tcg_out_insn(s, RRE, LRVGR, a0, a1);
1966 if (a2 & TCG_BSWAP_OS) {
1967 tcg_out_sh64(s, RSY_SRAG, a0, a0, TCG_REG_NONE, 48);
1968 } else {
1969 tcg_out_sh64(s, RSY_SRLG, a0, a0, TCG_REG_NONE, 48);
1970 }
1971 break;
1972
1973 case INDEX_op_bswap32_i32:
1974 tcg_out_insn(s, RRE, LRVR, args[0], args[1]);
1975 break;
1976 case INDEX_op_bswap32_i64:
1977 a0 = args[0], a1 = args[1], a2 = args[2];
1978 tcg_out_insn(s, RRE, LRVR, a0, a1);
1979 if (a2 & TCG_BSWAP_OS) {
1980 tgen_ext32s(s, a0, a0);
1981 } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
1982 tgen_ext32u(s, a0, a0);
1983 }
1984 break;
1985
1986 case INDEX_op_add2_i32:
1987 if (const_args[4]) {
1988 tcg_out_insn(s, RIL, ALFI, args[0], args[4]);
1989 } else {
1990 tcg_out_insn(s, RR, ALR, args[0], args[4]);
1991 }
1992 tcg_out_insn(s, RRE, ALCR, args[1], args[5]);
1993 break;
1994 case INDEX_op_sub2_i32:
1995 if (const_args[4]) {
1996 tcg_out_insn(s, RIL, SLFI, args[0], args[4]);
1997 } else {
1998 tcg_out_insn(s, RR, SLR, args[0], args[4]);
1999 }
2000 tcg_out_insn(s, RRE, SLBR, args[1], args[5]);
2001 break;
2002
2003 case INDEX_op_br:
2004 tgen_branch(s, S390_CC_ALWAYS, arg_label(args[0]));
2005 break;
2006
2007 case INDEX_op_brcond_i32:
2008 tgen_brcond(s, TCG_TYPE_I32, args[2], args[0],
2009 args[1], const_args[1], arg_label(args[3]));
2010 break;
2011 case INDEX_op_setcond_i32:
2012 tgen_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1],
2013 args[2], const_args[2]);
2014 break;
2015 case INDEX_op_movcond_i32:
2016 tgen_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1],
2017 args[2], const_args[2], args[3], const_args[3]);
2018 break;
2019
2020 case INDEX_op_qemu_ld_i32:
2021 /* ??? Technically we can use a non-extending instruction. */
2022 case INDEX_op_qemu_ld_i64:
2023 tcg_out_qemu_ld(s, args[0], args[1], args[2]);
2024 break;
2025 case INDEX_op_qemu_st_i32:
2026 case INDEX_op_qemu_st_i64:
2027 tcg_out_qemu_st(s, args[0], args[1], args[2]);
2028 break;
2029
2030 case INDEX_op_ld16s_i64:
2031 tcg_out_mem(s, 0, RXY_LGH, args[0], args[1], TCG_REG_NONE, args[2]);
2032 break;
2033 case INDEX_op_ld32u_i64:
2034 tcg_out_mem(s, 0, RXY_LLGF, args[0], args[1], TCG_REG_NONE, args[2]);
2035 break;
2036 case INDEX_op_ld32s_i64:
2037 tcg_out_mem(s, 0, RXY_LGF, args[0], args[1], TCG_REG_NONE, args[2]);
2038 break;
2039 case INDEX_op_ld_i64:
2040 tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
2041 break;
2042
2043 case INDEX_op_st32_i64:
2044 tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
2045 break;
2046 case INDEX_op_st_i64:
2047 tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
2048 break;
2049
2050 case INDEX_op_add_i64:
2051 a0 = args[0], a1 = args[1], a2 = args[2];
2052 if (const_args[2]) {
2053 do_addi_64:
2054 if (a0 == a1) {
2055 if (a2 == (int16_t)a2) {
2056 tcg_out_insn(s, RI, AGHI, a0, a2);
2057 break;
2058 }
2059 if (s390_facilities & FACILITY_EXT_IMM) {
2060 if (a2 == (int32_t)a2) {
2061 tcg_out_insn(s, RIL, AGFI, a0, a2);
2062 break;
2063 } else if (a2 == (uint32_t)a2) {
2064 tcg_out_insn(s, RIL, ALGFI, a0, a2);
2065 break;
2066 } else if (-a2 == (uint32_t)-a2) {
2067 tcg_out_insn(s, RIL, SLGFI, a0, -a2);
2068 break;
2069 }
2070 }
2071 }
2072 tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
2073 } else if (a0 == a1) {
2074 tcg_out_insn(s, RRE, AGR, a0, a2);
2075 } else {
2076 tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
2077 }
2078 break;
2079 case INDEX_op_sub_i64:
2080 a0 = args[0], a1 = args[1], a2 = args[2];
2081 if (const_args[2]) {
2082 a2 = -a2;
2083 goto do_addi_64;
2084 } else if (a0 == a1) {
2085 tcg_out_insn(s, RRE, SGR, a0, a2);
2086 } else {
2087 tcg_out_insn(s, RRF, SGRK, a0, a1, a2);
2088 }
2089 break;
2090
2091 case INDEX_op_and_i64:
2092 a0 = args[0], a1 = args[1], a2 = args[2];
2093 if (const_args[2]) {
2094 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
2095 tgen_andi(s, TCG_TYPE_I64, args[0], args[2]);
2096 } else if (a0 == a1) {
2097 tcg_out_insn(s, RRE, NGR, args[0], args[2]);
2098 } else {
2099 tcg_out_insn(s, RRF, NGRK, a0, a1, a2);
2100 }
2101 break;
2102 case INDEX_op_or_i64:
2103 a0 = args[0], a1 = args[1], a2 = args[2];
2104 if (const_args[2]) {
2105 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
2106 tgen_ori(s, TCG_TYPE_I64, a0, a2);
2107 } else if (a0 == a1) {
2108 tcg_out_insn(s, RRE, OGR, a0, a2);
2109 } else {
2110 tcg_out_insn(s, RRF, OGRK, a0, a1, a2);
2111 }
2112 break;
2113 case INDEX_op_xor_i64:
2114 a0 = args[0], a1 = args[1], a2 = args[2];
2115 if (const_args[2]) {
2116 tcg_out_mov(s, TCG_TYPE_I64, a0, a1);
2117 tgen_xori(s, TCG_TYPE_I64, a0, a2);
2118 } else if (a0 == a1) {
2119 tcg_out_insn(s, RRE, XGR, a0, a2);
2120 } else {
2121 tcg_out_insn(s, RRF, XGRK, a0, a1, a2);
2122 }
2123 break;
2124
2125 case INDEX_op_neg_i64:
2126 tcg_out_insn(s, RRE, LCGR, args[0], args[1]);
2127 break;
2128 case INDEX_op_bswap64_i64:
2129 tcg_out_insn(s, RRE, LRVGR, args[0], args[1]);
2130 break;
2131
2132 case INDEX_op_mul_i64:
2133 if (const_args[2]) {
2134 if (args[2] == (int16_t)args[2]) {
2135 tcg_out_insn(s, RI, MGHI, args[0], args[2]);
2136 } else {
2137 tcg_out_insn(s, RIL, MSGFI, args[0], args[2]);
2138 }
2139 } else {
2140 tcg_out_insn(s, RRE, MSGR, args[0], args[2]);
2141 }
2142 break;
2143
2144 case INDEX_op_div2_i64:
2145 /* ??? We get an unnecessary sign-extension of the dividend
2146 into R3 with this definition, but as we do in fact always
2147 produce both quotient and remainder using INDEX_op_div_i64
2148 instead requires jumping through even more hoops. */
2149 tcg_out_insn(s, RRE, DSGR, TCG_REG_R2, args[4]);
2150 break;
2151 case INDEX_op_divu2_i64:
2152 tcg_out_insn(s, RRE, DLGR, TCG_REG_R2, args[4]);
2153 break;
2154 case INDEX_op_mulu2_i64:
2155 tcg_out_insn(s, RRE, MLGR, TCG_REG_R2, args[3]);
2156 break;
2157
2158 case INDEX_op_shl_i64:
2159 op = RSY_SLLG;
2160 do_shift64:
2161 if (const_args[2]) {
2162 tcg_out_sh64(s, op, args[0], args[1], TCG_REG_NONE, args[2]);
2163 } else {
2164 tcg_out_sh64(s, op, args[0], args[1], args[2], 0);
2165 }
2166 break;
2167 case INDEX_op_shr_i64:
2168 op = RSY_SRLG;
2169 goto do_shift64;
2170 case INDEX_op_sar_i64:
2171 op = RSY_SRAG;
2172 goto do_shift64;
2173
2174 case INDEX_op_rotl_i64:
2175 if (const_args[2]) {
2176 tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
2177 TCG_REG_NONE, args[2]);
2178 } else {
2179 tcg_out_sh64(s, RSY_RLLG, args[0], args[1], args[2], 0);
2180 }
2181 break;
2182 case INDEX_op_rotr_i64:
2183 if (const_args[2]) {
2184 tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
2185 TCG_REG_NONE, (64 - args[2]) & 63);
2186 } else {
2187 /* We can use the smaller 32-bit negate because only the
2188 low 6 bits are examined for the rotate. */
2189 tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
2190 tcg_out_sh64(s, RSY_RLLG, args[0], args[1], TCG_TMP0, 0);
2191 }
2192 break;
2193
2194 case INDEX_op_ext8s_i64:
2195 tgen_ext8s(s, TCG_TYPE_I64, args[0], args[1]);
2196 break;
2197 case INDEX_op_ext16s_i64:
2198 tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
2199 break;
2200 case INDEX_op_ext_i32_i64:
2201 case INDEX_op_ext32s_i64:
2202 tgen_ext32s(s, args[0], args[1]);
2203 break;
2204 case INDEX_op_ext8u_i64:
2205 tgen_ext8u(s, TCG_TYPE_I64, args[0], args[1]);
2206 break;
2207 case INDEX_op_ext16u_i64:
2208 tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
2209 break;
2210 case INDEX_op_extu_i32_i64:
2211 case INDEX_op_ext32u_i64:
2212 tgen_ext32u(s, args[0], args[1]);
2213 break;
2214
2215 case INDEX_op_add2_i64:
2216 if (const_args[4]) {
2217 if ((int64_t)args[4] >= 0) {
2218 tcg_out_insn(s, RIL, ALGFI, args[0], args[4]);
2219 } else {
2220 tcg_out_insn(s, RIL, SLGFI, args[0], -args[4]);
2221 }
2222 } else {
2223 tcg_out_insn(s, RRE, ALGR, args[0], args[4]);
2224 }
2225 tcg_out_insn(s, RRE, ALCGR, args[1], args[5]);
2226 break;
2227 case INDEX_op_sub2_i64:
2228 if (const_args[4]) {
2229 if ((int64_t)args[4] >= 0) {
2230 tcg_out_insn(s, RIL, SLGFI, args[0], args[4]);
2231 } else {
2232 tcg_out_insn(s, RIL, ALGFI, args[0], -args[4]);
2233 }
2234 } else {
2235 tcg_out_insn(s, RRE, SLGR, args[0], args[4]);
2236 }
2237 tcg_out_insn(s, RRE, SLBGR, args[1], args[5]);
2238 break;
2239
2240 case INDEX_op_brcond_i64:
2241 tgen_brcond(s, TCG_TYPE_I64, args[2], args[0],
2242 args[1], const_args[1], arg_label(args[3]));
2243 break;
2244 case INDEX_op_setcond_i64:
2245 tgen_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1],
2246 args[2], const_args[2]);
2247 break;
2248 case INDEX_op_movcond_i64:
2249 tgen_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1],
2250 args[2], const_args[2], args[3], const_args[3]);
2251 break;
2252
2253 OP_32_64(deposit):
2254 a0 = args[0], a1 = args[1], a2 = args[2];
2255 if (const_args[1]) {
2256 tgen_deposit(s, a0, a2, args[3], args[4], 1);
2257 } else {
2258 /* Since we can't support "0Z" as a constraint, we allow a1 in
2259 any register. Fix things up as if a matching constraint. */
2260 if (a0 != a1) {
2261 TCGType type = (opc == INDEX_op_deposit_i64);
2262 if (a0 == a2) {
2263 tcg_out_mov(s, type, TCG_TMP0, a2);
2264 a2 = TCG_TMP0;
2265 }
2266 tcg_out_mov(s, type, a0, a1);
2267 }
2268 tgen_deposit(s, a0, a2, args[3], args[4], 0);
2269 }
2270 break;
2271
2272 OP_32_64(extract):
2273 tgen_extract(s, args[0], args[1], args[2], args[3]);
2274 break;
2275
2276 case INDEX_op_clz_i64:
2277 tgen_clz(s, args[0], args[1], args[2], const_args[2]);
2278 break;
2279
2280 case INDEX_op_mb:
2281 /* The host memory model is quite strong, we simply need to
2282 serialize the instruction stream. */
2283 if (args[0] & TCG_MO_ST_LD) {
2284 tcg_out_insn(s, RR, BCR,
2285 s390_facilities & FACILITY_FAST_BCR_SER ? 14 : 15, 0);
2286 }
2287 break;
2288
2289 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
2290 case INDEX_op_mov_i64:
2291 case INDEX_op_call: /* Always emitted via tcg_out_call. */
2292 default:
2293 tcg_abort();
2294 }
2295 }
2296
2297 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op)
2298 {
2299 switch (op) {
2300 case INDEX_op_goto_ptr:
2301 return C_O0_I1(r);
2302
2303 case INDEX_op_ld8u_i32:
2304 case INDEX_op_ld8u_i64:
2305 case INDEX_op_ld8s_i32:
2306 case INDEX_op_ld8s_i64:
2307 case INDEX_op_ld16u_i32:
2308 case INDEX_op_ld16u_i64:
2309 case INDEX_op_ld16s_i32:
2310 case INDEX_op_ld16s_i64:
2311 case INDEX_op_ld_i32:
2312 case INDEX_op_ld32u_i64:
2313 case INDEX_op_ld32s_i64:
2314 case INDEX_op_ld_i64:
2315 return C_O1_I1(r, r);
2316
2317 case INDEX_op_st8_i32:
2318 case INDEX_op_st8_i64:
2319 case INDEX_op_st16_i32:
2320 case INDEX_op_st16_i64:
2321 case INDEX_op_st_i32:
2322 case INDEX_op_st32_i64:
2323 case INDEX_op_st_i64:
2324 return C_O0_I2(r, r);
2325
2326 case INDEX_op_add_i32:
2327 case INDEX_op_add_i64:
2328 case INDEX_op_shl_i64:
2329 case INDEX_op_shr_i64:
2330 case INDEX_op_sar_i64:
2331 case INDEX_op_rotl_i32:
2332 case INDEX_op_rotl_i64:
2333 case INDEX_op_rotr_i32:
2334 case INDEX_op_rotr_i64:
2335 case INDEX_op_clz_i64:
2336 case INDEX_op_setcond_i32:
2337 case INDEX_op_setcond_i64:
2338 return C_O1_I2(r, r, ri);
2339
2340 case INDEX_op_sub_i32:
2341 case INDEX_op_sub_i64:
2342 case INDEX_op_and_i32:
2343 case INDEX_op_and_i64:
2344 case INDEX_op_or_i32:
2345 case INDEX_op_or_i64:
2346 case INDEX_op_xor_i32:
2347 case INDEX_op_xor_i64:
2348 return (s390_facilities & FACILITY_DISTINCT_OPS
2349 ? C_O1_I2(r, r, ri)
2350 : C_O1_I2(r, 0, ri));
2351
2352 case INDEX_op_mul_i32:
2353 /* If we have the general-instruction-extensions, then we have
2354 MULTIPLY SINGLE IMMEDIATE with a signed 32-bit, otherwise we
2355 have only MULTIPLY HALFWORD IMMEDIATE, with a signed 16-bit. */
2356 return (s390_facilities & FACILITY_GEN_INST_EXT
2357 ? C_O1_I2(r, 0, ri)
2358 : C_O1_I2(r, 0, rI));
2359
2360 case INDEX_op_mul_i64:
2361 return (s390_facilities & FACILITY_GEN_INST_EXT
2362 ? C_O1_I2(r, 0, rJ)
2363 : C_O1_I2(r, 0, rI));
2364
2365 case INDEX_op_shl_i32:
2366 case INDEX_op_shr_i32:
2367 case INDEX_op_sar_i32:
2368 return (s390_facilities & FACILITY_DISTINCT_OPS
2369 ? C_O1_I2(r, r, ri)
2370 : C_O1_I2(r, 0, ri));
2371
2372 case INDEX_op_brcond_i32:
2373 case INDEX_op_brcond_i64:
2374 return C_O0_I2(r, ri);
2375
2376 case INDEX_op_bswap16_i32:
2377 case INDEX_op_bswap16_i64:
2378 case INDEX_op_bswap32_i32:
2379 case INDEX_op_bswap32_i64:
2380 case INDEX_op_bswap64_i64:
2381 case INDEX_op_neg_i32:
2382 case INDEX_op_neg_i64:
2383 case INDEX_op_ext8s_i32:
2384 case INDEX_op_ext8s_i64:
2385 case INDEX_op_ext8u_i32:
2386 case INDEX_op_ext8u_i64:
2387 case INDEX_op_ext16s_i32:
2388 case INDEX_op_ext16s_i64:
2389 case INDEX_op_ext16u_i32:
2390 case INDEX_op_ext16u_i64:
2391 case INDEX_op_ext32s_i64:
2392 case INDEX_op_ext32u_i64:
2393 case INDEX_op_ext_i32_i64:
2394 case INDEX_op_extu_i32_i64:
2395 case INDEX_op_extract_i32:
2396 case INDEX_op_extract_i64:
2397 return C_O1_I1(r, r);
2398
2399 case INDEX_op_qemu_ld_i32:
2400 case INDEX_op_qemu_ld_i64:
2401 return C_O1_I1(r, L);
2402 case INDEX_op_qemu_st_i64:
2403 case INDEX_op_qemu_st_i32:
2404 return C_O0_I2(L, L);
2405
2406 case INDEX_op_deposit_i32:
2407 case INDEX_op_deposit_i64:
2408 return C_O1_I2(r, rZ, r);
2409
2410 case INDEX_op_movcond_i32:
2411 case INDEX_op_movcond_i64:
2412 return (s390_facilities & FACILITY_LOAD_ON_COND2
2413 ? C_O1_I4(r, r, ri, rI, 0)
2414 : C_O1_I4(r, r, ri, r, 0));
2415
2416 case INDEX_op_div2_i32:
2417 case INDEX_op_div2_i64:
2418 case INDEX_op_divu2_i32:
2419 case INDEX_op_divu2_i64:
2420 return C_O2_I3(b, a, 0, 1, r);
2421
2422 case INDEX_op_mulu2_i64:
2423 return C_O2_I2(b, a, 0, r);
2424
2425 case INDEX_op_add2_i32:
2426 case INDEX_op_sub2_i32:
2427 return (s390_facilities & FACILITY_EXT_IMM
2428 ? C_O2_I4(r, r, 0, 1, ri, r)
2429 : C_O2_I4(r, r, 0, 1, r, r));
2430
2431 case INDEX_op_add2_i64:
2432 case INDEX_op_sub2_i64:
2433 return (s390_facilities & FACILITY_EXT_IMM
2434 ? C_O2_I4(r, r, 0, 1, rA, r)
2435 : C_O2_I4(r, r, 0, 1, r, r));
2436
2437 default:
2438 g_assert_not_reached();
2439 }
2440 }
2441
2442 static void query_s390_facilities(void)
2443 {
2444 unsigned long hwcap = qemu_getauxval(AT_HWCAP);
2445
2446 /* Is STORE FACILITY LIST EXTENDED available? Honestly, I believe this
2447 is present on all 64-bit systems, but let's check for it anyway. */
2448 if (hwcap & HWCAP_S390_STFLE) {
2449 register int r0 __asm__("0");
2450 register void *r1 __asm__("1");
2451
2452 /* stfle 0(%r1) */
2453 r1 = &s390_facilities;
2454 asm volatile(".word 0xb2b0,0x1000"
2455 : "=r"(r0) : "0"(0), "r"(r1) : "memory", "cc");
2456 }
2457 }
2458
2459 static void tcg_target_init(TCGContext *s)
2460 {
2461 query_s390_facilities();
2462
2463 tcg_target_available_regs[TCG_TYPE_I32] = 0xffff;
2464 tcg_target_available_regs[TCG_TYPE_I64] = 0xffff;
2465
2466 tcg_target_call_clobber_regs = 0;
2467 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
2468 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R1);
2469 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
2470 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
2471 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
2472 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
2473 /* The r6 register is technically call-saved, but it's also a parameter
2474 register, so it can get killed by setup for the qemu_st helper. */
2475 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
2476 /* The return register can be considered call-clobbered. */
2477 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14);
2478
2479 s->reserved_regs = 0;
2480 tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);
2481 /* XXX many insns can't be used with R0, so we better avoid it for now */
2482 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);
2483 tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
2484 if (USE_REG_TB) {
2485 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB);
2486 }
2487 }
2488
2489 #define FRAME_SIZE ((int)(TCG_TARGET_CALL_STACK_OFFSET \
2490 + TCG_STATIC_CALL_ARGS_SIZE \
2491 + CPU_TEMP_BUF_NLONGS * sizeof(long)))
2492
2493 static void tcg_target_qemu_prologue(TCGContext *s)
2494 {
2495 /* stmg %r6,%r15,48(%r15) (save registers) */
2496 tcg_out_insn(s, RXY, STMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15, 48);
2497
2498 /* aghi %r15,-frame_size */
2499 tcg_out_insn(s, RI, AGHI, TCG_REG_R15, -FRAME_SIZE);
2500
2501 tcg_set_frame(s, TCG_REG_CALL_STACK,
2502 TCG_STATIC_CALL_ARGS_SIZE + TCG_TARGET_CALL_STACK_OFFSET,
2503 CPU_TEMP_BUF_NLONGS * sizeof(long));
2504
2505 #ifndef CONFIG_SOFTMMU
2506 if (guest_base >= 0x80000) {
2507 tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base, true);
2508 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
2509 }
2510 #endif
2511
2512 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
2513 if (USE_REG_TB) {
2514 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB,
2515 tcg_target_call_iarg_regs[1]);
2516 }
2517
2518 /* br %r3 (go to TB) */
2519 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, tcg_target_call_iarg_regs[1]);
2520
2521 /*
2522 * Return path for goto_ptr. Set return value to 0, a-la exit_tb,
2523 * and fall through to the rest of the epilogue.
2524 */
2525 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
2526 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, 0);
2527
2528 /* TB epilogue */
2529 tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
2530
2531 /* lmg %r6,%r15,fs+48(%r15) (restore registers) */
2532 tcg_out_insn(s, RXY, LMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15,
2533 FRAME_SIZE + 48);
2534
2535 /* br %r14 (return) */
2536 tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_R14);
2537 }
2538
2539 static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
2540 {
2541 memset(p, 0x07, count * sizeof(tcg_insn_unit));
2542 }
2543
2544 typedef struct {
2545 DebugFrameHeader h;
2546 uint8_t fde_def_cfa[4];
2547 uint8_t fde_reg_ofs[18];
2548 } DebugFrame;
2549
2550 /* We're expecting a 2 byte uleb128 encoded value. */
2551 QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
2552
2553 #define ELF_HOST_MACHINE EM_S390
2554
2555 static const DebugFrame debug_frame = {
2556 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
2557 .h.cie.id = -1,
2558 .h.cie.version = 1,
2559 .h.cie.code_align = 1,
2560 .h.cie.data_align = 8, /* sleb128 8 */
2561 .h.cie.return_column = TCG_REG_R14,
2562
2563 /* Total FDE size does not include the "len" member. */
2564 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
2565
2566 .fde_def_cfa = {
2567 12, TCG_REG_CALL_STACK, /* DW_CFA_def_cfa %r15, ... */
2568 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
2569 (FRAME_SIZE >> 7)
2570 },
2571 .fde_reg_ofs = {
2572 0x86, 6, /* DW_CFA_offset, %r6, 48 */
2573 0x87, 7, /* DW_CFA_offset, %r7, 56 */
2574 0x88, 8, /* DW_CFA_offset, %r8, 64 */
2575 0x89, 9, /* DW_CFA_offset, %r92, 72 */
2576 0x8a, 10, /* DW_CFA_offset, %r10, 80 */
2577 0x8b, 11, /* DW_CFA_offset, %r11, 88 */
2578 0x8c, 12, /* DW_CFA_offset, %r12, 96 */
2579 0x8d, 13, /* DW_CFA_offset, %r13, 104 */
2580 0x8e, 14, /* DW_CFA_offset, %r14, 112 */
2581 }
2582 };
2583
2584 void tcg_register_jit(const void *buf, size_t buf_size)
2585 {
2586 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
2587 }
QEMU mirror