]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg-op.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
target/s390x: Remove g_out, g_out2, g_in1, g_in2 from DisasContext
[mirror_qemu.git] / tcg / tcg-op.c
CommitLineData
951c6300
RH		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
757e725b 25#include "qemu/osdep.h"
63c91552 26#include "exec/exec-all.h"
dcb32f1d
PMD		 27#include "tcg/tcg.h"
28#include "tcg/tcg-op.h"
29#include "tcg/tcg-mo.h"
e6d86bed 30#include "exec/plugin-gen.h"
d56fea79 31#include "tcg-internal.h"
951c6300
RH		 32
33
b7e8b17a 34void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
951c6300 35{
d4478943 36 TCGOp *op = tcg_emit_op(opc, 1);
75e8b9b7 37 op->args[0] = a1;
951c6300
RH		 38}
39
b7e8b17a 40void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
951c6300 41{
d4478943 42 TCGOp *op = tcg_emit_op(opc, 2);
75e8b9b7
RH		 43 op->args[0] = a1;
44 op->args[1] = a2;
951c6300
RH		 45}
46
b7e8b17a 47void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
951c6300 48{
d4478943 49 TCGOp *op = tcg_emit_op(opc, 3);
75e8b9b7
RH		 50 op->args[0] = a1;
51 op->args[1] = a2;
52 op->args[2] = a3;
951c6300
RH		 53}
54
b7e8b17a 55void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
951c6300 56{
d4478943 57 TCGOp *op = tcg_emit_op(opc, 4);
75e8b9b7
RH		 58 op->args[0] = a1;
59 op->args[1] = a2;
60 op->args[2] = a3;
61 op->args[3] = a4;
951c6300
RH		 62}
63
b7e8b17a
RH		 64void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
65 TCGArg a4, TCGArg a5)
951c6300 66{
d4478943 67 TCGOp *op = tcg_emit_op(opc, 5);
75e8b9b7
RH		 68 op->args[0] = a1;
69 op->args[1] = a2;
70 op->args[2] = a3;
71 op->args[3] = a4;
72 op->args[4] = a5;
951c6300
RH		 73}
74
b7e8b17a
RH		 75void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
76 TCGArg a4, TCGArg a5, TCGArg a6)
951c6300 77{
d4478943 78 TCGOp *op = tcg_emit_op(opc, 6);
75e8b9b7
RH		 79 op->args[0] = a1;
80 op->args[1] = a2;
81 op->args[2] = a3;
82 op->args[3] = a4;
83 op->args[4] = a5;
84 op->args[5] = a6;
951c6300
RH		 85}
86
f85b1fc4
RH		 87/* Generic ops. */
88
89static void add_last_as_label_use(TCGLabel *l)
90{
91 TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse));
92
93 u->op = tcg_last_op();
94 QSIMPLEQ_INSERT_TAIL(&l->branches, u, next);
95}
96
97void tcg_gen_br(TCGLabel *l)
98{
99 tcg_gen_op1(INDEX_op_br, label_arg(l));
100 add_last_as_label_use(l);
101}
102
f65e19bc
PK		 103void tcg_gen_mb(TCGBar mb_type)
104{
b7e4afbd 105 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) {
b7e8b17a 106 tcg_gen_op1(INDEX_op_mb, mb_type);
f65e19bc
PK		 107 }
108}
109
951c6300
RH		 110/* 32 bit ops */
111
11d11d61
RH		 112void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
113{
114 tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
115}
116
951c6300
RH		 117void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
118{
119 /* some cases can be optimized here */
120 if (arg2 == 0) {
121 tcg_gen_mov_i32(ret, arg1);
122 } else {
11d11d61 123 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 124 }
125}
126
127void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
128{
129 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
130 /* Don't recurse with tcg_gen_neg_i32. */
131 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
132 } else {
11d11d61 133 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
951c6300
RH		 134 }
135}
136
137void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
138{
139 /* some cases can be optimized here */
140 if (arg2 == 0) {
141 tcg_gen_mov_i32(ret, arg1);
142 } else {
11d11d61 143 tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 144 }
145}
146
474b2e8f 147void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 148{
951c6300
RH		 149 /* Some cases can be optimized here. */
150 switch (arg2) {
151 case 0:
152 tcg_gen_movi_i32(ret, 0);
153 return;
474b2e8f 154 case -1:
951c6300
RH		 155 tcg_gen_mov_i32(ret, arg1);
156 return;
474b2e8f 157 case 0xff:
951c6300
RH		 158 /* Don't recurse with tcg_gen_ext8u_i32. */
159 if (TCG_TARGET_HAS_ext8u_i32) {
160 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
161 return;
162 }
163 break;
474b2e8f 164 case 0xffff:
951c6300
RH		 165 if (TCG_TARGET_HAS_ext16u_i32) {
166 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
167 return;
168 }
169 break;
170 }
11d11d61
RH		 171
172 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 173}
174
175void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
176{
177 /* Some cases can be optimized here. */
178 if (arg2 == -1) {
179 tcg_gen_movi_i32(ret, -1);
180 } else if (arg2 == 0) {
181 tcg_gen_mov_i32(ret, arg1);
182 } else {
11d11d61 183 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 184 }
185}
186
187void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
188{
189 /* Some cases can be optimized here. */
190 if (arg2 == 0) {
191 tcg_gen_mov_i32(ret, arg1);
192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
193 /* Don't recurse with tcg_gen_not_i32. */
194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
195 } else {
11d11d61 196 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 197 }
198}
199
474b2e8f 200void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 201{
474b2e8f 202 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 203 if (arg2 == 0) {
204 tcg_gen_mov_i32(ret, arg1);
205 } else {
11d11d61 206 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 207 }
208}
209
474b2e8f 210void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 211{
474b2e8f 212 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 213 if (arg2 == 0) {
214 tcg_gen_mov_i32(ret, arg1);
215 } else {
11d11d61 216 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 217 }
218}
219
474b2e8f 220void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 221{
474b2e8f 222 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 223 if (arg2 == 0) {
224 tcg_gen_mov_i32(ret, arg1);
225 } else {
11d11d61 226 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 227 }
228}
229
42a268c2 230void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
951c6300
RH		 231{
232 if (cond == TCG_COND_ALWAYS) {
42a268c2 233 tcg_gen_br(l);
951c6300 234 } else if (cond != TCG_COND_NEVER) {
42a268c2 235 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
f85b1fc4 236 add_last_as_label_use(l);
951c6300
RH		 237 }
238}
239
42a268c2 240void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
951c6300 241{
37ed3bf1
RH		 242 if (cond == TCG_COND_ALWAYS) {
243 tcg_gen_br(l);
244 } else if (cond != TCG_COND_NEVER) {
11d11d61 245 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
37ed3bf1 246 }
951c6300
RH		 247}
248
249void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
250 TCGv_i32 arg1, TCGv_i32 arg2)
251{
252 if (cond == TCG_COND_ALWAYS) {
253 tcg_gen_movi_i32(ret, 1);
254 } else if (cond == TCG_COND_NEVER) {
255 tcg_gen_movi_i32(ret, 0);
256 } else {
257 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
258 }
259}
260
261void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
262 TCGv_i32 arg1, int32_t arg2)
263{
11d11d61 264 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 265}
266
267void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
268{
b2e3ae94
RH		 269 if (arg2 == 0) {
270 tcg_gen_movi_i32(ret, 0);
271 } else if (is_power_of_2(arg2)) {
272 tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
273 } else {
11d11d61 274 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
b2e3ae94 275 }
951c6300
RH		 276}
277
278void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
279{
280 if (TCG_TARGET_HAS_div_i32) {
281 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
282 } else if (TCG_TARGET_HAS_div2_i32) {
5dd48602 283 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951c6300
RH		 284 tcg_gen_sari_i32(t0, arg1, 31);
285 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
286 tcg_temp_free_i32(t0);
287 } else {
288 gen_helper_div_i32(ret, arg1, arg2);
289 }
290}
291
292void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
293{
294 if (TCG_TARGET_HAS_rem_i32) {
295 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
296 } else if (TCG_TARGET_HAS_div_i32) {
5dd48602 297 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951c6300
RH		 298 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
299 tcg_gen_mul_i32(t0, t0, arg2);
300 tcg_gen_sub_i32(ret, arg1, t0);
301 tcg_temp_free_i32(t0);
302 } else if (TCG_TARGET_HAS_div2_i32) {
5dd48602 303 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951c6300
RH		 304 tcg_gen_sari_i32(t0, arg1, 31);
305 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
306 tcg_temp_free_i32(t0);
307 } else {
308 gen_helper_rem_i32(ret, arg1, arg2);
309 }
310}
311
312void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
313{
314 if (TCG_TARGET_HAS_div_i32) {
315 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
316 } else if (TCG_TARGET_HAS_div2_i32) {
5dd48602 317 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951c6300
RH		 318 tcg_gen_movi_i32(t0, 0);
319 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
320 tcg_temp_free_i32(t0);
321 } else {
322 gen_helper_divu_i32(ret, arg1, arg2);
323 }
324}
325
326void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
327{
328 if (TCG_TARGET_HAS_rem_i32) {
329 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
330 } else if (TCG_TARGET_HAS_div_i32) {
5dd48602 331 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951c6300
RH		 332 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
333 tcg_gen_mul_i32(t0, t0, arg2);
334 tcg_gen_sub_i32(ret, arg1, t0);
335 tcg_temp_free_i32(t0);
336 } else if (TCG_TARGET_HAS_div2_i32) {
5dd48602 337 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951c6300
RH		 338 tcg_gen_movi_i32(t0, 0);
339 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
340 tcg_temp_free_i32(t0);
341 } else {
342 gen_helper_remu_i32(ret, arg1, arg2);
343 }
344}
345
346void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
347{
348 if (TCG_TARGET_HAS_andc_i32) {
349 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
350 } else {
5dd48602 351 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951c6300
RH		 352 tcg_gen_not_i32(t0, arg2);
353 tcg_gen_and_i32(ret, arg1, t0);
354 tcg_temp_free_i32(t0);
355 }
356}
357
358void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
359{
360 if (TCG_TARGET_HAS_eqv_i32) {
361 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
362 } else {
363 tcg_gen_xor_i32(ret, arg1, arg2);
364 tcg_gen_not_i32(ret, ret);
365 }
366}
367
368void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
369{
370 if (TCG_TARGET_HAS_nand_i32) {
371 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
372 } else {
373 tcg_gen_and_i32(ret, arg1, arg2);
374 tcg_gen_not_i32(ret, ret);
375 }
376}
377
378void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
379{
380 if (TCG_TARGET_HAS_nor_i32) {
381 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
382 } else {
383 tcg_gen_or_i32(ret, arg1, arg2);
384 tcg_gen_not_i32(ret, ret);
385 }
386}
387
388void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
389{
390 if (TCG_TARGET_HAS_orc_i32) {
391 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
392 } else {
5dd48602 393 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951c6300
RH		 394 tcg_gen_not_i32(t0, arg2);
395 tcg_gen_or_i32(ret, arg1, t0);
396 tcg_temp_free_i32(t0);
397 }
398}
399
0e28d006
RH		 400void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
401{
402 if (TCG_TARGET_HAS_clz_i32) {
403 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
404 } else if (TCG_TARGET_HAS_clz_i64) {
5dd48602
RH		 405 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
406 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
0e28d006
RH		 407 tcg_gen_extu_i32_i64(t1, arg1);
408 tcg_gen_extu_i32_i64(t2, arg2);
409 tcg_gen_addi_i64(t2, t2, 32);
410 tcg_gen_clz_i64(t1, t1, t2);
411 tcg_gen_extrl_i64_i32(ret, t1);
412 tcg_temp_free_i64(t1);
413 tcg_temp_free_i64(t2);
414 tcg_gen_subi_i32(ret, ret, 32);
415 } else {
416 gen_helper_clz_i32(ret, arg1, arg2);
417 }
418}
419
420void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
421{
11d11d61 422 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
0e28d006
RH		 423}
424
425void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
426{
427 if (TCG_TARGET_HAS_ctz_i32) {
428 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
429 } else if (TCG_TARGET_HAS_ctz_i64) {
5dd48602
RH		 430 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
431 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
0e28d006
RH		 432 tcg_gen_extu_i32_i64(t1, arg1);
433 tcg_gen_extu_i32_i64(t2, arg2);
434 tcg_gen_ctz_i64(t1, t1, t2);
435 tcg_gen_extrl_i64_i32(ret, t1);
436 tcg_temp_free_i64(t1);
437 tcg_temp_free_i64(t2);
14e99210
RH		 438 } else if (TCG_TARGET_HAS_ctpop_i32
439 || TCG_TARGET_HAS_ctpop_i64
440 || TCG_TARGET_HAS_clz_i32
441 || TCG_TARGET_HAS_clz_i64) {
5dd48602 442 TCGv_i32 z, t = tcg_temp_ebb_new_i32();
14e99210
RH		 443
444 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
445 tcg_gen_subi_i32(t, arg1, 1);
446 tcg_gen_andc_i32(t, t, arg1);
447 tcg_gen_ctpop_i32(t, t);
448 } else {
449 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
450 tcg_gen_neg_i32(t, arg1);
451 tcg_gen_and_i32(t, t, arg1);
452 tcg_gen_clzi_i32(t, t, 32);
453 tcg_gen_xori_i32(t, t, 31);
454 }
11d11d61 455 z = tcg_constant_i32(0);
14e99210
RH		 456 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
457 tcg_temp_free_i32(t);
0e28d006
RH		 458 } else {
459 gen_helper_ctz_i32(ret, arg1, arg2);
460 }
461}
462
463void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
464{
14e99210
RH		 465 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
466 /* This equivalence has the advantage of not requiring a fixup. */
5dd48602 467 TCGv_i32 t = tcg_temp_ebb_new_i32();
14e99210
RH		 468 tcg_gen_subi_i32(t, arg1, 1);
469 tcg_gen_andc_i32(t, t, arg1);
470 tcg_gen_ctpop_i32(ret, t);
471 tcg_temp_free_i32(t);
472 } else {
11d11d61 473 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
14e99210 474 }
0e28d006
RH		 475}
476
086920c2
RH		 477void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
478{
479 if (TCG_TARGET_HAS_clz_i32) {
5dd48602 480 TCGv_i32 t = tcg_temp_ebb_new_i32();
086920c2
RH		 481 tcg_gen_sari_i32(t, arg, 31);
482 tcg_gen_xor_i32(t, t, arg);
483 tcg_gen_clzi_i32(t, t, 32);
484 tcg_gen_subi_i32(ret, t, 1);
485 tcg_temp_free_i32(t);
486 } else {
487 gen_helper_clrsb_i32(ret, arg);
488 }
489}
490
a768e4e9
RH		 491void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
492{
493 if (TCG_TARGET_HAS_ctpop_i32) {
494 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
495 } else if (TCG_TARGET_HAS_ctpop_i64) {
5dd48602 496 TCGv_i64 t = tcg_temp_ebb_new_i64();
a768e4e9
RH		 497 tcg_gen_extu_i32_i64(t, arg1);
498 tcg_gen_ctpop_i64(t, t);
499 tcg_gen_extrl_i64_i32(ret, t);
500 tcg_temp_free_i64(t);
501 } else {
502 gen_helper_ctpop_i32(ret, arg1);
503 }
504}
505
951c6300
RH		 506void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
507{
508 if (TCG_TARGET_HAS_rot_i32) {
509 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
510 } else {
511 TCGv_i32 t0, t1;
512
5dd48602
RH		 513 t0 = tcg_temp_ebb_new_i32();
514 t1 = tcg_temp_ebb_new_i32();
951c6300
RH		 515 tcg_gen_shl_i32(t0, arg1, arg2);
516 tcg_gen_subfi_i32(t1, 32, arg2);
517 tcg_gen_shr_i32(t1, arg1, t1);
518 tcg_gen_or_i32(ret, t0, t1);
519 tcg_temp_free_i32(t0);
520 tcg_temp_free_i32(t1);
521 }
522}
523
07dada03 524void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 525{
07dada03 526 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 527 /* some cases can be optimized here */
528 if (arg2 == 0) {
529 tcg_gen_mov_i32(ret, arg1);
530 } else if (TCG_TARGET_HAS_rot_i32) {
11d11d61 531 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 532 } else {
533 TCGv_i32 t0, t1;
5dd48602
RH		 534 t0 = tcg_temp_ebb_new_i32();
535 t1 = tcg_temp_ebb_new_i32();
951c6300
RH		 536 tcg_gen_shli_i32(t0, arg1, arg2);
537 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
538 tcg_gen_or_i32(ret, t0, t1);
539 tcg_temp_free_i32(t0);
540 tcg_temp_free_i32(t1);
541 }
542}
543
544void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
545{
546 if (TCG_TARGET_HAS_rot_i32) {
547 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
548 } else {
549 TCGv_i32 t0, t1;
550
5dd48602
RH		 551 t0 = tcg_temp_ebb_new_i32();
552 t1 = tcg_temp_ebb_new_i32();
951c6300
RH		 553 tcg_gen_shr_i32(t0, arg1, arg2);
554 tcg_gen_subfi_i32(t1, 32, arg2);
555 tcg_gen_shl_i32(t1, arg1, t1);
556 tcg_gen_or_i32(ret, t0, t1);
557 tcg_temp_free_i32(t0);
558 tcg_temp_free_i32(t1);
559 }
560}
561
07dada03 562void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 563{
07dada03 564 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 565 /* some cases can be optimized here */
566 if (arg2 == 0) {
567 tcg_gen_mov_i32(ret, arg1);
568 } else {
569 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
570 }
571}
572
573void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
574 unsigned int ofs, unsigned int len)
575{
576 uint32_t mask;
577 TCGv_i32 t1;
578
579 tcg_debug_assert(ofs < 32);
0d0d309d 580 tcg_debug_assert(len > 0);
951c6300
RH		 581 tcg_debug_assert(len <= 32);
582 tcg_debug_assert(ofs + len <= 32);
583
0d0d309d 584 if (len == 32) {
951c6300
RH		 585 tcg_gen_mov_i32(ret, arg2);
586 return;
587 }
588 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
589 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
590 return;
591 }
592
5dd48602 593 t1 = tcg_temp_ebb_new_i32();
951c6300 594
b0a60567
RH		 595 if (TCG_TARGET_HAS_extract2_i32) {
596 if (ofs + len == 32) {
597 tcg_gen_shli_i32(t1, arg1, len);
598 tcg_gen_extract2_i32(ret, t1, arg2, len);
599 goto done;
600 }
601 if (ofs == 0) {
602 tcg_gen_extract2_i32(ret, arg1, arg2, len);
603 tcg_gen_rotli_i32(ret, ret, len);
604 goto done;
605 }
606 }
607
608 mask = (1u << len) - 1;
951c6300
RH		 609 if (ofs + len < 32) {
610 tcg_gen_andi_i32(t1, arg2, mask);
611 tcg_gen_shli_i32(t1, t1, ofs);
612 } else {
613 tcg_gen_shli_i32(t1, arg2, ofs);
614 }
615 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
616 tcg_gen_or_i32(ret, ret, t1);
b0a60567 617 done:
951c6300
RH		 618 tcg_temp_free_i32(t1);
619}
620
07cc68d5
RH		 621void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
622 unsigned int ofs, unsigned int len)
623{
624 tcg_debug_assert(ofs < 32);
625 tcg_debug_assert(len > 0);
626 tcg_debug_assert(len <= 32);
627 tcg_debug_assert(ofs + len <= 32);
628
629 if (ofs + len == 32) {
630 tcg_gen_shli_i32(ret, arg, ofs);
631 } else if (ofs == 0) {
632 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
633 } else if (TCG_TARGET_HAS_deposit_i32
634 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
11d11d61 635 TCGv_i32 zero = tcg_constant_i32(0);
07cc68d5 636 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
07cc68d5
RH		 637 } else {
638 /* To help two-operand hosts we prefer to zero-extend first,
639 which allows ARG to stay live. */
640 switch (len) {
641 case 16:
642 if (TCG_TARGET_HAS_ext16u_i32) {
643 tcg_gen_ext16u_i32(ret, arg);
644 tcg_gen_shli_i32(ret, ret, ofs);
645 return;
646 }
647 break;
648 case 8:
649 if (TCG_TARGET_HAS_ext8u_i32) {
650 tcg_gen_ext8u_i32(ret, arg);
651 tcg_gen_shli_i32(ret, ret, ofs);
652 return;
653 }
654 break;
655 }
656 /* Otherwise prefer zero-extension over AND for code size. */
657 switch (ofs + len) {
658 case 16:
659 if (TCG_TARGET_HAS_ext16u_i32) {
660 tcg_gen_shli_i32(ret, arg, ofs);
661 tcg_gen_ext16u_i32(ret, ret);
662 return;
663 }
664 break;
665 case 8:
666 if (TCG_TARGET_HAS_ext8u_i32) {
667 tcg_gen_shli_i32(ret, arg, ofs);
668 tcg_gen_ext8u_i32(ret, ret);
669 return;
670 }
671 break;
672 }
673 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
674 tcg_gen_shli_i32(ret, ret, ofs);
675 }
676}
677
7ec8bab3
RH		 678void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
679 unsigned int ofs, unsigned int len)
680{
681 tcg_debug_assert(ofs < 32);
682 tcg_debug_assert(len > 0);
683 tcg_debug_assert(len <= 32);
684 tcg_debug_assert(ofs + len <= 32);
685
686 /* Canonicalize certain special cases, even if extract is supported. */
687 if (ofs + len == 32) {
688 tcg_gen_shri_i32(ret, arg, 32 - len);
689 return;
690 }
691 if (ofs == 0) {
692 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
693 return;
694 }
695
696 if (TCG_TARGET_HAS_extract_i32
697 && TCG_TARGET_extract_i32_valid(ofs, len)) {
698 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
699 return;
700 }
701
702 /* Assume that zero-extension, if available, is cheaper than a shift. */
703 switch (ofs + len) {
704 case 16:
705 if (TCG_TARGET_HAS_ext16u_i32) {
706 tcg_gen_ext16u_i32(ret, arg);
707 tcg_gen_shri_i32(ret, ret, ofs);
708 return;
709 }
710 break;
711 case 8:
712 if (TCG_TARGET_HAS_ext8u_i32) {
713 tcg_gen_ext8u_i32(ret, arg);
714 tcg_gen_shri_i32(ret, ret, ofs);
715 return;
716 }
717 break;
718 }
719
720 /* ??? Ideally we'd know what values are available for immediate AND.
721 Assume that 8 bits are available, plus the special case of 16,
722 so that we get ext8u, ext16u. */
723 switch (len) {
724 case 1 ... 8: case 16:
725 tcg_gen_shri_i32(ret, arg, ofs);
726 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
727 break;
728 default:
729 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
730 tcg_gen_shri_i32(ret, ret, 32 - len);
731 break;
732 }
733}
734
735void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
736 unsigned int ofs, unsigned int len)
737{
738 tcg_debug_assert(ofs < 32);
739 tcg_debug_assert(len > 0);
740 tcg_debug_assert(len <= 32);
741 tcg_debug_assert(ofs + len <= 32);
742
743 /* Canonicalize certain special cases, even if extract is supported. */
744 if (ofs + len == 32) {
745 tcg_gen_sari_i32(ret, arg, 32 - len);
746 return;
747 }
748 if (ofs == 0) {
749 switch (len) {
750 case 16:
751 tcg_gen_ext16s_i32(ret, arg);
752 return;
753 case 8:
754 tcg_gen_ext8s_i32(ret, arg);
755 return;
756 }
757 }
758
759 if (TCG_TARGET_HAS_sextract_i32
760 && TCG_TARGET_extract_i32_valid(ofs, len)) {
761 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
762 return;
763 }
764
765 /* Assume that sign-extension, if available, is cheaper than a shift. */
766 switch (ofs + len) {
767 case 16:
768 if (TCG_TARGET_HAS_ext16s_i32) {
769 tcg_gen_ext16s_i32(ret, arg);
770 tcg_gen_sari_i32(ret, ret, ofs);
771 return;
772 }
773 break;
774 case 8:
775 if (TCG_TARGET_HAS_ext8s_i32) {
776 tcg_gen_ext8s_i32(ret, arg);
777 tcg_gen_sari_i32(ret, ret, ofs);
778 return;
779 }
780 break;
781 }
782 switch (len) {
783 case 16:
784 if (TCG_TARGET_HAS_ext16s_i32) {
785 tcg_gen_shri_i32(ret, arg, ofs);
786 tcg_gen_ext16s_i32(ret, ret);
787 return;
788 }
789 break;
790 case 8:
791 if (TCG_TARGET_HAS_ext8s_i32) {
792 tcg_gen_shri_i32(ret, arg, ofs);
793 tcg_gen_ext8s_i32(ret, ret);
794 return;
795 }
796 break;
797 }
798
799 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
800 tcg_gen_sari_i32(ret, ret, 32 - len);
801}
802
2089fcc9
DH		 803/*
804 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
805 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
806 */
807void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
808 unsigned int ofs)
809{
810 tcg_debug_assert(ofs <= 32);
811 if (ofs == 0) {
812 tcg_gen_mov_i32(ret, al);
813 } else if (ofs == 32) {
814 tcg_gen_mov_i32(ret, ah);
815 } else if (al == ah) {
816 tcg_gen_rotri_i32(ret, al, ofs);
fce1296f
RH		 817 } else if (TCG_TARGET_HAS_extract2_i32) {
818 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
2089fcc9 819 } else {
5dd48602 820 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
2089fcc9
DH		 821 tcg_gen_shri_i32(t0, al, ofs);
822 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
823 tcg_temp_free_i32(t0);
824 }
825}
826
951c6300
RH		 827void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
828 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
829{
37ed3bf1
RH		 830 if (cond == TCG_COND_ALWAYS) {
831 tcg_gen_mov_i32(ret, v1);
832 } else if (cond == TCG_COND_NEVER) {
833 tcg_gen_mov_i32(ret, v2);
834 } else if (TCG_TARGET_HAS_movcond_i32) {
951c6300
RH		 835 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
836 } else {
5dd48602
RH		 837 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
838 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
951c6300
RH		 839 tcg_gen_setcond_i32(cond, t0, c1, c2);
840 tcg_gen_neg_i32(t0, t0);
841 tcg_gen_and_i32(t1, v1, t0);
842 tcg_gen_andc_i32(ret, v2, t0);
843 tcg_gen_or_i32(ret, ret, t1);
844 tcg_temp_free_i32(t0);
845 tcg_temp_free_i32(t1);
846 }
847}
848
849void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
850 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
851{
852 if (TCG_TARGET_HAS_add2_i32) {
853 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
951c6300 854 } else {
5dd48602
RH		 855 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
856 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 857 tcg_gen_concat_i32_i64(t0, al, ah);
858 tcg_gen_concat_i32_i64(t1, bl, bh);
859 tcg_gen_add_i64(t0, t0, t1);
860 tcg_gen_extr_i64_i32(rl, rh, t0);
861 tcg_temp_free_i64(t0);
862 tcg_temp_free_i64(t1);
863 }
864}
865
866void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
867 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
868{
869 if (TCG_TARGET_HAS_sub2_i32) {
870 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
951c6300 871 } else {
5dd48602
RH		 872 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
873 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 874 tcg_gen_concat_i32_i64(t0, al, ah);
875 tcg_gen_concat_i32_i64(t1, bl, bh);
876 tcg_gen_sub_i64(t0, t0, t1);
877 tcg_gen_extr_i64_i32(rl, rh, t0);
878 tcg_temp_free_i64(t0);
879 tcg_temp_free_i64(t1);
880 }
881}
882
883void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
884{
885 if (TCG_TARGET_HAS_mulu2_i32) {
886 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
951c6300 887 } else if (TCG_TARGET_HAS_muluh_i32) {
5dd48602 888 TCGv_i32 t = tcg_temp_ebb_new_i32();
951c6300
RH		 889 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
890 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
891 tcg_gen_mov_i32(rl, t);
892 tcg_temp_free_i32(t);
9fd86b51 893 } else if (TCG_TARGET_REG_BITS == 64) {
5dd48602
RH		 894 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
895 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 896 tcg_gen_extu_i32_i64(t0, arg1);
897 tcg_gen_extu_i32_i64(t1, arg2);
898 tcg_gen_mul_i64(t0, t0, t1);
899 tcg_gen_extr_i64_i32(rl, rh, t0);
900 tcg_temp_free_i64(t0);
901 tcg_temp_free_i64(t1);
9fd86b51
RH		 902 } else {
903 qemu_build_not_reached();
951c6300
RH		 904 }
905}
906
907void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
908{
909 if (TCG_TARGET_HAS_muls2_i32) {
910 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
951c6300 911 } else if (TCG_TARGET_HAS_mulsh_i32) {
5dd48602 912 TCGv_i32 t = tcg_temp_ebb_new_i32();
951c6300
RH		 913 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
914 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
915 tcg_gen_mov_i32(rl, t);
916 tcg_temp_free_i32(t);
917 } else if (TCG_TARGET_REG_BITS == 32) {
5dd48602
RH		 918 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
919 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
920 TCGv_i32 t2 = tcg_temp_ebb_new_i32();
921 TCGv_i32 t3 = tcg_temp_ebb_new_i32();
951c6300
RH		 922 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
923 /* Adjust for negative inputs. */
924 tcg_gen_sari_i32(t2, arg1, 31);
925 tcg_gen_sari_i32(t3, arg2, 31);
926 tcg_gen_and_i32(t2, t2, arg2);
927 tcg_gen_and_i32(t3, t3, arg1);
928 tcg_gen_sub_i32(rh, t1, t2);
929 tcg_gen_sub_i32(rh, rh, t3);
930 tcg_gen_mov_i32(rl, t0);
931 tcg_temp_free_i32(t0);
932 tcg_temp_free_i32(t1);
933 tcg_temp_free_i32(t2);
934 tcg_temp_free_i32(t3);
935 } else {
5dd48602
RH		 936 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
937 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 938 tcg_gen_ext_i32_i64(t0, arg1);
939 tcg_gen_ext_i32_i64(t1, arg2);
940 tcg_gen_mul_i64(t0, t0, t1);
941 tcg_gen_extr_i64_i32(rl, rh, t0);
942 tcg_temp_free_i64(t0);
943 tcg_temp_free_i64(t1);
944 }
945}
946
5087abfb
RH		 947void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
948{
949 if (TCG_TARGET_REG_BITS == 32) {
5dd48602
RH		 950 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
952 TCGv_i32 t2 = tcg_temp_ebb_new_i32();
5087abfb
RH		 953 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
954 /* Adjust for negative input for the signed arg1. */
955 tcg_gen_sari_i32(t2, arg1, 31);
956 tcg_gen_and_i32(t2, t2, arg2);
957 tcg_gen_sub_i32(rh, t1, t2);
958 tcg_gen_mov_i32(rl, t0);
959 tcg_temp_free_i32(t0);
960 tcg_temp_free_i32(t1);
961 tcg_temp_free_i32(t2);
962 } else {
5dd48602
RH		 963 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
964 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
5087abfb
RH		 965 tcg_gen_ext_i32_i64(t0, arg1);
966 tcg_gen_extu_i32_i64(t1, arg2);
967 tcg_gen_mul_i64(t0, t0, t1);
968 tcg_gen_extr_i64_i32(rl, rh, t0);
969 tcg_temp_free_i64(t0);
970 tcg_temp_free_i64(t1);
971 }
972}
973
951c6300
RH		 974void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
975{
976 if (TCG_TARGET_HAS_ext8s_i32) {
977 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
978 } else {
979 tcg_gen_shli_i32(ret, arg, 24);
980 tcg_gen_sari_i32(ret, ret, 24);
981 }
982}
983
984void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
985{
986 if (TCG_TARGET_HAS_ext16s_i32) {
987 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
988 } else {
989 tcg_gen_shli_i32(ret, arg, 16);
990 tcg_gen_sari_i32(ret, ret, 16);
991 }
992}
993
994void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
995{
996 if (TCG_TARGET_HAS_ext8u_i32) {
997 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
998 } else {
999 tcg_gen_andi_i32(ret, arg, 0xffu);
1000 }
1001}
1002
1003void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
1004{
1005 if (TCG_TARGET_HAS_ext16u_i32) {
1006 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
1007 } else {
1008 tcg_gen_andi_i32(ret, arg, 0xffffu);
1009 }
1010}
1011
2b836c2a 1012void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
951c6300 1013{
2b836c2a
RH		 1014 /* Only one extension flag may be present. */
1015 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1016
951c6300 1017 if (TCG_TARGET_HAS_bswap16_i32) {
2b836c2a 1018 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags);
951c6300 1019 } else {
5dd48602
RH		 1020 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1021 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
2b836c2a
RH		 1022
1023 tcg_gen_shri_i32(t0, arg, 8);
1024 if (!(flags & TCG_BSWAP_IZ)) {
1025 tcg_gen_ext8u_i32(t0, t0);
1026 }
951c6300 1027
2b836c2a
RH		 1028 if (flags & TCG_BSWAP_OS) {
1029 tcg_gen_shli_i32(t1, arg, 24);
1030 tcg_gen_sari_i32(t1, t1, 16);
1031 } else if (flags & TCG_BSWAP_OZ) {
1032 tcg_gen_ext8u_i32(t1, arg);
1033 tcg_gen_shli_i32(t1, t1, 8);
1034 } else {
1035 tcg_gen_shli_i32(t1, arg, 8);
1036 }
1037
1038 tcg_gen_or_i32(ret, t0, t1);
951c6300 1039 tcg_temp_free_i32(t0);
2b836c2a 1040 tcg_temp_free_i32(t1);
951c6300
RH		 1041 }
1042}
1043
1044void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1045{
1046 if (TCG_TARGET_HAS_bswap32_i32) {
587195bd 1047 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0);
951c6300 1048 } else {
5dd48602
RH		 1049 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1050 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
11d11d61 1051 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff);
951c6300 1052
a686dc71
RH		 1053 /* arg = abcd */
1054 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */
1055 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */
1056 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */
1057 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */
1058 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */
951c6300 1059
a686dc71
RH		 1060 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */
1061 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */
1062 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */
951c6300 1063
951c6300
RH		 1064 tcg_temp_free_i32(t0);
1065 tcg_temp_free_i32(t1);
1066 }
1067}
1068
46be8425
RH		 1069void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg)
1070{
1071 /* Swapping 2 16-bit elements is a rotate. */
1072 tcg_gen_rotli_i32(ret, arg, 16);
1073}
1074
b87fb8cd
RH		 1075void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1076{
1077 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1078}
1079
1080void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1081{
1082 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1083}
1084
1085void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1086{
1087 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1088}
1089
1090void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1091{
1092 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1093}
1094
ff1f11f7
RH		 1095void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1096{
5dd48602 1097 TCGv_i32 t = tcg_temp_ebb_new_i32();
ff1f11f7
RH		 1098
1099 tcg_gen_sari_i32(t, a, 31);
1100 tcg_gen_xor_i32(ret, a, t);
1101 tcg_gen_sub_i32(ret, ret, t);
1102 tcg_temp_free_i32(t);
1103}
1104
951c6300
RH		 1105/* 64-bit ops */
1106
1107#if TCG_TARGET_REG_BITS == 32
1108/* These are all inline for TCG_TARGET_REG_BITS == 64. */
1109
1110void tcg_gen_discard_i64(TCGv_i64 arg)
1111{
1112 tcg_gen_discard_i32(TCGV_LOW(arg));
1113 tcg_gen_discard_i32(TCGV_HIGH(arg));
1114}
1115
1116void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1117{
11d11d61
RH		 1118 TCGTemp *ts = tcgv_i64_temp(arg);
1119
1120 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1121 if (ts->kind == TEMP_CONST) {
1122 tcg_gen_movi_i64(ret, ts->val);
1123 } else {
1124 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1125 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1126 }
951c6300
RH		 1127}
1128
1129void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1130{
1131 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1132 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1133}
1134
1135void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1136{
1137 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1138 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1139}
1140
1141void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1142{
1143 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
3ff91d7e 1144 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
951c6300
RH		 1145}
1146
1147void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1148{
1149 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1150 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1151}
1152
1153void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1154{
1155 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1156 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1157}
1158
1159void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1160{
1161 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1162 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1163}
1164
1165void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1166{
1167 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1168 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1169}
1170
1171void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1172{
1173 /* Since arg2 and ret have different types,
1174 they cannot be the same temporary */
e03b5686 1175#if HOST_BIG_ENDIAN
951c6300
RH		 1176 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1177 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1178#else
1179 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1180 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1181#endif
1182}
1183
d56fea79
RH		 1184void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1185{
1186 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset);
1187}
1188
1189void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1190{
1191 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset);
1192}
1193
1194void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1195{
1196 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1197}
1198
951c6300
RH		 1199void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1200{
e03b5686 1201#if HOST_BIG_ENDIAN
951c6300
RH		 1202 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1203 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1204#else
1205 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1206 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1207#endif
1208}
1209
d56fea79
RH		 1210void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1211{
1212 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1),
1213 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2));
1214}
1215
1216void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1217{
1218 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1),
1219 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2));
1220}
1221
951c6300
RH		 1222void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1223{
1224 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1225 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1226}
1227
1228void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1229{
1230 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1231 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1232}
1233
1234void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1235{
1236 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1237 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1238}
1239
1240void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1241{
1242 gen_helper_shl_i64(ret, arg1, arg2);
1243}
1244
1245void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1246{
1247 gen_helper_shr_i64(ret, arg1, arg2);
1248}
1249
1250void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1251{
1252 gen_helper_sar_i64(ret, arg1, arg2);
1253}
1254
1255void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1256{
1257 TCGv_i64 t0;
1258 TCGv_i32 t1;
1259
5dd48602
RH		 1260 t0 = tcg_temp_ebb_new_i64();
1261 t1 = tcg_temp_ebb_new_i32();
951c6300
RH		 1262
1263 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1264 TCGV_LOW(arg1), TCGV_LOW(arg2));
1265
1266 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1267 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1268 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1269 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1270
1271 tcg_gen_mov_i64(ret, t0);
1272 tcg_temp_free_i64(t0);
1273 tcg_temp_free_i32(t1);
1274}
11d11d61
RH		 1275
1276#else
1277
1278void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1279{
1280 tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
1281}
1282
951c6300
RH		 1283#endif /* TCG_TARGET_REG_SIZE == 32 */
1284
1285void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1286{
1287 /* some cases can be optimized here */
1288 if (arg2 == 0) {
1289 tcg_gen_mov_i64(ret, arg1);
11d11d61
RH		 1290 } else if (TCG_TARGET_REG_BITS == 64) {
1291 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300 1292 } else {
11d11d61
RH		 1293 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1294 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1295 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
951c6300
RH		 1296 }
1297}
1298
1299void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1300{
1301 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1302 /* Don't recurse with tcg_gen_neg_i64. */
1303 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
11d11d61
RH		 1304 } else if (TCG_TARGET_REG_BITS == 64) {
1305 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
951c6300 1306 } else {
11d11d61
RH		 1307 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1308 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32),
1309 TCGV_LOW(arg2), TCGV_HIGH(arg2));
951c6300
RH		 1310 }
1311}
1312
1313void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1314{
1315 /* some cases can be optimized here */
1316 if (arg2 == 0) {
1317 tcg_gen_mov_i64(ret, arg1);
11d11d61
RH		 1318 } else if (TCG_TARGET_REG_BITS == 64) {
1319 tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300 1320 } else {
11d11d61
RH		 1321 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1322 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1323 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
951c6300
RH		 1324 }
1325}
1326
474b2e8f 1327void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1328{
3a13c3f3
RH		 1329 if (TCG_TARGET_REG_BITS == 32) {
1330 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1331 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1332 return;
1333 }
1334
951c6300
RH		 1335 /* Some cases can be optimized here. */
1336 switch (arg2) {
1337 case 0:
1338 tcg_gen_movi_i64(ret, 0);
1339 return;
474b2e8f 1340 case -1:
951c6300
RH		 1341 tcg_gen_mov_i64(ret, arg1);
1342 return;
474b2e8f 1343 case 0xff:
951c6300
RH		 1344 /* Don't recurse with tcg_gen_ext8u_i64. */
1345 if (TCG_TARGET_HAS_ext8u_i64) {
1346 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1347 return;
1348 }
1349 break;
474b2e8f 1350 case 0xffff:
951c6300
RH		 1351 if (TCG_TARGET_HAS_ext16u_i64) {
1352 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1353 return;
1354 }
1355 break;
474b2e8f 1356 case 0xffffffffu:
951c6300
RH		 1357 if (TCG_TARGET_HAS_ext32u_i64) {
1358 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1359 return;
1360 }
1361 break;
1362 }
11d11d61
RH		 1363
1364 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 1365}
1366
1367void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1368{
3a13c3f3
RH		 1369 if (TCG_TARGET_REG_BITS == 32) {
1370 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1371 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1372 return;
1373 }
951c6300
RH		 1374 /* Some cases can be optimized here. */
1375 if (arg2 == -1) {
1376 tcg_gen_movi_i64(ret, -1);
1377 } else if (arg2 == 0) {
1378 tcg_gen_mov_i64(ret, arg1);
1379 } else {
11d11d61 1380 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300 1381 }
951c6300
RH		 1382}
1383
1384void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1385{
3a13c3f3
RH		 1386 if (TCG_TARGET_REG_BITS == 32) {
1387 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1388 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1389 return;
1390 }
951c6300
RH		 1391 /* Some cases can be optimized here. */
1392 if (arg2 == 0) {
1393 tcg_gen_mov_i64(ret, arg1);
1394 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1395 /* Don't recurse with tcg_gen_not_i64. */
1396 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1397 } else {
11d11d61 1398 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300 1399 }
951c6300
RH		 1400}
1401
951c6300
RH		 1402static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1403 unsigned c, bool right, bool arith)
1404{
1405 tcg_debug_assert(c < 64);
1406 if (c == 0) {
1407 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1408 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1409 } else if (c >= 32) {
1410 c -= 32;
1411 if (right) {
1412 if (arith) {
1413 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1414 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1415 } else {
1416 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1417 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1418 }
1419 } else {
1420 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1421 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1422 }
02616bad
RH		 1423 } else if (right) {
1424 if (TCG_TARGET_HAS_extract2_i32) {
1425 tcg_gen_extract2_i32(TCGV_LOW(ret),
1426 TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1427 } else {
951c6300 1428 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
02616bad
RH		 1429 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1430 TCGV_HIGH(arg1), 32 - c, c);
1431 }
1432 if (arith) {
1433 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1434 } else {
1435 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1436 }
1437 } else {
1438 if (TCG_TARGET_HAS_extract2_i32) {
1439 tcg_gen_extract2_i32(TCGV_HIGH(ret),
1440 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
951c6300 1441 } else {
5dd48602 1442 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
951c6300 1443 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
02616bad
RH		 1444 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1445 TCGV_HIGH(arg1), c, 32 - c);
1446 tcg_temp_free_i32(t0);
951c6300 1447 }
02616bad 1448 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
951c6300
RH		 1449 }
1450}
1451
474b2e8f 1452void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1453{
474b2e8f 1454 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1455 if (TCG_TARGET_REG_BITS == 32) {
1456 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1457 } else if (arg2 == 0) {
951c6300
RH		 1458 tcg_gen_mov_i64(ret, arg1);
1459 } else {
11d11d61 1460 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 1461 }
1462}
1463
474b2e8f 1464void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1465{
474b2e8f 1466 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1467 if (TCG_TARGET_REG_BITS == 32) {
1468 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1469 } else if (arg2 == 0) {
951c6300
RH		 1470 tcg_gen_mov_i64(ret, arg1);
1471 } else {
11d11d61 1472 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 1473 }
1474}
1475
474b2e8f 1476void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1477{
474b2e8f 1478 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1479 if (TCG_TARGET_REG_BITS == 32) {
1480 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1481 } else if (arg2 == 0) {
951c6300
RH		 1482 tcg_gen_mov_i64(ret, arg1);
1483 } else {
11d11d61 1484 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 1485 }
1486}
951c6300 1487
42a268c2 1488void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
951c6300
RH		 1489{
1490 if (cond == TCG_COND_ALWAYS) {
42a268c2 1491 tcg_gen_br(l);
951c6300 1492 } else if (cond != TCG_COND_NEVER) {
3a13c3f3
RH		 1493 if (TCG_TARGET_REG_BITS == 32) {
1494 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1495 TCGV_HIGH(arg1), TCGV_LOW(arg2),
42a268c2 1496 TCGV_HIGH(arg2), cond, label_arg(l));
3a13c3f3 1497 } else {
42a268c2
RH		 1498 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1499 label_arg(l));
3a13c3f3 1500 }
f85b1fc4 1501 add_last_as_label_use(l);
951c6300
RH		 1502 }
1503}
1504
42a268c2 1505void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
951c6300 1506{
11d11d61
RH		 1507 if (TCG_TARGET_REG_BITS == 64) {
1508 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
1509 } else if (cond == TCG_COND_ALWAYS) {
42a268c2 1510 tcg_gen_br(l);
951c6300 1511 } else if (cond != TCG_COND_NEVER) {
11d11d61
RH		 1512 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
1513 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1514 tcg_constant_i32(arg2),
1515 tcg_constant_i32(arg2 >> 32),
1516 cond, label_arg(l));
f85b1fc4 1517 add_last_as_label_use(l);
951c6300
RH		 1518 }
1519}
1520
1521void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1522 TCGv_i64 arg1, TCGv_i64 arg2)
1523{
1524 if (cond == TCG_COND_ALWAYS) {
1525 tcg_gen_movi_i64(ret, 1);
1526 } else if (cond == TCG_COND_NEVER) {
1527 tcg_gen_movi_i64(ret, 0);
1528 } else {
3a13c3f3
RH		 1529 if (TCG_TARGET_REG_BITS == 32) {
1530 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1531 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1532 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1533 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1534 } else {
1535 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1536 }
951c6300
RH		 1537 }
1538}
1539
1540void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1541 TCGv_i64 arg1, int64_t arg2)
1542{
11d11d61
RH		 1543 if (TCG_TARGET_REG_BITS == 64) {
1544 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1545 } else if (cond == TCG_COND_ALWAYS) {
1546 tcg_gen_movi_i64(ret, 1);
1547 } else if (cond == TCG_COND_NEVER) {
1548 tcg_gen_movi_i64(ret, 0);
1549 } else {
1550 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1551 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1552 tcg_constant_i32(arg2),
1553 tcg_constant_i32(arg2 >> 32), cond);
1554 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1555 }
951c6300
RH		 1556}
1557
1558void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1559{
b2e3ae94
RH		 1560 if (arg2 == 0) {
1561 tcg_gen_movi_i64(ret, 0);
1562 } else if (is_power_of_2(arg2)) {
1563 tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1564 } else {
1565 TCGv_i64 t0 = tcg_const_i64(arg2);
1566 tcg_gen_mul_i64(ret, arg1, t0);
1567 tcg_temp_free_i64(t0);
1568 }
951c6300
RH		 1569}
1570
1571void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1572{
1573 if (TCG_TARGET_HAS_div_i64) {
1574 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1575 } else if (TCG_TARGET_HAS_div2_i64) {
5dd48602 1576 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 1577 tcg_gen_sari_i64(t0, arg1, 63);
1578 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1579 tcg_temp_free_i64(t0);
1580 } else {
1581 gen_helper_div_i64(ret, arg1, arg2);
1582 }
1583}
1584
1585void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1586{
1587 if (TCG_TARGET_HAS_rem_i64) {
1588 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1589 } else if (TCG_TARGET_HAS_div_i64) {
5dd48602 1590 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 1591 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1592 tcg_gen_mul_i64(t0, t0, arg2);
1593 tcg_gen_sub_i64(ret, arg1, t0);
1594 tcg_temp_free_i64(t0);
1595 } else if (TCG_TARGET_HAS_div2_i64) {
5dd48602 1596 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 1597 tcg_gen_sari_i64(t0, arg1, 63);
1598 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1599 tcg_temp_free_i64(t0);
1600 } else {
1601 gen_helper_rem_i64(ret, arg1, arg2);
1602 }
1603}
1604
1605void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1606{
1607 if (TCG_TARGET_HAS_div_i64) {
1608 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1609 } else if (TCG_TARGET_HAS_div2_i64) {
5dd48602 1610 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 1611 tcg_gen_movi_i64(t0, 0);
1612 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1613 tcg_temp_free_i64(t0);
1614 } else {
1615 gen_helper_divu_i64(ret, arg1, arg2);
1616 }
1617}
1618
1619void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1620{
1621 if (TCG_TARGET_HAS_rem_i64) {
1622 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1623 } else if (TCG_TARGET_HAS_div_i64) {
5dd48602 1624 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 1625 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1626 tcg_gen_mul_i64(t0, t0, arg2);
1627 tcg_gen_sub_i64(ret, arg1, t0);
1628 tcg_temp_free_i64(t0);
1629 } else if (TCG_TARGET_HAS_div2_i64) {
5dd48602 1630 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 1631 tcg_gen_movi_i64(t0, 0);
1632 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1633 tcg_temp_free_i64(t0);
1634 } else {
1635 gen_helper_remu_i64(ret, arg1, arg2);
1636 }
1637}
1638
1639void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1640{
3a13c3f3
RH		 1641 if (TCG_TARGET_REG_BITS == 32) {
1642 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1643 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1644 } else if (TCG_TARGET_HAS_ext8s_i64) {
951c6300
RH		 1645 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1646 } else {
1647 tcg_gen_shli_i64(ret, arg, 56);
1648 tcg_gen_sari_i64(ret, ret, 56);
1649 }
951c6300
RH		 1650}
1651
1652void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1653{
3a13c3f3
RH		 1654 if (TCG_TARGET_REG_BITS == 32) {
1655 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1656 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1657 } else if (TCG_TARGET_HAS_ext16s_i64) {
951c6300
RH		 1658 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1659 } else {
1660 tcg_gen_shli_i64(ret, arg, 48);
1661 tcg_gen_sari_i64(ret, ret, 48);
1662 }
951c6300
RH		 1663}
1664
1665void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1666{
3a13c3f3
RH		 1667 if (TCG_TARGET_REG_BITS == 32) {
1668 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1669 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1670 } else if (TCG_TARGET_HAS_ext32s_i64) {
951c6300
RH		 1671 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1672 } else {
1673 tcg_gen_shli_i64(ret, arg, 32);
1674 tcg_gen_sari_i64(ret, ret, 32);
1675 }
951c6300
RH		 1676}
1677
1678void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1679{
3a13c3f3
RH		 1680 if (TCG_TARGET_REG_BITS == 32) {
1681 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1682 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1683 } else if (TCG_TARGET_HAS_ext8u_i64) {
951c6300
RH		 1684 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1685 } else {
1686 tcg_gen_andi_i64(ret, arg, 0xffu);
1687 }
951c6300
RH		 1688}
1689
1690void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1691{
3a13c3f3
RH		 1692 if (TCG_TARGET_REG_BITS == 32) {
1693 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1694 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1695 } else if (TCG_TARGET_HAS_ext16u_i64) {
951c6300
RH		 1696 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1697 } else {
1698 tcg_gen_andi_i64(ret, arg, 0xffffu);
1699 }
951c6300
RH		 1700}
1701
1702void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1703{
3a13c3f3
RH		 1704 if (TCG_TARGET_REG_BITS == 32) {
1705 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1706 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1707 } else if (TCG_TARGET_HAS_ext32u_i64) {
951c6300
RH		 1708 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1709 } else {
1710 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1711 }
951c6300
RH		 1712}
1713
2b836c2a 1714void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
951c6300 1715{
2b836c2a
RH		 1716 /* Only one extension flag may be present. */
1717 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1718
3a13c3f3 1719 if (TCG_TARGET_REG_BITS == 32) {
2b836c2a
RH		 1720 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags);
1721 if (flags & TCG_BSWAP_OS) {
1722 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1723 } else {
1724 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1725 }
3a13c3f3 1726 } else if (TCG_TARGET_HAS_bswap16_i64) {
2b836c2a 1727 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags);
951c6300 1728 } else {
5dd48602
RH		 1729 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1730 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
951c6300 1731
2b836c2a
RH		 1732 tcg_gen_shri_i64(t0, arg, 8);
1733 if (!(flags & TCG_BSWAP_IZ)) {
1734 tcg_gen_ext8u_i64(t0, t0);
1735 }
1736
1737 if (flags & TCG_BSWAP_OS) {
1738 tcg_gen_shli_i64(t1, arg, 56);
1739 tcg_gen_sari_i64(t1, t1, 48);
1740 } else if (flags & TCG_BSWAP_OZ) {
1741 tcg_gen_ext8u_i64(t1, arg);
1742 tcg_gen_shli_i64(t1, t1, 8);
1743 } else {
1744 tcg_gen_shli_i64(t1, arg, 8);
1745 }
1746
1747 tcg_gen_or_i64(ret, t0, t1);
951c6300 1748 tcg_temp_free_i64(t0);
2b836c2a 1749 tcg_temp_free_i64(t1);
951c6300 1750 }
951c6300
RH		 1751}
1752
2b836c2a 1753void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
951c6300 1754{
2b836c2a
RH		 1755 /* Only one extension flag may be present. */
1756 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1757
3a13c3f3
RH		 1758 if (TCG_TARGET_REG_BITS == 32) {
1759 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2b836c2a
RH		 1760 if (flags & TCG_BSWAP_OS) {
1761 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1762 } else {
1763 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1764 }
3a13c3f3 1765 } else if (TCG_TARGET_HAS_bswap32_i64) {
2b836c2a 1766 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags);
951c6300 1767 } else {
5dd48602
RH		 1768 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1769 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
11d11d61 1770 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff);
951c6300 1771
2b836c2a
RH		 1772 /* arg = xxxxabcd */
1773 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */
1774 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */
1775 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */
1776 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */
1777 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */
1778
1779 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */
1780 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */
1781 if (flags & TCG_BSWAP_OS) {
1782 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */
1783 } else {
1784 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */
1785 }
1786 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba */
951c6300 1787
951c6300
RH		 1788 tcg_temp_free_i64(t0);
1789 tcg_temp_free_i64(t1);
1790 }
951c6300
RH		 1791}
1792
1793void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1794{
3a13c3f3
RH		 1795 if (TCG_TARGET_REG_BITS == 32) {
1796 TCGv_i32 t0, t1;
5dd48602
RH		 1797 t0 = tcg_temp_ebb_new_i32();
1798 t1 = tcg_temp_ebb_new_i32();
951c6300 1799
3a13c3f3
RH		 1800 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1801 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1802 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1803 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1804 tcg_temp_free_i32(t0);
1805 tcg_temp_free_i32(t1);
1806 } else if (TCG_TARGET_HAS_bswap64_i64) {
587195bd 1807 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0);
951c6300 1808 } else {
5dd48602
RH		 1809 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1810 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1811 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
951c6300 1812
9e821eab
RH		 1813 /* arg = abcdefgh */
1814 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1815 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */
1816 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */
1817 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */
1818 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */
1819 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */
1820
1821 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1822 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */
1823 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */
1824 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */
1825 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */
1826 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */
1827
1828 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */
1829 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */
1830 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */
951c6300 1831
951c6300
RH		 1832 tcg_temp_free_i64(t0);
1833 tcg_temp_free_i64(t1);
9e821eab 1834 tcg_temp_free_i64(t2);
951c6300 1835 }
951c6300
RH		 1836}
1837
46be8425
RH		 1838void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1839{
1840 uint64_t m = 0x0000ffff0000ffffull;
5dd48602
RH		 1841 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1842 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
46be8425
RH		 1843
1844 /* See include/qemu/bitops.h, hswap64. */
1845 tcg_gen_rotli_i64(t1, arg, 32);
1846 tcg_gen_andi_i64(t0, t1, m);
1847 tcg_gen_shli_i64(t0, t0, 16);
1848 tcg_gen_shri_i64(t1, t1, 16);
1849 tcg_gen_andi_i64(t1, t1, m);
1850 tcg_gen_or_i64(ret, t0, t1);
1851
1852 tcg_temp_free_i64(t0);
1853 tcg_temp_free_i64(t1);
1854}
1855
1856void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1857{
1858 /* Swapping 2 32-bit elements is a rotate. */
1859 tcg_gen_rotli_i64(ret, arg, 32);
1860}
1861
951c6300
RH		 1862void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1863{
3a13c3f3
RH		 1864 if (TCG_TARGET_REG_BITS == 32) {
1865 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1866 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1867 } else if (TCG_TARGET_HAS_not_i64) {
951c6300
RH		 1868 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1869 } else {
1870 tcg_gen_xori_i64(ret, arg, -1);
1871 }
951c6300
RH		 1872}
1873
1874void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1875{
3a13c3f3
RH		 1876 if (TCG_TARGET_REG_BITS == 32) {
1877 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1878 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1879 } else if (TCG_TARGET_HAS_andc_i64) {
951c6300
RH		 1880 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1881 } else {
5dd48602 1882 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 1883 tcg_gen_not_i64(t0, arg2);
1884 tcg_gen_and_i64(ret, arg1, t0);
1885 tcg_temp_free_i64(t0);
1886 }
951c6300
RH		 1887}
1888
1889void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1890{
3a13c3f3
RH		 1891 if (TCG_TARGET_REG_BITS == 32) {
1892 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1893 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1894 } else if (TCG_TARGET_HAS_eqv_i64) {
951c6300
RH		 1895 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1896 } else {
1897 tcg_gen_xor_i64(ret, arg1, arg2);
1898 tcg_gen_not_i64(ret, ret);
1899 }
951c6300
RH		 1900}
1901
1902void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1903{
3a13c3f3
RH		 1904 if (TCG_TARGET_REG_BITS == 32) {
1905 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1906 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1907 } else if (TCG_TARGET_HAS_nand_i64) {
951c6300
RH		 1908 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1909 } else {
1910 tcg_gen_and_i64(ret, arg1, arg2);
1911 tcg_gen_not_i64(ret, ret);
1912 }
951c6300
RH		 1913}
1914
1915void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1916{
3a13c3f3
RH		 1917 if (TCG_TARGET_REG_BITS == 32) {
1918 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1919 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1920 } else if (TCG_TARGET_HAS_nor_i64) {
951c6300
RH		 1921 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1922 } else {
1923 tcg_gen_or_i64(ret, arg1, arg2);
1924 tcg_gen_not_i64(ret, ret);
1925 }
951c6300
RH		 1926}
1927
1928void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1929{
3a13c3f3
RH		 1930 if (TCG_TARGET_REG_BITS == 32) {
1931 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1932 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1933 } else if (TCG_TARGET_HAS_orc_i64) {
951c6300
RH		 1934 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1935 } else {
5dd48602 1936 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 1937 tcg_gen_not_i64(t0, arg2);
1938 tcg_gen_or_i64(ret, arg1, t0);
1939 tcg_temp_free_i64(t0);
1940 }
951c6300
RH		 1941}
1942
0e28d006
RH		 1943void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1944{
1945 if (TCG_TARGET_HAS_clz_i64) {
1946 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1947 } else {
1948 gen_helper_clz_i64(ret, arg1, arg2);
1949 }
1950}
1951
1952void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1953{
1954 if (TCG_TARGET_REG_BITS == 32
1955 && TCG_TARGET_HAS_clz_i32
1956 && arg2 <= 0xffffffffu) {
5dd48602 1957 TCGv_i32 t = tcg_temp_ebb_new_i32();
11d11d61 1958 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32);
0e28d006
RH		 1959 tcg_gen_addi_i32(t, t, 32);
1960 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1961 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1962 tcg_temp_free_i32(t);
1963 } else {
11d11d61
RH		 1964 TCGv_i64 t0 = tcg_const_i64(arg2);
1965 tcg_gen_clz_i64(ret, arg1, t0);
1966 tcg_temp_free_i64(t0);
0e28d006
RH		 1967 }
1968}
1969
1970void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1971{
1972 if (TCG_TARGET_HAS_ctz_i64) {
1973 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
14e99210 1974 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
5dd48602 1975 TCGv_i64 z, t = tcg_temp_ebb_new_i64();
14e99210
RH		 1976
1977 if (TCG_TARGET_HAS_ctpop_i64) {
1978 tcg_gen_subi_i64(t, arg1, 1);
1979 tcg_gen_andc_i64(t, t, arg1);
1980 tcg_gen_ctpop_i64(t, t);
1981 } else {
1982 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1983 tcg_gen_neg_i64(t, arg1);
1984 tcg_gen_and_i64(t, t, arg1);
1985 tcg_gen_clzi_i64(t, t, 64);
1986 tcg_gen_xori_i64(t, t, 63);
1987 }
11d11d61 1988 z = tcg_constant_i64(0);
14e99210
RH		 1989 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1990 tcg_temp_free_i64(t);
1991 tcg_temp_free_i64(z);
0e28d006
RH		 1992 } else {
1993 gen_helper_ctz_i64(ret, arg1, arg2);
1994 }
1995}
1996
1997void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1998{
1999 if (TCG_TARGET_REG_BITS == 32
2000 && TCG_TARGET_HAS_ctz_i32
2001 && arg2 <= 0xffffffffu) {
5dd48602 2002 TCGv_i32 t32 = tcg_temp_ebb_new_i32();
11d11d61 2003 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32);
0e28d006
RH		 2004 tcg_gen_addi_i32(t32, t32, 32);
2005 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
2006 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2007 tcg_temp_free_i32(t32);
14e99210
RH		 2008 } else if (!TCG_TARGET_HAS_ctz_i64
2009 && TCG_TARGET_HAS_ctpop_i64
2010 && arg2 == 64) {
2011 /* This equivalence has the advantage of not requiring a fixup. */
5dd48602 2012 TCGv_i64 t = tcg_temp_ebb_new_i64();
14e99210
RH		 2013 tcg_gen_subi_i64(t, arg1, 1);
2014 tcg_gen_andc_i64(t, t, arg1);
2015 tcg_gen_ctpop_i64(ret, t);
2016 tcg_temp_free_i64(t);
0e28d006 2017 } else {
11d11d61
RH		 2018 TCGv_i64 t0 = tcg_const_i64(arg2);
2019 tcg_gen_ctz_i64(ret, arg1, t0);
2020 tcg_temp_free_i64(t0);
0e28d006
RH		 2021 }
2022}
2023
086920c2
RH		 2024void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
2025{
2026 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
5dd48602 2027 TCGv_i64 t = tcg_temp_ebb_new_i64();
086920c2
RH		 2028 tcg_gen_sari_i64(t, arg, 63);
2029 tcg_gen_xor_i64(t, t, arg);
2030 tcg_gen_clzi_i64(t, t, 64);
2031 tcg_gen_subi_i64(ret, t, 1);
2032 tcg_temp_free_i64(t);
2033 } else {
2034 gen_helper_clrsb_i64(ret, arg);
2035 }
2036}
2037
a768e4e9
RH		 2038void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
2039{
2040 if (TCG_TARGET_HAS_ctpop_i64) {
2041 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
2042 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
2043 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2044 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2045 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
2046 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2047 } else {
2048 gen_helper_ctpop_i64(ret, arg1);
2049 }
2050}
2051
951c6300
RH		 2052void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2053{
2054 if (TCG_TARGET_HAS_rot_i64) {
2055 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
2056 } else {
2057 TCGv_i64 t0, t1;
5dd48602
RH		 2058 t0 = tcg_temp_ebb_new_i64();
2059 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 2060 tcg_gen_shl_i64(t0, arg1, arg2);
2061 tcg_gen_subfi_i64(t1, 64, arg2);
2062 tcg_gen_shr_i64(t1, arg1, t1);
2063 tcg_gen_or_i64(ret, t0, t1);
2064 tcg_temp_free_i64(t0);
2065 tcg_temp_free_i64(t1);
2066 }
2067}
2068
07dada03 2069void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 2070{
07dada03 2071 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
951c6300
RH		 2072 /* some cases can be optimized here */
2073 if (arg2 == 0) {
2074 tcg_gen_mov_i64(ret, arg1);
2075 } else if (TCG_TARGET_HAS_rot_i64) {
11d11d61 2076 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 2077 } else {
2078 TCGv_i64 t0, t1;
5dd48602
RH		 2079 t0 = tcg_temp_ebb_new_i64();
2080 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 2081 tcg_gen_shli_i64(t0, arg1, arg2);
2082 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
2083 tcg_gen_or_i64(ret, t0, t1);
2084 tcg_temp_free_i64(t0);
2085 tcg_temp_free_i64(t1);
2086 }
2087}
2088
2089void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2090{
2091 if (TCG_TARGET_HAS_rot_i64) {
2092 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
2093 } else {
2094 TCGv_i64 t0, t1;
5dd48602
RH		 2095 t0 = tcg_temp_ebb_new_i64();
2096 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 2097 tcg_gen_shr_i64(t0, arg1, arg2);
2098 tcg_gen_subfi_i64(t1, 64, arg2);
2099 tcg_gen_shl_i64(t1, arg1, t1);
2100 tcg_gen_or_i64(ret, t0, t1);
2101 tcg_temp_free_i64(t0);
2102 tcg_temp_free_i64(t1);
2103 }
2104}
2105
07dada03 2106void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 2107{
07dada03 2108 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
951c6300
RH		 2109 /* some cases can be optimized here */
2110 if (arg2 == 0) {
2111 tcg_gen_mov_i64(ret, arg1);
2112 } else {
2113 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2114 }
2115}
2116
2117void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2118 unsigned int ofs, unsigned int len)
2119{
2120 uint64_t mask;
2121 TCGv_i64 t1;
2122
2123 tcg_debug_assert(ofs < 64);
0d0d309d 2124 tcg_debug_assert(len > 0);
951c6300
RH		 2125 tcg_debug_assert(len <= 64);
2126 tcg_debug_assert(ofs + len <= 64);
2127
0d0d309d 2128 if (len == 64) {
951c6300
RH		 2129 tcg_gen_mov_i64(ret, arg2);
2130 return;
2131 }
2132 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2133 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2134 return;
2135 }
2136
3a13c3f3
RH		 2137 if (TCG_TARGET_REG_BITS == 32) {
2138 if (ofs >= 32) {
2139 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2140 TCGV_LOW(arg2), ofs - 32, len);
2141 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2142 return;
2143 }
2144 if (ofs + len <= 32) {
2145 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2146 TCGV_LOW(arg2), ofs, len);
2147 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2148 return;
2149 }
951c6300 2150 }
951c6300 2151
5dd48602 2152 t1 = tcg_temp_ebb_new_i64();
951c6300 2153
b0a60567
RH		 2154 if (TCG_TARGET_HAS_extract2_i64) {
2155 if (ofs + len == 64) {
2156 tcg_gen_shli_i64(t1, arg1, len);
2157 tcg_gen_extract2_i64(ret, t1, arg2, len);
2158 goto done;
2159 }
2160 if (ofs == 0) {
2161 tcg_gen_extract2_i64(ret, arg1, arg2, len);
2162 tcg_gen_rotli_i64(ret, ret, len);
2163 goto done;
2164 }
2165 }
2166
2167 mask = (1ull << len) - 1;
951c6300
RH		 2168 if (ofs + len < 64) {
2169 tcg_gen_andi_i64(t1, arg2, mask);
2170 tcg_gen_shli_i64(t1, t1, ofs);
2171 } else {
2172 tcg_gen_shli_i64(t1, arg2, ofs);
2173 }
2174 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2175 tcg_gen_or_i64(ret, ret, t1);
b0a60567 2176 done:
951c6300
RH		 2177 tcg_temp_free_i64(t1);
2178}
2179
07cc68d5
RH		 2180void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2181 unsigned int ofs, unsigned int len)
2182{
2183 tcg_debug_assert(ofs < 64);
2184 tcg_debug_assert(len > 0);
2185 tcg_debug_assert(len <= 64);
2186 tcg_debug_assert(ofs + len <= 64);
2187
2188 if (ofs + len == 64) {
2189 tcg_gen_shli_i64(ret, arg, ofs);
2190 } else if (ofs == 0) {
2191 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2192 } else if (TCG_TARGET_HAS_deposit_i64
2193 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
11d11d61 2194 TCGv_i64 zero = tcg_constant_i64(0);
07cc68d5 2195 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
07cc68d5
RH		 2196 } else {
2197 if (TCG_TARGET_REG_BITS == 32) {
2198 if (ofs >= 32) {
2199 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2200 ofs - 32, len);
2201 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2202 return;
2203 }
2204 if (ofs + len <= 32) {
2205 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2206 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2207 return;
2208 }
2209 }
2210 /* To help two-operand hosts we prefer to zero-extend first,
2211 which allows ARG to stay live. */
2212 switch (len) {
2213 case 32:
2214 if (TCG_TARGET_HAS_ext32u_i64) {
2215 tcg_gen_ext32u_i64(ret, arg);
2216 tcg_gen_shli_i64(ret, ret, ofs);
2217 return;
2218 }
2219 break;
2220 case 16:
2221 if (TCG_TARGET_HAS_ext16u_i64) {
2222 tcg_gen_ext16u_i64(ret, arg);
2223 tcg_gen_shli_i64(ret, ret, ofs);
2224 return;
2225 }
2226 break;
2227 case 8:
2228 if (TCG_TARGET_HAS_ext8u_i64) {
2229 tcg_gen_ext8u_i64(ret, arg);
2230 tcg_gen_shli_i64(ret, ret, ofs);
2231 return;
2232 }
2233 break;
2234 }
2235 /* Otherwise prefer zero-extension over AND for code size. */
2236 switch (ofs + len) {
2237 case 32:
2238 if (TCG_TARGET_HAS_ext32u_i64) {
2239 tcg_gen_shli_i64(ret, arg, ofs);
2240 tcg_gen_ext32u_i64(ret, ret);
2241 return;
2242 }
2243 break;
2244 case 16:
2245 if (TCG_TARGET_HAS_ext16u_i64) {
2246 tcg_gen_shli_i64(ret, arg, ofs);
2247 tcg_gen_ext16u_i64(ret, ret);
2248 return;
2249 }
2250 break;
2251 case 8:
2252 if (TCG_TARGET_HAS_ext8u_i64) {
2253 tcg_gen_shli_i64(ret, arg, ofs);
2254 tcg_gen_ext8u_i64(ret, ret);
2255 return;
2256 }
2257 break;
2258 }
2259 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2260 tcg_gen_shli_i64(ret, ret, ofs);
2261 }
2262}
2263
7ec8bab3
RH		 2264void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2265 unsigned int ofs, unsigned int len)
2266{
2267 tcg_debug_assert(ofs < 64);
2268 tcg_debug_assert(len > 0);
2269 tcg_debug_assert(len <= 64);
2270 tcg_debug_assert(ofs + len <= 64);
2271
2272 /* Canonicalize certain special cases, even if extract is supported. */
2273 if (ofs + len == 64) {
2274 tcg_gen_shri_i64(ret, arg, 64 - len);
2275 return;
2276 }
2277 if (ofs == 0) {
2278 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2279 return;
2280 }
2281
2282 if (TCG_TARGET_REG_BITS == 32) {
2283 /* Look for a 32-bit extract within one of the two words. */
2284 if (ofs >= 32) {
2285 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2286 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2287 return;
2288 }
2289 if (ofs + len <= 32) {
2290 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2291 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2292 return;
2293 }
2294 /* The field is split across two words. One double-word
2295 shift is better than two double-word shifts. */
2296 goto do_shift_and;
2297 }
2298
2299 if (TCG_TARGET_HAS_extract_i64
2300 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2301 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2302 return;
2303 }
2304
2305 /* Assume that zero-extension, if available, is cheaper than a shift. */
2306 switch (ofs + len) {
2307 case 32:
2308 if (TCG_TARGET_HAS_ext32u_i64) {
2309 tcg_gen_ext32u_i64(ret, arg);
2310 tcg_gen_shri_i64(ret, ret, ofs);
2311 return;
2312 }
2313 break;
2314 case 16:
2315 if (TCG_TARGET_HAS_ext16u_i64) {
2316 tcg_gen_ext16u_i64(ret, arg);
2317 tcg_gen_shri_i64(ret, ret, ofs);
2318 return;
2319 }
2320 break;
2321 case 8:
2322 if (TCG_TARGET_HAS_ext8u_i64) {
2323 tcg_gen_ext8u_i64(ret, arg);
2324 tcg_gen_shri_i64(ret, ret, ofs);
2325 return;
2326 }
2327 break;
2328 }
2329
2330 /* ??? Ideally we'd know what values are available for immediate AND.
2331 Assume that 8 bits are available, plus the special cases of 16 and 32,
2332 so that we get ext8u, ext16u, and ext32u. */
2333 switch (len) {
2334 case 1 ... 8: case 16: case 32:
2335 do_shift_and:
2336 tcg_gen_shri_i64(ret, arg, ofs);
2337 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2338 break;
2339 default:
2340 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2341 tcg_gen_shri_i64(ret, ret, 64 - len);
2342 break;
2343 }
2344}
2345
2346void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2347 unsigned int ofs, unsigned int len)
2348{
2349 tcg_debug_assert(ofs < 64);
2350 tcg_debug_assert(len > 0);
2351 tcg_debug_assert(len <= 64);
2352 tcg_debug_assert(ofs + len <= 64);
2353
2354 /* Canonicalize certain special cases, even if sextract is supported. */
2355 if (ofs + len == 64) {
2356 tcg_gen_sari_i64(ret, arg, 64 - len);
2357 return;
2358 }
2359 if (ofs == 0) {
2360 switch (len) {
2361 case 32:
2362 tcg_gen_ext32s_i64(ret, arg);
2363 return;
2364 case 16:
2365 tcg_gen_ext16s_i64(ret, arg);
2366 return;
2367 case 8:
2368 tcg_gen_ext8s_i64(ret, arg);
2369 return;
2370 }
2371 }
2372
2373 if (TCG_TARGET_REG_BITS == 32) {
2374 /* Look for a 32-bit extract within one of the two words. */
2375 if (ofs >= 32) {
2376 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2377 } else if (ofs + len <= 32) {
2378 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2379 } else if (ofs == 0) {
2380 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2381 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2382 return;
2383 } else if (len > 32) {
5dd48602 2384 TCGv_i32 t = tcg_temp_ebb_new_i32();
7ec8bab3
RH		 2385 /* Extract the bits for the high word normally. */
2386 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2387 /* Shift the field down for the low part. */
2388 tcg_gen_shri_i64(ret, arg, ofs);
2389 /* Overwrite the shift into the high part. */
2390 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2391 tcg_temp_free_i32(t);
2392 return;
2393 } else {
2394 /* Shift the field down for the low part, such that the
2395 field sits at the MSB. */
2396 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2397 /* Shift the field down from the MSB, sign extending. */
2398 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2399 }
2400 /* Sign-extend the field from 32 bits. */
2401 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2402 return;
2403 }
2404
2405 if (TCG_TARGET_HAS_sextract_i64
2406 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2407 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2408 return;
2409 }
2410
2411 /* Assume that sign-extension, if available, is cheaper than a shift. */
2412 switch (ofs + len) {
2413 case 32:
2414 if (TCG_TARGET_HAS_ext32s_i64) {
2415 tcg_gen_ext32s_i64(ret, arg);
2416 tcg_gen_sari_i64(ret, ret, ofs);
2417 return;
2418 }
2419 break;
2420 case 16:
2421 if (TCG_TARGET_HAS_ext16s_i64) {
2422 tcg_gen_ext16s_i64(ret, arg);
2423 tcg_gen_sari_i64(ret, ret, ofs);
2424 return;
2425 }
2426 break;
2427 case 8:
2428 if (TCG_TARGET_HAS_ext8s_i64) {
2429 tcg_gen_ext8s_i64(ret, arg);
2430 tcg_gen_sari_i64(ret, ret, ofs);
2431 return;
2432 }
2433 break;
2434 }
2435 switch (len) {
2436 case 32:
2437 if (TCG_TARGET_HAS_ext32s_i64) {
2438 tcg_gen_shri_i64(ret, arg, ofs);
2439 tcg_gen_ext32s_i64(ret, ret);
2440 return;
2441 }
2442 break;
2443 case 16:
2444 if (TCG_TARGET_HAS_ext16s_i64) {
2445 tcg_gen_shri_i64(ret, arg, ofs);
2446 tcg_gen_ext16s_i64(ret, ret);
2447 return;
2448 }
2449 break;
2450 case 8:
2451 if (TCG_TARGET_HAS_ext8s_i64) {
2452 tcg_gen_shri_i64(ret, arg, ofs);
2453 tcg_gen_ext8s_i64(ret, ret);
2454 return;
2455 }
2456 break;
2457 }
2458 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2459 tcg_gen_sari_i64(ret, ret, 64 - len);
2460}
2461
2089fcc9
DH		 2462/*
2463 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2464 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2465 */
2466void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2467 unsigned int ofs)
2468{
2469 tcg_debug_assert(ofs <= 64);
2470 if (ofs == 0) {
2471 tcg_gen_mov_i64(ret, al);
2472 } else if (ofs == 64) {
2473 tcg_gen_mov_i64(ret, ah);
2474 } else if (al == ah) {
2475 tcg_gen_rotri_i64(ret, al, ofs);
fce1296f
RH		 2476 } else if (TCG_TARGET_HAS_extract2_i64) {
2477 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2089fcc9 2478 } else {
5dd48602 2479 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2089fcc9
DH		 2480 tcg_gen_shri_i64(t0, al, ofs);
2481 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2482 tcg_temp_free_i64(t0);
2483 }
2484}
2485
951c6300
RH		 2486void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2487 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2488{
37ed3bf1
RH		 2489 if (cond == TCG_COND_ALWAYS) {
2490 tcg_gen_mov_i64(ret, v1);
2491 } else if (cond == TCG_COND_NEVER) {
2492 tcg_gen_mov_i64(ret, v2);
2493 } else if (TCG_TARGET_REG_BITS == 32) {
5dd48602
RH		 2494 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
2495 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
3a13c3f3
RH		 2496 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2497 TCGV_LOW(c1), TCGV_HIGH(c1),
2498 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2499
2500 if (TCG_TARGET_HAS_movcond_i32) {
2501 tcg_gen_movi_i32(t1, 0);
2502 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2503 TCGV_LOW(v1), TCGV_LOW(v2));
2504 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2505 TCGV_HIGH(v1), TCGV_HIGH(v2));
2506 } else {
2507 tcg_gen_neg_i32(t0, t0);
951c6300 2508
3a13c3f3
RH		 2509 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2510 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2511 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
951c6300 2512
3a13c3f3
RH		 2513 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2514 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2515 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2516 }
2517 tcg_temp_free_i32(t0);
2518 tcg_temp_free_i32(t1);
2519 } else if (TCG_TARGET_HAS_movcond_i64) {
951c6300
RH		 2520 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2521 } else {
5dd48602
RH		 2522 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2523 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 2524 tcg_gen_setcond_i64(cond, t0, c1, c2);
2525 tcg_gen_neg_i64(t0, t0);
2526 tcg_gen_and_i64(t1, v1, t0);
2527 tcg_gen_andc_i64(ret, v2, t0);
2528 tcg_gen_or_i64(ret, ret, t1);
2529 tcg_temp_free_i64(t0);
2530 tcg_temp_free_i64(t1);
2531 }
951c6300
RH		 2532}
2533
2534void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2535 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2536{
2537 if (TCG_TARGET_HAS_add2_i64) {
2538 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
951c6300 2539 } else {
5dd48602
RH		 2540 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2541 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 2542 tcg_gen_add_i64(t0, al, bl);
2543 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2544 tcg_gen_add_i64(rh, ah, bh);
2545 tcg_gen_add_i64(rh, rh, t1);
2546 tcg_gen_mov_i64(rl, t0);
2547 tcg_temp_free_i64(t0);
2548 tcg_temp_free_i64(t1);
2549 }
2550}
2551
2552void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2553 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2554{
2555 if (TCG_TARGET_HAS_sub2_i64) {
2556 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
951c6300 2557 } else {
5dd48602
RH		 2558 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2559 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
951c6300
RH		 2560 tcg_gen_sub_i64(t0, al, bl);
2561 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2562 tcg_gen_sub_i64(rh, ah, bh);
2563 tcg_gen_sub_i64(rh, rh, t1);
2564 tcg_gen_mov_i64(rl, t0);
2565 tcg_temp_free_i64(t0);
2566 tcg_temp_free_i64(t1);
2567 }
2568}
2569
2570void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2571{
2572 if (TCG_TARGET_HAS_mulu2_i64) {
2573 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
951c6300 2574 } else if (TCG_TARGET_HAS_muluh_i64) {
5dd48602 2575 TCGv_i64 t = tcg_temp_ebb_new_i64();
951c6300
RH		 2576 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2577 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2578 tcg_gen_mov_i64(rl, t);
2579 tcg_temp_free_i64(t);
2580 } else {
5dd48602 2581 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 2582 tcg_gen_mul_i64(t0, arg1, arg2);
2583 gen_helper_muluh_i64(rh, arg1, arg2);
2584 tcg_gen_mov_i64(rl, t0);
2585 tcg_temp_free_i64(t0);
2586 }
2587}
2588
2589void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2590{
2591 if (TCG_TARGET_HAS_muls2_i64) {
2592 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
951c6300 2593 } else if (TCG_TARGET_HAS_mulsh_i64) {
5dd48602 2594 TCGv_i64 t = tcg_temp_ebb_new_i64();
951c6300
RH		 2595 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2596 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2597 tcg_gen_mov_i64(rl, t);
2598 tcg_temp_free_i64(t);
2599 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
5dd48602
RH		 2600 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2601 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2602 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
2603 TCGv_i64 t3 = tcg_temp_ebb_new_i64();
951c6300
RH		 2604 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2605 /* Adjust for negative inputs. */
2606 tcg_gen_sari_i64(t2, arg1, 63);
2607 tcg_gen_sari_i64(t3, arg2, 63);
2608 tcg_gen_and_i64(t2, t2, arg2);
2609 tcg_gen_and_i64(t3, t3, arg1);
2610 tcg_gen_sub_i64(rh, t1, t2);
2611 tcg_gen_sub_i64(rh, rh, t3);
2612 tcg_gen_mov_i64(rl, t0);
2613 tcg_temp_free_i64(t0);
2614 tcg_temp_free_i64(t1);
2615 tcg_temp_free_i64(t2);
2616 tcg_temp_free_i64(t3);
2617 } else {
5dd48602 2618 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
951c6300
RH		 2619 tcg_gen_mul_i64(t0, arg1, arg2);
2620 gen_helper_mulsh_i64(rh, arg1, arg2);
2621 tcg_gen_mov_i64(rl, t0);
2622 tcg_temp_free_i64(t0);
2623 }
2624}
2625
5087abfb
RH		 2626void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2627{
5dd48602
RH		 2628 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2629 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2630 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
5087abfb
RH		 2631 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2632 /* Adjust for negative input for the signed arg1. */
2633 tcg_gen_sari_i64(t2, arg1, 63);
2634 tcg_gen_and_i64(t2, t2, arg2);
2635 tcg_gen_sub_i64(rh, t1, t2);
2636 tcg_gen_mov_i64(rl, t0);
2637 tcg_temp_free_i64(t0);
2638 tcg_temp_free_i64(t1);
2639 tcg_temp_free_i64(t2);
2640}
2641
b87fb8cd
RH		 2642void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2643{
2644 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2645}
2646
2647void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2648{
2649 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2650}
2651
2652void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2653{
2654 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2655}
2656
2657void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2658{
2659 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2660}
2661
ff1f11f7
RH		 2662void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2663{
5dd48602 2664 TCGv_i64 t = tcg_temp_ebb_new_i64();
ff1f11f7
RH		 2665
2666 tcg_gen_sari_i64(t, a, 63);
2667 tcg_gen_xor_i64(ret, a, t);
2668 tcg_gen_sub_i64(ret, ret, t);
2669 tcg_temp_free_i64(t);
2670}
2671
951c6300
RH		 2672/* Size changing operations. */
2673
609ad705 2674void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
951c6300 2675{
3a13c3f3 2676 if (TCG_TARGET_REG_BITS == 32) {
609ad705
RH		 2677 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2678 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
b7e8b17a 2679 tcg_gen_op2(INDEX_op_extrl_i64_i32,
ae8b75dc 2680 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
609ad705 2681 } else {
dc41aa7d 2682 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
609ad705
RH		 2683 }
2684}
2685
2686void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2687{
2688 if (TCG_TARGET_REG_BITS == 32) {
2689 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2690 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
b7e8b17a 2691 tcg_gen_op2(INDEX_op_extrh_i64_i32,
ae8b75dc 2692 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
951c6300 2693 } else {
5dd48602 2694 TCGv_i64 t = tcg_temp_ebb_new_i64();
609ad705 2695 tcg_gen_shri_i64(t, arg, 32);
dc41aa7d 2696 tcg_gen_mov_i32(ret, (TCGv_i32)t);
951c6300
RH		 2697 tcg_temp_free_i64(t);
2698 }
951c6300
RH		 2699}
2700
2701void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2702{
3a13c3f3
RH		 2703 if (TCG_TARGET_REG_BITS == 32) {
2704 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2705 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2706 } else {
b7e8b17a 2707 tcg_gen_op2(INDEX_op_extu_i32_i64,
ae8b75dc 2708 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2709 }
951c6300
RH		 2710}
2711
2712void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2713{
3a13c3f3
RH		 2714 if (TCG_TARGET_REG_BITS == 32) {
2715 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2716 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2717 } else {
b7e8b17a 2718 tcg_gen_op2(INDEX_op_ext_i32_i64,
ae8b75dc 2719 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2720 }
951c6300
RH		 2721}
2722
2723void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2724{
3a13c3f3
RH		 2725 TCGv_i64 tmp;
2726
2727 if (TCG_TARGET_REG_BITS == 32) {
2728 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2729 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2730 return;
2731 }
2732
5dd48602 2733 tmp = tcg_temp_ebb_new_i64();
951c6300
RH		 2734 /* These extensions are only needed for type correctness.
2735 We may be able to do better given target specific information. */
2736 tcg_gen_extu_i32_i64(tmp, high);
2737 tcg_gen_extu_i32_i64(dest, low);
2738 /* If deposit is available, use it. Otherwise use the extra
2739 knowledge that we have of the zero-extensions above. */
2740 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2741 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2742 } else {
2743 tcg_gen_shli_i64(tmp, tmp, 32);
2744 tcg_gen_or_i64(dest, dest, tmp);
2745 }
2746 tcg_temp_free_i64(tmp);
951c6300
RH		 2747}
2748
2749void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2750{
3a13c3f3
RH		 2751 if (TCG_TARGET_REG_BITS == 32) {
2752 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2753 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2754 } else {
609ad705
RH		 2755 tcg_gen_extrl_i64_i32(lo, arg);
2756 tcg_gen_extrh_i64_i32(hi, arg);
3a13c3f3 2757 }
951c6300
RH		 2758}
2759
2760void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2761{
2762 tcg_gen_ext32u_i64(lo, arg);
2763 tcg_gen_shri_i64(hi, arg, 32);
2764}
2765
4771e71c
RH		 2766void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg)
2767{
2768 tcg_gen_mov_i64(lo, TCGV128_LOW(arg));
2769 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg));
2770}
2771
2772void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi)
2773{
2774 tcg_gen_mov_i64(TCGV128_LOW(ret), lo);
2775 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi);
2776}
2777
2778void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src)
2779{
2780 if (dst != src) {
2781 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src));
2782 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src));
2783 }
2784}
2785
951c6300
RH		 2786/* QEMU specific operations. */
2787
d9971435 2788void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
07ea28b4 2789{
eba40358
RH		 2790 /*
2791 * Let the jit code return the read-only version of the
2792 * TranslationBlock, so that we minimize the pc-relative
2793 * distance of the address of the exit_tb code to TB.
2794 * This will improve utilization of pc-relative address loads.
2795 *
2796 * TODO: Move this to translator_loop, so that all const
2797 * TranslationBlock pointers refer to read-only memory.
2798 * This requires coordination with targets that do not use
2799 * the translator_loop.
2800 */
2801 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
07ea28b4
RH		 2802
2803 if (tb == NULL) {
2804 tcg_debug_assert(idx == 0);
2805 } else if (idx <= TB_EXIT_IDXMAX) {
2806#ifdef CONFIG_DEBUG_TCG
2807 /* This is an exit following a goto_tb. Verify that we have
2808 seen this numbered exit before, via tcg_gen_goto_tb. */
2809 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2810#endif
2811 } else {
2812 /* This is an exit via the exitreq label. */
2813 tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2814 }
2815
e6d86bed 2816 plugin_gen_disable_mem_helpers();
07ea28b4
RH		 2817 tcg_gen_op1i(INDEX_op_exit_tb, val);
2818}
2819
951c6300
RH		 2820void tcg_gen_goto_tb(unsigned idx)
2821{
84f15616 2822 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
b7e4afbd 2823 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB));
951c6300 2824 /* We only support two chained exits. */
07ea28b4 2825 tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
951c6300 2826#ifdef CONFIG_DEBUG_TCG
a4761232 2827 /* Verify that we haven't seen this numbered exit before. */
b1311c4a
EC		 2828 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2829 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
951c6300 2830#endif
e6d86bed 2831 plugin_gen_disable_mem_helpers();
84f15616 2832 tcg_gen_op1i(INDEX_op_goto_tb, idx);
951c6300
RH		 2833}
2834
7f11636d 2835void tcg_gen_lookup_and_goto_ptr(void)
cedbcb01 2836{
84f15616 2837 TCGv_ptr ptr;
e6d86bed 2838
b7e4afbd 2839 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) {
07ea28b4 2840 tcg_gen_exit_tb(NULL, 0);
84f15616 2841 return;
cedbcb01 2842 }
84f15616
RH		 2843
2844 plugin_gen_disable_mem_helpers();
5dd48602 2845 ptr = tcg_temp_ebb_new_ptr();
84f15616
RH		 2846 gen_helper_lookup_tb_ptr(ptr, cpu_env);
2847 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2848 tcg_temp_free_ptr(ptr);
cedbcb01
EC		 2849}
2850
14776ab5 2851static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
951c6300 2852{
1f00b27f 2853 /* Trigger the asserts within as early as possible. */
76e366e7
RH		 2854 unsigned a_bits = get_alignment_bits(op);
2855
2856 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
2857 if (a_bits == (op & MO_SIZE)) {
2858 op = (op & ~MO_AMASK) | MO_ALIGN;
2859 }
1f00b27f 2860
951c6300
RH		 2861 switch (op & MO_SIZE) {
2862 case MO_8:
2863 op &= ~MO_BSWAP;
2864 break;
2865 case MO_16:
2866 break;
2867 case MO_32:
2868 if (!is64) {
2869 op &= ~MO_SIGN;
2870 }
2871 break;
2872 case MO_64:
4b473e0c
RH		 2873 if (is64) {
2874 op &= ~MO_SIGN;
2875 break;
951c6300 2876 }
4b473e0c
RH		 2877 /* fall through */
2878 default:
2879 g_assert_not_reached();
951c6300
RH		 2880 }
2881 if (st) {
2882 op &= ~MO_SIGN;
2883 }
2884 return op;
2885}
2886
c45cb8bb 2887static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
14776ab5 2888 MemOp memop, TCGArg idx)
951c6300 2889{
9002ffcb 2890 MemOpIdx oi = make_memop_idx(memop, idx);
c45cb8bb 2891#if TARGET_LONG_BITS == 32
59227d5d 2892 tcg_gen_op3i_i32(opc, val, addr, oi);
c45cb8bb 2893#else
3a13c3f3 2894 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2895 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
3a13c3f3 2896 } else {
ae8b75dc 2897 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
3a13c3f3 2898 }
c45cb8bb 2899#endif
951c6300
RH		 2900}
2901
c45cb8bb 2902static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
14776ab5 2903 MemOp memop, TCGArg idx)
c45cb8bb 2904{
9002ffcb 2905 MemOpIdx oi = make_memop_idx(memop, idx);
951c6300 2906#if TARGET_LONG_BITS == 32
c45cb8bb 2907 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2908 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
c45cb8bb 2909 } else {
ae8b75dc 2910 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
c45cb8bb 2911 }
951c6300 2912#else
c45cb8bb 2913 if (TCG_TARGET_REG_BITS == 32) {
59227d5d
RH		 2914 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2915 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
c45cb8bb 2916 } else {
59227d5d 2917 tcg_gen_op3i_i64(opc, val, addr, oi);
c45cb8bb 2918 }
951c6300 2919#endif
c45cb8bb 2920}
951c6300 2921
b32dc337
PK		 2922static void tcg_gen_req_mo(TCGBar type)
2923{
2924#ifdef TCG_GUEST_DEFAULT_MO
2925 type &= TCG_GUEST_DEFAULT_MO;
2926#endif
2927 type &= ~TCG_TARGET_DEFAULT_MO;
2928 if (type) {
2929 tcg_gen_mb(type | TCG_BAR_SC);
2930 }
2931}
2932
fcc54ab5
AB		 2933static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr)
2934{
2935#ifdef CONFIG_PLUGIN
2936 if (tcg_ctx->plugin_insn != NULL) {
2937 /* Save a copy of the vaddr for use after a load. */
2938 TCGv temp = tcg_temp_new();
2939 tcg_gen_mov_tl(temp, vaddr);
2940 return temp;
2941 }
2942#endif
2943 return vaddr;
2944}
2945
37aff087
RH		 2946static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi,
2947 enum qemu_plugin_mem_rw rw)
e6d86bed
EC		 2948{
2949#ifdef CONFIG_PLUGIN
fcc54ab5 2950 if (tcg_ctx->plugin_insn != NULL) {
37aff087 2951 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw);
fcc54ab5
AB		 2952 plugin_gen_empty_mem_callback(vaddr, info);
2953 tcg_temp_free(vaddr);
e6d86bed 2954 }
e6d86bed
EC		 2955#endif
2956}
2957
14776ab5 2958void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
951c6300 2959{
14776ab5 2960 MemOp orig_memop;
37aff087 2961 MemOpIdx oi;
e1dcf352 2962
b32dc337 2963 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
951c6300 2964 memop = tcg_canonicalize_memop(memop, 0, 0);
37aff087 2965 oi = make_memop_idx(memop, idx);
e1dcf352
RH		 2966
2967 orig_memop = memop;
2968 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2969 memop &= ~MO_BSWAP;
359feba5 2970 /* The bswap primitive benefits from zero-extended input. */
e1dcf352
RH		 2971 if ((memop & MO_SSIZE) == MO_SW) {
2972 memop &= ~MO_SIGN;
2973 }
2974 }
2975
fcc54ab5 2976 addr = plugin_prep_mem_callbacks(addr);
c45cb8bb 2977 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
37aff087 2978 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
e1dcf352
RH		 2979
2980 if ((orig_memop ^ memop) & MO_BSWAP) {
2981 switch (orig_memop & MO_SIZE) {
2982 case MO_16:
359feba5
RH		 2983 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
2984 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2985 : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
e1dcf352
RH		 2986 break;
2987 case MO_32:
2988 tcg_gen_bswap32_i32(val, val);
2989 break;
2990 default:
2991 g_assert_not_reached();
2992 }
2993 }
951c6300
RH		 2994}
2995
14776ab5 2996void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
951c6300 2997{
e1dcf352 2998 TCGv_i32 swap = NULL;
37aff087 2999 MemOpIdx oi;
e1dcf352 3000
b32dc337 3001 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
951c6300 3002 memop = tcg_canonicalize_memop(memop, 0, 1);
37aff087 3003 oi = make_memop_idx(memop, idx);
e1dcf352
RH		 3004
3005 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
5dd48602 3006 swap = tcg_temp_ebb_new_i32();
e1dcf352
RH		 3007 switch (memop & MO_SIZE) {
3008 case MO_16:
b53357ac 3009 tcg_gen_bswap16_i32(swap, val, 0);
e1dcf352
RH		 3010 break;
3011 case MO_32:
3012 tcg_gen_bswap32_i32(swap, val);
3013 break;
3014 default:
3015 g_assert_not_reached();
3016 }
3017 val = swap;
3018 memop &= ~MO_BSWAP;
3019 }
3020
fcc54ab5 3021 addr = plugin_prep_mem_callbacks(addr);
07ce0b05
RH		 3022 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
3023 gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx);
3024 } else {
3025 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
3026 }
37aff087 3027 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
e1dcf352
RH		 3028
3029 if (swap) {
3030 tcg_temp_free_i32(swap);
3031 }
951c6300
RH		 3032}
3033
14776ab5 3034void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
951c6300 3035{
14776ab5 3036 MemOp orig_memop;
37aff087 3037 MemOpIdx oi;
e1dcf352 3038
3a13c3f3 3039 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 3040 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
3041 if (memop & MO_SIGN) {
3042 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
3043 } else {
3044 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
3045 }
3046 return;
3047 }
951c6300 3048
e1dcf352 3049 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
c45cb8bb 3050 memop = tcg_canonicalize_memop(memop, 1, 0);
37aff087 3051 oi = make_memop_idx(memop, idx);
e1dcf352
RH		 3052
3053 orig_memop = memop;
3054 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
3055 memop &= ~MO_BSWAP;
359feba5 3056 /* The bswap primitive benefits from zero-extended input. */
e1dcf352
RH		 3057 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
3058 memop &= ~MO_SIGN;
3059 }
3060 }
3061
fcc54ab5 3062 addr = plugin_prep_mem_callbacks(addr);
c45cb8bb 3063 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
37aff087 3064 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
e1dcf352
RH		 3065
3066 if ((orig_memop ^ memop) & MO_BSWAP) {
359feba5
RH		 3067 int flags = (orig_memop & MO_SIGN
3068 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
3069 : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
e1dcf352
RH		 3070 switch (orig_memop & MO_SIZE) {
3071 case MO_16:
359feba5 3072 tcg_gen_bswap16_i64(val, val, flags);
e1dcf352
RH		 3073 break;
3074 case MO_32:
359feba5 3075 tcg_gen_bswap32_i64(val, val, flags);
e1dcf352
RH		 3076 break;
3077 case MO_64:
3078 tcg_gen_bswap64_i64(val, val);
3079 break;
3080 default:
3081 g_assert_not_reached();
3082 }
3083 }
951c6300
RH		 3084}
3085
14776ab5 3086void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
951c6300 3087{
e1dcf352 3088 TCGv_i64 swap = NULL;
37aff087 3089 MemOpIdx oi;
e1dcf352 3090
3a13c3f3 3091 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 3092 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
3093 return;
3094 }
951c6300 3095
e1dcf352 3096 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
c45cb8bb 3097 memop = tcg_canonicalize_memop(memop, 1, 1);
37aff087 3098 oi = make_memop_idx(memop, idx);
e1dcf352
RH		 3099
3100 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
5dd48602 3101 swap = tcg_temp_ebb_new_i64();
e1dcf352
RH		 3102 switch (memop & MO_SIZE) {
3103 case MO_16:
b53357ac 3104 tcg_gen_bswap16_i64(swap, val, 0);
e1dcf352
RH		 3105 break;
3106 case MO_32:
b53357ac 3107 tcg_gen_bswap32_i64(swap, val, 0);
e1dcf352
RH		 3108 break;
3109 case MO_64:
3110 tcg_gen_bswap64_i64(swap, val);
3111 break;
3112 default:
3113 g_assert_not_reached();
3114 }
3115 val = swap;
3116 memop &= ~MO_BSWAP;
3117 }
3118
fcc54ab5 3119 addr = plugin_prep_mem_callbacks(addr);
c45cb8bb 3120 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
37aff087 3121 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
e1dcf352
RH		 3122
3123 if (swap) {
3124 tcg_temp_free_i64(swap);
3125 }
951c6300 3126}
c482cb11 3127
cb48f365
RH		 3128static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig)
3129{
3130 MemOp mop_1 = orig, mop_2;
3131
3132 tcg_debug_assert((orig & MO_SIZE) == MO_128);
3133 tcg_debug_assert((orig & MO_SIGN) == 0);
3134
3135 /* Use a memory ordering implemented by the host. */
3136 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (orig & MO_BSWAP)) {
3137 mop_1 &= ~MO_BSWAP;
3138 }
3139
3140 /* Reduce the size to 64-bit. */
3141 mop_1 = (mop_1 & ~MO_SIZE) | MO_64;
3142
3143 /* Retain the alignment constraints of the original. */
3144 switch (orig & MO_AMASK) {
3145 case MO_UNALN:
3146 case MO_ALIGN_2:
3147 case MO_ALIGN_4:
3148 mop_2 = mop_1;
3149 break;
3150 case MO_ALIGN_8:
3151 /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */
3152 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN;
3153 mop_2 = mop_1;
3154 break;
3155 case MO_ALIGN:
3156 /* Second has 8-byte alignment; first has 16-byte alignment. */
3157 mop_2 = mop_1;
3158 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16;
3159 break;
3160 case MO_ALIGN_16:
3161 case MO_ALIGN_32:
3162 case MO_ALIGN_64:
3163 /* Second has 8-byte alignment; first retains original. */
3164 mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN;
3165 break;
3166 default:
3167 g_assert_not_reached();
3168 }
3169 ret[0] = mop_1;
3170 ret[1] = mop_2;
3171}
3172
3173void tcg_gen_qemu_ld_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop)
3174{
3175 MemOp mop[2];
3176 TCGv addr_p8;
3177 TCGv_i64 x, y;
3178
3179 canonicalize_memop_i128_as_i64(mop, memop);
3180
3181 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
3182 addr = plugin_prep_mem_callbacks(addr);
3183
3184 /* TODO: respect atomicity of the operation. */
3185 /* TODO: allow the tcg backend to see the whole operation. */
3186
3187 /*
3188 * Since there are no global TCGv_i128, there is no visible state
3189 * changed if the second load faults. Load directly into the two
3190 * subwords.
3191 */
3192 if ((memop & MO_BSWAP) == MO_LE) {
3193 x = TCGV128_LOW(val);
3194 y = TCGV128_HIGH(val);
3195 } else {
3196 x = TCGV128_HIGH(val);
3197 y = TCGV128_LOW(val);
3198 }
3199
3200 gen_ldst_i64(INDEX_op_qemu_ld_i64, x, addr, mop[0], idx);
3201
3202 if ((mop[0] ^ memop) & MO_BSWAP) {
3203 tcg_gen_bswap64_i64(x, x);
3204 }
3205
3206 addr_p8 = tcg_temp_new();
3207 tcg_gen_addi_tl(addr_p8, addr, 8);
3208 gen_ldst_i64(INDEX_op_qemu_ld_i64, y, addr_p8, mop[1], idx);
3209 tcg_temp_free(addr_p8);
3210
3211 if ((mop[0] ^ memop) & MO_BSWAP) {
3212 tcg_gen_bswap64_i64(y, y);
3213 }
3214
3215 plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx),
3216 QEMU_PLUGIN_MEM_R);
3217}
3218
3219void tcg_gen_qemu_st_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop)
3220{
3221 MemOp mop[2];
3222 TCGv addr_p8;
3223 TCGv_i64 x, y;
3224
3225 canonicalize_memop_i128_as_i64(mop, memop);
3226
3227 tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST);
3228 addr = plugin_prep_mem_callbacks(addr);
3229
3230 /* TODO: respect atomicity of the operation. */
3231 /* TODO: allow the tcg backend to see the whole operation. */
3232
3233 if ((memop & MO_BSWAP) == MO_LE) {
3234 x = TCGV128_LOW(val);
3235 y = TCGV128_HIGH(val);
3236 } else {
3237 x = TCGV128_HIGH(val);
3238 y = TCGV128_LOW(val);
3239 }
3240
3241 addr_p8 = tcg_temp_new();
3242 if ((mop[0] ^ memop) & MO_BSWAP) {
5dd48602 3243 TCGv_i64 t = tcg_temp_ebb_new_i64();
cb48f365
RH		 3244
3245 tcg_gen_bswap64_i64(t, x);
3246 gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr, mop[0], idx);
3247 tcg_gen_bswap64_i64(t, y);
3248 tcg_gen_addi_tl(addr_p8, addr, 8);
3249 gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr_p8, mop[1], idx);
3250 tcg_temp_free_i64(t);
3251 } else {
3252 gen_ldst_i64(INDEX_op_qemu_st_i64, x, addr, mop[0], idx);
3253 tcg_gen_addi_tl(addr_p8, addr, 8);
3254 gen_ldst_i64(INDEX_op_qemu_st_i64, y, addr_p8, mop[1], idx);
3255 }
3256 tcg_temp_free(addr_p8);
3257
3258 plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx),
3259 QEMU_PLUGIN_MEM_W);
3260}
3261
14776ab5 3262static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
c482cb11
RH		 3263{
3264 switch (opc & MO_SSIZE) {
3265 case MO_SB:
3266 tcg_gen_ext8s_i32(ret, val);
3267 break;
3268 case MO_UB:
3269 tcg_gen_ext8u_i32(ret, val);
3270 break;
3271 case MO_SW:
3272 tcg_gen_ext16s_i32(ret, val);
3273 break;
3274 case MO_UW:
3275 tcg_gen_ext16u_i32(ret, val);
3276 break;
3277 default:
3278 tcg_gen_mov_i32(ret, val);
3279 break;
3280 }
3281}
3282
14776ab5 3283static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
c482cb11
RH		 3284{
3285 switch (opc & MO_SSIZE) {
3286 case MO_SB:
3287 tcg_gen_ext8s_i64(ret, val);
3288 break;
3289 case MO_UB:
3290 tcg_gen_ext8u_i64(ret, val);
3291 break;
3292 case MO_SW:
3293 tcg_gen_ext16s_i64(ret, val);
3294 break;
3295 case MO_UW:
3296 tcg_gen_ext16u_i64(ret, val);
3297 break;
3298 case MO_SL:
3299 tcg_gen_ext32s_i64(ret, val);
3300 break;
3301 case MO_UL:
3302 tcg_gen_ext32u_i64(ret, val);
3303 break;
3304 default:
3305 tcg_gen_mov_i64(ret, val);
3306 break;
3307 }
3308}
3309
c482cb11
RH		 3310typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
3311 TCGv_i32, TCGv_i32, TCGv_i32);
3312typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
3313 TCGv_i64, TCGv_i64, TCGv_i32);
123ae568
RH		 3314typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv,
3315 TCGv_i128, TCGv_i128, TCGv_i32);
c482cb11
RH		 3316typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
3317 TCGv_i32, TCGv_i32);
3318typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
3319 TCGv_i64, TCGv_i32);
c482cb11 3320
df79b996
RH		 3321#ifdef CONFIG_ATOMIC64
3322# define WITH_ATOMIC64(X) X,
3323#else
3324# define WITH_ATOMIC64(X)
3325#endif
123ae568
RH		 3326#ifdef CONFIG_CMPXCHG128
3327# define WITH_ATOMIC128(X) X,
3328#else
3329# define WITH_ATOMIC128(X)
3330#endif
df79b996 3331
4b473e0c 3332static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
c482cb11
RH		 3333 [MO_8] = gen_helper_atomic_cmpxchgb,
3334 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
3335 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
3336 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
3337 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
df79b996
RH		 3338 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
3339 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
123ae568
RH		 3340 WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le)
3341 WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be)
c482cb11
RH		 3342};
3343
d1beee4d
RH		 3344void tcg_gen_nonatomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
3345 TCGv_i32 newv, TCGArg idx, MemOp memop)
3346{
5dd48602
RH		 3347 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
3348 TCGv_i32 t2 = tcg_temp_ebb_new_i32();
d1beee4d
RH		 3349
3350 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
3351
3352 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3353 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
3354 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3355 tcg_temp_free_i32(t2);
3356
3357 if (memop & MO_SIGN) {
3358 tcg_gen_ext_i32(retv, t1, memop);
3359 } else {
3360 tcg_gen_mov_i32(retv, t1);
3361 }
3362 tcg_temp_free_i32(t1);
3363}
3364
c482cb11 3365void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
14776ab5 3366 TCGv_i32 newv, TCGArg idx, MemOp memop)
c482cb11 3367{
d1beee4d
RH		 3368 gen_atomic_cx_i32 gen;
3369 MemOpIdx oi;
c482cb11 3370
b7e4afbd 3371 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
d1beee4d
RH		 3372 tcg_gen_nonatomic_cmpxchg_i32(retv, addr, cmpv, newv, idx, memop);
3373 return;
3374 }
c482cb11 3375
d1beee4d
RH		 3376 memop = tcg_canonicalize_memop(memop, 0, 0);
3377 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3378 tcg_debug_assert(gen != NULL);
c482cb11 3379
d1beee4d
RH		 3380 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3381 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3382
3383 if (memop & MO_SIGN) {
3384 tcg_gen_ext_i32(retv, retv, memop);
3385 }
3386}
3387
3388void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
3389 TCGv_i64 newv, TCGArg idx, MemOp memop)
3390{
3391 TCGv_i64 t1, t2;
c482cb11 3392
d1beee4d
RH		 3393 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
3394 tcg_gen_nonatomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
3395 TCGV_LOW(newv), idx, memop);
c482cb11 3396 if (memop & MO_SIGN) {
d1beee4d 3397 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
c482cb11 3398 } else {
d1beee4d 3399 tcg_gen_movi_i32(TCGV_HIGH(retv), 0);
c482cb11 3400 }
d1beee4d
RH		 3401 return;
3402 }
c482cb11 3403
5dd48602
RH		 3404 t1 = tcg_temp_ebb_new_i64();
3405 t2 = tcg_temp_ebb_new_i64();
c482cb11 3406
d1beee4d 3407 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
c482cb11 3408
d1beee4d
RH		 3409 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3410 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
3411 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3412 tcg_temp_free_i64(t2);
3413
3414 if (memop & MO_SIGN) {
3415 tcg_gen_ext_i64(retv, t1, memop);
3416 } else {
3417 tcg_gen_mov_i64(retv, t1);
c482cb11 3418 }
d1beee4d 3419 tcg_temp_free_i64(t1);
c482cb11
RH		 3420}
3421
3422void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
14776ab5 3423 TCGv_i64 newv, TCGArg idx, MemOp memop)
c482cb11 3424{
b7e4afbd 3425 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
d1beee4d
RH		 3426 tcg_gen_nonatomic_cmpxchg_i64(retv, addr, cmpv, newv, idx, memop);
3427 return;
3428 }
c482cb11 3429
d1beee4d 3430 if ((memop & MO_SIZE) == MO_64) {
c482cb11
RH		 3431 gen_atomic_cx_i64 gen;
3432
d1beee4d 3433 memop = tcg_canonicalize_memop(memop, 1, 0);
c482cb11 3434 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
d1beee4d
RH		 3435 if (gen) {
3436 MemOpIdx oi = make_memop_idx(memop, idx);
3437 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3438 return;
3439 }
c482cb11 3440
1c2adb95 3441 gen_helper_exit_atomic(cpu_env);
d1beee4d
RH		 3442
3443 /*
3444 * Produce a result for a well-formed opcode stream. This satisfies
3445 * liveness for set before used, which happens before this dead code
3446 * is removed.
3447 */
79b1af90 3448 tcg_gen_movi_i64(retv, 0);
d1beee4d
RH		 3449 return;
3450 }
3451
3452 if (TCG_TARGET_REG_BITS == 32) {
3453 tcg_gen_atomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
3454 TCGV_LOW(newv), idx, memop);
3455 if (memop & MO_SIGN) {
3456 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
3457 } else {
3458 tcg_gen_movi_i32(TCGV_HIGH(retv), 0);
3459 }
c482cb11 3460 } else {
5dd48602
RH		 3461 TCGv_i32 c32 = tcg_temp_ebb_new_i32();
3462 TCGv_i32 n32 = tcg_temp_ebb_new_i32();
3463 TCGv_i32 r32 = tcg_temp_ebb_new_i32();
c482cb11
RH		 3464
3465 tcg_gen_extrl_i64_i32(c32, cmpv);
3466 tcg_gen_extrl_i64_i32(n32, newv);
3467 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3468 tcg_temp_free_i32(c32);
3469 tcg_temp_free_i32(n32);
3470
3471 tcg_gen_extu_i32_i64(retv, r32);
3472 tcg_temp_free_i32(r32);
3473
3474 if (memop & MO_SIGN) {
3475 tcg_gen_ext_i64(retv, retv, memop);
3476 }
3477 }
3478}
3479
123ae568
RH		 3480void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
3481 TCGv_i128 newv, TCGArg idx, MemOp memop)
3482{
3483 if (TCG_TARGET_REG_BITS == 32) {
3484 /* Inline expansion below is simply too large for 32-bit hosts. */
3485 gen_atomic_cx_i128 gen = ((memop & MO_BSWAP) == MO_LE
3486 ? gen_helper_nonatomic_cmpxchgo_le
3487 : gen_helper_nonatomic_cmpxchgo_be);
3488 MemOpIdx oi = make_memop_idx(memop, idx);
3489
3490 tcg_debug_assert((memop & MO_SIZE) == MO_128);
3491 tcg_debug_assert((memop & MO_SIGN) == 0);
3492
3493 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3494 } else {
5dd48602
RH		 3495 TCGv_i128 oldv = tcg_temp_ebb_new_i128();
3496 TCGv_i128 tmpv = tcg_temp_ebb_new_i128();
3497 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
3498 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
123ae568
RH		 3499 TCGv_i64 z = tcg_constant_i64(0);
3500
3501 tcg_gen_qemu_ld_i128(oldv, addr, idx, memop);
3502
3503 /* Compare i128 */
3504 tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv));
3505 tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv));
3506 tcg_gen_or_i64(t0, t0, t1);
3507
3508 /* tmpv = equal ? newv : oldv */
3509 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z,
3510 TCGV128_LOW(newv), TCGV128_LOW(oldv));
3511 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z,
3512 TCGV128_HIGH(newv), TCGV128_HIGH(oldv));
3513
3514 /* Unconditional writeback. */
3515 tcg_gen_qemu_st_i128(tmpv, addr, idx, memop);
3516 tcg_gen_mov_i128(retv, oldv);
3517
3518 tcg_temp_free_i64(t0);
3519 tcg_temp_free_i64(t1);
3520 tcg_temp_free_i128(tmpv);
3521 tcg_temp_free_i128(oldv);
3522 }
3523}
3524
3525void tcg_gen_atomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
3526 TCGv_i128 newv, TCGArg idx, MemOp memop)
3527{
3528 gen_atomic_cx_i128 gen;
3529
3530 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
3531 tcg_gen_nonatomic_cmpxchg_i128(retv, addr, cmpv, newv, idx, memop);
3532 return;
3533 }
3534
3535 tcg_debug_assert((memop & MO_SIZE) == MO_128);
3536 tcg_debug_assert((memop & MO_SIGN) == 0);
3537 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3538
3539 if (gen) {
3540 MemOpIdx oi = make_memop_idx(memop, idx);
3541 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3542 return;
3543 }
3544
3545 gen_helper_exit_atomic(cpu_env);
3546
3547 /*
3548 * Produce a result for a well-formed opcode stream. This satisfies
3549 * liveness for set before used, which happens before this dead code
3550 * is removed.
3551 */
3552 tcg_gen_movi_i64(TCGV128_LOW(retv), 0);
3553 tcg_gen_movi_i64(TCGV128_HIGH(retv), 0);
3554}
3555
c482cb11 3556static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
14776ab5 3557 TCGArg idx, MemOp memop, bool new_val,
c482cb11
RH		 3558 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3559{
5dd48602
RH		 3560 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
3561 TCGv_i32 t2 = tcg_temp_ebb_new_i32();
c482cb11
RH		 3562
3563 memop = tcg_canonicalize_memop(memop, 0, 0);
3564
852f933e
RH		 3565 tcg_gen_qemu_ld_i32(t1, addr, idx, memop);
3566 tcg_gen_ext_i32(t2, val, memop);
3567 gen(t2, t1, t2);
c482cb11
RH		 3568 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3569
3570 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3571 tcg_temp_free_i32(t1);
3572 tcg_temp_free_i32(t2);
3573}
3574
3575static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
14776ab5 3576 TCGArg idx, MemOp memop, void * const table[])
c482cb11
RH		 3577{
3578 gen_atomic_op_i32 gen;
9002ffcb 3579 MemOpIdx oi;
c482cb11
RH		 3580
3581 memop = tcg_canonicalize_memop(memop, 0, 0);
3582
3583 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3584 tcg_debug_assert(gen != NULL);
3585
e28a8664
RH		 3586 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3587 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
c482cb11
RH		 3588
3589 if (memop & MO_SIGN) {
3590 tcg_gen_ext_i32(ret, ret, memop);
3591 }
3592}
3593
3594static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
14776ab5 3595 TCGArg idx, MemOp memop, bool new_val,
c482cb11
RH		 3596 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3597{
5dd48602
RH		 3598 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
3599 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
c482cb11
RH		 3600
3601 memop = tcg_canonicalize_memop(memop, 1, 0);
3602
852f933e
RH		 3603 tcg_gen_qemu_ld_i64(t1, addr, idx, memop);
3604 tcg_gen_ext_i64(t2, val, memop);
3605 gen(t2, t1, t2);
c482cb11
RH		 3606 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3607
3608 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3609 tcg_temp_free_i64(t1);
3610 tcg_temp_free_i64(t2);
3611}
3612
3613static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
14776ab5 3614 TCGArg idx, MemOp memop, void * const table[])
c482cb11
RH		 3615{
3616 memop = tcg_canonicalize_memop(memop, 1, 0);
3617
3618 if ((memop & MO_SIZE) == MO_64) {
df79b996 3619#ifdef CONFIG_ATOMIC64
c482cb11 3620 gen_atomic_op_i64 gen;
9002ffcb 3621 MemOpIdx oi;
c482cb11
RH		 3622
3623 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3624 tcg_debug_assert(gen != NULL);
3625
e28a8664
RH		 3626 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3627 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
df79b996 3628#else
1c2adb95 3629 gen_helper_exit_atomic(cpu_env);
79b1af90
RH		 3630 /* Produce a result, so that we have a well-formed opcode stream
3631 with respect to uses of the result in the (dead) code following. */
3632 tcg_gen_movi_i64(ret, 0);
df79b996 3633#endif /* CONFIG_ATOMIC64 */
c482cb11 3634 } else {
5dd48602
RH		 3635 TCGv_i32 v32 = tcg_temp_ebb_new_i32();
3636 TCGv_i32 r32 = tcg_temp_ebb_new_i32();
c482cb11
RH		 3637
3638 tcg_gen_extrl_i64_i32(v32, val);
3639 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3640 tcg_temp_free_i32(v32);
3641
3642 tcg_gen_extu_i32_i64(ret, r32);
3643 tcg_temp_free_i32(r32);
3644
3645 if (memop & MO_SIGN) {
3646 tcg_gen_ext_i64(ret, ret, memop);
3647 }
3648 }
3649}
3650
3651#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
4b473e0c 3652static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
c482cb11
RH		 3653 [MO_8] = gen_helper_atomic_##NAME##b, \
3654 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3655 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3656 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3657 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
df79b996
RH		 3658 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3659 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
c482cb11
RH		 3660}; \
3661void tcg_gen_atomic_##NAME##_i32 \
14776ab5 3662 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \
c482cb11 3663{ \
b7e4afbd 3664 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
c482cb11
RH		 3665 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3666 } else { \
3667 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3668 tcg_gen_##OP##_i32); \
3669 } \
3670} \
3671void tcg_gen_atomic_##NAME##_i64 \
14776ab5 3672 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \
c482cb11 3673{ \
b7e4afbd 3674 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
c482cb11
RH		 3675 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3676 } else { \
3677 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3678 tcg_gen_##OP##_i64); \
3679 } \
3680}
3681
3682GEN_ATOMIC_HELPER(fetch_add, add, 0)
3683GEN_ATOMIC_HELPER(fetch_and, and, 0)
3684GEN_ATOMIC_HELPER(fetch_or, or, 0)
3685GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
5507c2bf
RH		 3686GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3687GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3688GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3689GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
c482cb11
RH		 3690
3691GEN_ATOMIC_HELPER(add_fetch, add, 1)
3692GEN_ATOMIC_HELPER(and_fetch, and, 1)
3693GEN_ATOMIC_HELPER(or_fetch, or, 1)
3694GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
5507c2bf
RH		 3695GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3696GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3697GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3698GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
c482cb11
RH		 3699
3700static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3701{
3702 tcg_gen_mov_i32(r, b);
3703}
3704
3705static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3706{
3707 tcg_gen_mov_i64(r, b);
3708}
3709
3710GEN_ATOMIC_HELPER(xchg, mov2, 0)
3711
3712#undef GEN_ATOMIC_HELPER
QEMU mirror