]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg-op.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
nios2-softmmu.mak: express dependencies with Kconfig
[mirror_qemu.git] / tcg / tcg-op.c
CommitLineData
951c6300
RH		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
757e725b 25#include "qemu/osdep.h"
33c11879
PB		 26#include "qemu-common.h"
27#include "cpu.h"
63c91552 28#include "exec/exec-all.h"
951c6300
RH		 29#include "tcg.h"
30#include "tcg-op.h"
b32dc337 31#include "tcg-mo.h"
dcdaadb6
LV		 32#include "trace-tcg.h"
33#include "trace/mem.h"
951c6300 34
3a13c3f3
RH		 35/* Reduce the number of ifdefs below. This assumes that all uses of
36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
37 the compiler can eliminate. */
38#if TCG_TARGET_REG_BITS == 64
39extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
40extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
41#define TCGV_LOW TCGV_LOW_link_error
42#define TCGV_HIGH TCGV_HIGH_link_error
43#endif
951c6300 44
b7e8b17a 45void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
951c6300 46{
b7e8b17a 47 TCGOp *op = tcg_emit_op(opc);
75e8b9b7 48 op->args[0] = a1;
951c6300
RH		 49}
50
b7e8b17a 51void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
951c6300 52{
b7e8b17a 53 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 54 op->args[0] = a1;
55 op->args[1] = a2;
951c6300
RH		 56}
57
b7e8b17a 58void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
951c6300 59{
b7e8b17a 60 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 61 op->args[0] = a1;
62 op->args[1] = a2;
63 op->args[2] = a3;
951c6300
RH		 64}
65
b7e8b17a 66void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
951c6300 67{
b7e8b17a 68 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 69 op->args[0] = a1;
70 op->args[1] = a2;
71 op->args[2] = a3;
72 op->args[3] = a4;
951c6300
RH		 73}
74
b7e8b17a
RH		 75void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
76 TCGArg a4, TCGArg a5)
951c6300 77{
b7e8b17a 78 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 79 op->args[0] = a1;
80 op->args[1] = a2;
81 op->args[2] = a3;
82 op->args[3] = a4;
83 op->args[4] = a5;
951c6300
RH		 84}
85
b7e8b17a
RH		 86void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
87 TCGArg a4, TCGArg a5, TCGArg a6)
951c6300 88{
b7e8b17a 89 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 90 op->args[0] = a1;
91 op->args[1] = a2;
92 op->args[2] = a3;
93 op->args[3] = a4;
94 op->args[4] = a5;
95 op->args[5] = a6;
951c6300
RH		 96}
97
f65e19bc
PK		 98void tcg_gen_mb(TCGBar mb_type)
99{
b1311c4a 100 if (tcg_ctx->tb_cflags & CF_PARALLEL) {
b7e8b17a 101 tcg_gen_op1(INDEX_op_mb, mb_type);
f65e19bc
PK		 102 }
103}
104
951c6300
RH		 105/* 32 bit ops */
106
107void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
108{
109 /* some cases can be optimized here */
110 if (arg2 == 0) {
111 tcg_gen_mov_i32(ret, arg1);
112 } else {
113 TCGv_i32 t0 = tcg_const_i32(arg2);
114 tcg_gen_add_i32(ret, arg1, t0);
115 tcg_temp_free_i32(t0);
116 }
117}
118
119void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
120{
121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
122 /* Don't recurse with tcg_gen_neg_i32. */
123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
124 } else {
125 TCGv_i32 t0 = tcg_const_i32(arg1);
126 tcg_gen_sub_i32(ret, t0, arg2);
127 tcg_temp_free_i32(t0);
128 }
129}
130
131void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
132{
133 /* some cases can be optimized here */
134 if (arg2 == 0) {
135 tcg_gen_mov_i32(ret, arg1);
136 } else {
137 TCGv_i32 t0 = tcg_const_i32(arg2);
138 tcg_gen_sub_i32(ret, arg1, t0);
139 tcg_temp_free_i32(t0);
140 }
141}
142
474b2e8f 143void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300
RH		 144{
145 TCGv_i32 t0;
146 /* Some cases can be optimized here. */
147 switch (arg2) {
148 case 0:
149 tcg_gen_movi_i32(ret, 0);
150 return;
474b2e8f 151 case -1:
951c6300
RH		 152 tcg_gen_mov_i32(ret, arg1);
153 return;
474b2e8f 154 case 0xff:
951c6300
RH		 155 /* Don't recurse with tcg_gen_ext8u_i32. */
156 if (TCG_TARGET_HAS_ext8u_i32) {
157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
158 return;
159 }
160 break;
474b2e8f 161 case 0xffff:
951c6300
RH		 162 if (TCG_TARGET_HAS_ext16u_i32) {
163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
164 return;
165 }
166 break;
167 }
168 t0 = tcg_const_i32(arg2);
169 tcg_gen_and_i32(ret, arg1, t0);
170 tcg_temp_free_i32(t0);
171}
172
173void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
174{
175 /* Some cases can be optimized here. */
176 if (arg2 == -1) {
177 tcg_gen_movi_i32(ret, -1);
178 } else if (arg2 == 0) {
179 tcg_gen_mov_i32(ret, arg1);
180 } else {
181 TCGv_i32 t0 = tcg_const_i32(arg2);
182 tcg_gen_or_i32(ret, arg1, t0);
183 tcg_temp_free_i32(t0);
184 }
185}
186
187void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
188{
189 /* Some cases can be optimized here. */
190 if (arg2 == 0) {
191 tcg_gen_mov_i32(ret, arg1);
192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
193 /* Don't recurse with tcg_gen_not_i32. */
194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
195 } else {
196 TCGv_i32 t0 = tcg_const_i32(arg2);
197 tcg_gen_xor_i32(ret, arg1, t0);
198 tcg_temp_free_i32(t0);
199 }
200}
201
474b2e8f 202void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 203{
474b2e8f 204 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 205 if (arg2 == 0) {
206 tcg_gen_mov_i32(ret, arg1);
207 } else {
208 TCGv_i32 t0 = tcg_const_i32(arg2);
209 tcg_gen_shl_i32(ret, arg1, t0);
210 tcg_temp_free_i32(t0);
211 }
212}
213
474b2e8f 214void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 215{
474b2e8f 216 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 217 if (arg2 == 0) {
218 tcg_gen_mov_i32(ret, arg1);
219 } else {
220 TCGv_i32 t0 = tcg_const_i32(arg2);
221 tcg_gen_shr_i32(ret, arg1, t0);
222 tcg_temp_free_i32(t0);
223 }
224}
225
474b2e8f 226void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 227{
474b2e8f 228 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 229 if (arg2 == 0) {
230 tcg_gen_mov_i32(ret, arg1);
231 } else {
232 TCGv_i32 t0 = tcg_const_i32(arg2);
233 tcg_gen_sar_i32(ret, arg1, t0);
234 tcg_temp_free_i32(t0);
235 }
236}
237
42a268c2 238void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
951c6300
RH		 239{
240 if (cond == TCG_COND_ALWAYS) {
42a268c2 241 tcg_gen_br(l);
951c6300 242 } else if (cond != TCG_COND_NEVER) {
d88a117e 243 l->refs++;
42a268c2 244 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
951c6300
RH		 245 }
246}
247
42a268c2 248void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
951c6300 249{
37ed3bf1
RH		 250 if (cond == TCG_COND_ALWAYS) {
251 tcg_gen_br(l);
252 } else if (cond != TCG_COND_NEVER) {
253 TCGv_i32 t0 = tcg_const_i32(arg2);
254 tcg_gen_brcond_i32(cond, arg1, t0, l);
255 tcg_temp_free_i32(t0);
256 }
951c6300
RH		 257}
258
259void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
260 TCGv_i32 arg1, TCGv_i32 arg2)
261{
262 if (cond == TCG_COND_ALWAYS) {
263 tcg_gen_movi_i32(ret, 1);
264 } else if (cond == TCG_COND_NEVER) {
265 tcg_gen_movi_i32(ret, 0);
266 } else {
267 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
268 }
269}
270
271void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
272 TCGv_i32 arg1, int32_t arg2)
273{
274 TCGv_i32 t0 = tcg_const_i32(arg2);
275 tcg_gen_setcond_i32(cond, ret, arg1, t0);
276 tcg_temp_free_i32(t0);
277}
278
279void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
280{
b2e3ae94
RH		 281 if (arg2 == 0) {
282 tcg_gen_movi_i32(ret, 0);
283 } else if (is_power_of_2(arg2)) {
284 tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
285 } else {
286 TCGv_i32 t0 = tcg_const_i32(arg2);
287 tcg_gen_mul_i32(ret, arg1, t0);
288 tcg_temp_free_i32(t0);
289 }
951c6300
RH		 290}
291
292void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
293{
294 if (TCG_TARGET_HAS_div_i32) {
295 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
296 } else if (TCG_TARGET_HAS_div2_i32) {
297 TCGv_i32 t0 = tcg_temp_new_i32();
298 tcg_gen_sari_i32(t0, arg1, 31);
299 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
300 tcg_temp_free_i32(t0);
301 } else {
302 gen_helper_div_i32(ret, arg1, arg2);
303 }
304}
305
306void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
307{
308 if (TCG_TARGET_HAS_rem_i32) {
309 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
310 } else if (TCG_TARGET_HAS_div_i32) {
311 TCGv_i32 t0 = tcg_temp_new_i32();
312 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
313 tcg_gen_mul_i32(t0, t0, arg2);
314 tcg_gen_sub_i32(ret, arg1, t0);
315 tcg_temp_free_i32(t0);
316 } else if (TCG_TARGET_HAS_div2_i32) {
317 TCGv_i32 t0 = tcg_temp_new_i32();
318 tcg_gen_sari_i32(t0, arg1, 31);
319 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
320 tcg_temp_free_i32(t0);
321 } else {
322 gen_helper_rem_i32(ret, arg1, arg2);
323 }
324}
325
326void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
327{
328 if (TCG_TARGET_HAS_div_i32) {
329 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
330 } else if (TCG_TARGET_HAS_div2_i32) {
331 TCGv_i32 t0 = tcg_temp_new_i32();
332 tcg_gen_movi_i32(t0, 0);
333 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
334 tcg_temp_free_i32(t0);
335 } else {
336 gen_helper_divu_i32(ret, arg1, arg2);
337 }
338}
339
340void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
341{
342 if (TCG_TARGET_HAS_rem_i32) {
343 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
344 } else if (TCG_TARGET_HAS_div_i32) {
345 TCGv_i32 t0 = tcg_temp_new_i32();
346 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
347 tcg_gen_mul_i32(t0, t0, arg2);
348 tcg_gen_sub_i32(ret, arg1, t0);
349 tcg_temp_free_i32(t0);
350 } else if (TCG_TARGET_HAS_div2_i32) {
351 TCGv_i32 t0 = tcg_temp_new_i32();
352 tcg_gen_movi_i32(t0, 0);
353 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
354 tcg_temp_free_i32(t0);
355 } else {
356 gen_helper_remu_i32(ret, arg1, arg2);
357 }
358}
359
360void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
361{
362 if (TCG_TARGET_HAS_andc_i32) {
363 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
364 } else {
365 TCGv_i32 t0 = tcg_temp_new_i32();
366 tcg_gen_not_i32(t0, arg2);
367 tcg_gen_and_i32(ret, arg1, t0);
368 tcg_temp_free_i32(t0);
369 }
370}
371
372void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
373{
374 if (TCG_TARGET_HAS_eqv_i32) {
375 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
376 } else {
377 tcg_gen_xor_i32(ret, arg1, arg2);
378 tcg_gen_not_i32(ret, ret);
379 }
380}
381
382void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
383{
384 if (TCG_TARGET_HAS_nand_i32) {
385 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
386 } else {
387 tcg_gen_and_i32(ret, arg1, arg2);
388 tcg_gen_not_i32(ret, ret);
389 }
390}
391
392void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
393{
394 if (TCG_TARGET_HAS_nor_i32) {
395 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
396 } else {
397 tcg_gen_or_i32(ret, arg1, arg2);
398 tcg_gen_not_i32(ret, ret);
399 }
400}
401
402void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
403{
404 if (TCG_TARGET_HAS_orc_i32) {
405 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
406 } else {
407 TCGv_i32 t0 = tcg_temp_new_i32();
408 tcg_gen_not_i32(t0, arg2);
409 tcg_gen_or_i32(ret, arg1, t0);
410 tcg_temp_free_i32(t0);
411 }
412}
413
0e28d006
RH		 414void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
415{
416 if (TCG_TARGET_HAS_clz_i32) {
417 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
418 } else if (TCG_TARGET_HAS_clz_i64) {
419 TCGv_i64 t1 = tcg_temp_new_i64();
420 TCGv_i64 t2 = tcg_temp_new_i64();
421 tcg_gen_extu_i32_i64(t1, arg1);
422 tcg_gen_extu_i32_i64(t2, arg2);
423 tcg_gen_addi_i64(t2, t2, 32);
424 tcg_gen_clz_i64(t1, t1, t2);
425 tcg_gen_extrl_i64_i32(ret, t1);
426 tcg_temp_free_i64(t1);
427 tcg_temp_free_i64(t2);
428 tcg_gen_subi_i32(ret, ret, 32);
429 } else {
430 gen_helper_clz_i32(ret, arg1, arg2);
431 }
432}
433
434void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
435{
436 TCGv_i32 t = tcg_const_i32(arg2);
437 tcg_gen_clz_i32(ret, arg1, t);
438 tcg_temp_free_i32(t);
439}
440
441void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
442{
443 if (TCG_TARGET_HAS_ctz_i32) {
444 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
445 } else if (TCG_TARGET_HAS_ctz_i64) {
446 TCGv_i64 t1 = tcg_temp_new_i64();
447 TCGv_i64 t2 = tcg_temp_new_i64();
448 tcg_gen_extu_i32_i64(t1, arg1);
449 tcg_gen_extu_i32_i64(t2, arg2);
450 tcg_gen_ctz_i64(t1, t1, t2);
451 tcg_gen_extrl_i64_i32(ret, t1);
452 tcg_temp_free_i64(t1);
453 tcg_temp_free_i64(t2);
14e99210
RH		 454 } else if (TCG_TARGET_HAS_ctpop_i32
455 || TCG_TARGET_HAS_ctpop_i64
456 || TCG_TARGET_HAS_clz_i32
457 || TCG_TARGET_HAS_clz_i64) {
458 TCGv_i32 z, t = tcg_temp_new_i32();
459
460 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
461 tcg_gen_subi_i32(t, arg1, 1);
462 tcg_gen_andc_i32(t, t, arg1);
463 tcg_gen_ctpop_i32(t, t);
464 } else {
465 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
466 tcg_gen_neg_i32(t, arg1);
467 tcg_gen_and_i32(t, t, arg1);
468 tcg_gen_clzi_i32(t, t, 32);
469 tcg_gen_xori_i32(t, t, 31);
470 }
471 z = tcg_const_i32(0);
472 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
473 tcg_temp_free_i32(t);
474 tcg_temp_free_i32(z);
0e28d006
RH		 475 } else {
476 gen_helper_ctz_i32(ret, arg1, arg2);
477 }
478}
479
480void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
481{
14e99210
RH		 482 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
483 /* This equivalence has the advantage of not requiring a fixup. */
484 TCGv_i32 t = tcg_temp_new_i32();
485 tcg_gen_subi_i32(t, arg1, 1);
486 tcg_gen_andc_i32(t, t, arg1);
487 tcg_gen_ctpop_i32(ret, t);
488 tcg_temp_free_i32(t);
489 } else {
490 TCGv_i32 t = tcg_const_i32(arg2);
491 tcg_gen_ctz_i32(ret, arg1, t);
492 tcg_temp_free_i32(t);
493 }
0e28d006
RH		 494}
495
086920c2
RH		 496void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
497{
498 if (TCG_TARGET_HAS_clz_i32) {
499 TCGv_i32 t = tcg_temp_new_i32();
500 tcg_gen_sari_i32(t, arg, 31);
501 tcg_gen_xor_i32(t, t, arg);
502 tcg_gen_clzi_i32(t, t, 32);
503 tcg_gen_subi_i32(ret, t, 1);
504 tcg_temp_free_i32(t);
505 } else {
506 gen_helper_clrsb_i32(ret, arg);
507 }
508}
509
a768e4e9
RH		 510void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
511{
512 if (TCG_TARGET_HAS_ctpop_i32) {
513 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
514 } else if (TCG_TARGET_HAS_ctpop_i64) {
515 TCGv_i64 t = tcg_temp_new_i64();
516 tcg_gen_extu_i32_i64(t, arg1);
517 tcg_gen_ctpop_i64(t, t);
518 tcg_gen_extrl_i64_i32(ret, t);
519 tcg_temp_free_i64(t);
520 } else {
521 gen_helper_ctpop_i32(ret, arg1);
522 }
523}
524
951c6300
RH		 525void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
526{
527 if (TCG_TARGET_HAS_rot_i32) {
528 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
529 } else {
530 TCGv_i32 t0, t1;
531
532 t0 = tcg_temp_new_i32();
533 t1 = tcg_temp_new_i32();
534 tcg_gen_shl_i32(t0, arg1, arg2);
535 tcg_gen_subfi_i32(t1, 32, arg2);
536 tcg_gen_shr_i32(t1, arg1, t1);
537 tcg_gen_or_i32(ret, t0, t1);
538 tcg_temp_free_i32(t0);
539 tcg_temp_free_i32(t1);
540 }
541}
542
543void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
544{
545 tcg_debug_assert(arg2 < 32);
546 /* some cases can be optimized here */
547 if (arg2 == 0) {
548 tcg_gen_mov_i32(ret, arg1);
549 } else if (TCG_TARGET_HAS_rot_i32) {
550 TCGv_i32 t0 = tcg_const_i32(arg2);
551 tcg_gen_rotl_i32(ret, arg1, t0);
552 tcg_temp_free_i32(t0);
553 } else {
554 TCGv_i32 t0, t1;
555 t0 = tcg_temp_new_i32();
556 t1 = tcg_temp_new_i32();
557 tcg_gen_shli_i32(t0, arg1, arg2);
558 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
559 tcg_gen_or_i32(ret, t0, t1);
560 tcg_temp_free_i32(t0);
561 tcg_temp_free_i32(t1);
562 }
563}
564
565void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
566{
567 if (TCG_TARGET_HAS_rot_i32) {
568 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
569 } else {
570 TCGv_i32 t0, t1;
571
572 t0 = tcg_temp_new_i32();
573 t1 = tcg_temp_new_i32();
574 tcg_gen_shr_i32(t0, arg1, arg2);
575 tcg_gen_subfi_i32(t1, 32, arg2);
576 tcg_gen_shl_i32(t1, arg1, t1);
577 tcg_gen_or_i32(ret, t0, t1);
578 tcg_temp_free_i32(t0);
579 tcg_temp_free_i32(t1);
580 }
581}
582
583void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
584{
585 tcg_debug_assert(arg2 < 32);
586 /* some cases can be optimized here */
587 if (arg2 == 0) {
588 tcg_gen_mov_i32(ret, arg1);
589 } else {
590 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
591 }
592}
593
594void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
595 unsigned int ofs, unsigned int len)
596{
597 uint32_t mask;
598 TCGv_i32 t1;
599
600 tcg_debug_assert(ofs < 32);
0d0d309d 601 tcg_debug_assert(len > 0);
951c6300
RH		 602 tcg_debug_assert(len <= 32);
603 tcg_debug_assert(ofs + len <= 32);
604
0d0d309d 605 if (len == 32) {
951c6300
RH		 606 tcg_gen_mov_i32(ret, arg2);
607 return;
608 }
609 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
610 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
611 return;
612 }
613
614 mask = (1u << len) - 1;
615 t1 = tcg_temp_new_i32();
616
617 if (ofs + len < 32) {
618 tcg_gen_andi_i32(t1, arg2, mask);
619 tcg_gen_shli_i32(t1, t1, ofs);
620 } else {
621 tcg_gen_shli_i32(t1, arg2, ofs);
622 }
623 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
624 tcg_gen_or_i32(ret, ret, t1);
625
626 tcg_temp_free_i32(t1);
627}
628
07cc68d5
RH		 629void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
630 unsigned int ofs, unsigned int len)
631{
632 tcg_debug_assert(ofs < 32);
633 tcg_debug_assert(len > 0);
634 tcg_debug_assert(len <= 32);
635 tcg_debug_assert(ofs + len <= 32);
636
637 if (ofs + len == 32) {
638 tcg_gen_shli_i32(ret, arg, ofs);
639 } else if (ofs == 0) {
640 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
641 } else if (TCG_TARGET_HAS_deposit_i32
642 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
643 TCGv_i32 zero = tcg_const_i32(0);
644 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
645 tcg_temp_free_i32(zero);
646 } else {
647 /* To help two-operand hosts we prefer to zero-extend first,
648 which allows ARG to stay live. */
649 switch (len) {
650 case 16:
651 if (TCG_TARGET_HAS_ext16u_i32) {
652 tcg_gen_ext16u_i32(ret, arg);
653 tcg_gen_shli_i32(ret, ret, ofs);
654 return;
655 }
656 break;
657 case 8:
658 if (TCG_TARGET_HAS_ext8u_i32) {
659 tcg_gen_ext8u_i32(ret, arg);
660 tcg_gen_shli_i32(ret, ret, ofs);
661 return;
662 }
663 break;
664 }
665 /* Otherwise prefer zero-extension over AND for code size. */
666 switch (ofs + len) {
667 case 16:
668 if (TCG_TARGET_HAS_ext16u_i32) {
669 tcg_gen_shli_i32(ret, arg, ofs);
670 tcg_gen_ext16u_i32(ret, ret);
671 return;
672 }
673 break;
674 case 8:
675 if (TCG_TARGET_HAS_ext8u_i32) {
676 tcg_gen_shli_i32(ret, arg, ofs);
677 tcg_gen_ext8u_i32(ret, ret);
678 return;
679 }
680 break;
681 }
682 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
683 tcg_gen_shli_i32(ret, ret, ofs);
684 }
685}
686
7ec8bab3
RH		 687void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
688 unsigned int ofs, unsigned int len)
689{
690 tcg_debug_assert(ofs < 32);
691 tcg_debug_assert(len > 0);
692 tcg_debug_assert(len <= 32);
693 tcg_debug_assert(ofs + len <= 32);
694
695 /* Canonicalize certain special cases, even if extract is supported. */
696 if (ofs + len == 32) {
697 tcg_gen_shri_i32(ret, arg, 32 - len);
698 return;
699 }
700 if (ofs == 0) {
701 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
702 return;
703 }
704
705 if (TCG_TARGET_HAS_extract_i32
706 && TCG_TARGET_extract_i32_valid(ofs, len)) {
707 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
708 return;
709 }
710
711 /* Assume that zero-extension, if available, is cheaper than a shift. */
712 switch (ofs + len) {
713 case 16:
714 if (TCG_TARGET_HAS_ext16u_i32) {
715 tcg_gen_ext16u_i32(ret, arg);
716 tcg_gen_shri_i32(ret, ret, ofs);
717 return;
718 }
719 break;
720 case 8:
721 if (TCG_TARGET_HAS_ext8u_i32) {
722 tcg_gen_ext8u_i32(ret, arg);
723 tcg_gen_shri_i32(ret, ret, ofs);
724 return;
725 }
726 break;
727 }
728
729 /* ??? Ideally we'd know what values are available for immediate AND.
730 Assume that 8 bits are available, plus the special case of 16,
731 so that we get ext8u, ext16u. */
732 switch (len) {
733 case 1 ... 8: case 16:
734 tcg_gen_shri_i32(ret, arg, ofs);
735 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
736 break;
737 default:
738 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
739 tcg_gen_shri_i32(ret, ret, 32 - len);
740 break;
741 }
742}
743
744void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
745 unsigned int ofs, unsigned int len)
746{
747 tcg_debug_assert(ofs < 32);
748 tcg_debug_assert(len > 0);
749 tcg_debug_assert(len <= 32);
750 tcg_debug_assert(ofs + len <= 32);
751
752 /* Canonicalize certain special cases, even if extract is supported. */
753 if (ofs + len == 32) {
754 tcg_gen_sari_i32(ret, arg, 32 - len);
755 return;
756 }
757 if (ofs == 0) {
758 switch (len) {
759 case 16:
760 tcg_gen_ext16s_i32(ret, arg);
761 return;
762 case 8:
763 tcg_gen_ext8s_i32(ret, arg);
764 return;
765 }
766 }
767
768 if (TCG_TARGET_HAS_sextract_i32
769 && TCG_TARGET_extract_i32_valid(ofs, len)) {
770 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
771 return;
772 }
773
774 /* Assume that sign-extension, if available, is cheaper than a shift. */
775 switch (ofs + len) {
776 case 16:
777 if (TCG_TARGET_HAS_ext16s_i32) {
778 tcg_gen_ext16s_i32(ret, arg);
779 tcg_gen_sari_i32(ret, ret, ofs);
780 return;
781 }
782 break;
783 case 8:
784 if (TCG_TARGET_HAS_ext8s_i32) {
785 tcg_gen_ext8s_i32(ret, arg);
786 tcg_gen_sari_i32(ret, ret, ofs);
787 return;
788 }
789 break;
790 }
791 switch (len) {
792 case 16:
793 if (TCG_TARGET_HAS_ext16s_i32) {
794 tcg_gen_shri_i32(ret, arg, ofs);
795 tcg_gen_ext16s_i32(ret, ret);
796 return;
797 }
798 break;
799 case 8:
800 if (TCG_TARGET_HAS_ext8s_i32) {
801 tcg_gen_shri_i32(ret, arg, ofs);
802 tcg_gen_ext8s_i32(ret, ret);
803 return;
804 }
805 break;
806 }
807
808 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
809 tcg_gen_sari_i32(ret, ret, 32 - len);
810}
811
951c6300
RH		 812void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
813 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
814{
37ed3bf1
RH		 815 if (cond == TCG_COND_ALWAYS) {
816 tcg_gen_mov_i32(ret, v1);
817 } else if (cond == TCG_COND_NEVER) {
818 tcg_gen_mov_i32(ret, v2);
819 } else if (TCG_TARGET_HAS_movcond_i32) {
951c6300
RH		 820 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
821 } else {
822 TCGv_i32 t0 = tcg_temp_new_i32();
823 TCGv_i32 t1 = tcg_temp_new_i32();
824 tcg_gen_setcond_i32(cond, t0, c1, c2);
825 tcg_gen_neg_i32(t0, t0);
826 tcg_gen_and_i32(t1, v1, t0);
827 tcg_gen_andc_i32(ret, v2, t0);
828 tcg_gen_or_i32(ret, ret, t1);
829 tcg_temp_free_i32(t0);
830 tcg_temp_free_i32(t1);
831 }
832}
833
834void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
835 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
836{
837 if (TCG_TARGET_HAS_add2_i32) {
838 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 839 } else {
840 TCGv_i64 t0 = tcg_temp_new_i64();
841 TCGv_i64 t1 = tcg_temp_new_i64();
842 tcg_gen_concat_i32_i64(t0, al, ah);
843 tcg_gen_concat_i32_i64(t1, bl, bh);
844 tcg_gen_add_i64(t0, t0, t1);
845 tcg_gen_extr_i64_i32(rl, rh, t0);
846 tcg_temp_free_i64(t0);
847 tcg_temp_free_i64(t1);
848 }
849}
850
851void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
852 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
853{
854 if (TCG_TARGET_HAS_sub2_i32) {
855 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 856 } else {
857 TCGv_i64 t0 = tcg_temp_new_i64();
858 TCGv_i64 t1 = tcg_temp_new_i64();
859 tcg_gen_concat_i32_i64(t0, al, ah);
860 tcg_gen_concat_i32_i64(t1, bl, bh);
861 tcg_gen_sub_i64(t0, t0, t1);
862 tcg_gen_extr_i64_i32(rl, rh, t0);
863 tcg_temp_free_i64(t0);
864 tcg_temp_free_i64(t1);
865 }
866}
867
868void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
869{
870 if (TCG_TARGET_HAS_mulu2_i32) {
871 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
951c6300
RH		 872 } else if (TCG_TARGET_HAS_muluh_i32) {
873 TCGv_i32 t = tcg_temp_new_i32();
874 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
875 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
876 tcg_gen_mov_i32(rl, t);
877 tcg_temp_free_i32(t);
878 } else {
879 TCGv_i64 t0 = tcg_temp_new_i64();
880 TCGv_i64 t1 = tcg_temp_new_i64();
881 tcg_gen_extu_i32_i64(t0, arg1);
882 tcg_gen_extu_i32_i64(t1, arg2);
883 tcg_gen_mul_i64(t0, t0, t1);
884 tcg_gen_extr_i64_i32(rl, rh, t0);
885 tcg_temp_free_i64(t0);
886 tcg_temp_free_i64(t1);
887 }
888}
889
890void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
891{
892 if (TCG_TARGET_HAS_muls2_i32) {
893 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
951c6300
RH		 894 } else if (TCG_TARGET_HAS_mulsh_i32) {
895 TCGv_i32 t = tcg_temp_new_i32();
896 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
897 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
898 tcg_gen_mov_i32(rl, t);
899 tcg_temp_free_i32(t);
900 } else if (TCG_TARGET_REG_BITS == 32) {
901 TCGv_i32 t0 = tcg_temp_new_i32();
902 TCGv_i32 t1 = tcg_temp_new_i32();
903 TCGv_i32 t2 = tcg_temp_new_i32();
904 TCGv_i32 t3 = tcg_temp_new_i32();
905 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
906 /* Adjust for negative inputs. */
907 tcg_gen_sari_i32(t2, arg1, 31);
908 tcg_gen_sari_i32(t3, arg2, 31);
909 tcg_gen_and_i32(t2, t2, arg2);
910 tcg_gen_and_i32(t3, t3, arg1);
911 tcg_gen_sub_i32(rh, t1, t2);
912 tcg_gen_sub_i32(rh, rh, t3);
913 tcg_gen_mov_i32(rl, t0);
914 tcg_temp_free_i32(t0);
915 tcg_temp_free_i32(t1);
916 tcg_temp_free_i32(t2);
917 tcg_temp_free_i32(t3);
918 } else {
919 TCGv_i64 t0 = tcg_temp_new_i64();
920 TCGv_i64 t1 = tcg_temp_new_i64();
921 tcg_gen_ext_i32_i64(t0, arg1);
922 tcg_gen_ext_i32_i64(t1, arg2);
923 tcg_gen_mul_i64(t0, t0, t1);
924 tcg_gen_extr_i64_i32(rl, rh, t0);
925 tcg_temp_free_i64(t0);
926 tcg_temp_free_i64(t1);
927 }
928}
929
5087abfb
RH		 930void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
931{
932 if (TCG_TARGET_REG_BITS == 32) {
933 TCGv_i32 t0 = tcg_temp_new_i32();
934 TCGv_i32 t1 = tcg_temp_new_i32();
935 TCGv_i32 t2 = tcg_temp_new_i32();
936 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
937 /* Adjust for negative input for the signed arg1. */
938 tcg_gen_sari_i32(t2, arg1, 31);
939 tcg_gen_and_i32(t2, t2, arg2);
940 tcg_gen_sub_i32(rh, t1, t2);
941 tcg_gen_mov_i32(rl, t0);
942 tcg_temp_free_i32(t0);
943 tcg_temp_free_i32(t1);
944 tcg_temp_free_i32(t2);
945 } else {
946 TCGv_i64 t0 = tcg_temp_new_i64();
947 TCGv_i64 t1 = tcg_temp_new_i64();
948 tcg_gen_ext_i32_i64(t0, arg1);
949 tcg_gen_extu_i32_i64(t1, arg2);
950 tcg_gen_mul_i64(t0, t0, t1);
951 tcg_gen_extr_i64_i32(rl, rh, t0);
952 tcg_temp_free_i64(t0);
953 tcg_temp_free_i64(t1);
954 }
955}
956
951c6300
RH		 957void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
958{
959 if (TCG_TARGET_HAS_ext8s_i32) {
960 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
961 } else {
962 tcg_gen_shli_i32(ret, arg, 24);
963 tcg_gen_sari_i32(ret, ret, 24);
964 }
965}
966
967void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
968{
969 if (TCG_TARGET_HAS_ext16s_i32) {
970 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
971 } else {
972 tcg_gen_shli_i32(ret, arg, 16);
973 tcg_gen_sari_i32(ret, ret, 16);
974 }
975}
976
977void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
978{
979 if (TCG_TARGET_HAS_ext8u_i32) {
980 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
981 } else {
982 tcg_gen_andi_i32(ret, arg, 0xffu);
983 }
984}
985
986void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
987{
988 if (TCG_TARGET_HAS_ext16u_i32) {
989 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
990 } else {
991 tcg_gen_andi_i32(ret, arg, 0xffffu);
992 }
993}
994
995/* Note: we assume the two high bytes are set to zero */
996void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
997{
998 if (TCG_TARGET_HAS_bswap16_i32) {
999 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
1000 } else {
1001 TCGv_i32 t0 = tcg_temp_new_i32();
1002
1003 tcg_gen_ext8u_i32(t0, arg);
1004 tcg_gen_shli_i32(t0, t0, 8);
1005 tcg_gen_shri_i32(ret, arg, 8);
1006 tcg_gen_or_i32(ret, ret, t0);
1007 tcg_temp_free_i32(t0);
1008 }
1009}
1010
1011void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1012{
1013 if (TCG_TARGET_HAS_bswap32_i32) {
1014 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1015 } else {
a686dc71
RH		 1016 TCGv_i32 t0 = tcg_temp_new_i32();
1017 TCGv_i32 t1 = tcg_temp_new_i32();
1018 TCGv_i32 t2 = tcg_const_i32(0x00ff00ff);
951c6300 1019
a686dc71
RH		 1020 /* arg = abcd */
1021 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */
1022 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */
1023 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */
1024 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */
1025 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */
951c6300 1026
a686dc71
RH		 1027 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */
1028 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */
1029 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */
951c6300 1030
951c6300
RH		 1031 tcg_temp_free_i32(t0);
1032 tcg_temp_free_i32(t1);
a686dc71 1033 tcg_temp_free_i32(t2);
951c6300
RH		 1034 }
1035}
1036
b87fb8cd
RH		 1037void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1038{
1039 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1040}
1041
1042void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1043{
1044 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1045}
1046
1047void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1048{
1049 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1050}
1051
1052void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1053{
1054 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1055}
1056
951c6300
RH		 1057/* 64-bit ops */
1058
1059#if TCG_TARGET_REG_BITS == 32
1060/* These are all inline for TCG_TARGET_REG_BITS == 64. */
1061
1062void tcg_gen_discard_i64(TCGv_i64 arg)
1063{
1064 tcg_gen_discard_i32(TCGV_LOW(arg));
1065 tcg_gen_discard_i32(TCGV_HIGH(arg));
1066}
1067
1068void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1069{
1070 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1071 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1072}
1073
1074void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1075{
1076 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1077 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1078}
1079
1080void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1081{
1082 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1083 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1084}
1085
1086void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1087{
1088 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
3ff91d7e 1089 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
951c6300
RH		 1090}
1091
1092void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1093{
1094 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1095 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1096}
1097
1098void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1099{
1100 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1101 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1102}
1103
1104void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1105{
1106 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1107 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1108}
1109
1110void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1111{
1112 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1113 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1114}
1115
1116void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1117{
1118 /* Since arg2 and ret have different types,
1119 they cannot be the same temporary */
cf811fff 1120#ifdef HOST_WORDS_BIGENDIAN
951c6300
RH		 1121 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1122 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1123#else
1124 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1125 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1126#endif
1127}
1128
1129void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1130{
cf811fff 1131#ifdef HOST_WORDS_BIGENDIAN
951c6300
RH		 1132 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1133 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1134#else
1135 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1136 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1137#endif
1138}
1139
1140void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1141{
1142 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1143 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1144}
1145
1146void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1147{
1148 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1149 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1150}
1151
1152void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1153{
1154 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1155 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1156}
1157
1158void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1159{
1160 gen_helper_shl_i64(ret, arg1, arg2);
1161}
1162
1163void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1164{
1165 gen_helper_shr_i64(ret, arg1, arg2);
1166}
1167
1168void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1169{
1170 gen_helper_sar_i64(ret, arg1, arg2);
1171}
1172
1173void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1174{
1175 TCGv_i64 t0;
1176 TCGv_i32 t1;
1177
1178 t0 = tcg_temp_new_i64();
1179 t1 = tcg_temp_new_i32();
1180
1181 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1182 TCGV_LOW(arg1), TCGV_LOW(arg2));
1183
1184 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1185 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1186 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1187 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1188
1189 tcg_gen_mov_i64(ret, t0);
1190 tcg_temp_free_i64(t0);
1191 tcg_temp_free_i32(t1);
1192}
1193#endif /* TCG_TARGET_REG_SIZE == 32 */
1194
1195void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1196{
1197 /* some cases can be optimized here */
1198 if (arg2 == 0) {
1199 tcg_gen_mov_i64(ret, arg1);
1200 } else {
1201 TCGv_i64 t0 = tcg_const_i64(arg2);
1202 tcg_gen_add_i64(ret, arg1, t0);
1203 tcg_temp_free_i64(t0);
1204 }
1205}
1206
1207void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1208{
1209 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1210 /* Don't recurse with tcg_gen_neg_i64. */
1211 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1212 } else {
1213 TCGv_i64 t0 = tcg_const_i64(arg1);
1214 tcg_gen_sub_i64(ret, t0, arg2);
1215 tcg_temp_free_i64(t0);
1216 }
1217}
1218
1219void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1220{
1221 /* some cases can be optimized here */
1222 if (arg2 == 0) {
1223 tcg_gen_mov_i64(ret, arg1);
1224 } else {
1225 TCGv_i64 t0 = tcg_const_i64(arg2);
1226 tcg_gen_sub_i64(ret, arg1, t0);
1227 tcg_temp_free_i64(t0);
1228 }
1229}
1230
474b2e8f 1231void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1232{
951c6300 1233 TCGv_i64 t0;
3a13c3f3
RH		 1234
1235 if (TCG_TARGET_REG_BITS == 32) {
1236 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1237 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1238 return;
1239 }
1240
951c6300
RH		 1241 /* Some cases can be optimized here. */
1242 switch (arg2) {
1243 case 0:
1244 tcg_gen_movi_i64(ret, 0);
1245 return;
474b2e8f 1246 case -1:
951c6300
RH		 1247 tcg_gen_mov_i64(ret, arg1);
1248 return;
474b2e8f 1249 case 0xff:
951c6300
RH		 1250 /* Don't recurse with tcg_gen_ext8u_i64. */
1251 if (TCG_TARGET_HAS_ext8u_i64) {
1252 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1253 return;
1254 }
1255 break;
474b2e8f 1256 case 0xffff:
951c6300
RH		 1257 if (TCG_TARGET_HAS_ext16u_i64) {
1258 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1259 return;
1260 }
1261 break;
474b2e8f 1262 case 0xffffffffu:
951c6300
RH		 1263 if (TCG_TARGET_HAS_ext32u_i64) {
1264 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1265 return;
1266 }
1267 break;
1268 }
1269 t0 = tcg_const_i64(arg2);
1270 tcg_gen_and_i64(ret, arg1, t0);
1271 tcg_temp_free_i64(t0);
951c6300
RH		 1272}
1273
1274void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1275{
3a13c3f3
RH		 1276 if (TCG_TARGET_REG_BITS == 32) {
1277 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1278 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1279 return;
1280 }
951c6300
RH		 1281 /* Some cases can be optimized here. */
1282 if (arg2 == -1) {
1283 tcg_gen_movi_i64(ret, -1);
1284 } else if (arg2 == 0) {
1285 tcg_gen_mov_i64(ret, arg1);
1286 } else {
1287 TCGv_i64 t0 = tcg_const_i64(arg2);
1288 tcg_gen_or_i64(ret, arg1, t0);
1289 tcg_temp_free_i64(t0);
1290 }
951c6300
RH		 1291}
1292
1293void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1294{
3a13c3f3
RH		 1295 if (TCG_TARGET_REG_BITS == 32) {
1296 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1297 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1298 return;
1299 }
951c6300
RH		 1300 /* Some cases can be optimized here. */
1301 if (arg2 == 0) {
1302 tcg_gen_mov_i64(ret, arg1);
1303 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1304 /* Don't recurse with tcg_gen_not_i64. */
1305 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1306 } else {
1307 TCGv_i64 t0 = tcg_const_i64(arg2);
1308 tcg_gen_xor_i64(ret, arg1, t0);
1309 tcg_temp_free_i64(t0);
1310 }
951c6300
RH		 1311}
1312
951c6300
RH		 1313static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1314 unsigned c, bool right, bool arith)
1315{
1316 tcg_debug_assert(c < 64);
1317 if (c == 0) {
1318 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1319 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1320 } else if (c >= 32) {
1321 c -= 32;
1322 if (right) {
1323 if (arith) {
1324 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1325 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1326 } else {
1327 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1328 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1329 }
1330 } else {
1331 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1332 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1333 }
1334 } else {
1335 TCGv_i32 t0, t1;
1336
1337 t0 = tcg_temp_new_i32();
1338 t1 = tcg_temp_new_i32();
1339 if (right) {
1340 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1341 if (arith) {
1342 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1343 } else {
1344 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1345 }
1346 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1347 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1348 tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1349 } else {
1350 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1351 /* Note: ret can be the same as arg1, so we use t1 */
1352 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1353 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1354 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1355 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1356 }
1357 tcg_temp_free_i32(t0);
1358 tcg_temp_free_i32(t1);
1359 }
1360}
1361
474b2e8f 1362void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1363{
474b2e8f 1364 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1365 if (TCG_TARGET_REG_BITS == 32) {
1366 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1367 } else if (arg2 == 0) {
951c6300
RH		 1368 tcg_gen_mov_i64(ret, arg1);
1369 } else {
1370 TCGv_i64 t0 = tcg_const_i64(arg2);
1371 tcg_gen_shl_i64(ret, arg1, t0);
1372 tcg_temp_free_i64(t0);
1373 }
1374}
1375
474b2e8f 1376void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1377{
474b2e8f 1378 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1379 if (TCG_TARGET_REG_BITS == 32) {
1380 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1381 } else if (arg2 == 0) {
951c6300
RH		 1382 tcg_gen_mov_i64(ret, arg1);
1383 } else {
1384 TCGv_i64 t0 = tcg_const_i64(arg2);
1385 tcg_gen_shr_i64(ret, arg1, t0);
1386 tcg_temp_free_i64(t0);
1387 }
1388}
1389
474b2e8f 1390void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1391{
474b2e8f 1392 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1393 if (TCG_TARGET_REG_BITS == 32) {
1394 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1395 } else if (arg2 == 0) {
951c6300
RH		 1396 tcg_gen_mov_i64(ret, arg1);
1397 } else {
1398 TCGv_i64 t0 = tcg_const_i64(arg2);
1399 tcg_gen_sar_i64(ret, arg1, t0);
1400 tcg_temp_free_i64(t0);
1401 }
1402}
951c6300 1403
42a268c2 1404void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
951c6300
RH		 1405{
1406 if (cond == TCG_COND_ALWAYS) {
42a268c2 1407 tcg_gen_br(l);
951c6300 1408 } else if (cond != TCG_COND_NEVER) {
d88a117e 1409 l->refs++;
3a13c3f3
RH		 1410 if (TCG_TARGET_REG_BITS == 32) {
1411 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1412 TCGV_HIGH(arg1), TCGV_LOW(arg2),
42a268c2 1413 TCGV_HIGH(arg2), cond, label_arg(l));
3a13c3f3 1414 } else {
42a268c2
RH		 1415 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1416 label_arg(l));
3a13c3f3 1417 }
951c6300
RH		 1418 }
1419}
1420
42a268c2 1421void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
951c6300
RH		 1422{
1423 if (cond == TCG_COND_ALWAYS) {
42a268c2 1424 tcg_gen_br(l);
951c6300
RH		 1425 } else if (cond != TCG_COND_NEVER) {
1426 TCGv_i64 t0 = tcg_const_i64(arg2);
42a268c2 1427 tcg_gen_brcond_i64(cond, arg1, t0, l);
951c6300
RH		 1428 tcg_temp_free_i64(t0);
1429 }
1430}
1431
1432void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1433 TCGv_i64 arg1, TCGv_i64 arg2)
1434{
1435 if (cond == TCG_COND_ALWAYS) {
1436 tcg_gen_movi_i64(ret, 1);
1437 } else if (cond == TCG_COND_NEVER) {
1438 tcg_gen_movi_i64(ret, 0);
1439 } else {
3a13c3f3
RH		 1440 if (TCG_TARGET_REG_BITS == 32) {
1441 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1442 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1443 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1444 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1445 } else {
1446 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1447 }
951c6300
RH		 1448 }
1449}
1450
1451void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1452 TCGv_i64 arg1, int64_t arg2)
1453{
1454 TCGv_i64 t0 = tcg_const_i64(arg2);
1455 tcg_gen_setcond_i64(cond, ret, arg1, t0);
1456 tcg_temp_free_i64(t0);
1457}
1458
1459void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1460{
b2e3ae94
RH		 1461 if (arg2 == 0) {
1462 tcg_gen_movi_i64(ret, 0);
1463 } else if (is_power_of_2(arg2)) {
1464 tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1465 } else {
1466 TCGv_i64 t0 = tcg_const_i64(arg2);
1467 tcg_gen_mul_i64(ret, arg1, t0);
1468 tcg_temp_free_i64(t0);
1469 }
951c6300
RH		 1470}
1471
1472void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1473{
1474 if (TCG_TARGET_HAS_div_i64) {
1475 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1476 } else if (TCG_TARGET_HAS_div2_i64) {
1477 TCGv_i64 t0 = tcg_temp_new_i64();
1478 tcg_gen_sari_i64(t0, arg1, 63);
1479 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1480 tcg_temp_free_i64(t0);
1481 } else {
1482 gen_helper_div_i64(ret, arg1, arg2);
1483 }
1484}
1485
1486void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1487{
1488 if (TCG_TARGET_HAS_rem_i64) {
1489 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1490 } else if (TCG_TARGET_HAS_div_i64) {
1491 TCGv_i64 t0 = tcg_temp_new_i64();
1492 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1493 tcg_gen_mul_i64(t0, t0, arg2);
1494 tcg_gen_sub_i64(ret, arg1, t0);
1495 tcg_temp_free_i64(t0);
1496 } else if (TCG_TARGET_HAS_div2_i64) {
1497 TCGv_i64 t0 = tcg_temp_new_i64();
1498 tcg_gen_sari_i64(t0, arg1, 63);
1499 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1500 tcg_temp_free_i64(t0);
1501 } else {
1502 gen_helper_rem_i64(ret, arg1, arg2);
1503 }
1504}
1505
1506void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1507{
1508 if (TCG_TARGET_HAS_div_i64) {
1509 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1510 } else if (TCG_TARGET_HAS_div2_i64) {
1511 TCGv_i64 t0 = tcg_temp_new_i64();
1512 tcg_gen_movi_i64(t0, 0);
1513 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1514 tcg_temp_free_i64(t0);
1515 } else {
1516 gen_helper_divu_i64(ret, arg1, arg2);
1517 }
1518}
1519
1520void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1521{
1522 if (TCG_TARGET_HAS_rem_i64) {
1523 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1524 } else if (TCG_TARGET_HAS_div_i64) {
1525 TCGv_i64 t0 = tcg_temp_new_i64();
1526 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1527 tcg_gen_mul_i64(t0, t0, arg2);
1528 tcg_gen_sub_i64(ret, arg1, t0);
1529 tcg_temp_free_i64(t0);
1530 } else if (TCG_TARGET_HAS_div2_i64) {
1531 TCGv_i64 t0 = tcg_temp_new_i64();
1532 tcg_gen_movi_i64(t0, 0);
1533 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1534 tcg_temp_free_i64(t0);
1535 } else {
1536 gen_helper_remu_i64(ret, arg1, arg2);
1537 }
1538}
1539
1540void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1541{
3a13c3f3
RH		 1542 if (TCG_TARGET_REG_BITS == 32) {
1543 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1544 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1545 } else if (TCG_TARGET_HAS_ext8s_i64) {
951c6300
RH		 1546 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1547 } else {
1548 tcg_gen_shli_i64(ret, arg, 56);
1549 tcg_gen_sari_i64(ret, ret, 56);
1550 }
951c6300
RH		 1551}
1552
1553void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1554{
3a13c3f3
RH		 1555 if (TCG_TARGET_REG_BITS == 32) {
1556 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1557 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1558 } else if (TCG_TARGET_HAS_ext16s_i64) {
951c6300
RH		 1559 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1560 } else {
1561 tcg_gen_shli_i64(ret, arg, 48);
1562 tcg_gen_sari_i64(ret, ret, 48);
1563 }
951c6300
RH		 1564}
1565
1566void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1567{
3a13c3f3
RH		 1568 if (TCG_TARGET_REG_BITS == 32) {
1569 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1570 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1571 } else if (TCG_TARGET_HAS_ext32s_i64) {
951c6300
RH		 1572 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1573 } else {
1574 tcg_gen_shli_i64(ret, arg, 32);
1575 tcg_gen_sari_i64(ret, ret, 32);
1576 }
951c6300
RH		 1577}
1578
1579void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1580{
3a13c3f3
RH		 1581 if (TCG_TARGET_REG_BITS == 32) {
1582 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1583 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1584 } else if (TCG_TARGET_HAS_ext8u_i64) {
951c6300
RH		 1585 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1586 } else {
1587 tcg_gen_andi_i64(ret, arg, 0xffu);
1588 }
951c6300
RH		 1589}
1590
1591void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1592{
3a13c3f3
RH		 1593 if (TCG_TARGET_REG_BITS == 32) {
1594 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1595 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1596 } else if (TCG_TARGET_HAS_ext16u_i64) {
951c6300
RH		 1597 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1598 } else {
1599 tcg_gen_andi_i64(ret, arg, 0xffffu);
1600 }
951c6300
RH		 1601}
1602
1603void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1604{
3a13c3f3
RH		 1605 if (TCG_TARGET_REG_BITS == 32) {
1606 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1607 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1608 } else if (TCG_TARGET_HAS_ext32u_i64) {
951c6300
RH		 1609 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1610 } else {
1611 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1612 }
951c6300
RH		 1613}
1614
1615/* Note: we assume the six high bytes are set to zero */
1616void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1617{
3a13c3f3
RH		 1618 if (TCG_TARGET_REG_BITS == 32) {
1619 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1620 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1621 } else if (TCG_TARGET_HAS_bswap16_i64) {
951c6300
RH		 1622 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1623 } else {
1624 TCGv_i64 t0 = tcg_temp_new_i64();
1625
1626 tcg_gen_ext8u_i64(t0, arg);
1627 tcg_gen_shli_i64(t0, t0, 8);
1628 tcg_gen_shri_i64(ret, arg, 8);
1629 tcg_gen_or_i64(ret, ret, t0);
1630 tcg_temp_free_i64(t0);
1631 }
951c6300
RH		 1632}
1633
1634/* Note: we assume the four high bytes are set to zero */
1635void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1636{
3a13c3f3
RH		 1637 if (TCG_TARGET_REG_BITS == 32) {
1638 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1639 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1640 } else if (TCG_TARGET_HAS_bswap32_i64) {
951c6300
RH		 1641 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1642 } else {
a686dc71
RH		 1643 TCGv_i64 t0 = tcg_temp_new_i64();
1644 TCGv_i64 t1 = tcg_temp_new_i64();
1645 TCGv_i64 t2 = tcg_const_i64(0x00ff00ff);
951c6300 1646
a686dc71
RH		 1647 /* arg = ....abcd */
1648 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .....abc */
1649 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */
1650 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */
1651 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */
1652 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */
951c6300 1653
a686dc71
RH		 1654 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */
1655 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */
1656 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */
1657 tcg_gen_or_i64(ret, t0, t1); /* ret = ....dcba */
951c6300 1658
951c6300
RH		 1659 tcg_temp_free_i64(t0);
1660 tcg_temp_free_i64(t1);
a686dc71 1661 tcg_temp_free_i64(t2);
951c6300 1662 }
951c6300
RH		 1663}
1664
1665void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1666{
3a13c3f3
RH		 1667 if (TCG_TARGET_REG_BITS == 32) {
1668 TCGv_i32 t0, t1;
1669 t0 = tcg_temp_new_i32();
1670 t1 = tcg_temp_new_i32();
951c6300 1671
3a13c3f3
RH		 1672 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1673 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1674 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1675 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1676 tcg_temp_free_i32(t0);
1677 tcg_temp_free_i32(t1);
1678 } else if (TCG_TARGET_HAS_bswap64_i64) {
951c6300
RH		 1679 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1680 } else {
1681 TCGv_i64 t0 = tcg_temp_new_i64();
1682 TCGv_i64 t1 = tcg_temp_new_i64();
9e821eab 1683 TCGv_i64 t2 = tcg_temp_new_i64();
951c6300 1684
9e821eab
RH		 1685 /* arg = abcdefgh */
1686 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1687 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */
1688 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */
1689 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */
1690 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */
1691 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */
1692
1693 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1694 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */
1695 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */
1696 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */
1697 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */
1698 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */
1699
1700 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */
1701 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */
1702 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */
951c6300 1703
951c6300
RH		 1704 tcg_temp_free_i64(t0);
1705 tcg_temp_free_i64(t1);
9e821eab 1706 tcg_temp_free_i64(t2);
951c6300 1707 }
951c6300
RH		 1708}
1709
1710void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1711{
3a13c3f3
RH		 1712 if (TCG_TARGET_REG_BITS == 32) {
1713 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1714 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1715 } else if (TCG_TARGET_HAS_not_i64) {
951c6300
RH		 1716 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1717 } else {
1718 tcg_gen_xori_i64(ret, arg, -1);
1719 }
951c6300
RH		 1720}
1721
1722void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1723{
3a13c3f3
RH		 1724 if (TCG_TARGET_REG_BITS == 32) {
1725 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1726 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1727 } else if (TCG_TARGET_HAS_andc_i64) {
951c6300
RH		 1728 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1729 } else {
1730 TCGv_i64 t0 = tcg_temp_new_i64();
1731 tcg_gen_not_i64(t0, arg2);
1732 tcg_gen_and_i64(ret, arg1, t0);
1733 tcg_temp_free_i64(t0);
1734 }
951c6300
RH		 1735}
1736
1737void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1738{
3a13c3f3
RH		 1739 if (TCG_TARGET_REG_BITS == 32) {
1740 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1741 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1742 } else if (TCG_TARGET_HAS_eqv_i64) {
951c6300
RH		 1743 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1744 } else {
1745 tcg_gen_xor_i64(ret, arg1, arg2);
1746 tcg_gen_not_i64(ret, ret);
1747 }
951c6300
RH		 1748}
1749
1750void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1751{
3a13c3f3
RH		 1752 if (TCG_TARGET_REG_BITS == 32) {
1753 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1754 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1755 } else if (TCG_TARGET_HAS_nand_i64) {
951c6300
RH		 1756 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1757 } else {
1758 tcg_gen_and_i64(ret, arg1, arg2);
1759 tcg_gen_not_i64(ret, ret);
1760 }
951c6300
RH		 1761}
1762
1763void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1764{
3a13c3f3
RH		 1765 if (TCG_TARGET_REG_BITS == 32) {
1766 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1767 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1768 } else if (TCG_TARGET_HAS_nor_i64) {
951c6300
RH		 1769 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1770 } else {
1771 tcg_gen_or_i64(ret, arg1, arg2);
1772 tcg_gen_not_i64(ret, ret);
1773 }
951c6300
RH		 1774}
1775
1776void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1777{
3a13c3f3
RH		 1778 if (TCG_TARGET_REG_BITS == 32) {
1779 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1780 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1781 } else if (TCG_TARGET_HAS_orc_i64) {
951c6300
RH		 1782 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1783 } else {
1784 TCGv_i64 t0 = tcg_temp_new_i64();
1785 tcg_gen_not_i64(t0, arg2);
1786 tcg_gen_or_i64(ret, arg1, t0);
1787 tcg_temp_free_i64(t0);
1788 }
951c6300
RH		 1789}
1790
0e28d006
RH		 1791void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1792{
1793 if (TCG_TARGET_HAS_clz_i64) {
1794 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1795 } else {
1796 gen_helper_clz_i64(ret, arg1, arg2);
1797 }
1798}
1799
1800void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1801{
1802 if (TCG_TARGET_REG_BITS == 32
1803 && TCG_TARGET_HAS_clz_i32
1804 && arg2 <= 0xffffffffu) {
1805 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1806 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1807 tcg_gen_addi_i32(t, t, 32);
1808 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1809 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1810 tcg_temp_free_i32(t);
1811 } else {
1812 TCGv_i64 t = tcg_const_i64(arg2);
1813 tcg_gen_clz_i64(ret, arg1, t);
1814 tcg_temp_free_i64(t);
1815 }
1816}
1817
1818void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1819{
1820 if (TCG_TARGET_HAS_ctz_i64) {
1821 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
14e99210
RH		 1822 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1823 TCGv_i64 z, t = tcg_temp_new_i64();
1824
1825 if (TCG_TARGET_HAS_ctpop_i64) {
1826 tcg_gen_subi_i64(t, arg1, 1);
1827 tcg_gen_andc_i64(t, t, arg1);
1828 tcg_gen_ctpop_i64(t, t);
1829 } else {
1830 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1831 tcg_gen_neg_i64(t, arg1);
1832 tcg_gen_and_i64(t, t, arg1);
1833 tcg_gen_clzi_i64(t, t, 64);
1834 tcg_gen_xori_i64(t, t, 63);
1835 }
1836 z = tcg_const_i64(0);
1837 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1838 tcg_temp_free_i64(t);
1839 tcg_temp_free_i64(z);
0e28d006
RH		 1840 } else {
1841 gen_helper_ctz_i64(ret, arg1, arg2);
1842 }
1843}
1844
1845void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1846{
1847 if (TCG_TARGET_REG_BITS == 32
1848 && TCG_TARGET_HAS_ctz_i32
1849 && arg2 <= 0xffffffffu) {
1850 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1851 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1852 tcg_gen_addi_i32(t32, t32, 32);
1853 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1854 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1855 tcg_temp_free_i32(t32);
14e99210
RH		 1856 } else if (!TCG_TARGET_HAS_ctz_i64
1857 && TCG_TARGET_HAS_ctpop_i64
1858 && arg2 == 64) {
1859 /* This equivalence has the advantage of not requiring a fixup. */
1860 TCGv_i64 t = tcg_temp_new_i64();
1861 tcg_gen_subi_i64(t, arg1, 1);
1862 tcg_gen_andc_i64(t, t, arg1);
1863 tcg_gen_ctpop_i64(ret, t);
1864 tcg_temp_free_i64(t);
0e28d006
RH		 1865 } else {
1866 TCGv_i64 t64 = tcg_const_i64(arg2);
1867 tcg_gen_ctz_i64(ret, arg1, t64);
1868 tcg_temp_free_i64(t64);
1869 }
1870}
1871
086920c2
RH		 1872void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1873{
1874 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1875 TCGv_i64 t = tcg_temp_new_i64();
1876 tcg_gen_sari_i64(t, arg, 63);
1877 tcg_gen_xor_i64(t, t, arg);
1878 tcg_gen_clzi_i64(t, t, 64);
1879 tcg_gen_subi_i64(ret, t, 1);
1880 tcg_temp_free_i64(t);
1881 } else {
1882 gen_helper_clrsb_i64(ret, arg);
1883 }
1884}
1885
a768e4e9
RH		 1886void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1887{
1888 if (TCG_TARGET_HAS_ctpop_i64) {
1889 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1890 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1891 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1892 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1893 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1894 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1895 } else {
1896 gen_helper_ctpop_i64(ret, arg1);
1897 }
1898}
1899
951c6300
RH		 1900void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1901{
1902 if (TCG_TARGET_HAS_rot_i64) {
1903 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1904 } else {
1905 TCGv_i64 t0, t1;
1906 t0 = tcg_temp_new_i64();
1907 t1 = tcg_temp_new_i64();
1908 tcg_gen_shl_i64(t0, arg1, arg2);
1909 tcg_gen_subfi_i64(t1, 64, arg2);
1910 tcg_gen_shr_i64(t1, arg1, t1);
1911 tcg_gen_or_i64(ret, t0, t1);
1912 tcg_temp_free_i64(t0);
1913 tcg_temp_free_i64(t1);
1914 }
1915}
1916
1917void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1918{
1919 tcg_debug_assert(arg2 < 64);
1920 /* some cases can be optimized here */
1921 if (arg2 == 0) {
1922 tcg_gen_mov_i64(ret, arg1);
1923 } else if (TCG_TARGET_HAS_rot_i64) {
1924 TCGv_i64 t0 = tcg_const_i64(arg2);
1925 tcg_gen_rotl_i64(ret, arg1, t0);
1926 tcg_temp_free_i64(t0);
1927 } else {
1928 TCGv_i64 t0, t1;
1929 t0 = tcg_temp_new_i64();
1930 t1 = tcg_temp_new_i64();
1931 tcg_gen_shli_i64(t0, arg1, arg2);
1932 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1933 tcg_gen_or_i64(ret, t0, t1);
1934 tcg_temp_free_i64(t0);
1935 tcg_temp_free_i64(t1);
1936 }
1937}
1938
1939void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1940{
1941 if (TCG_TARGET_HAS_rot_i64) {
1942 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1943 } else {
1944 TCGv_i64 t0, t1;
1945 t0 = tcg_temp_new_i64();
1946 t1 = tcg_temp_new_i64();
1947 tcg_gen_shr_i64(t0, arg1, arg2);
1948 tcg_gen_subfi_i64(t1, 64, arg2);
1949 tcg_gen_shl_i64(t1, arg1, t1);
1950 tcg_gen_or_i64(ret, t0, t1);
1951 tcg_temp_free_i64(t0);
1952 tcg_temp_free_i64(t1);
1953 }
1954}
1955
1956void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1957{
1958 tcg_debug_assert(arg2 < 64);
1959 /* some cases can be optimized here */
1960 if (arg2 == 0) {
1961 tcg_gen_mov_i64(ret, arg1);
1962 } else {
1963 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1964 }
1965}
1966
1967void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1968 unsigned int ofs, unsigned int len)
1969{
1970 uint64_t mask;
1971 TCGv_i64 t1;
1972
1973 tcg_debug_assert(ofs < 64);
0d0d309d 1974 tcg_debug_assert(len > 0);
951c6300
RH		 1975 tcg_debug_assert(len <= 64);
1976 tcg_debug_assert(ofs + len <= 64);
1977
0d0d309d 1978 if (len == 64) {
951c6300
RH		 1979 tcg_gen_mov_i64(ret, arg2);
1980 return;
1981 }
1982 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1983 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
1984 return;
1985 }
1986
3a13c3f3
RH		 1987 if (TCG_TARGET_REG_BITS == 32) {
1988 if (ofs >= 32) {
1989 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
1990 TCGV_LOW(arg2), ofs - 32, len);
1991 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1992 return;
1993 }
1994 if (ofs + len <= 32) {
1995 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
1996 TCGV_LOW(arg2), ofs, len);
1997 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1998 return;
1999 }
951c6300 2000 }
951c6300
RH		 2001
2002 mask = (1ull << len) - 1;
2003 t1 = tcg_temp_new_i64();
2004
2005 if (ofs + len < 64) {
2006 tcg_gen_andi_i64(t1, arg2, mask);
2007 tcg_gen_shli_i64(t1, t1, ofs);
2008 } else {
2009 tcg_gen_shli_i64(t1, arg2, ofs);
2010 }
2011 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2012 tcg_gen_or_i64(ret, ret, t1);
2013
2014 tcg_temp_free_i64(t1);
2015}
2016
07cc68d5
RH		 2017void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2018 unsigned int ofs, unsigned int len)
2019{
2020 tcg_debug_assert(ofs < 64);
2021 tcg_debug_assert(len > 0);
2022 tcg_debug_assert(len <= 64);
2023 tcg_debug_assert(ofs + len <= 64);
2024
2025 if (ofs + len == 64) {
2026 tcg_gen_shli_i64(ret, arg, ofs);
2027 } else if (ofs == 0) {
2028 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2029 } else if (TCG_TARGET_HAS_deposit_i64
2030 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2031 TCGv_i64 zero = tcg_const_i64(0);
2032 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2033 tcg_temp_free_i64(zero);
2034 } else {
2035 if (TCG_TARGET_REG_BITS == 32) {
2036 if (ofs >= 32) {
2037 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2038 ofs - 32, len);
2039 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2040 return;
2041 }
2042 if (ofs + len <= 32) {
2043 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2044 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2045 return;
2046 }
2047 }
2048 /* To help two-operand hosts we prefer to zero-extend first,
2049 which allows ARG to stay live. */
2050 switch (len) {
2051 case 32:
2052 if (TCG_TARGET_HAS_ext32u_i64) {
2053 tcg_gen_ext32u_i64(ret, arg);
2054 tcg_gen_shli_i64(ret, ret, ofs);
2055 return;
2056 }
2057 break;
2058 case 16:
2059 if (TCG_TARGET_HAS_ext16u_i64) {
2060 tcg_gen_ext16u_i64(ret, arg);
2061 tcg_gen_shli_i64(ret, ret, ofs);
2062 return;
2063 }
2064 break;
2065 case 8:
2066 if (TCG_TARGET_HAS_ext8u_i64) {
2067 tcg_gen_ext8u_i64(ret, arg);
2068 tcg_gen_shli_i64(ret, ret, ofs);
2069 return;
2070 }
2071 break;
2072 }
2073 /* Otherwise prefer zero-extension over AND for code size. */
2074 switch (ofs + len) {
2075 case 32:
2076 if (TCG_TARGET_HAS_ext32u_i64) {
2077 tcg_gen_shli_i64(ret, arg, ofs);
2078 tcg_gen_ext32u_i64(ret, ret);
2079 return;
2080 }
2081 break;
2082 case 16:
2083 if (TCG_TARGET_HAS_ext16u_i64) {
2084 tcg_gen_shli_i64(ret, arg, ofs);
2085 tcg_gen_ext16u_i64(ret, ret);
2086 return;
2087 }
2088 break;
2089 case 8:
2090 if (TCG_TARGET_HAS_ext8u_i64) {
2091 tcg_gen_shli_i64(ret, arg, ofs);
2092 tcg_gen_ext8u_i64(ret, ret);
2093 return;
2094 }
2095 break;
2096 }
2097 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2098 tcg_gen_shli_i64(ret, ret, ofs);
2099 }
2100}
2101
7ec8bab3
RH		 2102void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2103 unsigned int ofs, unsigned int len)
2104{
2105 tcg_debug_assert(ofs < 64);
2106 tcg_debug_assert(len > 0);
2107 tcg_debug_assert(len <= 64);
2108 tcg_debug_assert(ofs + len <= 64);
2109
2110 /* Canonicalize certain special cases, even if extract is supported. */
2111 if (ofs + len == 64) {
2112 tcg_gen_shri_i64(ret, arg, 64 - len);
2113 return;
2114 }
2115 if (ofs == 0) {
2116 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2117 return;
2118 }
2119
2120 if (TCG_TARGET_REG_BITS == 32) {
2121 /* Look for a 32-bit extract within one of the two words. */
2122 if (ofs >= 32) {
2123 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2124 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2125 return;
2126 }
2127 if (ofs + len <= 32) {
2128 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2129 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2130 return;
2131 }
2132 /* The field is split across two words. One double-word
2133 shift is better than two double-word shifts. */
2134 goto do_shift_and;
2135 }
2136
2137 if (TCG_TARGET_HAS_extract_i64
2138 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2139 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2140 return;
2141 }
2142
2143 /* Assume that zero-extension, if available, is cheaper than a shift. */
2144 switch (ofs + len) {
2145 case 32:
2146 if (TCG_TARGET_HAS_ext32u_i64) {
2147 tcg_gen_ext32u_i64(ret, arg);
2148 tcg_gen_shri_i64(ret, ret, ofs);
2149 return;
2150 }
2151 break;
2152 case 16:
2153 if (TCG_TARGET_HAS_ext16u_i64) {
2154 tcg_gen_ext16u_i64(ret, arg);
2155 tcg_gen_shri_i64(ret, ret, ofs);
2156 return;
2157 }
2158 break;
2159 case 8:
2160 if (TCG_TARGET_HAS_ext8u_i64) {
2161 tcg_gen_ext8u_i64(ret, arg);
2162 tcg_gen_shri_i64(ret, ret, ofs);
2163 return;
2164 }
2165 break;
2166 }
2167
2168 /* ??? Ideally we'd know what values are available for immediate AND.
2169 Assume that 8 bits are available, plus the special cases of 16 and 32,
2170 so that we get ext8u, ext16u, and ext32u. */
2171 switch (len) {
2172 case 1 ... 8: case 16: case 32:
2173 do_shift_and:
2174 tcg_gen_shri_i64(ret, arg, ofs);
2175 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2176 break;
2177 default:
2178 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2179 tcg_gen_shri_i64(ret, ret, 64 - len);
2180 break;
2181 }
2182}
2183
2184void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2185 unsigned int ofs, unsigned int len)
2186{
2187 tcg_debug_assert(ofs < 64);
2188 tcg_debug_assert(len > 0);
2189 tcg_debug_assert(len <= 64);
2190 tcg_debug_assert(ofs + len <= 64);
2191
2192 /* Canonicalize certain special cases, even if sextract is supported. */
2193 if (ofs + len == 64) {
2194 tcg_gen_sari_i64(ret, arg, 64 - len);
2195 return;
2196 }
2197 if (ofs == 0) {
2198 switch (len) {
2199 case 32:
2200 tcg_gen_ext32s_i64(ret, arg);
2201 return;
2202 case 16:
2203 tcg_gen_ext16s_i64(ret, arg);
2204 return;
2205 case 8:
2206 tcg_gen_ext8s_i64(ret, arg);
2207 return;
2208 }
2209 }
2210
2211 if (TCG_TARGET_REG_BITS == 32) {
2212 /* Look for a 32-bit extract within one of the two words. */
2213 if (ofs >= 32) {
2214 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2215 } else if (ofs + len <= 32) {
2216 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2217 } else if (ofs == 0) {
2218 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2219 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2220 return;
2221 } else if (len > 32) {
2222 TCGv_i32 t = tcg_temp_new_i32();
2223 /* Extract the bits for the high word normally. */
2224 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2225 /* Shift the field down for the low part. */
2226 tcg_gen_shri_i64(ret, arg, ofs);
2227 /* Overwrite the shift into the high part. */
2228 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2229 tcg_temp_free_i32(t);
2230 return;
2231 } else {
2232 /* Shift the field down for the low part, such that the
2233 field sits at the MSB. */
2234 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2235 /* Shift the field down from the MSB, sign extending. */
2236 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2237 }
2238 /* Sign-extend the field from 32 bits. */
2239 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2240 return;
2241 }
2242
2243 if (TCG_TARGET_HAS_sextract_i64
2244 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2245 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2246 return;
2247 }
2248
2249 /* Assume that sign-extension, if available, is cheaper than a shift. */
2250 switch (ofs + len) {
2251 case 32:
2252 if (TCG_TARGET_HAS_ext32s_i64) {
2253 tcg_gen_ext32s_i64(ret, arg);
2254 tcg_gen_sari_i64(ret, ret, ofs);
2255 return;
2256 }
2257 break;
2258 case 16:
2259 if (TCG_TARGET_HAS_ext16s_i64) {
2260 tcg_gen_ext16s_i64(ret, arg);
2261 tcg_gen_sari_i64(ret, ret, ofs);
2262 return;
2263 }
2264 break;
2265 case 8:
2266 if (TCG_TARGET_HAS_ext8s_i64) {
2267 tcg_gen_ext8s_i64(ret, arg);
2268 tcg_gen_sari_i64(ret, ret, ofs);
2269 return;
2270 }
2271 break;
2272 }
2273 switch (len) {
2274 case 32:
2275 if (TCG_TARGET_HAS_ext32s_i64) {
2276 tcg_gen_shri_i64(ret, arg, ofs);
2277 tcg_gen_ext32s_i64(ret, ret);
2278 return;
2279 }
2280 break;
2281 case 16:
2282 if (TCG_TARGET_HAS_ext16s_i64) {
2283 tcg_gen_shri_i64(ret, arg, ofs);
2284 tcg_gen_ext16s_i64(ret, ret);
2285 return;
2286 }
2287 break;
2288 case 8:
2289 if (TCG_TARGET_HAS_ext8s_i64) {
2290 tcg_gen_shri_i64(ret, arg, ofs);
2291 tcg_gen_ext8s_i64(ret, ret);
2292 return;
2293 }
2294 break;
2295 }
2296 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2297 tcg_gen_sari_i64(ret, ret, 64 - len);
2298}
2299
951c6300
RH		 2300void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2301 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2302{
37ed3bf1
RH		 2303 if (cond == TCG_COND_ALWAYS) {
2304 tcg_gen_mov_i64(ret, v1);
2305 } else if (cond == TCG_COND_NEVER) {
2306 tcg_gen_mov_i64(ret, v2);
2307 } else if (TCG_TARGET_REG_BITS == 32) {
3a13c3f3
RH		 2308 TCGv_i32 t0 = tcg_temp_new_i32();
2309 TCGv_i32 t1 = tcg_temp_new_i32();
2310 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2311 TCGV_LOW(c1), TCGV_HIGH(c1),
2312 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2313
2314 if (TCG_TARGET_HAS_movcond_i32) {
2315 tcg_gen_movi_i32(t1, 0);
2316 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2317 TCGV_LOW(v1), TCGV_LOW(v2));
2318 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2319 TCGV_HIGH(v1), TCGV_HIGH(v2));
2320 } else {
2321 tcg_gen_neg_i32(t0, t0);
951c6300 2322
3a13c3f3
RH		 2323 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2324 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2325 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
951c6300 2326
3a13c3f3
RH		 2327 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2328 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2329 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2330 }
2331 tcg_temp_free_i32(t0);
2332 tcg_temp_free_i32(t1);
2333 } else if (TCG_TARGET_HAS_movcond_i64) {
951c6300
RH		 2334 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2335 } else {
2336 TCGv_i64 t0 = tcg_temp_new_i64();
2337 TCGv_i64 t1 = tcg_temp_new_i64();
2338 tcg_gen_setcond_i64(cond, t0, c1, c2);
2339 tcg_gen_neg_i64(t0, t0);
2340 tcg_gen_and_i64(t1, v1, t0);
2341 tcg_gen_andc_i64(ret, v2, t0);
2342 tcg_gen_or_i64(ret, ret, t1);
2343 tcg_temp_free_i64(t0);
2344 tcg_temp_free_i64(t1);
2345 }
951c6300
RH		 2346}
2347
2348void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2349 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2350{
2351 if (TCG_TARGET_HAS_add2_i64) {
2352 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2353 } else {
2354 TCGv_i64 t0 = tcg_temp_new_i64();
2355 TCGv_i64 t1 = tcg_temp_new_i64();
2356 tcg_gen_add_i64(t0, al, bl);
2357 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2358 tcg_gen_add_i64(rh, ah, bh);
2359 tcg_gen_add_i64(rh, rh, t1);
2360 tcg_gen_mov_i64(rl, t0);
2361 tcg_temp_free_i64(t0);
2362 tcg_temp_free_i64(t1);
2363 }
2364}
2365
2366void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2367 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2368{
2369 if (TCG_TARGET_HAS_sub2_i64) {
2370 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2371 } else {
2372 TCGv_i64 t0 = tcg_temp_new_i64();
2373 TCGv_i64 t1 = tcg_temp_new_i64();
2374 tcg_gen_sub_i64(t0, al, bl);
2375 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2376 tcg_gen_sub_i64(rh, ah, bh);
2377 tcg_gen_sub_i64(rh, rh, t1);
2378 tcg_gen_mov_i64(rl, t0);
2379 tcg_temp_free_i64(t0);
2380 tcg_temp_free_i64(t1);
2381 }
2382}
2383
2384void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2385{
2386 if (TCG_TARGET_HAS_mulu2_i64) {
2387 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2388 } else if (TCG_TARGET_HAS_muluh_i64) {
2389 TCGv_i64 t = tcg_temp_new_i64();
2390 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2391 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2392 tcg_gen_mov_i64(rl, t);
2393 tcg_temp_free_i64(t);
2394 } else {
2395 TCGv_i64 t0 = tcg_temp_new_i64();
2396 tcg_gen_mul_i64(t0, arg1, arg2);
2397 gen_helper_muluh_i64(rh, arg1, arg2);
2398 tcg_gen_mov_i64(rl, t0);
2399 tcg_temp_free_i64(t0);
2400 }
2401}
2402
2403void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2404{
2405 if (TCG_TARGET_HAS_muls2_i64) {
2406 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2407 } else if (TCG_TARGET_HAS_mulsh_i64) {
2408 TCGv_i64 t = tcg_temp_new_i64();
2409 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2410 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2411 tcg_gen_mov_i64(rl, t);
2412 tcg_temp_free_i64(t);
2413 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2414 TCGv_i64 t0 = tcg_temp_new_i64();
2415 TCGv_i64 t1 = tcg_temp_new_i64();
2416 TCGv_i64 t2 = tcg_temp_new_i64();
2417 TCGv_i64 t3 = tcg_temp_new_i64();
2418 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2419 /* Adjust for negative inputs. */
2420 tcg_gen_sari_i64(t2, arg1, 63);
2421 tcg_gen_sari_i64(t3, arg2, 63);
2422 tcg_gen_and_i64(t2, t2, arg2);
2423 tcg_gen_and_i64(t3, t3, arg1);
2424 tcg_gen_sub_i64(rh, t1, t2);
2425 tcg_gen_sub_i64(rh, rh, t3);
2426 tcg_gen_mov_i64(rl, t0);
2427 tcg_temp_free_i64(t0);
2428 tcg_temp_free_i64(t1);
2429 tcg_temp_free_i64(t2);
2430 tcg_temp_free_i64(t3);
2431 } else {
2432 TCGv_i64 t0 = tcg_temp_new_i64();
2433 tcg_gen_mul_i64(t0, arg1, arg2);
2434 gen_helper_mulsh_i64(rh, arg1, arg2);
2435 tcg_gen_mov_i64(rl, t0);
2436 tcg_temp_free_i64(t0);
2437 }
2438}
2439
5087abfb
RH		 2440void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2441{
2442 TCGv_i64 t0 = tcg_temp_new_i64();
2443 TCGv_i64 t1 = tcg_temp_new_i64();
2444 TCGv_i64 t2 = tcg_temp_new_i64();
2445 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2446 /* Adjust for negative input for the signed arg1. */
2447 tcg_gen_sari_i64(t2, arg1, 63);
2448 tcg_gen_and_i64(t2, t2, arg2);
2449 tcg_gen_sub_i64(rh, t1, t2);
2450 tcg_gen_mov_i64(rl, t0);
2451 tcg_temp_free_i64(t0);
2452 tcg_temp_free_i64(t1);
2453 tcg_temp_free_i64(t2);
2454}
2455
b87fb8cd
RH		 2456void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2457{
2458 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2459}
2460
2461void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2462{
2463 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2464}
2465
2466void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2467{
2468 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2469}
2470
2471void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2472{
2473 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2474}
2475
951c6300
RH		 2476/* Size changing operations. */
2477
609ad705 2478void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
951c6300 2479{
3a13c3f3 2480 if (TCG_TARGET_REG_BITS == 32) {
609ad705
RH		 2481 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2482 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
b7e8b17a 2483 tcg_gen_op2(INDEX_op_extrl_i64_i32,
ae8b75dc 2484 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
609ad705 2485 } else {
dc41aa7d 2486 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
609ad705
RH		 2487 }
2488}
2489
2490void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2491{
2492 if (TCG_TARGET_REG_BITS == 32) {
2493 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2494 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
b7e8b17a 2495 tcg_gen_op2(INDEX_op_extrh_i64_i32,
ae8b75dc 2496 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
951c6300
RH		 2497 } else {
2498 TCGv_i64 t = tcg_temp_new_i64();
609ad705 2499 tcg_gen_shri_i64(t, arg, 32);
dc41aa7d 2500 tcg_gen_mov_i32(ret, (TCGv_i32)t);
951c6300
RH		 2501 tcg_temp_free_i64(t);
2502 }
951c6300
RH		 2503}
2504
2505void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2506{
3a13c3f3
RH		 2507 if (TCG_TARGET_REG_BITS == 32) {
2508 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2509 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2510 } else {
b7e8b17a 2511 tcg_gen_op2(INDEX_op_extu_i32_i64,
ae8b75dc 2512 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2513 }
951c6300
RH		 2514}
2515
2516void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2517{
3a13c3f3
RH		 2518 if (TCG_TARGET_REG_BITS == 32) {
2519 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2520 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2521 } else {
b7e8b17a 2522 tcg_gen_op2(INDEX_op_ext_i32_i64,
ae8b75dc 2523 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2524 }
951c6300
RH		 2525}
2526
2527void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2528{
3a13c3f3
RH		 2529 TCGv_i64 tmp;
2530
2531 if (TCG_TARGET_REG_BITS == 32) {
2532 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2533 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2534 return;
2535 }
2536
2537 tmp = tcg_temp_new_i64();
951c6300
RH		 2538 /* These extensions are only needed for type correctness.
2539 We may be able to do better given target specific information. */
2540 tcg_gen_extu_i32_i64(tmp, high);
2541 tcg_gen_extu_i32_i64(dest, low);
2542 /* If deposit is available, use it. Otherwise use the extra
2543 knowledge that we have of the zero-extensions above. */
2544 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2545 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2546 } else {
2547 tcg_gen_shli_i64(tmp, tmp, 32);
2548 tcg_gen_or_i64(dest, dest, tmp);
2549 }
2550 tcg_temp_free_i64(tmp);
951c6300
RH		 2551}
2552
2553void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2554{
3a13c3f3
RH		 2555 if (TCG_TARGET_REG_BITS == 32) {
2556 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2557 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2558 } else {
609ad705
RH		 2559 tcg_gen_extrl_i64_i32(lo, arg);
2560 tcg_gen_extrh_i64_i32(hi, arg);
3a13c3f3 2561 }
951c6300
RH		 2562}
2563
2564void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2565{
2566 tcg_gen_ext32u_i64(lo, arg);
2567 tcg_gen_shri_i64(hi, arg, 32);
2568}
2569
2570/* QEMU specific operations. */
2571
07ea28b4
RH		 2572void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
2573{
2574 uintptr_t val = (uintptr_t)tb + idx;
2575
2576 if (tb == NULL) {
2577 tcg_debug_assert(idx == 0);
2578 } else if (idx <= TB_EXIT_IDXMAX) {
2579#ifdef CONFIG_DEBUG_TCG
2580 /* This is an exit following a goto_tb. Verify that we have
2581 seen this numbered exit before, via tcg_gen_goto_tb. */
2582 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2583#endif
d7f425fd
RH		 2584 /* When not chaining, exit without indicating a link. */
2585 if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2586 val = 0;
2587 }
07ea28b4
RH		 2588 } else {
2589 /* This is an exit via the exitreq label. */
2590 tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2591 }
2592
2593 tcg_gen_op1i(INDEX_op_exit_tb, val);
2594}
2595
951c6300
RH		 2596void tcg_gen_goto_tb(unsigned idx)
2597{
2598 /* We only support two chained exits. */
07ea28b4 2599 tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
951c6300
RH		 2600#ifdef CONFIG_DEBUG_TCG
2601 /* Verify that we havn't seen this numbered exit before. */
b1311c4a
EC		 2602 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2603 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
951c6300 2604#endif
d7f425fd
RH		 2605 /* When not chaining, we simply fall through to the "fallback" exit. */
2606 if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2607 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2608 }
951c6300
RH		 2609}
2610
7f11636d 2611void tcg_gen_lookup_and_goto_ptr(void)
cedbcb01
EC		 2612{
2613 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2614 TCGv_ptr ptr = tcg_temp_new_ptr();
1c2adb95 2615 gen_helper_lookup_tb_ptr(ptr, cpu_env);
ae8b75dc 2616 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
cedbcb01
EC		 2617 tcg_temp_free_ptr(ptr);
2618 } else {
07ea28b4 2619 tcg_gen_exit_tb(NULL, 0);
cedbcb01
EC		 2620 }
2621}
2622
951c6300
RH		 2623static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2624{
1f00b27f
SS		 2625 /* Trigger the asserts within as early as possible. */
2626 (void)get_alignment_bits(op);
2627
951c6300
RH		 2628 switch (op & MO_SIZE) {
2629 case MO_8:
2630 op &= ~MO_BSWAP;
2631 break;
2632 case MO_16:
2633 break;
2634 case MO_32:
2635 if (!is64) {
2636 op &= ~MO_SIGN;
2637 }
2638 break;
2639 case MO_64:
2640 if (!is64) {
2641 tcg_abort();
2642 }
2643 break;
2644 }
2645 if (st) {
2646 op &= ~MO_SIGN;
2647 }
2648 return op;
2649}
2650
c45cb8bb
RH		 2651static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2652 TCGMemOp memop, TCGArg idx)
951c6300 2653{
59227d5d 2654 TCGMemOpIdx oi = make_memop_idx(memop, idx);
c45cb8bb 2655#if TARGET_LONG_BITS == 32
59227d5d 2656 tcg_gen_op3i_i32(opc, val, addr, oi);
c45cb8bb 2657#else
3a13c3f3 2658 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2659 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
3a13c3f3 2660 } else {
ae8b75dc 2661 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
3a13c3f3 2662 }
c45cb8bb 2663#endif
951c6300
RH		 2664}
2665
c45cb8bb
RH		 2666static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2667 TCGMemOp memop, TCGArg idx)
2668{
59227d5d 2669 TCGMemOpIdx oi = make_memop_idx(memop, idx);
951c6300 2670#if TARGET_LONG_BITS == 32
c45cb8bb 2671 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2672 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
c45cb8bb 2673 } else {
ae8b75dc 2674 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
c45cb8bb 2675 }
951c6300 2676#else
c45cb8bb 2677 if (TCG_TARGET_REG_BITS == 32) {
59227d5d
RH		 2678 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2679 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
c45cb8bb 2680 } else {
59227d5d 2681 tcg_gen_op3i_i64(opc, val, addr, oi);
c45cb8bb 2682 }
951c6300 2683#endif
c45cb8bb 2684}
951c6300 2685
b32dc337
PK		 2686static void tcg_gen_req_mo(TCGBar type)
2687{
2688#ifdef TCG_GUEST_DEFAULT_MO
2689 type &= TCG_GUEST_DEFAULT_MO;
2690#endif
2691 type &= ~TCG_TARGET_DEFAULT_MO;
2692 if (type) {
2693 tcg_gen_mb(type | TCG_BAR_SC);
2694 }
2695}
2696
951c6300
RH		 2697void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2698{
e1dcf352
RH		 2699 TCGMemOp orig_memop;
2700
b32dc337 2701 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
951c6300 2702 memop = tcg_canonicalize_memop(memop, 0, 0);
1c2adb95 2703 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2704 addr, trace_mem_get_info(memop, 0));
e1dcf352
RH		 2705
2706 orig_memop = memop;
2707 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2708 memop &= ~MO_BSWAP;
2709 /* The bswap primitive requires zero-extended input. */
2710 if ((memop & MO_SSIZE) == MO_SW) {
2711 memop &= ~MO_SIGN;
2712 }
2713 }
2714
c45cb8bb 2715 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
e1dcf352
RH		 2716
2717 if ((orig_memop ^ memop) & MO_BSWAP) {
2718 switch (orig_memop & MO_SIZE) {
2719 case MO_16:
2720 tcg_gen_bswap16_i32(val, val);
2721 if (orig_memop & MO_SIGN) {
2722 tcg_gen_ext16s_i32(val, val);
2723 }
2724 break;
2725 case MO_32:
2726 tcg_gen_bswap32_i32(val, val);
2727 break;
2728 default:
2729 g_assert_not_reached();
2730 }
2731 }
951c6300
RH		 2732}
2733
2734void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2735{
e1dcf352
RH		 2736 TCGv_i32 swap = NULL;
2737
b32dc337 2738 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
951c6300 2739 memop = tcg_canonicalize_memop(memop, 0, 1);
1c2adb95 2740 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2741 addr, trace_mem_get_info(memop, 1));
e1dcf352
RH		 2742
2743 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2744 swap = tcg_temp_new_i32();
2745 switch (memop & MO_SIZE) {
2746 case MO_16:
2747 tcg_gen_ext16u_i32(swap, val);
2748 tcg_gen_bswap16_i32(swap, swap);
2749 break;
2750 case MO_32:
2751 tcg_gen_bswap32_i32(swap, val);
2752 break;
2753 default:
2754 g_assert_not_reached();
2755 }
2756 val = swap;
2757 memop &= ~MO_BSWAP;
2758 }
2759
c45cb8bb 2760 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
e1dcf352
RH		 2761
2762 if (swap) {
2763 tcg_temp_free_i32(swap);
2764 }
951c6300
RH		 2765}
2766
2767void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2768{
e1dcf352
RH		 2769 TCGMemOp orig_memop;
2770
3a13c3f3 2771 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 2772 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2773 if (memop & MO_SIGN) {
2774 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2775 } else {
2776 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2777 }
2778 return;
2779 }
951c6300 2780
e1dcf352 2781 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
c45cb8bb 2782 memop = tcg_canonicalize_memop(memop, 1, 0);
1c2adb95 2783 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2784 addr, trace_mem_get_info(memop, 0));
e1dcf352
RH		 2785
2786 orig_memop = memop;
2787 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2788 memop &= ~MO_BSWAP;
2789 /* The bswap primitive requires zero-extended input. */
2790 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
2791 memop &= ~MO_SIGN;
2792 }
2793 }
2794
c45cb8bb 2795 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
e1dcf352
RH		 2796
2797 if ((orig_memop ^ memop) & MO_BSWAP) {
2798 switch (orig_memop & MO_SIZE) {
2799 case MO_16:
2800 tcg_gen_bswap16_i64(val, val);
2801 if (orig_memop & MO_SIGN) {
2802 tcg_gen_ext16s_i64(val, val);
2803 }
2804 break;
2805 case MO_32:
2806 tcg_gen_bswap32_i64(val, val);
2807 if (orig_memop & MO_SIGN) {
2808 tcg_gen_ext32s_i64(val, val);
2809 }
2810 break;
2811 case MO_64:
2812 tcg_gen_bswap64_i64(val, val);
2813 break;
2814 default:
2815 g_assert_not_reached();
2816 }
2817 }
951c6300
RH		 2818}
2819
2820void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2821{
e1dcf352
RH		 2822 TCGv_i64 swap = NULL;
2823
3a13c3f3 2824 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 2825 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2826 return;
2827 }
951c6300 2828
e1dcf352 2829 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
c45cb8bb 2830 memop = tcg_canonicalize_memop(memop, 1, 1);
1c2adb95 2831 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2832 addr, trace_mem_get_info(memop, 1));
e1dcf352
RH		 2833
2834 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2835 swap = tcg_temp_new_i64();
2836 switch (memop & MO_SIZE) {
2837 case MO_16:
2838 tcg_gen_ext16u_i64(swap, val);
2839 tcg_gen_bswap16_i64(swap, swap);
2840 break;
2841 case MO_32:
2842 tcg_gen_ext32u_i64(swap, val);
2843 tcg_gen_bswap32_i64(swap, swap);
2844 break;
2845 case MO_64:
2846 tcg_gen_bswap64_i64(swap, val);
2847 break;
2848 default:
2849 g_assert_not_reached();
2850 }
2851 val = swap;
2852 memop &= ~MO_BSWAP;
2853 }
2854
c45cb8bb 2855 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
e1dcf352
RH		 2856
2857 if (swap) {
2858 tcg_temp_free_i64(swap);
2859 }
951c6300 2860}
c482cb11
RH		 2861
2862static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2863{
2864 switch (opc & MO_SSIZE) {
2865 case MO_SB:
2866 tcg_gen_ext8s_i32(ret, val);
2867 break;
2868 case MO_UB:
2869 tcg_gen_ext8u_i32(ret, val);
2870 break;
2871 case MO_SW:
2872 tcg_gen_ext16s_i32(ret, val);
2873 break;
2874 case MO_UW:
2875 tcg_gen_ext16u_i32(ret, val);
2876 break;
2877 default:
2878 tcg_gen_mov_i32(ret, val);
2879 break;
2880 }
2881}
2882
2883static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2884{
2885 switch (opc & MO_SSIZE) {
2886 case MO_SB:
2887 tcg_gen_ext8s_i64(ret, val);
2888 break;
2889 case MO_UB:
2890 tcg_gen_ext8u_i64(ret, val);
2891 break;
2892 case MO_SW:
2893 tcg_gen_ext16s_i64(ret, val);
2894 break;
2895 case MO_UW:
2896 tcg_gen_ext16u_i64(ret, val);
2897 break;
2898 case MO_SL:
2899 tcg_gen_ext32s_i64(ret, val);
2900 break;
2901 case MO_UL:
2902 tcg_gen_ext32u_i64(ret, val);
2903 break;
2904 default:
2905 tcg_gen_mov_i64(ret, val);
2906 break;
2907 }
2908}
2909
2910#ifdef CONFIG_SOFTMMU
2911typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2912 TCGv_i32, TCGv_i32, TCGv_i32);
2913typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2914 TCGv_i64, TCGv_i64, TCGv_i32);
2915typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2916 TCGv_i32, TCGv_i32);
2917typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2918 TCGv_i64, TCGv_i32);
2919#else
2920typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2921typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2922typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2923typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2924#endif
2925
df79b996
RH		 2926#ifdef CONFIG_ATOMIC64
2927# define WITH_ATOMIC64(X) X,
2928#else
2929# define WITH_ATOMIC64(X)
2930#endif
2931
c482cb11
RH		 2932static void * const table_cmpxchg[16] = {
2933 [MO_8] = gen_helper_atomic_cmpxchgb,
2934 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2935 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2936 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2937 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
df79b996
RH		 2938 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2939 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
c482cb11
RH		 2940};
2941
2942void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2943 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2944{
2945 memop = tcg_canonicalize_memop(memop, 0, 0);
2946
b1311c4a 2947 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
c482cb11
RH		 2948 TCGv_i32 t1 = tcg_temp_new_i32();
2949 TCGv_i32 t2 = tcg_temp_new_i32();
2950
2951 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2952
2953 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2954 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2955 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2956 tcg_temp_free_i32(t2);
2957
2958 if (memop & MO_SIGN) {
2959 tcg_gen_ext_i32(retv, t1, memop);
2960 } else {
2961 tcg_gen_mov_i32(retv, t1);
2962 }
2963 tcg_temp_free_i32(t1);
2964 } else {
2965 gen_atomic_cx_i32 gen;
2966
2967 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2968 tcg_debug_assert(gen != NULL);
2969
2970#ifdef CONFIG_SOFTMMU
2971 {
2972 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
1c2adb95 2973 gen(retv, cpu_env, addr, cmpv, newv, oi);
c482cb11
RH		 2974 tcg_temp_free_i32(oi);
2975 }
2976#else
1c2adb95 2977 gen(retv, cpu_env, addr, cmpv, newv);
c482cb11
RH		 2978#endif
2979
2980 if (memop & MO_SIGN) {
2981 tcg_gen_ext_i32(retv, retv, memop);
2982 }
2983 }
2984}
2985
2986void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2987 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2988{
2989 memop = tcg_canonicalize_memop(memop, 1, 0);
2990
b1311c4a 2991 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
c482cb11
RH		 2992 TCGv_i64 t1 = tcg_temp_new_i64();
2993 TCGv_i64 t2 = tcg_temp_new_i64();
2994
2995 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2996
2997 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2998 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2999 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3000 tcg_temp_free_i64(t2);
3001
3002 if (memop & MO_SIGN) {
3003 tcg_gen_ext_i64(retv, t1, memop);
3004 } else {
3005 tcg_gen_mov_i64(retv, t1);
3006 }
3007 tcg_temp_free_i64(t1);
3008 } else if ((memop & MO_SIZE) == MO_64) {
df79b996 3009#ifdef CONFIG_ATOMIC64
c482cb11
RH		 3010 gen_atomic_cx_i64 gen;
3011
3012 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3013 tcg_debug_assert(gen != NULL);
3014
3015#ifdef CONFIG_SOFTMMU
3016 {
3017 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
1c2adb95 3018 gen(retv, cpu_env, addr, cmpv, newv, oi);
c482cb11
RH		 3019 tcg_temp_free_i32(oi);
3020 }
3021#else
1c2adb95 3022 gen(retv, cpu_env, addr, cmpv, newv);
c482cb11 3023#endif
df79b996 3024#else
1c2adb95 3025 gen_helper_exit_atomic(cpu_env);
79b1af90
RH		 3026 /* Produce a result, so that we have a well-formed opcode stream
3027 with respect to uses of the result in the (dead) code following. */
3028 tcg_gen_movi_i64(retv, 0);
df79b996 3029#endif /* CONFIG_ATOMIC64 */
c482cb11
RH		 3030 } else {
3031 TCGv_i32 c32 = tcg_temp_new_i32();
3032 TCGv_i32 n32 = tcg_temp_new_i32();
3033 TCGv_i32 r32 = tcg_temp_new_i32();
3034
3035 tcg_gen_extrl_i64_i32(c32, cmpv);
3036 tcg_gen_extrl_i64_i32(n32, newv);
3037 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3038 tcg_temp_free_i32(c32);
3039 tcg_temp_free_i32(n32);
3040
3041 tcg_gen_extu_i32_i64(retv, r32);
3042 tcg_temp_free_i32(r32);
3043
3044 if (memop & MO_SIGN) {
3045 tcg_gen_ext_i64(retv, retv, memop);
3046 }
3047 }
3048}
3049
3050static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3051 TCGArg idx, TCGMemOp memop, bool new_val,
3052 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3053{
3054 TCGv_i32 t1 = tcg_temp_new_i32();
3055 TCGv_i32 t2 = tcg_temp_new_i32();
3056
3057 memop = tcg_canonicalize_memop(memop, 0, 0);
3058
3059 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3060 gen(t2, t1, val);
3061 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3062
3063 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3064 tcg_temp_free_i32(t1);
3065 tcg_temp_free_i32(t2);
3066}
3067
3068static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3069 TCGArg idx, TCGMemOp memop, void * const table[])
3070{
3071 gen_atomic_op_i32 gen;
3072
3073 memop = tcg_canonicalize_memop(memop, 0, 0);
3074
3075 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3076 tcg_debug_assert(gen != NULL);
3077
3078#ifdef CONFIG_SOFTMMU
3079 {
3080 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
1c2adb95 3081 gen(ret, cpu_env, addr, val, oi);
c482cb11
RH		 3082 tcg_temp_free_i32(oi);
3083 }
3084#else
1c2adb95 3085 gen(ret, cpu_env, addr, val);
c482cb11
RH		 3086#endif
3087
3088 if (memop & MO_SIGN) {
3089 tcg_gen_ext_i32(ret, ret, memop);
3090 }
3091}
3092
3093static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3094 TCGArg idx, TCGMemOp memop, bool new_val,
3095 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3096{
3097 TCGv_i64 t1 = tcg_temp_new_i64();
3098 TCGv_i64 t2 = tcg_temp_new_i64();
3099
3100 memop = tcg_canonicalize_memop(memop, 1, 0);
3101
3102 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3103 gen(t2, t1, val);
3104 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3105
3106 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3107 tcg_temp_free_i64(t1);
3108 tcg_temp_free_i64(t2);
3109}
3110
3111static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3112 TCGArg idx, TCGMemOp memop, void * const table[])
3113{
3114 memop = tcg_canonicalize_memop(memop, 1, 0);
3115
3116 if ((memop & MO_SIZE) == MO_64) {
df79b996 3117#ifdef CONFIG_ATOMIC64
c482cb11
RH		 3118 gen_atomic_op_i64 gen;
3119
3120 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3121 tcg_debug_assert(gen != NULL);
3122
3123#ifdef CONFIG_SOFTMMU
3124 {
3125 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
1c2adb95 3126 gen(ret, cpu_env, addr, val, oi);
c482cb11
RH		 3127 tcg_temp_free_i32(oi);
3128 }
3129#else
1c2adb95 3130 gen(ret, cpu_env, addr, val);
c482cb11 3131#endif
df79b996 3132#else
1c2adb95 3133 gen_helper_exit_atomic(cpu_env);
79b1af90
RH		 3134 /* Produce a result, so that we have a well-formed opcode stream
3135 with respect to uses of the result in the (dead) code following. */
3136 tcg_gen_movi_i64(ret, 0);
df79b996 3137#endif /* CONFIG_ATOMIC64 */
c482cb11
RH		 3138 } else {
3139 TCGv_i32 v32 = tcg_temp_new_i32();
3140 TCGv_i32 r32 = tcg_temp_new_i32();
3141
3142 tcg_gen_extrl_i64_i32(v32, val);
3143 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3144 tcg_temp_free_i32(v32);
3145
3146 tcg_gen_extu_i32_i64(ret, r32);
3147 tcg_temp_free_i32(r32);
3148
3149 if (memop & MO_SIGN) {
3150 tcg_gen_ext_i64(ret, ret, memop);
3151 }
3152 }
3153}
3154
3155#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3156static void * const table_##NAME[16] = { \
3157 [MO_8] = gen_helper_atomic_##NAME##b, \
3158 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3159 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3160 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3161 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
df79b996
RH		 3162 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3163 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
c482cb11
RH		 3164}; \
3165void tcg_gen_atomic_##NAME##_i32 \
3166 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
3167{ \
b1311c4a 3168 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
c482cb11
RH		 3169 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3170 } else { \
3171 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3172 tcg_gen_##OP##_i32); \
3173 } \
3174} \
3175void tcg_gen_atomic_##NAME##_i64 \
3176 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
3177{ \
b1311c4a 3178 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
c482cb11
RH		 3179 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3180 } else { \
3181 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3182 tcg_gen_##OP##_i64); \
3183 } \
3184}
3185
3186GEN_ATOMIC_HELPER(fetch_add, add, 0)
3187GEN_ATOMIC_HELPER(fetch_and, and, 0)
3188GEN_ATOMIC_HELPER(fetch_or, or, 0)
3189GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
5507c2bf
RH		 3190GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3191GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3192GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3193GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
c482cb11
RH		 3194
3195GEN_ATOMIC_HELPER(add_fetch, add, 1)
3196GEN_ATOMIC_HELPER(and_fetch, and, 1)
3197GEN_ATOMIC_HELPER(or_fetch, or, 1)
3198GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
5507c2bf
RH		 3199GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3200GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3201GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3202GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
c482cb11
RH		 3203
3204static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3205{
3206 tcg_gen_mov_i32(r, b);
3207}
3208
3209static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3210{
3211 tcg_gen_mov_i64(r, b);
3212}
3213
3214GEN_ATOMIC_HELPER(xchg, mov2, 0)
3215
3216#undef GEN_ATOMIC_HELPER
QEMU mirror