]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg-op.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg/riscv: Add the immediate encoders
[mirror_qemu.git] / tcg / tcg-op.c
CommitLineData
951c6300
RH		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
757e725b 25#include "qemu/osdep.h"
33c11879
PB		 26#include "qemu-common.h"
27#include "cpu.h"
63c91552 28#include "exec/exec-all.h"
951c6300
RH		 29#include "tcg.h"
30#include "tcg-op.h"
b32dc337 31#include "tcg-mo.h"
dcdaadb6
LV		 32#include "trace-tcg.h"
33#include "trace/mem.h"
951c6300 34
3a13c3f3
RH		 35/* Reduce the number of ifdefs below. This assumes that all uses of
36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
37 the compiler can eliminate. */
38#if TCG_TARGET_REG_BITS == 64
39extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
40extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
41#define TCGV_LOW TCGV_LOW_link_error
42#define TCGV_HIGH TCGV_HIGH_link_error
43#endif
951c6300 44
b7e8b17a 45void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
951c6300 46{
b7e8b17a 47 TCGOp *op = tcg_emit_op(opc);
75e8b9b7 48 op->args[0] = a1;
951c6300
RH		 49}
50
b7e8b17a 51void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
951c6300 52{
b7e8b17a 53 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 54 op->args[0] = a1;
55 op->args[1] = a2;
951c6300
RH		 56}
57
b7e8b17a 58void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
951c6300 59{
b7e8b17a 60 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 61 op->args[0] = a1;
62 op->args[1] = a2;
63 op->args[2] = a3;
951c6300
RH		 64}
65
b7e8b17a 66void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
951c6300 67{
b7e8b17a 68 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 69 op->args[0] = a1;
70 op->args[1] = a2;
71 op->args[2] = a3;
72 op->args[3] = a4;
951c6300
RH		 73}
74
b7e8b17a
RH		 75void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
76 TCGArg a4, TCGArg a5)
951c6300 77{
b7e8b17a 78 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 79 op->args[0] = a1;
80 op->args[1] = a2;
81 op->args[2] = a3;
82 op->args[3] = a4;
83 op->args[4] = a5;
951c6300
RH		 84}
85
b7e8b17a
RH		 86void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
87 TCGArg a4, TCGArg a5, TCGArg a6)
951c6300 88{
b7e8b17a 89 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 90 op->args[0] = a1;
91 op->args[1] = a2;
92 op->args[2] = a3;
93 op->args[3] = a4;
94 op->args[4] = a5;
95 op->args[5] = a6;
951c6300
RH		 96}
97
f65e19bc
PK		 98void tcg_gen_mb(TCGBar mb_type)
99{
b1311c4a 100 if (tcg_ctx->tb_cflags & CF_PARALLEL) {
b7e8b17a 101 tcg_gen_op1(INDEX_op_mb, mb_type);
f65e19bc
PK		 102 }
103}
104
951c6300
RH		 105/* 32 bit ops */
106
107void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
108{
109 /* some cases can be optimized here */
110 if (arg2 == 0) {
111 tcg_gen_mov_i32(ret, arg1);
112 } else {
113 TCGv_i32 t0 = tcg_const_i32(arg2);
114 tcg_gen_add_i32(ret, arg1, t0);
115 tcg_temp_free_i32(t0);
116 }
117}
118
119void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
120{
121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
122 /* Don't recurse with tcg_gen_neg_i32. */
123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
124 } else {
125 TCGv_i32 t0 = tcg_const_i32(arg1);
126 tcg_gen_sub_i32(ret, t0, arg2);
127 tcg_temp_free_i32(t0);
128 }
129}
130
131void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
132{
133 /* some cases can be optimized here */
134 if (arg2 == 0) {
135 tcg_gen_mov_i32(ret, arg1);
136 } else {
137 TCGv_i32 t0 = tcg_const_i32(arg2);
138 tcg_gen_sub_i32(ret, arg1, t0);
139 tcg_temp_free_i32(t0);
140 }
141}
142
474b2e8f 143void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300
RH		 144{
145 TCGv_i32 t0;
146 /* Some cases can be optimized here. */
147 switch (arg2) {
148 case 0:
149 tcg_gen_movi_i32(ret, 0);
150 return;
474b2e8f 151 case -1:
951c6300
RH		 152 tcg_gen_mov_i32(ret, arg1);
153 return;
474b2e8f 154 case 0xff:
951c6300
RH		 155 /* Don't recurse with tcg_gen_ext8u_i32. */
156 if (TCG_TARGET_HAS_ext8u_i32) {
157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
158 return;
159 }
160 break;
474b2e8f 161 case 0xffff:
951c6300
RH		 162 if (TCG_TARGET_HAS_ext16u_i32) {
163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
164 return;
165 }
166 break;
167 }
168 t0 = tcg_const_i32(arg2);
169 tcg_gen_and_i32(ret, arg1, t0);
170 tcg_temp_free_i32(t0);
171}
172
173void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
174{
175 /* Some cases can be optimized here. */
176 if (arg2 == -1) {
177 tcg_gen_movi_i32(ret, -1);
178 } else if (arg2 == 0) {
179 tcg_gen_mov_i32(ret, arg1);
180 } else {
181 TCGv_i32 t0 = tcg_const_i32(arg2);
182 tcg_gen_or_i32(ret, arg1, t0);
183 tcg_temp_free_i32(t0);
184 }
185}
186
187void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
188{
189 /* Some cases can be optimized here. */
190 if (arg2 == 0) {
191 tcg_gen_mov_i32(ret, arg1);
192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
193 /* Don't recurse with tcg_gen_not_i32. */
194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
195 } else {
196 TCGv_i32 t0 = tcg_const_i32(arg2);
197 tcg_gen_xor_i32(ret, arg1, t0);
198 tcg_temp_free_i32(t0);
199 }
200}
201
474b2e8f 202void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 203{
474b2e8f 204 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 205 if (arg2 == 0) {
206 tcg_gen_mov_i32(ret, arg1);
207 } else {
208 TCGv_i32 t0 = tcg_const_i32(arg2);
209 tcg_gen_shl_i32(ret, arg1, t0);
210 tcg_temp_free_i32(t0);
211 }
212}
213
474b2e8f 214void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 215{
474b2e8f 216 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 217 if (arg2 == 0) {
218 tcg_gen_mov_i32(ret, arg1);
219 } else {
220 TCGv_i32 t0 = tcg_const_i32(arg2);
221 tcg_gen_shr_i32(ret, arg1, t0);
222 tcg_temp_free_i32(t0);
223 }
224}
225
474b2e8f 226void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 227{
474b2e8f 228 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 229 if (arg2 == 0) {
230 tcg_gen_mov_i32(ret, arg1);
231 } else {
232 TCGv_i32 t0 = tcg_const_i32(arg2);
233 tcg_gen_sar_i32(ret, arg1, t0);
234 tcg_temp_free_i32(t0);
235 }
236}
237
42a268c2 238void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
951c6300
RH		 239{
240 if (cond == TCG_COND_ALWAYS) {
42a268c2 241 tcg_gen_br(l);
951c6300 242 } else if (cond != TCG_COND_NEVER) {
42a268c2 243 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
951c6300
RH		 244 }
245}
246
42a268c2 247void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
951c6300 248{
37ed3bf1
RH		 249 if (cond == TCG_COND_ALWAYS) {
250 tcg_gen_br(l);
251 } else if (cond != TCG_COND_NEVER) {
252 TCGv_i32 t0 = tcg_const_i32(arg2);
253 tcg_gen_brcond_i32(cond, arg1, t0, l);
254 tcg_temp_free_i32(t0);
255 }
951c6300
RH		 256}
257
258void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
259 TCGv_i32 arg1, TCGv_i32 arg2)
260{
261 if (cond == TCG_COND_ALWAYS) {
262 tcg_gen_movi_i32(ret, 1);
263 } else if (cond == TCG_COND_NEVER) {
264 tcg_gen_movi_i32(ret, 0);
265 } else {
266 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
267 }
268}
269
270void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
271 TCGv_i32 arg1, int32_t arg2)
272{
273 TCGv_i32 t0 = tcg_const_i32(arg2);
274 tcg_gen_setcond_i32(cond, ret, arg1, t0);
275 tcg_temp_free_i32(t0);
276}
277
278void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
279{
b2e3ae94
RH		 280 if (arg2 == 0) {
281 tcg_gen_movi_i32(ret, 0);
282 } else if (is_power_of_2(arg2)) {
283 tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
284 } else {
285 TCGv_i32 t0 = tcg_const_i32(arg2);
286 tcg_gen_mul_i32(ret, arg1, t0);
287 tcg_temp_free_i32(t0);
288 }
951c6300
RH		 289}
290
291void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
292{
293 if (TCG_TARGET_HAS_div_i32) {
294 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
295 } else if (TCG_TARGET_HAS_div2_i32) {
296 TCGv_i32 t0 = tcg_temp_new_i32();
297 tcg_gen_sari_i32(t0, arg1, 31);
298 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
299 tcg_temp_free_i32(t0);
300 } else {
301 gen_helper_div_i32(ret, arg1, arg2);
302 }
303}
304
305void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
306{
307 if (TCG_TARGET_HAS_rem_i32) {
308 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
309 } else if (TCG_TARGET_HAS_div_i32) {
310 TCGv_i32 t0 = tcg_temp_new_i32();
311 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
312 tcg_gen_mul_i32(t0, t0, arg2);
313 tcg_gen_sub_i32(ret, arg1, t0);
314 tcg_temp_free_i32(t0);
315 } else if (TCG_TARGET_HAS_div2_i32) {
316 TCGv_i32 t0 = tcg_temp_new_i32();
317 tcg_gen_sari_i32(t0, arg1, 31);
318 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
319 tcg_temp_free_i32(t0);
320 } else {
321 gen_helper_rem_i32(ret, arg1, arg2);
322 }
323}
324
325void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
326{
327 if (TCG_TARGET_HAS_div_i32) {
328 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
329 } else if (TCG_TARGET_HAS_div2_i32) {
330 TCGv_i32 t0 = tcg_temp_new_i32();
331 tcg_gen_movi_i32(t0, 0);
332 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
333 tcg_temp_free_i32(t0);
334 } else {
335 gen_helper_divu_i32(ret, arg1, arg2);
336 }
337}
338
339void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
340{
341 if (TCG_TARGET_HAS_rem_i32) {
342 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
343 } else if (TCG_TARGET_HAS_div_i32) {
344 TCGv_i32 t0 = tcg_temp_new_i32();
345 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
346 tcg_gen_mul_i32(t0, t0, arg2);
347 tcg_gen_sub_i32(ret, arg1, t0);
348 tcg_temp_free_i32(t0);
349 } else if (TCG_TARGET_HAS_div2_i32) {
350 TCGv_i32 t0 = tcg_temp_new_i32();
351 tcg_gen_movi_i32(t0, 0);
352 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
353 tcg_temp_free_i32(t0);
354 } else {
355 gen_helper_remu_i32(ret, arg1, arg2);
356 }
357}
358
359void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
360{
361 if (TCG_TARGET_HAS_andc_i32) {
362 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
363 } else {
364 TCGv_i32 t0 = tcg_temp_new_i32();
365 tcg_gen_not_i32(t0, arg2);
366 tcg_gen_and_i32(ret, arg1, t0);
367 tcg_temp_free_i32(t0);
368 }
369}
370
371void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
372{
373 if (TCG_TARGET_HAS_eqv_i32) {
374 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
375 } else {
376 tcg_gen_xor_i32(ret, arg1, arg2);
377 tcg_gen_not_i32(ret, ret);
378 }
379}
380
381void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
382{
383 if (TCG_TARGET_HAS_nand_i32) {
384 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
385 } else {
386 tcg_gen_and_i32(ret, arg1, arg2);
387 tcg_gen_not_i32(ret, ret);
388 }
389}
390
391void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
392{
393 if (TCG_TARGET_HAS_nor_i32) {
394 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
395 } else {
396 tcg_gen_or_i32(ret, arg1, arg2);
397 tcg_gen_not_i32(ret, ret);
398 }
399}
400
401void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
402{
403 if (TCG_TARGET_HAS_orc_i32) {
404 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
405 } else {
406 TCGv_i32 t0 = tcg_temp_new_i32();
407 tcg_gen_not_i32(t0, arg2);
408 tcg_gen_or_i32(ret, arg1, t0);
409 tcg_temp_free_i32(t0);
410 }
411}
412
0e28d006
RH		 413void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
414{
415 if (TCG_TARGET_HAS_clz_i32) {
416 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
417 } else if (TCG_TARGET_HAS_clz_i64) {
418 TCGv_i64 t1 = tcg_temp_new_i64();
419 TCGv_i64 t2 = tcg_temp_new_i64();
420 tcg_gen_extu_i32_i64(t1, arg1);
421 tcg_gen_extu_i32_i64(t2, arg2);
422 tcg_gen_addi_i64(t2, t2, 32);
423 tcg_gen_clz_i64(t1, t1, t2);
424 tcg_gen_extrl_i64_i32(ret, t1);
425 tcg_temp_free_i64(t1);
426 tcg_temp_free_i64(t2);
427 tcg_gen_subi_i32(ret, ret, 32);
428 } else {
429 gen_helper_clz_i32(ret, arg1, arg2);
430 }
431}
432
433void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
434{
435 TCGv_i32 t = tcg_const_i32(arg2);
436 tcg_gen_clz_i32(ret, arg1, t);
437 tcg_temp_free_i32(t);
438}
439
440void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
441{
442 if (TCG_TARGET_HAS_ctz_i32) {
443 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
444 } else if (TCG_TARGET_HAS_ctz_i64) {
445 TCGv_i64 t1 = tcg_temp_new_i64();
446 TCGv_i64 t2 = tcg_temp_new_i64();
447 tcg_gen_extu_i32_i64(t1, arg1);
448 tcg_gen_extu_i32_i64(t2, arg2);
449 tcg_gen_ctz_i64(t1, t1, t2);
450 tcg_gen_extrl_i64_i32(ret, t1);
451 tcg_temp_free_i64(t1);
452 tcg_temp_free_i64(t2);
14e99210
RH		 453 } else if (TCG_TARGET_HAS_ctpop_i32
454 || TCG_TARGET_HAS_ctpop_i64
455 || TCG_TARGET_HAS_clz_i32
456 || TCG_TARGET_HAS_clz_i64) {
457 TCGv_i32 z, t = tcg_temp_new_i32();
458
459 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
460 tcg_gen_subi_i32(t, arg1, 1);
461 tcg_gen_andc_i32(t, t, arg1);
462 tcg_gen_ctpop_i32(t, t);
463 } else {
464 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
465 tcg_gen_neg_i32(t, arg1);
466 tcg_gen_and_i32(t, t, arg1);
467 tcg_gen_clzi_i32(t, t, 32);
468 tcg_gen_xori_i32(t, t, 31);
469 }
470 z = tcg_const_i32(0);
471 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
472 tcg_temp_free_i32(t);
473 tcg_temp_free_i32(z);
0e28d006
RH		 474 } else {
475 gen_helper_ctz_i32(ret, arg1, arg2);
476 }
477}
478
479void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
480{
14e99210
RH		 481 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
482 /* This equivalence has the advantage of not requiring a fixup. */
483 TCGv_i32 t = tcg_temp_new_i32();
484 tcg_gen_subi_i32(t, arg1, 1);
485 tcg_gen_andc_i32(t, t, arg1);
486 tcg_gen_ctpop_i32(ret, t);
487 tcg_temp_free_i32(t);
488 } else {
489 TCGv_i32 t = tcg_const_i32(arg2);
490 tcg_gen_ctz_i32(ret, arg1, t);
491 tcg_temp_free_i32(t);
492 }
0e28d006
RH		 493}
494
086920c2
RH		 495void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
496{
497 if (TCG_TARGET_HAS_clz_i32) {
498 TCGv_i32 t = tcg_temp_new_i32();
499 tcg_gen_sari_i32(t, arg, 31);
500 tcg_gen_xor_i32(t, t, arg);
501 tcg_gen_clzi_i32(t, t, 32);
502 tcg_gen_subi_i32(ret, t, 1);
503 tcg_temp_free_i32(t);
504 } else {
505 gen_helper_clrsb_i32(ret, arg);
506 }
507}
508
a768e4e9
RH		 509void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
510{
511 if (TCG_TARGET_HAS_ctpop_i32) {
512 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
513 } else if (TCG_TARGET_HAS_ctpop_i64) {
514 TCGv_i64 t = tcg_temp_new_i64();
515 tcg_gen_extu_i32_i64(t, arg1);
516 tcg_gen_ctpop_i64(t, t);
517 tcg_gen_extrl_i64_i32(ret, t);
518 tcg_temp_free_i64(t);
519 } else {
520 gen_helper_ctpop_i32(ret, arg1);
521 }
522}
523
951c6300
RH		 524void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
525{
526 if (TCG_TARGET_HAS_rot_i32) {
527 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
528 } else {
529 TCGv_i32 t0, t1;
530
531 t0 = tcg_temp_new_i32();
532 t1 = tcg_temp_new_i32();
533 tcg_gen_shl_i32(t0, arg1, arg2);
534 tcg_gen_subfi_i32(t1, 32, arg2);
535 tcg_gen_shr_i32(t1, arg1, t1);
536 tcg_gen_or_i32(ret, t0, t1);
537 tcg_temp_free_i32(t0);
538 tcg_temp_free_i32(t1);
539 }
540}
541
542void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
543{
544 tcg_debug_assert(arg2 < 32);
545 /* some cases can be optimized here */
546 if (arg2 == 0) {
547 tcg_gen_mov_i32(ret, arg1);
548 } else if (TCG_TARGET_HAS_rot_i32) {
549 TCGv_i32 t0 = tcg_const_i32(arg2);
550 tcg_gen_rotl_i32(ret, arg1, t0);
551 tcg_temp_free_i32(t0);
552 } else {
553 TCGv_i32 t0, t1;
554 t0 = tcg_temp_new_i32();
555 t1 = tcg_temp_new_i32();
556 tcg_gen_shli_i32(t0, arg1, arg2);
557 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
558 tcg_gen_or_i32(ret, t0, t1);
559 tcg_temp_free_i32(t0);
560 tcg_temp_free_i32(t1);
561 }
562}
563
564void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
565{
566 if (TCG_TARGET_HAS_rot_i32) {
567 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
568 } else {
569 TCGv_i32 t0, t1;
570
571 t0 = tcg_temp_new_i32();
572 t1 = tcg_temp_new_i32();
573 tcg_gen_shr_i32(t0, arg1, arg2);
574 tcg_gen_subfi_i32(t1, 32, arg2);
575 tcg_gen_shl_i32(t1, arg1, t1);
576 tcg_gen_or_i32(ret, t0, t1);
577 tcg_temp_free_i32(t0);
578 tcg_temp_free_i32(t1);
579 }
580}
581
582void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
583{
584 tcg_debug_assert(arg2 < 32);
585 /* some cases can be optimized here */
586 if (arg2 == 0) {
587 tcg_gen_mov_i32(ret, arg1);
588 } else {
589 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
590 }
591}
592
593void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
594 unsigned int ofs, unsigned int len)
595{
596 uint32_t mask;
597 TCGv_i32 t1;
598
599 tcg_debug_assert(ofs < 32);
0d0d309d 600 tcg_debug_assert(len > 0);
951c6300
RH		 601 tcg_debug_assert(len <= 32);
602 tcg_debug_assert(ofs + len <= 32);
603
0d0d309d 604 if (len == 32) {
951c6300
RH		 605 tcg_gen_mov_i32(ret, arg2);
606 return;
607 }
608 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
609 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
610 return;
611 }
612
613 mask = (1u << len) - 1;
614 t1 = tcg_temp_new_i32();
615
616 if (ofs + len < 32) {
617 tcg_gen_andi_i32(t1, arg2, mask);
618 tcg_gen_shli_i32(t1, t1, ofs);
619 } else {
620 tcg_gen_shli_i32(t1, arg2, ofs);
621 }
622 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
623 tcg_gen_or_i32(ret, ret, t1);
624
625 tcg_temp_free_i32(t1);
626}
627
07cc68d5
RH		 628void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
629 unsigned int ofs, unsigned int len)
630{
631 tcg_debug_assert(ofs < 32);
632 tcg_debug_assert(len > 0);
633 tcg_debug_assert(len <= 32);
634 tcg_debug_assert(ofs + len <= 32);
635
636 if (ofs + len == 32) {
637 tcg_gen_shli_i32(ret, arg, ofs);
638 } else if (ofs == 0) {
639 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
640 } else if (TCG_TARGET_HAS_deposit_i32
641 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
642 TCGv_i32 zero = tcg_const_i32(0);
643 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
644 tcg_temp_free_i32(zero);
645 } else {
646 /* To help two-operand hosts we prefer to zero-extend first,
647 which allows ARG to stay live. */
648 switch (len) {
649 case 16:
650 if (TCG_TARGET_HAS_ext16u_i32) {
651 tcg_gen_ext16u_i32(ret, arg);
652 tcg_gen_shli_i32(ret, ret, ofs);
653 return;
654 }
655 break;
656 case 8:
657 if (TCG_TARGET_HAS_ext8u_i32) {
658 tcg_gen_ext8u_i32(ret, arg);
659 tcg_gen_shli_i32(ret, ret, ofs);
660 return;
661 }
662 break;
663 }
664 /* Otherwise prefer zero-extension over AND for code size. */
665 switch (ofs + len) {
666 case 16:
667 if (TCG_TARGET_HAS_ext16u_i32) {
668 tcg_gen_shli_i32(ret, arg, ofs);
669 tcg_gen_ext16u_i32(ret, ret);
670 return;
671 }
672 break;
673 case 8:
674 if (TCG_TARGET_HAS_ext8u_i32) {
675 tcg_gen_shli_i32(ret, arg, ofs);
676 tcg_gen_ext8u_i32(ret, ret);
677 return;
678 }
679 break;
680 }
681 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
682 tcg_gen_shli_i32(ret, ret, ofs);
683 }
684}
685
7ec8bab3
RH		 686void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
687 unsigned int ofs, unsigned int len)
688{
689 tcg_debug_assert(ofs < 32);
690 tcg_debug_assert(len > 0);
691 tcg_debug_assert(len <= 32);
692 tcg_debug_assert(ofs + len <= 32);
693
694 /* Canonicalize certain special cases, even if extract is supported. */
695 if (ofs + len == 32) {
696 tcg_gen_shri_i32(ret, arg, 32 - len);
697 return;
698 }
699 if (ofs == 0) {
700 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
701 return;
702 }
703
704 if (TCG_TARGET_HAS_extract_i32
705 && TCG_TARGET_extract_i32_valid(ofs, len)) {
706 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
707 return;
708 }
709
710 /* Assume that zero-extension, if available, is cheaper than a shift. */
711 switch (ofs + len) {
712 case 16:
713 if (TCG_TARGET_HAS_ext16u_i32) {
714 tcg_gen_ext16u_i32(ret, arg);
715 tcg_gen_shri_i32(ret, ret, ofs);
716 return;
717 }
718 break;
719 case 8:
720 if (TCG_TARGET_HAS_ext8u_i32) {
721 tcg_gen_ext8u_i32(ret, arg);
722 tcg_gen_shri_i32(ret, ret, ofs);
723 return;
724 }
725 break;
726 }
727
728 /* ??? Ideally we'd know what values are available for immediate AND.
729 Assume that 8 bits are available, plus the special case of 16,
730 so that we get ext8u, ext16u. */
731 switch (len) {
732 case 1 ... 8: case 16:
733 tcg_gen_shri_i32(ret, arg, ofs);
734 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
735 break;
736 default:
737 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
738 tcg_gen_shri_i32(ret, ret, 32 - len);
739 break;
740 }
741}
742
743void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
744 unsigned int ofs, unsigned int len)
745{
746 tcg_debug_assert(ofs < 32);
747 tcg_debug_assert(len > 0);
748 tcg_debug_assert(len <= 32);
749 tcg_debug_assert(ofs + len <= 32);
750
751 /* Canonicalize certain special cases, even if extract is supported. */
752 if (ofs + len == 32) {
753 tcg_gen_sari_i32(ret, arg, 32 - len);
754 return;
755 }
756 if (ofs == 0) {
757 switch (len) {
758 case 16:
759 tcg_gen_ext16s_i32(ret, arg);
760 return;
761 case 8:
762 tcg_gen_ext8s_i32(ret, arg);
763 return;
764 }
765 }
766
767 if (TCG_TARGET_HAS_sextract_i32
768 && TCG_TARGET_extract_i32_valid(ofs, len)) {
769 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
770 return;
771 }
772
773 /* Assume that sign-extension, if available, is cheaper than a shift. */
774 switch (ofs + len) {
775 case 16:
776 if (TCG_TARGET_HAS_ext16s_i32) {
777 tcg_gen_ext16s_i32(ret, arg);
778 tcg_gen_sari_i32(ret, ret, ofs);
779 return;
780 }
781 break;
782 case 8:
783 if (TCG_TARGET_HAS_ext8s_i32) {
784 tcg_gen_ext8s_i32(ret, arg);
785 tcg_gen_sari_i32(ret, ret, ofs);
786 return;
787 }
788 break;
789 }
790 switch (len) {
791 case 16:
792 if (TCG_TARGET_HAS_ext16s_i32) {
793 tcg_gen_shri_i32(ret, arg, ofs);
794 tcg_gen_ext16s_i32(ret, ret);
795 return;
796 }
797 break;
798 case 8:
799 if (TCG_TARGET_HAS_ext8s_i32) {
800 tcg_gen_shri_i32(ret, arg, ofs);
801 tcg_gen_ext8s_i32(ret, ret);
802 return;
803 }
804 break;
805 }
806
807 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
808 tcg_gen_sari_i32(ret, ret, 32 - len);
809}
810
951c6300
RH		 811void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
812 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
813{
37ed3bf1
RH		 814 if (cond == TCG_COND_ALWAYS) {
815 tcg_gen_mov_i32(ret, v1);
816 } else if (cond == TCG_COND_NEVER) {
817 tcg_gen_mov_i32(ret, v2);
818 } else if (TCG_TARGET_HAS_movcond_i32) {
951c6300
RH		 819 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
820 } else {
821 TCGv_i32 t0 = tcg_temp_new_i32();
822 TCGv_i32 t1 = tcg_temp_new_i32();
823 tcg_gen_setcond_i32(cond, t0, c1, c2);
824 tcg_gen_neg_i32(t0, t0);
825 tcg_gen_and_i32(t1, v1, t0);
826 tcg_gen_andc_i32(ret, v2, t0);
827 tcg_gen_or_i32(ret, ret, t1);
828 tcg_temp_free_i32(t0);
829 tcg_temp_free_i32(t1);
830 }
831}
832
833void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
834 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
835{
836 if (TCG_TARGET_HAS_add2_i32) {
837 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 838 } else {
839 TCGv_i64 t0 = tcg_temp_new_i64();
840 TCGv_i64 t1 = tcg_temp_new_i64();
841 tcg_gen_concat_i32_i64(t0, al, ah);
842 tcg_gen_concat_i32_i64(t1, bl, bh);
843 tcg_gen_add_i64(t0, t0, t1);
844 tcg_gen_extr_i64_i32(rl, rh, t0);
845 tcg_temp_free_i64(t0);
846 tcg_temp_free_i64(t1);
847 }
848}
849
850void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
851 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
852{
853 if (TCG_TARGET_HAS_sub2_i32) {
854 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 855 } else {
856 TCGv_i64 t0 = tcg_temp_new_i64();
857 TCGv_i64 t1 = tcg_temp_new_i64();
858 tcg_gen_concat_i32_i64(t0, al, ah);
859 tcg_gen_concat_i32_i64(t1, bl, bh);
860 tcg_gen_sub_i64(t0, t0, t1);
861 tcg_gen_extr_i64_i32(rl, rh, t0);
862 tcg_temp_free_i64(t0);
863 tcg_temp_free_i64(t1);
864 }
865}
866
867void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
868{
869 if (TCG_TARGET_HAS_mulu2_i32) {
870 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
951c6300
RH		 871 } else if (TCG_TARGET_HAS_muluh_i32) {
872 TCGv_i32 t = tcg_temp_new_i32();
873 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
874 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
875 tcg_gen_mov_i32(rl, t);
876 tcg_temp_free_i32(t);
877 } else {
878 TCGv_i64 t0 = tcg_temp_new_i64();
879 TCGv_i64 t1 = tcg_temp_new_i64();
880 tcg_gen_extu_i32_i64(t0, arg1);
881 tcg_gen_extu_i32_i64(t1, arg2);
882 tcg_gen_mul_i64(t0, t0, t1);
883 tcg_gen_extr_i64_i32(rl, rh, t0);
884 tcg_temp_free_i64(t0);
885 tcg_temp_free_i64(t1);
886 }
887}
888
889void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
890{
891 if (TCG_TARGET_HAS_muls2_i32) {
892 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
951c6300
RH		 893 } else if (TCG_TARGET_HAS_mulsh_i32) {
894 TCGv_i32 t = tcg_temp_new_i32();
895 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
896 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
897 tcg_gen_mov_i32(rl, t);
898 tcg_temp_free_i32(t);
899 } else if (TCG_TARGET_REG_BITS == 32) {
900 TCGv_i32 t0 = tcg_temp_new_i32();
901 TCGv_i32 t1 = tcg_temp_new_i32();
902 TCGv_i32 t2 = tcg_temp_new_i32();
903 TCGv_i32 t3 = tcg_temp_new_i32();
904 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
905 /* Adjust for negative inputs. */
906 tcg_gen_sari_i32(t2, arg1, 31);
907 tcg_gen_sari_i32(t3, arg2, 31);
908 tcg_gen_and_i32(t2, t2, arg2);
909 tcg_gen_and_i32(t3, t3, arg1);
910 tcg_gen_sub_i32(rh, t1, t2);
911 tcg_gen_sub_i32(rh, rh, t3);
912 tcg_gen_mov_i32(rl, t0);
913 tcg_temp_free_i32(t0);
914 tcg_temp_free_i32(t1);
915 tcg_temp_free_i32(t2);
916 tcg_temp_free_i32(t3);
917 } else {
918 TCGv_i64 t0 = tcg_temp_new_i64();
919 TCGv_i64 t1 = tcg_temp_new_i64();
920 tcg_gen_ext_i32_i64(t0, arg1);
921 tcg_gen_ext_i32_i64(t1, arg2);
922 tcg_gen_mul_i64(t0, t0, t1);
923 tcg_gen_extr_i64_i32(rl, rh, t0);
924 tcg_temp_free_i64(t0);
925 tcg_temp_free_i64(t1);
926 }
927}
928
5087abfb
RH		 929void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
930{
931 if (TCG_TARGET_REG_BITS == 32) {
932 TCGv_i32 t0 = tcg_temp_new_i32();
933 TCGv_i32 t1 = tcg_temp_new_i32();
934 TCGv_i32 t2 = tcg_temp_new_i32();
935 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
936 /* Adjust for negative input for the signed arg1. */
937 tcg_gen_sari_i32(t2, arg1, 31);
938 tcg_gen_and_i32(t2, t2, arg2);
939 tcg_gen_sub_i32(rh, t1, t2);
940 tcg_gen_mov_i32(rl, t0);
941 tcg_temp_free_i32(t0);
942 tcg_temp_free_i32(t1);
943 tcg_temp_free_i32(t2);
944 } else {
945 TCGv_i64 t0 = tcg_temp_new_i64();
946 TCGv_i64 t1 = tcg_temp_new_i64();
947 tcg_gen_ext_i32_i64(t0, arg1);
948 tcg_gen_extu_i32_i64(t1, arg2);
949 tcg_gen_mul_i64(t0, t0, t1);
950 tcg_gen_extr_i64_i32(rl, rh, t0);
951 tcg_temp_free_i64(t0);
952 tcg_temp_free_i64(t1);
953 }
954}
955
951c6300
RH		 956void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
957{
958 if (TCG_TARGET_HAS_ext8s_i32) {
959 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
960 } else {
961 tcg_gen_shli_i32(ret, arg, 24);
962 tcg_gen_sari_i32(ret, ret, 24);
963 }
964}
965
966void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
967{
968 if (TCG_TARGET_HAS_ext16s_i32) {
969 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
970 } else {
971 tcg_gen_shli_i32(ret, arg, 16);
972 tcg_gen_sari_i32(ret, ret, 16);
973 }
974}
975
976void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
977{
978 if (TCG_TARGET_HAS_ext8u_i32) {
979 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
980 } else {
981 tcg_gen_andi_i32(ret, arg, 0xffu);
982 }
983}
984
985void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
986{
987 if (TCG_TARGET_HAS_ext16u_i32) {
988 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
989 } else {
990 tcg_gen_andi_i32(ret, arg, 0xffffu);
991 }
992}
993
994/* Note: we assume the two high bytes are set to zero */
995void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
996{
997 if (TCG_TARGET_HAS_bswap16_i32) {
998 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
999 } else {
1000 TCGv_i32 t0 = tcg_temp_new_i32();
1001
1002 tcg_gen_ext8u_i32(t0, arg);
1003 tcg_gen_shli_i32(t0, t0, 8);
1004 tcg_gen_shri_i32(ret, arg, 8);
1005 tcg_gen_or_i32(ret, ret, t0);
1006 tcg_temp_free_i32(t0);
1007 }
1008}
1009
1010void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1011{
1012 if (TCG_TARGET_HAS_bswap32_i32) {
1013 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1014 } else {
a686dc71
RH		 1015 TCGv_i32 t0 = tcg_temp_new_i32();
1016 TCGv_i32 t1 = tcg_temp_new_i32();
1017 TCGv_i32 t2 = tcg_const_i32(0x00ff00ff);
951c6300 1018
a686dc71
RH		 1019 /* arg = abcd */
1020 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */
1021 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */
1022 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */
1023 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */
1024 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */
951c6300 1025
a686dc71
RH		 1026 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */
1027 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */
1028 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */
951c6300 1029
951c6300
RH		 1030 tcg_temp_free_i32(t0);
1031 tcg_temp_free_i32(t1);
a686dc71 1032 tcg_temp_free_i32(t2);
951c6300
RH		 1033 }
1034}
1035
b87fb8cd
RH		 1036void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1037{
1038 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1039}
1040
1041void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1042{
1043 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1044}
1045
1046void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1047{
1048 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1049}
1050
1051void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1052{
1053 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1054}
1055
951c6300
RH		 1056/* 64-bit ops */
1057
1058#if TCG_TARGET_REG_BITS == 32
1059/* These are all inline for TCG_TARGET_REG_BITS == 64. */
1060
1061void tcg_gen_discard_i64(TCGv_i64 arg)
1062{
1063 tcg_gen_discard_i32(TCGV_LOW(arg));
1064 tcg_gen_discard_i32(TCGV_HIGH(arg));
1065}
1066
1067void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1068{
1069 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1070 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1071}
1072
1073void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1074{
1075 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1076 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1077}
1078
1079void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1080{
1081 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1082 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1083}
1084
1085void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1086{
1087 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
3ff91d7e 1088 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
951c6300
RH		 1089}
1090
1091void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1092{
1093 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1094 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1095}
1096
1097void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1098{
1099 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1100 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1101}
1102
1103void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1104{
1105 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1106 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1107}
1108
1109void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1110{
1111 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1112 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1113}
1114
1115void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1116{
1117 /* Since arg2 and ret have different types,
1118 they cannot be the same temporary */
cf811fff 1119#ifdef HOST_WORDS_BIGENDIAN
951c6300
RH		 1120 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1121 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1122#else
1123 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1124 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1125#endif
1126}
1127
1128void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1129{
cf811fff 1130#ifdef HOST_WORDS_BIGENDIAN
951c6300
RH		 1131 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1132 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1133#else
1134 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1135 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1136#endif
1137}
1138
1139void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1140{
1141 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1142 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1143}
1144
1145void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1146{
1147 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1148 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1149}
1150
1151void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1152{
1153 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1154 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1155}
1156
1157void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1158{
1159 gen_helper_shl_i64(ret, arg1, arg2);
1160}
1161
1162void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1163{
1164 gen_helper_shr_i64(ret, arg1, arg2);
1165}
1166
1167void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1168{
1169 gen_helper_sar_i64(ret, arg1, arg2);
1170}
1171
1172void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1173{
1174 TCGv_i64 t0;
1175 TCGv_i32 t1;
1176
1177 t0 = tcg_temp_new_i64();
1178 t1 = tcg_temp_new_i32();
1179
1180 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1181 TCGV_LOW(arg1), TCGV_LOW(arg2));
1182
1183 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1184 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1185 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1186 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1187
1188 tcg_gen_mov_i64(ret, t0);
1189 tcg_temp_free_i64(t0);
1190 tcg_temp_free_i32(t1);
1191}
1192#endif /* TCG_TARGET_REG_SIZE == 32 */
1193
1194void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1195{
1196 /* some cases can be optimized here */
1197 if (arg2 == 0) {
1198 tcg_gen_mov_i64(ret, arg1);
1199 } else {
1200 TCGv_i64 t0 = tcg_const_i64(arg2);
1201 tcg_gen_add_i64(ret, arg1, t0);
1202 tcg_temp_free_i64(t0);
1203 }
1204}
1205
1206void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1207{
1208 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1209 /* Don't recurse with tcg_gen_neg_i64. */
1210 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1211 } else {
1212 TCGv_i64 t0 = tcg_const_i64(arg1);
1213 tcg_gen_sub_i64(ret, t0, arg2);
1214 tcg_temp_free_i64(t0);
1215 }
1216}
1217
1218void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1219{
1220 /* some cases can be optimized here */
1221 if (arg2 == 0) {
1222 tcg_gen_mov_i64(ret, arg1);
1223 } else {
1224 TCGv_i64 t0 = tcg_const_i64(arg2);
1225 tcg_gen_sub_i64(ret, arg1, t0);
1226 tcg_temp_free_i64(t0);
1227 }
1228}
1229
474b2e8f 1230void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1231{
951c6300 1232 TCGv_i64 t0;
3a13c3f3
RH		 1233
1234 if (TCG_TARGET_REG_BITS == 32) {
1235 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1236 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1237 return;
1238 }
1239
951c6300
RH		 1240 /* Some cases can be optimized here. */
1241 switch (arg2) {
1242 case 0:
1243 tcg_gen_movi_i64(ret, 0);
1244 return;
474b2e8f 1245 case -1:
951c6300
RH		 1246 tcg_gen_mov_i64(ret, arg1);
1247 return;
474b2e8f 1248 case 0xff:
951c6300
RH		 1249 /* Don't recurse with tcg_gen_ext8u_i64. */
1250 if (TCG_TARGET_HAS_ext8u_i64) {
1251 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1252 return;
1253 }
1254 break;
474b2e8f 1255 case 0xffff:
951c6300
RH		 1256 if (TCG_TARGET_HAS_ext16u_i64) {
1257 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1258 return;
1259 }
1260 break;
474b2e8f 1261 case 0xffffffffu:
951c6300
RH		 1262 if (TCG_TARGET_HAS_ext32u_i64) {
1263 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1264 return;
1265 }
1266 break;
1267 }
1268 t0 = tcg_const_i64(arg2);
1269 tcg_gen_and_i64(ret, arg1, t0);
1270 tcg_temp_free_i64(t0);
951c6300
RH		 1271}
1272
1273void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1274{
3a13c3f3
RH		 1275 if (TCG_TARGET_REG_BITS == 32) {
1276 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1277 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1278 return;
1279 }
951c6300
RH		 1280 /* Some cases can be optimized here. */
1281 if (arg2 == -1) {
1282 tcg_gen_movi_i64(ret, -1);
1283 } else if (arg2 == 0) {
1284 tcg_gen_mov_i64(ret, arg1);
1285 } else {
1286 TCGv_i64 t0 = tcg_const_i64(arg2);
1287 tcg_gen_or_i64(ret, arg1, t0);
1288 tcg_temp_free_i64(t0);
1289 }
951c6300
RH		 1290}
1291
1292void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1293{
3a13c3f3
RH		 1294 if (TCG_TARGET_REG_BITS == 32) {
1295 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1296 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1297 return;
1298 }
951c6300
RH		 1299 /* Some cases can be optimized here. */
1300 if (arg2 == 0) {
1301 tcg_gen_mov_i64(ret, arg1);
1302 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1303 /* Don't recurse with tcg_gen_not_i64. */
1304 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1305 } else {
1306 TCGv_i64 t0 = tcg_const_i64(arg2);
1307 tcg_gen_xor_i64(ret, arg1, t0);
1308 tcg_temp_free_i64(t0);
1309 }
951c6300
RH		 1310}
1311
951c6300
RH		 1312static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1313 unsigned c, bool right, bool arith)
1314{
1315 tcg_debug_assert(c < 64);
1316 if (c == 0) {
1317 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1318 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1319 } else if (c >= 32) {
1320 c -= 32;
1321 if (right) {
1322 if (arith) {
1323 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1324 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1325 } else {
1326 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1327 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1328 }
1329 } else {
1330 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1331 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1332 }
1333 } else {
1334 TCGv_i32 t0, t1;
1335
1336 t0 = tcg_temp_new_i32();
1337 t1 = tcg_temp_new_i32();
1338 if (right) {
1339 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1340 if (arith) {
1341 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1342 } else {
1343 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1344 }
1345 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1346 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1347 tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1348 } else {
1349 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1350 /* Note: ret can be the same as arg1, so we use t1 */
1351 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1352 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1353 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1354 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1355 }
1356 tcg_temp_free_i32(t0);
1357 tcg_temp_free_i32(t1);
1358 }
1359}
1360
474b2e8f 1361void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1362{
474b2e8f 1363 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1364 if (TCG_TARGET_REG_BITS == 32) {
1365 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1366 } else if (arg2 == 0) {
951c6300
RH		 1367 tcg_gen_mov_i64(ret, arg1);
1368 } else {
1369 TCGv_i64 t0 = tcg_const_i64(arg2);
1370 tcg_gen_shl_i64(ret, arg1, t0);
1371 tcg_temp_free_i64(t0);
1372 }
1373}
1374
474b2e8f 1375void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1376{
474b2e8f 1377 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1378 if (TCG_TARGET_REG_BITS == 32) {
1379 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1380 } else if (arg2 == 0) {
951c6300
RH		 1381 tcg_gen_mov_i64(ret, arg1);
1382 } else {
1383 TCGv_i64 t0 = tcg_const_i64(arg2);
1384 tcg_gen_shr_i64(ret, arg1, t0);
1385 tcg_temp_free_i64(t0);
1386 }
1387}
1388
474b2e8f 1389void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1390{
474b2e8f 1391 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1392 if (TCG_TARGET_REG_BITS == 32) {
1393 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1394 } else if (arg2 == 0) {
951c6300
RH		 1395 tcg_gen_mov_i64(ret, arg1);
1396 } else {
1397 TCGv_i64 t0 = tcg_const_i64(arg2);
1398 tcg_gen_sar_i64(ret, arg1, t0);
1399 tcg_temp_free_i64(t0);
1400 }
1401}
951c6300 1402
42a268c2 1403void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
951c6300
RH		 1404{
1405 if (cond == TCG_COND_ALWAYS) {
42a268c2 1406 tcg_gen_br(l);
951c6300 1407 } else if (cond != TCG_COND_NEVER) {
3a13c3f3
RH		 1408 if (TCG_TARGET_REG_BITS == 32) {
1409 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1410 TCGV_HIGH(arg1), TCGV_LOW(arg2),
42a268c2 1411 TCGV_HIGH(arg2), cond, label_arg(l));
3a13c3f3 1412 } else {
42a268c2
RH		 1413 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1414 label_arg(l));
3a13c3f3 1415 }
951c6300
RH		 1416 }
1417}
1418
42a268c2 1419void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
951c6300
RH		 1420{
1421 if (cond == TCG_COND_ALWAYS) {
42a268c2 1422 tcg_gen_br(l);
951c6300
RH		 1423 } else if (cond != TCG_COND_NEVER) {
1424 TCGv_i64 t0 = tcg_const_i64(arg2);
42a268c2 1425 tcg_gen_brcond_i64(cond, arg1, t0, l);
951c6300
RH		 1426 tcg_temp_free_i64(t0);
1427 }
1428}
1429
1430void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1431 TCGv_i64 arg1, TCGv_i64 arg2)
1432{
1433 if (cond == TCG_COND_ALWAYS) {
1434 tcg_gen_movi_i64(ret, 1);
1435 } else if (cond == TCG_COND_NEVER) {
1436 tcg_gen_movi_i64(ret, 0);
1437 } else {
3a13c3f3
RH		 1438 if (TCG_TARGET_REG_BITS == 32) {
1439 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1440 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1441 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1442 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1443 } else {
1444 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1445 }
951c6300
RH		 1446 }
1447}
1448
1449void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1450 TCGv_i64 arg1, int64_t arg2)
1451{
1452 TCGv_i64 t0 = tcg_const_i64(arg2);
1453 tcg_gen_setcond_i64(cond, ret, arg1, t0);
1454 tcg_temp_free_i64(t0);
1455}
1456
1457void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1458{
b2e3ae94
RH		 1459 if (arg2 == 0) {
1460 tcg_gen_movi_i64(ret, 0);
1461 } else if (is_power_of_2(arg2)) {
1462 tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1463 } else {
1464 TCGv_i64 t0 = tcg_const_i64(arg2);
1465 tcg_gen_mul_i64(ret, arg1, t0);
1466 tcg_temp_free_i64(t0);
1467 }
951c6300
RH		 1468}
1469
1470void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1471{
1472 if (TCG_TARGET_HAS_div_i64) {
1473 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1474 } else if (TCG_TARGET_HAS_div2_i64) {
1475 TCGv_i64 t0 = tcg_temp_new_i64();
1476 tcg_gen_sari_i64(t0, arg1, 63);
1477 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1478 tcg_temp_free_i64(t0);
1479 } else {
1480 gen_helper_div_i64(ret, arg1, arg2);
1481 }
1482}
1483
1484void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1485{
1486 if (TCG_TARGET_HAS_rem_i64) {
1487 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1488 } else if (TCG_TARGET_HAS_div_i64) {
1489 TCGv_i64 t0 = tcg_temp_new_i64();
1490 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1491 tcg_gen_mul_i64(t0, t0, arg2);
1492 tcg_gen_sub_i64(ret, arg1, t0);
1493 tcg_temp_free_i64(t0);
1494 } else if (TCG_TARGET_HAS_div2_i64) {
1495 TCGv_i64 t0 = tcg_temp_new_i64();
1496 tcg_gen_sari_i64(t0, arg1, 63);
1497 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1498 tcg_temp_free_i64(t0);
1499 } else {
1500 gen_helper_rem_i64(ret, arg1, arg2);
1501 }
1502}
1503
1504void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1505{
1506 if (TCG_TARGET_HAS_div_i64) {
1507 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1508 } else if (TCG_TARGET_HAS_div2_i64) {
1509 TCGv_i64 t0 = tcg_temp_new_i64();
1510 tcg_gen_movi_i64(t0, 0);
1511 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1512 tcg_temp_free_i64(t0);
1513 } else {
1514 gen_helper_divu_i64(ret, arg1, arg2);
1515 }
1516}
1517
1518void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1519{
1520 if (TCG_TARGET_HAS_rem_i64) {
1521 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1522 } else if (TCG_TARGET_HAS_div_i64) {
1523 TCGv_i64 t0 = tcg_temp_new_i64();
1524 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1525 tcg_gen_mul_i64(t0, t0, arg2);
1526 tcg_gen_sub_i64(ret, arg1, t0);
1527 tcg_temp_free_i64(t0);
1528 } else if (TCG_TARGET_HAS_div2_i64) {
1529 TCGv_i64 t0 = tcg_temp_new_i64();
1530 tcg_gen_movi_i64(t0, 0);
1531 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1532 tcg_temp_free_i64(t0);
1533 } else {
1534 gen_helper_remu_i64(ret, arg1, arg2);
1535 }
1536}
1537
1538void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1539{
3a13c3f3
RH		 1540 if (TCG_TARGET_REG_BITS == 32) {
1541 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1542 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1543 } else if (TCG_TARGET_HAS_ext8s_i64) {
951c6300
RH		 1544 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1545 } else {
1546 tcg_gen_shli_i64(ret, arg, 56);
1547 tcg_gen_sari_i64(ret, ret, 56);
1548 }
951c6300
RH		 1549}
1550
1551void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1552{
3a13c3f3
RH		 1553 if (TCG_TARGET_REG_BITS == 32) {
1554 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1555 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1556 } else if (TCG_TARGET_HAS_ext16s_i64) {
951c6300
RH		 1557 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1558 } else {
1559 tcg_gen_shli_i64(ret, arg, 48);
1560 tcg_gen_sari_i64(ret, ret, 48);
1561 }
951c6300
RH		 1562}
1563
1564void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1565{
3a13c3f3
RH		 1566 if (TCG_TARGET_REG_BITS == 32) {
1567 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1568 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1569 } else if (TCG_TARGET_HAS_ext32s_i64) {
951c6300
RH		 1570 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1571 } else {
1572 tcg_gen_shli_i64(ret, arg, 32);
1573 tcg_gen_sari_i64(ret, ret, 32);
1574 }
951c6300
RH		 1575}
1576
1577void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1578{
3a13c3f3
RH		 1579 if (TCG_TARGET_REG_BITS == 32) {
1580 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1581 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1582 } else if (TCG_TARGET_HAS_ext8u_i64) {
951c6300
RH		 1583 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1584 } else {
1585 tcg_gen_andi_i64(ret, arg, 0xffu);
1586 }
951c6300
RH		 1587}
1588
1589void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1590{
3a13c3f3
RH		 1591 if (TCG_TARGET_REG_BITS == 32) {
1592 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1593 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1594 } else if (TCG_TARGET_HAS_ext16u_i64) {
951c6300
RH		 1595 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1596 } else {
1597 tcg_gen_andi_i64(ret, arg, 0xffffu);
1598 }
951c6300
RH		 1599}
1600
1601void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1602{
3a13c3f3
RH		 1603 if (TCG_TARGET_REG_BITS == 32) {
1604 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1605 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1606 } else if (TCG_TARGET_HAS_ext32u_i64) {
951c6300
RH		 1607 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1608 } else {
1609 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1610 }
951c6300
RH		 1611}
1612
1613/* Note: we assume the six high bytes are set to zero */
1614void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1615{
3a13c3f3
RH		 1616 if (TCG_TARGET_REG_BITS == 32) {
1617 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1618 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1619 } else if (TCG_TARGET_HAS_bswap16_i64) {
951c6300
RH		 1620 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1621 } else {
1622 TCGv_i64 t0 = tcg_temp_new_i64();
1623
1624 tcg_gen_ext8u_i64(t0, arg);
1625 tcg_gen_shli_i64(t0, t0, 8);
1626 tcg_gen_shri_i64(ret, arg, 8);
1627 tcg_gen_or_i64(ret, ret, t0);
1628 tcg_temp_free_i64(t0);
1629 }
951c6300
RH		 1630}
1631
1632/* Note: we assume the four high bytes are set to zero */
1633void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1634{
3a13c3f3
RH		 1635 if (TCG_TARGET_REG_BITS == 32) {
1636 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1637 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1638 } else if (TCG_TARGET_HAS_bswap32_i64) {
951c6300
RH		 1639 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1640 } else {
a686dc71
RH		 1641 TCGv_i64 t0 = tcg_temp_new_i64();
1642 TCGv_i64 t1 = tcg_temp_new_i64();
1643 TCGv_i64 t2 = tcg_const_i64(0x00ff00ff);
951c6300 1644
a686dc71
RH		 1645 /* arg = ....abcd */
1646 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .....abc */
1647 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */
1648 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */
1649 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */
1650 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */
951c6300 1651
a686dc71
RH		 1652 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */
1653 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */
1654 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */
1655 tcg_gen_or_i64(ret, t0, t1); /* ret = ....dcba */
951c6300 1656
951c6300
RH		 1657 tcg_temp_free_i64(t0);
1658 tcg_temp_free_i64(t1);
a686dc71 1659 tcg_temp_free_i64(t2);
951c6300 1660 }
951c6300
RH		 1661}
1662
1663void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1664{
3a13c3f3
RH		 1665 if (TCG_TARGET_REG_BITS == 32) {
1666 TCGv_i32 t0, t1;
1667 t0 = tcg_temp_new_i32();
1668 t1 = tcg_temp_new_i32();
951c6300 1669
3a13c3f3
RH		 1670 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1671 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1672 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1673 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1674 tcg_temp_free_i32(t0);
1675 tcg_temp_free_i32(t1);
1676 } else if (TCG_TARGET_HAS_bswap64_i64) {
951c6300
RH		 1677 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1678 } else {
1679 TCGv_i64 t0 = tcg_temp_new_i64();
1680 TCGv_i64 t1 = tcg_temp_new_i64();
9e821eab 1681 TCGv_i64 t2 = tcg_temp_new_i64();
951c6300 1682
9e821eab
RH		 1683 /* arg = abcdefgh */
1684 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1685 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */
1686 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */
1687 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */
1688 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */
1689 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */
1690
1691 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1692 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */
1693 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */
1694 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */
1695 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */
1696 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */
1697
1698 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */
1699 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */
1700 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */
951c6300 1701
951c6300
RH		 1702 tcg_temp_free_i64(t0);
1703 tcg_temp_free_i64(t1);
9e821eab 1704 tcg_temp_free_i64(t2);
951c6300 1705 }
951c6300
RH		 1706}
1707
1708void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1709{
3a13c3f3
RH		 1710 if (TCG_TARGET_REG_BITS == 32) {
1711 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1712 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1713 } else if (TCG_TARGET_HAS_not_i64) {
951c6300
RH		 1714 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1715 } else {
1716 tcg_gen_xori_i64(ret, arg, -1);
1717 }
951c6300
RH		 1718}
1719
1720void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1721{
3a13c3f3
RH		 1722 if (TCG_TARGET_REG_BITS == 32) {
1723 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1724 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1725 } else if (TCG_TARGET_HAS_andc_i64) {
951c6300
RH		 1726 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1727 } else {
1728 TCGv_i64 t0 = tcg_temp_new_i64();
1729 tcg_gen_not_i64(t0, arg2);
1730 tcg_gen_and_i64(ret, arg1, t0);
1731 tcg_temp_free_i64(t0);
1732 }
951c6300
RH		 1733}
1734
1735void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1736{
3a13c3f3
RH		 1737 if (TCG_TARGET_REG_BITS == 32) {
1738 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1739 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1740 } else if (TCG_TARGET_HAS_eqv_i64) {
951c6300
RH		 1741 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1742 } else {
1743 tcg_gen_xor_i64(ret, arg1, arg2);
1744 tcg_gen_not_i64(ret, ret);
1745 }
951c6300
RH		 1746}
1747
1748void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1749{
3a13c3f3
RH		 1750 if (TCG_TARGET_REG_BITS == 32) {
1751 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1752 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1753 } else if (TCG_TARGET_HAS_nand_i64) {
951c6300
RH		 1754 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1755 } else {
1756 tcg_gen_and_i64(ret, arg1, arg2);
1757 tcg_gen_not_i64(ret, ret);
1758 }
951c6300
RH		 1759}
1760
1761void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1762{
3a13c3f3
RH		 1763 if (TCG_TARGET_REG_BITS == 32) {
1764 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1765 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1766 } else if (TCG_TARGET_HAS_nor_i64) {
951c6300
RH		 1767 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1768 } else {
1769 tcg_gen_or_i64(ret, arg1, arg2);
1770 tcg_gen_not_i64(ret, ret);
1771 }
951c6300
RH		 1772}
1773
1774void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1775{
3a13c3f3
RH		 1776 if (TCG_TARGET_REG_BITS == 32) {
1777 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1778 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1779 } else if (TCG_TARGET_HAS_orc_i64) {
951c6300
RH		 1780 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1781 } else {
1782 TCGv_i64 t0 = tcg_temp_new_i64();
1783 tcg_gen_not_i64(t0, arg2);
1784 tcg_gen_or_i64(ret, arg1, t0);
1785 tcg_temp_free_i64(t0);
1786 }
951c6300
RH		 1787}
1788
0e28d006
RH		 1789void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1790{
1791 if (TCG_TARGET_HAS_clz_i64) {
1792 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1793 } else {
1794 gen_helper_clz_i64(ret, arg1, arg2);
1795 }
1796}
1797
1798void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1799{
1800 if (TCG_TARGET_REG_BITS == 32
1801 && TCG_TARGET_HAS_clz_i32
1802 && arg2 <= 0xffffffffu) {
1803 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1804 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1805 tcg_gen_addi_i32(t, t, 32);
1806 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1807 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1808 tcg_temp_free_i32(t);
1809 } else {
1810 TCGv_i64 t = tcg_const_i64(arg2);
1811 tcg_gen_clz_i64(ret, arg1, t);
1812 tcg_temp_free_i64(t);
1813 }
1814}
1815
1816void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1817{
1818 if (TCG_TARGET_HAS_ctz_i64) {
1819 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
14e99210
RH		 1820 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1821 TCGv_i64 z, t = tcg_temp_new_i64();
1822
1823 if (TCG_TARGET_HAS_ctpop_i64) {
1824 tcg_gen_subi_i64(t, arg1, 1);
1825 tcg_gen_andc_i64(t, t, arg1);
1826 tcg_gen_ctpop_i64(t, t);
1827 } else {
1828 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1829 tcg_gen_neg_i64(t, arg1);
1830 tcg_gen_and_i64(t, t, arg1);
1831 tcg_gen_clzi_i64(t, t, 64);
1832 tcg_gen_xori_i64(t, t, 63);
1833 }
1834 z = tcg_const_i64(0);
1835 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1836 tcg_temp_free_i64(t);
1837 tcg_temp_free_i64(z);
0e28d006
RH		 1838 } else {
1839 gen_helper_ctz_i64(ret, arg1, arg2);
1840 }
1841}
1842
1843void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1844{
1845 if (TCG_TARGET_REG_BITS == 32
1846 && TCG_TARGET_HAS_ctz_i32
1847 && arg2 <= 0xffffffffu) {
1848 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1849 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1850 tcg_gen_addi_i32(t32, t32, 32);
1851 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1852 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1853 tcg_temp_free_i32(t32);
14e99210
RH		 1854 } else if (!TCG_TARGET_HAS_ctz_i64
1855 && TCG_TARGET_HAS_ctpop_i64
1856 && arg2 == 64) {
1857 /* This equivalence has the advantage of not requiring a fixup. */
1858 TCGv_i64 t = tcg_temp_new_i64();
1859 tcg_gen_subi_i64(t, arg1, 1);
1860 tcg_gen_andc_i64(t, t, arg1);
1861 tcg_gen_ctpop_i64(ret, t);
1862 tcg_temp_free_i64(t);
0e28d006
RH		 1863 } else {
1864 TCGv_i64 t64 = tcg_const_i64(arg2);
1865 tcg_gen_ctz_i64(ret, arg1, t64);
1866 tcg_temp_free_i64(t64);
1867 }
1868}
1869
086920c2
RH		 1870void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1871{
1872 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1873 TCGv_i64 t = tcg_temp_new_i64();
1874 tcg_gen_sari_i64(t, arg, 63);
1875 tcg_gen_xor_i64(t, t, arg);
1876 tcg_gen_clzi_i64(t, t, 64);
1877 tcg_gen_subi_i64(ret, t, 1);
1878 tcg_temp_free_i64(t);
1879 } else {
1880 gen_helper_clrsb_i64(ret, arg);
1881 }
1882}
1883
a768e4e9
RH		 1884void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1885{
1886 if (TCG_TARGET_HAS_ctpop_i64) {
1887 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1888 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1889 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1890 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1891 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1892 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1893 } else {
1894 gen_helper_ctpop_i64(ret, arg1);
1895 }
1896}
1897
951c6300
RH		 1898void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1899{
1900 if (TCG_TARGET_HAS_rot_i64) {
1901 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1902 } else {
1903 TCGv_i64 t0, t1;
1904 t0 = tcg_temp_new_i64();
1905 t1 = tcg_temp_new_i64();
1906 tcg_gen_shl_i64(t0, arg1, arg2);
1907 tcg_gen_subfi_i64(t1, 64, arg2);
1908 tcg_gen_shr_i64(t1, arg1, t1);
1909 tcg_gen_or_i64(ret, t0, t1);
1910 tcg_temp_free_i64(t0);
1911 tcg_temp_free_i64(t1);
1912 }
1913}
1914
1915void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1916{
1917 tcg_debug_assert(arg2 < 64);
1918 /* some cases can be optimized here */
1919 if (arg2 == 0) {
1920 tcg_gen_mov_i64(ret, arg1);
1921 } else if (TCG_TARGET_HAS_rot_i64) {
1922 TCGv_i64 t0 = tcg_const_i64(arg2);
1923 tcg_gen_rotl_i64(ret, arg1, t0);
1924 tcg_temp_free_i64(t0);
1925 } else {
1926 TCGv_i64 t0, t1;
1927 t0 = tcg_temp_new_i64();
1928 t1 = tcg_temp_new_i64();
1929 tcg_gen_shli_i64(t0, arg1, arg2);
1930 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1931 tcg_gen_or_i64(ret, t0, t1);
1932 tcg_temp_free_i64(t0);
1933 tcg_temp_free_i64(t1);
1934 }
1935}
1936
1937void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1938{
1939 if (TCG_TARGET_HAS_rot_i64) {
1940 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1941 } else {
1942 TCGv_i64 t0, t1;
1943 t0 = tcg_temp_new_i64();
1944 t1 = tcg_temp_new_i64();
1945 tcg_gen_shr_i64(t0, arg1, arg2);
1946 tcg_gen_subfi_i64(t1, 64, arg2);
1947 tcg_gen_shl_i64(t1, arg1, t1);
1948 tcg_gen_or_i64(ret, t0, t1);
1949 tcg_temp_free_i64(t0);
1950 tcg_temp_free_i64(t1);
1951 }
1952}
1953
1954void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1955{
1956 tcg_debug_assert(arg2 < 64);
1957 /* some cases can be optimized here */
1958 if (arg2 == 0) {
1959 tcg_gen_mov_i64(ret, arg1);
1960 } else {
1961 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1962 }
1963}
1964
1965void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1966 unsigned int ofs, unsigned int len)
1967{
1968 uint64_t mask;
1969 TCGv_i64 t1;
1970
1971 tcg_debug_assert(ofs < 64);
0d0d309d 1972 tcg_debug_assert(len > 0);
951c6300
RH		 1973 tcg_debug_assert(len <= 64);
1974 tcg_debug_assert(ofs + len <= 64);
1975
0d0d309d 1976 if (len == 64) {
951c6300
RH		 1977 tcg_gen_mov_i64(ret, arg2);
1978 return;
1979 }
1980 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1981 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
1982 return;
1983 }
1984
3a13c3f3
RH		 1985 if (TCG_TARGET_REG_BITS == 32) {
1986 if (ofs >= 32) {
1987 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
1988 TCGV_LOW(arg2), ofs - 32, len);
1989 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1990 return;
1991 }
1992 if (ofs + len <= 32) {
1993 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
1994 TCGV_LOW(arg2), ofs, len);
1995 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1996 return;
1997 }
951c6300 1998 }
951c6300
RH		 1999
2000 mask = (1ull << len) - 1;
2001 t1 = tcg_temp_new_i64();
2002
2003 if (ofs + len < 64) {
2004 tcg_gen_andi_i64(t1, arg2, mask);
2005 tcg_gen_shli_i64(t1, t1, ofs);
2006 } else {
2007 tcg_gen_shli_i64(t1, arg2, ofs);
2008 }
2009 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2010 tcg_gen_or_i64(ret, ret, t1);
2011
2012 tcg_temp_free_i64(t1);
2013}
2014
07cc68d5
RH		 2015void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2016 unsigned int ofs, unsigned int len)
2017{
2018 tcg_debug_assert(ofs < 64);
2019 tcg_debug_assert(len > 0);
2020 tcg_debug_assert(len <= 64);
2021 tcg_debug_assert(ofs + len <= 64);
2022
2023 if (ofs + len == 64) {
2024 tcg_gen_shli_i64(ret, arg, ofs);
2025 } else if (ofs == 0) {
2026 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2027 } else if (TCG_TARGET_HAS_deposit_i64
2028 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2029 TCGv_i64 zero = tcg_const_i64(0);
2030 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2031 tcg_temp_free_i64(zero);
2032 } else {
2033 if (TCG_TARGET_REG_BITS == 32) {
2034 if (ofs >= 32) {
2035 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2036 ofs - 32, len);
2037 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2038 return;
2039 }
2040 if (ofs + len <= 32) {
2041 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2042 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2043 return;
2044 }
2045 }
2046 /* To help two-operand hosts we prefer to zero-extend first,
2047 which allows ARG to stay live. */
2048 switch (len) {
2049 case 32:
2050 if (TCG_TARGET_HAS_ext32u_i64) {
2051 tcg_gen_ext32u_i64(ret, arg);
2052 tcg_gen_shli_i64(ret, ret, ofs);
2053 return;
2054 }
2055 break;
2056 case 16:
2057 if (TCG_TARGET_HAS_ext16u_i64) {
2058 tcg_gen_ext16u_i64(ret, arg);
2059 tcg_gen_shli_i64(ret, ret, ofs);
2060 return;
2061 }
2062 break;
2063 case 8:
2064 if (TCG_TARGET_HAS_ext8u_i64) {
2065 tcg_gen_ext8u_i64(ret, arg);
2066 tcg_gen_shli_i64(ret, ret, ofs);
2067 return;
2068 }
2069 break;
2070 }
2071 /* Otherwise prefer zero-extension over AND for code size. */
2072 switch (ofs + len) {
2073 case 32:
2074 if (TCG_TARGET_HAS_ext32u_i64) {
2075 tcg_gen_shli_i64(ret, arg, ofs);
2076 tcg_gen_ext32u_i64(ret, ret);
2077 return;
2078 }
2079 break;
2080 case 16:
2081 if (TCG_TARGET_HAS_ext16u_i64) {
2082 tcg_gen_shli_i64(ret, arg, ofs);
2083 tcg_gen_ext16u_i64(ret, ret);
2084 return;
2085 }
2086 break;
2087 case 8:
2088 if (TCG_TARGET_HAS_ext8u_i64) {
2089 tcg_gen_shli_i64(ret, arg, ofs);
2090 tcg_gen_ext8u_i64(ret, ret);
2091 return;
2092 }
2093 break;
2094 }
2095 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2096 tcg_gen_shli_i64(ret, ret, ofs);
2097 }
2098}
2099
7ec8bab3
RH		 2100void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2101 unsigned int ofs, unsigned int len)
2102{
2103 tcg_debug_assert(ofs < 64);
2104 tcg_debug_assert(len > 0);
2105 tcg_debug_assert(len <= 64);
2106 tcg_debug_assert(ofs + len <= 64);
2107
2108 /* Canonicalize certain special cases, even if extract is supported. */
2109 if (ofs + len == 64) {
2110 tcg_gen_shri_i64(ret, arg, 64 - len);
2111 return;
2112 }
2113 if (ofs == 0) {
2114 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2115 return;
2116 }
2117
2118 if (TCG_TARGET_REG_BITS == 32) {
2119 /* Look for a 32-bit extract within one of the two words. */
2120 if (ofs >= 32) {
2121 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2122 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2123 return;
2124 }
2125 if (ofs + len <= 32) {
2126 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2127 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2128 return;
2129 }
2130 /* The field is split across two words. One double-word
2131 shift is better than two double-word shifts. */
2132 goto do_shift_and;
2133 }
2134
2135 if (TCG_TARGET_HAS_extract_i64
2136 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2137 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2138 return;
2139 }
2140
2141 /* Assume that zero-extension, if available, is cheaper than a shift. */
2142 switch (ofs + len) {
2143 case 32:
2144 if (TCG_TARGET_HAS_ext32u_i64) {
2145 tcg_gen_ext32u_i64(ret, arg);
2146 tcg_gen_shri_i64(ret, ret, ofs);
2147 return;
2148 }
2149 break;
2150 case 16:
2151 if (TCG_TARGET_HAS_ext16u_i64) {
2152 tcg_gen_ext16u_i64(ret, arg);
2153 tcg_gen_shri_i64(ret, ret, ofs);
2154 return;
2155 }
2156 break;
2157 case 8:
2158 if (TCG_TARGET_HAS_ext8u_i64) {
2159 tcg_gen_ext8u_i64(ret, arg);
2160 tcg_gen_shri_i64(ret, ret, ofs);
2161 return;
2162 }
2163 break;
2164 }
2165
2166 /* ??? Ideally we'd know what values are available for immediate AND.
2167 Assume that 8 bits are available, plus the special cases of 16 and 32,
2168 so that we get ext8u, ext16u, and ext32u. */
2169 switch (len) {
2170 case 1 ... 8: case 16: case 32:
2171 do_shift_and:
2172 tcg_gen_shri_i64(ret, arg, ofs);
2173 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2174 break;
2175 default:
2176 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2177 tcg_gen_shri_i64(ret, ret, 64 - len);
2178 break;
2179 }
2180}
2181
2182void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2183 unsigned int ofs, unsigned int len)
2184{
2185 tcg_debug_assert(ofs < 64);
2186 tcg_debug_assert(len > 0);
2187 tcg_debug_assert(len <= 64);
2188 tcg_debug_assert(ofs + len <= 64);
2189
2190 /* Canonicalize certain special cases, even if sextract is supported. */
2191 if (ofs + len == 64) {
2192 tcg_gen_sari_i64(ret, arg, 64 - len);
2193 return;
2194 }
2195 if (ofs == 0) {
2196 switch (len) {
2197 case 32:
2198 tcg_gen_ext32s_i64(ret, arg);
2199 return;
2200 case 16:
2201 tcg_gen_ext16s_i64(ret, arg);
2202 return;
2203 case 8:
2204 tcg_gen_ext8s_i64(ret, arg);
2205 return;
2206 }
2207 }
2208
2209 if (TCG_TARGET_REG_BITS == 32) {
2210 /* Look for a 32-bit extract within one of the two words. */
2211 if (ofs >= 32) {
2212 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2213 } else if (ofs + len <= 32) {
2214 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2215 } else if (ofs == 0) {
2216 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2217 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2218 return;
2219 } else if (len > 32) {
2220 TCGv_i32 t = tcg_temp_new_i32();
2221 /* Extract the bits for the high word normally. */
2222 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2223 /* Shift the field down for the low part. */
2224 tcg_gen_shri_i64(ret, arg, ofs);
2225 /* Overwrite the shift into the high part. */
2226 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2227 tcg_temp_free_i32(t);
2228 return;
2229 } else {
2230 /* Shift the field down for the low part, such that the
2231 field sits at the MSB. */
2232 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2233 /* Shift the field down from the MSB, sign extending. */
2234 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2235 }
2236 /* Sign-extend the field from 32 bits. */
2237 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2238 return;
2239 }
2240
2241 if (TCG_TARGET_HAS_sextract_i64
2242 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2243 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2244 return;
2245 }
2246
2247 /* Assume that sign-extension, if available, is cheaper than a shift. */
2248 switch (ofs + len) {
2249 case 32:
2250 if (TCG_TARGET_HAS_ext32s_i64) {
2251 tcg_gen_ext32s_i64(ret, arg);
2252 tcg_gen_sari_i64(ret, ret, ofs);
2253 return;
2254 }
2255 break;
2256 case 16:
2257 if (TCG_TARGET_HAS_ext16s_i64) {
2258 tcg_gen_ext16s_i64(ret, arg);
2259 tcg_gen_sari_i64(ret, ret, ofs);
2260 return;
2261 }
2262 break;
2263 case 8:
2264 if (TCG_TARGET_HAS_ext8s_i64) {
2265 tcg_gen_ext8s_i64(ret, arg);
2266 tcg_gen_sari_i64(ret, ret, ofs);
2267 return;
2268 }
2269 break;
2270 }
2271 switch (len) {
2272 case 32:
2273 if (TCG_TARGET_HAS_ext32s_i64) {
2274 tcg_gen_shri_i64(ret, arg, ofs);
2275 tcg_gen_ext32s_i64(ret, ret);
2276 return;
2277 }
2278 break;
2279 case 16:
2280 if (TCG_TARGET_HAS_ext16s_i64) {
2281 tcg_gen_shri_i64(ret, arg, ofs);
2282 tcg_gen_ext16s_i64(ret, ret);
2283 return;
2284 }
2285 break;
2286 case 8:
2287 if (TCG_TARGET_HAS_ext8s_i64) {
2288 tcg_gen_shri_i64(ret, arg, ofs);
2289 tcg_gen_ext8s_i64(ret, ret);
2290 return;
2291 }
2292 break;
2293 }
2294 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2295 tcg_gen_sari_i64(ret, ret, 64 - len);
2296}
2297
951c6300
RH		 2298void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2299 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2300{
37ed3bf1
RH		 2301 if (cond == TCG_COND_ALWAYS) {
2302 tcg_gen_mov_i64(ret, v1);
2303 } else if (cond == TCG_COND_NEVER) {
2304 tcg_gen_mov_i64(ret, v2);
2305 } else if (TCG_TARGET_REG_BITS == 32) {
3a13c3f3
RH		 2306 TCGv_i32 t0 = tcg_temp_new_i32();
2307 TCGv_i32 t1 = tcg_temp_new_i32();
2308 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2309 TCGV_LOW(c1), TCGV_HIGH(c1),
2310 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2311
2312 if (TCG_TARGET_HAS_movcond_i32) {
2313 tcg_gen_movi_i32(t1, 0);
2314 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2315 TCGV_LOW(v1), TCGV_LOW(v2));
2316 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2317 TCGV_HIGH(v1), TCGV_HIGH(v2));
2318 } else {
2319 tcg_gen_neg_i32(t0, t0);
951c6300 2320
3a13c3f3
RH		 2321 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2322 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2323 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
951c6300 2324
3a13c3f3
RH		 2325 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2326 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2327 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2328 }
2329 tcg_temp_free_i32(t0);
2330 tcg_temp_free_i32(t1);
2331 } else if (TCG_TARGET_HAS_movcond_i64) {
951c6300
RH		 2332 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2333 } else {
2334 TCGv_i64 t0 = tcg_temp_new_i64();
2335 TCGv_i64 t1 = tcg_temp_new_i64();
2336 tcg_gen_setcond_i64(cond, t0, c1, c2);
2337 tcg_gen_neg_i64(t0, t0);
2338 tcg_gen_and_i64(t1, v1, t0);
2339 tcg_gen_andc_i64(ret, v2, t0);
2340 tcg_gen_or_i64(ret, ret, t1);
2341 tcg_temp_free_i64(t0);
2342 tcg_temp_free_i64(t1);
2343 }
951c6300
RH		 2344}
2345
2346void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2347 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2348{
2349 if (TCG_TARGET_HAS_add2_i64) {
2350 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2351 } else {
2352 TCGv_i64 t0 = tcg_temp_new_i64();
2353 TCGv_i64 t1 = tcg_temp_new_i64();
2354 tcg_gen_add_i64(t0, al, bl);
2355 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2356 tcg_gen_add_i64(rh, ah, bh);
2357 tcg_gen_add_i64(rh, rh, t1);
2358 tcg_gen_mov_i64(rl, t0);
2359 tcg_temp_free_i64(t0);
2360 tcg_temp_free_i64(t1);
2361 }
2362}
2363
2364void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2365 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2366{
2367 if (TCG_TARGET_HAS_sub2_i64) {
2368 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2369 } else {
2370 TCGv_i64 t0 = tcg_temp_new_i64();
2371 TCGv_i64 t1 = tcg_temp_new_i64();
2372 tcg_gen_sub_i64(t0, al, bl);
2373 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2374 tcg_gen_sub_i64(rh, ah, bh);
2375 tcg_gen_sub_i64(rh, rh, t1);
2376 tcg_gen_mov_i64(rl, t0);
2377 tcg_temp_free_i64(t0);
2378 tcg_temp_free_i64(t1);
2379 }
2380}
2381
2382void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2383{
2384 if (TCG_TARGET_HAS_mulu2_i64) {
2385 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2386 } else if (TCG_TARGET_HAS_muluh_i64) {
2387 TCGv_i64 t = tcg_temp_new_i64();
2388 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2389 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2390 tcg_gen_mov_i64(rl, t);
2391 tcg_temp_free_i64(t);
2392 } else {
2393 TCGv_i64 t0 = tcg_temp_new_i64();
2394 tcg_gen_mul_i64(t0, arg1, arg2);
2395 gen_helper_muluh_i64(rh, arg1, arg2);
2396 tcg_gen_mov_i64(rl, t0);
2397 tcg_temp_free_i64(t0);
2398 }
2399}
2400
2401void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2402{
2403 if (TCG_TARGET_HAS_muls2_i64) {
2404 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2405 } else if (TCG_TARGET_HAS_mulsh_i64) {
2406 TCGv_i64 t = tcg_temp_new_i64();
2407 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2408 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2409 tcg_gen_mov_i64(rl, t);
2410 tcg_temp_free_i64(t);
2411 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2412 TCGv_i64 t0 = tcg_temp_new_i64();
2413 TCGv_i64 t1 = tcg_temp_new_i64();
2414 TCGv_i64 t2 = tcg_temp_new_i64();
2415 TCGv_i64 t3 = tcg_temp_new_i64();
2416 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2417 /* Adjust for negative inputs. */
2418 tcg_gen_sari_i64(t2, arg1, 63);
2419 tcg_gen_sari_i64(t3, arg2, 63);
2420 tcg_gen_and_i64(t2, t2, arg2);
2421 tcg_gen_and_i64(t3, t3, arg1);
2422 tcg_gen_sub_i64(rh, t1, t2);
2423 tcg_gen_sub_i64(rh, rh, t3);
2424 tcg_gen_mov_i64(rl, t0);
2425 tcg_temp_free_i64(t0);
2426 tcg_temp_free_i64(t1);
2427 tcg_temp_free_i64(t2);
2428 tcg_temp_free_i64(t3);
2429 } else {
2430 TCGv_i64 t0 = tcg_temp_new_i64();
2431 tcg_gen_mul_i64(t0, arg1, arg2);
2432 gen_helper_mulsh_i64(rh, arg1, arg2);
2433 tcg_gen_mov_i64(rl, t0);
2434 tcg_temp_free_i64(t0);
2435 }
2436}
2437
5087abfb
RH		 2438void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2439{
2440 TCGv_i64 t0 = tcg_temp_new_i64();
2441 TCGv_i64 t1 = tcg_temp_new_i64();
2442 TCGv_i64 t2 = tcg_temp_new_i64();
2443 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2444 /* Adjust for negative input for the signed arg1. */
2445 tcg_gen_sari_i64(t2, arg1, 63);
2446 tcg_gen_and_i64(t2, t2, arg2);
2447 tcg_gen_sub_i64(rh, t1, t2);
2448 tcg_gen_mov_i64(rl, t0);
2449 tcg_temp_free_i64(t0);
2450 tcg_temp_free_i64(t1);
2451 tcg_temp_free_i64(t2);
2452}
2453
b87fb8cd
RH		 2454void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2455{
2456 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2457}
2458
2459void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2460{
2461 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2462}
2463
2464void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2465{
2466 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2467}
2468
2469void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2470{
2471 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2472}
2473
951c6300
RH		 2474/* Size changing operations. */
2475
609ad705 2476void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
951c6300 2477{
3a13c3f3 2478 if (TCG_TARGET_REG_BITS == 32) {
609ad705
RH		 2479 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2480 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
b7e8b17a 2481 tcg_gen_op2(INDEX_op_extrl_i64_i32,
ae8b75dc 2482 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
609ad705 2483 } else {
dc41aa7d 2484 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
609ad705
RH		 2485 }
2486}
2487
2488void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2489{
2490 if (TCG_TARGET_REG_BITS == 32) {
2491 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2492 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
b7e8b17a 2493 tcg_gen_op2(INDEX_op_extrh_i64_i32,
ae8b75dc 2494 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
951c6300
RH		 2495 } else {
2496 TCGv_i64 t = tcg_temp_new_i64();
609ad705 2497 tcg_gen_shri_i64(t, arg, 32);
dc41aa7d 2498 tcg_gen_mov_i32(ret, (TCGv_i32)t);
951c6300
RH		 2499 tcg_temp_free_i64(t);
2500 }
951c6300
RH		 2501}
2502
2503void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2504{
3a13c3f3
RH		 2505 if (TCG_TARGET_REG_BITS == 32) {
2506 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2507 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2508 } else {
b7e8b17a 2509 tcg_gen_op2(INDEX_op_extu_i32_i64,
ae8b75dc 2510 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2511 }
951c6300
RH		 2512}
2513
2514void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2515{
3a13c3f3
RH		 2516 if (TCG_TARGET_REG_BITS == 32) {
2517 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2518 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2519 } else {
b7e8b17a 2520 tcg_gen_op2(INDEX_op_ext_i32_i64,
ae8b75dc 2521 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2522 }
951c6300
RH		 2523}
2524
2525void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2526{
3a13c3f3
RH		 2527 TCGv_i64 tmp;
2528
2529 if (TCG_TARGET_REG_BITS == 32) {
2530 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2531 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2532 return;
2533 }
2534
2535 tmp = tcg_temp_new_i64();
951c6300
RH		 2536 /* These extensions are only needed for type correctness.
2537 We may be able to do better given target specific information. */
2538 tcg_gen_extu_i32_i64(tmp, high);
2539 tcg_gen_extu_i32_i64(dest, low);
2540 /* If deposit is available, use it. Otherwise use the extra
2541 knowledge that we have of the zero-extensions above. */
2542 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2543 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2544 } else {
2545 tcg_gen_shli_i64(tmp, tmp, 32);
2546 tcg_gen_or_i64(dest, dest, tmp);
2547 }
2548 tcg_temp_free_i64(tmp);
951c6300
RH		 2549}
2550
2551void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2552{
3a13c3f3
RH		 2553 if (TCG_TARGET_REG_BITS == 32) {
2554 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2555 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2556 } else {
609ad705
RH		 2557 tcg_gen_extrl_i64_i32(lo, arg);
2558 tcg_gen_extrh_i64_i32(hi, arg);
3a13c3f3 2559 }
951c6300
RH		 2560}
2561
2562void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2563{
2564 tcg_gen_ext32u_i64(lo, arg);
2565 tcg_gen_shri_i64(hi, arg, 32);
2566}
2567
2568/* QEMU specific operations. */
2569
07ea28b4
RH		 2570void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
2571{
2572 uintptr_t val = (uintptr_t)tb + idx;
2573
2574 if (tb == NULL) {
2575 tcg_debug_assert(idx == 0);
2576 } else if (idx <= TB_EXIT_IDXMAX) {
2577#ifdef CONFIG_DEBUG_TCG
2578 /* This is an exit following a goto_tb. Verify that we have
2579 seen this numbered exit before, via tcg_gen_goto_tb. */
2580 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2581#endif
d7f425fd
RH		 2582 /* When not chaining, exit without indicating a link. */
2583 if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2584 val = 0;
2585 }
07ea28b4
RH		 2586 } else {
2587 /* This is an exit via the exitreq label. */
2588 tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2589 }
2590
2591 tcg_gen_op1i(INDEX_op_exit_tb, val);
2592}
2593
951c6300
RH		 2594void tcg_gen_goto_tb(unsigned idx)
2595{
2596 /* We only support two chained exits. */
07ea28b4 2597 tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
951c6300
RH		 2598#ifdef CONFIG_DEBUG_TCG
2599 /* Verify that we havn't seen this numbered exit before. */
b1311c4a
EC		 2600 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2601 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
951c6300 2602#endif
d7f425fd
RH		 2603 /* When not chaining, we simply fall through to the "fallback" exit. */
2604 if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2605 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2606 }
951c6300
RH		 2607}
2608
7f11636d 2609void tcg_gen_lookup_and_goto_ptr(void)
cedbcb01
EC		 2610{
2611 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2612 TCGv_ptr ptr = tcg_temp_new_ptr();
1c2adb95 2613 gen_helper_lookup_tb_ptr(ptr, cpu_env);
ae8b75dc 2614 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
cedbcb01
EC		 2615 tcg_temp_free_ptr(ptr);
2616 } else {
07ea28b4 2617 tcg_gen_exit_tb(NULL, 0);
cedbcb01
EC		 2618 }
2619}
2620
951c6300
RH		 2621static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2622{
1f00b27f
SS		 2623 /* Trigger the asserts within as early as possible. */
2624 (void)get_alignment_bits(op);
2625
951c6300
RH		 2626 switch (op & MO_SIZE) {
2627 case MO_8:
2628 op &= ~MO_BSWAP;
2629 break;
2630 case MO_16:
2631 break;
2632 case MO_32:
2633 if (!is64) {
2634 op &= ~MO_SIGN;
2635 }
2636 break;
2637 case MO_64:
2638 if (!is64) {
2639 tcg_abort();
2640 }
2641 break;
2642 }
2643 if (st) {
2644 op &= ~MO_SIGN;
2645 }
2646 return op;
2647}
2648
c45cb8bb
RH		 2649static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2650 TCGMemOp memop, TCGArg idx)
951c6300 2651{
59227d5d 2652 TCGMemOpIdx oi = make_memop_idx(memop, idx);
c45cb8bb 2653#if TARGET_LONG_BITS == 32
59227d5d 2654 tcg_gen_op3i_i32(opc, val, addr, oi);
c45cb8bb 2655#else
3a13c3f3 2656 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2657 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
3a13c3f3 2658 } else {
ae8b75dc 2659 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
3a13c3f3 2660 }
c45cb8bb 2661#endif
951c6300
RH		 2662}
2663
c45cb8bb
RH		 2664static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2665 TCGMemOp memop, TCGArg idx)
2666{
59227d5d 2667 TCGMemOpIdx oi = make_memop_idx(memop, idx);
951c6300 2668#if TARGET_LONG_BITS == 32
c45cb8bb 2669 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2670 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
c45cb8bb 2671 } else {
ae8b75dc 2672 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
c45cb8bb 2673 }
951c6300 2674#else
c45cb8bb 2675 if (TCG_TARGET_REG_BITS == 32) {
59227d5d
RH		 2676 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2677 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
c45cb8bb 2678 } else {
59227d5d 2679 tcg_gen_op3i_i64(opc, val, addr, oi);
c45cb8bb 2680 }
951c6300 2681#endif
c45cb8bb 2682}
951c6300 2683
b32dc337
PK		 2684static void tcg_gen_req_mo(TCGBar type)
2685{
2686#ifdef TCG_GUEST_DEFAULT_MO
2687 type &= TCG_GUEST_DEFAULT_MO;
2688#endif
2689 type &= ~TCG_TARGET_DEFAULT_MO;
2690 if (type) {
2691 tcg_gen_mb(type | TCG_BAR_SC);
2692 }
2693}
2694
951c6300
RH		 2695void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2696{
e1dcf352
RH		 2697 TCGMemOp orig_memop;
2698
b32dc337 2699 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
951c6300 2700 memop = tcg_canonicalize_memop(memop, 0, 0);
1c2adb95 2701 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2702 addr, trace_mem_get_info(memop, 0));
e1dcf352
RH		 2703
2704 orig_memop = memop;
2705 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2706 memop &= ~MO_BSWAP;
2707 /* The bswap primitive requires zero-extended input. */
2708 if ((memop & MO_SSIZE) == MO_SW) {
2709 memop &= ~MO_SIGN;
2710 }
2711 }
2712
c45cb8bb 2713 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
e1dcf352
RH		 2714
2715 if ((orig_memop ^ memop) & MO_BSWAP) {
2716 switch (orig_memop & MO_SIZE) {
2717 case MO_16:
2718 tcg_gen_bswap16_i32(val, val);
2719 if (orig_memop & MO_SIGN) {
2720 tcg_gen_ext16s_i32(val, val);
2721 }
2722 break;
2723 case MO_32:
2724 tcg_gen_bswap32_i32(val, val);
2725 break;
2726 default:
2727 g_assert_not_reached();
2728 }
2729 }
951c6300
RH		 2730}
2731
2732void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2733{
e1dcf352
RH		 2734 TCGv_i32 swap = NULL;
2735
b32dc337 2736 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
951c6300 2737 memop = tcg_canonicalize_memop(memop, 0, 1);
1c2adb95 2738 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2739 addr, trace_mem_get_info(memop, 1));
e1dcf352
RH		 2740
2741 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2742 swap = tcg_temp_new_i32();
2743 switch (memop & MO_SIZE) {
2744 case MO_16:
2745 tcg_gen_ext16u_i32(swap, val);
2746 tcg_gen_bswap16_i32(swap, swap);
2747 break;
2748 case MO_32:
2749 tcg_gen_bswap32_i32(swap, val);
2750 break;
2751 default:
2752 g_assert_not_reached();
2753 }
2754 val = swap;
2755 memop &= ~MO_BSWAP;
2756 }
2757
c45cb8bb 2758 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
e1dcf352
RH		 2759
2760 if (swap) {
2761 tcg_temp_free_i32(swap);
2762 }
951c6300
RH		 2763}
2764
2765void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2766{
e1dcf352
RH		 2767 TCGMemOp orig_memop;
2768
3a13c3f3 2769 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 2770 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2771 if (memop & MO_SIGN) {
2772 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2773 } else {
2774 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2775 }
2776 return;
2777 }
951c6300 2778
e1dcf352 2779 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
c45cb8bb 2780 memop = tcg_canonicalize_memop(memop, 1, 0);
1c2adb95 2781 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2782 addr, trace_mem_get_info(memop, 0));
e1dcf352
RH		 2783
2784 orig_memop = memop;
2785 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2786 memop &= ~MO_BSWAP;
2787 /* The bswap primitive requires zero-extended input. */
2788 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
2789 memop &= ~MO_SIGN;
2790 }
2791 }
2792
c45cb8bb 2793 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
e1dcf352
RH		 2794
2795 if ((orig_memop ^ memop) & MO_BSWAP) {
2796 switch (orig_memop & MO_SIZE) {
2797 case MO_16:
2798 tcg_gen_bswap16_i64(val, val);
2799 if (orig_memop & MO_SIGN) {
2800 tcg_gen_ext16s_i64(val, val);
2801 }
2802 break;
2803 case MO_32:
2804 tcg_gen_bswap32_i64(val, val);
2805 if (orig_memop & MO_SIGN) {
2806 tcg_gen_ext32s_i64(val, val);
2807 }
2808 break;
2809 case MO_64:
2810 tcg_gen_bswap64_i64(val, val);
2811 break;
2812 default:
2813 g_assert_not_reached();
2814 }
2815 }
951c6300
RH		 2816}
2817
2818void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2819{
e1dcf352
RH		 2820 TCGv_i64 swap = NULL;
2821
3a13c3f3 2822 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 2823 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2824 return;
2825 }
951c6300 2826
e1dcf352 2827 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
c45cb8bb 2828 memop = tcg_canonicalize_memop(memop, 1, 1);
1c2adb95 2829 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2830 addr, trace_mem_get_info(memop, 1));
e1dcf352
RH		 2831
2832 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2833 swap = tcg_temp_new_i64();
2834 switch (memop & MO_SIZE) {
2835 case MO_16:
2836 tcg_gen_ext16u_i64(swap, val);
2837 tcg_gen_bswap16_i64(swap, swap);
2838 break;
2839 case MO_32:
2840 tcg_gen_ext32u_i64(swap, val);
2841 tcg_gen_bswap32_i64(swap, swap);
2842 break;
2843 case MO_64:
2844 tcg_gen_bswap64_i64(swap, val);
2845 break;
2846 default:
2847 g_assert_not_reached();
2848 }
2849 val = swap;
2850 memop &= ~MO_BSWAP;
2851 }
2852
c45cb8bb 2853 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
e1dcf352
RH		 2854
2855 if (swap) {
2856 tcg_temp_free_i64(swap);
2857 }
951c6300 2858}
c482cb11
RH		 2859
2860static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2861{
2862 switch (opc & MO_SSIZE) {
2863 case MO_SB:
2864 tcg_gen_ext8s_i32(ret, val);
2865 break;
2866 case MO_UB:
2867 tcg_gen_ext8u_i32(ret, val);
2868 break;
2869 case MO_SW:
2870 tcg_gen_ext16s_i32(ret, val);
2871 break;
2872 case MO_UW:
2873 tcg_gen_ext16u_i32(ret, val);
2874 break;
2875 default:
2876 tcg_gen_mov_i32(ret, val);
2877 break;
2878 }
2879}
2880
2881static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2882{
2883 switch (opc & MO_SSIZE) {
2884 case MO_SB:
2885 tcg_gen_ext8s_i64(ret, val);
2886 break;
2887 case MO_UB:
2888 tcg_gen_ext8u_i64(ret, val);
2889 break;
2890 case MO_SW:
2891 tcg_gen_ext16s_i64(ret, val);
2892 break;
2893 case MO_UW:
2894 tcg_gen_ext16u_i64(ret, val);
2895 break;
2896 case MO_SL:
2897 tcg_gen_ext32s_i64(ret, val);
2898 break;
2899 case MO_UL:
2900 tcg_gen_ext32u_i64(ret, val);
2901 break;
2902 default:
2903 tcg_gen_mov_i64(ret, val);
2904 break;
2905 }
2906}
2907
2908#ifdef CONFIG_SOFTMMU
2909typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2910 TCGv_i32, TCGv_i32, TCGv_i32);
2911typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2912 TCGv_i64, TCGv_i64, TCGv_i32);
2913typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2914 TCGv_i32, TCGv_i32);
2915typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2916 TCGv_i64, TCGv_i32);
2917#else
2918typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2919typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2920typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2921typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2922#endif
2923
df79b996
RH		 2924#ifdef CONFIG_ATOMIC64
2925# define WITH_ATOMIC64(X) X,
2926#else
2927# define WITH_ATOMIC64(X)
2928#endif
2929
c482cb11
RH		 2930static void * const table_cmpxchg[16] = {
2931 [MO_8] = gen_helper_atomic_cmpxchgb,
2932 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2933 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2934 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2935 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
df79b996
RH		 2936 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2937 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
c482cb11
RH		 2938};
2939
2940void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2941 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2942{
2943 memop = tcg_canonicalize_memop(memop, 0, 0);
2944
b1311c4a 2945 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
c482cb11
RH		 2946 TCGv_i32 t1 = tcg_temp_new_i32();
2947 TCGv_i32 t2 = tcg_temp_new_i32();
2948
2949 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2950
2951 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2952 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2953 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2954 tcg_temp_free_i32(t2);
2955
2956 if (memop & MO_SIGN) {
2957 tcg_gen_ext_i32(retv, t1, memop);
2958 } else {
2959 tcg_gen_mov_i32(retv, t1);
2960 }
2961 tcg_temp_free_i32(t1);
2962 } else {
2963 gen_atomic_cx_i32 gen;
2964
2965 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2966 tcg_debug_assert(gen != NULL);
2967
2968#ifdef CONFIG_SOFTMMU
2969 {
2970 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
1c2adb95 2971 gen(retv, cpu_env, addr, cmpv, newv, oi);
c482cb11
RH		 2972 tcg_temp_free_i32(oi);
2973 }
2974#else
1c2adb95 2975 gen(retv, cpu_env, addr, cmpv, newv);
c482cb11
RH		 2976#endif
2977
2978 if (memop & MO_SIGN) {
2979 tcg_gen_ext_i32(retv, retv, memop);
2980 }
2981 }
2982}
2983
2984void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2985 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2986{
2987 memop = tcg_canonicalize_memop(memop, 1, 0);
2988
b1311c4a 2989 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
c482cb11
RH		 2990 TCGv_i64 t1 = tcg_temp_new_i64();
2991 TCGv_i64 t2 = tcg_temp_new_i64();
2992
2993 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2994
2995 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2996 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2997 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2998 tcg_temp_free_i64(t2);
2999
3000 if (memop & MO_SIGN) {
3001 tcg_gen_ext_i64(retv, t1, memop);
3002 } else {
3003 tcg_gen_mov_i64(retv, t1);
3004 }
3005 tcg_temp_free_i64(t1);
3006 } else if ((memop & MO_SIZE) == MO_64) {
df79b996 3007#ifdef CONFIG_ATOMIC64
c482cb11
RH		 3008 gen_atomic_cx_i64 gen;
3009
3010 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3011 tcg_debug_assert(gen != NULL);
3012
3013#ifdef CONFIG_SOFTMMU
3014 {
3015 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
1c2adb95 3016 gen(retv, cpu_env, addr, cmpv, newv, oi);
c482cb11
RH		 3017 tcg_temp_free_i32(oi);
3018 }
3019#else
1c2adb95 3020 gen(retv, cpu_env, addr, cmpv, newv);
c482cb11 3021#endif
df79b996 3022#else
1c2adb95 3023 gen_helper_exit_atomic(cpu_env);
79b1af90
RH		 3024 /* Produce a result, so that we have a well-formed opcode stream
3025 with respect to uses of the result in the (dead) code following. */
3026 tcg_gen_movi_i64(retv, 0);
df79b996 3027#endif /* CONFIG_ATOMIC64 */
c482cb11
RH		 3028 } else {
3029 TCGv_i32 c32 = tcg_temp_new_i32();
3030 TCGv_i32 n32 = tcg_temp_new_i32();
3031 TCGv_i32 r32 = tcg_temp_new_i32();
3032
3033 tcg_gen_extrl_i64_i32(c32, cmpv);
3034 tcg_gen_extrl_i64_i32(n32, newv);
3035 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3036 tcg_temp_free_i32(c32);
3037 tcg_temp_free_i32(n32);
3038
3039 tcg_gen_extu_i32_i64(retv, r32);
3040 tcg_temp_free_i32(r32);
3041
3042 if (memop & MO_SIGN) {
3043 tcg_gen_ext_i64(retv, retv, memop);
3044 }
3045 }
3046}
3047
3048static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3049 TCGArg idx, TCGMemOp memop, bool new_val,
3050 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3051{
3052 TCGv_i32 t1 = tcg_temp_new_i32();
3053 TCGv_i32 t2 = tcg_temp_new_i32();
3054
3055 memop = tcg_canonicalize_memop(memop, 0, 0);
3056
3057 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3058 gen(t2, t1, val);
3059 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3060
3061 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3062 tcg_temp_free_i32(t1);
3063 tcg_temp_free_i32(t2);
3064}
3065
3066static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3067 TCGArg idx, TCGMemOp memop, void * const table[])
3068{
3069 gen_atomic_op_i32 gen;
3070
3071 memop = tcg_canonicalize_memop(memop, 0, 0);
3072
3073 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3074 tcg_debug_assert(gen != NULL);
3075
3076#ifdef CONFIG_SOFTMMU
3077 {
3078 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
1c2adb95 3079 gen(ret, cpu_env, addr, val, oi);
c482cb11
RH		 3080 tcg_temp_free_i32(oi);
3081 }
3082#else
1c2adb95 3083 gen(ret, cpu_env, addr, val);
c482cb11
RH		 3084#endif
3085
3086 if (memop & MO_SIGN) {
3087 tcg_gen_ext_i32(ret, ret, memop);
3088 }
3089}
3090
3091static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3092 TCGArg idx, TCGMemOp memop, bool new_val,
3093 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3094{
3095 TCGv_i64 t1 = tcg_temp_new_i64();
3096 TCGv_i64 t2 = tcg_temp_new_i64();
3097
3098 memop = tcg_canonicalize_memop(memop, 1, 0);
3099
3100 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3101 gen(t2, t1, val);
3102 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3103
3104 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3105 tcg_temp_free_i64(t1);
3106 tcg_temp_free_i64(t2);
3107}
3108
3109static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3110 TCGArg idx, TCGMemOp memop, void * const table[])
3111{
3112 memop = tcg_canonicalize_memop(memop, 1, 0);
3113
3114 if ((memop & MO_SIZE) == MO_64) {
df79b996 3115#ifdef CONFIG_ATOMIC64
c482cb11
RH		 3116 gen_atomic_op_i64 gen;
3117
3118 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3119 tcg_debug_assert(gen != NULL);
3120
3121#ifdef CONFIG_SOFTMMU
3122 {
3123 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
1c2adb95 3124 gen(ret, cpu_env, addr, val, oi);
c482cb11
RH		 3125 tcg_temp_free_i32(oi);
3126 }
3127#else
1c2adb95 3128 gen(ret, cpu_env, addr, val);
c482cb11 3129#endif
df79b996 3130#else
1c2adb95 3131 gen_helper_exit_atomic(cpu_env);
79b1af90
RH		 3132 /* Produce a result, so that we have a well-formed opcode stream
3133 with respect to uses of the result in the (dead) code following. */
3134 tcg_gen_movi_i64(ret, 0);
df79b996 3135#endif /* CONFIG_ATOMIC64 */
c482cb11
RH		 3136 } else {
3137 TCGv_i32 v32 = tcg_temp_new_i32();
3138 TCGv_i32 r32 = tcg_temp_new_i32();
3139
3140 tcg_gen_extrl_i64_i32(v32, val);
3141 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3142 tcg_temp_free_i32(v32);
3143
3144 tcg_gen_extu_i32_i64(ret, r32);
3145 tcg_temp_free_i32(r32);
3146
3147 if (memop & MO_SIGN) {
3148 tcg_gen_ext_i64(ret, ret, memop);
3149 }
3150 }
3151}
3152
3153#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3154static void * const table_##NAME[16] = { \
3155 [MO_8] = gen_helper_atomic_##NAME##b, \
3156 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3157 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3158 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3159 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
df79b996
RH		 3160 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3161 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
c482cb11
RH		 3162}; \
3163void tcg_gen_atomic_##NAME##_i32 \
3164 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
3165{ \
b1311c4a 3166 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
c482cb11
RH		 3167 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3168 } else { \
3169 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3170 tcg_gen_##OP##_i32); \
3171 } \
3172} \
3173void tcg_gen_atomic_##NAME##_i64 \
3174 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
3175{ \
b1311c4a 3176 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
c482cb11
RH		 3177 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3178 } else { \
3179 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3180 tcg_gen_##OP##_i64); \
3181 } \
3182}
3183
3184GEN_ATOMIC_HELPER(fetch_add, add, 0)
3185GEN_ATOMIC_HELPER(fetch_and, and, 0)
3186GEN_ATOMIC_HELPER(fetch_or, or, 0)
3187GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
5507c2bf
RH		 3188GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3189GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3190GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3191GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
c482cb11
RH		 3192
3193GEN_ATOMIC_HELPER(add_fetch, add, 1)
3194GEN_ATOMIC_HELPER(and_fetch, and, 1)
3195GEN_ATOMIC_HELPER(or_fetch, or, 1)
3196GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
5507c2bf
RH		 3197GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3198GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3199GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3200GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
c482cb11
RH		 3201
3202static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3203{
3204 tcg_gen_mov_i32(r, b);
3205}
3206
3207static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3208{
3209 tcg_gen_mov_i64(r, b);
3210}
3211
3212GEN_ATOMIC_HELPER(xchg, mov2, 0)
3213
3214#undef GEN_ATOMIC_HELPER
QEMU mirror