]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/tcg-op.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
target/riscv: Allow debugger to access seed CSR
[mirror_qemu.git] / tcg / tcg-op.c
CommitLineData
951c6300
RH		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
757e725b 25#include "qemu/osdep.h"
63c91552 26#include "exec/exec-all.h"
dcb32f1d
PMD		 27#include "tcg/tcg.h"
28#include "tcg/tcg-op.h"
29#include "tcg/tcg-mo.h"
e6d86bed 30#include "exec/plugin-gen.h"
d56fea79 31#include "tcg-internal.h"
951c6300
RH		 32
33
b7e8b17a 34void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
951c6300 35{
d4478943 36 TCGOp *op = tcg_emit_op(opc, 1);
75e8b9b7 37 op->args[0] = a1;
951c6300
RH		 38}
39
b7e8b17a 40void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
951c6300 41{
d4478943 42 TCGOp *op = tcg_emit_op(opc, 2);
75e8b9b7
RH		 43 op->args[0] = a1;
44 op->args[1] = a2;
951c6300
RH		 45}
46
b7e8b17a 47void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
951c6300 48{
d4478943 49 TCGOp *op = tcg_emit_op(opc, 3);
75e8b9b7
RH		 50 op->args[0] = a1;
51 op->args[1] = a2;
52 op->args[2] = a3;
951c6300
RH		 53}
54
b7e8b17a 55void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
951c6300 56{
d4478943 57 TCGOp *op = tcg_emit_op(opc, 4);
75e8b9b7
RH		 58 op->args[0] = a1;
59 op->args[1] = a2;
60 op->args[2] = a3;
61 op->args[3] = a4;
951c6300
RH		 62}
63
b7e8b17a
RH		 64void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
65 TCGArg a4, TCGArg a5)
951c6300 66{
d4478943 67 TCGOp *op = tcg_emit_op(opc, 5);
75e8b9b7
RH		 68 op->args[0] = a1;
69 op->args[1] = a2;
70 op->args[2] = a3;
71 op->args[3] = a4;
72 op->args[4] = a5;
951c6300
RH		 73}
74
b7e8b17a
RH		 75void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
76 TCGArg a4, TCGArg a5, TCGArg a6)
951c6300 77{
d4478943 78 TCGOp *op = tcg_emit_op(opc, 6);
75e8b9b7
RH		 79 op->args[0] = a1;
80 op->args[1] = a2;
81 op->args[2] = a3;
82 op->args[3] = a4;
83 op->args[4] = a5;
84 op->args[5] = a6;
951c6300
RH		 85}
86
f65e19bc
PK		 87void tcg_gen_mb(TCGBar mb_type)
88{
b7e4afbd 89 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) {
b7e8b17a 90 tcg_gen_op1(INDEX_op_mb, mb_type);
f65e19bc
PK		 91 }
92}
93
951c6300
RH		 94/* 32 bit ops */
95
11d11d61
RH		 96void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
97{
98 tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
99}
100
951c6300
RH		 101void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
102{
103 /* some cases can be optimized here */
104 if (arg2 == 0) {
105 tcg_gen_mov_i32(ret, arg1);
106 } else {
11d11d61 107 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 108 }
109}
110
111void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
112{
113 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
114 /* Don't recurse with tcg_gen_neg_i32. */
115 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
116 } else {
11d11d61 117 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
951c6300
RH		 118 }
119}
120
121void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
122{
123 /* some cases can be optimized here */
124 if (arg2 == 0) {
125 tcg_gen_mov_i32(ret, arg1);
126 } else {
11d11d61 127 tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 128 }
129}
130
474b2e8f 131void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 132{
951c6300
RH		 133 /* Some cases can be optimized here. */
134 switch (arg2) {
135 case 0:
136 tcg_gen_movi_i32(ret, 0);
137 return;
474b2e8f 138 case -1:
951c6300
RH		 139 tcg_gen_mov_i32(ret, arg1);
140 return;
474b2e8f 141 case 0xff:
951c6300
RH		 142 /* Don't recurse with tcg_gen_ext8u_i32. */
143 if (TCG_TARGET_HAS_ext8u_i32) {
144 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
145 return;
146 }
147 break;
474b2e8f 148 case 0xffff:
951c6300
RH		 149 if (TCG_TARGET_HAS_ext16u_i32) {
150 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
151 return;
152 }
153 break;
154 }
11d11d61
RH		 155
156 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 157}
158
159void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
160{
161 /* Some cases can be optimized here. */
162 if (arg2 == -1) {
163 tcg_gen_movi_i32(ret, -1);
164 } else if (arg2 == 0) {
165 tcg_gen_mov_i32(ret, arg1);
166 } else {
11d11d61 167 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 168 }
169}
170
171void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
172{
173 /* Some cases can be optimized here. */
174 if (arg2 == 0) {
175 tcg_gen_mov_i32(ret, arg1);
176 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
177 /* Don't recurse with tcg_gen_not_i32. */
178 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
179 } else {
11d11d61 180 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 181 }
182}
183
474b2e8f 184void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 185{
474b2e8f 186 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 187 if (arg2 == 0) {
188 tcg_gen_mov_i32(ret, arg1);
189 } else {
11d11d61 190 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 191 }
192}
193
474b2e8f 194void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 195{
474b2e8f 196 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 197 if (arg2 == 0) {
198 tcg_gen_mov_i32(ret, arg1);
199 } else {
11d11d61 200 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 201 }
202}
203
474b2e8f 204void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 205{
474b2e8f 206 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 207 if (arg2 == 0) {
208 tcg_gen_mov_i32(ret, arg1);
209 } else {
11d11d61 210 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 211 }
212}
213
42a268c2 214void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
951c6300
RH		 215{
216 if (cond == TCG_COND_ALWAYS) {
42a268c2 217 tcg_gen_br(l);
951c6300 218 } else if (cond != TCG_COND_NEVER) {
d88a117e 219 l->refs++;
42a268c2 220 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
951c6300
RH		 221 }
222}
223
42a268c2 224void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
951c6300 225{
37ed3bf1
RH		 226 if (cond == TCG_COND_ALWAYS) {
227 tcg_gen_br(l);
228 } else if (cond != TCG_COND_NEVER) {
11d11d61 229 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
37ed3bf1 230 }
951c6300
RH		 231}
232
233void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
234 TCGv_i32 arg1, TCGv_i32 arg2)
235{
236 if (cond == TCG_COND_ALWAYS) {
237 tcg_gen_movi_i32(ret, 1);
238 } else if (cond == TCG_COND_NEVER) {
239 tcg_gen_movi_i32(ret, 0);
240 } else {
241 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
242 }
243}
244
245void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
246 TCGv_i32 arg1, int32_t arg2)
247{
11d11d61 248 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 249}
250
251void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
252{
b2e3ae94
RH		 253 if (arg2 == 0) {
254 tcg_gen_movi_i32(ret, 0);
255 } else if (is_power_of_2(arg2)) {
256 tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
257 } else {
11d11d61 258 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
b2e3ae94 259 }
951c6300
RH		 260}
261
262void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
263{
264 if (TCG_TARGET_HAS_div_i32) {
265 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
266 } else if (TCG_TARGET_HAS_div2_i32) {
267 TCGv_i32 t0 = tcg_temp_new_i32();
268 tcg_gen_sari_i32(t0, arg1, 31);
269 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
270 tcg_temp_free_i32(t0);
271 } else {
272 gen_helper_div_i32(ret, arg1, arg2);
273 }
274}
275
276void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
277{
278 if (TCG_TARGET_HAS_rem_i32) {
279 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
280 } else if (TCG_TARGET_HAS_div_i32) {
281 TCGv_i32 t0 = tcg_temp_new_i32();
282 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
283 tcg_gen_mul_i32(t0, t0, arg2);
284 tcg_gen_sub_i32(ret, arg1, t0);
285 tcg_temp_free_i32(t0);
286 } else if (TCG_TARGET_HAS_div2_i32) {
287 TCGv_i32 t0 = tcg_temp_new_i32();
288 tcg_gen_sari_i32(t0, arg1, 31);
289 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
290 tcg_temp_free_i32(t0);
291 } else {
292 gen_helper_rem_i32(ret, arg1, arg2);
293 }
294}
295
296void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
297{
298 if (TCG_TARGET_HAS_div_i32) {
299 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
300 } else if (TCG_TARGET_HAS_div2_i32) {
301 TCGv_i32 t0 = tcg_temp_new_i32();
302 tcg_gen_movi_i32(t0, 0);
303 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
304 tcg_temp_free_i32(t0);
305 } else {
306 gen_helper_divu_i32(ret, arg1, arg2);
307 }
308}
309
310void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
311{
312 if (TCG_TARGET_HAS_rem_i32) {
313 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
314 } else if (TCG_TARGET_HAS_div_i32) {
315 TCGv_i32 t0 = tcg_temp_new_i32();
316 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
317 tcg_gen_mul_i32(t0, t0, arg2);
318 tcg_gen_sub_i32(ret, arg1, t0);
319 tcg_temp_free_i32(t0);
320 } else if (TCG_TARGET_HAS_div2_i32) {
321 TCGv_i32 t0 = tcg_temp_new_i32();
322 tcg_gen_movi_i32(t0, 0);
323 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
324 tcg_temp_free_i32(t0);
325 } else {
326 gen_helper_remu_i32(ret, arg1, arg2);
327 }
328}
329
330void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
331{
332 if (TCG_TARGET_HAS_andc_i32) {
333 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
334 } else {
335 TCGv_i32 t0 = tcg_temp_new_i32();
336 tcg_gen_not_i32(t0, arg2);
337 tcg_gen_and_i32(ret, arg1, t0);
338 tcg_temp_free_i32(t0);
339 }
340}
341
342void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
343{
344 if (TCG_TARGET_HAS_eqv_i32) {
345 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
346 } else {
347 tcg_gen_xor_i32(ret, arg1, arg2);
348 tcg_gen_not_i32(ret, ret);
349 }
350}
351
352void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
353{
354 if (TCG_TARGET_HAS_nand_i32) {
355 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
356 } else {
357 tcg_gen_and_i32(ret, arg1, arg2);
358 tcg_gen_not_i32(ret, ret);
359 }
360}
361
362void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
363{
364 if (TCG_TARGET_HAS_nor_i32) {
365 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
366 } else {
367 tcg_gen_or_i32(ret, arg1, arg2);
368 tcg_gen_not_i32(ret, ret);
369 }
370}
371
372void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
373{
374 if (TCG_TARGET_HAS_orc_i32) {
375 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
376 } else {
377 TCGv_i32 t0 = tcg_temp_new_i32();
378 tcg_gen_not_i32(t0, arg2);
379 tcg_gen_or_i32(ret, arg1, t0);
380 tcg_temp_free_i32(t0);
381 }
382}
383
0e28d006
RH		 384void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
385{
386 if (TCG_TARGET_HAS_clz_i32) {
387 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
388 } else if (TCG_TARGET_HAS_clz_i64) {
389 TCGv_i64 t1 = tcg_temp_new_i64();
390 TCGv_i64 t2 = tcg_temp_new_i64();
391 tcg_gen_extu_i32_i64(t1, arg1);
392 tcg_gen_extu_i32_i64(t2, arg2);
393 tcg_gen_addi_i64(t2, t2, 32);
394 tcg_gen_clz_i64(t1, t1, t2);
395 tcg_gen_extrl_i64_i32(ret, t1);
396 tcg_temp_free_i64(t1);
397 tcg_temp_free_i64(t2);
398 tcg_gen_subi_i32(ret, ret, 32);
399 } else {
400 gen_helper_clz_i32(ret, arg1, arg2);
401 }
402}
403
404void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
405{
11d11d61 406 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
0e28d006
RH		 407}
408
409void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
410{
411 if (TCG_TARGET_HAS_ctz_i32) {
412 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
413 } else if (TCG_TARGET_HAS_ctz_i64) {
414 TCGv_i64 t1 = tcg_temp_new_i64();
415 TCGv_i64 t2 = tcg_temp_new_i64();
416 tcg_gen_extu_i32_i64(t1, arg1);
417 tcg_gen_extu_i32_i64(t2, arg2);
418 tcg_gen_ctz_i64(t1, t1, t2);
419 tcg_gen_extrl_i64_i32(ret, t1);
420 tcg_temp_free_i64(t1);
421 tcg_temp_free_i64(t2);
14e99210
RH		 422 } else if (TCG_TARGET_HAS_ctpop_i32
423 || TCG_TARGET_HAS_ctpop_i64
424 || TCG_TARGET_HAS_clz_i32
425 || TCG_TARGET_HAS_clz_i64) {
426 TCGv_i32 z, t = tcg_temp_new_i32();
427
428 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
429 tcg_gen_subi_i32(t, arg1, 1);
430 tcg_gen_andc_i32(t, t, arg1);
431 tcg_gen_ctpop_i32(t, t);
432 } else {
433 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
434 tcg_gen_neg_i32(t, arg1);
435 tcg_gen_and_i32(t, t, arg1);
436 tcg_gen_clzi_i32(t, t, 32);
437 tcg_gen_xori_i32(t, t, 31);
438 }
11d11d61 439 z = tcg_constant_i32(0);
14e99210
RH		 440 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
441 tcg_temp_free_i32(t);
0e28d006
RH		 442 } else {
443 gen_helper_ctz_i32(ret, arg1, arg2);
444 }
445}
446
447void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
448{
14e99210
RH		 449 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
450 /* This equivalence has the advantage of not requiring a fixup. */
451 TCGv_i32 t = tcg_temp_new_i32();
452 tcg_gen_subi_i32(t, arg1, 1);
453 tcg_gen_andc_i32(t, t, arg1);
454 tcg_gen_ctpop_i32(ret, t);
455 tcg_temp_free_i32(t);
456 } else {
11d11d61 457 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
14e99210 458 }
0e28d006
RH		 459}
460
086920c2
RH		 461void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
462{
463 if (TCG_TARGET_HAS_clz_i32) {
464 TCGv_i32 t = tcg_temp_new_i32();
465 tcg_gen_sari_i32(t, arg, 31);
466 tcg_gen_xor_i32(t, t, arg);
467 tcg_gen_clzi_i32(t, t, 32);
468 tcg_gen_subi_i32(ret, t, 1);
469 tcg_temp_free_i32(t);
470 } else {
471 gen_helper_clrsb_i32(ret, arg);
472 }
473}
474
a768e4e9
RH		 475void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
476{
477 if (TCG_TARGET_HAS_ctpop_i32) {
478 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
479 } else if (TCG_TARGET_HAS_ctpop_i64) {
480 TCGv_i64 t = tcg_temp_new_i64();
481 tcg_gen_extu_i32_i64(t, arg1);
482 tcg_gen_ctpop_i64(t, t);
483 tcg_gen_extrl_i64_i32(ret, t);
484 tcg_temp_free_i64(t);
485 } else {
486 gen_helper_ctpop_i32(ret, arg1);
487 }
488}
489
951c6300
RH		 490void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
491{
492 if (TCG_TARGET_HAS_rot_i32) {
493 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
494 } else {
495 TCGv_i32 t0, t1;
496
497 t0 = tcg_temp_new_i32();
498 t1 = tcg_temp_new_i32();
499 tcg_gen_shl_i32(t0, arg1, arg2);
500 tcg_gen_subfi_i32(t1, 32, arg2);
501 tcg_gen_shr_i32(t1, arg1, t1);
502 tcg_gen_or_i32(ret, t0, t1);
503 tcg_temp_free_i32(t0);
504 tcg_temp_free_i32(t1);
505 }
506}
507
07dada03 508void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 509{
07dada03 510 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 511 /* some cases can be optimized here */
512 if (arg2 == 0) {
513 tcg_gen_mov_i32(ret, arg1);
514 } else if (TCG_TARGET_HAS_rot_i32) {
11d11d61 515 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2));
951c6300
RH		 516 } else {
517 TCGv_i32 t0, t1;
518 t0 = tcg_temp_new_i32();
519 t1 = tcg_temp_new_i32();
520 tcg_gen_shli_i32(t0, arg1, arg2);
521 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
522 tcg_gen_or_i32(ret, t0, t1);
523 tcg_temp_free_i32(t0);
524 tcg_temp_free_i32(t1);
525 }
526}
527
528void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
529{
530 if (TCG_TARGET_HAS_rot_i32) {
531 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
532 } else {
533 TCGv_i32 t0, t1;
534
535 t0 = tcg_temp_new_i32();
536 t1 = tcg_temp_new_i32();
537 tcg_gen_shr_i32(t0, arg1, arg2);
538 tcg_gen_subfi_i32(t1, 32, arg2);
539 tcg_gen_shl_i32(t1, arg1, t1);
540 tcg_gen_or_i32(ret, t0, t1);
541 tcg_temp_free_i32(t0);
542 tcg_temp_free_i32(t1);
543 }
544}
545
07dada03 546void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 547{
07dada03 548 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 549 /* some cases can be optimized here */
550 if (arg2 == 0) {
551 tcg_gen_mov_i32(ret, arg1);
552 } else {
553 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
554 }
555}
556
557void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
558 unsigned int ofs, unsigned int len)
559{
560 uint32_t mask;
561 TCGv_i32 t1;
562
563 tcg_debug_assert(ofs < 32);
0d0d309d 564 tcg_debug_assert(len > 0);
951c6300
RH		 565 tcg_debug_assert(len <= 32);
566 tcg_debug_assert(ofs + len <= 32);
567
0d0d309d 568 if (len == 32) {
951c6300
RH		 569 tcg_gen_mov_i32(ret, arg2);
570 return;
571 }
572 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
573 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
574 return;
575 }
576
951c6300
RH		 577 t1 = tcg_temp_new_i32();
578
b0a60567
RH		 579 if (TCG_TARGET_HAS_extract2_i32) {
580 if (ofs + len == 32) {
581 tcg_gen_shli_i32(t1, arg1, len);
582 tcg_gen_extract2_i32(ret, t1, arg2, len);
583 goto done;
584 }
585 if (ofs == 0) {
586 tcg_gen_extract2_i32(ret, arg1, arg2, len);
587 tcg_gen_rotli_i32(ret, ret, len);
588 goto done;
589 }
590 }
591
592 mask = (1u << len) - 1;
951c6300
RH		 593 if (ofs + len < 32) {
594 tcg_gen_andi_i32(t1, arg2, mask);
595 tcg_gen_shli_i32(t1, t1, ofs);
596 } else {
597 tcg_gen_shli_i32(t1, arg2, ofs);
598 }
599 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
600 tcg_gen_or_i32(ret, ret, t1);
b0a60567 601 done:
951c6300
RH		 602 tcg_temp_free_i32(t1);
603}
604
07cc68d5
RH		 605void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
606 unsigned int ofs, unsigned int len)
607{
608 tcg_debug_assert(ofs < 32);
609 tcg_debug_assert(len > 0);
610 tcg_debug_assert(len <= 32);
611 tcg_debug_assert(ofs + len <= 32);
612
613 if (ofs + len == 32) {
614 tcg_gen_shli_i32(ret, arg, ofs);
615 } else if (ofs == 0) {
616 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
617 } else if (TCG_TARGET_HAS_deposit_i32
618 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
11d11d61 619 TCGv_i32 zero = tcg_constant_i32(0);
07cc68d5 620 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
07cc68d5
RH		 621 } else {
622 /* To help two-operand hosts we prefer to zero-extend first,
623 which allows ARG to stay live. */
624 switch (len) {
625 case 16:
626 if (TCG_TARGET_HAS_ext16u_i32) {
627 tcg_gen_ext16u_i32(ret, arg);
628 tcg_gen_shli_i32(ret, ret, ofs);
629 return;
630 }
631 break;
632 case 8:
633 if (TCG_TARGET_HAS_ext8u_i32) {
634 tcg_gen_ext8u_i32(ret, arg);
635 tcg_gen_shli_i32(ret, ret, ofs);
636 return;
637 }
638 break;
639 }
640 /* Otherwise prefer zero-extension over AND for code size. */
641 switch (ofs + len) {
642 case 16:
643 if (TCG_TARGET_HAS_ext16u_i32) {
644 tcg_gen_shli_i32(ret, arg, ofs);
645 tcg_gen_ext16u_i32(ret, ret);
646 return;
647 }
648 break;
649 case 8:
650 if (TCG_TARGET_HAS_ext8u_i32) {
651 tcg_gen_shli_i32(ret, arg, ofs);
652 tcg_gen_ext8u_i32(ret, ret);
653 return;
654 }
655 break;
656 }
657 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
658 tcg_gen_shli_i32(ret, ret, ofs);
659 }
660}
661
7ec8bab3
RH		 662void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
663 unsigned int ofs, unsigned int len)
664{
665 tcg_debug_assert(ofs < 32);
666 tcg_debug_assert(len > 0);
667 tcg_debug_assert(len <= 32);
668 tcg_debug_assert(ofs + len <= 32);
669
670 /* Canonicalize certain special cases, even if extract is supported. */
671 if (ofs + len == 32) {
672 tcg_gen_shri_i32(ret, arg, 32 - len);
673 return;
674 }
675 if (ofs == 0) {
676 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
677 return;
678 }
679
680 if (TCG_TARGET_HAS_extract_i32
681 && TCG_TARGET_extract_i32_valid(ofs, len)) {
682 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
683 return;
684 }
685
686 /* Assume that zero-extension, if available, is cheaper than a shift. */
687 switch (ofs + len) {
688 case 16:
689 if (TCG_TARGET_HAS_ext16u_i32) {
690 tcg_gen_ext16u_i32(ret, arg);
691 tcg_gen_shri_i32(ret, ret, ofs);
692 return;
693 }
694 break;
695 case 8:
696 if (TCG_TARGET_HAS_ext8u_i32) {
697 tcg_gen_ext8u_i32(ret, arg);
698 tcg_gen_shri_i32(ret, ret, ofs);
699 return;
700 }
701 break;
702 }
703
704 /* ??? Ideally we'd know what values are available for immediate AND.
705 Assume that 8 bits are available, plus the special case of 16,
706 so that we get ext8u, ext16u. */
707 switch (len) {
708 case 1 ... 8: case 16:
709 tcg_gen_shri_i32(ret, arg, ofs);
710 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
711 break;
712 default:
713 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
714 tcg_gen_shri_i32(ret, ret, 32 - len);
715 break;
716 }
717}
718
719void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
720 unsigned int ofs, unsigned int len)
721{
722 tcg_debug_assert(ofs < 32);
723 tcg_debug_assert(len > 0);
724 tcg_debug_assert(len <= 32);
725 tcg_debug_assert(ofs + len <= 32);
726
727 /* Canonicalize certain special cases, even if extract is supported. */
728 if (ofs + len == 32) {
729 tcg_gen_sari_i32(ret, arg, 32 - len);
730 return;
731 }
732 if (ofs == 0) {
733 switch (len) {
734 case 16:
735 tcg_gen_ext16s_i32(ret, arg);
736 return;
737 case 8:
738 tcg_gen_ext8s_i32(ret, arg);
739 return;
740 }
741 }
742
743 if (TCG_TARGET_HAS_sextract_i32
744 && TCG_TARGET_extract_i32_valid(ofs, len)) {
745 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
746 return;
747 }
748
749 /* Assume that sign-extension, if available, is cheaper than a shift. */
750 switch (ofs + len) {
751 case 16:
752 if (TCG_TARGET_HAS_ext16s_i32) {
753 tcg_gen_ext16s_i32(ret, arg);
754 tcg_gen_sari_i32(ret, ret, ofs);
755 return;
756 }
757 break;
758 case 8:
759 if (TCG_TARGET_HAS_ext8s_i32) {
760 tcg_gen_ext8s_i32(ret, arg);
761 tcg_gen_sari_i32(ret, ret, ofs);
762 return;
763 }
764 break;
765 }
766 switch (len) {
767 case 16:
768 if (TCG_TARGET_HAS_ext16s_i32) {
769 tcg_gen_shri_i32(ret, arg, ofs);
770 tcg_gen_ext16s_i32(ret, ret);
771 return;
772 }
773 break;
774 case 8:
775 if (TCG_TARGET_HAS_ext8s_i32) {
776 tcg_gen_shri_i32(ret, arg, ofs);
777 tcg_gen_ext8s_i32(ret, ret);
778 return;
779 }
780 break;
781 }
782
783 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
784 tcg_gen_sari_i32(ret, ret, 32 - len);
785}
786
2089fcc9
DH		 787/*
788 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
789 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
790 */
791void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
792 unsigned int ofs)
793{
794 tcg_debug_assert(ofs <= 32);
795 if (ofs == 0) {
796 tcg_gen_mov_i32(ret, al);
797 } else if (ofs == 32) {
798 tcg_gen_mov_i32(ret, ah);
799 } else if (al == ah) {
800 tcg_gen_rotri_i32(ret, al, ofs);
fce1296f
RH		 801 } else if (TCG_TARGET_HAS_extract2_i32) {
802 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
2089fcc9
DH		 803 } else {
804 TCGv_i32 t0 = tcg_temp_new_i32();
805 tcg_gen_shri_i32(t0, al, ofs);
806 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
807 tcg_temp_free_i32(t0);
808 }
809}
810
951c6300
RH		 811void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
812 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
813{
37ed3bf1
RH		 814 if (cond == TCG_COND_ALWAYS) {
815 tcg_gen_mov_i32(ret, v1);
816 } else if (cond == TCG_COND_NEVER) {
817 tcg_gen_mov_i32(ret, v2);
818 } else if (TCG_TARGET_HAS_movcond_i32) {
951c6300
RH		 819 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
820 } else {
821 TCGv_i32 t0 = tcg_temp_new_i32();
822 TCGv_i32 t1 = tcg_temp_new_i32();
823 tcg_gen_setcond_i32(cond, t0, c1, c2);
824 tcg_gen_neg_i32(t0, t0);
825 tcg_gen_and_i32(t1, v1, t0);
826 tcg_gen_andc_i32(ret, v2, t0);
827 tcg_gen_or_i32(ret, ret, t1);
828 tcg_temp_free_i32(t0);
829 tcg_temp_free_i32(t1);
830 }
831}
832
833void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
834 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
835{
836 if (TCG_TARGET_HAS_add2_i32) {
837 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 838 } else {
839 TCGv_i64 t0 = tcg_temp_new_i64();
840 TCGv_i64 t1 = tcg_temp_new_i64();
841 tcg_gen_concat_i32_i64(t0, al, ah);
842 tcg_gen_concat_i32_i64(t1, bl, bh);
843 tcg_gen_add_i64(t0, t0, t1);
844 tcg_gen_extr_i64_i32(rl, rh, t0);
845 tcg_temp_free_i64(t0);
846 tcg_temp_free_i64(t1);
847 }
848}
849
850void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
851 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
852{
853 if (TCG_TARGET_HAS_sub2_i32) {
854 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 855 } else {
856 TCGv_i64 t0 = tcg_temp_new_i64();
857 TCGv_i64 t1 = tcg_temp_new_i64();
858 tcg_gen_concat_i32_i64(t0, al, ah);
859 tcg_gen_concat_i32_i64(t1, bl, bh);
860 tcg_gen_sub_i64(t0, t0, t1);
861 tcg_gen_extr_i64_i32(rl, rh, t0);
862 tcg_temp_free_i64(t0);
863 tcg_temp_free_i64(t1);
864 }
865}
866
867void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
868{
869 if (TCG_TARGET_HAS_mulu2_i32) {
870 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
951c6300
RH		 871 } else if (TCG_TARGET_HAS_muluh_i32) {
872 TCGv_i32 t = tcg_temp_new_i32();
873 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
874 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
875 tcg_gen_mov_i32(rl, t);
876 tcg_temp_free_i32(t);
9fd86b51 877 } else if (TCG_TARGET_REG_BITS == 64) {
951c6300
RH		 878 TCGv_i64 t0 = tcg_temp_new_i64();
879 TCGv_i64 t1 = tcg_temp_new_i64();
880 tcg_gen_extu_i32_i64(t0, arg1);
881 tcg_gen_extu_i32_i64(t1, arg2);
882 tcg_gen_mul_i64(t0, t0, t1);
883 tcg_gen_extr_i64_i32(rl, rh, t0);
884 tcg_temp_free_i64(t0);
885 tcg_temp_free_i64(t1);
9fd86b51
RH		 886 } else {
887 qemu_build_not_reached();
951c6300
RH		 888 }
889}
890
891void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
892{
893 if (TCG_TARGET_HAS_muls2_i32) {
894 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
951c6300
RH		 895 } else if (TCG_TARGET_HAS_mulsh_i32) {
896 TCGv_i32 t = tcg_temp_new_i32();
897 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
898 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
899 tcg_gen_mov_i32(rl, t);
900 tcg_temp_free_i32(t);
901 } else if (TCG_TARGET_REG_BITS == 32) {
902 TCGv_i32 t0 = tcg_temp_new_i32();
903 TCGv_i32 t1 = tcg_temp_new_i32();
904 TCGv_i32 t2 = tcg_temp_new_i32();
905 TCGv_i32 t3 = tcg_temp_new_i32();
906 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
907 /* Adjust for negative inputs. */
908 tcg_gen_sari_i32(t2, arg1, 31);
909 tcg_gen_sari_i32(t3, arg2, 31);
910 tcg_gen_and_i32(t2, t2, arg2);
911 tcg_gen_and_i32(t3, t3, arg1);
912 tcg_gen_sub_i32(rh, t1, t2);
913 tcg_gen_sub_i32(rh, rh, t3);
914 tcg_gen_mov_i32(rl, t0);
915 tcg_temp_free_i32(t0);
916 tcg_temp_free_i32(t1);
917 tcg_temp_free_i32(t2);
918 tcg_temp_free_i32(t3);
919 } else {
920 TCGv_i64 t0 = tcg_temp_new_i64();
921 TCGv_i64 t1 = tcg_temp_new_i64();
922 tcg_gen_ext_i32_i64(t0, arg1);
923 tcg_gen_ext_i32_i64(t1, arg2);
924 tcg_gen_mul_i64(t0, t0, t1);
925 tcg_gen_extr_i64_i32(rl, rh, t0);
926 tcg_temp_free_i64(t0);
927 tcg_temp_free_i64(t1);
928 }
929}
930
5087abfb
RH		 931void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
932{
933 if (TCG_TARGET_REG_BITS == 32) {
934 TCGv_i32 t0 = tcg_temp_new_i32();
935 TCGv_i32 t1 = tcg_temp_new_i32();
936 TCGv_i32 t2 = tcg_temp_new_i32();
937 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
938 /* Adjust for negative input for the signed arg1. */
939 tcg_gen_sari_i32(t2, arg1, 31);
940 tcg_gen_and_i32(t2, t2, arg2);
941 tcg_gen_sub_i32(rh, t1, t2);
942 tcg_gen_mov_i32(rl, t0);
943 tcg_temp_free_i32(t0);
944 tcg_temp_free_i32(t1);
945 tcg_temp_free_i32(t2);
946 } else {
947 TCGv_i64 t0 = tcg_temp_new_i64();
948 TCGv_i64 t1 = tcg_temp_new_i64();
949 tcg_gen_ext_i32_i64(t0, arg1);
950 tcg_gen_extu_i32_i64(t1, arg2);
951 tcg_gen_mul_i64(t0, t0, t1);
952 tcg_gen_extr_i64_i32(rl, rh, t0);
953 tcg_temp_free_i64(t0);
954 tcg_temp_free_i64(t1);
955 }
956}
957
951c6300
RH		 958void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
959{
960 if (TCG_TARGET_HAS_ext8s_i32) {
961 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
962 } else {
963 tcg_gen_shli_i32(ret, arg, 24);
964 tcg_gen_sari_i32(ret, ret, 24);
965 }
966}
967
968void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
969{
970 if (TCG_TARGET_HAS_ext16s_i32) {
971 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
972 } else {
973 tcg_gen_shli_i32(ret, arg, 16);
974 tcg_gen_sari_i32(ret, ret, 16);
975 }
976}
977
978void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
979{
980 if (TCG_TARGET_HAS_ext8u_i32) {
981 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
982 } else {
983 tcg_gen_andi_i32(ret, arg, 0xffu);
984 }
985}
986
987void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
988{
989 if (TCG_TARGET_HAS_ext16u_i32) {
990 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
991 } else {
992 tcg_gen_andi_i32(ret, arg, 0xffffu);
993 }
994}
995
2b836c2a 996void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
951c6300 997{
2b836c2a
RH		 998 /* Only one extension flag may be present. */
999 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1000
951c6300 1001 if (TCG_TARGET_HAS_bswap16_i32) {
2b836c2a 1002 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags);
951c6300
RH		 1003 } else {
1004 TCGv_i32 t0 = tcg_temp_new_i32();
2b836c2a
RH		 1005 TCGv_i32 t1 = tcg_temp_new_i32();
1006
1007 tcg_gen_shri_i32(t0, arg, 8);
1008 if (!(flags & TCG_BSWAP_IZ)) {
1009 tcg_gen_ext8u_i32(t0, t0);
1010 }
951c6300 1011
2b836c2a
RH		 1012 if (flags & TCG_BSWAP_OS) {
1013 tcg_gen_shli_i32(t1, arg, 24);
1014 tcg_gen_sari_i32(t1, t1, 16);
1015 } else if (flags & TCG_BSWAP_OZ) {
1016 tcg_gen_ext8u_i32(t1, arg);
1017 tcg_gen_shli_i32(t1, t1, 8);
1018 } else {
1019 tcg_gen_shli_i32(t1, arg, 8);
1020 }
1021
1022 tcg_gen_or_i32(ret, t0, t1);
951c6300 1023 tcg_temp_free_i32(t0);
2b836c2a 1024 tcg_temp_free_i32(t1);
951c6300
RH		 1025 }
1026}
1027
1028void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1029{
1030 if (TCG_TARGET_HAS_bswap32_i32) {
587195bd 1031 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0);
951c6300 1032 } else {
a686dc71
RH		 1033 TCGv_i32 t0 = tcg_temp_new_i32();
1034 TCGv_i32 t1 = tcg_temp_new_i32();
11d11d61 1035 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff);
951c6300 1036
a686dc71
RH		 1037 /* arg = abcd */
1038 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */
1039 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */
1040 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */
1041 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */
1042 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */
951c6300 1043
a686dc71
RH		 1044 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */
1045 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */
1046 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */
951c6300 1047
951c6300
RH		 1048 tcg_temp_free_i32(t0);
1049 tcg_temp_free_i32(t1);
1050 }
1051}
1052
46be8425
RH		 1053void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg)
1054{
1055 /* Swapping 2 16-bit elements is a rotate. */
1056 tcg_gen_rotli_i32(ret, arg, 16);
1057}
1058
b87fb8cd
RH		 1059void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1060{
1061 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1062}
1063
1064void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1065{
1066 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1067}
1068
1069void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1070{
1071 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1072}
1073
1074void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1075{
1076 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1077}
1078
ff1f11f7
RH		 1079void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1080{
1081 TCGv_i32 t = tcg_temp_new_i32();
1082
1083 tcg_gen_sari_i32(t, a, 31);
1084 tcg_gen_xor_i32(ret, a, t);
1085 tcg_gen_sub_i32(ret, ret, t);
1086 tcg_temp_free_i32(t);
1087}
1088
951c6300
RH		 1089/* 64-bit ops */
1090
1091#if TCG_TARGET_REG_BITS == 32
1092/* These are all inline for TCG_TARGET_REG_BITS == 64. */
1093
1094void tcg_gen_discard_i64(TCGv_i64 arg)
1095{
1096 tcg_gen_discard_i32(TCGV_LOW(arg));
1097 tcg_gen_discard_i32(TCGV_HIGH(arg));
1098}
1099
1100void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1101{
11d11d61
RH		 1102 TCGTemp *ts = tcgv_i64_temp(arg);
1103
1104 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1105 if (ts->kind == TEMP_CONST) {
1106 tcg_gen_movi_i64(ret, ts->val);
1107 } else {
1108 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1109 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1110 }
951c6300
RH		 1111}
1112
1113void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1114{
1115 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1116 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1117}
1118
1119void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1120{
1121 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1122 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1123}
1124
1125void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1126{
1127 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
3ff91d7e 1128 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
951c6300
RH		 1129}
1130
1131void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1132{
1133 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1134 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1135}
1136
1137void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1138{
1139 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1140 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1141}
1142
1143void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1144{
1145 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1146 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1147}
1148
1149void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1150{
1151 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1152 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1153}
1154
1155void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1156{
1157 /* Since arg2 and ret have different types,
1158 they cannot be the same temporary */
e03b5686 1159#if HOST_BIG_ENDIAN
951c6300
RH		 1160 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1161 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1162#else
1163 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1164 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1165#endif
1166}
1167
d56fea79
RH		 1168void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1169{
1170 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset);
1171}
1172
1173void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1174{
1175 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset);
1176}
1177
1178void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1179{
1180 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1181}
1182
951c6300
RH		 1183void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1184{
e03b5686 1185#if HOST_BIG_ENDIAN
951c6300
RH		 1186 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1187 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1188#else
1189 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1190 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1191#endif
1192}
1193
d56fea79
RH		 1194void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1195{
1196 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1),
1197 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2));
1198}
1199
1200void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1201{
1202 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1),
1203 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2));
1204}
1205
951c6300
RH		 1206void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1207{
1208 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1209 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1210}
1211
1212void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1213{
1214 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1215 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1216}
1217
1218void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1219{
1220 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1221 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1222}
1223
1224void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1225{
1226 gen_helper_shl_i64(ret, arg1, arg2);
1227}
1228
1229void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1230{
1231 gen_helper_shr_i64(ret, arg1, arg2);
1232}
1233
1234void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1235{
1236 gen_helper_sar_i64(ret, arg1, arg2);
1237}
1238
1239void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1240{
1241 TCGv_i64 t0;
1242 TCGv_i32 t1;
1243
1244 t0 = tcg_temp_new_i64();
1245 t1 = tcg_temp_new_i32();
1246
1247 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1248 TCGV_LOW(arg1), TCGV_LOW(arg2));
1249
1250 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1251 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1252 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1253 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1254
1255 tcg_gen_mov_i64(ret, t0);
1256 tcg_temp_free_i64(t0);
1257 tcg_temp_free_i32(t1);
1258}
11d11d61
RH		 1259
1260#else
1261
1262void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1263{
1264 tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
1265}
1266
951c6300
RH		 1267#endif /* TCG_TARGET_REG_SIZE == 32 */
1268
1269void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1270{
1271 /* some cases can be optimized here */
1272 if (arg2 == 0) {
1273 tcg_gen_mov_i64(ret, arg1);
11d11d61
RH		 1274 } else if (TCG_TARGET_REG_BITS == 64) {
1275 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300 1276 } else {
11d11d61
RH		 1277 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1278 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1279 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
951c6300
RH		 1280 }
1281}
1282
1283void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1284{
1285 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1286 /* Don't recurse with tcg_gen_neg_i64. */
1287 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
11d11d61
RH		 1288 } else if (TCG_TARGET_REG_BITS == 64) {
1289 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
951c6300 1290 } else {
11d11d61
RH		 1291 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1292 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32),
1293 TCGV_LOW(arg2), TCGV_HIGH(arg2));
951c6300
RH		 1294 }
1295}
1296
1297void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1298{
1299 /* some cases can be optimized here */
1300 if (arg2 == 0) {
1301 tcg_gen_mov_i64(ret, arg1);
11d11d61
RH		 1302 } else if (TCG_TARGET_REG_BITS == 64) {
1303 tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300 1304 } else {
11d11d61
RH		 1305 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1306 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1307 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
951c6300
RH		 1308 }
1309}
1310
474b2e8f 1311void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1312{
3a13c3f3
RH		 1313 if (TCG_TARGET_REG_BITS == 32) {
1314 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1315 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1316 return;
1317 }
1318
951c6300
RH		 1319 /* Some cases can be optimized here. */
1320 switch (arg2) {
1321 case 0:
1322 tcg_gen_movi_i64(ret, 0);
1323 return;
474b2e8f 1324 case -1:
951c6300
RH		 1325 tcg_gen_mov_i64(ret, arg1);
1326 return;
474b2e8f 1327 case 0xff:
951c6300
RH		 1328 /* Don't recurse with tcg_gen_ext8u_i64. */
1329 if (TCG_TARGET_HAS_ext8u_i64) {
1330 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1331 return;
1332 }
1333 break;
474b2e8f 1334 case 0xffff:
951c6300
RH		 1335 if (TCG_TARGET_HAS_ext16u_i64) {
1336 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1337 return;
1338 }
1339 break;
474b2e8f 1340 case 0xffffffffu:
951c6300
RH		 1341 if (TCG_TARGET_HAS_ext32u_i64) {
1342 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1343 return;
1344 }
1345 break;
1346 }
11d11d61
RH		 1347
1348 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 1349}
1350
1351void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1352{
3a13c3f3
RH		 1353 if (TCG_TARGET_REG_BITS == 32) {
1354 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1355 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1356 return;
1357 }
951c6300
RH		 1358 /* Some cases can be optimized here. */
1359 if (arg2 == -1) {
1360 tcg_gen_movi_i64(ret, -1);
1361 } else if (arg2 == 0) {
1362 tcg_gen_mov_i64(ret, arg1);
1363 } else {
11d11d61 1364 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300 1365 }
951c6300
RH		 1366}
1367
1368void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1369{
3a13c3f3
RH		 1370 if (TCG_TARGET_REG_BITS == 32) {
1371 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1372 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1373 return;
1374 }
951c6300
RH		 1375 /* Some cases can be optimized here. */
1376 if (arg2 == 0) {
1377 tcg_gen_mov_i64(ret, arg1);
1378 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1379 /* Don't recurse with tcg_gen_not_i64. */
1380 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1381 } else {
11d11d61 1382 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300 1383 }
951c6300
RH		 1384}
1385
951c6300
RH		 1386static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1387 unsigned c, bool right, bool arith)
1388{
1389 tcg_debug_assert(c < 64);
1390 if (c == 0) {
1391 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1392 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1393 } else if (c >= 32) {
1394 c -= 32;
1395 if (right) {
1396 if (arith) {
1397 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1398 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1399 } else {
1400 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1401 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1402 }
1403 } else {
1404 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1405 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1406 }
02616bad
RH		 1407 } else if (right) {
1408 if (TCG_TARGET_HAS_extract2_i32) {
1409 tcg_gen_extract2_i32(TCGV_LOW(ret),
1410 TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1411 } else {
951c6300 1412 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
02616bad
RH		 1413 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1414 TCGV_HIGH(arg1), 32 - c, c);
1415 }
1416 if (arith) {
1417 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1418 } else {
1419 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1420 }
1421 } else {
1422 if (TCG_TARGET_HAS_extract2_i32) {
1423 tcg_gen_extract2_i32(TCGV_HIGH(ret),
1424 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
951c6300 1425 } else {
02616bad 1426 TCGv_i32 t0 = tcg_temp_new_i32();
951c6300 1427 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
02616bad
RH		 1428 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1429 TCGV_HIGH(arg1), c, 32 - c);
1430 tcg_temp_free_i32(t0);
951c6300 1431 }
02616bad 1432 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
951c6300
RH		 1433 }
1434}
1435
474b2e8f 1436void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1437{
474b2e8f 1438 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1439 if (TCG_TARGET_REG_BITS == 32) {
1440 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1441 } else if (arg2 == 0) {
951c6300
RH		 1442 tcg_gen_mov_i64(ret, arg1);
1443 } else {
11d11d61 1444 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 1445 }
1446}
1447
474b2e8f 1448void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1449{
474b2e8f 1450 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1451 if (TCG_TARGET_REG_BITS == 32) {
1452 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1453 } else if (arg2 == 0) {
951c6300
RH		 1454 tcg_gen_mov_i64(ret, arg1);
1455 } else {
11d11d61 1456 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 1457 }
1458}
1459
474b2e8f 1460void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1461{
474b2e8f 1462 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1463 if (TCG_TARGET_REG_BITS == 32) {
1464 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1465 } else if (arg2 == 0) {
951c6300
RH		 1466 tcg_gen_mov_i64(ret, arg1);
1467 } else {
11d11d61 1468 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 1469 }
1470}
951c6300 1471
42a268c2 1472void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
951c6300
RH		 1473{
1474 if (cond == TCG_COND_ALWAYS) {
42a268c2 1475 tcg_gen_br(l);
951c6300 1476 } else if (cond != TCG_COND_NEVER) {
d88a117e 1477 l->refs++;
3a13c3f3
RH		 1478 if (TCG_TARGET_REG_BITS == 32) {
1479 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1480 TCGV_HIGH(arg1), TCGV_LOW(arg2),
42a268c2 1481 TCGV_HIGH(arg2), cond, label_arg(l));
3a13c3f3 1482 } else {
42a268c2
RH		 1483 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1484 label_arg(l));
3a13c3f3 1485 }
951c6300
RH		 1486 }
1487}
1488
42a268c2 1489void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
951c6300 1490{
11d11d61
RH		 1491 if (TCG_TARGET_REG_BITS == 64) {
1492 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
1493 } else if (cond == TCG_COND_ALWAYS) {
42a268c2 1494 tcg_gen_br(l);
951c6300 1495 } else if (cond != TCG_COND_NEVER) {
11d11d61
RH		 1496 l->refs++;
1497 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
1498 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1499 tcg_constant_i32(arg2),
1500 tcg_constant_i32(arg2 >> 32),
1501 cond, label_arg(l));
951c6300
RH		 1502 }
1503}
1504
1505void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1506 TCGv_i64 arg1, TCGv_i64 arg2)
1507{
1508 if (cond == TCG_COND_ALWAYS) {
1509 tcg_gen_movi_i64(ret, 1);
1510 } else if (cond == TCG_COND_NEVER) {
1511 tcg_gen_movi_i64(ret, 0);
1512 } else {
3a13c3f3
RH		 1513 if (TCG_TARGET_REG_BITS == 32) {
1514 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1515 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1516 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1517 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1518 } else {
1519 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1520 }
951c6300
RH		 1521 }
1522}
1523
1524void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1525 TCGv_i64 arg1, int64_t arg2)
1526{
11d11d61
RH		 1527 if (TCG_TARGET_REG_BITS == 64) {
1528 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1529 } else if (cond == TCG_COND_ALWAYS) {
1530 tcg_gen_movi_i64(ret, 1);
1531 } else if (cond == TCG_COND_NEVER) {
1532 tcg_gen_movi_i64(ret, 0);
1533 } else {
1534 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1535 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1536 tcg_constant_i32(arg2),
1537 tcg_constant_i32(arg2 >> 32), cond);
1538 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1539 }
951c6300
RH		 1540}
1541
1542void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1543{
b2e3ae94
RH		 1544 if (arg2 == 0) {
1545 tcg_gen_movi_i64(ret, 0);
1546 } else if (is_power_of_2(arg2)) {
1547 tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1548 } else {
1549 TCGv_i64 t0 = tcg_const_i64(arg2);
1550 tcg_gen_mul_i64(ret, arg1, t0);
1551 tcg_temp_free_i64(t0);
1552 }
951c6300
RH		 1553}
1554
1555void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1556{
1557 if (TCG_TARGET_HAS_div_i64) {
1558 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1559 } else if (TCG_TARGET_HAS_div2_i64) {
1560 TCGv_i64 t0 = tcg_temp_new_i64();
1561 tcg_gen_sari_i64(t0, arg1, 63);
1562 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1563 tcg_temp_free_i64(t0);
1564 } else {
1565 gen_helper_div_i64(ret, arg1, arg2);
1566 }
1567}
1568
1569void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1570{
1571 if (TCG_TARGET_HAS_rem_i64) {
1572 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1573 } else if (TCG_TARGET_HAS_div_i64) {
1574 TCGv_i64 t0 = tcg_temp_new_i64();
1575 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1576 tcg_gen_mul_i64(t0, t0, arg2);
1577 tcg_gen_sub_i64(ret, arg1, t0);
1578 tcg_temp_free_i64(t0);
1579 } else if (TCG_TARGET_HAS_div2_i64) {
1580 TCGv_i64 t0 = tcg_temp_new_i64();
1581 tcg_gen_sari_i64(t0, arg1, 63);
1582 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1583 tcg_temp_free_i64(t0);
1584 } else {
1585 gen_helper_rem_i64(ret, arg1, arg2);
1586 }
1587}
1588
1589void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1590{
1591 if (TCG_TARGET_HAS_div_i64) {
1592 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1593 } else if (TCG_TARGET_HAS_div2_i64) {
1594 TCGv_i64 t0 = tcg_temp_new_i64();
1595 tcg_gen_movi_i64(t0, 0);
1596 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1597 tcg_temp_free_i64(t0);
1598 } else {
1599 gen_helper_divu_i64(ret, arg1, arg2);
1600 }
1601}
1602
1603void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1604{
1605 if (TCG_TARGET_HAS_rem_i64) {
1606 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1607 } else if (TCG_TARGET_HAS_div_i64) {
1608 TCGv_i64 t0 = tcg_temp_new_i64();
1609 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1610 tcg_gen_mul_i64(t0, t0, arg2);
1611 tcg_gen_sub_i64(ret, arg1, t0);
1612 tcg_temp_free_i64(t0);
1613 } else if (TCG_TARGET_HAS_div2_i64) {
1614 TCGv_i64 t0 = tcg_temp_new_i64();
1615 tcg_gen_movi_i64(t0, 0);
1616 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1617 tcg_temp_free_i64(t0);
1618 } else {
1619 gen_helper_remu_i64(ret, arg1, arg2);
1620 }
1621}
1622
1623void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1624{
3a13c3f3
RH		 1625 if (TCG_TARGET_REG_BITS == 32) {
1626 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1627 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1628 } else if (TCG_TARGET_HAS_ext8s_i64) {
951c6300
RH		 1629 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1630 } else {
1631 tcg_gen_shli_i64(ret, arg, 56);
1632 tcg_gen_sari_i64(ret, ret, 56);
1633 }
951c6300
RH		 1634}
1635
1636void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1637{
3a13c3f3
RH		 1638 if (TCG_TARGET_REG_BITS == 32) {
1639 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1640 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1641 } else if (TCG_TARGET_HAS_ext16s_i64) {
951c6300
RH		 1642 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1643 } else {
1644 tcg_gen_shli_i64(ret, arg, 48);
1645 tcg_gen_sari_i64(ret, ret, 48);
1646 }
951c6300
RH		 1647}
1648
1649void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1650{
3a13c3f3
RH		 1651 if (TCG_TARGET_REG_BITS == 32) {
1652 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1653 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1654 } else if (TCG_TARGET_HAS_ext32s_i64) {
951c6300
RH		 1655 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1656 } else {
1657 tcg_gen_shli_i64(ret, arg, 32);
1658 tcg_gen_sari_i64(ret, ret, 32);
1659 }
951c6300
RH		 1660}
1661
1662void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1663{
3a13c3f3
RH		 1664 if (TCG_TARGET_REG_BITS == 32) {
1665 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1666 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1667 } else if (TCG_TARGET_HAS_ext8u_i64) {
951c6300
RH		 1668 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1669 } else {
1670 tcg_gen_andi_i64(ret, arg, 0xffu);
1671 }
951c6300
RH		 1672}
1673
1674void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1675{
3a13c3f3
RH		 1676 if (TCG_TARGET_REG_BITS == 32) {
1677 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1678 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1679 } else if (TCG_TARGET_HAS_ext16u_i64) {
951c6300
RH		 1680 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1681 } else {
1682 tcg_gen_andi_i64(ret, arg, 0xffffu);
1683 }
951c6300
RH		 1684}
1685
1686void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1687{
3a13c3f3
RH		 1688 if (TCG_TARGET_REG_BITS == 32) {
1689 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1690 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1691 } else if (TCG_TARGET_HAS_ext32u_i64) {
951c6300
RH		 1692 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1693 } else {
1694 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1695 }
951c6300
RH		 1696}
1697
2b836c2a 1698void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
951c6300 1699{
2b836c2a
RH		 1700 /* Only one extension flag may be present. */
1701 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1702
3a13c3f3 1703 if (TCG_TARGET_REG_BITS == 32) {
2b836c2a
RH		 1704 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags);
1705 if (flags & TCG_BSWAP_OS) {
1706 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1707 } else {
1708 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1709 }
3a13c3f3 1710 } else if (TCG_TARGET_HAS_bswap16_i64) {
2b836c2a 1711 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags);
951c6300
RH		 1712 } else {
1713 TCGv_i64 t0 = tcg_temp_new_i64();
2b836c2a 1714 TCGv_i64 t1 = tcg_temp_new_i64();
951c6300 1715
2b836c2a
RH		 1716 tcg_gen_shri_i64(t0, arg, 8);
1717 if (!(flags & TCG_BSWAP_IZ)) {
1718 tcg_gen_ext8u_i64(t0, t0);
1719 }
1720
1721 if (flags & TCG_BSWAP_OS) {
1722 tcg_gen_shli_i64(t1, arg, 56);
1723 tcg_gen_sari_i64(t1, t1, 48);
1724 } else if (flags & TCG_BSWAP_OZ) {
1725 tcg_gen_ext8u_i64(t1, arg);
1726 tcg_gen_shli_i64(t1, t1, 8);
1727 } else {
1728 tcg_gen_shli_i64(t1, arg, 8);
1729 }
1730
1731 tcg_gen_or_i64(ret, t0, t1);
951c6300 1732 tcg_temp_free_i64(t0);
2b836c2a 1733 tcg_temp_free_i64(t1);
951c6300 1734 }
951c6300
RH		 1735}
1736
2b836c2a 1737void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
951c6300 1738{
2b836c2a
RH		 1739 /* Only one extension flag may be present. */
1740 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1741
3a13c3f3
RH		 1742 if (TCG_TARGET_REG_BITS == 32) {
1743 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2b836c2a
RH		 1744 if (flags & TCG_BSWAP_OS) {
1745 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1746 } else {
1747 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1748 }
3a13c3f3 1749 } else if (TCG_TARGET_HAS_bswap32_i64) {
2b836c2a 1750 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags);
951c6300 1751 } else {
a686dc71
RH		 1752 TCGv_i64 t0 = tcg_temp_new_i64();
1753 TCGv_i64 t1 = tcg_temp_new_i64();
11d11d61 1754 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff);
951c6300 1755
2b836c2a
RH		 1756 /* arg = xxxxabcd */
1757 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */
1758 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */
1759 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */
1760 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */
1761 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */
1762
1763 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */
1764 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */
1765 if (flags & TCG_BSWAP_OS) {
1766 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */
1767 } else {
1768 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */
1769 }
1770 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba */
951c6300 1771
951c6300
RH		 1772 tcg_temp_free_i64(t0);
1773 tcg_temp_free_i64(t1);
1774 }
951c6300
RH		 1775}
1776
1777void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1778{
3a13c3f3
RH		 1779 if (TCG_TARGET_REG_BITS == 32) {
1780 TCGv_i32 t0, t1;
1781 t0 = tcg_temp_new_i32();
1782 t1 = tcg_temp_new_i32();
951c6300 1783
3a13c3f3
RH		 1784 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1785 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1786 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1787 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1788 tcg_temp_free_i32(t0);
1789 tcg_temp_free_i32(t1);
1790 } else if (TCG_TARGET_HAS_bswap64_i64) {
587195bd 1791 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0);
951c6300
RH		 1792 } else {
1793 TCGv_i64 t0 = tcg_temp_new_i64();
1794 TCGv_i64 t1 = tcg_temp_new_i64();
9e821eab 1795 TCGv_i64 t2 = tcg_temp_new_i64();
951c6300 1796
9e821eab
RH		 1797 /* arg = abcdefgh */
1798 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1799 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */
1800 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */
1801 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */
1802 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */
1803 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */
1804
1805 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1806 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */
1807 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */
1808 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */
1809 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */
1810 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */
1811
1812 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */
1813 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */
1814 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */
951c6300 1815
951c6300
RH		 1816 tcg_temp_free_i64(t0);
1817 tcg_temp_free_i64(t1);
9e821eab 1818 tcg_temp_free_i64(t2);
951c6300 1819 }
951c6300
RH		 1820}
1821
46be8425
RH		 1822void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1823{
1824 uint64_t m = 0x0000ffff0000ffffull;
1825 TCGv_i64 t0 = tcg_temp_new_i64();
1826 TCGv_i64 t1 = tcg_temp_new_i64();
1827
1828 /* See include/qemu/bitops.h, hswap64. */
1829 tcg_gen_rotli_i64(t1, arg, 32);
1830 tcg_gen_andi_i64(t0, t1, m);
1831 tcg_gen_shli_i64(t0, t0, 16);
1832 tcg_gen_shri_i64(t1, t1, 16);
1833 tcg_gen_andi_i64(t1, t1, m);
1834 tcg_gen_or_i64(ret, t0, t1);
1835
1836 tcg_temp_free_i64(t0);
1837 tcg_temp_free_i64(t1);
1838}
1839
1840void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1841{
1842 /* Swapping 2 32-bit elements is a rotate. */
1843 tcg_gen_rotli_i64(ret, arg, 32);
1844}
1845
951c6300
RH		 1846void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1847{
3a13c3f3
RH		 1848 if (TCG_TARGET_REG_BITS == 32) {
1849 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1850 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1851 } else if (TCG_TARGET_HAS_not_i64) {
951c6300
RH		 1852 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1853 } else {
1854 tcg_gen_xori_i64(ret, arg, -1);
1855 }
951c6300
RH		 1856}
1857
1858void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1859{
3a13c3f3
RH		 1860 if (TCG_TARGET_REG_BITS == 32) {
1861 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1862 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1863 } else if (TCG_TARGET_HAS_andc_i64) {
951c6300
RH		 1864 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1865 } else {
1866 TCGv_i64 t0 = tcg_temp_new_i64();
1867 tcg_gen_not_i64(t0, arg2);
1868 tcg_gen_and_i64(ret, arg1, t0);
1869 tcg_temp_free_i64(t0);
1870 }
951c6300
RH		 1871}
1872
1873void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1874{
3a13c3f3
RH		 1875 if (TCG_TARGET_REG_BITS == 32) {
1876 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1877 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1878 } else if (TCG_TARGET_HAS_eqv_i64) {
951c6300
RH		 1879 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1880 } else {
1881 tcg_gen_xor_i64(ret, arg1, arg2);
1882 tcg_gen_not_i64(ret, ret);
1883 }
951c6300
RH		 1884}
1885
1886void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1887{
3a13c3f3
RH		 1888 if (TCG_TARGET_REG_BITS == 32) {
1889 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1890 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1891 } else if (TCG_TARGET_HAS_nand_i64) {
951c6300
RH		 1892 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1893 } else {
1894 tcg_gen_and_i64(ret, arg1, arg2);
1895 tcg_gen_not_i64(ret, ret);
1896 }
951c6300
RH		 1897}
1898
1899void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1900{
3a13c3f3
RH		 1901 if (TCG_TARGET_REG_BITS == 32) {
1902 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1903 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1904 } else if (TCG_TARGET_HAS_nor_i64) {
951c6300
RH		 1905 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1906 } else {
1907 tcg_gen_or_i64(ret, arg1, arg2);
1908 tcg_gen_not_i64(ret, ret);
1909 }
951c6300
RH		 1910}
1911
1912void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1913{
3a13c3f3
RH		 1914 if (TCG_TARGET_REG_BITS == 32) {
1915 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1916 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1917 } else if (TCG_TARGET_HAS_orc_i64) {
951c6300
RH		 1918 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1919 } else {
1920 TCGv_i64 t0 = tcg_temp_new_i64();
1921 tcg_gen_not_i64(t0, arg2);
1922 tcg_gen_or_i64(ret, arg1, t0);
1923 tcg_temp_free_i64(t0);
1924 }
951c6300
RH		 1925}
1926
0e28d006
RH		 1927void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1928{
1929 if (TCG_TARGET_HAS_clz_i64) {
1930 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1931 } else {
1932 gen_helper_clz_i64(ret, arg1, arg2);
1933 }
1934}
1935
1936void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1937{
1938 if (TCG_TARGET_REG_BITS == 32
1939 && TCG_TARGET_HAS_clz_i32
1940 && arg2 <= 0xffffffffu) {
11d11d61
RH		 1941 TCGv_i32 t = tcg_temp_new_i32();
1942 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32);
0e28d006
RH		 1943 tcg_gen_addi_i32(t, t, 32);
1944 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1945 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1946 tcg_temp_free_i32(t);
1947 } else {
11d11d61
RH		 1948 TCGv_i64 t0 = tcg_const_i64(arg2);
1949 tcg_gen_clz_i64(ret, arg1, t0);
1950 tcg_temp_free_i64(t0);
0e28d006
RH		 1951 }
1952}
1953
1954void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1955{
1956 if (TCG_TARGET_HAS_ctz_i64) {
1957 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
14e99210
RH		 1958 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1959 TCGv_i64 z, t = tcg_temp_new_i64();
1960
1961 if (TCG_TARGET_HAS_ctpop_i64) {
1962 tcg_gen_subi_i64(t, arg1, 1);
1963 tcg_gen_andc_i64(t, t, arg1);
1964 tcg_gen_ctpop_i64(t, t);
1965 } else {
1966 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1967 tcg_gen_neg_i64(t, arg1);
1968 tcg_gen_and_i64(t, t, arg1);
1969 tcg_gen_clzi_i64(t, t, 64);
1970 tcg_gen_xori_i64(t, t, 63);
1971 }
11d11d61 1972 z = tcg_constant_i64(0);
14e99210
RH		 1973 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1974 tcg_temp_free_i64(t);
1975 tcg_temp_free_i64(z);
0e28d006
RH		 1976 } else {
1977 gen_helper_ctz_i64(ret, arg1, arg2);
1978 }
1979}
1980
1981void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1982{
1983 if (TCG_TARGET_REG_BITS == 32
1984 && TCG_TARGET_HAS_ctz_i32
1985 && arg2 <= 0xffffffffu) {
11d11d61
RH		 1986 TCGv_i32 t32 = tcg_temp_new_i32();
1987 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32);
0e28d006
RH		 1988 tcg_gen_addi_i32(t32, t32, 32);
1989 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1990 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1991 tcg_temp_free_i32(t32);
14e99210
RH		 1992 } else if (!TCG_TARGET_HAS_ctz_i64
1993 && TCG_TARGET_HAS_ctpop_i64
1994 && arg2 == 64) {
1995 /* This equivalence has the advantage of not requiring a fixup. */
1996 TCGv_i64 t = tcg_temp_new_i64();
1997 tcg_gen_subi_i64(t, arg1, 1);
1998 tcg_gen_andc_i64(t, t, arg1);
1999 tcg_gen_ctpop_i64(ret, t);
2000 tcg_temp_free_i64(t);
0e28d006 2001 } else {
11d11d61
RH		 2002 TCGv_i64 t0 = tcg_const_i64(arg2);
2003 tcg_gen_ctz_i64(ret, arg1, t0);
2004 tcg_temp_free_i64(t0);
0e28d006
RH		 2005 }
2006}
2007
086920c2
RH		 2008void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
2009{
2010 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
2011 TCGv_i64 t = tcg_temp_new_i64();
2012 tcg_gen_sari_i64(t, arg, 63);
2013 tcg_gen_xor_i64(t, t, arg);
2014 tcg_gen_clzi_i64(t, t, 64);
2015 tcg_gen_subi_i64(ret, t, 1);
2016 tcg_temp_free_i64(t);
2017 } else {
2018 gen_helper_clrsb_i64(ret, arg);
2019 }
2020}
2021
a768e4e9
RH		 2022void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
2023{
2024 if (TCG_TARGET_HAS_ctpop_i64) {
2025 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
2026 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
2027 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2028 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2029 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
2030 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2031 } else {
2032 gen_helper_ctpop_i64(ret, arg1);
2033 }
2034}
2035
951c6300
RH		 2036void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2037{
2038 if (TCG_TARGET_HAS_rot_i64) {
2039 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
2040 } else {
2041 TCGv_i64 t0, t1;
2042 t0 = tcg_temp_new_i64();
2043 t1 = tcg_temp_new_i64();
2044 tcg_gen_shl_i64(t0, arg1, arg2);
2045 tcg_gen_subfi_i64(t1, 64, arg2);
2046 tcg_gen_shr_i64(t1, arg1, t1);
2047 tcg_gen_or_i64(ret, t0, t1);
2048 tcg_temp_free_i64(t0);
2049 tcg_temp_free_i64(t1);
2050 }
2051}
2052
07dada03 2053void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 2054{
07dada03 2055 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
951c6300
RH		 2056 /* some cases can be optimized here */
2057 if (arg2 == 0) {
2058 tcg_gen_mov_i64(ret, arg1);
2059 } else if (TCG_TARGET_HAS_rot_i64) {
11d11d61 2060 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2));
951c6300
RH		 2061 } else {
2062 TCGv_i64 t0, t1;
2063 t0 = tcg_temp_new_i64();
2064 t1 = tcg_temp_new_i64();
2065 tcg_gen_shli_i64(t0, arg1, arg2);
2066 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
2067 tcg_gen_or_i64(ret, t0, t1);
2068 tcg_temp_free_i64(t0);
2069 tcg_temp_free_i64(t1);
2070 }
2071}
2072
2073void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2074{
2075 if (TCG_TARGET_HAS_rot_i64) {
2076 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
2077 } else {
2078 TCGv_i64 t0, t1;
2079 t0 = tcg_temp_new_i64();
2080 t1 = tcg_temp_new_i64();
2081 tcg_gen_shr_i64(t0, arg1, arg2);
2082 tcg_gen_subfi_i64(t1, 64, arg2);
2083 tcg_gen_shl_i64(t1, arg1, t1);
2084 tcg_gen_or_i64(ret, t0, t1);
2085 tcg_temp_free_i64(t0);
2086 tcg_temp_free_i64(t1);
2087 }
2088}
2089
07dada03 2090void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 2091{
07dada03 2092 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
951c6300
RH		 2093 /* some cases can be optimized here */
2094 if (arg2 == 0) {
2095 tcg_gen_mov_i64(ret, arg1);
2096 } else {
2097 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2098 }
2099}
2100
2101void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2102 unsigned int ofs, unsigned int len)
2103{
2104 uint64_t mask;
2105 TCGv_i64 t1;
2106
2107 tcg_debug_assert(ofs < 64);
0d0d309d 2108 tcg_debug_assert(len > 0);
951c6300
RH		 2109 tcg_debug_assert(len <= 64);
2110 tcg_debug_assert(ofs + len <= 64);
2111
0d0d309d 2112 if (len == 64) {
951c6300
RH		 2113 tcg_gen_mov_i64(ret, arg2);
2114 return;
2115 }
2116 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2117 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2118 return;
2119 }
2120
3a13c3f3
RH		 2121 if (TCG_TARGET_REG_BITS == 32) {
2122 if (ofs >= 32) {
2123 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2124 TCGV_LOW(arg2), ofs - 32, len);
2125 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2126 return;
2127 }
2128 if (ofs + len <= 32) {
2129 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2130 TCGV_LOW(arg2), ofs, len);
2131 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2132 return;
2133 }
951c6300 2134 }
951c6300 2135
951c6300
RH		 2136 t1 = tcg_temp_new_i64();
2137
b0a60567
RH		 2138 if (TCG_TARGET_HAS_extract2_i64) {
2139 if (ofs + len == 64) {
2140 tcg_gen_shli_i64(t1, arg1, len);
2141 tcg_gen_extract2_i64(ret, t1, arg2, len);
2142 goto done;
2143 }
2144 if (ofs == 0) {
2145 tcg_gen_extract2_i64(ret, arg1, arg2, len);
2146 tcg_gen_rotli_i64(ret, ret, len);
2147 goto done;
2148 }
2149 }
2150
2151 mask = (1ull << len) - 1;
951c6300
RH		 2152 if (ofs + len < 64) {
2153 tcg_gen_andi_i64(t1, arg2, mask);
2154 tcg_gen_shli_i64(t1, t1, ofs);
2155 } else {
2156 tcg_gen_shli_i64(t1, arg2, ofs);
2157 }
2158 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2159 tcg_gen_or_i64(ret, ret, t1);
b0a60567 2160 done:
951c6300
RH		 2161 tcg_temp_free_i64(t1);
2162}
2163
07cc68d5
RH		 2164void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2165 unsigned int ofs, unsigned int len)
2166{
2167 tcg_debug_assert(ofs < 64);
2168 tcg_debug_assert(len > 0);
2169 tcg_debug_assert(len <= 64);
2170 tcg_debug_assert(ofs + len <= 64);
2171
2172 if (ofs + len == 64) {
2173 tcg_gen_shli_i64(ret, arg, ofs);
2174 } else if (ofs == 0) {
2175 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2176 } else if (TCG_TARGET_HAS_deposit_i64
2177 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
11d11d61 2178 TCGv_i64 zero = tcg_constant_i64(0);
07cc68d5 2179 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
07cc68d5
RH		 2180 } else {
2181 if (TCG_TARGET_REG_BITS == 32) {
2182 if (ofs >= 32) {
2183 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2184 ofs - 32, len);
2185 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2186 return;
2187 }
2188 if (ofs + len <= 32) {
2189 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2190 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2191 return;
2192 }
2193 }
2194 /* To help two-operand hosts we prefer to zero-extend first,
2195 which allows ARG to stay live. */
2196 switch (len) {
2197 case 32:
2198 if (TCG_TARGET_HAS_ext32u_i64) {
2199 tcg_gen_ext32u_i64(ret, arg);
2200 tcg_gen_shli_i64(ret, ret, ofs);
2201 return;
2202 }
2203 break;
2204 case 16:
2205 if (TCG_TARGET_HAS_ext16u_i64) {
2206 tcg_gen_ext16u_i64(ret, arg);
2207 tcg_gen_shli_i64(ret, ret, ofs);
2208 return;
2209 }
2210 break;
2211 case 8:
2212 if (TCG_TARGET_HAS_ext8u_i64) {
2213 tcg_gen_ext8u_i64(ret, arg);
2214 tcg_gen_shli_i64(ret, ret, ofs);
2215 return;
2216 }
2217 break;
2218 }
2219 /* Otherwise prefer zero-extension over AND for code size. */
2220 switch (ofs + len) {
2221 case 32:
2222 if (TCG_TARGET_HAS_ext32u_i64) {
2223 tcg_gen_shli_i64(ret, arg, ofs);
2224 tcg_gen_ext32u_i64(ret, ret);
2225 return;
2226 }
2227 break;
2228 case 16:
2229 if (TCG_TARGET_HAS_ext16u_i64) {
2230 tcg_gen_shli_i64(ret, arg, ofs);
2231 tcg_gen_ext16u_i64(ret, ret);
2232 return;
2233 }
2234 break;
2235 case 8:
2236 if (TCG_TARGET_HAS_ext8u_i64) {
2237 tcg_gen_shli_i64(ret, arg, ofs);
2238 tcg_gen_ext8u_i64(ret, ret);
2239 return;
2240 }
2241 break;
2242 }
2243 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2244 tcg_gen_shli_i64(ret, ret, ofs);
2245 }
2246}
2247
7ec8bab3
RH		 2248void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2249 unsigned int ofs, unsigned int len)
2250{
2251 tcg_debug_assert(ofs < 64);
2252 tcg_debug_assert(len > 0);
2253 tcg_debug_assert(len <= 64);
2254 tcg_debug_assert(ofs + len <= 64);
2255
2256 /* Canonicalize certain special cases, even if extract is supported. */
2257 if (ofs + len == 64) {
2258 tcg_gen_shri_i64(ret, arg, 64 - len);
2259 return;
2260 }
2261 if (ofs == 0) {
2262 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2263 return;
2264 }
2265
2266 if (TCG_TARGET_REG_BITS == 32) {
2267 /* Look for a 32-bit extract within one of the two words. */
2268 if (ofs >= 32) {
2269 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2270 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2271 return;
2272 }
2273 if (ofs + len <= 32) {
2274 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2275 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2276 return;
2277 }
2278 /* The field is split across two words. One double-word
2279 shift is better than two double-word shifts. */
2280 goto do_shift_and;
2281 }
2282
2283 if (TCG_TARGET_HAS_extract_i64
2284 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2285 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2286 return;
2287 }
2288
2289 /* Assume that zero-extension, if available, is cheaper than a shift. */
2290 switch (ofs + len) {
2291 case 32:
2292 if (TCG_TARGET_HAS_ext32u_i64) {
2293 tcg_gen_ext32u_i64(ret, arg);
2294 tcg_gen_shri_i64(ret, ret, ofs);
2295 return;
2296 }
2297 break;
2298 case 16:
2299 if (TCG_TARGET_HAS_ext16u_i64) {
2300 tcg_gen_ext16u_i64(ret, arg);
2301 tcg_gen_shri_i64(ret, ret, ofs);
2302 return;
2303 }
2304 break;
2305 case 8:
2306 if (TCG_TARGET_HAS_ext8u_i64) {
2307 tcg_gen_ext8u_i64(ret, arg);
2308 tcg_gen_shri_i64(ret, ret, ofs);
2309 return;
2310 }
2311 break;
2312 }
2313
2314 /* ??? Ideally we'd know what values are available for immediate AND.
2315 Assume that 8 bits are available, plus the special cases of 16 and 32,
2316 so that we get ext8u, ext16u, and ext32u. */
2317 switch (len) {
2318 case 1 ... 8: case 16: case 32:
2319 do_shift_and:
2320 tcg_gen_shri_i64(ret, arg, ofs);
2321 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2322 break;
2323 default:
2324 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2325 tcg_gen_shri_i64(ret, ret, 64 - len);
2326 break;
2327 }
2328}
2329
2330void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2331 unsigned int ofs, unsigned int len)
2332{
2333 tcg_debug_assert(ofs < 64);
2334 tcg_debug_assert(len > 0);
2335 tcg_debug_assert(len <= 64);
2336 tcg_debug_assert(ofs + len <= 64);
2337
2338 /* Canonicalize certain special cases, even if sextract is supported. */
2339 if (ofs + len == 64) {
2340 tcg_gen_sari_i64(ret, arg, 64 - len);
2341 return;
2342 }
2343 if (ofs == 0) {
2344 switch (len) {
2345 case 32:
2346 tcg_gen_ext32s_i64(ret, arg);
2347 return;
2348 case 16:
2349 tcg_gen_ext16s_i64(ret, arg);
2350 return;
2351 case 8:
2352 tcg_gen_ext8s_i64(ret, arg);
2353 return;
2354 }
2355 }
2356
2357 if (TCG_TARGET_REG_BITS == 32) {
2358 /* Look for a 32-bit extract within one of the two words. */
2359 if (ofs >= 32) {
2360 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2361 } else if (ofs + len <= 32) {
2362 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2363 } else if (ofs == 0) {
2364 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2365 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2366 return;
2367 } else if (len > 32) {
2368 TCGv_i32 t = tcg_temp_new_i32();
2369 /* Extract the bits for the high word normally. */
2370 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2371 /* Shift the field down for the low part. */
2372 tcg_gen_shri_i64(ret, arg, ofs);
2373 /* Overwrite the shift into the high part. */
2374 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2375 tcg_temp_free_i32(t);
2376 return;
2377 } else {
2378 /* Shift the field down for the low part, such that the
2379 field sits at the MSB. */
2380 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2381 /* Shift the field down from the MSB, sign extending. */
2382 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2383 }
2384 /* Sign-extend the field from 32 bits. */
2385 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2386 return;
2387 }
2388
2389 if (TCG_TARGET_HAS_sextract_i64
2390 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2391 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2392 return;
2393 }
2394
2395 /* Assume that sign-extension, if available, is cheaper than a shift. */
2396 switch (ofs + len) {
2397 case 32:
2398 if (TCG_TARGET_HAS_ext32s_i64) {
2399 tcg_gen_ext32s_i64(ret, arg);
2400 tcg_gen_sari_i64(ret, ret, ofs);
2401 return;
2402 }
2403 break;
2404 case 16:
2405 if (TCG_TARGET_HAS_ext16s_i64) {
2406 tcg_gen_ext16s_i64(ret, arg);
2407 tcg_gen_sari_i64(ret, ret, ofs);
2408 return;
2409 }
2410 break;
2411 case 8:
2412 if (TCG_TARGET_HAS_ext8s_i64) {
2413 tcg_gen_ext8s_i64(ret, arg);
2414 tcg_gen_sari_i64(ret, ret, ofs);
2415 return;
2416 }
2417 break;
2418 }
2419 switch (len) {
2420 case 32:
2421 if (TCG_TARGET_HAS_ext32s_i64) {
2422 tcg_gen_shri_i64(ret, arg, ofs);
2423 tcg_gen_ext32s_i64(ret, ret);
2424 return;
2425 }
2426 break;
2427 case 16:
2428 if (TCG_TARGET_HAS_ext16s_i64) {
2429 tcg_gen_shri_i64(ret, arg, ofs);
2430 tcg_gen_ext16s_i64(ret, ret);
2431 return;
2432 }
2433 break;
2434 case 8:
2435 if (TCG_TARGET_HAS_ext8s_i64) {
2436 tcg_gen_shri_i64(ret, arg, ofs);
2437 tcg_gen_ext8s_i64(ret, ret);
2438 return;
2439 }
2440 break;
2441 }
2442 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2443 tcg_gen_sari_i64(ret, ret, 64 - len);
2444}
2445
2089fcc9
DH		 2446/*
2447 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2448 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2449 */
2450void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2451 unsigned int ofs)
2452{
2453 tcg_debug_assert(ofs <= 64);
2454 if (ofs == 0) {
2455 tcg_gen_mov_i64(ret, al);
2456 } else if (ofs == 64) {
2457 tcg_gen_mov_i64(ret, ah);
2458 } else if (al == ah) {
2459 tcg_gen_rotri_i64(ret, al, ofs);
fce1296f
RH		 2460 } else if (TCG_TARGET_HAS_extract2_i64) {
2461 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2089fcc9
DH		 2462 } else {
2463 TCGv_i64 t0 = tcg_temp_new_i64();
2464 tcg_gen_shri_i64(t0, al, ofs);
2465 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2466 tcg_temp_free_i64(t0);
2467 }
2468}
2469
951c6300
RH		 2470void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2471 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2472{
37ed3bf1
RH		 2473 if (cond == TCG_COND_ALWAYS) {
2474 tcg_gen_mov_i64(ret, v1);
2475 } else if (cond == TCG_COND_NEVER) {
2476 tcg_gen_mov_i64(ret, v2);
2477 } else if (TCG_TARGET_REG_BITS == 32) {
3a13c3f3
RH		 2478 TCGv_i32 t0 = tcg_temp_new_i32();
2479 TCGv_i32 t1 = tcg_temp_new_i32();
2480 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2481 TCGV_LOW(c1), TCGV_HIGH(c1),
2482 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2483
2484 if (TCG_TARGET_HAS_movcond_i32) {
2485 tcg_gen_movi_i32(t1, 0);
2486 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2487 TCGV_LOW(v1), TCGV_LOW(v2));
2488 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2489 TCGV_HIGH(v1), TCGV_HIGH(v2));
2490 } else {
2491 tcg_gen_neg_i32(t0, t0);
951c6300 2492
3a13c3f3
RH		 2493 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2494 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2495 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
951c6300 2496
3a13c3f3
RH		 2497 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2498 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2499 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2500 }
2501 tcg_temp_free_i32(t0);
2502 tcg_temp_free_i32(t1);
2503 } else if (TCG_TARGET_HAS_movcond_i64) {
951c6300
RH		 2504 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2505 } else {
2506 TCGv_i64 t0 = tcg_temp_new_i64();
2507 TCGv_i64 t1 = tcg_temp_new_i64();
2508 tcg_gen_setcond_i64(cond, t0, c1, c2);
2509 tcg_gen_neg_i64(t0, t0);
2510 tcg_gen_and_i64(t1, v1, t0);
2511 tcg_gen_andc_i64(ret, v2, t0);
2512 tcg_gen_or_i64(ret, ret, t1);
2513 tcg_temp_free_i64(t0);
2514 tcg_temp_free_i64(t1);
2515 }
951c6300
RH		 2516}
2517
2518void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2519 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2520{
2521 if (TCG_TARGET_HAS_add2_i64) {
2522 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2523 } else {
2524 TCGv_i64 t0 = tcg_temp_new_i64();
2525 TCGv_i64 t1 = tcg_temp_new_i64();
2526 tcg_gen_add_i64(t0, al, bl);
2527 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2528 tcg_gen_add_i64(rh, ah, bh);
2529 tcg_gen_add_i64(rh, rh, t1);
2530 tcg_gen_mov_i64(rl, t0);
2531 tcg_temp_free_i64(t0);
2532 tcg_temp_free_i64(t1);
2533 }
2534}
2535
2536void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2537 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2538{
2539 if (TCG_TARGET_HAS_sub2_i64) {
2540 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2541 } else {
2542 TCGv_i64 t0 = tcg_temp_new_i64();
2543 TCGv_i64 t1 = tcg_temp_new_i64();
2544 tcg_gen_sub_i64(t0, al, bl);
2545 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2546 tcg_gen_sub_i64(rh, ah, bh);
2547 tcg_gen_sub_i64(rh, rh, t1);
2548 tcg_gen_mov_i64(rl, t0);
2549 tcg_temp_free_i64(t0);
2550 tcg_temp_free_i64(t1);
2551 }
2552}
2553
2554void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2555{
2556 if (TCG_TARGET_HAS_mulu2_i64) {
2557 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2558 } else if (TCG_TARGET_HAS_muluh_i64) {
2559 TCGv_i64 t = tcg_temp_new_i64();
2560 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2561 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2562 tcg_gen_mov_i64(rl, t);
2563 tcg_temp_free_i64(t);
2564 } else {
2565 TCGv_i64 t0 = tcg_temp_new_i64();
2566 tcg_gen_mul_i64(t0, arg1, arg2);
2567 gen_helper_muluh_i64(rh, arg1, arg2);
2568 tcg_gen_mov_i64(rl, t0);
2569 tcg_temp_free_i64(t0);
2570 }
2571}
2572
2573void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2574{
2575 if (TCG_TARGET_HAS_muls2_i64) {
2576 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2577 } else if (TCG_TARGET_HAS_mulsh_i64) {
2578 TCGv_i64 t = tcg_temp_new_i64();
2579 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2580 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2581 tcg_gen_mov_i64(rl, t);
2582 tcg_temp_free_i64(t);
2583 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2584 TCGv_i64 t0 = tcg_temp_new_i64();
2585 TCGv_i64 t1 = tcg_temp_new_i64();
2586 TCGv_i64 t2 = tcg_temp_new_i64();
2587 TCGv_i64 t3 = tcg_temp_new_i64();
2588 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2589 /* Adjust for negative inputs. */
2590 tcg_gen_sari_i64(t2, arg1, 63);
2591 tcg_gen_sari_i64(t3, arg2, 63);
2592 tcg_gen_and_i64(t2, t2, arg2);
2593 tcg_gen_and_i64(t3, t3, arg1);
2594 tcg_gen_sub_i64(rh, t1, t2);
2595 tcg_gen_sub_i64(rh, rh, t3);
2596 tcg_gen_mov_i64(rl, t0);
2597 tcg_temp_free_i64(t0);
2598 tcg_temp_free_i64(t1);
2599 tcg_temp_free_i64(t2);
2600 tcg_temp_free_i64(t3);
2601 } else {
2602 TCGv_i64 t0 = tcg_temp_new_i64();
2603 tcg_gen_mul_i64(t0, arg1, arg2);
2604 gen_helper_mulsh_i64(rh, arg1, arg2);
2605 tcg_gen_mov_i64(rl, t0);
2606 tcg_temp_free_i64(t0);
2607 }
2608}
2609
5087abfb
RH		 2610void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2611{
2612 TCGv_i64 t0 = tcg_temp_new_i64();
2613 TCGv_i64 t1 = tcg_temp_new_i64();
2614 TCGv_i64 t2 = tcg_temp_new_i64();
2615 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2616 /* Adjust for negative input for the signed arg1. */
2617 tcg_gen_sari_i64(t2, arg1, 63);
2618 tcg_gen_and_i64(t2, t2, arg2);
2619 tcg_gen_sub_i64(rh, t1, t2);
2620 tcg_gen_mov_i64(rl, t0);
2621 tcg_temp_free_i64(t0);
2622 tcg_temp_free_i64(t1);
2623 tcg_temp_free_i64(t2);
2624}
2625
b87fb8cd
RH		 2626void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2627{
2628 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2629}
2630
2631void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2632{
2633 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2634}
2635
2636void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2637{
2638 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2639}
2640
2641void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2642{
2643 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2644}
2645
ff1f11f7
RH		 2646void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2647{
2648 TCGv_i64 t = tcg_temp_new_i64();
2649
2650 tcg_gen_sari_i64(t, a, 63);
2651 tcg_gen_xor_i64(ret, a, t);
2652 tcg_gen_sub_i64(ret, ret, t);
2653 tcg_temp_free_i64(t);
2654}
2655
951c6300
RH		 2656/* Size changing operations. */
2657
609ad705 2658void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
951c6300 2659{
3a13c3f3 2660 if (TCG_TARGET_REG_BITS == 32) {
609ad705
RH		 2661 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2662 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
b7e8b17a 2663 tcg_gen_op2(INDEX_op_extrl_i64_i32,
ae8b75dc 2664 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
609ad705 2665 } else {
dc41aa7d 2666 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
609ad705
RH		 2667 }
2668}
2669
2670void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2671{
2672 if (TCG_TARGET_REG_BITS == 32) {
2673 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2674 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
b7e8b17a 2675 tcg_gen_op2(INDEX_op_extrh_i64_i32,
ae8b75dc 2676 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
951c6300
RH		 2677 } else {
2678 TCGv_i64 t = tcg_temp_new_i64();
609ad705 2679 tcg_gen_shri_i64(t, arg, 32);
dc41aa7d 2680 tcg_gen_mov_i32(ret, (TCGv_i32)t);
951c6300
RH		 2681 tcg_temp_free_i64(t);
2682 }
951c6300
RH		 2683}
2684
2685void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2686{
3a13c3f3
RH		 2687 if (TCG_TARGET_REG_BITS == 32) {
2688 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2689 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2690 } else {
b7e8b17a 2691 tcg_gen_op2(INDEX_op_extu_i32_i64,
ae8b75dc 2692 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2693 }
951c6300
RH		 2694}
2695
2696void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2697{
3a13c3f3
RH		 2698 if (TCG_TARGET_REG_BITS == 32) {
2699 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2700 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2701 } else {
b7e8b17a 2702 tcg_gen_op2(INDEX_op_ext_i32_i64,
ae8b75dc 2703 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2704 }
951c6300
RH		 2705}
2706
2707void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2708{
3a13c3f3
RH		 2709 TCGv_i64 tmp;
2710
2711 if (TCG_TARGET_REG_BITS == 32) {
2712 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2713 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2714 return;
2715 }
2716
2717 tmp = tcg_temp_new_i64();
951c6300
RH		 2718 /* These extensions are only needed for type correctness.
2719 We may be able to do better given target specific information. */
2720 tcg_gen_extu_i32_i64(tmp, high);
2721 tcg_gen_extu_i32_i64(dest, low);
2722 /* If deposit is available, use it. Otherwise use the extra
2723 knowledge that we have of the zero-extensions above. */
2724 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2725 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2726 } else {
2727 tcg_gen_shli_i64(tmp, tmp, 32);
2728 tcg_gen_or_i64(dest, dest, tmp);
2729 }
2730 tcg_temp_free_i64(tmp);
951c6300
RH		 2731}
2732
2733void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2734{
3a13c3f3
RH		 2735 if (TCG_TARGET_REG_BITS == 32) {
2736 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2737 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2738 } else {
609ad705
RH		 2739 tcg_gen_extrl_i64_i32(lo, arg);
2740 tcg_gen_extrh_i64_i32(hi, arg);
3a13c3f3 2741 }
951c6300
RH		 2742}
2743
2744void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2745{
2746 tcg_gen_ext32u_i64(lo, arg);
2747 tcg_gen_shri_i64(hi, arg, 32);
2748}
2749
4771e71c
RH		 2750void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg)
2751{
2752 tcg_gen_mov_i64(lo, TCGV128_LOW(arg));
2753 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg));
2754}
2755
2756void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi)
2757{
2758 tcg_gen_mov_i64(TCGV128_LOW(ret), lo);
2759 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi);
2760}
2761
2762void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src)
2763{
2764 if (dst != src) {
2765 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src));
2766 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src));
2767 }
2768}
2769
951c6300
RH		 2770/* QEMU specific operations. */
2771
d9971435 2772void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
07ea28b4 2773{
eba40358
RH		 2774 /*
2775 * Let the jit code return the read-only version of the
2776 * TranslationBlock, so that we minimize the pc-relative
2777 * distance of the address of the exit_tb code to TB.
2778 * This will improve utilization of pc-relative address loads.
2779 *
2780 * TODO: Move this to translator_loop, so that all const
2781 * TranslationBlock pointers refer to read-only memory.
2782 * This requires coordination with targets that do not use
2783 * the translator_loop.
2784 */
2785 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
07ea28b4
RH		 2786
2787 if (tb == NULL) {
2788 tcg_debug_assert(idx == 0);
2789 } else if (idx <= TB_EXIT_IDXMAX) {
2790#ifdef CONFIG_DEBUG_TCG
2791 /* This is an exit following a goto_tb. Verify that we have
2792 seen this numbered exit before, via tcg_gen_goto_tb. */
2793 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2794#endif
2795 } else {
2796 /* This is an exit via the exitreq label. */
2797 tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2798 }
2799
e6d86bed 2800 plugin_gen_disable_mem_helpers();
07ea28b4
RH		 2801 tcg_gen_op1i(INDEX_op_exit_tb, val);
2802}
2803
951c6300
RH		 2804void tcg_gen_goto_tb(unsigned idx)
2805{
84f15616 2806 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
b7e4afbd 2807 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB));
951c6300 2808 /* We only support two chained exits. */
07ea28b4 2809 tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
951c6300 2810#ifdef CONFIG_DEBUG_TCG
a4761232 2811 /* Verify that we haven't seen this numbered exit before. */
b1311c4a
EC		 2812 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2813 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
951c6300 2814#endif
e6d86bed 2815 plugin_gen_disable_mem_helpers();
84f15616 2816 tcg_gen_op1i(INDEX_op_goto_tb, idx);
951c6300
RH		 2817}
2818
7f11636d 2819void tcg_gen_lookup_and_goto_ptr(void)
cedbcb01 2820{
84f15616 2821 TCGv_ptr ptr;
e6d86bed 2822
b7e4afbd 2823 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) {
07ea28b4 2824 tcg_gen_exit_tb(NULL, 0);
84f15616 2825 return;
cedbcb01 2826 }
84f15616
RH		 2827
2828 plugin_gen_disable_mem_helpers();
2829 ptr = tcg_temp_new_ptr();
2830 gen_helper_lookup_tb_ptr(ptr, cpu_env);
2831 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2832 tcg_temp_free_ptr(ptr);
cedbcb01
EC		 2833}
2834
14776ab5 2835static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
951c6300 2836{
1f00b27f 2837 /* Trigger the asserts within as early as possible. */
76e366e7
RH		 2838 unsigned a_bits = get_alignment_bits(op);
2839
2840 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
2841 if (a_bits == (op & MO_SIZE)) {
2842 op = (op & ~MO_AMASK) | MO_ALIGN;
2843 }
1f00b27f 2844
951c6300
RH		 2845 switch (op & MO_SIZE) {
2846 case MO_8:
2847 op &= ~MO_BSWAP;
2848 break;
2849 case MO_16:
2850 break;
2851 case MO_32:
2852 if (!is64) {
2853 op &= ~MO_SIGN;
2854 }
2855 break;
2856 case MO_64:
4b473e0c
RH		 2857 if (is64) {
2858 op &= ~MO_SIGN;
2859 break;
951c6300 2860 }
4b473e0c
RH		 2861 /* fall through */
2862 default:
2863 g_assert_not_reached();
951c6300
RH		 2864 }
2865 if (st) {
2866 op &= ~MO_SIGN;
2867 }
2868 return op;
2869}
2870
c45cb8bb 2871static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
14776ab5 2872 MemOp memop, TCGArg idx)
951c6300 2873{
9002ffcb 2874 MemOpIdx oi = make_memop_idx(memop, idx);
c45cb8bb 2875#if TARGET_LONG_BITS == 32
59227d5d 2876 tcg_gen_op3i_i32(opc, val, addr, oi);
c45cb8bb 2877#else
3a13c3f3 2878 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2879 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
3a13c3f3 2880 } else {
ae8b75dc 2881 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
3a13c3f3 2882 }
c45cb8bb 2883#endif
951c6300
RH		 2884}
2885
c45cb8bb 2886static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
14776ab5 2887 MemOp memop, TCGArg idx)
c45cb8bb 2888{
9002ffcb 2889 MemOpIdx oi = make_memop_idx(memop, idx);
951c6300 2890#if TARGET_LONG_BITS == 32
c45cb8bb 2891 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2892 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
c45cb8bb 2893 } else {
ae8b75dc 2894 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
c45cb8bb 2895 }
951c6300 2896#else
c45cb8bb 2897 if (TCG_TARGET_REG_BITS == 32) {
59227d5d
RH		 2898 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2899 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
c45cb8bb 2900 } else {
59227d5d 2901 tcg_gen_op3i_i64(opc, val, addr, oi);
c45cb8bb 2902 }
951c6300 2903#endif
c45cb8bb 2904}
951c6300 2905
b32dc337
PK		 2906static void tcg_gen_req_mo(TCGBar type)
2907{
2908#ifdef TCG_GUEST_DEFAULT_MO
2909 type &= TCG_GUEST_DEFAULT_MO;
2910#endif
2911 type &= ~TCG_TARGET_DEFAULT_MO;
2912 if (type) {
2913 tcg_gen_mb(type | TCG_BAR_SC);
2914 }
2915}
2916
fcc54ab5
AB		 2917static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr)
2918{
2919#ifdef CONFIG_PLUGIN
2920 if (tcg_ctx->plugin_insn != NULL) {
2921 /* Save a copy of the vaddr for use after a load. */
2922 TCGv temp = tcg_temp_new();
2923 tcg_gen_mov_tl(temp, vaddr);
2924 return temp;
2925 }
2926#endif
2927 return vaddr;
2928}
2929
37aff087
RH		 2930static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi,
2931 enum qemu_plugin_mem_rw rw)
e6d86bed
EC		 2932{
2933#ifdef CONFIG_PLUGIN
fcc54ab5 2934 if (tcg_ctx->plugin_insn != NULL) {
37aff087 2935 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw);
fcc54ab5
AB		 2936 plugin_gen_empty_mem_callback(vaddr, info);
2937 tcg_temp_free(vaddr);
e6d86bed 2938 }
e6d86bed
EC		 2939#endif
2940}
2941
14776ab5 2942void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
951c6300 2943{
14776ab5 2944 MemOp orig_memop;
37aff087 2945 MemOpIdx oi;
e1dcf352 2946
b32dc337 2947 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
951c6300 2948 memop = tcg_canonicalize_memop(memop, 0, 0);
37aff087 2949 oi = make_memop_idx(memop, idx);
e1dcf352
RH		 2950
2951 orig_memop = memop;
2952 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2953 memop &= ~MO_BSWAP;
359feba5 2954 /* The bswap primitive benefits from zero-extended input. */
e1dcf352
RH		 2955 if ((memop & MO_SSIZE) == MO_SW) {
2956 memop &= ~MO_SIGN;
2957 }
2958 }
2959
fcc54ab5 2960 addr = plugin_prep_mem_callbacks(addr);
c45cb8bb 2961 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
37aff087 2962 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
e1dcf352
RH		 2963
2964 if ((orig_memop ^ memop) & MO_BSWAP) {
2965 switch (orig_memop & MO_SIZE) {
2966 case MO_16:
359feba5
RH		 2967 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
2968 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2969 : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
e1dcf352
RH		 2970 break;
2971 case MO_32:
2972 tcg_gen_bswap32_i32(val, val);
2973 break;
2974 default:
2975 g_assert_not_reached();
2976 }
2977 }
951c6300
RH		 2978}
2979
14776ab5 2980void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
951c6300 2981{
e1dcf352 2982 TCGv_i32 swap = NULL;
37aff087 2983 MemOpIdx oi;
e1dcf352 2984
b32dc337 2985 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
951c6300 2986 memop = tcg_canonicalize_memop(memop, 0, 1);
37aff087 2987 oi = make_memop_idx(memop, idx);
e1dcf352
RH		 2988
2989 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2990 swap = tcg_temp_new_i32();
2991 switch (memop & MO_SIZE) {
2992 case MO_16:
b53357ac 2993 tcg_gen_bswap16_i32(swap, val, 0);
e1dcf352
RH		 2994 break;
2995 case MO_32:
2996 tcg_gen_bswap32_i32(swap, val);
2997 break;
2998 default:
2999 g_assert_not_reached();
3000 }
3001 val = swap;
3002 memop &= ~MO_BSWAP;
3003 }
3004
fcc54ab5 3005 addr = plugin_prep_mem_callbacks(addr);
07ce0b05
RH		 3006 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
3007 gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx);
3008 } else {
3009 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
3010 }
37aff087 3011 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
e1dcf352
RH		 3012
3013 if (swap) {
3014 tcg_temp_free_i32(swap);
3015 }
951c6300
RH		 3016}
3017
14776ab5 3018void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
951c6300 3019{
14776ab5 3020 MemOp orig_memop;
37aff087 3021 MemOpIdx oi;
e1dcf352 3022
3a13c3f3 3023 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 3024 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
3025 if (memop & MO_SIGN) {
3026 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
3027 } else {
3028 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
3029 }
3030 return;
3031 }
951c6300 3032
e1dcf352 3033 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
c45cb8bb 3034 memop = tcg_canonicalize_memop(memop, 1, 0);
37aff087 3035 oi = make_memop_idx(memop, idx);
e1dcf352
RH		 3036
3037 orig_memop = memop;
3038 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
3039 memop &= ~MO_BSWAP;
359feba5 3040 /* The bswap primitive benefits from zero-extended input. */
e1dcf352
RH		 3041 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
3042 memop &= ~MO_SIGN;
3043 }
3044 }
3045
fcc54ab5 3046 addr = plugin_prep_mem_callbacks(addr);
c45cb8bb 3047 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
37aff087 3048 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
e1dcf352
RH		 3049
3050 if ((orig_memop ^ memop) & MO_BSWAP) {
359feba5
RH		 3051 int flags = (orig_memop & MO_SIGN
3052 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
3053 : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
e1dcf352
RH		 3054 switch (orig_memop & MO_SIZE) {
3055 case MO_16:
359feba5 3056 tcg_gen_bswap16_i64(val, val, flags);
e1dcf352
RH		 3057 break;
3058 case MO_32:
359feba5 3059 tcg_gen_bswap32_i64(val, val, flags);
e1dcf352
RH		 3060 break;
3061 case MO_64:
3062 tcg_gen_bswap64_i64(val, val);
3063 break;
3064 default:
3065 g_assert_not_reached();
3066 }
3067 }
951c6300
RH		 3068}
3069
14776ab5 3070void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
951c6300 3071{
e1dcf352 3072 TCGv_i64 swap = NULL;
37aff087 3073 MemOpIdx oi;
e1dcf352 3074
3a13c3f3 3075 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 3076 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
3077 return;
3078 }
951c6300 3079
e1dcf352 3080 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
c45cb8bb 3081 memop = tcg_canonicalize_memop(memop, 1, 1);
37aff087 3082 oi = make_memop_idx(memop, idx);
e1dcf352
RH		 3083
3084 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
3085 swap = tcg_temp_new_i64();
3086 switch (memop & MO_SIZE) {
3087 case MO_16:
b53357ac 3088 tcg_gen_bswap16_i64(swap, val, 0);
e1dcf352
RH		 3089 break;
3090 case MO_32:
b53357ac 3091 tcg_gen_bswap32_i64(swap, val, 0);
e1dcf352
RH		 3092 break;
3093 case MO_64:
3094 tcg_gen_bswap64_i64(swap, val);
3095 break;
3096 default:
3097 g_assert_not_reached();
3098 }
3099 val = swap;
3100 memop &= ~MO_BSWAP;
3101 }
3102
fcc54ab5 3103 addr = plugin_prep_mem_callbacks(addr);
c45cb8bb 3104 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
37aff087 3105 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
e1dcf352
RH		 3106
3107 if (swap) {
3108 tcg_temp_free_i64(swap);
3109 }
951c6300 3110}
c482cb11 3111
cb48f365
RH		 3112static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig)
3113{
3114 MemOp mop_1 = orig, mop_2;
3115
3116 tcg_debug_assert((orig & MO_SIZE) == MO_128);
3117 tcg_debug_assert((orig & MO_SIGN) == 0);
3118
3119 /* Use a memory ordering implemented by the host. */
3120 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (orig & MO_BSWAP)) {
3121 mop_1 &= ~MO_BSWAP;
3122 }
3123
3124 /* Reduce the size to 64-bit. */
3125 mop_1 = (mop_1 & ~MO_SIZE) | MO_64;
3126
3127 /* Retain the alignment constraints of the original. */
3128 switch (orig & MO_AMASK) {
3129 case MO_UNALN:
3130 case MO_ALIGN_2:
3131 case MO_ALIGN_4:
3132 mop_2 = mop_1;
3133 break;
3134 case MO_ALIGN_8:
3135 /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */
3136 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN;
3137 mop_2 = mop_1;
3138 break;
3139 case MO_ALIGN:
3140 /* Second has 8-byte alignment; first has 16-byte alignment. */
3141 mop_2 = mop_1;
3142 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16;
3143 break;
3144 case MO_ALIGN_16:
3145 case MO_ALIGN_32:
3146 case MO_ALIGN_64:
3147 /* Second has 8-byte alignment; first retains original. */
3148 mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN;
3149 break;
3150 default:
3151 g_assert_not_reached();
3152 }
3153 ret[0] = mop_1;
3154 ret[1] = mop_2;
3155}
3156
3157void tcg_gen_qemu_ld_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop)
3158{
3159 MemOp mop[2];
3160 TCGv addr_p8;
3161 TCGv_i64 x, y;
3162
3163 canonicalize_memop_i128_as_i64(mop, memop);
3164
3165 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
3166 addr = plugin_prep_mem_callbacks(addr);
3167
3168 /* TODO: respect atomicity of the operation. */
3169 /* TODO: allow the tcg backend to see the whole operation. */
3170
3171 /*
3172 * Since there are no global TCGv_i128, there is no visible state
3173 * changed if the second load faults. Load directly into the two
3174 * subwords.
3175 */
3176 if ((memop & MO_BSWAP) == MO_LE) {
3177 x = TCGV128_LOW(val);
3178 y = TCGV128_HIGH(val);
3179 } else {
3180 x = TCGV128_HIGH(val);
3181 y = TCGV128_LOW(val);
3182 }
3183
3184 gen_ldst_i64(INDEX_op_qemu_ld_i64, x, addr, mop[0], idx);
3185
3186 if ((mop[0] ^ memop) & MO_BSWAP) {
3187 tcg_gen_bswap64_i64(x, x);
3188 }
3189
3190 addr_p8 = tcg_temp_new();
3191 tcg_gen_addi_tl(addr_p8, addr, 8);
3192 gen_ldst_i64(INDEX_op_qemu_ld_i64, y, addr_p8, mop[1], idx);
3193 tcg_temp_free(addr_p8);
3194
3195 if ((mop[0] ^ memop) & MO_BSWAP) {
3196 tcg_gen_bswap64_i64(y, y);
3197 }
3198
3199 plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx),
3200 QEMU_PLUGIN_MEM_R);
3201}
3202
3203void tcg_gen_qemu_st_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop)
3204{
3205 MemOp mop[2];
3206 TCGv addr_p8;
3207 TCGv_i64 x, y;
3208
3209 canonicalize_memop_i128_as_i64(mop, memop);
3210
3211 tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST);
3212 addr = plugin_prep_mem_callbacks(addr);
3213
3214 /* TODO: respect atomicity of the operation. */
3215 /* TODO: allow the tcg backend to see the whole operation. */
3216
3217 if ((memop & MO_BSWAP) == MO_LE) {
3218 x = TCGV128_LOW(val);
3219 y = TCGV128_HIGH(val);
3220 } else {
3221 x = TCGV128_HIGH(val);
3222 y = TCGV128_LOW(val);
3223 }
3224
3225 addr_p8 = tcg_temp_new();
3226 if ((mop[0] ^ memop) & MO_BSWAP) {
3227 TCGv_i64 t = tcg_temp_new_i64();
3228
3229 tcg_gen_bswap64_i64(t, x);
3230 gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr, mop[0], idx);
3231 tcg_gen_bswap64_i64(t, y);
3232 tcg_gen_addi_tl(addr_p8, addr, 8);
3233 gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr_p8, mop[1], idx);
3234 tcg_temp_free_i64(t);
3235 } else {
3236 gen_ldst_i64(INDEX_op_qemu_st_i64, x, addr, mop[0], idx);
3237 tcg_gen_addi_tl(addr_p8, addr, 8);
3238 gen_ldst_i64(INDEX_op_qemu_st_i64, y, addr_p8, mop[1], idx);
3239 }
3240 tcg_temp_free(addr_p8);
3241
3242 plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx),
3243 QEMU_PLUGIN_MEM_W);
3244}
3245
14776ab5 3246static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
c482cb11
RH		 3247{
3248 switch (opc & MO_SSIZE) {
3249 case MO_SB:
3250 tcg_gen_ext8s_i32(ret, val);
3251 break;
3252 case MO_UB:
3253 tcg_gen_ext8u_i32(ret, val);
3254 break;
3255 case MO_SW:
3256 tcg_gen_ext16s_i32(ret, val);
3257 break;
3258 case MO_UW:
3259 tcg_gen_ext16u_i32(ret, val);
3260 break;
3261 default:
3262 tcg_gen_mov_i32(ret, val);
3263 break;
3264 }
3265}
3266
14776ab5 3267static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
c482cb11
RH		 3268{
3269 switch (opc & MO_SSIZE) {
3270 case MO_SB:
3271 tcg_gen_ext8s_i64(ret, val);
3272 break;
3273 case MO_UB:
3274 tcg_gen_ext8u_i64(ret, val);
3275 break;
3276 case MO_SW:
3277 tcg_gen_ext16s_i64(ret, val);
3278 break;
3279 case MO_UW:
3280 tcg_gen_ext16u_i64(ret, val);
3281 break;
3282 case MO_SL:
3283 tcg_gen_ext32s_i64(ret, val);
3284 break;
3285 case MO_UL:
3286 tcg_gen_ext32u_i64(ret, val);
3287 break;
3288 default:
3289 tcg_gen_mov_i64(ret, val);
3290 break;
3291 }
3292}
3293
c482cb11
RH		 3294typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
3295 TCGv_i32, TCGv_i32, TCGv_i32);
3296typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
3297 TCGv_i64, TCGv_i64, TCGv_i32);
123ae568
RH		 3298typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv,
3299 TCGv_i128, TCGv_i128, TCGv_i32);
c482cb11
RH		 3300typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
3301 TCGv_i32, TCGv_i32);
3302typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
3303 TCGv_i64, TCGv_i32);
c482cb11 3304
df79b996
RH		 3305#ifdef CONFIG_ATOMIC64
3306# define WITH_ATOMIC64(X) X,
3307#else
3308# define WITH_ATOMIC64(X)
3309#endif
123ae568
RH		 3310#ifdef CONFIG_CMPXCHG128
3311# define WITH_ATOMIC128(X) X,
3312#else
3313# define WITH_ATOMIC128(X)
3314#endif
df79b996 3315
4b473e0c 3316static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
c482cb11
RH		 3317 [MO_8] = gen_helper_atomic_cmpxchgb,
3318 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
3319 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
3320 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
3321 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
df79b996
RH		 3322 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
3323 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
123ae568
RH		 3324 WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le)
3325 WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be)
c482cb11
RH		 3326};
3327
d1beee4d
RH		 3328void tcg_gen_nonatomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
3329 TCGv_i32 newv, TCGArg idx, MemOp memop)
3330{
3331 TCGv_i32 t1 = tcg_temp_new_i32();
3332 TCGv_i32 t2 = tcg_temp_new_i32();
3333
3334 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
3335
3336 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3337 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
3338 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3339 tcg_temp_free_i32(t2);
3340
3341 if (memop & MO_SIGN) {
3342 tcg_gen_ext_i32(retv, t1, memop);
3343 } else {
3344 tcg_gen_mov_i32(retv, t1);
3345 }
3346 tcg_temp_free_i32(t1);
3347}
3348
c482cb11 3349void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
14776ab5 3350 TCGv_i32 newv, TCGArg idx, MemOp memop)
c482cb11 3351{
d1beee4d
RH		 3352 gen_atomic_cx_i32 gen;
3353 MemOpIdx oi;
c482cb11 3354
b7e4afbd 3355 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
d1beee4d
RH		 3356 tcg_gen_nonatomic_cmpxchg_i32(retv, addr, cmpv, newv, idx, memop);
3357 return;
3358 }
c482cb11 3359
d1beee4d
RH		 3360 memop = tcg_canonicalize_memop(memop, 0, 0);
3361 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3362 tcg_debug_assert(gen != NULL);
c482cb11 3363
d1beee4d
RH		 3364 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3365 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3366
3367 if (memop & MO_SIGN) {
3368 tcg_gen_ext_i32(retv, retv, memop);
3369 }
3370}
3371
3372void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
3373 TCGv_i64 newv, TCGArg idx, MemOp memop)
3374{
3375 TCGv_i64 t1, t2;
c482cb11 3376
d1beee4d
RH		 3377 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
3378 tcg_gen_nonatomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
3379 TCGV_LOW(newv), idx, memop);
c482cb11 3380 if (memop & MO_SIGN) {
d1beee4d 3381 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
c482cb11 3382 } else {
d1beee4d 3383 tcg_gen_movi_i32(TCGV_HIGH(retv), 0);
c482cb11 3384 }
d1beee4d
RH		 3385 return;
3386 }
c482cb11 3387
d1beee4d
RH		 3388 t1 = tcg_temp_new_i64();
3389 t2 = tcg_temp_new_i64();
c482cb11 3390
d1beee4d 3391 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
c482cb11 3392
d1beee4d
RH		 3393 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3394 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
3395 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3396 tcg_temp_free_i64(t2);
3397
3398 if (memop & MO_SIGN) {
3399 tcg_gen_ext_i64(retv, t1, memop);
3400 } else {
3401 tcg_gen_mov_i64(retv, t1);
c482cb11 3402 }
d1beee4d 3403 tcg_temp_free_i64(t1);
c482cb11
RH		 3404}
3405
3406void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
14776ab5 3407 TCGv_i64 newv, TCGArg idx, MemOp memop)
c482cb11 3408{
b7e4afbd 3409 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
d1beee4d
RH		 3410 tcg_gen_nonatomic_cmpxchg_i64(retv, addr, cmpv, newv, idx, memop);
3411 return;
3412 }
c482cb11 3413
d1beee4d 3414 if ((memop & MO_SIZE) == MO_64) {
c482cb11
RH		 3415 gen_atomic_cx_i64 gen;
3416
d1beee4d 3417 memop = tcg_canonicalize_memop(memop, 1, 0);
c482cb11 3418 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
d1beee4d
RH		 3419 if (gen) {
3420 MemOpIdx oi = make_memop_idx(memop, idx);
3421 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3422 return;
3423 }
c482cb11 3424
1c2adb95 3425 gen_helper_exit_atomic(cpu_env);
d1beee4d
RH		 3426
3427 /*
3428 * Produce a result for a well-formed opcode stream. This satisfies
3429 * liveness for set before used, which happens before this dead code
3430 * is removed.
3431 */
79b1af90 3432 tcg_gen_movi_i64(retv, 0);
d1beee4d
RH		 3433 return;
3434 }
3435
3436 if (TCG_TARGET_REG_BITS == 32) {
3437 tcg_gen_atomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
3438 TCGV_LOW(newv), idx, memop);
3439 if (memop & MO_SIGN) {
3440 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
3441 } else {
3442 tcg_gen_movi_i32(TCGV_HIGH(retv), 0);
3443 }
c482cb11
RH		 3444 } else {
3445 TCGv_i32 c32 = tcg_temp_new_i32();
3446 TCGv_i32 n32 = tcg_temp_new_i32();
3447 TCGv_i32 r32 = tcg_temp_new_i32();
3448
3449 tcg_gen_extrl_i64_i32(c32, cmpv);
3450 tcg_gen_extrl_i64_i32(n32, newv);
3451 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3452 tcg_temp_free_i32(c32);
3453 tcg_temp_free_i32(n32);
3454
3455 tcg_gen_extu_i32_i64(retv, r32);
3456 tcg_temp_free_i32(r32);
3457
3458 if (memop & MO_SIGN) {
3459 tcg_gen_ext_i64(retv, retv, memop);
3460 }
3461 }
3462}
3463
123ae568
RH		 3464void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
3465 TCGv_i128 newv, TCGArg idx, MemOp memop)
3466{
3467 if (TCG_TARGET_REG_BITS == 32) {
3468 /* Inline expansion below is simply too large for 32-bit hosts. */
3469 gen_atomic_cx_i128 gen = ((memop & MO_BSWAP) == MO_LE
3470 ? gen_helper_nonatomic_cmpxchgo_le
3471 : gen_helper_nonatomic_cmpxchgo_be);
3472 MemOpIdx oi = make_memop_idx(memop, idx);
3473
3474 tcg_debug_assert((memop & MO_SIZE) == MO_128);
3475 tcg_debug_assert((memop & MO_SIGN) == 0);
3476
3477 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3478 } else {
3479 TCGv_i128 oldv = tcg_temp_new_i128();
3480 TCGv_i128 tmpv = tcg_temp_new_i128();
3481 TCGv_i64 t0 = tcg_temp_new_i64();
3482 TCGv_i64 t1 = tcg_temp_new_i64();
3483 TCGv_i64 z = tcg_constant_i64(0);
3484
3485 tcg_gen_qemu_ld_i128(oldv, addr, idx, memop);
3486
3487 /* Compare i128 */
3488 tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv));
3489 tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv));
3490 tcg_gen_or_i64(t0, t0, t1);
3491
3492 /* tmpv = equal ? newv : oldv */
3493 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z,
3494 TCGV128_LOW(newv), TCGV128_LOW(oldv));
3495 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z,
3496 TCGV128_HIGH(newv), TCGV128_HIGH(oldv));
3497
3498 /* Unconditional writeback. */
3499 tcg_gen_qemu_st_i128(tmpv, addr, idx, memop);
3500 tcg_gen_mov_i128(retv, oldv);
3501
3502 tcg_temp_free_i64(t0);
3503 tcg_temp_free_i64(t1);
3504 tcg_temp_free_i128(tmpv);
3505 tcg_temp_free_i128(oldv);
3506 }
3507}
3508
3509void tcg_gen_atomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
3510 TCGv_i128 newv, TCGArg idx, MemOp memop)
3511{
3512 gen_atomic_cx_i128 gen;
3513
3514 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
3515 tcg_gen_nonatomic_cmpxchg_i128(retv, addr, cmpv, newv, idx, memop);
3516 return;
3517 }
3518
3519 tcg_debug_assert((memop & MO_SIZE) == MO_128);
3520 tcg_debug_assert((memop & MO_SIGN) == 0);
3521 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3522
3523 if (gen) {
3524 MemOpIdx oi = make_memop_idx(memop, idx);
3525 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3526 return;
3527 }
3528
3529 gen_helper_exit_atomic(cpu_env);
3530
3531 /*
3532 * Produce a result for a well-formed opcode stream. This satisfies
3533 * liveness for set before used, which happens before this dead code
3534 * is removed.
3535 */
3536 tcg_gen_movi_i64(TCGV128_LOW(retv), 0);
3537 tcg_gen_movi_i64(TCGV128_HIGH(retv), 0);
3538}
3539
c482cb11 3540static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
14776ab5 3541 TCGArg idx, MemOp memop, bool new_val,
c482cb11
RH		 3542 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3543{
3544 TCGv_i32 t1 = tcg_temp_new_i32();
3545 TCGv_i32 t2 = tcg_temp_new_i32();
3546
3547 memop = tcg_canonicalize_memop(memop, 0, 0);
3548
852f933e
RH		 3549 tcg_gen_qemu_ld_i32(t1, addr, idx, memop);
3550 tcg_gen_ext_i32(t2, val, memop);
3551 gen(t2, t1, t2);
c482cb11
RH		 3552 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3553
3554 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3555 tcg_temp_free_i32(t1);
3556 tcg_temp_free_i32(t2);
3557}
3558
3559static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
14776ab5 3560 TCGArg idx, MemOp memop, void * const table[])
c482cb11
RH		 3561{
3562 gen_atomic_op_i32 gen;
9002ffcb 3563 MemOpIdx oi;
c482cb11
RH		 3564
3565 memop = tcg_canonicalize_memop(memop, 0, 0);
3566
3567 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3568 tcg_debug_assert(gen != NULL);
3569
e28a8664
RH		 3570 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3571 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
c482cb11
RH		 3572
3573 if (memop & MO_SIGN) {
3574 tcg_gen_ext_i32(ret, ret, memop);
3575 }
3576}
3577
3578static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
14776ab5 3579 TCGArg idx, MemOp memop, bool new_val,
c482cb11
RH		 3580 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3581{
3582 TCGv_i64 t1 = tcg_temp_new_i64();
3583 TCGv_i64 t2 = tcg_temp_new_i64();
3584
3585 memop = tcg_canonicalize_memop(memop, 1, 0);
3586
852f933e
RH		 3587 tcg_gen_qemu_ld_i64(t1, addr, idx, memop);
3588 tcg_gen_ext_i64(t2, val, memop);
3589 gen(t2, t1, t2);
c482cb11
RH		 3590 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3591
3592 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3593 tcg_temp_free_i64(t1);
3594 tcg_temp_free_i64(t2);
3595}
3596
3597static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
14776ab5 3598 TCGArg idx, MemOp memop, void * const table[])
c482cb11
RH		 3599{
3600 memop = tcg_canonicalize_memop(memop, 1, 0);
3601
3602 if ((memop & MO_SIZE) == MO_64) {
df79b996 3603#ifdef CONFIG_ATOMIC64
c482cb11 3604 gen_atomic_op_i64 gen;
9002ffcb 3605 MemOpIdx oi;
c482cb11
RH		 3606
3607 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3608 tcg_debug_assert(gen != NULL);
3609
e28a8664
RH		 3610 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3611 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
df79b996 3612#else
1c2adb95 3613 gen_helper_exit_atomic(cpu_env);
79b1af90
RH		 3614 /* Produce a result, so that we have a well-formed opcode stream
3615 with respect to uses of the result in the (dead) code following. */
3616 tcg_gen_movi_i64(ret, 0);
df79b996 3617#endif /* CONFIG_ATOMIC64 */
c482cb11
RH		 3618 } else {
3619 TCGv_i32 v32 = tcg_temp_new_i32();
3620 TCGv_i32 r32 = tcg_temp_new_i32();
3621
3622 tcg_gen_extrl_i64_i32(v32, val);
3623 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3624 tcg_temp_free_i32(v32);
3625
3626 tcg_gen_extu_i32_i64(ret, r32);
3627 tcg_temp_free_i32(r32);
3628
3629 if (memop & MO_SIGN) {
3630 tcg_gen_ext_i64(ret, ret, memop);
3631 }
3632 }
3633}
3634
3635#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
4b473e0c 3636static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
c482cb11
RH		 3637 [MO_8] = gen_helper_atomic_##NAME##b, \
3638 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3639 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3640 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3641 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
df79b996
RH		 3642 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3643 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
c482cb11
RH		 3644}; \
3645void tcg_gen_atomic_##NAME##_i32 \
14776ab5 3646 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \
c482cb11 3647{ \
b7e4afbd 3648 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
c482cb11
RH		 3649 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3650 } else { \
3651 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3652 tcg_gen_##OP##_i32); \
3653 } \
3654} \
3655void tcg_gen_atomic_##NAME##_i64 \
14776ab5 3656 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \
c482cb11 3657{ \
b7e4afbd 3658 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
c482cb11
RH		 3659 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3660 } else { \
3661 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3662 tcg_gen_##OP##_i64); \
3663 } \
3664}
3665
3666GEN_ATOMIC_HELPER(fetch_add, add, 0)
3667GEN_ATOMIC_HELPER(fetch_and, and, 0)
3668GEN_ATOMIC_HELPER(fetch_or, or, 0)
3669GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
5507c2bf
RH		 3670GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3671GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3672GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3673GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
c482cb11
RH		 3674
3675GEN_ATOMIC_HELPER(add_fetch, add, 1)
3676GEN_ATOMIC_HELPER(and_fetch, and, 1)
3677GEN_ATOMIC_HELPER(or_fetch, or, 1)
3678GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
5507c2bf
RH		 3679GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3680GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3681GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3682GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
c482cb11
RH		 3683
3684static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3685{
3686 tcg_gen_mov_i32(r, b);
3687}
3688
3689static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3690{
3691 tcg_gen_mov_i64(r, b);
3692}
3693
3694GEN_ATOMIC_HELPER(xchg, mov2, 0)
3695
3696#undef GEN_ATOMIC_HELPER
QEMU mirror