]> git.proxmox.com Git - mirror_qemu.git/blob - tcg/tcg-op.c
projects / mirror_qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'remotes/vsementsov/tags/pull-jobs-2021-10-07-v2' into...
[mirror_qemu.git] / tcg / tcg-op.c
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "exec/exec-all.h"
27 #include "tcg/tcg.h"
28 #include "tcg/tcg-op.h"
29 #include "tcg/tcg-mo.h"
30 #include "trace-tcg.h"
31 #include "exec/plugin-gen.h"
32
33 /* Reduce the number of ifdefs below. This assumes that all uses of
34 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
35 the compiler can eliminate. */
36 #if TCG_TARGET_REG_BITS == 64
37 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
38 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
39 #define TCGV_LOW TCGV_LOW_link_error
40 #define TCGV_HIGH TCGV_HIGH_link_error
41 #endif
42
43 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
44 {
45 TCGOp *op = tcg_emit_op(opc);
46 op->args[0] = a1;
47 }
48
49 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
50 {
51 TCGOp *op = tcg_emit_op(opc);
52 op->args[0] = a1;
53 op->args[1] = a2;
54 }
55
56 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
57 {
58 TCGOp *op = tcg_emit_op(opc);
59 op->args[0] = a1;
60 op->args[1] = a2;
61 op->args[2] = a3;
62 }
63
64 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
65 {
66 TCGOp *op = tcg_emit_op(opc);
67 op->args[0] = a1;
68 op->args[1] = a2;
69 op->args[2] = a3;
70 op->args[3] = a4;
71 }
72
73 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
74 TCGArg a4, TCGArg a5)
75 {
76 TCGOp *op = tcg_emit_op(opc);
77 op->args[0] = a1;
78 op->args[1] = a2;
79 op->args[2] = a3;
80 op->args[3] = a4;
81 op->args[4] = a5;
82 }
83
84 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
85 TCGArg a4, TCGArg a5, TCGArg a6)
86 {
87 TCGOp *op = tcg_emit_op(opc);
88 op->args[0] = a1;
89 op->args[1] = a2;
90 op->args[2] = a3;
91 op->args[3] = a4;
92 op->args[4] = a5;
93 op->args[5] = a6;
94 }
95
96 void tcg_gen_mb(TCGBar mb_type)
97 {
98 if (tcg_ctx->tb_cflags & CF_PARALLEL) {
99 tcg_gen_op1(INDEX_op_mb, mb_type);
100 }
101 }
102
103 /* 32 bit ops */
104
105 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
106 {
107 tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
108 }
109
110 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
111 {
112 /* some cases can be optimized here */
113 if (arg2 == 0) {
114 tcg_gen_mov_i32(ret, arg1);
115 } else {
116 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
117 }
118 }
119
120 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
121 {
122 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
123 /* Don't recurse with tcg_gen_neg_i32. */
124 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
125 } else {
126 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
127 }
128 }
129
130 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
131 {
132 /* some cases can be optimized here */
133 if (arg2 == 0) {
134 tcg_gen_mov_i32(ret, arg1);
135 } else {
136 tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2));
137 }
138 }
139
140 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
141 {
142 /* Some cases can be optimized here. */
143 switch (arg2) {
144 case 0:
145 tcg_gen_movi_i32(ret, 0);
146 return;
147 case -1:
148 tcg_gen_mov_i32(ret, arg1);
149 return;
150 case 0xff:
151 /* Don't recurse with tcg_gen_ext8u_i32. */
152 if (TCG_TARGET_HAS_ext8u_i32) {
153 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
154 return;
155 }
156 break;
157 case 0xffff:
158 if (TCG_TARGET_HAS_ext16u_i32) {
159 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
160 return;
161 }
162 break;
163 }
164
165 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
166 }
167
168 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
169 {
170 /* Some cases can be optimized here. */
171 if (arg2 == -1) {
172 tcg_gen_movi_i32(ret, -1);
173 } else if (arg2 == 0) {
174 tcg_gen_mov_i32(ret, arg1);
175 } else {
176 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
177 }
178 }
179
180 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
181 {
182 /* Some cases can be optimized here. */
183 if (arg2 == 0) {
184 tcg_gen_mov_i32(ret, arg1);
185 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
186 /* Don't recurse with tcg_gen_not_i32. */
187 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
188 } else {
189 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
190 }
191 }
192
193 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
194 {
195 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
196 if (arg2 == 0) {
197 tcg_gen_mov_i32(ret, arg1);
198 } else {
199 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
200 }
201 }
202
203 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
204 {
205 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
206 if (arg2 == 0) {
207 tcg_gen_mov_i32(ret, arg1);
208 } else {
209 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
210 }
211 }
212
213 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
214 {
215 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
216 if (arg2 == 0) {
217 tcg_gen_mov_i32(ret, arg1);
218 } else {
219 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
220 }
221 }
222
223 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
224 {
225 if (cond == TCG_COND_ALWAYS) {
226 tcg_gen_br(l);
227 } else if (cond != TCG_COND_NEVER) {
228 l->refs++;
229 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
230 }
231 }
232
233 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
234 {
235 if (cond == TCG_COND_ALWAYS) {
236 tcg_gen_br(l);
237 } else if (cond != TCG_COND_NEVER) {
238 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
239 }
240 }
241
242 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
243 TCGv_i32 arg1, TCGv_i32 arg2)
244 {
245 if (cond == TCG_COND_ALWAYS) {
246 tcg_gen_movi_i32(ret, 1);
247 } else if (cond == TCG_COND_NEVER) {
248 tcg_gen_movi_i32(ret, 0);
249 } else {
250 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
251 }
252 }
253
254 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
255 TCGv_i32 arg1, int32_t arg2)
256 {
257 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
258 }
259
260 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
261 {
262 if (arg2 == 0) {
263 tcg_gen_movi_i32(ret, 0);
264 } else if (is_power_of_2(arg2)) {
265 tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
266 } else {
267 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
268 }
269 }
270
271 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
272 {
273 if (TCG_TARGET_HAS_div_i32) {
274 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
275 } else if (TCG_TARGET_HAS_div2_i32) {
276 TCGv_i32 t0 = tcg_temp_new_i32();
277 tcg_gen_sari_i32(t0, arg1, 31);
278 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
279 tcg_temp_free_i32(t0);
280 } else {
281 gen_helper_div_i32(ret, arg1, arg2);
282 }
283 }
284
285 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
286 {
287 if (TCG_TARGET_HAS_rem_i32) {
288 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
289 } else if (TCG_TARGET_HAS_div_i32) {
290 TCGv_i32 t0 = tcg_temp_new_i32();
291 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
292 tcg_gen_mul_i32(t0, t0, arg2);
293 tcg_gen_sub_i32(ret, arg1, t0);
294 tcg_temp_free_i32(t0);
295 } else if (TCG_TARGET_HAS_div2_i32) {
296 TCGv_i32 t0 = tcg_temp_new_i32();
297 tcg_gen_sari_i32(t0, arg1, 31);
298 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
299 tcg_temp_free_i32(t0);
300 } else {
301 gen_helper_rem_i32(ret, arg1, arg2);
302 }
303 }
304
305 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
306 {
307 if (TCG_TARGET_HAS_div_i32) {
308 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
309 } else if (TCG_TARGET_HAS_div2_i32) {
310 TCGv_i32 t0 = tcg_temp_new_i32();
311 tcg_gen_movi_i32(t0, 0);
312 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
313 tcg_temp_free_i32(t0);
314 } else {
315 gen_helper_divu_i32(ret, arg1, arg2);
316 }
317 }
318
319 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
320 {
321 if (TCG_TARGET_HAS_rem_i32) {
322 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
323 } else if (TCG_TARGET_HAS_div_i32) {
324 TCGv_i32 t0 = tcg_temp_new_i32();
325 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
326 tcg_gen_mul_i32(t0, t0, arg2);
327 tcg_gen_sub_i32(ret, arg1, t0);
328 tcg_temp_free_i32(t0);
329 } else if (TCG_TARGET_HAS_div2_i32) {
330 TCGv_i32 t0 = tcg_temp_new_i32();
331 tcg_gen_movi_i32(t0, 0);
332 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
333 tcg_temp_free_i32(t0);
334 } else {
335 gen_helper_remu_i32(ret, arg1, arg2);
336 }
337 }
338
339 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
340 {
341 if (TCG_TARGET_HAS_andc_i32) {
342 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
343 } else {
344 TCGv_i32 t0 = tcg_temp_new_i32();
345 tcg_gen_not_i32(t0, arg2);
346 tcg_gen_and_i32(ret, arg1, t0);
347 tcg_temp_free_i32(t0);
348 }
349 }
350
351 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
352 {
353 if (TCG_TARGET_HAS_eqv_i32) {
354 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
355 } else {
356 tcg_gen_xor_i32(ret, arg1, arg2);
357 tcg_gen_not_i32(ret, ret);
358 }
359 }
360
361 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
362 {
363 if (TCG_TARGET_HAS_nand_i32) {
364 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
365 } else {
366 tcg_gen_and_i32(ret, arg1, arg2);
367 tcg_gen_not_i32(ret, ret);
368 }
369 }
370
371 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
372 {
373 if (TCG_TARGET_HAS_nor_i32) {
374 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
375 } else {
376 tcg_gen_or_i32(ret, arg1, arg2);
377 tcg_gen_not_i32(ret, ret);
378 }
379 }
380
381 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
382 {
383 if (TCG_TARGET_HAS_orc_i32) {
384 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
385 } else {
386 TCGv_i32 t0 = tcg_temp_new_i32();
387 tcg_gen_not_i32(t0, arg2);
388 tcg_gen_or_i32(ret, arg1, t0);
389 tcg_temp_free_i32(t0);
390 }
391 }
392
393 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
394 {
395 if (TCG_TARGET_HAS_clz_i32) {
396 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
397 } else if (TCG_TARGET_HAS_clz_i64) {
398 TCGv_i64 t1 = tcg_temp_new_i64();
399 TCGv_i64 t2 = tcg_temp_new_i64();
400 tcg_gen_extu_i32_i64(t1, arg1);
401 tcg_gen_extu_i32_i64(t2, arg2);
402 tcg_gen_addi_i64(t2, t2, 32);
403 tcg_gen_clz_i64(t1, t1, t2);
404 tcg_gen_extrl_i64_i32(ret, t1);
405 tcg_temp_free_i64(t1);
406 tcg_temp_free_i64(t2);
407 tcg_gen_subi_i32(ret, ret, 32);
408 } else {
409 gen_helper_clz_i32(ret, arg1, arg2);
410 }
411 }
412
413 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
414 {
415 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
416 }
417
418 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
419 {
420 if (TCG_TARGET_HAS_ctz_i32) {
421 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
422 } else if (TCG_TARGET_HAS_ctz_i64) {
423 TCGv_i64 t1 = tcg_temp_new_i64();
424 TCGv_i64 t2 = tcg_temp_new_i64();
425 tcg_gen_extu_i32_i64(t1, arg1);
426 tcg_gen_extu_i32_i64(t2, arg2);
427 tcg_gen_ctz_i64(t1, t1, t2);
428 tcg_gen_extrl_i64_i32(ret, t1);
429 tcg_temp_free_i64(t1);
430 tcg_temp_free_i64(t2);
431 } else if (TCG_TARGET_HAS_ctpop_i32
432 || TCG_TARGET_HAS_ctpop_i64
433 || TCG_TARGET_HAS_clz_i32
434 || TCG_TARGET_HAS_clz_i64) {
435 TCGv_i32 z, t = tcg_temp_new_i32();
436
437 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
438 tcg_gen_subi_i32(t, arg1, 1);
439 tcg_gen_andc_i32(t, t, arg1);
440 tcg_gen_ctpop_i32(t, t);
441 } else {
442 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
443 tcg_gen_neg_i32(t, arg1);
444 tcg_gen_and_i32(t, t, arg1);
445 tcg_gen_clzi_i32(t, t, 32);
446 tcg_gen_xori_i32(t, t, 31);
447 }
448 z = tcg_constant_i32(0);
449 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
450 tcg_temp_free_i32(t);
451 } else {
452 gen_helper_ctz_i32(ret, arg1, arg2);
453 }
454 }
455
456 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
457 {
458 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
459 /* This equivalence has the advantage of not requiring a fixup. */
460 TCGv_i32 t = tcg_temp_new_i32();
461 tcg_gen_subi_i32(t, arg1, 1);
462 tcg_gen_andc_i32(t, t, arg1);
463 tcg_gen_ctpop_i32(ret, t);
464 tcg_temp_free_i32(t);
465 } else {
466 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
467 }
468 }
469
470 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
471 {
472 if (TCG_TARGET_HAS_clz_i32) {
473 TCGv_i32 t = tcg_temp_new_i32();
474 tcg_gen_sari_i32(t, arg, 31);
475 tcg_gen_xor_i32(t, t, arg);
476 tcg_gen_clzi_i32(t, t, 32);
477 tcg_gen_subi_i32(ret, t, 1);
478 tcg_temp_free_i32(t);
479 } else {
480 gen_helper_clrsb_i32(ret, arg);
481 }
482 }
483
484 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
485 {
486 if (TCG_TARGET_HAS_ctpop_i32) {
487 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
488 } else if (TCG_TARGET_HAS_ctpop_i64) {
489 TCGv_i64 t = tcg_temp_new_i64();
490 tcg_gen_extu_i32_i64(t, arg1);
491 tcg_gen_ctpop_i64(t, t);
492 tcg_gen_extrl_i64_i32(ret, t);
493 tcg_temp_free_i64(t);
494 } else {
495 gen_helper_ctpop_i32(ret, arg1);
496 }
497 }
498
499 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
500 {
501 if (TCG_TARGET_HAS_rot_i32) {
502 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
503 } else {
504 TCGv_i32 t0, t1;
505
506 t0 = tcg_temp_new_i32();
507 t1 = tcg_temp_new_i32();
508 tcg_gen_shl_i32(t0, arg1, arg2);
509 tcg_gen_subfi_i32(t1, 32, arg2);
510 tcg_gen_shr_i32(t1, arg1, t1);
511 tcg_gen_or_i32(ret, t0, t1);
512 tcg_temp_free_i32(t0);
513 tcg_temp_free_i32(t1);
514 }
515 }
516
517 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
518 {
519 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
520 /* some cases can be optimized here */
521 if (arg2 == 0) {
522 tcg_gen_mov_i32(ret, arg1);
523 } else if (TCG_TARGET_HAS_rot_i32) {
524 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2));
525 } else {
526 TCGv_i32 t0, t1;
527 t0 = tcg_temp_new_i32();
528 t1 = tcg_temp_new_i32();
529 tcg_gen_shli_i32(t0, arg1, arg2);
530 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
531 tcg_gen_or_i32(ret, t0, t1);
532 tcg_temp_free_i32(t0);
533 tcg_temp_free_i32(t1);
534 }
535 }
536
537 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
538 {
539 if (TCG_TARGET_HAS_rot_i32) {
540 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
541 } else {
542 TCGv_i32 t0, t1;
543
544 t0 = tcg_temp_new_i32();
545 t1 = tcg_temp_new_i32();
546 tcg_gen_shr_i32(t0, arg1, arg2);
547 tcg_gen_subfi_i32(t1, 32, arg2);
548 tcg_gen_shl_i32(t1, arg1, t1);
549 tcg_gen_or_i32(ret, t0, t1);
550 tcg_temp_free_i32(t0);
551 tcg_temp_free_i32(t1);
552 }
553 }
554
555 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
556 {
557 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
558 /* some cases can be optimized here */
559 if (arg2 == 0) {
560 tcg_gen_mov_i32(ret, arg1);
561 } else {
562 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
563 }
564 }
565
566 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
567 unsigned int ofs, unsigned int len)
568 {
569 uint32_t mask;
570 TCGv_i32 t1;
571
572 tcg_debug_assert(ofs < 32);
573 tcg_debug_assert(len > 0);
574 tcg_debug_assert(len <= 32);
575 tcg_debug_assert(ofs + len <= 32);
576
577 if (len == 32) {
578 tcg_gen_mov_i32(ret, arg2);
579 return;
580 }
581 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
582 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
583 return;
584 }
585
586 t1 = tcg_temp_new_i32();
587
588 if (TCG_TARGET_HAS_extract2_i32) {
589 if (ofs + len == 32) {
590 tcg_gen_shli_i32(t1, arg1, len);
591 tcg_gen_extract2_i32(ret, t1, arg2, len);
592 goto done;
593 }
594 if (ofs == 0) {
595 tcg_gen_extract2_i32(ret, arg1, arg2, len);
596 tcg_gen_rotli_i32(ret, ret, len);
597 goto done;
598 }
599 }
600
601 mask = (1u << len) - 1;
602 if (ofs + len < 32) {
603 tcg_gen_andi_i32(t1, arg2, mask);
604 tcg_gen_shli_i32(t1, t1, ofs);
605 } else {
606 tcg_gen_shli_i32(t1, arg2, ofs);
607 }
608 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
609 tcg_gen_or_i32(ret, ret, t1);
610 done:
611 tcg_temp_free_i32(t1);
612 }
613
614 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
615 unsigned int ofs, unsigned int len)
616 {
617 tcg_debug_assert(ofs < 32);
618 tcg_debug_assert(len > 0);
619 tcg_debug_assert(len <= 32);
620 tcg_debug_assert(ofs + len <= 32);
621
622 if (ofs + len == 32) {
623 tcg_gen_shli_i32(ret, arg, ofs);
624 } else if (ofs == 0) {
625 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
626 } else if (TCG_TARGET_HAS_deposit_i32
627 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
628 TCGv_i32 zero = tcg_constant_i32(0);
629 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
630 } else {
631 /* To help two-operand hosts we prefer to zero-extend first,
632 which allows ARG to stay live. */
633 switch (len) {
634 case 16:
635 if (TCG_TARGET_HAS_ext16u_i32) {
636 tcg_gen_ext16u_i32(ret, arg);
637 tcg_gen_shli_i32(ret, ret, ofs);
638 return;
639 }
640 break;
641 case 8:
642 if (TCG_TARGET_HAS_ext8u_i32) {
643 tcg_gen_ext8u_i32(ret, arg);
644 tcg_gen_shli_i32(ret, ret, ofs);
645 return;
646 }
647 break;
648 }
649 /* Otherwise prefer zero-extension over AND for code size. */
650 switch (ofs + len) {
651 case 16:
652 if (TCG_TARGET_HAS_ext16u_i32) {
653 tcg_gen_shli_i32(ret, arg, ofs);
654 tcg_gen_ext16u_i32(ret, ret);
655 return;
656 }
657 break;
658 case 8:
659 if (TCG_TARGET_HAS_ext8u_i32) {
660 tcg_gen_shli_i32(ret, arg, ofs);
661 tcg_gen_ext8u_i32(ret, ret);
662 return;
663 }
664 break;
665 }
666 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
667 tcg_gen_shli_i32(ret, ret, ofs);
668 }
669 }
670
671 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
672 unsigned int ofs, unsigned int len)
673 {
674 tcg_debug_assert(ofs < 32);
675 tcg_debug_assert(len > 0);
676 tcg_debug_assert(len <= 32);
677 tcg_debug_assert(ofs + len <= 32);
678
679 /* Canonicalize certain special cases, even if extract is supported. */
680 if (ofs + len == 32) {
681 tcg_gen_shri_i32(ret, arg, 32 - len);
682 return;
683 }
684 if (ofs == 0) {
685 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
686 return;
687 }
688
689 if (TCG_TARGET_HAS_extract_i32
690 && TCG_TARGET_extract_i32_valid(ofs, len)) {
691 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
692 return;
693 }
694
695 /* Assume that zero-extension, if available, is cheaper than a shift. */
696 switch (ofs + len) {
697 case 16:
698 if (TCG_TARGET_HAS_ext16u_i32) {
699 tcg_gen_ext16u_i32(ret, arg);
700 tcg_gen_shri_i32(ret, ret, ofs);
701 return;
702 }
703 break;
704 case 8:
705 if (TCG_TARGET_HAS_ext8u_i32) {
706 tcg_gen_ext8u_i32(ret, arg);
707 tcg_gen_shri_i32(ret, ret, ofs);
708 return;
709 }
710 break;
711 }
712
713 /* ??? Ideally we'd know what values are available for immediate AND.
714 Assume that 8 bits are available, plus the special case of 16,
715 so that we get ext8u, ext16u. */
716 switch (len) {
717 case 1 ... 8: case 16:
718 tcg_gen_shri_i32(ret, arg, ofs);
719 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
720 break;
721 default:
722 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
723 tcg_gen_shri_i32(ret, ret, 32 - len);
724 break;
725 }
726 }
727
728 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
729 unsigned int ofs, unsigned int len)
730 {
731 tcg_debug_assert(ofs < 32);
732 tcg_debug_assert(len > 0);
733 tcg_debug_assert(len <= 32);
734 tcg_debug_assert(ofs + len <= 32);
735
736 /* Canonicalize certain special cases, even if extract is supported. */
737 if (ofs + len == 32) {
738 tcg_gen_sari_i32(ret, arg, 32 - len);
739 return;
740 }
741 if (ofs == 0) {
742 switch (len) {
743 case 16:
744 tcg_gen_ext16s_i32(ret, arg);
745 return;
746 case 8:
747 tcg_gen_ext8s_i32(ret, arg);
748 return;
749 }
750 }
751
752 if (TCG_TARGET_HAS_sextract_i32
753 && TCG_TARGET_extract_i32_valid(ofs, len)) {
754 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
755 return;
756 }
757
758 /* Assume that sign-extension, if available, is cheaper than a shift. */
759 switch (ofs + len) {
760 case 16:
761 if (TCG_TARGET_HAS_ext16s_i32) {
762 tcg_gen_ext16s_i32(ret, arg);
763 tcg_gen_sari_i32(ret, ret, ofs);
764 return;
765 }
766 break;
767 case 8:
768 if (TCG_TARGET_HAS_ext8s_i32) {
769 tcg_gen_ext8s_i32(ret, arg);
770 tcg_gen_sari_i32(ret, ret, ofs);
771 return;
772 }
773 break;
774 }
775 switch (len) {
776 case 16:
777 if (TCG_TARGET_HAS_ext16s_i32) {
778 tcg_gen_shri_i32(ret, arg, ofs);
779 tcg_gen_ext16s_i32(ret, ret);
780 return;
781 }
782 break;
783 case 8:
784 if (TCG_TARGET_HAS_ext8s_i32) {
785 tcg_gen_shri_i32(ret, arg, ofs);
786 tcg_gen_ext8s_i32(ret, ret);
787 return;
788 }
789 break;
790 }
791
792 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
793 tcg_gen_sari_i32(ret, ret, 32 - len);
794 }
795
796 /*
797 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
798 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
799 */
800 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
801 unsigned int ofs)
802 {
803 tcg_debug_assert(ofs <= 32);
804 if (ofs == 0) {
805 tcg_gen_mov_i32(ret, al);
806 } else if (ofs == 32) {
807 tcg_gen_mov_i32(ret, ah);
808 } else if (al == ah) {
809 tcg_gen_rotri_i32(ret, al, ofs);
810 } else if (TCG_TARGET_HAS_extract2_i32) {
811 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
812 } else {
813 TCGv_i32 t0 = tcg_temp_new_i32();
814 tcg_gen_shri_i32(t0, al, ofs);
815 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
816 tcg_temp_free_i32(t0);
817 }
818 }
819
820 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
821 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
822 {
823 if (cond == TCG_COND_ALWAYS) {
824 tcg_gen_mov_i32(ret, v1);
825 } else if (cond == TCG_COND_NEVER) {
826 tcg_gen_mov_i32(ret, v2);
827 } else if (TCG_TARGET_HAS_movcond_i32) {
828 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
829 } else {
830 TCGv_i32 t0 = tcg_temp_new_i32();
831 TCGv_i32 t1 = tcg_temp_new_i32();
832 tcg_gen_setcond_i32(cond, t0, c1, c2);
833 tcg_gen_neg_i32(t0, t0);
834 tcg_gen_and_i32(t1, v1, t0);
835 tcg_gen_andc_i32(ret, v2, t0);
836 tcg_gen_or_i32(ret, ret, t1);
837 tcg_temp_free_i32(t0);
838 tcg_temp_free_i32(t1);
839 }
840 }
841
842 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
843 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
844 {
845 if (TCG_TARGET_HAS_add2_i32) {
846 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
847 } else {
848 TCGv_i64 t0 = tcg_temp_new_i64();
849 TCGv_i64 t1 = tcg_temp_new_i64();
850 tcg_gen_concat_i32_i64(t0, al, ah);
851 tcg_gen_concat_i32_i64(t1, bl, bh);
852 tcg_gen_add_i64(t0, t0, t1);
853 tcg_gen_extr_i64_i32(rl, rh, t0);
854 tcg_temp_free_i64(t0);
855 tcg_temp_free_i64(t1);
856 }
857 }
858
859 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
860 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
861 {
862 if (TCG_TARGET_HAS_sub2_i32) {
863 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
864 } else {
865 TCGv_i64 t0 = tcg_temp_new_i64();
866 TCGv_i64 t1 = tcg_temp_new_i64();
867 tcg_gen_concat_i32_i64(t0, al, ah);
868 tcg_gen_concat_i32_i64(t1, bl, bh);
869 tcg_gen_sub_i64(t0, t0, t1);
870 tcg_gen_extr_i64_i32(rl, rh, t0);
871 tcg_temp_free_i64(t0);
872 tcg_temp_free_i64(t1);
873 }
874 }
875
876 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
877 {
878 if (TCG_TARGET_HAS_mulu2_i32) {
879 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
880 } else if (TCG_TARGET_HAS_muluh_i32) {
881 TCGv_i32 t = tcg_temp_new_i32();
882 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
883 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
884 tcg_gen_mov_i32(rl, t);
885 tcg_temp_free_i32(t);
886 } else {
887 TCGv_i64 t0 = tcg_temp_new_i64();
888 TCGv_i64 t1 = tcg_temp_new_i64();
889 tcg_gen_extu_i32_i64(t0, arg1);
890 tcg_gen_extu_i32_i64(t1, arg2);
891 tcg_gen_mul_i64(t0, t0, t1);
892 tcg_gen_extr_i64_i32(rl, rh, t0);
893 tcg_temp_free_i64(t0);
894 tcg_temp_free_i64(t1);
895 }
896 }
897
898 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
899 {
900 if (TCG_TARGET_HAS_muls2_i32) {
901 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
902 } else if (TCG_TARGET_HAS_mulsh_i32) {
903 TCGv_i32 t = tcg_temp_new_i32();
904 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
905 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
906 tcg_gen_mov_i32(rl, t);
907 tcg_temp_free_i32(t);
908 } else if (TCG_TARGET_REG_BITS == 32) {
909 TCGv_i32 t0 = tcg_temp_new_i32();
910 TCGv_i32 t1 = tcg_temp_new_i32();
911 TCGv_i32 t2 = tcg_temp_new_i32();
912 TCGv_i32 t3 = tcg_temp_new_i32();
913 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
914 /* Adjust for negative inputs. */
915 tcg_gen_sari_i32(t2, arg1, 31);
916 tcg_gen_sari_i32(t3, arg2, 31);
917 tcg_gen_and_i32(t2, t2, arg2);
918 tcg_gen_and_i32(t3, t3, arg1);
919 tcg_gen_sub_i32(rh, t1, t2);
920 tcg_gen_sub_i32(rh, rh, t3);
921 tcg_gen_mov_i32(rl, t0);
922 tcg_temp_free_i32(t0);
923 tcg_temp_free_i32(t1);
924 tcg_temp_free_i32(t2);
925 tcg_temp_free_i32(t3);
926 } else {
927 TCGv_i64 t0 = tcg_temp_new_i64();
928 TCGv_i64 t1 = tcg_temp_new_i64();
929 tcg_gen_ext_i32_i64(t0, arg1);
930 tcg_gen_ext_i32_i64(t1, arg2);
931 tcg_gen_mul_i64(t0, t0, t1);
932 tcg_gen_extr_i64_i32(rl, rh, t0);
933 tcg_temp_free_i64(t0);
934 tcg_temp_free_i64(t1);
935 }
936 }
937
938 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
939 {
940 if (TCG_TARGET_REG_BITS == 32) {
941 TCGv_i32 t0 = tcg_temp_new_i32();
942 TCGv_i32 t1 = tcg_temp_new_i32();
943 TCGv_i32 t2 = tcg_temp_new_i32();
944 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
945 /* Adjust for negative input for the signed arg1. */
946 tcg_gen_sari_i32(t2, arg1, 31);
947 tcg_gen_and_i32(t2, t2, arg2);
948 tcg_gen_sub_i32(rh, t1, t2);
949 tcg_gen_mov_i32(rl, t0);
950 tcg_temp_free_i32(t0);
951 tcg_temp_free_i32(t1);
952 tcg_temp_free_i32(t2);
953 } else {
954 TCGv_i64 t0 = tcg_temp_new_i64();
955 TCGv_i64 t1 = tcg_temp_new_i64();
956 tcg_gen_ext_i32_i64(t0, arg1);
957 tcg_gen_extu_i32_i64(t1, arg2);
958 tcg_gen_mul_i64(t0, t0, t1);
959 tcg_gen_extr_i64_i32(rl, rh, t0);
960 tcg_temp_free_i64(t0);
961 tcg_temp_free_i64(t1);
962 }
963 }
964
965 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
966 {
967 if (TCG_TARGET_HAS_ext8s_i32) {
968 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
969 } else {
970 tcg_gen_shli_i32(ret, arg, 24);
971 tcg_gen_sari_i32(ret, ret, 24);
972 }
973 }
974
975 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
976 {
977 if (TCG_TARGET_HAS_ext16s_i32) {
978 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
979 } else {
980 tcg_gen_shli_i32(ret, arg, 16);
981 tcg_gen_sari_i32(ret, ret, 16);
982 }
983 }
984
985 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
986 {
987 if (TCG_TARGET_HAS_ext8u_i32) {
988 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
989 } else {
990 tcg_gen_andi_i32(ret, arg, 0xffu);
991 }
992 }
993
994 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
995 {
996 if (TCG_TARGET_HAS_ext16u_i32) {
997 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
998 } else {
999 tcg_gen_andi_i32(ret, arg, 0xffffu);
1000 }
1001 }
1002
1003 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
1004 {
1005 /* Only one extension flag may be present. */
1006 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1007
1008 if (TCG_TARGET_HAS_bswap16_i32) {
1009 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags);
1010 } else {
1011 TCGv_i32 t0 = tcg_temp_new_i32();
1012 TCGv_i32 t1 = tcg_temp_new_i32();
1013
1014 tcg_gen_shri_i32(t0, arg, 8);
1015 if (!(flags & TCG_BSWAP_IZ)) {
1016 tcg_gen_ext8u_i32(t0, t0);
1017 }
1018
1019 if (flags & TCG_BSWAP_OS) {
1020 tcg_gen_shli_i32(t1, arg, 24);
1021 tcg_gen_sari_i32(t1, t1, 16);
1022 } else if (flags & TCG_BSWAP_OZ) {
1023 tcg_gen_ext8u_i32(t1, arg);
1024 tcg_gen_shli_i32(t1, t1, 8);
1025 } else {
1026 tcg_gen_shli_i32(t1, arg, 8);
1027 }
1028
1029 tcg_gen_or_i32(ret, t0, t1);
1030 tcg_temp_free_i32(t0);
1031 tcg_temp_free_i32(t1);
1032 }
1033 }
1034
1035 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1036 {
1037 if (TCG_TARGET_HAS_bswap32_i32) {
1038 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0);
1039 } else {
1040 TCGv_i32 t0 = tcg_temp_new_i32();
1041 TCGv_i32 t1 = tcg_temp_new_i32();
1042 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff);
1043
1044 /* arg = abcd */
1045 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */
1046 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */
1047 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */
1048 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */
1049 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */
1050
1051 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */
1052 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */
1053 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */
1054
1055 tcg_temp_free_i32(t0);
1056 tcg_temp_free_i32(t1);
1057 }
1058 }
1059
1060 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1061 {
1062 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1063 }
1064
1065 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1066 {
1067 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1068 }
1069
1070 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1071 {
1072 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1073 }
1074
1075 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1076 {
1077 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1078 }
1079
1080 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1081 {
1082 TCGv_i32 t = tcg_temp_new_i32();
1083
1084 tcg_gen_sari_i32(t, a, 31);
1085 tcg_gen_xor_i32(ret, a, t);
1086 tcg_gen_sub_i32(ret, ret, t);
1087 tcg_temp_free_i32(t);
1088 }
1089
1090 /* 64-bit ops */
1091
1092 #if TCG_TARGET_REG_BITS == 32
1093 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1094
1095 void tcg_gen_discard_i64(TCGv_i64 arg)
1096 {
1097 tcg_gen_discard_i32(TCGV_LOW(arg));
1098 tcg_gen_discard_i32(TCGV_HIGH(arg));
1099 }
1100
1101 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1102 {
1103 TCGTemp *ts = tcgv_i64_temp(arg);
1104
1105 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1106 if (ts->kind == TEMP_CONST) {
1107 tcg_gen_movi_i64(ret, ts->val);
1108 } else {
1109 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1110 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1111 }
1112 }
1113
1114 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1115 {
1116 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1117 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1118 }
1119
1120 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1121 {
1122 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1123 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1124 }
1125
1126 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1127 {
1128 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1129 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1130 }
1131
1132 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1133 {
1134 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1135 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1136 }
1137
1138 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1139 {
1140 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1141 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1142 }
1143
1144 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1145 {
1146 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1147 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1148 }
1149
1150 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1151 {
1152 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1153 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1154 }
1155
1156 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1157 {
1158 /* Since arg2 and ret have different types,
1159 they cannot be the same temporary */
1160 #ifdef HOST_WORDS_BIGENDIAN
1161 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1162 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1163 #else
1164 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1165 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1166 #endif
1167 }
1168
1169 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1170 {
1171 #ifdef HOST_WORDS_BIGENDIAN
1172 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1173 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1174 #else
1175 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1176 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1177 #endif
1178 }
1179
1180 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1181 {
1182 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1183 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1184 }
1185
1186 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1187 {
1188 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1189 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1190 }
1191
1192 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1193 {
1194 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1195 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1196 }
1197
1198 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1199 {
1200 gen_helper_shl_i64(ret, arg1, arg2);
1201 }
1202
1203 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1204 {
1205 gen_helper_shr_i64(ret, arg1, arg2);
1206 }
1207
1208 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1209 {
1210 gen_helper_sar_i64(ret, arg1, arg2);
1211 }
1212
1213 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1214 {
1215 TCGv_i64 t0;
1216 TCGv_i32 t1;
1217
1218 t0 = tcg_temp_new_i64();
1219 t1 = tcg_temp_new_i32();
1220
1221 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1222 TCGV_LOW(arg1), TCGV_LOW(arg2));
1223
1224 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1225 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1226 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1227 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1228
1229 tcg_gen_mov_i64(ret, t0);
1230 tcg_temp_free_i64(t0);
1231 tcg_temp_free_i32(t1);
1232 }
1233
1234 #else
1235
1236 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1237 {
1238 tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
1239 }
1240
1241 #endif /* TCG_TARGET_REG_SIZE == 32 */
1242
1243 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1244 {
1245 /* some cases can be optimized here */
1246 if (arg2 == 0) {
1247 tcg_gen_mov_i64(ret, arg1);
1248 } else if (TCG_TARGET_REG_BITS == 64) {
1249 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
1250 } else {
1251 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1252 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1253 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1254 }
1255 }
1256
1257 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1258 {
1259 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1260 /* Don't recurse with tcg_gen_neg_i64. */
1261 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1262 } else if (TCG_TARGET_REG_BITS == 64) {
1263 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
1264 } else {
1265 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1266 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32),
1267 TCGV_LOW(arg2), TCGV_HIGH(arg2));
1268 }
1269 }
1270
1271 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1272 {
1273 /* some cases can be optimized here */
1274 if (arg2 == 0) {
1275 tcg_gen_mov_i64(ret, arg1);
1276 } else if (TCG_TARGET_REG_BITS == 64) {
1277 tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2));
1278 } else {
1279 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1280 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1281 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1282 }
1283 }
1284
1285 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1286 {
1287 if (TCG_TARGET_REG_BITS == 32) {
1288 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1289 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1290 return;
1291 }
1292
1293 /* Some cases can be optimized here. */
1294 switch (arg2) {
1295 case 0:
1296 tcg_gen_movi_i64(ret, 0);
1297 return;
1298 case -1:
1299 tcg_gen_mov_i64(ret, arg1);
1300 return;
1301 case 0xff:
1302 /* Don't recurse with tcg_gen_ext8u_i64. */
1303 if (TCG_TARGET_HAS_ext8u_i64) {
1304 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1305 return;
1306 }
1307 break;
1308 case 0xffff:
1309 if (TCG_TARGET_HAS_ext16u_i64) {
1310 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1311 return;
1312 }
1313 break;
1314 case 0xffffffffu:
1315 if (TCG_TARGET_HAS_ext32u_i64) {
1316 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1317 return;
1318 }
1319 break;
1320 }
1321
1322 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
1323 }
1324
1325 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1326 {
1327 if (TCG_TARGET_REG_BITS == 32) {
1328 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1329 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1330 return;
1331 }
1332 /* Some cases can be optimized here. */
1333 if (arg2 == -1) {
1334 tcg_gen_movi_i64(ret, -1);
1335 } else if (arg2 == 0) {
1336 tcg_gen_mov_i64(ret, arg1);
1337 } else {
1338 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
1339 }
1340 }
1341
1342 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1343 {
1344 if (TCG_TARGET_REG_BITS == 32) {
1345 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1346 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1347 return;
1348 }
1349 /* Some cases can be optimized here. */
1350 if (arg2 == 0) {
1351 tcg_gen_mov_i64(ret, arg1);
1352 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1353 /* Don't recurse with tcg_gen_not_i64. */
1354 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1355 } else {
1356 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
1357 }
1358 }
1359
1360 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1361 unsigned c, bool right, bool arith)
1362 {
1363 tcg_debug_assert(c < 64);
1364 if (c == 0) {
1365 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1366 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1367 } else if (c >= 32) {
1368 c -= 32;
1369 if (right) {
1370 if (arith) {
1371 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1372 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1373 } else {
1374 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1375 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1376 }
1377 } else {
1378 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1379 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1380 }
1381 } else if (right) {
1382 if (TCG_TARGET_HAS_extract2_i32) {
1383 tcg_gen_extract2_i32(TCGV_LOW(ret),
1384 TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1385 } else {
1386 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1387 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1388 TCGV_HIGH(arg1), 32 - c, c);
1389 }
1390 if (arith) {
1391 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1392 } else {
1393 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1394 }
1395 } else {
1396 if (TCG_TARGET_HAS_extract2_i32) {
1397 tcg_gen_extract2_i32(TCGV_HIGH(ret),
1398 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
1399 } else {
1400 TCGv_i32 t0 = tcg_temp_new_i32();
1401 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1402 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1403 TCGV_HIGH(arg1), c, 32 - c);
1404 tcg_temp_free_i32(t0);
1405 }
1406 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1407 }
1408 }
1409
1410 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1411 {
1412 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1413 if (TCG_TARGET_REG_BITS == 32) {
1414 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1415 } else if (arg2 == 0) {
1416 tcg_gen_mov_i64(ret, arg1);
1417 } else {
1418 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
1419 }
1420 }
1421
1422 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1423 {
1424 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1425 if (TCG_TARGET_REG_BITS == 32) {
1426 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1427 } else if (arg2 == 0) {
1428 tcg_gen_mov_i64(ret, arg1);
1429 } else {
1430 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
1431 }
1432 }
1433
1434 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1435 {
1436 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1437 if (TCG_TARGET_REG_BITS == 32) {
1438 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1439 } else if (arg2 == 0) {
1440 tcg_gen_mov_i64(ret, arg1);
1441 } else {
1442 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
1443 }
1444 }
1445
1446 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1447 {
1448 if (cond == TCG_COND_ALWAYS) {
1449 tcg_gen_br(l);
1450 } else if (cond != TCG_COND_NEVER) {
1451 l->refs++;
1452 if (TCG_TARGET_REG_BITS == 32) {
1453 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1454 TCGV_HIGH(arg1), TCGV_LOW(arg2),
1455 TCGV_HIGH(arg2), cond, label_arg(l));
1456 } else {
1457 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1458 label_arg(l));
1459 }
1460 }
1461 }
1462
1463 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1464 {
1465 if (TCG_TARGET_REG_BITS == 64) {
1466 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
1467 } else if (cond == TCG_COND_ALWAYS) {
1468 tcg_gen_br(l);
1469 } else if (cond != TCG_COND_NEVER) {
1470 l->refs++;
1471 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
1472 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1473 tcg_constant_i32(arg2),
1474 tcg_constant_i32(arg2 >> 32),
1475 cond, label_arg(l));
1476 }
1477 }
1478
1479 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1480 TCGv_i64 arg1, TCGv_i64 arg2)
1481 {
1482 if (cond == TCG_COND_ALWAYS) {
1483 tcg_gen_movi_i64(ret, 1);
1484 } else if (cond == TCG_COND_NEVER) {
1485 tcg_gen_movi_i64(ret, 0);
1486 } else {
1487 if (TCG_TARGET_REG_BITS == 32) {
1488 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1489 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1490 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1491 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1492 } else {
1493 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1494 }
1495 }
1496 }
1497
1498 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1499 TCGv_i64 arg1, int64_t arg2)
1500 {
1501 if (TCG_TARGET_REG_BITS == 64) {
1502 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1503 } else if (cond == TCG_COND_ALWAYS) {
1504 tcg_gen_movi_i64(ret, 1);
1505 } else if (cond == TCG_COND_NEVER) {
1506 tcg_gen_movi_i64(ret, 0);
1507 } else {
1508 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1509 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1510 tcg_constant_i32(arg2),
1511 tcg_constant_i32(arg2 >> 32), cond);
1512 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1513 }
1514 }
1515
1516 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1517 {
1518 if (arg2 == 0) {
1519 tcg_gen_movi_i64(ret, 0);
1520 } else if (is_power_of_2(arg2)) {
1521 tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1522 } else {
1523 TCGv_i64 t0 = tcg_const_i64(arg2);
1524 tcg_gen_mul_i64(ret, arg1, t0);
1525 tcg_temp_free_i64(t0);
1526 }
1527 }
1528
1529 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1530 {
1531 if (TCG_TARGET_HAS_div_i64) {
1532 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1533 } else if (TCG_TARGET_HAS_div2_i64) {
1534 TCGv_i64 t0 = tcg_temp_new_i64();
1535 tcg_gen_sari_i64(t0, arg1, 63);
1536 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1537 tcg_temp_free_i64(t0);
1538 } else {
1539 gen_helper_div_i64(ret, arg1, arg2);
1540 }
1541 }
1542
1543 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1544 {
1545 if (TCG_TARGET_HAS_rem_i64) {
1546 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1547 } else if (TCG_TARGET_HAS_div_i64) {
1548 TCGv_i64 t0 = tcg_temp_new_i64();
1549 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1550 tcg_gen_mul_i64(t0, t0, arg2);
1551 tcg_gen_sub_i64(ret, arg1, t0);
1552 tcg_temp_free_i64(t0);
1553 } else if (TCG_TARGET_HAS_div2_i64) {
1554 TCGv_i64 t0 = tcg_temp_new_i64();
1555 tcg_gen_sari_i64(t0, arg1, 63);
1556 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1557 tcg_temp_free_i64(t0);
1558 } else {
1559 gen_helper_rem_i64(ret, arg1, arg2);
1560 }
1561 }
1562
1563 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1564 {
1565 if (TCG_TARGET_HAS_div_i64) {
1566 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1567 } else if (TCG_TARGET_HAS_div2_i64) {
1568 TCGv_i64 t0 = tcg_temp_new_i64();
1569 tcg_gen_movi_i64(t0, 0);
1570 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1571 tcg_temp_free_i64(t0);
1572 } else {
1573 gen_helper_divu_i64(ret, arg1, arg2);
1574 }
1575 }
1576
1577 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1578 {
1579 if (TCG_TARGET_HAS_rem_i64) {
1580 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1581 } else if (TCG_TARGET_HAS_div_i64) {
1582 TCGv_i64 t0 = tcg_temp_new_i64();
1583 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1584 tcg_gen_mul_i64(t0, t0, arg2);
1585 tcg_gen_sub_i64(ret, arg1, t0);
1586 tcg_temp_free_i64(t0);
1587 } else if (TCG_TARGET_HAS_div2_i64) {
1588 TCGv_i64 t0 = tcg_temp_new_i64();
1589 tcg_gen_movi_i64(t0, 0);
1590 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1591 tcg_temp_free_i64(t0);
1592 } else {
1593 gen_helper_remu_i64(ret, arg1, arg2);
1594 }
1595 }
1596
1597 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1598 {
1599 if (TCG_TARGET_REG_BITS == 32) {
1600 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1601 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1602 } else if (TCG_TARGET_HAS_ext8s_i64) {
1603 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1604 } else {
1605 tcg_gen_shli_i64(ret, arg, 56);
1606 tcg_gen_sari_i64(ret, ret, 56);
1607 }
1608 }
1609
1610 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1611 {
1612 if (TCG_TARGET_REG_BITS == 32) {
1613 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1614 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1615 } else if (TCG_TARGET_HAS_ext16s_i64) {
1616 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1617 } else {
1618 tcg_gen_shli_i64(ret, arg, 48);
1619 tcg_gen_sari_i64(ret, ret, 48);
1620 }
1621 }
1622
1623 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1624 {
1625 if (TCG_TARGET_REG_BITS == 32) {
1626 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1627 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1628 } else if (TCG_TARGET_HAS_ext32s_i64) {
1629 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1630 } else {
1631 tcg_gen_shli_i64(ret, arg, 32);
1632 tcg_gen_sari_i64(ret, ret, 32);
1633 }
1634 }
1635
1636 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1637 {
1638 if (TCG_TARGET_REG_BITS == 32) {
1639 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1640 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1641 } else if (TCG_TARGET_HAS_ext8u_i64) {
1642 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1643 } else {
1644 tcg_gen_andi_i64(ret, arg, 0xffu);
1645 }
1646 }
1647
1648 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1649 {
1650 if (TCG_TARGET_REG_BITS == 32) {
1651 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1652 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1653 } else if (TCG_TARGET_HAS_ext16u_i64) {
1654 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1655 } else {
1656 tcg_gen_andi_i64(ret, arg, 0xffffu);
1657 }
1658 }
1659
1660 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1661 {
1662 if (TCG_TARGET_REG_BITS == 32) {
1663 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1664 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1665 } else if (TCG_TARGET_HAS_ext32u_i64) {
1666 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1667 } else {
1668 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1669 }
1670 }
1671
1672 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1673 {
1674 /* Only one extension flag may be present. */
1675 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1676
1677 if (TCG_TARGET_REG_BITS == 32) {
1678 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags);
1679 if (flags & TCG_BSWAP_OS) {
1680 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1681 } else {
1682 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1683 }
1684 } else if (TCG_TARGET_HAS_bswap16_i64) {
1685 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags);
1686 } else {
1687 TCGv_i64 t0 = tcg_temp_new_i64();
1688 TCGv_i64 t1 = tcg_temp_new_i64();
1689
1690 tcg_gen_shri_i64(t0, arg, 8);
1691 if (!(flags & TCG_BSWAP_IZ)) {
1692 tcg_gen_ext8u_i64(t0, t0);
1693 }
1694
1695 if (flags & TCG_BSWAP_OS) {
1696 tcg_gen_shli_i64(t1, arg, 56);
1697 tcg_gen_sari_i64(t1, t1, 48);
1698 } else if (flags & TCG_BSWAP_OZ) {
1699 tcg_gen_ext8u_i64(t1, arg);
1700 tcg_gen_shli_i64(t1, t1, 8);
1701 } else {
1702 tcg_gen_shli_i64(t1, arg, 8);
1703 }
1704
1705 tcg_gen_or_i64(ret, t0, t1);
1706 tcg_temp_free_i64(t0);
1707 tcg_temp_free_i64(t1);
1708 }
1709 }
1710
1711 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1712 {
1713 /* Only one extension flag may be present. */
1714 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1715
1716 if (TCG_TARGET_REG_BITS == 32) {
1717 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1718 if (flags & TCG_BSWAP_OS) {
1719 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1720 } else {
1721 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1722 }
1723 } else if (TCG_TARGET_HAS_bswap32_i64) {
1724 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags);
1725 } else {
1726 TCGv_i64 t0 = tcg_temp_new_i64();
1727 TCGv_i64 t1 = tcg_temp_new_i64();
1728 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff);
1729
1730 /* arg = xxxxabcd */
1731 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */
1732 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */
1733 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */
1734 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */
1735 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */
1736
1737 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */
1738 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */
1739 if (flags & TCG_BSWAP_OS) {
1740 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */
1741 } else {
1742 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */
1743 }
1744 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba */
1745
1746 tcg_temp_free_i64(t0);
1747 tcg_temp_free_i64(t1);
1748 }
1749 }
1750
1751 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1752 {
1753 if (TCG_TARGET_REG_BITS == 32) {
1754 TCGv_i32 t0, t1;
1755 t0 = tcg_temp_new_i32();
1756 t1 = tcg_temp_new_i32();
1757
1758 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1759 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1760 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1761 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1762 tcg_temp_free_i32(t0);
1763 tcg_temp_free_i32(t1);
1764 } else if (TCG_TARGET_HAS_bswap64_i64) {
1765 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0);
1766 } else {
1767 TCGv_i64 t0 = tcg_temp_new_i64();
1768 TCGv_i64 t1 = tcg_temp_new_i64();
1769 TCGv_i64 t2 = tcg_temp_new_i64();
1770
1771 /* arg = abcdefgh */
1772 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1773 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */
1774 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */
1775 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */
1776 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */
1777 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */
1778
1779 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1780 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */
1781 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */
1782 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */
1783 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */
1784 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */
1785
1786 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */
1787 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */
1788 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */
1789
1790 tcg_temp_free_i64(t0);
1791 tcg_temp_free_i64(t1);
1792 tcg_temp_free_i64(t2);
1793 }
1794 }
1795
1796 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1797 {
1798 if (TCG_TARGET_REG_BITS == 32) {
1799 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1800 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1801 } else if (TCG_TARGET_HAS_not_i64) {
1802 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1803 } else {
1804 tcg_gen_xori_i64(ret, arg, -1);
1805 }
1806 }
1807
1808 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1809 {
1810 if (TCG_TARGET_REG_BITS == 32) {
1811 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1812 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1813 } else if (TCG_TARGET_HAS_andc_i64) {
1814 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1815 } else {
1816 TCGv_i64 t0 = tcg_temp_new_i64();
1817 tcg_gen_not_i64(t0, arg2);
1818 tcg_gen_and_i64(ret, arg1, t0);
1819 tcg_temp_free_i64(t0);
1820 }
1821 }
1822
1823 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1824 {
1825 if (TCG_TARGET_REG_BITS == 32) {
1826 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1827 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1828 } else if (TCG_TARGET_HAS_eqv_i64) {
1829 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1830 } else {
1831 tcg_gen_xor_i64(ret, arg1, arg2);
1832 tcg_gen_not_i64(ret, ret);
1833 }
1834 }
1835
1836 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1837 {
1838 if (TCG_TARGET_REG_BITS == 32) {
1839 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1840 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1841 } else if (TCG_TARGET_HAS_nand_i64) {
1842 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1843 } else {
1844 tcg_gen_and_i64(ret, arg1, arg2);
1845 tcg_gen_not_i64(ret, ret);
1846 }
1847 }
1848
1849 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1850 {
1851 if (TCG_TARGET_REG_BITS == 32) {
1852 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1853 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1854 } else if (TCG_TARGET_HAS_nor_i64) {
1855 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1856 } else {
1857 tcg_gen_or_i64(ret, arg1, arg2);
1858 tcg_gen_not_i64(ret, ret);
1859 }
1860 }
1861
1862 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1863 {
1864 if (TCG_TARGET_REG_BITS == 32) {
1865 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1866 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1867 } else if (TCG_TARGET_HAS_orc_i64) {
1868 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1869 } else {
1870 TCGv_i64 t0 = tcg_temp_new_i64();
1871 tcg_gen_not_i64(t0, arg2);
1872 tcg_gen_or_i64(ret, arg1, t0);
1873 tcg_temp_free_i64(t0);
1874 }
1875 }
1876
1877 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1878 {
1879 if (TCG_TARGET_HAS_clz_i64) {
1880 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1881 } else {
1882 gen_helper_clz_i64(ret, arg1, arg2);
1883 }
1884 }
1885
1886 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1887 {
1888 if (TCG_TARGET_REG_BITS == 32
1889 && TCG_TARGET_HAS_clz_i32
1890 && arg2 <= 0xffffffffu) {
1891 TCGv_i32 t = tcg_temp_new_i32();
1892 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32);
1893 tcg_gen_addi_i32(t, t, 32);
1894 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1895 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1896 tcg_temp_free_i32(t);
1897 } else {
1898 TCGv_i64 t0 = tcg_const_i64(arg2);
1899 tcg_gen_clz_i64(ret, arg1, t0);
1900 tcg_temp_free_i64(t0);
1901 }
1902 }
1903
1904 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1905 {
1906 if (TCG_TARGET_HAS_ctz_i64) {
1907 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1908 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1909 TCGv_i64 z, t = tcg_temp_new_i64();
1910
1911 if (TCG_TARGET_HAS_ctpop_i64) {
1912 tcg_gen_subi_i64(t, arg1, 1);
1913 tcg_gen_andc_i64(t, t, arg1);
1914 tcg_gen_ctpop_i64(t, t);
1915 } else {
1916 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1917 tcg_gen_neg_i64(t, arg1);
1918 tcg_gen_and_i64(t, t, arg1);
1919 tcg_gen_clzi_i64(t, t, 64);
1920 tcg_gen_xori_i64(t, t, 63);
1921 }
1922 z = tcg_constant_i64(0);
1923 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1924 tcg_temp_free_i64(t);
1925 tcg_temp_free_i64(z);
1926 } else {
1927 gen_helper_ctz_i64(ret, arg1, arg2);
1928 }
1929 }
1930
1931 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1932 {
1933 if (TCG_TARGET_REG_BITS == 32
1934 && TCG_TARGET_HAS_ctz_i32
1935 && arg2 <= 0xffffffffu) {
1936 TCGv_i32 t32 = tcg_temp_new_i32();
1937 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32);
1938 tcg_gen_addi_i32(t32, t32, 32);
1939 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1940 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1941 tcg_temp_free_i32(t32);
1942 } else if (!TCG_TARGET_HAS_ctz_i64
1943 && TCG_TARGET_HAS_ctpop_i64
1944 && arg2 == 64) {
1945 /* This equivalence has the advantage of not requiring a fixup. */
1946 TCGv_i64 t = tcg_temp_new_i64();
1947 tcg_gen_subi_i64(t, arg1, 1);
1948 tcg_gen_andc_i64(t, t, arg1);
1949 tcg_gen_ctpop_i64(ret, t);
1950 tcg_temp_free_i64(t);
1951 } else {
1952 TCGv_i64 t0 = tcg_const_i64(arg2);
1953 tcg_gen_ctz_i64(ret, arg1, t0);
1954 tcg_temp_free_i64(t0);
1955 }
1956 }
1957
1958 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1959 {
1960 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1961 TCGv_i64 t = tcg_temp_new_i64();
1962 tcg_gen_sari_i64(t, arg, 63);
1963 tcg_gen_xor_i64(t, t, arg);
1964 tcg_gen_clzi_i64(t, t, 64);
1965 tcg_gen_subi_i64(ret, t, 1);
1966 tcg_temp_free_i64(t);
1967 } else {
1968 gen_helper_clrsb_i64(ret, arg);
1969 }
1970 }
1971
1972 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1973 {
1974 if (TCG_TARGET_HAS_ctpop_i64) {
1975 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1976 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1977 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1978 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1979 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1980 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1981 } else {
1982 gen_helper_ctpop_i64(ret, arg1);
1983 }
1984 }
1985
1986 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1987 {
1988 if (TCG_TARGET_HAS_rot_i64) {
1989 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1990 } else {
1991 TCGv_i64 t0, t1;
1992 t0 = tcg_temp_new_i64();
1993 t1 = tcg_temp_new_i64();
1994 tcg_gen_shl_i64(t0, arg1, arg2);
1995 tcg_gen_subfi_i64(t1, 64, arg2);
1996 tcg_gen_shr_i64(t1, arg1, t1);
1997 tcg_gen_or_i64(ret, t0, t1);
1998 tcg_temp_free_i64(t0);
1999 tcg_temp_free_i64(t1);
2000 }
2001 }
2002
2003 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2004 {
2005 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2006 /* some cases can be optimized here */
2007 if (arg2 == 0) {
2008 tcg_gen_mov_i64(ret, arg1);
2009 } else if (TCG_TARGET_HAS_rot_i64) {
2010 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2));
2011 } else {
2012 TCGv_i64 t0, t1;
2013 t0 = tcg_temp_new_i64();
2014 t1 = tcg_temp_new_i64();
2015 tcg_gen_shli_i64(t0, arg1, arg2);
2016 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
2017 tcg_gen_or_i64(ret, t0, t1);
2018 tcg_temp_free_i64(t0);
2019 tcg_temp_free_i64(t1);
2020 }
2021 }
2022
2023 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2024 {
2025 if (TCG_TARGET_HAS_rot_i64) {
2026 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
2027 } else {
2028 TCGv_i64 t0, t1;
2029 t0 = tcg_temp_new_i64();
2030 t1 = tcg_temp_new_i64();
2031 tcg_gen_shr_i64(t0, arg1, arg2);
2032 tcg_gen_subfi_i64(t1, 64, arg2);
2033 tcg_gen_shl_i64(t1, arg1, t1);
2034 tcg_gen_or_i64(ret, t0, t1);
2035 tcg_temp_free_i64(t0);
2036 tcg_temp_free_i64(t1);
2037 }
2038 }
2039
2040 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2041 {
2042 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2043 /* some cases can be optimized here */
2044 if (arg2 == 0) {
2045 tcg_gen_mov_i64(ret, arg1);
2046 } else {
2047 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2048 }
2049 }
2050
2051 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2052 unsigned int ofs, unsigned int len)
2053 {
2054 uint64_t mask;
2055 TCGv_i64 t1;
2056
2057 tcg_debug_assert(ofs < 64);
2058 tcg_debug_assert(len > 0);
2059 tcg_debug_assert(len <= 64);
2060 tcg_debug_assert(ofs + len <= 64);
2061
2062 if (len == 64) {
2063 tcg_gen_mov_i64(ret, arg2);
2064 return;
2065 }
2066 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2067 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2068 return;
2069 }
2070
2071 if (TCG_TARGET_REG_BITS == 32) {
2072 if (ofs >= 32) {
2073 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2074 TCGV_LOW(arg2), ofs - 32, len);
2075 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2076 return;
2077 }
2078 if (ofs + len <= 32) {
2079 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2080 TCGV_LOW(arg2), ofs, len);
2081 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2082 return;
2083 }
2084 }
2085
2086 t1 = tcg_temp_new_i64();
2087
2088 if (TCG_TARGET_HAS_extract2_i64) {
2089 if (ofs + len == 64) {
2090 tcg_gen_shli_i64(t1, arg1, len);
2091 tcg_gen_extract2_i64(ret, t1, arg2, len);
2092 goto done;
2093 }
2094 if (ofs == 0) {
2095 tcg_gen_extract2_i64(ret, arg1, arg2, len);
2096 tcg_gen_rotli_i64(ret, ret, len);
2097 goto done;
2098 }
2099 }
2100
2101 mask = (1ull << len) - 1;
2102 if (ofs + len < 64) {
2103 tcg_gen_andi_i64(t1, arg2, mask);
2104 tcg_gen_shli_i64(t1, t1, ofs);
2105 } else {
2106 tcg_gen_shli_i64(t1, arg2, ofs);
2107 }
2108 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2109 tcg_gen_or_i64(ret, ret, t1);
2110 done:
2111 tcg_temp_free_i64(t1);
2112 }
2113
2114 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2115 unsigned int ofs, unsigned int len)
2116 {
2117 tcg_debug_assert(ofs < 64);
2118 tcg_debug_assert(len > 0);
2119 tcg_debug_assert(len <= 64);
2120 tcg_debug_assert(ofs + len <= 64);
2121
2122 if (ofs + len == 64) {
2123 tcg_gen_shli_i64(ret, arg, ofs);
2124 } else if (ofs == 0) {
2125 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2126 } else if (TCG_TARGET_HAS_deposit_i64
2127 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2128 TCGv_i64 zero = tcg_constant_i64(0);
2129 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2130 } else {
2131 if (TCG_TARGET_REG_BITS == 32) {
2132 if (ofs >= 32) {
2133 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2134 ofs - 32, len);
2135 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2136 return;
2137 }
2138 if (ofs + len <= 32) {
2139 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2140 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2141 return;
2142 }
2143 }
2144 /* To help two-operand hosts we prefer to zero-extend first,
2145 which allows ARG to stay live. */
2146 switch (len) {
2147 case 32:
2148 if (TCG_TARGET_HAS_ext32u_i64) {
2149 tcg_gen_ext32u_i64(ret, arg);
2150 tcg_gen_shli_i64(ret, ret, ofs);
2151 return;
2152 }
2153 break;
2154 case 16:
2155 if (TCG_TARGET_HAS_ext16u_i64) {
2156 tcg_gen_ext16u_i64(ret, arg);
2157 tcg_gen_shli_i64(ret, ret, ofs);
2158 return;
2159 }
2160 break;
2161 case 8:
2162 if (TCG_TARGET_HAS_ext8u_i64) {
2163 tcg_gen_ext8u_i64(ret, arg);
2164 tcg_gen_shli_i64(ret, ret, ofs);
2165 return;
2166 }
2167 break;
2168 }
2169 /* Otherwise prefer zero-extension over AND for code size. */
2170 switch (ofs + len) {
2171 case 32:
2172 if (TCG_TARGET_HAS_ext32u_i64) {
2173 tcg_gen_shli_i64(ret, arg, ofs);
2174 tcg_gen_ext32u_i64(ret, ret);
2175 return;
2176 }
2177 break;
2178 case 16:
2179 if (TCG_TARGET_HAS_ext16u_i64) {
2180 tcg_gen_shli_i64(ret, arg, ofs);
2181 tcg_gen_ext16u_i64(ret, ret);
2182 return;
2183 }
2184 break;
2185 case 8:
2186 if (TCG_TARGET_HAS_ext8u_i64) {
2187 tcg_gen_shli_i64(ret, arg, ofs);
2188 tcg_gen_ext8u_i64(ret, ret);
2189 return;
2190 }
2191 break;
2192 }
2193 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2194 tcg_gen_shli_i64(ret, ret, ofs);
2195 }
2196 }
2197
2198 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2199 unsigned int ofs, unsigned int len)
2200 {
2201 tcg_debug_assert(ofs < 64);
2202 tcg_debug_assert(len > 0);
2203 tcg_debug_assert(len <= 64);
2204 tcg_debug_assert(ofs + len <= 64);
2205
2206 /* Canonicalize certain special cases, even if extract is supported. */
2207 if (ofs + len == 64) {
2208 tcg_gen_shri_i64(ret, arg, 64 - len);
2209 return;
2210 }
2211 if (ofs == 0) {
2212 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2213 return;
2214 }
2215
2216 if (TCG_TARGET_REG_BITS == 32) {
2217 /* Look for a 32-bit extract within one of the two words. */
2218 if (ofs >= 32) {
2219 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2220 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2221 return;
2222 }
2223 if (ofs + len <= 32) {
2224 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2225 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2226 return;
2227 }
2228 /* The field is split across two words. One double-word
2229 shift is better than two double-word shifts. */
2230 goto do_shift_and;
2231 }
2232
2233 if (TCG_TARGET_HAS_extract_i64
2234 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2235 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2236 return;
2237 }
2238
2239 /* Assume that zero-extension, if available, is cheaper than a shift. */
2240 switch (ofs + len) {
2241 case 32:
2242 if (TCG_TARGET_HAS_ext32u_i64) {
2243 tcg_gen_ext32u_i64(ret, arg);
2244 tcg_gen_shri_i64(ret, ret, ofs);
2245 return;
2246 }
2247 break;
2248 case 16:
2249 if (TCG_TARGET_HAS_ext16u_i64) {
2250 tcg_gen_ext16u_i64(ret, arg);
2251 tcg_gen_shri_i64(ret, ret, ofs);
2252 return;
2253 }
2254 break;
2255 case 8:
2256 if (TCG_TARGET_HAS_ext8u_i64) {
2257 tcg_gen_ext8u_i64(ret, arg);
2258 tcg_gen_shri_i64(ret, ret, ofs);
2259 return;
2260 }
2261 break;
2262 }
2263
2264 /* ??? Ideally we'd know what values are available for immediate AND.
2265 Assume that 8 bits are available, plus the special cases of 16 and 32,
2266 so that we get ext8u, ext16u, and ext32u. */
2267 switch (len) {
2268 case 1 ... 8: case 16: case 32:
2269 do_shift_and:
2270 tcg_gen_shri_i64(ret, arg, ofs);
2271 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2272 break;
2273 default:
2274 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2275 tcg_gen_shri_i64(ret, ret, 64 - len);
2276 break;
2277 }
2278 }
2279
2280 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2281 unsigned int ofs, unsigned int len)
2282 {
2283 tcg_debug_assert(ofs < 64);
2284 tcg_debug_assert(len > 0);
2285 tcg_debug_assert(len <= 64);
2286 tcg_debug_assert(ofs + len <= 64);
2287
2288 /* Canonicalize certain special cases, even if sextract is supported. */
2289 if (ofs + len == 64) {
2290 tcg_gen_sari_i64(ret, arg, 64 - len);
2291 return;
2292 }
2293 if (ofs == 0) {
2294 switch (len) {
2295 case 32:
2296 tcg_gen_ext32s_i64(ret, arg);
2297 return;
2298 case 16:
2299 tcg_gen_ext16s_i64(ret, arg);
2300 return;
2301 case 8:
2302 tcg_gen_ext8s_i64(ret, arg);
2303 return;
2304 }
2305 }
2306
2307 if (TCG_TARGET_REG_BITS == 32) {
2308 /* Look for a 32-bit extract within one of the two words. */
2309 if (ofs >= 32) {
2310 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2311 } else if (ofs + len <= 32) {
2312 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2313 } else if (ofs == 0) {
2314 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2315 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2316 return;
2317 } else if (len > 32) {
2318 TCGv_i32 t = tcg_temp_new_i32();
2319 /* Extract the bits for the high word normally. */
2320 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2321 /* Shift the field down for the low part. */
2322 tcg_gen_shri_i64(ret, arg, ofs);
2323 /* Overwrite the shift into the high part. */
2324 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2325 tcg_temp_free_i32(t);
2326 return;
2327 } else {
2328 /* Shift the field down for the low part, such that the
2329 field sits at the MSB. */
2330 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2331 /* Shift the field down from the MSB, sign extending. */
2332 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2333 }
2334 /* Sign-extend the field from 32 bits. */
2335 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2336 return;
2337 }
2338
2339 if (TCG_TARGET_HAS_sextract_i64
2340 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2341 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2342 return;
2343 }
2344
2345 /* Assume that sign-extension, if available, is cheaper than a shift. */
2346 switch (ofs + len) {
2347 case 32:
2348 if (TCG_TARGET_HAS_ext32s_i64) {
2349 tcg_gen_ext32s_i64(ret, arg);
2350 tcg_gen_sari_i64(ret, ret, ofs);
2351 return;
2352 }
2353 break;
2354 case 16:
2355 if (TCG_TARGET_HAS_ext16s_i64) {
2356 tcg_gen_ext16s_i64(ret, arg);
2357 tcg_gen_sari_i64(ret, ret, ofs);
2358 return;
2359 }
2360 break;
2361 case 8:
2362 if (TCG_TARGET_HAS_ext8s_i64) {
2363 tcg_gen_ext8s_i64(ret, arg);
2364 tcg_gen_sari_i64(ret, ret, ofs);
2365 return;
2366 }
2367 break;
2368 }
2369 switch (len) {
2370 case 32:
2371 if (TCG_TARGET_HAS_ext32s_i64) {
2372 tcg_gen_shri_i64(ret, arg, ofs);
2373 tcg_gen_ext32s_i64(ret, ret);
2374 return;
2375 }
2376 break;
2377 case 16:
2378 if (TCG_TARGET_HAS_ext16s_i64) {
2379 tcg_gen_shri_i64(ret, arg, ofs);
2380 tcg_gen_ext16s_i64(ret, ret);
2381 return;
2382 }
2383 break;
2384 case 8:
2385 if (TCG_TARGET_HAS_ext8s_i64) {
2386 tcg_gen_shri_i64(ret, arg, ofs);
2387 tcg_gen_ext8s_i64(ret, ret);
2388 return;
2389 }
2390 break;
2391 }
2392 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2393 tcg_gen_sari_i64(ret, ret, 64 - len);
2394 }
2395
2396 /*
2397 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2398 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2399 */
2400 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2401 unsigned int ofs)
2402 {
2403 tcg_debug_assert(ofs <= 64);
2404 if (ofs == 0) {
2405 tcg_gen_mov_i64(ret, al);
2406 } else if (ofs == 64) {
2407 tcg_gen_mov_i64(ret, ah);
2408 } else if (al == ah) {
2409 tcg_gen_rotri_i64(ret, al, ofs);
2410 } else if (TCG_TARGET_HAS_extract2_i64) {
2411 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2412 } else {
2413 TCGv_i64 t0 = tcg_temp_new_i64();
2414 tcg_gen_shri_i64(t0, al, ofs);
2415 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2416 tcg_temp_free_i64(t0);
2417 }
2418 }
2419
2420 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2421 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2422 {
2423 if (cond == TCG_COND_ALWAYS) {
2424 tcg_gen_mov_i64(ret, v1);
2425 } else if (cond == TCG_COND_NEVER) {
2426 tcg_gen_mov_i64(ret, v2);
2427 } else if (TCG_TARGET_REG_BITS == 32) {
2428 TCGv_i32 t0 = tcg_temp_new_i32();
2429 TCGv_i32 t1 = tcg_temp_new_i32();
2430 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2431 TCGV_LOW(c1), TCGV_HIGH(c1),
2432 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2433
2434 if (TCG_TARGET_HAS_movcond_i32) {
2435 tcg_gen_movi_i32(t1, 0);
2436 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2437 TCGV_LOW(v1), TCGV_LOW(v2));
2438 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2439 TCGV_HIGH(v1), TCGV_HIGH(v2));
2440 } else {
2441 tcg_gen_neg_i32(t0, t0);
2442
2443 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2444 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2445 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2446
2447 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2448 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2449 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2450 }
2451 tcg_temp_free_i32(t0);
2452 tcg_temp_free_i32(t1);
2453 } else if (TCG_TARGET_HAS_movcond_i64) {
2454 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2455 } else {
2456 TCGv_i64 t0 = tcg_temp_new_i64();
2457 TCGv_i64 t1 = tcg_temp_new_i64();
2458 tcg_gen_setcond_i64(cond, t0, c1, c2);
2459 tcg_gen_neg_i64(t0, t0);
2460 tcg_gen_and_i64(t1, v1, t0);
2461 tcg_gen_andc_i64(ret, v2, t0);
2462 tcg_gen_or_i64(ret, ret, t1);
2463 tcg_temp_free_i64(t0);
2464 tcg_temp_free_i64(t1);
2465 }
2466 }
2467
2468 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2469 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2470 {
2471 if (TCG_TARGET_HAS_add2_i64) {
2472 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2473 } else {
2474 TCGv_i64 t0 = tcg_temp_new_i64();
2475 TCGv_i64 t1 = tcg_temp_new_i64();
2476 tcg_gen_add_i64(t0, al, bl);
2477 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2478 tcg_gen_add_i64(rh, ah, bh);
2479 tcg_gen_add_i64(rh, rh, t1);
2480 tcg_gen_mov_i64(rl, t0);
2481 tcg_temp_free_i64(t0);
2482 tcg_temp_free_i64(t1);
2483 }
2484 }
2485
2486 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2487 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2488 {
2489 if (TCG_TARGET_HAS_sub2_i64) {
2490 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2491 } else {
2492 TCGv_i64 t0 = tcg_temp_new_i64();
2493 TCGv_i64 t1 = tcg_temp_new_i64();
2494 tcg_gen_sub_i64(t0, al, bl);
2495 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2496 tcg_gen_sub_i64(rh, ah, bh);
2497 tcg_gen_sub_i64(rh, rh, t1);
2498 tcg_gen_mov_i64(rl, t0);
2499 tcg_temp_free_i64(t0);
2500 tcg_temp_free_i64(t1);
2501 }
2502 }
2503
2504 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2505 {
2506 if (TCG_TARGET_HAS_mulu2_i64) {
2507 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2508 } else if (TCG_TARGET_HAS_muluh_i64) {
2509 TCGv_i64 t = tcg_temp_new_i64();
2510 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2511 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2512 tcg_gen_mov_i64(rl, t);
2513 tcg_temp_free_i64(t);
2514 } else {
2515 TCGv_i64 t0 = tcg_temp_new_i64();
2516 tcg_gen_mul_i64(t0, arg1, arg2);
2517 gen_helper_muluh_i64(rh, arg1, arg2);
2518 tcg_gen_mov_i64(rl, t0);
2519 tcg_temp_free_i64(t0);
2520 }
2521 }
2522
2523 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2524 {
2525 if (TCG_TARGET_HAS_muls2_i64) {
2526 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2527 } else if (TCG_TARGET_HAS_mulsh_i64) {
2528 TCGv_i64 t = tcg_temp_new_i64();
2529 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2530 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2531 tcg_gen_mov_i64(rl, t);
2532 tcg_temp_free_i64(t);
2533 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2534 TCGv_i64 t0 = tcg_temp_new_i64();
2535 TCGv_i64 t1 = tcg_temp_new_i64();
2536 TCGv_i64 t2 = tcg_temp_new_i64();
2537 TCGv_i64 t3 = tcg_temp_new_i64();
2538 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2539 /* Adjust for negative inputs. */
2540 tcg_gen_sari_i64(t2, arg1, 63);
2541 tcg_gen_sari_i64(t3, arg2, 63);
2542 tcg_gen_and_i64(t2, t2, arg2);
2543 tcg_gen_and_i64(t3, t3, arg1);
2544 tcg_gen_sub_i64(rh, t1, t2);
2545 tcg_gen_sub_i64(rh, rh, t3);
2546 tcg_gen_mov_i64(rl, t0);
2547 tcg_temp_free_i64(t0);
2548 tcg_temp_free_i64(t1);
2549 tcg_temp_free_i64(t2);
2550 tcg_temp_free_i64(t3);
2551 } else {
2552 TCGv_i64 t0 = tcg_temp_new_i64();
2553 tcg_gen_mul_i64(t0, arg1, arg2);
2554 gen_helper_mulsh_i64(rh, arg1, arg2);
2555 tcg_gen_mov_i64(rl, t0);
2556 tcg_temp_free_i64(t0);
2557 }
2558 }
2559
2560 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2561 {
2562 TCGv_i64 t0 = tcg_temp_new_i64();
2563 TCGv_i64 t1 = tcg_temp_new_i64();
2564 TCGv_i64 t2 = tcg_temp_new_i64();
2565 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2566 /* Adjust for negative input for the signed arg1. */
2567 tcg_gen_sari_i64(t2, arg1, 63);
2568 tcg_gen_and_i64(t2, t2, arg2);
2569 tcg_gen_sub_i64(rh, t1, t2);
2570 tcg_gen_mov_i64(rl, t0);
2571 tcg_temp_free_i64(t0);
2572 tcg_temp_free_i64(t1);
2573 tcg_temp_free_i64(t2);
2574 }
2575
2576 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2577 {
2578 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2579 }
2580
2581 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2582 {
2583 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2584 }
2585
2586 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2587 {
2588 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2589 }
2590
2591 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2592 {
2593 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2594 }
2595
2596 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2597 {
2598 TCGv_i64 t = tcg_temp_new_i64();
2599
2600 tcg_gen_sari_i64(t, a, 63);
2601 tcg_gen_xor_i64(ret, a, t);
2602 tcg_gen_sub_i64(ret, ret, t);
2603 tcg_temp_free_i64(t);
2604 }
2605
2606 /* Size changing operations. */
2607
2608 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2609 {
2610 if (TCG_TARGET_REG_BITS == 32) {
2611 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2612 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2613 tcg_gen_op2(INDEX_op_extrl_i64_i32,
2614 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2615 } else {
2616 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2617 }
2618 }
2619
2620 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2621 {
2622 if (TCG_TARGET_REG_BITS == 32) {
2623 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2624 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2625 tcg_gen_op2(INDEX_op_extrh_i64_i32,
2626 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2627 } else {
2628 TCGv_i64 t = tcg_temp_new_i64();
2629 tcg_gen_shri_i64(t, arg, 32);
2630 tcg_gen_mov_i32(ret, (TCGv_i32)t);
2631 tcg_temp_free_i64(t);
2632 }
2633 }
2634
2635 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2636 {
2637 if (TCG_TARGET_REG_BITS == 32) {
2638 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2639 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2640 } else {
2641 tcg_gen_op2(INDEX_op_extu_i32_i64,
2642 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2643 }
2644 }
2645
2646 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2647 {
2648 if (TCG_TARGET_REG_BITS == 32) {
2649 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2650 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2651 } else {
2652 tcg_gen_op2(INDEX_op_ext_i32_i64,
2653 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2654 }
2655 }
2656
2657 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2658 {
2659 TCGv_i64 tmp;
2660
2661 if (TCG_TARGET_REG_BITS == 32) {
2662 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2663 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2664 return;
2665 }
2666
2667 tmp = tcg_temp_new_i64();
2668 /* These extensions are only needed for type correctness.
2669 We may be able to do better given target specific information. */
2670 tcg_gen_extu_i32_i64(tmp, high);
2671 tcg_gen_extu_i32_i64(dest, low);
2672 /* If deposit is available, use it. Otherwise use the extra
2673 knowledge that we have of the zero-extensions above. */
2674 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2675 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2676 } else {
2677 tcg_gen_shli_i64(tmp, tmp, 32);
2678 tcg_gen_or_i64(dest, dest, tmp);
2679 }
2680 tcg_temp_free_i64(tmp);
2681 }
2682
2683 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2684 {
2685 if (TCG_TARGET_REG_BITS == 32) {
2686 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2687 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2688 } else {
2689 tcg_gen_extrl_i64_i32(lo, arg);
2690 tcg_gen_extrh_i64_i32(hi, arg);
2691 }
2692 }
2693
2694 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2695 {
2696 tcg_gen_ext32u_i64(lo, arg);
2697 tcg_gen_shri_i64(hi, arg, 32);
2698 }
2699
2700 /* QEMU specific operations. */
2701
2702 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
2703 {
2704 /*
2705 * Let the jit code return the read-only version of the
2706 * TranslationBlock, so that we minimize the pc-relative
2707 * distance of the address of the exit_tb code to TB.
2708 * This will improve utilization of pc-relative address loads.
2709 *
2710 * TODO: Move this to translator_loop, so that all const
2711 * TranslationBlock pointers refer to read-only memory.
2712 * This requires coordination with targets that do not use
2713 * the translator_loop.
2714 */
2715 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
2716
2717 if (tb == NULL) {
2718 tcg_debug_assert(idx == 0);
2719 } else if (idx <= TB_EXIT_IDXMAX) {
2720 #ifdef CONFIG_DEBUG_TCG
2721 /* This is an exit following a goto_tb. Verify that we have
2722 seen this numbered exit before, via tcg_gen_goto_tb. */
2723 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2724 #endif
2725 } else {
2726 /* This is an exit via the exitreq label. */
2727 tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2728 }
2729
2730 plugin_gen_disable_mem_helpers();
2731 tcg_gen_op1i(INDEX_op_exit_tb, val);
2732 }
2733
2734 void tcg_gen_goto_tb(unsigned idx)
2735 {
2736 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
2737 tcg_debug_assert(!(tcg_ctx->tb_cflags & CF_NO_GOTO_TB));
2738 /* We only support two chained exits. */
2739 tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2740 #ifdef CONFIG_DEBUG_TCG
2741 /* Verify that we haven't seen this numbered exit before. */
2742 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2743 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2744 #endif
2745 plugin_gen_disable_mem_helpers();
2746 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2747 }
2748
2749 void tcg_gen_lookup_and_goto_ptr(void)
2750 {
2751 TCGv_ptr ptr;
2752
2753 if (tcg_ctx->tb_cflags & CF_NO_GOTO_PTR) {
2754 tcg_gen_exit_tb(NULL, 0);
2755 return;
2756 }
2757
2758 plugin_gen_disable_mem_helpers();
2759 ptr = tcg_temp_new_ptr();
2760 gen_helper_lookup_tb_ptr(ptr, cpu_env);
2761 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2762 tcg_temp_free_ptr(ptr);
2763 }
2764
2765 static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
2766 {
2767 /* Trigger the asserts within as early as possible. */
2768 (void)get_alignment_bits(op);
2769
2770 switch (op & MO_SIZE) {
2771 case MO_8:
2772 op &= ~MO_BSWAP;
2773 break;
2774 case MO_16:
2775 break;
2776 case MO_32:
2777 if (!is64) {
2778 op &= ~MO_SIGN;
2779 }
2780 break;
2781 case MO_64:
2782 if (is64) {
2783 op &= ~MO_SIGN;
2784 break;
2785 }
2786 /* fall through */
2787 default:
2788 g_assert_not_reached();
2789 }
2790 if (st) {
2791 op &= ~MO_SIGN;
2792 }
2793 return op;
2794 }
2795
2796 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2797 MemOp memop, TCGArg idx)
2798 {
2799 MemOpIdx oi = make_memop_idx(memop, idx);
2800 #if TARGET_LONG_BITS == 32
2801 tcg_gen_op3i_i32(opc, val, addr, oi);
2802 #else
2803 if (TCG_TARGET_REG_BITS == 32) {
2804 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2805 } else {
2806 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2807 }
2808 #endif
2809 }
2810
2811 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2812 MemOp memop, TCGArg idx)
2813 {
2814 MemOpIdx oi = make_memop_idx(memop, idx);
2815 #if TARGET_LONG_BITS == 32
2816 if (TCG_TARGET_REG_BITS == 32) {
2817 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2818 } else {
2819 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2820 }
2821 #else
2822 if (TCG_TARGET_REG_BITS == 32) {
2823 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2824 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2825 } else {
2826 tcg_gen_op3i_i64(opc, val, addr, oi);
2827 }
2828 #endif
2829 }
2830
2831 static void tcg_gen_req_mo(TCGBar type)
2832 {
2833 #ifdef TCG_GUEST_DEFAULT_MO
2834 type &= TCG_GUEST_DEFAULT_MO;
2835 #endif
2836 type &= ~TCG_TARGET_DEFAULT_MO;
2837 if (type) {
2838 tcg_gen_mb(type | TCG_BAR_SC);
2839 }
2840 }
2841
2842 static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr)
2843 {
2844 #ifdef CONFIG_PLUGIN
2845 if (tcg_ctx->plugin_insn != NULL) {
2846 /* Save a copy of the vaddr for use after a load. */
2847 TCGv temp = tcg_temp_new();
2848 tcg_gen_mov_tl(temp, vaddr);
2849 return temp;
2850 }
2851 #endif
2852 return vaddr;
2853 }
2854
2855 static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi,
2856 enum qemu_plugin_mem_rw rw)
2857 {
2858 #ifdef CONFIG_PLUGIN
2859 if (tcg_ctx->plugin_insn != NULL) {
2860 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw);
2861 plugin_gen_empty_mem_callback(vaddr, info);
2862 tcg_temp_free(vaddr);
2863 }
2864 #endif
2865 }
2866
2867 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2868 {
2869 MemOp orig_memop;
2870 MemOpIdx oi;
2871
2872 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2873 memop = tcg_canonicalize_memop(memop, 0, 0);
2874 oi = make_memop_idx(memop, idx);
2875 trace_guest_ld_before_tcg(tcg_ctx->cpu, cpu_env, addr, oi);
2876
2877 orig_memop = memop;
2878 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2879 memop &= ~MO_BSWAP;
2880 /* The bswap primitive benefits from zero-extended input. */
2881 if ((memop & MO_SSIZE) == MO_SW) {
2882 memop &= ~MO_SIGN;
2883 }
2884 }
2885
2886 addr = plugin_prep_mem_callbacks(addr);
2887 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2888 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
2889
2890 if ((orig_memop ^ memop) & MO_BSWAP) {
2891 switch (orig_memop & MO_SIZE) {
2892 case MO_16:
2893 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
2894 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2895 : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
2896 break;
2897 case MO_32:
2898 tcg_gen_bswap32_i32(val, val);
2899 break;
2900 default:
2901 g_assert_not_reached();
2902 }
2903 }
2904 }
2905
2906 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2907 {
2908 TCGv_i32 swap = NULL;
2909 MemOpIdx oi;
2910
2911 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2912 memop = tcg_canonicalize_memop(memop, 0, 1);
2913 oi = make_memop_idx(memop, idx);
2914 trace_guest_st_before_tcg(tcg_ctx->cpu, cpu_env, addr, oi);
2915
2916 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2917 swap = tcg_temp_new_i32();
2918 switch (memop & MO_SIZE) {
2919 case MO_16:
2920 tcg_gen_bswap16_i32(swap, val, 0);
2921 break;
2922 case MO_32:
2923 tcg_gen_bswap32_i32(swap, val);
2924 break;
2925 default:
2926 g_assert_not_reached();
2927 }
2928 val = swap;
2929 memop &= ~MO_BSWAP;
2930 }
2931
2932 addr = plugin_prep_mem_callbacks(addr);
2933 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
2934 gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx);
2935 } else {
2936 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2937 }
2938 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
2939
2940 if (swap) {
2941 tcg_temp_free_i32(swap);
2942 }
2943 }
2944
2945 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
2946 {
2947 MemOp orig_memop;
2948 MemOpIdx oi;
2949
2950 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2951 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2952 if (memop & MO_SIGN) {
2953 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2954 } else {
2955 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2956 }
2957 return;
2958 }
2959
2960 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2961 memop = tcg_canonicalize_memop(memop, 1, 0);
2962 oi = make_memop_idx(memop, idx);
2963 trace_guest_ld_before_tcg(tcg_ctx->cpu, cpu_env, addr, oi);
2964
2965 orig_memop = memop;
2966 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2967 memop &= ~MO_BSWAP;
2968 /* The bswap primitive benefits from zero-extended input. */
2969 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
2970 memop &= ~MO_SIGN;
2971 }
2972 }
2973
2974 addr = plugin_prep_mem_callbacks(addr);
2975 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2976 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
2977
2978 if ((orig_memop ^ memop) & MO_BSWAP) {
2979 int flags = (orig_memop & MO_SIGN
2980 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2981 : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
2982 switch (orig_memop & MO_SIZE) {
2983 case MO_16:
2984 tcg_gen_bswap16_i64(val, val, flags);
2985 break;
2986 case MO_32:
2987 tcg_gen_bswap32_i64(val, val, flags);
2988 break;
2989 case MO_64:
2990 tcg_gen_bswap64_i64(val, val);
2991 break;
2992 default:
2993 g_assert_not_reached();
2994 }
2995 }
2996 }
2997
2998 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
2999 {
3000 TCGv_i64 swap = NULL;
3001 MemOpIdx oi;
3002
3003 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
3004 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
3005 return;
3006 }
3007
3008 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
3009 memop = tcg_canonicalize_memop(memop, 1, 1);
3010 oi = make_memop_idx(memop, idx);
3011 trace_guest_st_before_tcg(tcg_ctx->cpu, cpu_env, addr, oi);
3012
3013 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
3014 swap = tcg_temp_new_i64();
3015 switch (memop & MO_SIZE) {
3016 case MO_16:
3017 tcg_gen_bswap16_i64(swap, val, 0);
3018 break;
3019 case MO_32:
3020 tcg_gen_bswap32_i64(swap, val, 0);
3021 break;
3022 case MO_64:
3023 tcg_gen_bswap64_i64(swap, val);
3024 break;
3025 default:
3026 g_assert_not_reached();
3027 }
3028 val = swap;
3029 memop &= ~MO_BSWAP;
3030 }
3031
3032 addr = plugin_prep_mem_callbacks(addr);
3033 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
3034 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
3035
3036 if (swap) {
3037 tcg_temp_free_i64(swap);
3038 }
3039 }
3040
3041 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
3042 {
3043 switch (opc & MO_SSIZE) {
3044 case MO_SB:
3045 tcg_gen_ext8s_i32(ret, val);
3046 break;
3047 case MO_UB:
3048 tcg_gen_ext8u_i32(ret, val);
3049 break;
3050 case MO_SW:
3051 tcg_gen_ext16s_i32(ret, val);
3052 break;
3053 case MO_UW:
3054 tcg_gen_ext16u_i32(ret, val);
3055 break;
3056 default:
3057 tcg_gen_mov_i32(ret, val);
3058 break;
3059 }
3060 }
3061
3062 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
3063 {
3064 switch (opc & MO_SSIZE) {
3065 case MO_SB:
3066 tcg_gen_ext8s_i64(ret, val);
3067 break;
3068 case MO_UB:
3069 tcg_gen_ext8u_i64(ret, val);
3070 break;
3071 case MO_SW:
3072 tcg_gen_ext16s_i64(ret, val);
3073 break;
3074 case MO_UW:
3075 tcg_gen_ext16u_i64(ret, val);
3076 break;
3077 case MO_SL:
3078 tcg_gen_ext32s_i64(ret, val);
3079 break;
3080 case MO_UL:
3081 tcg_gen_ext32u_i64(ret, val);
3082 break;
3083 default:
3084 tcg_gen_mov_i64(ret, val);
3085 break;
3086 }
3087 }
3088
3089 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
3090 TCGv_i32, TCGv_i32, TCGv_i32);
3091 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
3092 TCGv_i64, TCGv_i64, TCGv_i32);
3093 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
3094 TCGv_i32, TCGv_i32);
3095 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
3096 TCGv_i64, TCGv_i32);
3097
3098 #ifdef CONFIG_ATOMIC64
3099 # define WITH_ATOMIC64(X) X,
3100 #else
3101 # define WITH_ATOMIC64(X)
3102 #endif
3103
3104 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
3105 [MO_8] = gen_helper_atomic_cmpxchgb,
3106 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
3107 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
3108 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
3109 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
3110 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
3111 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
3112 };
3113
3114 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
3115 TCGv_i32 newv, TCGArg idx, MemOp memop)
3116 {
3117 memop = tcg_canonicalize_memop(memop, 0, 0);
3118
3119 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3120 TCGv_i32 t1 = tcg_temp_new_i32();
3121 TCGv_i32 t2 = tcg_temp_new_i32();
3122
3123 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
3124
3125 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3126 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
3127 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3128 tcg_temp_free_i32(t2);
3129
3130 if (memop & MO_SIGN) {
3131 tcg_gen_ext_i32(retv, t1, memop);
3132 } else {
3133 tcg_gen_mov_i32(retv, t1);
3134 }
3135 tcg_temp_free_i32(t1);
3136 } else {
3137 gen_atomic_cx_i32 gen;
3138 MemOpIdx oi;
3139
3140 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3141 tcg_debug_assert(gen != NULL);
3142
3143 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3144 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3145
3146 if (memop & MO_SIGN) {
3147 tcg_gen_ext_i32(retv, retv, memop);
3148 }
3149 }
3150 }
3151
3152 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
3153 TCGv_i64 newv, TCGArg idx, MemOp memop)
3154 {
3155 memop = tcg_canonicalize_memop(memop, 1, 0);
3156
3157 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3158 TCGv_i64 t1 = tcg_temp_new_i64();
3159 TCGv_i64 t2 = tcg_temp_new_i64();
3160
3161 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
3162
3163 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3164 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
3165 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3166 tcg_temp_free_i64(t2);
3167
3168 if (memop & MO_SIGN) {
3169 tcg_gen_ext_i64(retv, t1, memop);
3170 } else {
3171 tcg_gen_mov_i64(retv, t1);
3172 }
3173 tcg_temp_free_i64(t1);
3174 } else if ((memop & MO_SIZE) == MO_64) {
3175 #ifdef CONFIG_ATOMIC64
3176 gen_atomic_cx_i64 gen;
3177 MemOpIdx oi;
3178
3179 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3180 tcg_debug_assert(gen != NULL);
3181
3182 oi = make_memop_idx(memop, idx);
3183 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3184 #else
3185 gen_helper_exit_atomic(cpu_env);
3186 /* Produce a result, so that we have a well-formed opcode stream
3187 with respect to uses of the result in the (dead) code following. */
3188 tcg_gen_movi_i64(retv, 0);
3189 #endif /* CONFIG_ATOMIC64 */
3190 } else {
3191 TCGv_i32 c32 = tcg_temp_new_i32();
3192 TCGv_i32 n32 = tcg_temp_new_i32();
3193 TCGv_i32 r32 = tcg_temp_new_i32();
3194
3195 tcg_gen_extrl_i64_i32(c32, cmpv);
3196 tcg_gen_extrl_i64_i32(n32, newv);
3197 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3198 tcg_temp_free_i32(c32);
3199 tcg_temp_free_i32(n32);
3200
3201 tcg_gen_extu_i32_i64(retv, r32);
3202 tcg_temp_free_i32(r32);
3203
3204 if (memop & MO_SIGN) {
3205 tcg_gen_ext_i64(retv, retv, memop);
3206 }
3207 }
3208 }
3209
3210 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3211 TCGArg idx, MemOp memop, bool new_val,
3212 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3213 {
3214 TCGv_i32 t1 = tcg_temp_new_i32();
3215 TCGv_i32 t2 = tcg_temp_new_i32();
3216
3217 memop = tcg_canonicalize_memop(memop, 0, 0);
3218
3219 tcg_gen_qemu_ld_i32(t1, addr, idx, memop);
3220 tcg_gen_ext_i32(t2, val, memop);
3221 gen(t2, t1, t2);
3222 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3223
3224 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3225 tcg_temp_free_i32(t1);
3226 tcg_temp_free_i32(t2);
3227 }
3228
3229 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3230 TCGArg idx, MemOp memop, void * const table[])
3231 {
3232 gen_atomic_op_i32 gen;
3233 MemOpIdx oi;
3234
3235 memop = tcg_canonicalize_memop(memop, 0, 0);
3236
3237 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3238 tcg_debug_assert(gen != NULL);
3239
3240 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3241 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3242
3243 if (memop & MO_SIGN) {
3244 tcg_gen_ext_i32(ret, ret, memop);
3245 }
3246 }
3247
3248 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3249 TCGArg idx, MemOp memop, bool new_val,
3250 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3251 {
3252 TCGv_i64 t1 = tcg_temp_new_i64();
3253 TCGv_i64 t2 = tcg_temp_new_i64();
3254
3255 memop = tcg_canonicalize_memop(memop, 1, 0);
3256
3257 tcg_gen_qemu_ld_i64(t1, addr, idx, memop);
3258 tcg_gen_ext_i64(t2, val, memop);
3259 gen(t2, t1, t2);
3260 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3261
3262 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3263 tcg_temp_free_i64(t1);
3264 tcg_temp_free_i64(t2);
3265 }
3266
3267 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3268 TCGArg idx, MemOp memop, void * const table[])
3269 {
3270 memop = tcg_canonicalize_memop(memop, 1, 0);
3271
3272 if ((memop & MO_SIZE) == MO_64) {
3273 #ifdef CONFIG_ATOMIC64
3274 gen_atomic_op_i64 gen;
3275 MemOpIdx oi;
3276
3277 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3278 tcg_debug_assert(gen != NULL);
3279
3280 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3281 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3282 #else
3283 gen_helper_exit_atomic(cpu_env);
3284 /* Produce a result, so that we have a well-formed opcode stream
3285 with respect to uses of the result in the (dead) code following. */
3286 tcg_gen_movi_i64(ret, 0);
3287 #endif /* CONFIG_ATOMIC64 */
3288 } else {
3289 TCGv_i32 v32 = tcg_temp_new_i32();
3290 TCGv_i32 r32 = tcg_temp_new_i32();
3291
3292 tcg_gen_extrl_i64_i32(v32, val);
3293 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3294 tcg_temp_free_i32(v32);
3295
3296 tcg_gen_extu_i32_i64(ret, r32);
3297 tcg_temp_free_i32(r32);
3298
3299 if (memop & MO_SIGN) {
3300 tcg_gen_ext_i64(ret, ret, memop);
3301 }
3302 }
3303 }
3304
3305 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3306 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
3307 [MO_8] = gen_helper_atomic_##NAME##b, \
3308 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3309 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3310 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3311 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
3312 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3313 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
3314 }; \
3315 void tcg_gen_atomic_##NAME##_i32 \
3316 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \
3317 { \
3318 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3319 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3320 } else { \
3321 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3322 tcg_gen_##OP##_i32); \
3323 } \
3324 } \
3325 void tcg_gen_atomic_##NAME##_i64 \
3326 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \
3327 { \
3328 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3329 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3330 } else { \
3331 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3332 tcg_gen_##OP##_i64); \
3333 } \
3334 }
3335
3336 GEN_ATOMIC_HELPER(fetch_add, add, 0)
3337 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3338 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3339 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3340 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3341 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3342 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3343 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3344
3345 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3346 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3347 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3348 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3349 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3350 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3351 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3352 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3353
3354 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3355 {
3356 tcg_gen_mov_i32(r, b);
3357 }
3358
3359 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3360 {
3361 tcg_gen_mov_i64(r, b);
3362 }
3363
3364 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3365
3366 #undef GEN_ATOMIC_HELPER
QEMU mirror