]> git.proxmox.com Git - mirror_qemu.git/blob - tcg/tcg-op.c
projects / mirror_qemu.git / blob
? search:


2b520c19536239f2287d2327f4a00adc472581bd
[mirror_qemu.git] / tcg / tcg-op.c
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "qemu-common.h"
27 #include "cpu.h"
28 #include "exec/exec-all.h"
29 #include "tcg.h"
30 #include "tcg-op.h"
31 #include "trace-tcg.h"
32 #include "trace/mem.h"
33
34 /* Reduce the number of ifdefs below. This assumes that all uses of
35 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
36 the compiler can eliminate. */
37 #if TCG_TARGET_REG_BITS == 64
38 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
39 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
40 #define TCGV_LOW TCGV_LOW_link_error
41 #define TCGV_HIGH TCGV_HIGH_link_error
42 #endif
43
44 /* Note that this is optimized for sequential allocation during translate.
45 Up to and including filling in the forward link immediately. We'll do
46 proper termination of the end of the list after we finish translation. */
47
48 static void tcg_emit_op(TCGContext *ctx, TCGOpcode opc, int args)
49 {
50 int oi = ctx->gen_next_op_idx;
51 int ni = oi + 1;
52 int pi = oi - 1;
53
54 tcg_debug_assert(oi < OPC_BUF_SIZE);
55 ctx->gen_op_buf[0].prev = oi;
56 ctx->gen_next_op_idx = ni;
57
58 ctx->gen_op_buf[oi] = (TCGOp){
59 .opc = opc,
60 .args = args,
61 .prev = pi,
62 .next = ni
63 };
64 }
65
66 void tcg_gen_op1(TCGContext *ctx, TCGOpcode opc, TCGArg a1)
67 {
68 int pi = ctx->gen_next_parm_idx;
69
70 tcg_debug_assert(pi + 1 <= OPPARAM_BUF_SIZE);
71 ctx->gen_next_parm_idx = pi + 1;
72 ctx->gen_opparam_buf[pi] = a1;
73
74 tcg_emit_op(ctx, opc, pi);
75 }
76
77 void tcg_gen_op2(TCGContext *ctx, TCGOpcode opc, TCGArg a1, TCGArg a2)
78 {
79 int pi = ctx->gen_next_parm_idx;
80
81 tcg_debug_assert(pi + 2 <= OPPARAM_BUF_SIZE);
82 ctx->gen_next_parm_idx = pi + 2;
83 ctx->gen_opparam_buf[pi + 0] = a1;
84 ctx->gen_opparam_buf[pi + 1] = a2;
85
86 tcg_emit_op(ctx, opc, pi);
87 }
88
89 void tcg_gen_op3(TCGContext *ctx, TCGOpcode opc, TCGArg a1,
90 TCGArg a2, TCGArg a3)
91 {
92 int pi = ctx->gen_next_parm_idx;
93
94 tcg_debug_assert(pi + 3 <= OPPARAM_BUF_SIZE);
95 ctx->gen_next_parm_idx = pi + 3;
96 ctx->gen_opparam_buf[pi + 0] = a1;
97 ctx->gen_opparam_buf[pi + 1] = a2;
98 ctx->gen_opparam_buf[pi + 2] = a3;
99
100 tcg_emit_op(ctx, opc, pi);
101 }
102
103 void tcg_gen_op4(TCGContext *ctx, TCGOpcode opc, TCGArg a1,
104 TCGArg a2, TCGArg a3, TCGArg a4)
105 {
106 int pi = ctx->gen_next_parm_idx;
107
108 tcg_debug_assert(pi + 4 <= OPPARAM_BUF_SIZE);
109 ctx->gen_next_parm_idx = pi + 4;
110 ctx->gen_opparam_buf[pi + 0] = a1;
111 ctx->gen_opparam_buf[pi + 1] = a2;
112 ctx->gen_opparam_buf[pi + 2] = a3;
113 ctx->gen_opparam_buf[pi + 3] = a4;
114
115 tcg_emit_op(ctx, opc, pi);
116 }
117
118 void tcg_gen_op5(TCGContext *ctx, TCGOpcode opc, TCGArg a1,
119 TCGArg a2, TCGArg a3, TCGArg a4, TCGArg a5)
120 {
121 int pi = ctx->gen_next_parm_idx;
122
123 tcg_debug_assert(pi + 5 <= OPPARAM_BUF_SIZE);
124 ctx->gen_next_parm_idx = pi + 5;
125 ctx->gen_opparam_buf[pi + 0] = a1;
126 ctx->gen_opparam_buf[pi + 1] = a2;
127 ctx->gen_opparam_buf[pi + 2] = a3;
128 ctx->gen_opparam_buf[pi + 3] = a4;
129 ctx->gen_opparam_buf[pi + 4] = a5;
130
131 tcg_emit_op(ctx, opc, pi);
132 }
133
134 void tcg_gen_op6(TCGContext *ctx, TCGOpcode opc, TCGArg a1, TCGArg a2,
135 TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6)
136 {
137 int pi = ctx->gen_next_parm_idx;
138
139 tcg_debug_assert(pi + 6 <= OPPARAM_BUF_SIZE);
140 ctx->gen_next_parm_idx = pi + 6;
141 ctx->gen_opparam_buf[pi + 0] = a1;
142 ctx->gen_opparam_buf[pi + 1] = a2;
143 ctx->gen_opparam_buf[pi + 2] = a3;
144 ctx->gen_opparam_buf[pi + 3] = a4;
145 ctx->gen_opparam_buf[pi + 4] = a5;
146 ctx->gen_opparam_buf[pi + 5] = a6;
147
148 tcg_emit_op(ctx, opc, pi);
149 }
150
151 void tcg_gen_mb(TCGBar mb_type)
152 {
153 if (parallel_cpus) {
154 tcg_gen_op1(&tcg_ctx, INDEX_op_mb, mb_type);
155 }
156 }
157
158 /* 32 bit ops */
159
160 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
161 {
162 /* some cases can be optimized here */
163 if (arg2 == 0) {
164 tcg_gen_mov_i32(ret, arg1);
165 } else {
166 TCGv_i32 t0 = tcg_const_i32(arg2);
167 tcg_gen_add_i32(ret, arg1, t0);
168 tcg_temp_free_i32(t0);
169 }
170 }
171
172 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
173 {
174 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
175 /* Don't recurse with tcg_gen_neg_i32. */
176 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
177 } else {
178 TCGv_i32 t0 = tcg_const_i32(arg1);
179 tcg_gen_sub_i32(ret, t0, arg2);
180 tcg_temp_free_i32(t0);
181 }
182 }
183
184 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
185 {
186 /* some cases can be optimized here */
187 if (arg2 == 0) {
188 tcg_gen_mov_i32(ret, arg1);
189 } else {
190 TCGv_i32 t0 = tcg_const_i32(arg2);
191 tcg_gen_sub_i32(ret, arg1, t0);
192 tcg_temp_free_i32(t0);
193 }
194 }
195
196 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
197 {
198 TCGv_i32 t0;
199 /* Some cases can be optimized here. */
200 switch (arg2) {
201 case 0:
202 tcg_gen_movi_i32(ret, 0);
203 return;
204 case 0xffffffffu:
205 tcg_gen_mov_i32(ret, arg1);
206 return;
207 case 0xffu:
208 /* Don't recurse with tcg_gen_ext8u_i32. */
209 if (TCG_TARGET_HAS_ext8u_i32) {
210 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
211 return;
212 }
213 break;
214 case 0xffffu:
215 if (TCG_TARGET_HAS_ext16u_i32) {
216 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
217 return;
218 }
219 break;
220 }
221 t0 = tcg_const_i32(arg2);
222 tcg_gen_and_i32(ret, arg1, t0);
223 tcg_temp_free_i32(t0);
224 }
225
226 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
227 {
228 /* Some cases can be optimized here. */
229 if (arg2 == -1) {
230 tcg_gen_movi_i32(ret, -1);
231 } else if (arg2 == 0) {
232 tcg_gen_mov_i32(ret, arg1);
233 } else {
234 TCGv_i32 t0 = tcg_const_i32(arg2);
235 tcg_gen_or_i32(ret, arg1, t0);
236 tcg_temp_free_i32(t0);
237 }
238 }
239
240 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
241 {
242 /* Some cases can be optimized here. */
243 if (arg2 == 0) {
244 tcg_gen_mov_i32(ret, arg1);
245 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
246 /* Don't recurse with tcg_gen_not_i32. */
247 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
248 } else {
249 TCGv_i32 t0 = tcg_const_i32(arg2);
250 tcg_gen_xor_i32(ret, arg1, t0);
251 tcg_temp_free_i32(t0);
252 }
253 }
254
255 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
256 {
257 tcg_debug_assert(arg2 < 32);
258 if (arg2 == 0) {
259 tcg_gen_mov_i32(ret, arg1);
260 } else {
261 TCGv_i32 t0 = tcg_const_i32(arg2);
262 tcg_gen_shl_i32(ret, arg1, t0);
263 tcg_temp_free_i32(t0);
264 }
265 }
266
267 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
268 {
269 tcg_debug_assert(arg2 < 32);
270 if (arg2 == 0) {
271 tcg_gen_mov_i32(ret, arg1);
272 } else {
273 TCGv_i32 t0 = tcg_const_i32(arg2);
274 tcg_gen_shr_i32(ret, arg1, t0);
275 tcg_temp_free_i32(t0);
276 }
277 }
278
279 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
280 {
281 tcg_debug_assert(arg2 < 32);
282 if (arg2 == 0) {
283 tcg_gen_mov_i32(ret, arg1);
284 } else {
285 TCGv_i32 t0 = tcg_const_i32(arg2);
286 tcg_gen_sar_i32(ret, arg1, t0);
287 tcg_temp_free_i32(t0);
288 }
289 }
290
291 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
292 {
293 if (cond == TCG_COND_ALWAYS) {
294 tcg_gen_br(l);
295 } else if (cond != TCG_COND_NEVER) {
296 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
297 }
298 }
299
300 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
301 {
302 if (cond == TCG_COND_ALWAYS) {
303 tcg_gen_br(l);
304 } else if (cond != TCG_COND_NEVER) {
305 TCGv_i32 t0 = tcg_const_i32(arg2);
306 tcg_gen_brcond_i32(cond, arg1, t0, l);
307 tcg_temp_free_i32(t0);
308 }
309 }
310
311 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
312 TCGv_i32 arg1, TCGv_i32 arg2)
313 {
314 if (cond == TCG_COND_ALWAYS) {
315 tcg_gen_movi_i32(ret, 1);
316 } else if (cond == TCG_COND_NEVER) {
317 tcg_gen_movi_i32(ret, 0);
318 } else {
319 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
320 }
321 }
322
323 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
324 TCGv_i32 arg1, int32_t arg2)
325 {
326 TCGv_i32 t0 = tcg_const_i32(arg2);
327 tcg_gen_setcond_i32(cond, ret, arg1, t0);
328 tcg_temp_free_i32(t0);
329 }
330
331 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
332 {
333 TCGv_i32 t0 = tcg_const_i32(arg2);
334 tcg_gen_mul_i32(ret, arg1, t0);
335 tcg_temp_free_i32(t0);
336 }
337
338 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
339 {
340 if (TCG_TARGET_HAS_div_i32) {
341 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
342 } else if (TCG_TARGET_HAS_div2_i32) {
343 TCGv_i32 t0 = tcg_temp_new_i32();
344 tcg_gen_sari_i32(t0, arg1, 31);
345 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
346 tcg_temp_free_i32(t0);
347 } else {
348 gen_helper_div_i32(ret, arg1, arg2);
349 }
350 }
351
352 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
353 {
354 if (TCG_TARGET_HAS_rem_i32) {
355 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
356 } else if (TCG_TARGET_HAS_div_i32) {
357 TCGv_i32 t0 = tcg_temp_new_i32();
358 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
359 tcg_gen_mul_i32(t0, t0, arg2);
360 tcg_gen_sub_i32(ret, arg1, t0);
361 tcg_temp_free_i32(t0);
362 } else if (TCG_TARGET_HAS_div2_i32) {
363 TCGv_i32 t0 = tcg_temp_new_i32();
364 tcg_gen_sari_i32(t0, arg1, 31);
365 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
366 tcg_temp_free_i32(t0);
367 } else {
368 gen_helper_rem_i32(ret, arg1, arg2);
369 }
370 }
371
372 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
373 {
374 if (TCG_TARGET_HAS_div_i32) {
375 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
376 } else if (TCG_TARGET_HAS_div2_i32) {
377 TCGv_i32 t0 = tcg_temp_new_i32();
378 tcg_gen_movi_i32(t0, 0);
379 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
380 tcg_temp_free_i32(t0);
381 } else {
382 gen_helper_divu_i32(ret, arg1, arg2);
383 }
384 }
385
386 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
387 {
388 if (TCG_TARGET_HAS_rem_i32) {
389 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
390 } else if (TCG_TARGET_HAS_div_i32) {
391 TCGv_i32 t0 = tcg_temp_new_i32();
392 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
393 tcg_gen_mul_i32(t0, t0, arg2);
394 tcg_gen_sub_i32(ret, arg1, t0);
395 tcg_temp_free_i32(t0);
396 } else if (TCG_TARGET_HAS_div2_i32) {
397 TCGv_i32 t0 = tcg_temp_new_i32();
398 tcg_gen_movi_i32(t0, 0);
399 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
400 tcg_temp_free_i32(t0);
401 } else {
402 gen_helper_remu_i32(ret, arg1, arg2);
403 }
404 }
405
406 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
407 {
408 if (TCG_TARGET_HAS_andc_i32) {
409 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
410 } else {
411 TCGv_i32 t0 = tcg_temp_new_i32();
412 tcg_gen_not_i32(t0, arg2);
413 tcg_gen_and_i32(ret, arg1, t0);
414 tcg_temp_free_i32(t0);
415 }
416 }
417
418 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
419 {
420 if (TCG_TARGET_HAS_eqv_i32) {
421 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
422 } else {
423 tcg_gen_xor_i32(ret, arg1, arg2);
424 tcg_gen_not_i32(ret, ret);
425 }
426 }
427
428 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
429 {
430 if (TCG_TARGET_HAS_nand_i32) {
431 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
432 } else {
433 tcg_gen_and_i32(ret, arg1, arg2);
434 tcg_gen_not_i32(ret, ret);
435 }
436 }
437
438 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
439 {
440 if (TCG_TARGET_HAS_nor_i32) {
441 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
442 } else {
443 tcg_gen_or_i32(ret, arg1, arg2);
444 tcg_gen_not_i32(ret, ret);
445 }
446 }
447
448 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
449 {
450 if (TCG_TARGET_HAS_orc_i32) {
451 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
452 } else {
453 TCGv_i32 t0 = tcg_temp_new_i32();
454 tcg_gen_not_i32(t0, arg2);
455 tcg_gen_or_i32(ret, arg1, t0);
456 tcg_temp_free_i32(t0);
457 }
458 }
459
460 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
461 {
462 if (TCG_TARGET_HAS_clz_i32) {
463 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
464 } else if (TCG_TARGET_HAS_clz_i64) {
465 TCGv_i64 t1 = tcg_temp_new_i64();
466 TCGv_i64 t2 = tcg_temp_new_i64();
467 tcg_gen_extu_i32_i64(t1, arg1);
468 tcg_gen_extu_i32_i64(t2, arg2);
469 tcg_gen_addi_i64(t2, t2, 32);
470 tcg_gen_clz_i64(t1, t1, t2);
471 tcg_gen_extrl_i64_i32(ret, t1);
472 tcg_temp_free_i64(t1);
473 tcg_temp_free_i64(t2);
474 tcg_gen_subi_i32(ret, ret, 32);
475 } else {
476 gen_helper_clz_i32(ret, arg1, arg2);
477 }
478 }
479
480 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
481 {
482 TCGv_i32 t = tcg_const_i32(arg2);
483 tcg_gen_clz_i32(ret, arg1, t);
484 tcg_temp_free_i32(t);
485 }
486
487 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
488 {
489 if (TCG_TARGET_HAS_ctz_i32) {
490 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
491 } else if (TCG_TARGET_HAS_ctz_i64) {
492 TCGv_i64 t1 = tcg_temp_new_i64();
493 TCGv_i64 t2 = tcg_temp_new_i64();
494 tcg_gen_extu_i32_i64(t1, arg1);
495 tcg_gen_extu_i32_i64(t2, arg2);
496 tcg_gen_ctz_i64(t1, t1, t2);
497 tcg_gen_extrl_i64_i32(ret, t1);
498 tcg_temp_free_i64(t1);
499 tcg_temp_free_i64(t2);
500 } else if (TCG_TARGET_HAS_clz_i32) {
501 TCGv_i32 t1 = tcg_temp_new_i32();
502 TCGv_i32 t2 = tcg_temp_new_i32();
503 tcg_gen_neg_i32(t1, arg1);
504 tcg_gen_xori_i32(t2, arg2, 31);
505 tcg_gen_and_i32(t1, t1, arg1);
506 tcg_gen_clz_i32(ret, t1, t2);
507 tcg_temp_free_i32(t1);
508 tcg_temp_free_i32(t2);
509 tcg_gen_xori_i32(ret, ret, 31);
510 } else if (TCG_TARGET_HAS_clz_i64) {
511 TCGv_i32 t1 = tcg_temp_new_i32();
512 TCGv_i32 t2 = tcg_temp_new_i32();
513 TCGv_i64 x1 = tcg_temp_new_i64();
514 TCGv_i64 x2 = tcg_temp_new_i64();
515 tcg_gen_neg_i32(t1, arg1);
516 tcg_gen_xori_i32(t2, arg2, 63);
517 tcg_gen_and_i32(t1, t1, arg1);
518 tcg_gen_extu_i32_i64(x1, t1);
519 tcg_gen_extu_i32_i64(x2, t2);
520 tcg_temp_free_i32(t1);
521 tcg_temp_free_i32(t2);
522 tcg_gen_clz_i64(x1, x1, x2);
523 tcg_gen_extrl_i64_i32(ret, x1);
524 tcg_temp_free_i64(x1);
525 tcg_temp_free_i64(x2);
526 tcg_gen_xori_i32(ret, ret, 63);
527 } else {
528 gen_helper_ctz_i32(ret, arg1, arg2);
529 }
530 }
531
532 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
533 {
534 TCGv_i32 t = tcg_const_i32(arg2);
535 tcg_gen_ctz_i32(ret, arg1, t);
536 tcg_temp_free_i32(t);
537 }
538
539 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
540 {
541 if (TCG_TARGET_HAS_rot_i32) {
542 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
543 } else {
544 TCGv_i32 t0, t1;
545
546 t0 = tcg_temp_new_i32();
547 t1 = tcg_temp_new_i32();
548 tcg_gen_shl_i32(t0, arg1, arg2);
549 tcg_gen_subfi_i32(t1, 32, arg2);
550 tcg_gen_shr_i32(t1, arg1, t1);
551 tcg_gen_or_i32(ret, t0, t1);
552 tcg_temp_free_i32(t0);
553 tcg_temp_free_i32(t1);
554 }
555 }
556
557 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
558 {
559 tcg_debug_assert(arg2 < 32);
560 /* some cases can be optimized here */
561 if (arg2 == 0) {
562 tcg_gen_mov_i32(ret, arg1);
563 } else if (TCG_TARGET_HAS_rot_i32) {
564 TCGv_i32 t0 = tcg_const_i32(arg2);
565 tcg_gen_rotl_i32(ret, arg1, t0);
566 tcg_temp_free_i32(t0);
567 } else {
568 TCGv_i32 t0, t1;
569 t0 = tcg_temp_new_i32();
570 t1 = tcg_temp_new_i32();
571 tcg_gen_shli_i32(t0, arg1, arg2);
572 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
573 tcg_gen_or_i32(ret, t0, t1);
574 tcg_temp_free_i32(t0);
575 tcg_temp_free_i32(t1);
576 }
577 }
578
579 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
580 {
581 if (TCG_TARGET_HAS_rot_i32) {
582 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
583 } else {
584 TCGv_i32 t0, t1;
585
586 t0 = tcg_temp_new_i32();
587 t1 = tcg_temp_new_i32();
588 tcg_gen_shr_i32(t0, arg1, arg2);
589 tcg_gen_subfi_i32(t1, 32, arg2);
590 tcg_gen_shl_i32(t1, arg1, t1);
591 tcg_gen_or_i32(ret, t0, t1);
592 tcg_temp_free_i32(t0);
593 tcg_temp_free_i32(t1);
594 }
595 }
596
597 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
598 {
599 tcg_debug_assert(arg2 < 32);
600 /* some cases can be optimized here */
601 if (arg2 == 0) {
602 tcg_gen_mov_i32(ret, arg1);
603 } else {
604 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
605 }
606 }
607
608 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
609 unsigned int ofs, unsigned int len)
610 {
611 uint32_t mask;
612 TCGv_i32 t1;
613
614 tcg_debug_assert(ofs < 32);
615 tcg_debug_assert(len > 0);
616 tcg_debug_assert(len <= 32);
617 tcg_debug_assert(ofs + len <= 32);
618
619 if (len == 32) {
620 tcg_gen_mov_i32(ret, arg2);
621 return;
622 }
623 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
624 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
625 return;
626 }
627
628 mask = (1u << len) - 1;
629 t1 = tcg_temp_new_i32();
630
631 if (ofs + len < 32) {
632 tcg_gen_andi_i32(t1, arg2, mask);
633 tcg_gen_shli_i32(t1, t1, ofs);
634 } else {
635 tcg_gen_shli_i32(t1, arg2, ofs);
636 }
637 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
638 tcg_gen_or_i32(ret, ret, t1);
639
640 tcg_temp_free_i32(t1);
641 }
642
643 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
644 unsigned int ofs, unsigned int len)
645 {
646 tcg_debug_assert(ofs < 32);
647 tcg_debug_assert(len > 0);
648 tcg_debug_assert(len <= 32);
649 tcg_debug_assert(ofs + len <= 32);
650
651 if (ofs + len == 32) {
652 tcg_gen_shli_i32(ret, arg, ofs);
653 } else if (ofs == 0) {
654 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
655 } else if (TCG_TARGET_HAS_deposit_i32
656 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
657 TCGv_i32 zero = tcg_const_i32(0);
658 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
659 tcg_temp_free_i32(zero);
660 } else {
661 /* To help two-operand hosts we prefer to zero-extend first,
662 which allows ARG to stay live. */
663 switch (len) {
664 case 16:
665 if (TCG_TARGET_HAS_ext16u_i32) {
666 tcg_gen_ext16u_i32(ret, arg);
667 tcg_gen_shli_i32(ret, ret, ofs);
668 return;
669 }
670 break;
671 case 8:
672 if (TCG_TARGET_HAS_ext8u_i32) {
673 tcg_gen_ext8u_i32(ret, arg);
674 tcg_gen_shli_i32(ret, ret, ofs);
675 return;
676 }
677 break;
678 }
679 /* Otherwise prefer zero-extension over AND for code size. */
680 switch (ofs + len) {
681 case 16:
682 if (TCG_TARGET_HAS_ext16u_i32) {
683 tcg_gen_shli_i32(ret, arg, ofs);
684 tcg_gen_ext16u_i32(ret, ret);
685 return;
686 }
687 break;
688 case 8:
689 if (TCG_TARGET_HAS_ext8u_i32) {
690 tcg_gen_shli_i32(ret, arg, ofs);
691 tcg_gen_ext8u_i32(ret, ret);
692 return;
693 }
694 break;
695 }
696 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
697 tcg_gen_shli_i32(ret, ret, ofs);
698 }
699 }
700
701 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
702 unsigned int ofs, unsigned int len)
703 {
704 tcg_debug_assert(ofs < 32);
705 tcg_debug_assert(len > 0);
706 tcg_debug_assert(len <= 32);
707 tcg_debug_assert(ofs + len <= 32);
708
709 /* Canonicalize certain special cases, even if extract is supported. */
710 if (ofs + len == 32) {
711 tcg_gen_shri_i32(ret, arg, 32 - len);
712 return;
713 }
714 if (ofs == 0) {
715 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
716 return;
717 }
718
719 if (TCG_TARGET_HAS_extract_i32
720 && TCG_TARGET_extract_i32_valid(ofs, len)) {
721 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
722 return;
723 }
724
725 /* Assume that zero-extension, if available, is cheaper than a shift. */
726 switch (ofs + len) {
727 case 16:
728 if (TCG_TARGET_HAS_ext16u_i32) {
729 tcg_gen_ext16u_i32(ret, arg);
730 tcg_gen_shri_i32(ret, ret, ofs);
731 return;
732 }
733 break;
734 case 8:
735 if (TCG_TARGET_HAS_ext8u_i32) {
736 tcg_gen_ext8u_i32(ret, arg);
737 tcg_gen_shri_i32(ret, ret, ofs);
738 return;
739 }
740 break;
741 }
742
743 /* ??? Ideally we'd know what values are available for immediate AND.
744 Assume that 8 bits are available, plus the special case of 16,
745 so that we get ext8u, ext16u. */
746 switch (len) {
747 case 1 ... 8: case 16:
748 tcg_gen_shri_i32(ret, arg, ofs);
749 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
750 break;
751 default:
752 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
753 tcg_gen_shri_i32(ret, ret, 32 - len);
754 break;
755 }
756 }
757
758 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
759 unsigned int ofs, unsigned int len)
760 {
761 tcg_debug_assert(ofs < 32);
762 tcg_debug_assert(len > 0);
763 tcg_debug_assert(len <= 32);
764 tcg_debug_assert(ofs + len <= 32);
765
766 /* Canonicalize certain special cases, even if extract is supported. */
767 if (ofs + len == 32) {
768 tcg_gen_sari_i32(ret, arg, 32 - len);
769 return;
770 }
771 if (ofs == 0) {
772 switch (len) {
773 case 16:
774 tcg_gen_ext16s_i32(ret, arg);
775 return;
776 case 8:
777 tcg_gen_ext8s_i32(ret, arg);
778 return;
779 }
780 }
781
782 if (TCG_TARGET_HAS_sextract_i32
783 && TCG_TARGET_extract_i32_valid(ofs, len)) {
784 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
785 return;
786 }
787
788 /* Assume that sign-extension, if available, is cheaper than a shift. */
789 switch (ofs + len) {
790 case 16:
791 if (TCG_TARGET_HAS_ext16s_i32) {
792 tcg_gen_ext16s_i32(ret, arg);
793 tcg_gen_sari_i32(ret, ret, ofs);
794 return;
795 }
796 break;
797 case 8:
798 if (TCG_TARGET_HAS_ext8s_i32) {
799 tcg_gen_ext8s_i32(ret, arg);
800 tcg_gen_sari_i32(ret, ret, ofs);
801 return;
802 }
803 break;
804 }
805 switch (len) {
806 case 16:
807 if (TCG_TARGET_HAS_ext16s_i32) {
808 tcg_gen_shri_i32(ret, arg, ofs);
809 tcg_gen_ext16s_i32(ret, ret);
810 return;
811 }
812 break;
813 case 8:
814 if (TCG_TARGET_HAS_ext8s_i32) {
815 tcg_gen_shri_i32(ret, arg, ofs);
816 tcg_gen_ext8s_i32(ret, ret);
817 return;
818 }
819 break;
820 }
821
822 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
823 tcg_gen_sari_i32(ret, ret, 32 - len);
824 }
825
826 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
827 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
828 {
829 if (cond == TCG_COND_ALWAYS) {
830 tcg_gen_mov_i32(ret, v1);
831 } else if (cond == TCG_COND_NEVER) {
832 tcg_gen_mov_i32(ret, v2);
833 } else if (TCG_TARGET_HAS_movcond_i32) {
834 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
835 } else {
836 TCGv_i32 t0 = tcg_temp_new_i32();
837 TCGv_i32 t1 = tcg_temp_new_i32();
838 tcg_gen_setcond_i32(cond, t0, c1, c2);
839 tcg_gen_neg_i32(t0, t0);
840 tcg_gen_and_i32(t1, v1, t0);
841 tcg_gen_andc_i32(ret, v2, t0);
842 tcg_gen_or_i32(ret, ret, t1);
843 tcg_temp_free_i32(t0);
844 tcg_temp_free_i32(t1);
845 }
846 }
847
848 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
849 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
850 {
851 if (TCG_TARGET_HAS_add2_i32) {
852 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
853 } else {
854 TCGv_i64 t0 = tcg_temp_new_i64();
855 TCGv_i64 t1 = tcg_temp_new_i64();
856 tcg_gen_concat_i32_i64(t0, al, ah);
857 tcg_gen_concat_i32_i64(t1, bl, bh);
858 tcg_gen_add_i64(t0, t0, t1);
859 tcg_gen_extr_i64_i32(rl, rh, t0);
860 tcg_temp_free_i64(t0);
861 tcg_temp_free_i64(t1);
862 }
863 }
864
865 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
866 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
867 {
868 if (TCG_TARGET_HAS_sub2_i32) {
869 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
870 } else {
871 TCGv_i64 t0 = tcg_temp_new_i64();
872 TCGv_i64 t1 = tcg_temp_new_i64();
873 tcg_gen_concat_i32_i64(t0, al, ah);
874 tcg_gen_concat_i32_i64(t1, bl, bh);
875 tcg_gen_sub_i64(t0, t0, t1);
876 tcg_gen_extr_i64_i32(rl, rh, t0);
877 tcg_temp_free_i64(t0);
878 tcg_temp_free_i64(t1);
879 }
880 }
881
882 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
883 {
884 if (TCG_TARGET_HAS_mulu2_i32) {
885 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
886 } else if (TCG_TARGET_HAS_muluh_i32) {
887 TCGv_i32 t = tcg_temp_new_i32();
888 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
889 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
890 tcg_gen_mov_i32(rl, t);
891 tcg_temp_free_i32(t);
892 } else {
893 TCGv_i64 t0 = tcg_temp_new_i64();
894 TCGv_i64 t1 = tcg_temp_new_i64();
895 tcg_gen_extu_i32_i64(t0, arg1);
896 tcg_gen_extu_i32_i64(t1, arg2);
897 tcg_gen_mul_i64(t0, t0, t1);
898 tcg_gen_extr_i64_i32(rl, rh, t0);
899 tcg_temp_free_i64(t0);
900 tcg_temp_free_i64(t1);
901 }
902 }
903
904 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
905 {
906 if (TCG_TARGET_HAS_muls2_i32) {
907 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
908 } else if (TCG_TARGET_HAS_mulsh_i32) {
909 TCGv_i32 t = tcg_temp_new_i32();
910 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
911 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
912 tcg_gen_mov_i32(rl, t);
913 tcg_temp_free_i32(t);
914 } else if (TCG_TARGET_REG_BITS == 32) {
915 TCGv_i32 t0 = tcg_temp_new_i32();
916 TCGv_i32 t1 = tcg_temp_new_i32();
917 TCGv_i32 t2 = tcg_temp_new_i32();
918 TCGv_i32 t3 = tcg_temp_new_i32();
919 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
920 /* Adjust for negative inputs. */
921 tcg_gen_sari_i32(t2, arg1, 31);
922 tcg_gen_sari_i32(t3, arg2, 31);
923 tcg_gen_and_i32(t2, t2, arg2);
924 tcg_gen_and_i32(t3, t3, arg1);
925 tcg_gen_sub_i32(rh, t1, t2);
926 tcg_gen_sub_i32(rh, rh, t3);
927 tcg_gen_mov_i32(rl, t0);
928 tcg_temp_free_i32(t0);
929 tcg_temp_free_i32(t1);
930 tcg_temp_free_i32(t2);
931 tcg_temp_free_i32(t3);
932 } else {
933 TCGv_i64 t0 = tcg_temp_new_i64();
934 TCGv_i64 t1 = tcg_temp_new_i64();
935 tcg_gen_ext_i32_i64(t0, arg1);
936 tcg_gen_ext_i32_i64(t1, arg2);
937 tcg_gen_mul_i64(t0, t0, t1);
938 tcg_gen_extr_i64_i32(rl, rh, t0);
939 tcg_temp_free_i64(t0);
940 tcg_temp_free_i64(t1);
941 }
942 }
943
944 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
945 {
946 if (TCG_TARGET_REG_BITS == 32) {
947 TCGv_i32 t0 = tcg_temp_new_i32();
948 TCGv_i32 t1 = tcg_temp_new_i32();
949 TCGv_i32 t2 = tcg_temp_new_i32();
950 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
951 /* Adjust for negative input for the signed arg1. */
952 tcg_gen_sari_i32(t2, arg1, 31);
953 tcg_gen_and_i32(t2, t2, arg2);
954 tcg_gen_sub_i32(rh, t1, t2);
955 tcg_gen_mov_i32(rl, t0);
956 tcg_temp_free_i32(t0);
957 tcg_temp_free_i32(t1);
958 tcg_temp_free_i32(t2);
959 } else {
960 TCGv_i64 t0 = tcg_temp_new_i64();
961 TCGv_i64 t1 = tcg_temp_new_i64();
962 tcg_gen_ext_i32_i64(t0, arg1);
963 tcg_gen_extu_i32_i64(t1, arg2);
964 tcg_gen_mul_i64(t0, t0, t1);
965 tcg_gen_extr_i64_i32(rl, rh, t0);
966 tcg_temp_free_i64(t0);
967 tcg_temp_free_i64(t1);
968 }
969 }
970
971 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
972 {
973 if (TCG_TARGET_HAS_ext8s_i32) {
974 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
975 } else {
976 tcg_gen_shli_i32(ret, arg, 24);
977 tcg_gen_sari_i32(ret, ret, 24);
978 }
979 }
980
981 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
982 {
983 if (TCG_TARGET_HAS_ext16s_i32) {
984 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
985 } else {
986 tcg_gen_shli_i32(ret, arg, 16);
987 tcg_gen_sari_i32(ret, ret, 16);
988 }
989 }
990
991 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
992 {
993 if (TCG_TARGET_HAS_ext8u_i32) {
994 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
995 } else {
996 tcg_gen_andi_i32(ret, arg, 0xffu);
997 }
998 }
999
1000 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
1001 {
1002 if (TCG_TARGET_HAS_ext16u_i32) {
1003 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
1004 } else {
1005 tcg_gen_andi_i32(ret, arg, 0xffffu);
1006 }
1007 }
1008
1009 /* Note: we assume the two high bytes are set to zero */
1010 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
1011 {
1012 if (TCG_TARGET_HAS_bswap16_i32) {
1013 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
1014 } else {
1015 TCGv_i32 t0 = tcg_temp_new_i32();
1016
1017 tcg_gen_ext8u_i32(t0, arg);
1018 tcg_gen_shli_i32(t0, t0, 8);
1019 tcg_gen_shri_i32(ret, arg, 8);
1020 tcg_gen_or_i32(ret, ret, t0);
1021 tcg_temp_free_i32(t0);
1022 }
1023 }
1024
1025 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1026 {
1027 if (TCG_TARGET_HAS_bswap32_i32) {
1028 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1029 } else {
1030 TCGv_i32 t0, t1;
1031 t0 = tcg_temp_new_i32();
1032 t1 = tcg_temp_new_i32();
1033
1034 tcg_gen_shli_i32(t0, arg, 24);
1035
1036 tcg_gen_andi_i32(t1, arg, 0x0000ff00);
1037 tcg_gen_shli_i32(t1, t1, 8);
1038 tcg_gen_or_i32(t0, t0, t1);
1039
1040 tcg_gen_shri_i32(t1, arg, 8);
1041 tcg_gen_andi_i32(t1, t1, 0x0000ff00);
1042 tcg_gen_or_i32(t0, t0, t1);
1043
1044 tcg_gen_shri_i32(t1, arg, 24);
1045 tcg_gen_or_i32(ret, t0, t1);
1046 tcg_temp_free_i32(t0);
1047 tcg_temp_free_i32(t1);
1048 }
1049 }
1050
1051 /* 64-bit ops */
1052
1053 #if TCG_TARGET_REG_BITS == 32
1054 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1055
1056 void tcg_gen_discard_i64(TCGv_i64 arg)
1057 {
1058 tcg_gen_discard_i32(TCGV_LOW(arg));
1059 tcg_gen_discard_i32(TCGV_HIGH(arg));
1060 }
1061
1062 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1063 {
1064 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1065 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1066 }
1067
1068 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1069 {
1070 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1071 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1072 }
1073
1074 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1075 {
1076 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1077 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1078 }
1079
1080 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1081 {
1082 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1083 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1084 }
1085
1086 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1087 {
1088 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1089 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1090 }
1091
1092 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1093 {
1094 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1095 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1096 }
1097
1098 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1099 {
1100 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1101 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1102 }
1103
1104 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1105 {
1106 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1107 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1108 }
1109
1110 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1111 {
1112 /* Since arg2 and ret have different types,
1113 they cannot be the same temporary */
1114 #ifdef HOST_WORDS_BIGENDIAN
1115 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1116 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1117 #else
1118 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1119 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1120 #endif
1121 }
1122
1123 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1124 {
1125 #ifdef HOST_WORDS_BIGENDIAN
1126 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1127 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1128 #else
1129 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1130 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1131 #endif
1132 }
1133
1134 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1135 {
1136 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1137 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1138 }
1139
1140 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1141 {
1142 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1143 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1144 }
1145
1146 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1147 {
1148 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1149 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1150 }
1151
1152 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1153 {
1154 gen_helper_shl_i64(ret, arg1, arg2);
1155 }
1156
1157 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1158 {
1159 gen_helper_shr_i64(ret, arg1, arg2);
1160 }
1161
1162 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1163 {
1164 gen_helper_sar_i64(ret, arg1, arg2);
1165 }
1166
1167 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1168 {
1169 TCGv_i64 t0;
1170 TCGv_i32 t1;
1171
1172 t0 = tcg_temp_new_i64();
1173 t1 = tcg_temp_new_i32();
1174
1175 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1176 TCGV_LOW(arg1), TCGV_LOW(arg2));
1177
1178 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1179 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1180 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1181 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1182
1183 tcg_gen_mov_i64(ret, t0);
1184 tcg_temp_free_i64(t0);
1185 tcg_temp_free_i32(t1);
1186 }
1187 #endif /* TCG_TARGET_REG_SIZE == 32 */
1188
1189 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1190 {
1191 /* some cases can be optimized here */
1192 if (arg2 == 0) {
1193 tcg_gen_mov_i64(ret, arg1);
1194 } else {
1195 TCGv_i64 t0 = tcg_const_i64(arg2);
1196 tcg_gen_add_i64(ret, arg1, t0);
1197 tcg_temp_free_i64(t0);
1198 }
1199 }
1200
1201 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1202 {
1203 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1204 /* Don't recurse with tcg_gen_neg_i64. */
1205 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1206 } else {
1207 TCGv_i64 t0 = tcg_const_i64(arg1);
1208 tcg_gen_sub_i64(ret, t0, arg2);
1209 tcg_temp_free_i64(t0);
1210 }
1211 }
1212
1213 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1214 {
1215 /* some cases can be optimized here */
1216 if (arg2 == 0) {
1217 tcg_gen_mov_i64(ret, arg1);
1218 } else {
1219 TCGv_i64 t0 = tcg_const_i64(arg2);
1220 tcg_gen_sub_i64(ret, arg1, t0);
1221 tcg_temp_free_i64(t0);
1222 }
1223 }
1224
1225 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1226 {
1227 TCGv_i64 t0;
1228
1229 if (TCG_TARGET_REG_BITS == 32) {
1230 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1231 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1232 return;
1233 }
1234
1235 /* Some cases can be optimized here. */
1236 switch (arg2) {
1237 case 0:
1238 tcg_gen_movi_i64(ret, 0);
1239 return;
1240 case 0xffffffffffffffffull:
1241 tcg_gen_mov_i64(ret, arg1);
1242 return;
1243 case 0xffull:
1244 /* Don't recurse with tcg_gen_ext8u_i64. */
1245 if (TCG_TARGET_HAS_ext8u_i64) {
1246 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1247 return;
1248 }
1249 break;
1250 case 0xffffu:
1251 if (TCG_TARGET_HAS_ext16u_i64) {
1252 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1253 return;
1254 }
1255 break;
1256 case 0xffffffffull:
1257 if (TCG_TARGET_HAS_ext32u_i64) {
1258 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1259 return;
1260 }
1261 break;
1262 }
1263 t0 = tcg_const_i64(arg2);
1264 tcg_gen_and_i64(ret, arg1, t0);
1265 tcg_temp_free_i64(t0);
1266 }
1267
1268 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1269 {
1270 if (TCG_TARGET_REG_BITS == 32) {
1271 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1272 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1273 return;
1274 }
1275 /* Some cases can be optimized here. */
1276 if (arg2 == -1) {
1277 tcg_gen_movi_i64(ret, -1);
1278 } else if (arg2 == 0) {
1279 tcg_gen_mov_i64(ret, arg1);
1280 } else {
1281 TCGv_i64 t0 = tcg_const_i64(arg2);
1282 tcg_gen_or_i64(ret, arg1, t0);
1283 tcg_temp_free_i64(t0);
1284 }
1285 }
1286
1287 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1288 {
1289 if (TCG_TARGET_REG_BITS == 32) {
1290 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1291 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1292 return;
1293 }
1294 /* Some cases can be optimized here. */
1295 if (arg2 == 0) {
1296 tcg_gen_mov_i64(ret, arg1);
1297 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1298 /* Don't recurse with tcg_gen_not_i64. */
1299 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1300 } else {
1301 TCGv_i64 t0 = tcg_const_i64(arg2);
1302 tcg_gen_xor_i64(ret, arg1, t0);
1303 tcg_temp_free_i64(t0);
1304 }
1305 }
1306
1307 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1308 unsigned c, bool right, bool arith)
1309 {
1310 tcg_debug_assert(c < 64);
1311 if (c == 0) {
1312 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1313 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1314 } else if (c >= 32) {
1315 c -= 32;
1316 if (right) {
1317 if (arith) {
1318 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1319 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1320 } else {
1321 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1322 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1323 }
1324 } else {
1325 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1326 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1327 }
1328 } else {
1329 TCGv_i32 t0, t1;
1330
1331 t0 = tcg_temp_new_i32();
1332 t1 = tcg_temp_new_i32();
1333 if (right) {
1334 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1335 if (arith) {
1336 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1337 } else {
1338 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1339 }
1340 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1341 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1342 tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1343 } else {
1344 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1345 /* Note: ret can be the same as arg1, so we use t1 */
1346 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1347 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1348 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1349 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1350 }
1351 tcg_temp_free_i32(t0);
1352 tcg_temp_free_i32(t1);
1353 }
1354 }
1355
1356 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1357 {
1358 tcg_debug_assert(arg2 < 64);
1359 if (TCG_TARGET_REG_BITS == 32) {
1360 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1361 } else if (arg2 == 0) {
1362 tcg_gen_mov_i64(ret, arg1);
1363 } else {
1364 TCGv_i64 t0 = tcg_const_i64(arg2);
1365 tcg_gen_shl_i64(ret, arg1, t0);
1366 tcg_temp_free_i64(t0);
1367 }
1368 }
1369
1370 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1371 {
1372 tcg_debug_assert(arg2 < 64);
1373 if (TCG_TARGET_REG_BITS == 32) {
1374 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1375 } else if (arg2 == 0) {
1376 tcg_gen_mov_i64(ret, arg1);
1377 } else {
1378 TCGv_i64 t0 = tcg_const_i64(arg2);
1379 tcg_gen_shr_i64(ret, arg1, t0);
1380 tcg_temp_free_i64(t0);
1381 }
1382 }
1383
1384 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1385 {
1386 tcg_debug_assert(arg2 < 64);
1387 if (TCG_TARGET_REG_BITS == 32) {
1388 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1389 } else if (arg2 == 0) {
1390 tcg_gen_mov_i64(ret, arg1);
1391 } else {
1392 TCGv_i64 t0 = tcg_const_i64(arg2);
1393 tcg_gen_sar_i64(ret, arg1, t0);
1394 tcg_temp_free_i64(t0);
1395 }
1396 }
1397
1398 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1399 {
1400 if (cond == TCG_COND_ALWAYS) {
1401 tcg_gen_br(l);
1402 } else if (cond != TCG_COND_NEVER) {
1403 if (TCG_TARGET_REG_BITS == 32) {
1404 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1405 TCGV_HIGH(arg1), TCGV_LOW(arg2),
1406 TCGV_HIGH(arg2), cond, label_arg(l));
1407 } else {
1408 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1409 label_arg(l));
1410 }
1411 }
1412 }
1413
1414 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1415 {
1416 if (cond == TCG_COND_ALWAYS) {
1417 tcg_gen_br(l);
1418 } else if (cond != TCG_COND_NEVER) {
1419 TCGv_i64 t0 = tcg_const_i64(arg2);
1420 tcg_gen_brcond_i64(cond, arg1, t0, l);
1421 tcg_temp_free_i64(t0);
1422 }
1423 }
1424
1425 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1426 TCGv_i64 arg1, TCGv_i64 arg2)
1427 {
1428 if (cond == TCG_COND_ALWAYS) {
1429 tcg_gen_movi_i64(ret, 1);
1430 } else if (cond == TCG_COND_NEVER) {
1431 tcg_gen_movi_i64(ret, 0);
1432 } else {
1433 if (TCG_TARGET_REG_BITS == 32) {
1434 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1435 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1436 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1437 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1438 } else {
1439 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1440 }
1441 }
1442 }
1443
1444 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1445 TCGv_i64 arg1, int64_t arg2)
1446 {
1447 TCGv_i64 t0 = tcg_const_i64(arg2);
1448 tcg_gen_setcond_i64(cond, ret, arg1, t0);
1449 tcg_temp_free_i64(t0);
1450 }
1451
1452 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1453 {
1454 TCGv_i64 t0 = tcg_const_i64(arg2);
1455 tcg_gen_mul_i64(ret, arg1, t0);
1456 tcg_temp_free_i64(t0);
1457 }
1458
1459 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1460 {
1461 if (TCG_TARGET_HAS_div_i64) {
1462 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1463 } else if (TCG_TARGET_HAS_div2_i64) {
1464 TCGv_i64 t0 = tcg_temp_new_i64();
1465 tcg_gen_sari_i64(t0, arg1, 63);
1466 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1467 tcg_temp_free_i64(t0);
1468 } else {
1469 gen_helper_div_i64(ret, arg1, arg2);
1470 }
1471 }
1472
1473 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1474 {
1475 if (TCG_TARGET_HAS_rem_i64) {
1476 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1477 } else if (TCG_TARGET_HAS_div_i64) {
1478 TCGv_i64 t0 = tcg_temp_new_i64();
1479 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1480 tcg_gen_mul_i64(t0, t0, arg2);
1481 tcg_gen_sub_i64(ret, arg1, t0);
1482 tcg_temp_free_i64(t0);
1483 } else if (TCG_TARGET_HAS_div2_i64) {
1484 TCGv_i64 t0 = tcg_temp_new_i64();
1485 tcg_gen_sari_i64(t0, arg1, 63);
1486 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1487 tcg_temp_free_i64(t0);
1488 } else {
1489 gen_helper_rem_i64(ret, arg1, arg2);
1490 }
1491 }
1492
1493 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1494 {
1495 if (TCG_TARGET_HAS_div_i64) {
1496 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1497 } else if (TCG_TARGET_HAS_div2_i64) {
1498 TCGv_i64 t0 = tcg_temp_new_i64();
1499 tcg_gen_movi_i64(t0, 0);
1500 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1501 tcg_temp_free_i64(t0);
1502 } else {
1503 gen_helper_divu_i64(ret, arg1, arg2);
1504 }
1505 }
1506
1507 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1508 {
1509 if (TCG_TARGET_HAS_rem_i64) {
1510 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1511 } else if (TCG_TARGET_HAS_div_i64) {
1512 TCGv_i64 t0 = tcg_temp_new_i64();
1513 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1514 tcg_gen_mul_i64(t0, t0, arg2);
1515 tcg_gen_sub_i64(ret, arg1, t0);
1516 tcg_temp_free_i64(t0);
1517 } else if (TCG_TARGET_HAS_div2_i64) {
1518 TCGv_i64 t0 = tcg_temp_new_i64();
1519 tcg_gen_movi_i64(t0, 0);
1520 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1521 tcg_temp_free_i64(t0);
1522 } else {
1523 gen_helper_remu_i64(ret, arg1, arg2);
1524 }
1525 }
1526
1527 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1528 {
1529 if (TCG_TARGET_REG_BITS == 32) {
1530 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1531 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1532 } else if (TCG_TARGET_HAS_ext8s_i64) {
1533 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1534 } else {
1535 tcg_gen_shli_i64(ret, arg, 56);
1536 tcg_gen_sari_i64(ret, ret, 56);
1537 }
1538 }
1539
1540 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1541 {
1542 if (TCG_TARGET_REG_BITS == 32) {
1543 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1544 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1545 } else if (TCG_TARGET_HAS_ext16s_i64) {
1546 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1547 } else {
1548 tcg_gen_shli_i64(ret, arg, 48);
1549 tcg_gen_sari_i64(ret, ret, 48);
1550 }
1551 }
1552
1553 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1554 {
1555 if (TCG_TARGET_REG_BITS == 32) {
1556 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1557 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1558 } else if (TCG_TARGET_HAS_ext32s_i64) {
1559 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1560 } else {
1561 tcg_gen_shli_i64(ret, arg, 32);
1562 tcg_gen_sari_i64(ret, ret, 32);
1563 }
1564 }
1565
1566 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1567 {
1568 if (TCG_TARGET_REG_BITS == 32) {
1569 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1570 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1571 } else if (TCG_TARGET_HAS_ext8u_i64) {
1572 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1573 } else {
1574 tcg_gen_andi_i64(ret, arg, 0xffu);
1575 }
1576 }
1577
1578 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1579 {
1580 if (TCG_TARGET_REG_BITS == 32) {
1581 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1582 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1583 } else if (TCG_TARGET_HAS_ext16u_i64) {
1584 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1585 } else {
1586 tcg_gen_andi_i64(ret, arg, 0xffffu);
1587 }
1588 }
1589
1590 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1591 {
1592 if (TCG_TARGET_REG_BITS == 32) {
1593 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1594 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1595 } else if (TCG_TARGET_HAS_ext32u_i64) {
1596 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1597 } else {
1598 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1599 }
1600 }
1601
1602 /* Note: we assume the six high bytes are set to zero */
1603 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1604 {
1605 if (TCG_TARGET_REG_BITS == 32) {
1606 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1607 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1608 } else if (TCG_TARGET_HAS_bswap16_i64) {
1609 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1610 } else {
1611 TCGv_i64 t0 = tcg_temp_new_i64();
1612
1613 tcg_gen_ext8u_i64(t0, arg);
1614 tcg_gen_shli_i64(t0, t0, 8);
1615 tcg_gen_shri_i64(ret, arg, 8);
1616 tcg_gen_or_i64(ret, ret, t0);
1617 tcg_temp_free_i64(t0);
1618 }
1619 }
1620
1621 /* Note: we assume the four high bytes are set to zero */
1622 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1623 {
1624 if (TCG_TARGET_REG_BITS == 32) {
1625 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1626 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1627 } else if (TCG_TARGET_HAS_bswap32_i64) {
1628 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1629 } else {
1630 TCGv_i64 t0, t1;
1631 t0 = tcg_temp_new_i64();
1632 t1 = tcg_temp_new_i64();
1633
1634 tcg_gen_shli_i64(t0, arg, 24);
1635 tcg_gen_ext32u_i64(t0, t0);
1636
1637 tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1638 tcg_gen_shli_i64(t1, t1, 8);
1639 tcg_gen_or_i64(t0, t0, t1);
1640
1641 tcg_gen_shri_i64(t1, arg, 8);
1642 tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1643 tcg_gen_or_i64(t0, t0, t1);
1644
1645 tcg_gen_shri_i64(t1, arg, 24);
1646 tcg_gen_or_i64(ret, t0, t1);
1647 tcg_temp_free_i64(t0);
1648 tcg_temp_free_i64(t1);
1649 }
1650 }
1651
1652 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1653 {
1654 if (TCG_TARGET_REG_BITS == 32) {
1655 TCGv_i32 t0, t1;
1656 t0 = tcg_temp_new_i32();
1657 t1 = tcg_temp_new_i32();
1658
1659 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1660 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1661 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1662 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1663 tcg_temp_free_i32(t0);
1664 tcg_temp_free_i32(t1);
1665 } else if (TCG_TARGET_HAS_bswap64_i64) {
1666 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1667 } else {
1668 TCGv_i64 t0 = tcg_temp_new_i64();
1669 TCGv_i64 t1 = tcg_temp_new_i64();
1670
1671 tcg_gen_shli_i64(t0, arg, 56);
1672
1673 tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1674 tcg_gen_shli_i64(t1, t1, 40);
1675 tcg_gen_or_i64(t0, t0, t1);
1676
1677 tcg_gen_andi_i64(t1, arg, 0x00ff0000);
1678 tcg_gen_shli_i64(t1, t1, 24);
1679 tcg_gen_or_i64(t0, t0, t1);
1680
1681 tcg_gen_andi_i64(t1, arg, 0xff000000);
1682 tcg_gen_shli_i64(t1, t1, 8);
1683 tcg_gen_or_i64(t0, t0, t1);
1684
1685 tcg_gen_shri_i64(t1, arg, 8);
1686 tcg_gen_andi_i64(t1, t1, 0xff000000);
1687 tcg_gen_or_i64(t0, t0, t1);
1688
1689 tcg_gen_shri_i64(t1, arg, 24);
1690 tcg_gen_andi_i64(t1, t1, 0x00ff0000);
1691 tcg_gen_or_i64(t0, t0, t1);
1692
1693 tcg_gen_shri_i64(t1, arg, 40);
1694 tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1695 tcg_gen_or_i64(t0, t0, t1);
1696
1697 tcg_gen_shri_i64(t1, arg, 56);
1698 tcg_gen_or_i64(ret, t0, t1);
1699 tcg_temp_free_i64(t0);
1700 tcg_temp_free_i64(t1);
1701 }
1702 }
1703
1704 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1705 {
1706 if (TCG_TARGET_REG_BITS == 32) {
1707 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1708 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1709 } else if (TCG_TARGET_HAS_not_i64) {
1710 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1711 } else {
1712 tcg_gen_xori_i64(ret, arg, -1);
1713 }
1714 }
1715
1716 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1717 {
1718 if (TCG_TARGET_REG_BITS == 32) {
1719 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1720 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1721 } else if (TCG_TARGET_HAS_andc_i64) {
1722 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1723 } else {
1724 TCGv_i64 t0 = tcg_temp_new_i64();
1725 tcg_gen_not_i64(t0, arg2);
1726 tcg_gen_and_i64(ret, arg1, t0);
1727 tcg_temp_free_i64(t0);
1728 }
1729 }
1730
1731 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1732 {
1733 if (TCG_TARGET_REG_BITS == 32) {
1734 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1735 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1736 } else if (TCG_TARGET_HAS_eqv_i64) {
1737 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1738 } else {
1739 tcg_gen_xor_i64(ret, arg1, arg2);
1740 tcg_gen_not_i64(ret, ret);
1741 }
1742 }
1743
1744 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1745 {
1746 if (TCG_TARGET_REG_BITS == 32) {
1747 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1748 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1749 } else if (TCG_TARGET_HAS_nand_i64) {
1750 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1751 } else {
1752 tcg_gen_and_i64(ret, arg1, arg2);
1753 tcg_gen_not_i64(ret, ret);
1754 }
1755 }
1756
1757 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1758 {
1759 if (TCG_TARGET_REG_BITS == 32) {
1760 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1761 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1762 } else if (TCG_TARGET_HAS_nor_i64) {
1763 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1764 } else {
1765 tcg_gen_or_i64(ret, arg1, arg2);
1766 tcg_gen_not_i64(ret, ret);
1767 }
1768 }
1769
1770 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1771 {
1772 if (TCG_TARGET_REG_BITS == 32) {
1773 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1774 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1775 } else if (TCG_TARGET_HAS_orc_i64) {
1776 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1777 } else {
1778 TCGv_i64 t0 = tcg_temp_new_i64();
1779 tcg_gen_not_i64(t0, arg2);
1780 tcg_gen_or_i64(ret, arg1, t0);
1781 tcg_temp_free_i64(t0);
1782 }
1783 }
1784
1785 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1786 {
1787 if (TCG_TARGET_HAS_clz_i64) {
1788 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1789 } else {
1790 gen_helper_clz_i64(ret, arg1, arg2);
1791 }
1792 }
1793
1794 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1795 {
1796 if (TCG_TARGET_REG_BITS == 32
1797 && TCG_TARGET_HAS_clz_i32
1798 && arg2 <= 0xffffffffu) {
1799 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1800 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1801 tcg_gen_addi_i32(t, t, 32);
1802 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1803 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1804 tcg_temp_free_i32(t);
1805 } else {
1806 TCGv_i64 t = tcg_const_i64(arg2);
1807 tcg_gen_clz_i64(ret, arg1, t);
1808 tcg_temp_free_i64(t);
1809 }
1810 }
1811
1812 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1813 {
1814 if (TCG_TARGET_HAS_ctz_i64) {
1815 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1816 } else if (TCG_TARGET_HAS_clz_i64) {
1817 TCGv_i64 t1 = tcg_temp_new_i64();
1818 TCGv_i64 t2 = tcg_temp_new_i64();
1819 tcg_gen_neg_i64(t1, arg1);
1820 tcg_gen_xori_i64(t2, arg2, 63);
1821 tcg_gen_and_i64(t1, t1, arg1);
1822 tcg_gen_clz_i64(ret, t1, t2);
1823 tcg_temp_free_i64(t1);
1824 tcg_temp_free_i64(t2);
1825 tcg_gen_xori_i64(ret, ret, 63);
1826 } else {
1827 gen_helper_ctz_i64(ret, arg1, arg2);
1828 }
1829 }
1830
1831 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1832 {
1833 if (TCG_TARGET_REG_BITS == 32
1834 && TCG_TARGET_HAS_ctz_i32
1835 && arg2 <= 0xffffffffu) {
1836 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1837 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1838 tcg_gen_addi_i32(t32, t32, 32);
1839 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1840 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1841 tcg_temp_free_i32(t32);
1842 } else {
1843 TCGv_i64 t64 = tcg_const_i64(arg2);
1844 tcg_gen_ctz_i64(ret, arg1, t64);
1845 tcg_temp_free_i64(t64);
1846 }
1847 }
1848
1849 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1850 {
1851 if (TCG_TARGET_HAS_rot_i64) {
1852 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1853 } else {
1854 TCGv_i64 t0, t1;
1855 t0 = tcg_temp_new_i64();
1856 t1 = tcg_temp_new_i64();
1857 tcg_gen_shl_i64(t0, arg1, arg2);
1858 tcg_gen_subfi_i64(t1, 64, arg2);
1859 tcg_gen_shr_i64(t1, arg1, t1);
1860 tcg_gen_or_i64(ret, t0, t1);
1861 tcg_temp_free_i64(t0);
1862 tcg_temp_free_i64(t1);
1863 }
1864 }
1865
1866 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1867 {
1868 tcg_debug_assert(arg2 < 64);
1869 /* some cases can be optimized here */
1870 if (arg2 == 0) {
1871 tcg_gen_mov_i64(ret, arg1);
1872 } else if (TCG_TARGET_HAS_rot_i64) {
1873 TCGv_i64 t0 = tcg_const_i64(arg2);
1874 tcg_gen_rotl_i64(ret, arg1, t0);
1875 tcg_temp_free_i64(t0);
1876 } else {
1877 TCGv_i64 t0, t1;
1878 t0 = tcg_temp_new_i64();
1879 t1 = tcg_temp_new_i64();
1880 tcg_gen_shli_i64(t0, arg1, arg2);
1881 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1882 tcg_gen_or_i64(ret, t0, t1);
1883 tcg_temp_free_i64(t0);
1884 tcg_temp_free_i64(t1);
1885 }
1886 }
1887
1888 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1889 {
1890 if (TCG_TARGET_HAS_rot_i64) {
1891 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1892 } else {
1893 TCGv_i64 t0, t1;
1894 t0 = tcg_temp_new_i64();
1895 t1 = tcg_temp_new_i64();
1896 tcg_gen_shr_i64(t0, arg1, arg2);
1897 tcg_gen_subfi_i64(t1, 64, arg2);
1898 tcg_gen_shl_i64(t1, arg1, t1);
1899 tcg_gen_or_i64(ret, t0, t1);
1900 tcg_temp_free_i64(t0);
1901 tcg_temp_free_i64(t1);
1902 }
1903 }
1904
1905 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1906 {
1907 tcg_debug_assert(arg2 < 64);
1908 /* some cases can be optimized here */
1909 if (arg2 == 0) {
1910 tcg_gen_mov_i64(ret, arg1);
1911 } else {
1912 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1913 }
1914 }
1915
1916 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1917 unsigned int ofs, unsigned int len)
1918 {
1919 uint64_t mask;
1920 TCGv_i64 t1;
1921
1922 tcg_debug_assert(ofs < 64);
1923 tcg_debug_assert(len > 0);
1924 tcg_debug_assert(len <= 64);
1925 tcg_debug_assert(ofs + len <= 64);
1926
1927 if (len == 64) {
1928 tcg_gen_mov_i64(ret, arg2);
1929 return;
1930 }
1931 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1932 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
1933 return;
1934 }
1935
1936 if (TCG_TARGET_REG_BITS == 32) {
1937 if (ofs >= 32) {
1938 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
1939 TCGV_LOW(arg2), ofs - 32, len);
1940 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1941 return;
1942 }
1943 if (ofs + len <= 32) {
1944 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
1945 TCGV_LOW(arg2), ofs, len);
1946 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1947 return;
1948 }
1949 }
1950
1951 mask = (1ull << len) - 1;
1952 t1 = tcg_temp_new_i64();
1953
1954 if (ofs + len < 64) {
1955 tcg_gen_andi_i64(t1, arg2, mask);
1956 tcg_gen_shli_i64(t1, t1, ofs);
1957 } else {
1958 tcg_gen_shli_i64(t1, arg2, ofs);
1959 }
1960 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
1961 tcg_gen_or_i64(ret, ret, t1);
1962
1963 tcg_temp_free_i64(t1);
1964 }
1965
1966 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
1967 unsigned int ofs, unsigned int len)
1968 {
1969 tcg_debug_assert(ofs < 64);
1970 tcg_debug_assert(len > 0);
1971 tcg_debug_assert(len <= 64);
1972 tcg_debug_assert(ofs + len <= 64);
1973
1974 if (ofs + len == 64) {
1975 tcg_gen_shli_i64(ret, arg, ofs);
1976 } else if (ofs == 0) {
1977 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
1978 } else if (TCG_TARGET_HAS_deposit_i64
1979 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1980 TCGv_i64 zero = tcg_const_i64(0);
1981 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
1982 tcg_temp_free_i64(zero);
1983 } else {
1984 if (TCG_TARGET_REG_BITS == 32) {
1985 if (ofs >= 32) {
1986 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
1987 ofs - 32, len);
1988 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1989 return;
1990 }
1991 if (ofs + len <= 32) {
1992 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
1993 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1994 return;
1995 }
1996 }
1997 /* To help two-operand hosts we prefer to zero-extend first,
1998 which allows ARG to stay live. */
1999 switch (len) {
2000 case 32:
2001 if (TCG_TARGET_HAS_ext32u_i64) {
2002 tcg_gen_ext32u_i64(ret, arg);
2003 tcg_gen_shli_i64(ret, ret, ofs);
2004 return;
2005 }
2006 break;
2007 case 16:
2008 if (TCG_TARGET_HAS_ext16u_i64) {
2009 tcg_gen_ext16u_i64(ret, arg);
2010 tcg_gen_shli_i64(ret, ret, ofs);
2011 return;
2012 }
2013 break;
2014 case 8:
2015 if (TCG_TARGET_HAS_ext8u_i64) {
2016 tcg_gen_ext8u_i64(ret, arg);
2017 tcg_gen_shli_i64(ret, ret, ofs);
2018 return;
2019 }
2020 break;
2021 }
2022 /* Otherwise prefer zero-extension over AND for code size. */
2023 switch (ofs + len) {
2024 case 32:
2025 if (TCG_TARGET_HAS_ext32u_i64) {
2026 tcg_gen_shli_i64(ret, arg, ofs);
2027 tcg_gen_ext32u_i64(ret, ret);
2028 return;
2029 }
2030 break;
2031 case 16:
2032 if (TCG_TARGET_HAS_ext16u_i64) {
2033 tcg_gen_shli_i64(ret, arg, ofs);
2034 tcg_gen_ext16u_i64(ret, ret);
2035 return;
2036 }
2037 break;
2038 case 8:
2039 if (TCG_TARGET_HAS_ext8u_i64) {
2040 tcg_gen_shli_i64(ret, arg, ofs);
2041 tcg_gen_ext8u_i64(ret, ret);
2042 return;
2043 }
2044 break;
2045 }
2046 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2047 tcg_gen_shli_i64(ret, ret, ofs);
2048 }
2049 }
2050
2051 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2052 unsigned int ofs, unsigned int len)
2053 {
2054 tcg_debug_assert(ofs < 64);
2055 tcg_debug_assert(len > 0);
2056 tcg_debug_assert(len <= 64);
2057 tcg_debug_assert(ofs + len <= 64);
2058
2059 /* Canonicalize certain special cases, even if extract is supported. */
2060 if (ofs + len == 64) {
2061 tcg_gen_shri_i64(ret, arg, 64 - len);
2062 return;
2063 }
2064 if (ofs == 0) {
2065 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2066 return;
2067 }
2068
2069 if (TCG_TARGET_REG_BITS == 32) {
2070 /* Look for a 32-bit extract within one of the two words. */
2071 if (ofs >= 32) {
2072 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2073 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2074 return;
2075 }
2076 if (ofs + len <= 32) {
2077 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2078 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2079 return;
2080 }
2081 /* The field is split across two words. One double-word
2082 shift is better than two double-word shifts. */
2083 goto do_shift_and;
2084 }
2085
2086 if (TCG_TARGET_HAS_extract_i64
2087 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2088 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2089 return;
2090 }
2091
2092 /* Assume that zero-extension, if available, is cheaper than a shift. */
2093 switch (ofs + len) {
2094 case 32:
2095 if (TCG_TARGET_HAS_ext32u_i64) {
2096 tcg_gen_ext32u_i64(ret, arg);
2097 tcg_gen_shri_i64(ret, ret, ofs);
2098 return;
2099 }
2100 break;
2101 case 16:
2102 if (TCG_TARGET_HAS_ext16u_i64) {
2103 tcg_gen_ext16u_i64(ret, arg);
2104 tcg_gen_shri_i64(ret, ret, ofs);
2105 return;
2106 }
2107 break;
2108 case 8:
2109 if (TCG_TARGET_HAS_ext8u_i64) {
2110 tcg_gen_ext8u_i64(ret, arg);
2111 tcg_gen_shri_i64(ret, ret, ofs);
2112 return;
2113 }
2114 break;
2115 }
2116
2117 /* ??? Ideally we'd know what values are available for immediate AND.
2118 Assume that 8 bits are available, plus the special cases of 16 and 32,
2119 so that we get ext8u, ext16u, and ext32u. */
2120 switch (len) {
2121 case 1 ... 8: case 16: case 32:
2122 do_shift_and:
2123 tcg_gen_shri_i64(ret, arg, ofs);
2124 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2125 break;
2126 default:
2127 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2128 tcg_gen_shri_i64(ret, ret, 64 - len);
2129 break;
2130 }
2131 }
2132
2133 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2134 unsigned int ofs, unsigned int len)
2135 {
2136 tcg_debug_assert(ofs < 64);
2137 tcg_debug_assert(len > 0);
2138 tcg_debug_assert(len <= 64);
2139 tcg_debug_assert(ofs + len <= 64);
2140
2141 /* Canonicalize certain special cases, even if sextract is supported. */
2142 if (ofs + len == 64) {
2143 tcg_gen_sari_i64(ret, arg, 64 - len);
2144 return;
2145 }
2146 if (ofs == 0) {
2147 switch (len) {
2148 case 32:
2149 tcg_gen_ext32s_i64(ret, arg);
2150 return;
2151 case 16:
2152 tcg_gen_ext16s_i64(ret, arg);
2153 return;
2154 case 8:
2155 tcg_gen_ext8s_i64(ret, arg);
2156 return;
2157 }
2158 }
2159
2160 if (TCG_TARGET_REG_BITS == 32) {
2161 /* Look for a 32-bit extract within one of the two words. */
2162 if (ofs >= 32) {
2163 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2164 } else if (ofs + len <= 32) {
2165 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2166 } else if (ofs == 0) {
2167 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2168 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2169 return;
2170 } else if (len > 32) {
2171 TCGv_i32 t = tcg_temp_new_i32();
2172 /* Extract the bits for the high word normally. */
2173 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2174 /* Shift the field down for the low part. */
2175 tcg_gen_shri_i64(ret, arg, ofs);
2176 /* Overwrite the shift into the high part. */
2177 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2178 tcg_temp_free_i32(t);
2179 return;
2180 } else {
2181 /* Shift the field down for the low part, such that the
2182 field sits at the MSB. */
2183 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2184 /* Shift the field down from the MSB, sign extending. */
2185 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2186 }
2187 /* Sign-extend the field from 32 bits. */
2188 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2189 return;
2190 }
2191
2192 if (TCG_TARGET_HAS_sextract_i64
2193 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2194 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2195 return;
2196 }
2197
2198 /* Assume that sign-extension, if available, is cheaper than a shift. */
2199 switch (ofs + len) {
2200 case 32:
2201 if (TCG_TARGET_HAS_ext32s_i64) {
2202 tcg_gen_ext32s_i64(ret, arg);
2203 tcg_gen_sari_i64(ret, ret, ofs);
2204 return;
2205 }
2206 break;
2207 case 16:
2208 if (TCG_TARGET_HAS_ext16s_i64) {
2209 tcg_gen_ext16s_i64(ret, arg);
2210 tcg_gen_sari_i64(ret, ret, ofs);
2211 return;
2212 }
2213 break;
2214 case 8:
2215 if (TCG_TARGET_HAS_ext8s_i64) {
2216 tcg_gen_ext8s_i64(ret, arg);
2217 tcg_gen_sari_i64(ret, ret, ofs);
2218 return;
2219 }
2220 break;
2221 }
2222 switch (len) {
2223 case 32:
2224 if (TCG_TARGET_HAS_ext32s_i64) {
2225 tcg_gen_shri_i64(ret, arg, ofs);
2226 tcg_gen_ext32s_i64(ret, ret);
2227 return;
2228 }
2229 break;
2230 case 16:
2231 if (TCG_TARGET_HAS_ext16s_i64) {
2232 tcg_gen_shri_i64(ret, arg, ofs);
2233 tcg_gen_ext16s_i64(ret, ret);
2234 return;
2235 }
2236 break;
2237 case 8:
2238 if (TCG_TARGET_HAS_ext8s_i64) {
2239 tcg_gen_shri_i64(ret, arg, ofs);
2240 tcg_gen_ext8s_i64(ret, ret);
2241 return;
2242 }
2243 break;
2244 }
2245 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2246 tcg_gen_sari_i64(ret, ret, 64 - len);
2247 }
2248
2249 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2250 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2251 {
2252 if (cond == TCG_COND_ALWAYS) {
2253 tcg_gen_mov_i64(ret, v1);
2254 } else if (cond == TCG_COND_NEVER) {
2255 tcg_gen_mov_i64(ret, v2);
2256 } else if (TCG_TARGET_REG_BITS == 32) {
2257 TCGv_i32 t0 = tcg_temp_new_i32();
2258 TCGv_i32 t1 = tcg_temp_new_i32();
2259 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2260 TCGV_LOW(c1), TCGV_HIGH(c1),
2261 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2262
2263 if (TCG_TARGET_HAS_movcond_i32) {
2264 tcg_gen_movi_i32(t1, 0);
2265 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2266 TCGV_LOW(v1), TCGV_LOW(v2));
2267 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2268 TCGV_HIGH(v1), TCGV_HIGH(v2));
2269 } else {
2270 tcg_gen_neg_i32(t0, t0);
2271
2272 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2273 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2274 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2275
2276 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2277 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2278 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2279 }
2280 tcg_temp_free_i32(t0);
2281 tcg_temp_free_i32(t1);
2282 } else if (TCG_TARGET_HAS_movcond_i64) {
2283 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2284 } else {
2285 TCGv_i64 t0 = tcg_temp_new_i64();
2286 TCGv_i64 t1 = tcg_temp_new_i64();
2287 tcg_gen_setcond_i64(cond, t0, c1, c2);
2288 tcg_gen_neg_i64(t0, t0);
2289 tcg_gen_and_i64(t1, v1, t0);
2290 tcg_gen_andc_i64(ret, v2, t0);
2291 tcg_gen_or_i64(ret, ret, t1);
2292 tcg_temp_free_i64(t0);
2293 tcg_temp_free_i64(t1);
2294 }
2295 }
2296
2297 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2298 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2299 {
2300 if (TCG_TARGET_HAS_add2_i64) {
2301 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2302 } else {
2303 TCGv_i64 t0 = tcg_temp_new_i64();
2304 TCGv_i64 t1 = tcg_temp_new_i64();
2305 tcg_gen_add_i64(t0, al, bl);
2306 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2307 tcg_gen_add_i64(rh, ah, bh);
2308 tcg_gen_add_i64(rh, rh, t1);
2309 tcg_gen_mov_i64(rl, t0);
2310 tcg_temp_free_i64(t0);
2311 tcg_temp_free_i64(t1);
2312 }
2313 }
2314
2315 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2316 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2317 {
2318 if (TCG_TARGET_HAS_sub2_i64) {
2319 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2320 } else {
2321 TCGv_i64 t0 = tcg_temp_new_i64();
2322 TCGv_i64 t1 = tcg_temp_new_i64();
2323 tcg_gen_sub_i64(t0, al, bl);
2324 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2325 tcg_gen_sub_i64(rh, ah, bh);
2326 tcg_gen_sub_i64(rh, rh, t1);
2327 tcg_gen_mov_i64(rl, t0);
2328 tcg_temp_free_i64(t0);
2329 tcg_temp_free_i64(t1);
2330 }
2331 }
2332
2333 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2334 {
2335 if (TCG_TARGET_HAS_mulu2_i64) {
2336 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2337 } else if (TCG_TARGET_HAS_muluh_i64) {
2338 TCGv_i64 t = tcg_temp_new_i64();
2339 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2340 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2341 tcg_gen_mov_i64(rl, t);
2342 tcg_temp_free_i64(t);
2343 } else {
2344 TCGv_i64 t0 = tcg_temp_new_i64();
2345 tcg_gen_mul_i64(t0, arg1, arg2);
2346 gen_helper_muluh_i64(rh, arg1, arg2);
2347 tcg_gen_mov_i64(rl, t0);
2348 tcg_temp_free_i64(t0);
2349 }
2350 }
2351
2352 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2353 {
2354 if (TCG_TARGET_HAS_muls2_i64) {
2355 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2356 } else if (TCG_TARGET_HAS_mulsh_i64) {
2357 TCGv_i64 t = tcg_temp_new_i64();
2358 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2359 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2360 tcg_gen_mov_i64(rl, t);
2361 tcg_temp_free_i64(t);
2362 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2363 TCGv_i64 t0 = tcg_temp_new_i64();
2364 TCGv_i64 t1 = tcg_temp_new_i64();
2365 TCGv_i64 t2 = tcg_temp_new_i64();
2366 TCGv_i64 t3 = tcg_temp_new_i64();
2367 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2368 /* Adjust for negative inputs. */
2369 tcg_gen_sari_i64(t2, arg1, 63);
2370 tcg_gen_sari_i64(t3, arg2, 63);
2371 tcg_gen_and_i64(t2, t2, arg2);
2372 tcg_gen_and_i64(t3, t3, arg1);
2373 tcg_gen_sub_i64(rh, t1, t2);
2374 tcg_gen_sub_i64(rh, rh, t3);
2375 tcg_gen_mov_i64(rl, t0);
2376 tcg_temp_free_i64(t0);
2377 tcg_temp_free_i64(t1);
2378 tcg_temp_free_i64(t2);
2379 tcg_temp_free_i64(t3);
2380 } else {
2381 TCGv_i64 t0 = tcg_temp_new_i64();
2382 tcg_gen_mul_i64(t0, arg1, arg2);
2383 gen_helper_mulsh_i64(rh, arg1, arg2);
2384 tcg_gen_mov_i64(rl, t0);
2385 tcg_temp_free_i64(t0);
2386 }
2387 }
2388
2389 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2390 {
2391 TCGv_i64 t0 = tcg_temp_new_i64();
2392 TCGv_i64 t1 = tcg_temp_new_i64();
2393 TCGv_i64 t2 = tcg_temp_new_i64();
2394 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2395 /* Adjust for negative input for the signed arg1. */
2396 tcg_gen_sari_i64(t2, arg1, 63);
2397 tcg_gen_and_i64(t2, t2, arg2);
2398 tcg_gen_sub_i64(rh, t1, t2);
2399 tcg_gen_mov_i64(rl, t0);
2400 tcg_temp_free_i64(t0);
2401 tcg_temp_free_i64(t1);
2402 tcg_temp_free_i64(t2);
2403 }
2404
2405 /* Size changing operations. */
2406
2407 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2408 {
2409 if (TCG_TARGET_REG_BITS == 32) {
2410 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2411 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2412 tcg_gen_op2(&tcg_ctx, INDEX_op_extrl_i64_i32,
2413 GET_TCGV_I32(ret), GET_TCGV_I64(arg));
2414 } else {
2415 tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(arg)));
2416 }
2417 }
2418
2419 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2420 {
2421 if (TCG_TARGET_REG_BITS == 32) {
2422 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2423 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2424 tcg_gen_op2(&tcg_ctx, INDEX_op_extrh_i64_i32,
2425 GET_TCGV_I32(ret), GET_TCGV_I64(arg));
2426 } else {
2427 TCGv_i64 t = tcg_temp_new_i64();
2428 tcg_gen_shri_i64(t, arg, 32);
2429 tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(t)));
2430 tcg_temp_free_i64(t);
2431 }
2432 }
2433
2434 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2435 {
2436 if (TCG_TARGET_REG_BITS == 32) {
2437 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2438 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2439 } else {
2440 tcg_gen_op2(&tcg_ctx, INDEX_op_extu_i32_i64,
2441 GET_TCGV_I64(ret), GET_TCGV_I32(arg));
2442 }
2443 }
2444
2445 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2446 {
2447 if (TCG_TARGET_REG_BITS == 32) {
2448 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2449 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2450 } else {
2451 tcg_gen_op2(&tcg_ctx, INDEX_op_ext_i32_i64,
2452 GET_TCGV_I64(ret), GET_TCGV_I32(arg));
2453 }
2454 }
2455
2456 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2457 {
2458 TCGv_i64 tmp;
2459
2460 if (TCG_TARGET_REG_BITS == 32) {
2461 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2462 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2463 return;
2464 }
2465
2466 tmp = tcg_temp_new_i64();
2467 /* These extensions are only needed for type correctness.
2468 We may be able to do better given target specific information. */
2469 tcg_gen_extu_i32_i64(tmp, high);
2470 tcg_gen_extu_i32_i64(dest, low);
2471 /* If deposit is available, use it. Otherwise use the extra
2472 knowledge that we have of the zero-extensions above. */
2473 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2474 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2475 } else {
2476 tcg_gen_shli_i64(tmp, tmp, 32);
2477 tcg_gen_or_i64(dest, dest, tmp);
2478 }
2479 tcg_temp_free_i64(tmp);
2480 }
2481
2482 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2483 {
2484 if (TCG_TARGET_REG_BITS == 32) {
2485 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2486 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2487 } else {
2488 tcg_gen_extrl_i64_i32(lo, arg);
2489 tcg_gen_extrh_i64_i32(hi, arg);
2490 }
2491 }
2492
2493 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2494 {
2495 tcg_gen_ext32u_i64(lo, arg);
2496 tcg_gen_shri_i64(hi, arg, 32);
2497 }
2498
2499 /* QEMU specific operations. */
2500
2501 void tcg_gen_goto_tb(unsigned idx)
2502 {
2503 /* We only support two chained exits. */
2504 tcg_debug_assert(idx <= 1);
2505 #ifdef CONFIG_DEBUG_TCG
2506 /* Verify that we havn't seen this numbered exit before. */
2507 tcg_debug_assert((tcg_ctx.goto_tb_issue_mask & (1 << idx)) == 0);
2508 tcg_ctx.goto_tb_issue_mask |= 1 << idx;
2509 #endif
2510 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2511 }
2512
2513 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2514 {
2515 /* Trigger the asserts within as early as possible. */
2516 (void)get_alignment_bits(op);
2517
2518 switch (op & MO_SIZE) {
2519 case MO_8:
2520 op &= ~MO_BSWAP;
2521 break;
2522 case MO_16:
2523 break;
2524 case MO_32:
2525 if (!is64) {
2526 op &= ~MO_SIGN;
2527 }
2528 break;
2529 case MO_64:
2530 if (!is64) {
2531 tcg_abort();
2532 }
2533 break;
2534 }
2535 if (st) {
2536 op &= ~MO_SIGN;
2537 }
2538 return op;
2539 }
2540
2541 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2542 TCGMemOp memop, TCGArg idx)
2543 {
2544 TCGMemOpIdx oi = make_memop_idx(memop, idx);
2545 #if TARGET_LONG_BITS == 32
2546 tcg_gen_op3i_i32(opc, val, addr, oi);
2547 #else
2548 if (TCG_TARGET_REG_BITS == 32) {
2549 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2550 } else {
2551 tcg_gen_op3(&tcg_ctx, opc, GET_TCGV_I32(val), GET_TCGV_I64(addr), oi);
2552 }
2553 #endif
2554 }
2555
2556 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2557 TCGMemOp memop, TCGArg idx)
2558 {
2559 TCGMemOpIdx oi = make_memop_idx(memop, idx);
2560 #if TARGET_LONG_BITS == 32
2561 if (TCG_TARGET_REG_BITS == 32) {
2562 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2563 } else {
2564 tcg_gen_op3(&tcg_ctx, opc, GET_TCGV_I64(val), GET_TCGV_I32(addr), oi);
2565 }
2566 #else
2567 if (TCG_TARGET_REG_BITS == 32) {
2568 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2569 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2570 } else {
2571 tcg_gen_op3i_i64(opc, val, addr, oi);
2572 }
2573 #endif
2574 }
2575
2576 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2577 {
2578 memop = tcg_canonicalize_memop(memop, 0, 0);
2579 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
2580 addr, trace_mem_get_info(memop, 0));
2581 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2582 }
2583
2584 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2585 {
2586 memop = tcg_canonicalize_memop(memop, 0, 1);
2587 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
2588 addr, trace_mem_get_info(memop, 1));
2589 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2590 }
2591
2592 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2593 {
2594 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2595 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2596 if (memop & MO_SIGN) {
2597 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2598 } else {
2599 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2600 }
2601 return;
2602 }
2603
2604 memop = tcg_canonicalize_memop(memop, 1, 0);
2605 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
2606 addr, trace_mem_get_info(memop, 0));
2607 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2608 }
2609
2610 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2611 {
2612 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2613 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2614 return;
2615 }
2616
2617 memop = tcg_canonicalize_memop(memop, 1, 1);
2618 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
2619 addr, trace_mem_get_info(memop, 1));
2620 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
2621 }
2622
2623 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2624 {
2625 switch (opc & MO_SSIZE) {
2626 case MO_SB:
2627 tcg_gen_ext8s_i32(ret, val);
2628 break;
2629 case MO_UB:
2630 tcg_gen_ext8u_i32(ret, val);
2631 break;
2632 case MO_SW:
2633 tcg_gen_ext16s_i32(ret, val);
2634 break;
2635 case MO_UW:
2636 tcg_gen_ext16u_i32(ret, val);
2637 break;
2638 default:
2639 tcg_gen_mov_i32(ret, val);
2640 break;
2641 }
2642 }
2643
2644 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2645 {
2646 switch (opc & MO_SSIZE) {
2647 case MO_SB:
2648 tcg_gen_ext8s_i64(ret, val);
2649 break;
2650 case MO_UB:
2651 tcg_gen_ext8u_i64(ret, val);
2652 break;
2653 case MO_SW:
2654 tcg_gen_ext16s_i64(ret, val);
2655 break;
2656 case MO_UW:
2657 tcg_gen_ext16u_i64(ret, val);
2658 break;
2659 case MO_SL:
2660 tcg_gen_ext32s_i64(ret, val);
2661 break;
2662 case MO_UL:
2663 tcg_gen_ext32u_i64(ret, val);
2664 break;
2665 default:
2666 tcg_gen_mov_i64(ret, val);
2667 break;
2668 }
2669 }
2670
2671 #ifdef CONFIG_SOFTMMU
2672 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2673 TCGv_i32, TCGv_i32, TCGv_i32);
2674 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2675 TCGv_i64, TCGv_i64, TCGv_i32);
2676 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2677 TCGv_i32, TCGv_i32);
2678 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2679 TCGv_i64, TCGv_i32);
2680 #else
2681 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2682 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2683 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2684 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2685 #endif
2686
2687 #ifdef CONFIG_ATOMIC64
2688 # define WITH_ATOMIC64(X) X,
2689 #else
2690 # define WITH_ATOMIC64(X)
2691 #endif
2692
2693 static void * const table_cmpxchg[16] = {
2694 [MO_8] = gen_helper_atomic_cmpxchgb,
2695 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2696 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2697 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2698 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
2699 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2700 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
2701 };
2702
2703 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2704 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2705 {
2706 memop = tcg_canonicalize_memop(memop, 0, 0);
2707
2708 if (!parallel_cpus) {
2709 TCGv_i32 t1 = tcg_temp_new_i32();
2710 TCGv_i32 t2 = tcg_temp_new_i32();
2711
2712 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2713
2714 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2715 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2716 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2717 tcg_temp_free_i32(t2);
2718
2719 if (memop & MO_SIGN) {
2720 tcg_gen_ext_i32(retv, t1, memop);
2721 } else {
2722 tcg_gen_mov_i32(retv, t1);
2723 }
2724 tcg_temp_free_i32(t1);
2725 } else {
2726 gen_atomic_cx_i32 gen;
2727
2728 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2729 tcg_debug_assert(gen != NULL);
2730
2731 #ifdef CONFIG_SOFTMMU
2732 {
2733 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2734 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi);
2735 tcg_temp_free_i32(oi);
2736 }
2737 #else
2738 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv);
2739 #endif
2740
2741 if (memop & MO_SIGN) {
2742 tcg_gen_ext_i32(retv, retv, memop);
2743 }
2744 }
2745 }
2746
2747 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2748 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2749 {
2750 memop = tcg_canonicalize_memop(memop, 1, 0);
2751
2752 if (!parallel_cpus) {
2753 TCGv_i64 t1 = tcg_temp_new_i64();
2754 TCGv_i64 t2 = tcg_temp_new_i64();
2755
2756 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2757
2758 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2759 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2760 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2761 tcg_temp_free_i64(t2);
2762
2763 if (memop & MO_SIGN) {
2764 tcg_gen_ext_i64(retv, t1, memop);
2765 } else {
2766 tcg_gen_mov_i64(retv, t1);
2767 }
2768 tcg_temp_free_i64(t1);
2769 } else if ((memop & MO_SIZE) == MO_64) {
2770 #ifdef CONFIG_ATOMIC64
2771 gen_atomic_cx_i64 gen;
2772
2773 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2774 tcg_debug_assert(gen != NULL);
2775
2776 #ifdef CONFIG_SOFTMMU
2777 {
2778 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
2779 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi);
2780 tcg_temp_free_i32(oi);
2781 }
2782 #else
2783 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv);
2784 #endif
2785 #else
2786 gen_helper_exit_atomic(tcg_ctx.tcg_env);
2787 #endif /* CONFIG_ATOMIC64 */
2788 } else {
2789 TCGv_i32 c32 = tcg_temp_new_i32();
2790 TCGv_i32 n32 = tcg_temp_new_i32();
2791 TCGv_i32 r32 = tcg_temp_new_i32();
2792
2793 tcg_gen_extrl_i64_i32(c32, cmpv);
2794 tcg_gen_extrl_i64_i32(n32, newv);
2795 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
2796 tcg_temp_free_i32(c32);
2797 tcg_temp_free_i32(n32);
2798
2799 tcg_gen_extu_i32_i64(retv, r32);
2800 tcg_temp_free_i32(r32);
2801
2802 if (memop & MO_SIGN) {
2803 tcg_gen_ext_i64(retv, retv, memop);
2804 }
2805 }
2806 }
2807
2808 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2809 TCGArg idx, TCGMemOp memop, bool new_val,
2810 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
2811 {
2812 TCGv_i32 t1 = tcg_temp_new_i32();
2813 TCGv_i32 t2 = tcg_temp_new_i32();
2814
2815 memop = tcg_canonicalize_memop(memop, 0, 0);
2816
2817 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2818 gen(t2, t1, val);
2819 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2820
2821 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
2822 tcg_temp_free_i32(t1);
2823 tcg_temp_free_i32(t2);
2824 }
2825
2826 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2827 TCGArg idx, TCGMemOp memop, void * const table[])
2828 {
2829 gen_atomic_op_i32 gen;
2830
2831 memop = tcg_canonicalize_memop(memop, 0, 0);
2832
2833 gen = table[memop & (MO_SIZE | MO_BSWAP)];
2834 tcg_debug_assert(gen != NULL);
2835
2836 #ifdef CONFIG_SOFTMMU
2837 {
2838 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2839 gen(ret, tcg_ctx.tcg_env, addr, val, oi);
2840 tcg_temp_free_i32(oi);
2841 }
2842 #else
2843 gen(ret, tcg_ctx.tcg_env, addr, val);
2844 #endif
2845
2846 if (memop & MO_SIGN) {
2847 tcg_gen_ext_i32(ret, ret, memop);
2848 }
2849 }
2850
2851 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2852 TCGArg idx, TCGMemOp memop, bool new_val,
2853 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
2854 {
2855 TCGv_i64 t1 = tcg_temp_new_i64();
2856 TCGv_i64 t2 = tcg_temp_new_i64();
2857
2858 memop = tcg_canonicalize_memop(memop, 1, 0);
2859
2860 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2861 gen(t2, t1, val);
2862 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2863
2864 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
2865 tcg_temp_free_i64(t1);
2866 tcg_temp_free_i64(t2);
2867 }
2868
2869 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2870 TCGArg idx, TCGMemOp memop, void * const table[])
2871 {
2872 memop = tcg_canonicalize_memop(memop, 1, 0);
2873
2874 if ((memop & MO_SIZE) == MO_64) {
2875 #ifdef CONFIG_ATOMIC64
2876 gen_atomic_op_i64 gen;
2877
2878 gen = table[memop & (MO_SIZE | MO_BSWAP)];
2879 tcg_debug_assert(gen != NULL);
2880
2881 #ifdef CONFIG_SOFTMMU
2882 {
2883 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2884 gen(ret, tcg_ctx.tcg_env, addr, val, oi);
2885 tcg_temp_free_i32(oi);
2886 }
2887 #else
2888 gen(ret, tcg_ctx.tcg_env, addr, val);
2889 #endif
2890 #else
2891 gen_helper_exit_atomic(tcg_ctx.tcg_env);
2892 #endif /* CONFIG_ATOMIC64 */
2893 } else {
2894 TCGv_i32 v32 = tcg_temp_new_i32();
2895 TCGv_i32 r32 = tcg_temp_new_i32();
2896
2897 tcg_gen_extrl_i64_i32(v32, val);
2898 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
2899 tcg_temp_free_i32(v32);
2900
2901 tcg_gen_extu_i32_i64(ret, r32);
2902 tcg_temp_free_i32(r32);
2903
2904 if (memop & MO_SIGN) {
2905 tcg_gen_ext_i64(ret, ret, memop);
2906 }
2907 }
2908 }
2909
2910 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
2911 static void * const table_##NAME[16] = { \
2912 [MO_8] = gen_helper_atomic_##NAME##b, \
2913 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
2914 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
2915 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
2916 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
2917 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
2918 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
2919 }; \
2920 void tcg_gen_atomic_##NAME##_i32 \
2921 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
2922 { \
2923 if (parallel_cpus) { \
2924 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
2925 } else { \
2926 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
2927 tcg_gen_##OP##_i32); \
2928 } \
2929 } \
2930 void tcg_gen_atomic_##NAME##_i64 \
2931 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
2932 { \
2933 if (parallel_cpus) { \
2934 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
2935 } else { \
2936 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
2937 tcg_gen_##OP##_i64); \
2938 } \
2939 }
2940
2941 GEN_ATOMIC_HELPER(fetch_add, add, 0)
2942 GEN_ATOMIC_HELPER(fetch_and, and, 0)
2943 GEN_ATOMIC_HELPER(fetch_or, or, 0)
2944 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
2945
2946 GEN_ATOMIC_HELPER(add_fetch, add, 1)
2947 GEN_ATOMIC_HELPER(and_fetch, and, 1)
2948 GEN_ATOMIC_HELPER(or_fetch, or, 1)
2949 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
2950
2951 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
2952 {
2953 tcg_gen_mov_i32(r, b);
2954 }
2955
2956 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
2957 {
2958 tcg_gen_mov_i64(r, b);
2959 }
2960
2961 GEN_ATOMIC_HELPER(xchg, mov2, 0)
2962
2963 #undef GEN_ATOMIC_HELPER
QEMU mirror