2 * User emulator execution
4 * Copyright (c) 2003-2005 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
20 #include "hw/core/tcg-cpu-ops.h"
21 #include "disas/disas.h"
22 #include "exec/exec-all.h"
24 #include "qemu/bitops.h"
25 #include "exec/cpu_ldst.h"
26 #include "exec/translate-all.h"
27 #include "exec/helper-proto.h"
28 #include "qemu/atomic128.h"
29 #include "trace/trace-root.h"
30 #include "tcg/tcg-ldst.h"
33 __thread
uintptr_t helper_retaddr
;
35 //#define DEBUG_SIGNAL
38 * Adjust the pc to pass to cpu_restore_state; return the memop type.
40 MMUAccessType
adjust_signal_pc(uintptr_t *pc
, bool is_write
)
42 switch (helper_retaddr
) {
45 * Fault during host memory operation within a helper function.
46 * The helper's host return address, saved here, gives us a
47 * pointer into the generated code that will unwind to the
55 * Fault during host memory operation within generated code.
56 * (Or, a unrelated bug within qemu, but we can't tell from here).
58 * We take the host pc from the signal frame. However, we cannot
59 * use that value directly. Within cpu_restore_state_from_tb, we
60 * assume PC comes from GETPC(), as used by the helper functions,
61 * so we adjust the address by -GETPC_ADJ to form an address that
62 * is within the call insn, so that the address does not accidentally
63 * match the beginning of the next guest insn. However, when the
64 * pc comes from the signal frame it points to the actual faulting
65 * host memory insn and not the return from a call insn.
67 * Therefore, adjust to compensate for what will be done later
68 * by cpu_restore_state_from_tb.
75 * Fault during host read for translation, or loosely, "execution".
77 * The guest pc is already pointing to the start of the TB for which
78 * code is being generated. If the guest translator manages the
79 * page crossings correctly, this is exactly the correct address
80 * (and if the translator doesn't handle page boundaries correctly
81 * there's little we can do about that here). Therefore, do not
82 * trigger the unwinder.
84 * Like tb_gen_code, release the memory lock before cpu_loop_exit.
88 return MMU_INST_FETCH
;
91 return is_write
? MMU_DATA_STORE
: MMU_DATA_LOAD
;
95 * handle_sigsegv_accerr_write:
96 * @cpu: the cpu context
97 * @old_set: the sigset_t from the signal ucontext_t
98 * @host_pc: the host pc, adjusted for the signal
99 * @guest_addr: the guest address of the fault
101 * Return true if the write fault has been handled, and should be re-tried.
103 * Note that it is important that we don't call page_unprotect() unless
104 * this is really a "write to nonwriteable page" fault, because
105 * page_unprotect() assumes that if it is called for an access to
106 * a page that's writeable this means we had two threads racing and
107 * another thread got there first and already made the page writeable;
108 * so we will retry the access. If we were to call page_unprotect()
109 * for some other kind of fault that should really be passed to the
110 * guest, we'd end up in an infinite loop of retrying the faulting access.
112 bool handle_sigsegv_accerr_write(CPUState
*cpu
, sigset_t
*old_set
,
113 uintptr_t host_pc
, abi_ptr guest_addr
)
115 switch (page_unprotect(guest_addr
, host_pc
)) {
118 * Fault not caused by a page marked unwritable to protect
119 * cached translations, must be the guest binary's problem.
124 * Fault caused by protection of cached translation; TBs
125 * invalidated, so resume execution.
130 * Fault caused by protection of cached translation, and the
131 * currently executing TB was modified and must be exited immediately.
133 sigprocmask(SIG_SETMASK
, old_set
, NULL
);
134 cpu_loop_exit_noexc(cpu
);
137 g_assert_not_reached();
141 static int probe_access_internal(CPUArchState
*env
, target_ulong addr
,
142 int fault_size
, MMUAccessType access_type
,
143 bool nonfault
, uintptr_t ra
)
148 switch (access_type
) {
150 acc_flag
= PAGE_WRITE_ORG
;
153 acc_flag
= PAGE_READ
;
156 acc_flag
= PAGE_EXEC
;
159 g_assert_not_reached();
162 if (guest_addr_valid_untagged(addr
)) {
163 int page_flags
= page_get_flags(addr
);
164 if (page_flags
& acc_flag
) {
165 return 0; /* success */
167 maperr
= !(page_flags
& PAGE_VALID
);
173 return TLB_INVALID_MASK
;
176 cpu_loop_exit_sigsegv(env_cpu(env
), addr
, access_type
, maperr
, ra
);
179 int probe_access_flags(CPUArchState
*env
, target_ulong addr
,
180 MMUAccessType access_type
, int mmu_idx
,
181 bool nonfault
, void **phost
, uintptr_t ra
)
185 flags
= probe_access_internal(env
, addr
, 0, access_type
, nonfault
, ra
);
186 *phost
= flags
? NULL
: g2h(env_cpu(env
), addr
);
190 void *probe_access(CPUArchState
*env
, target_ulong addr
, int size
,
191 MMUAccessType access_type
, int mmu_idx
, uintptr_t ra
)
195 g_assert(-(addr
| TARGET_PAGE_MASK
) >= size
);
196 flags
= probe_access_internal(env
, addr
, size
, access_type
, false, ra
);
197 g_assert(flags
== 0);
199 return size
? g2h(env_cpu(env
), addr
) : NULL
;
202 /* The softmmu versions of these helpers are in cputlb.c. */
205 * Verify that we have passed the correct MemOp to the correct function.
207 * We could present one function to target code, and dispatch based on
208 * the MemOp, but so far we have worked hard to avoid an indirect function
209 * call along the memory path.
211 static void validate_memop(MemOpIdx oi
, MemOp expected
)
213 #ifdef CONFIG_DEBUG_TCG
214 MemOp have
= get_memop(oi
) & (MO_SIZE
| MO_BSWAP
);
215 assert(have
== expected
);
219 void helper_unaligned_ld(CPUArchState
*env
, target_ulong addr
)
221 cpu_loop_exit_sigbus(env_cpu(env
), addr
, MMU_DATA_LOAD
, GETPC());
224 void helper_unaligned_st(CPUArchState
*env
, target_ulong addr
)
226 cpu_loop_exit_sigbus(env_cpu(env
), addr
, MMU_DATA_STORE
, GETPC());
229 static void *cpu_mmu_lookup(CPUArchState
*env
, target_ulong addr
,
230 MemOpIdx oi
, uintptr_t ra
, MMUAccessType type
)
232 MemOp mop
= get_memop(oi
);
233 int a_bits
= get_alignment_bits(mop
);
236 /* Enforce guest required alignment. */
237 if (unlikely(addr
& ((1 << a_bits
) - 1))) {
238 cpu_loop_exit_sigbus(env_cpu(env
), addr
, type
, ra
);
241 ret
= g2h(env_cpu(env
), addr
);
242 set_helper_retaddr(ra
);
246 uint8_t cpu_ldb_mmu(CPUArchState
*env
, abi_ptr addr
,
247 MemOpIdx oi
, uintptr_t ra
)
252 validate_memop(oi
, MO_UB
);
253 trace_guest_ld_before_exec(env_cpu(env
), addr
, oi
);
254 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
256 clear_helper_retaddr();
257 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_R
);
261 uint16_t cpu_ldw_be_mmu(CPUArchState
*env
, abi_ptr addr
,
262 MemOpIdx oi
, uintptr_t ra
)
267 validate_memop(oi
, MO_BEUW
);
268 trace_guest_ld_before_exec(env_cpu(env
), addr
, oi
);
269 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
270 ret
= lduw_be_p(haddr
);
271 clear_helper_retaddr();
272 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_R
);
276 uint32_t cpu_ldl_be_mmu(CPUArchState
*env
, abi_ptr addr
,
277 MemOpIdx oi
, uintptr_t ra
)
282 validate_memop(oi
, MO_BEUL
);
283 trace_guest_ld_before_exec(env_cpu(env
), addr
, oi
);
284 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
285 ret
= ldl_be_p(haddr
);
286 clear_helper_retaddr();
287 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_R
);
291 uint64_t cpu_ldq_be_mmu(CPUArchState
*env
, abi_ptr addr
,
292 MemOpIdx oi
, uintptr_t ra
)
297 validate_memop(oi
, MO_BEUQ
);
298 trace_guest_ld_before_exec(env_cpu(env
), addr
, oi
);
299 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
300 ret
= ldq_be_p(haddr
);
301 clear_helper_retaddr();
302 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_R
);
306 uint16_t cpu_ldw_le_mmu(CPUArchState
*env
, abi_ptr addr
,
307 MemOpIdx oi
, uintptr_t ra
)
312 validate_memop(oi
, MO_LEUW
);
313 trace_guest_ld_before_exec(env_cpu(env
), addr
, oi
);
314 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
315 ret
= lduw_le_p(haddr
);
316 clear_helper_retaddr();
317 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_R
);
321 uint32_t cpu_ldl_le_mmu(CPUArchState
*env
, abi_ptr addr
,
322 MemOpIdx oi
, uintptr_t ra
)
327 validate_memop(oi
, MO_LEUL
);
328 trace_guest_ld_before_exec(env_cpu(env
), addr
, oi
);
329 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
330 ret
= ldl_le_p(haddr
);
331 clear_helper_retaddr();
332 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_R
);
336 uint64_t cpu_ldq_le_mmu(CPUArchState
*env
, abi_ptr addr
,
337 MemOpIdx oi
, uintptr_t ra
)
342 validate_memop(oi
, MO_LEUQ
);
343 trace_guest_ld_before_exec(env_cpu(env
), addr
, oi
);
344 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
345 ret
= ldq_le_p(haddr
);
346 clear_helper_retaddr();
347 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_R
);
351 void cpu_stb_mmu(CPUArchState
*env
, abi_ptr addr
, uint8_t val
,
352 MemOpIdx oi
, uintptr_t ra
)
356 validate_memop(oi
, MO_UB
);
357 trace_guest_st_before_exec(env_cpu(env
), addr
, oi
);
358 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
);
360 clear_helper_retaddr();
361 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_W
);
364 void cpu_stw_be_mmu(CPUArchState
*env
, abi_ptr addr
, uint16_t val
,
365 MemOpIdx oi
, uintptr_t ra
)
369 validate_memop(oi
, MO_BEUW
);
370 trace_guest_st_before_exec(env_cpu(env
), addr
, oi
);
371 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
);
372 stw_be_p(haddr
, val
);
373 clear_helper_retaddr();
374 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_W
);
377 void cpu_stl_be_mmu(CPUArchState
*env
, abi_ptr addr
, uint32_t val
,
378 MemOpIdx oi
, uintptr_t ra
)
382 validate_memop(oi
, MO_BEUL
);
383 trace_guest_st_before_exec(env_cpu(env
), addr
, oi
);
384 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
);
385 stl_be_p(haddr
, val
);
386 clear_helper_retaddr();
387 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_W
);
390 void cpu_stq_be_mmu(CPUArchState
*env
, abi_ptr addr
, uint64_t val
,
391 MemOpIdx oi
, uintptr_t ra
)
395 validate_memop(oi
, MO_BEUQ
);
396 trace_guest_st_before_exec(env_cpu(env
), addr
, oi
);
397 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
);
398 stq_be_p(haddr
, val
);
399 clear_helper_retaddr();
400 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_W
);
403 void cpu_stw_le_mmu(CPUArchState
*env
, abi_ptr addr
, uint16_t val
,
404 MemOpIdx oi
, uintptr_t ra
)
408 validate_memop(oi
, MO_LEUW
);
409 trace_guest_st_before_exec(env_cpu(env
), addr
, oi
);
410 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
);
411 stw_le_p(haddr
, val
);
412 clear_helper_retaddr();
413 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_W
);
416 void cpu_stl_le_mmu(CPUArchState
*env
, abi_ptr addr
, uint32_t val
,
417 MemOpIdx oi
, uintptr_t ra
)
421 validate_memop(oi
, MO_LEUL
);
422 trace_guest_st_before_exec(env_cpu(env
), addr
, oi
);
423 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
);
424 stl_le_p(haddr
, val
);
425 clear_helper_retaddr();
426 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_W
);
429 void cpu_stq_le_mmu(CPUArchState
*env
, abi_ptr addr
, uint64_t val
,
430 MemOpIdx oi
, uintptr_t ra
)
434 validate_memop(oi
, MO_LEUQ
);
435 trace_guest_st_before_exec(env_cpu(env
), addr
, oi
);
436 haddr
= cpu_mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
);
437 stq_le_p(haddr
, val
);
438 clear_helper_retaddr();
439 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_W
);
442 uint32_t cpu_ldub_code(CPUArchState
*env
, abi_ptr ptr
)
446 set_helper_retaddr(1);
447 ret
= ldub_p(g2h_untagged(ptr
));
448 clear_helper_retaddr();
452 uint32_t cpu_lduw_code(CPUArchState
*env
, abi_ptr ptr
)
456 set_helper_retaddr(1);
457 ret
= lduw_p(g2h_untagged(ptr
));
458 clear_helper_retaddr();
462 uint32_t cpu_ldl_code(CPUArchState
*env
, abi_ptr ptr
)
466 set_helper_retaddr(1);
467 ret
= ldl_p(g2h_untagged(ptr
));
468 clear_helper_retaddr();
472 uint64_t cpu_ldq_code(CPUArchState
*env
, abi_ptr ptr
)
476 set_helper_retaddr(1);
477 ret
= ldq_p(g2h_untagged(ptr
));
478 clear_helper_retaddr();
482 #include "ldst_common.c.inc"
485 * Do not allow unaligned operations to proceed. Return the host address.
487 * @prot may be PAGE_READ, PAGE_WRITE, or PAGE_READ|PAGE_WRITE.
489 static void *atomic_mmu_lookup(CPUArchState
*env
, target_ulong addr
,
490 MemOpIdx oi
, int size
, int prot
,
493 MemOp mop
= get_memop(oi
);
494 int a_bits
= get_alignment_bits(mop
);
497 /* Enforce guest required alignment. */
498 if (unlikely(addr
& ((1 << a_bits
) - 1))) {
499 MMUAccessType t
= prot
== PAGE_READ
? MMU_DATA_LOAD
: MMU_DATA_STORE
;
500 cpu_loop_exit_sigbus(env_cpu(env
), addr
, t
, retaddr
);
503 /* Enforce qemu required alignment. */
504 if (unlikely(addr
& (size
- 1))) {
505 cpu_loop_exit_atomic(env_cpu(env
), retaddr
);
508 ret
= g2h(env_cpu(env
), addr
);
509 set_helper_retaddr(retaddr
);
513 #include "atomic_common.c.inc"
516 * First set of functions passes in OI and RETADDR.
517 * This makes them callable from other helpers.
520 #define ATOMIC_NAME(X) \
521 glue(glue(glue(cpu_atomic_ ## X, SUFFIX), END), _mmu)
522 #define ATOMIC_MMU_CLEANUP do { clear_helper_retaddr(); } while (0)
523 #define ATOMIC_MMU_IDX MMU_USER_IDX
526 #include "atomic_template.h"
529 #include "atomic_template.h"
532 #include "atomic_template.h"
534 #ifdef CONFIG_ATOMIC64
536 #include "atomic_template.h"
539 #if HAVE_ATOMIC128 || HAVE_CMPXCHG128
541 #include "atomic_template.h"