2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/address-spaces.h"
26 #include "exec/cpu_ldst.h"
27 #include "exec/cputlb.h"
28 #include "exec/memory-internal.h"
29 #include "exec/ram_addr.h"
31 #include "qemu/error-report.h"
33 #include "exec/helper-proto.h"
34 #include "qemu/atomic.h"
36 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
37 /* #define DEBUG_TLB */
38 /* #define DEBUG_TLB_LOG */
41 # define DEBUG_TLB_GATE 1
43 # define DEBUG_TLB_LOG_GATE 1
45 # define DEBUG_TLB_LOG_GATE 0
48 # define DEBUG_TLB_GATE 0
49 # define DEBUG_TLB_LOG_GATE 0
52 #define tlb_debug(fmt, ...) do { \
53 if (DEBUG_TLB_LOG_GATE) { \
54 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
56 } else if (DEBUG_TLB_GATE) { \
57 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
61 #define assert_cpu_is_self(this_cpu) do { \
62 if (DEBUG_TLB_GATE) { \
63 g_assert(!cpu->created || qemu_cpu_is_self(cpu)); \
67 /* run_on_cpu_data.target_ptr should always be big enough for a
68 * target_ulong even on 32 bit builds */
69 QEMU_BUILD_BUG_ON(sizeof(target_ulong
) > sizeof(run_on_cpu_data
));
71 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
73 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> 16);
74 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
79 /* This is OK because CPU architectures generally permit an
80 * implementation to drop entries from the TLB at any time, so
81 * flushing more entries than required is only an efficiency issue,
82 * not a correctness issue.
84 static void tlb_flush_nocheck(CPUState
*cpu
)
86 CPUArchState
*env
= cpu
->env_ptr
;
88 /* The QOM tests will trigger tlb_flushes without setting up TCG
89 * so we bug out here in that case.
95 assert_cpu_is_self(cpu
);
96 tlb_debug("(count: %d)\n", tlb_flush_count
++);
100 memset(env
->tlb_table
, -1, sizeof(env
->tlb_table
));
101 memset(env
->tlb_v_table
, -1, sizeof(env
->tlb_v_table
));
102 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
105 env
->tlb_flush_addr
= -1;
106 env
->tlb_flush_mask
= 0;
110 atomic_mb_set(&cpu
->pending_tlb_flush
, 0);
113 static void tlb_flush_global_async_work(CPUState
*cpu
, run_on_cpu_data data
)
115 tlb_flush_nocheck(cpu
);
118 void tlb_flush(CPUState
*cpu
)
120 if (cpu
->created
&& !qemu_cpu_is_self(cpu
)) {
121 if (atomic_mb_read(&cpu
->pending_tlb_flush
) != ALL_MMUIDX_BITS
) {
122 atomic_mb_set(&cpu
->pending_tlb_flush
, ALL_MMUIDX_BITS
);
123 async_run_on_cpu(cpu
, tlb_flush_global_async_work
,
127 tlb_flush_nocheck(cpu
);
131 static void tlb_flush_by_mmuidx_async_work(CPUState
*cpu
, run_on_cpu_data data
)
133 CPUArchState
*env
= cpu
->env_ptr
;
134 unsigned long mmu_idx_bitmask
= data
.host_int
;
137 assert_cpu_is_self(cpu
);
141 tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask
);
143 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
145 if (test_bit(mmu_idx
, &mmu_idx_bitmask
)) {
146 tlb_debug("%d\n", mmu_idx
);
148 memset(env
->tlb_table
[mmu_idx
], -1, sizeof(env
->tlb_table
[0]));
149 memset(env
->tlb_v_table
[mmu_idx
], -1, sizeof(env
->tlb_v_table
[0]));
153 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
160 void tlb_flush_by_mmuidx(CPUState
*cpu
, uint16_t idxmap
)
162 tlb_debug("mmu_idx: 0x%" PRIx16
"\n", idxmap
);
164 if (!qemu_cpu_is_self(cpu
)) {
165 uint16_t pending_flushes
= idxmap
;
166 pending_flushes
&= ~atomic_mb_read(&cpu
->pending_tlb_flush
);
168 if (pending_flushes
) {
169 tlb_debug("reduced mmu_idx: 0x%" PRIx16
"\n", pending_flushes
);
171 atomic_or(&cpu
->pending_tlb_flush
, pending_flushes
);
172 async_run_on_cpu(cpu
, tlb_flush_by_mmuidx_async_work
,
173 RUN_ON_CPU_HOST_INT(pending_flushes
));
176 tlb_flush_by_mmuidx_async_work(cpu
,
177 RUN_ON_CPU_HOST_INT(idxmap
));
181 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
183 if (addr
== (tlb_entry
->addr_read
&
184 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
185 addr
== (tlb_entry
->addr_write
&
186 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
187 addr
== (tlb_entry
->addr_code
&
188 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
189 memset(tlb_entry
, -1, sizeof(*tlb_entry
));
193 static void tlb_flush_page_async_work(CPUState
*cpu
, run_on_cpu_data data
)
195 CPUArchState
*env
= cpu
->env_ptr
;
196 target_ulong addr
= (target_ulong
) data
.target_ptr
;
200 assert_cpu_is_self(cpu
);
202 tlb_debug("page :" TARGET_FMT_lx
"\n", addr
);
204 /* Check if we need to flush due to large pages. */
205 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
206 tlb_debug("forcing full flush ("
207 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
208 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
214 addr
&= TARGET_PAGE_MASK
;
215 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
216 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
217 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
220 /* check whether there are entries that need to be flushed in the vtlb */
221 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
223 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
224 tlb_flush_entry(&env
->tlb_v_table
[mmu_idx
][k
], addr
);
228 tb_flush_jmp_cache(cpu
, addr
);
231 void tlb_flush_page(CPUState
*cpu
, target_ulong addr
)
233 tlb_debug("page :" TARGET_FMT_lx
"\n", addr
);
235 if (!qemu_cpu_is_self(cpu
)) {
236 async_run_on_cpu(cpu
, tlb_flush_page_async_work
,
237 RUN_ON_CPU_TARGET_PTR(addr
));
239 tlb_flush_page_async_work(cpu
, RUN_ON_CPU_TARGET_PTR(addr
));
243 /* As we are going to hijack the bottom bits of the page address for a
244 * mmuidx bit mask we need to fail to build if we can't do that
246 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> TARGET_PAGE_BITS_MIN
);
248 static void tlb_flush_page_by_mmuidx_async_work(CPUState
*cpu
,
249 run_on_cpu_data data
)
251 CPUArchState
*env
= cpu
->env_ptr
;
252 target_ulong addr_and_mmuidx
= (target_ulong
) data
.target_ptr
;
253 target_ulong addr
= addr_and_mmuidx
& TARGET_PAGE_MASK
;
254 unsigned long mmu_idx_bitmap
= addr_and_mmuidx
& ALL_MMUIDX_BITS
;
255 int page
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
259 assert_cpu_is_self(cpu
);
261 tlb_debug("page:%d addr:"TARGET_FMT_lx
" mmu_idx:0x%lx\n",
262 page
, addr
, mmu_idx_bitmap
);
264 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
265 if (test_bit(mmu_idx
, &mmu_idx_bitmap
)) {
266 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][page
], addr
);
268 /* check whether there are vltb entries that need to be flushed */
269 for (i
= 0; i
< CPU_VTLB_SIZE
; i
++) {
270 tlb_flush_entry(&env
->tlb_v_table
[mmu_idx
][i
], addr
);
275 tb_flush_jmp_cache(cpu
, addr
);
278 static void tlb_check_page_and_flush_by_mmuidx_async_work(CPUState
*cpu
,
279 run_on_cpu_data data
)
281 CPUArchState
*env
= cpu
->env_ptr
;
282 target_ulong addr_and_mmuidx
= (target_ulong
) data
.target_ptr
;
283 target_ulong addr
= addr_and_mmuidx
& TARGET_PAGE_MASK
;
284 unsigned long mmu_idx_bitmap
= addr_and_mmuidx
& ALL_MMUIDX_BITS
;
286 tlb_debug("addr:"TARGET_FMT_lx
" mmu_idx: %04lx\n", addr
, mmu_idx_bitmap
);
288 /* Check if we need to flush due to large pages. */
289 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
290 tlb_debug("forced full flush ("
291 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
292 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
294 tlb_flush_by_mmuidx_async_work(cpu
,
295 RUN_ON_CPU_HOST_INT(mmu_idx_bitmap
));
297 tlb_flush_page_by_mmuidx_async_work(cpu
, data
);
301 void tlb_flush_page_by_mmuidx(CPUState
*cpu
, target_ulong addr
, uint16_t idxmap
)
303 target_ulong addr_and_mmu_idx
;
305 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%" PRIx16
"\n", addr
, idxmap
);
307 /* This should already be page aligned */
308 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
309 addr_and_mmu_idx
|= idxmap
;
311 if (!qemu_cpu_is_self(cpu
)) {
312 async_run_on_cpu(cpu
, tlb_check_page_and_flush_by_mmuidx_async_work
,
313 RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
315 tlb_check_page_and_flush_by_mmuidx_async_work(
316 cpu
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
320 void tlb_flush_page_all(target_ulong addr
)
325 async_run_on_cpu(cpu
, tlb_flush_page_async_work
,
326 RUN_ON_CPU_TARGET_PTR(addr
));
330 /* update the TLBs so that writes to code in the virtual page 'addr'
332 void tlb_protect_code(ram_addr_t ram_addr
)
334 cpu_physical_memory_test_and_clear_dirty(ram_addr
, TARGET_PAGE_SIZE
,
338 /* update the TLB so that writes in physical page 'phys_addr' are no longer
339 tested for self modifying code */
340 void tlb_unprotect_code(ram_addr_t ram_addr
)
342 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
);
347 * Dirty write flag handling
349 * When the TCG code writes to a location it looks up the address in
350 * the TLB and uses that data to compute the final address. If any of
351 * the lower bits of the address are set then the slow path is forced.
352 * There are a number of reasons to do this but for normal RAM the
353 * most usual is detecting writes to code regions which may invalidate
356 * Because we want other vCPUs to respond to changes straight away we
357 * update the te->addr_write field atomically. If the TLB entry has
358 * been changed by the vCPU in the mean time we skip the update.
360 * As this function uses atomic accesses we also need to ensure
361 * updates to tlb_entries follow the same access rules. We don't need
362 * to worry about this for oversized guests as MTTCG is disabled for
366 static void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
, uintptr_t start
,
369 #if TCG_OVERSIZED_GUEST
370 uintptr_t addr
= tlb_entry
->addr_write
;
372 if ((addr
& (TLB_INVALID_MASK
| TLB_MMIO
| TLB_NOTDIRTY
)) == 0) {
373 addr
&= TARGET_PAGE_MASK
;
374 addr
+= tlb_entry
->addend
;
375 if ((addr
- start
) < length
) {
376 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
380 /* paired with atomic_mb_set in tlb_set_page_with_attrs */
381 uintptr_t orig_addr
= atomic_mb_read(&tlb_entry
->addr_write
);
382 uintptr_t addr
= orig_addr
;
384 if ((addr
& (TLB_INVALID_MASK
| TLB_MMIO
| TLB_NOTDIRTY
)) == 0) {
385 addr
&= TARGET_PAGE_MASK
;
386 addr
+= atomic_read(&tlb_entry
->addend
);
387 if ((addr
- start
) < length
) {
388 uintptr_t notdirty_addr
= orig_addr
| TLB_NOTDIRTY
;
389 atomic_cmpxchg(&tlb_entry
->addr_write
, orig_addr
, notdirty_addr
);
395 /* For atomic correctness when running MTTCG we need to use the right
396 * primitives when copying entries */
397 static inline void copy_tlb_helper(CPUTLBEntry
*d
, CPUTLBEntry
*s
,
400 #if TCG_OVERSIZED_GUEST
404 d
->addr_read
= s
->addr_read
;
405 d
->addr_code
= s
->addr_code
;
406 atomic_set(&d
->addend
, atomic_read(&s
->addend
));
407 /* Pairs with flag setting in tlb_reset_dirty_range */
408 atomic_mb_set(&d
->addr_write
, atomic_read(&s
->addr_write
));
410 d
->addr_read
= s
->addr_read
;
411 d
->addr_write
= atomic_read(&s
->addr_write
);
412 d
->addr_code
= s
->addr_code
;
413 d
->addend
= atomic_read(&s
->addend
);
418 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
419 * the target vCPU). As such care needs to be taken that we don't
420 * dangerously race with another vCPU update. The only thing actually
421 * updated is the target TLB entry ->addr_write flags.
423 void tlb_reset_dirty(CPUState
*cpu
, ram_addr_t start1
, ram_addr_t length
)
430 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
433 for (i
= 0; i
< CPU_TLB_SIZE
; i
++) {
434 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
438 for (i
= 0; i
< CPU_VTLB_SIZE
; i
++) {
439 tlb_reset_dirty_range(&env
->tlb_v_table
[mmu_idx
][i
],
445 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
447 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
)) {
448 tlb_entry
->addr_write
= vaddr
;
452 /* update the TLB corresponding to virtual page vaddr
453 so that it is no longer dirty */
454 void tlb_set_dirty(CPUState
*cpu
, target_ulong vaddr
)
456 CPUArchState
*env
= cpu
->env_ptr
;
460 assert_cpu_is_self(cpu
);
462 vaddr
&= TARGET_PAGE_MASK
;
463 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
464 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
465 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
468 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
470 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
471 tlb_set_dirty1(&env
->tlb_v_table
[mmu_idx
][k
], vaddr
);
476 /* Our TLB does not support large pages, so remember the area covered by
477 large pages and trigger a full TLB flush if these are invalidated. */
478 static void tlb_add_large_page(CPUArchState
*env
, target_ulong vaddr
,
481 target_ulong mask
= ~(size
- 1);
483 if (env
->tlb_flush_addr
== (target_ulong
)-1) {
484 env
->tlb_flush_addr
= vaddr
& mask
;
485 env
->tlb_flush_mask
= mask
;
488 /* Extend the existing region to include the new page.
489 This is a compromise between unnecessary flushes and the cost
490 of maintaining a full variable size TLB. */
491 mask
&= env
->tlb_flush_mask
;
492 while (((env
->tlb_flush_addr
^ vaddr
) & mask
) != 0) {
495 env
->tlb_flush_addr
&= mask
;
496 env
->tlb_flush_mask
= mask
;
499 /* Add a new TLB entry. At most one entry for a given virtual address
500 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
501 * supplied size is only used by tlb_flush_page.
503 * Called from TCG-generated code, which is under an RCU read-side
506 void tlb_set_page_with_attrs(CPUState
*cpu
, target_ulong vaddr
,
507 hwaddr paddr
, MemTxAttrs attrs
, int prot
,
508 int mmu_idx
, target_ulong size
)
510 CPUArchState
*env
= cpu
->env_ptr
;
511 MemoryRegionSection
*section
;
513 target_ulong address
;
514 target_ulong code_address
;
516 CPUTLBEntry
*te
, *tv
, tn
;
517 hwaddr iotlb
, xlat
, sz
;
518 unsigned vidx
= env
->vtlb_index
++ % CPU_VTLB_SIZE
;
519 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
521 assert_cpu_is_self(cpu
);
522 assert(size
>= TARGET_PAGE_SIZE
);
523 if (size
!= TARGET_PAGE_SIZE
) {
524 tlb_add_large_page(env
, vaddr
, size
);
528 section
= address_space_translate_for_iotlb(cpu
, asidx
, paddr
, &xlat
, &sz
);
529 assert(sz
>= TARGET_PAGE_SIZE
);
531 tlb_debug("vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
533 vaddr
, paddr
, prot
, mmu_idx
);
536 if (!memory_region_is_ram(section
->mr
) && !memory_region_is_romd(section
->mr
)) {
541 /* TLB_MMIO for rom/romd handled below */
542 addend
= (uintptr_t)memory_region_get_ram_ptr(section
->mr
) + xlat
;
545 code_address
= address
;
546 iotlb
= memory_region_section_get_iotlb(cpu
, section
, vaddr
, paddr
, xlat
,
549 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
550 te
= &env
->tlb_table
[mmu_idx
][index
];
551 /* do not discard the translation in te, evict it into a victim tlb */
552 tv
= &env
->tlb_v_table
[mmu_idx
][vidx
];
554 /* addr_write can race with tlb_reset_dirty_range */
555 copy_tlb_helper(tv
, te
, true);
557 env
->iotlb_v
[mmu_idx
][vidx
] = env
->iotlb
[mmu_idx
][index
];
560 env
->iotlb
[mmu_idx
][index
].addr
= iotlb
- vaddr
;
561 env
->iotlb
[mmu_idx
][index
].attrs
= attrs
;
563 /* Now calculate the new entry */
564 tn
.addend
= addend
- vaddr
;
565 if (prot
& PAGE_READ
) {
566 tn
.addr_read
= address
;
571 if (prot
& PAGE_EXEC
) {
572 tn
.addr_code
= code_address
;
578 if (prot
& PAGE_WRITE
) {
579 if ((memory_region_is_ram(section
->mr
) && section
->readonly
)
580 || memory_region_is_romd(section
->mr
)) {
581 /* Write access calls the I/O callback. */
582 tn
.addr_write
= address
| TLB_MMIO
;
583 } else if (memory_region_is_ram(section
->mr
)
584 && cpu_physical_memory_is_clean(
585 memory_region_get_ram_addr(section
->mr
) + xlat
)) {
586 tn
.addr_write
= address
| TLB_NOTDIRTY
;
588 tn
.addr_write
= address
;
592 /* Pairs with flag setting in tlb_reset_dirty_range */
593 copy_tlb_helper(te
, &tn
, true);
594 /* atomic_mb_set(&te->addr_write, write_address); */
597 /* Add a new TLB entry, but without specifying the memory
598 * transaction attributes to be used.
600 void tlb_set_page(CPUState
*cpu
, target_ulong vaddr
,
601 hwaddr paddr
, int prot
,
602 int mmu_idx
, target_ulong size
)
604 tlb_set_page_with_attrs(cpu
, vaddr
, paddr
, MEMTXATTRS_UNSPECIFIED
,
605 prot
, mmu_idx
, size
);
608 static void report_bad_exec(CPUState
*cpu
, target_ulong addr
)
610 /* Accidentally executing outside RAM or ROM is quite common for
611 * several user-error situations, so report it in a way that
612 * makes it clear that this isn't a QEMU bug and provide suggestions
613 * about what a user could do to fix things.
615 error_report("Trying to execute code outside RAM or ROM at 0x"
616 TARGET_FMT_lx
, addr
);
617 error_printf("This usually means one of the following happened:\n\n"
618 "(1) You told QEMU to execute a kernel for the wrong machine "
619 "type, and it crashed on startup (eg trying to run a "
620 "raspberry pi kernel on a versatilepb QEMU machine)\n"
621 "(2) You didn't give QEMU a kernel or BIOS filename at all, "
622 "and QEMU executed a ROM full of no-op instructions until "
623 "it fell off the end\n"
624 "(3) Your guest kernel has a bug and crashed by jumping "
625 "off into nowhere\n\n"
626 "This is almost always one of the first two, so check your "
627 "command line and that you are using the right type of kernel "
628 "for this machine.\n"
629 "If you think option (3) is likely then you can try debugging "
630 "your guest with the -d debug options; in particular "
631 "-d guest_errors will cause the log to include a dump of the "
632 "guest register state at this point.\n\n"
633 "Execution cannot continue; stopping here.\n\n");
635 /* Report also to the logs, with more detail including register dump */
636 qemu_log_mask(LOG_GUEST_ERROR
, "qemu: fatal: Trying to execute code "
637 "outside RAM or ROM at 0x" TARGET_FMT_lx
"\n", addr
);
638 log_cpu_state_mask(LOG_GUEST_ERROR
, cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
641 static inline ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
645 ram_addr
= qemu_ram_addr_from_host(ptr
);
646 if (ram_addr
== RAM_ADDR_INVALID
) {
647 error_report("Bad ram pointer %p", ptr
);
653 /* NOTE: this function can trigger an exception */
654 /* NOTE2: the returned address is not exactly the physical address: it
655 * is actually a ram_addr_t (in system mode; the user mode emulation
656 * version of this function returns a guest virtual address).
658 tb_page_addr_t
get_page_addr_code(CPUArchState
*env1
, target_ulong addr
)
660 int mmu_idx
, page_index
, pd
;
663 CPUState
*cpu
= ENV_GET_CPU(env1
);
664 CPUIOTLBEntry
*iotlbentry
;
666 page_index
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
667 mmu_idx
= cpu_mmu_index(env1
, true);
668 if (unlikely(env1
->tlb_table
[mmu_idx
][page_index
].addr_code
!=
669 (addr
& TARGET_PAGE_MASK
))) {
670 cpu_ldub_code(env1
, addr
);
672 iotlbentry
= &env1
->iotlb
[mmu_idx
][page_index
];
673 pd
= iotlbentry
->addr
& ~TARGET_PAGE_MASK
;
674 mr
= iotlb_to_region(cpu
, pd
, iotlbentry
->attrs
);
675 if (memory_region_is_unassigned(mr
)) {
676 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
678 if (cc
->do_unassigned_access
) {
679 cc
->do_unassigned_access(cpu
, addr
, false, true, 0, 4);
681 report_bad_exec(cpu
, addr
);
685 p
= (void *)((uintptr_t)addr
+ env1
->tlb_table
[mmu_idx
][page_index
].addend
);
686 return qemu_ram_addr_from_host_nofail(p
);
689 static uint64_t io_readx(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
690 target_ulong addr
, uintptr_t retaddr
, int size
)
692 CPUState
*cpu
= ENV_GET_CPU(env
);
693 hwaddr physaddr
= iotlbentry
->addr
;
694 MemoryRegion
*mr
= iotlb_to_region(cpu
, physaddr
, iotlbentry
->attrs
);
698 physaddr
= (physaddr
& TARGET_PAGE_MASK
) + addr
;
699 cpu
->mem_io_pc
= retaddr
;
700 if (mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !cpu
->can_do_io
) {
701 cpu_io_recompile(cpu
, retaddr
);
704 cpu
->mem_io_vaddr
= addr
;
706 if (mr
->global_locking
) {
707 qemu_mutex_lock_iothread();
710 memory_region_dispatch_read(mr
, physaddr
, &val
, size
, iotlbentry
->attrs
);
712 qemu_mutex_unlock_iothread();
718 static void io_writex(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
719 uint64_t val
, target_ulong addr
,
720 uintptr_t retaddr
, int size
)
722 CPUState
*cpu
= ENV_GET_CPU(env
);
723 hwaddr physaddr
= iotlbentry
->addr
;
724 MemoryRegion
*mr
= iotlb_to_region(cpu
, physaddr
, iotlbentry
->attrs
);
727 physaddr
= (physaddr
& TARGET_PAGE_MASK
) + addr
;
728 if (mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !cpu
->can_do_io
) {
729 cpu_io_recompile(cpu
, retaddr
);
731 cpu
->mem_io_vaddr
= addr
;
732 cpu
->mem_io_pc
= retaddr
;
734 if (mr
->global_locking
) {
735 qemu_mutex_lock_iothread();
738 memory_region_dispatch_write(mr
, physaddr
, val
, size
, iotlbentry
->attrs
);
740 qemu_mutex_unlock_iothread();
744 /* Return true if ADDR is present in the victim tlb, and has been copied
745 back to the main tlb. */
746 static bool victim_tlb_hit(CPUArchState
*env
, size_t mmu_idx
, size_t index
,
747 size_t elt_ofs
, target_ulong page
)
750 for (vidx
= 0; vidx
< CPU_VTLB_SIZE
; ++vidx
) {
751 CPUTLBEntry
*vtlb
= &env
->tlb_v_table
[mmu_idx
][vidx
];
752 target_ulong cmp
= *(target_ulong
*)((uintptr_t)vtlb
+ elt_ofs
);
755 /* Found entry in victim tlb, swap tlb and iotlb. */
756 CPUTLBEntry tmptlb
, *tlb
= &env
->tlb_table
[mmu_idx
][index
];
758 copy_tlb_helper(&tmptlb
, tlb
, false);
759 copy_tlb_helper(tlb
, vtlb
, true);
760 copy_tlb_helper(vtlb
, &tmptlb
, true);
762 CPUIOTLBEntry tmpio
, *io
= &env
->iotlb
[mmu_idx
][index
];
763 CPUIOTLBEntry
*vio
= &env
->iotlb_v
[mmu_idx
][vidx
];
764 tmpio
= *io
; *io
= *vio
; *vio
= tmpio
;
771 /* Macro to call the above, with local variables from the use context. */
772 #define VICTIM_TLB_HIT(TY, ADDR) \
773 victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
774 (ADDR) & TARGET_PAGE_MASK)
776 /* Probe for whether the specified guest write access is permitted.
777 * If it is not permitted then an exception will be taken in the same
778 * way as if this were a real write access (and we will not return).
779 * Otherwise the function will return, and there will be a valid
780 * entry in the TLB for this access.
782 void probe_write(CPUArchState
*env
, target_ulong addr
, int mmu_idx
,
785 int index
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
786 target_ulong tlb_addr
= env
->tlb_table
[mmu_idx
][index
].addr_write
;
788 if ((addr
& TARGET_PAGE_MASK
)
789 != (tlb_addr
& (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
790 /* TLB entry is for a different page */
791 if (!VICTIM_TLB_HIT(addr_write
, addr
)) {
792 tlb_fill(ENV_GET_CPU(env
), addr
, MMU_DATA_STORE
, mmu_idx
, retaddr
);
797 /* Probe for a read-modify-write atomic operation. Do not allow unaligned
798 * operations, or io operations to proceed. Return the host address. */
799 static void *atomic_mmu_lookup(CPUArchState
*env
, target_ulong addr
,
800 TCGMemOpIdx oi
, uintptr_t retaddr
)
802 size_t mmu_idx
= get_mmuidx(oi
);
803 size_t index
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
804 CPUTLBEntry
*tlbe
= &env
->tlb_table
[mmu_idx
][index
];
805 target_ulong tlb_addr
= tlbe
->addr_write
;
806 TCGMemOp mop
= get_memop(oi
);
807 int a_bits
= get_alignment_bits(mop
);
808 int s_bits
= mop
& MO_SIZE
;
810 /* Adjust the given return address. */
811 retaddr
-= GETPC_ADJ
;
813 /* Enforce guest required alignment. */
814 if (unlikely(a_bits
> 0 && (addr
& ((1 << a_bits
) - 1)))) {
815 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
816 cpu_unaligned_access(ENV_GET_CPU(env
), addr
, MMU_DATA_STORE
,
820 /* Enforce qemu required alignment. */
821 if (unlikely(addr
& ((1 << s_bits
) - 1))) {
822 /* We get here if guest alignment was not requested,
823 or was not enforced by cpu_unaligned_access above.
824 We might widen the access and emulate, but for now
825 mark an exception and exit the cpu loop. */
829 /* Check TLB entry and enforce page permissions. */
830 if ((addr
& TARGET_PAGE_MASK
)
831 != (tlb_addr
& (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
832 if (!VICTIM_TLB_HIT(addr_write
, addr
)) {
833 tlb_fill(ENV_GET_CPU(env
), addr
, MMU_DATA_STORE
, mmu_idx
, retaddr
);
835 tlb_addr
= tlbe
->addr_write
;
838 /* Notice an IO access, or a notdirty page. */
839 if (unlikely(tlb_addr
& ~TARGET_PAGE_MASK
)) {
840 /* There's really nothing that can be done to
841 support this apart from stop-the-world. */
845 /* Let the guest notice RMW on a write-only page. */
846 if (unlikely(tlbe
->addr_read
!= tlb_addr
)) {
847 tlb_fill(ENV_GET_CPU(env
), addr
, MMU_DATA_LOAD
, mmu_idx
, retaddr
);
848 /* Since we don't support reads and writes to different addresses,
849 and we do have the proper page loaded for write, this shouldn't
850 ever return. But just in case, handle via stop-the-world. */
854 return (void *)((uintptr_t)addr
+ tlbe
->addend
);
857 cpu_loop_exit_atomic(ENV_GET_CPU(env
), retaddr
);
860 #ifdef TARGET_WORDS_BIGENDIAN
861 # define TGT_BE(X) (X)
862 # define TGT_LE(X) BSWAP(X)
864 # define TGT_BE(X) BSWAP(X)
865 # define TGT_LE(X) (X)
868 #define MMUSUFFIX _mmu
871 #include "softmmu_template.h"
874 #include "softmmu_template.h"
877 #include "softmmu_template.h"
880 #include "softmmu_template.h"
882 /* First set of helpers allows passing in of OI and RETADDR. This makes
883 them callable from other helpers. */
885 #define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr
886 #define ATOMIC_NAME(X) \
887 HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
888 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr)
891 #include "atomic_template.h"
894 #include "atomic_template.h"
897 #include "atomic_template.h"
899 #ifdef CONFIG_ATOMIC64
901 #include "atomic_template.h"
904 #ifdef CONFIG_ATOMIC128
906 #include "atomic_template.h"
909 /* Second set of helpers are directly callable from TCG as helpers. */
913 #undef ATOMIC_MMU_LOOKUP
914 #define EXTRA_ARGS , TCGMemOpIdx oi
915 #define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
916 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC())
919 #include "atomic_template.h"
922 #include "atomic_template.h"
925 #include "atomic_template.h"
927 #ifdef CONFIG_ATOMIC64
929 #include "atomic_template.h"
932 /* Code access functions. */
935 #define MMUSUFFIX _cmmu
937 #define GETPC() ((uintptr_t)0)
938 #define SOFTMMU_CODE_ACCESS
941 #include "softmmu_template.h"
944 #include "softmmu_template.h"
947 #include "softmmu_template.h"
950 #include "softmmu_template.h"