2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/address-spaces.h"
26 #include "exec/cpu_ldst.h"
27 #include "exec/cputlb.h"
28 #include "exec/memory-internal.h"
29 #include "exec/ram_addr.h"
31 #include "qemu/error-report.h"
33 #include "exec/helper-proto.h"
34 #include "qemu/atomic.h"
35 #include "qemu/atomic128.h"
37 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
38 /* #define DEBUG_TLB */
39 /* #define DEBUG_TLB_LOG */
42 # define DEBUG_TLB_GATE 1
44 # define DEBUG_TLB_LOG_GATE 1
46 # define DEBUG_TLB_LOG_GATE 0
49 # define DEBUG_TLB_GATE 0
50 # define DEBUG_TLB_LOG_GATE 0
53 #define tlb_debug(fmt, ...) do { \
54 if (DEBUG_TLB_LOG_GATE) { \
55 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
57 } else if (DEBUG_TLB_GATE) { \
58 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
62 #define assert_cpu_is_self(cpu) do { \
63 if (DEBUG_TLB_GATE) { \
64 g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \
68 /* run_on_cpu_data.target_ptr should always be big enough for a
69 * target_ulong even on 32 bit builds */
70 QEMU_BUILD_BUG_ON(sizeof(target_ulong
) > sizeof(run_on_cpu_data
));
72 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
74 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> 16);
75 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
77 void tlb_init(CPUState
*cpu
)
79 CPUArchState
*env
= cpu
->env_ptr
;
81 qemu_spin_init(&env
->tlb_c
.lock
);
84 /* flush_all_helper: run fn across all cpus
86 * If the wait flag is set then the src cpu's helper will be queued as
87 * "safe" work and the loop exited creating a synchronisation point
88 * where all queued work will be finished before execution starts
91 static void flush_all_helper(CPUState
*src
, run_on_cpu_func fn
,
98 async_run_on_cpu(cpu
, fn
, d
);
103 void tlb_flush_counts(size_t *pfull
, size_t *ppart
, size_t *pelide
)
106 size_t full
= 0, part
= 0, elide
= 0;
109 CPUArchState
*env
= cpu
->env_ptr
;
111 full
+= atomic_read(&env
->tlb_c
.full_flush_count
);
112 part
+= atomic_read(&env
->tlb_c
.part_flush_count
);
113 elide
+= atomic_read(&env
->tlb_c
.elide_flush_count
);
120 static void tlb_flush_one_mmuidx_locked(CPUArchState
*env
, int mmu_idx
)
122 memset(env
->tlb_table
[mmu_idx
], -1, sizeof(env
->tlb_table
[0]));
123 memset(env
->tlb_v_table
[mmu_idx
], -1, sizeof(env
->tlb_v_table
[0]));
124 env
->tlb_d
[mmu_idx
].large_page_addr
= -1;
125 env
->tlb_d
[mmu_idx
].large_page_mask
= -1;
126 env
->tlb_d
[mmu_idx
].vindex
= 0;
129 static void tlb_flush_by_mmuidx_async_work(CPUState
*cpu
, run_on_cpu_data data
)
131 CPUArchState
*env
= cpu
->env_ptr
;
132 unsigned long mmu_idx_bitmask
= data
.host_int
;
135 assert_cpu_is_self(cpu
);
137 tlb_debug("mmu_idx:0x%04lx\n", mmu_idx_bitmask
);
139 qemu_spin_lock(&env
->tlb_c
.lock
);
140 env
->tlb_c
.pending_flush
&= ~mmu_idx_bitmask
;
142 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
143 if (test_bit(mmu_idx
, &mmu_idx_bitmask
)) {
144 tlb_flush_one_mmuidx_locked(env
, mmu_idx
);
147 qemu_spin_unlock(&env
->tlb_c
.lock
);
149 cpu_tb_jmp_cache_clear(cpu
);
151 if (mmu_idx_bitmask
== ALL_MMUIDX_BITS
) {
152 atomic_set(&env
->tlb_c
.full_flush_count
,
153 env
->tlb_c
.full_flush_count
+ 1);
155 atomic_set(&env
->tlb_c
.part_flush_count
,
156 env
->tlb_c
.part_flush_count
+ ctpop16(mmu_idx_bitmask
));
160 void tlb_flush_by_mmuidx(CPUState
*cpu
, uint16_t idxmap
)
162 tlb_debug("mmu_idx: 0x%" PRIx16
"\n", idxmap
);
164 if (cpu
->created
&& !qemu_cpu_is_self(cpu
)) {
165 CPUArchState
*env
= cpu
->env_ptr
;
166 uint16_t pending
, to_clean
;
168 qemu_spin_lock(&env
->tlb_c
.lock
);
169 pending
= env
->tlb_c
.pending_flush
;
170 to_clean
= idxmap
& ~pending
;
171 env
->tlb_c
.pending_flush
= pending
| idxmap
;
172 qemu_spin_unlock(&env
->tlb_c
.lock
);
175 tlb_debug("reduced mmu_idx: 0x%" PRIx16
"\n", to_clean
);
176 async_run_on_cpu(cpu
, tlb_flush_by_mmuidx_async_work
,
177 RUN_ON_CPU_HOST_INT(to_clean
));
180 tlb_flush_by_mmuidx_async_work(cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
184 void tlb_flush(CPUState
*cpu
)
186 tlb_flush_by_mmuidx(cpu
, ALL_MMUIDX_BITS
);
189 void tlb_flush_by_mmuidx_all_cpus(CPUState
*src_cpu
, uint16_t idxmap
)
191 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
193 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
195 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
196 fn(src_cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
199 void tlb_flush_all_cpus(CPUState
*src_cpu
)
201 tlb_flush_by_mmuidx_all_cpus(src_cpu
, ALL_MMUIDX_BITS
);
204 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
, uint16_t idxmap
)
206 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
208 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
210 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
211 async_safe_run_on_cpu(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
214 void tlb_flush_all_cpus_synced(CPUState
*src_cpu
)
216 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu
, ALL_MMUIDX_BITS
);
219 static inline bool tlb_hit_page_anyprot(CPUTLBEntry
*tlb_entry
,
222 return tlb_hit_page(tlb_entry
->addr_read
, page
) ||
223 tlb_hit_page(tlb_addr_write(tlb_entry
), page
) ||
224 tlb_hit_page(tlb_entry
->addr_code
, page
);
227 /* Called with tlb_c.lock held */
228 static inline void tlb_flush_entry_locked(CPUTLBEntry
*tlb_entry
,
231 if (tlb_hit_page_anyprot(tlb_entry
, page
)) {
232 memset(tlb_entry
, -1, sizeof(*tlb_entry
));
236 /* Called with tlb_c.lock held */
237 static inline void tlb_flush_vtlb_page_locked(CPUArchState
*env
, int mmu_idx
,
242 assert_cpu_is_self(ENV_GET_CPU(env
));
243 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
244 tlb_flush_entry_locked(&env
->tlb_v_table
[mmu_idx
][k
], page
);
248 static void tlb_flush_page_locked(CPUArchState
*env
, int midx
,
251 target_ulong lp_addr
= env
->tlb_d
[midx
].large_page_addr
;
252 target_ulong lp_mask
= env
->tlb_d
[midx
].large_page_mask
;
254 /* Check if we need to flush due to large pages. */
255 if ((page
& lp_mask
) == lp_addr
) {
256 tlb_debug("forcing full flush midx %d ("
257 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
258 midx
, lp_addr
, lp_mask
);
259 tlb_flush_one_mmuidx_locked(env
, midx
);
261 tlb_flush_entry_locked(tlb_entry(env
, midx
, page
), page
);
262 tlb_flush_vtlb_page_locked(env
, midx
, page
);
266 /* As we are going to hijack the bottom bits of the page address for a
267 * mmuidx bit mask we need to fail to build if we can't do that
269 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> TARGET_PAGE_BITS_MIN
);
271 static void tlb_flush_page_by_mmuidx_async_work(CPUState
*cpu
,
272 run_on_cpu_data data
)
274 CPUArchState
*env
= cpu
->env_ptr
;
275 target_ulong addr_and_mmuidx
= (target_ulong
) data
.target_ptr
;
276 target_ulong addr
= addr_and_mmuidx
& TARGET_PAGE_MASK
;
277 unsigned long mmu_idx_bitmap
= addr_and_mmuidx
& ALL_MMUIDX_BITS
;
280 assert_cpu_is_self(cpu
);
282 tlb_debug("page addr:" TARGET_FMT_lx
" mmu_map:0x%lx\n",
283 addr
, mmu_idx_bitmap
);
285 qemu_spin_lock(&env
->tlb_c
.lock
);
286 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
287 if (test_bit(mmu_idx
, &mmu_idx_bitmap
)) {
288 tlb_flush_page_locked(env
, mmu_idx
, addr
);
291 qemu_spin_unlock(&env
->tlb_c
.lock
);
293 tb_flush_jmp_cache(cpu
, addr
);
296 void tlb_flush_page_by_mmuidx(CPUState
*cpu
, target_ulong addr
, uint16_t idxmap
)
298 target_ulong addr_and_mmu_idx
;
300 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%" PRIx16
"\n", addr
, idxmap
);
302 /* This should already be page aligned */
303 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
304 addr_and_mmu_idx
|= idxmap
;
306 if (!qemu_cpu_is_self(cpu
)) {
307 async_run_on_cpu(cpu
, tlb_flush_page_by_mmuidx_async_work
,
308 RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
310 tlb_flush_page_by_mmuidx_async_work(
311 cpu
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
315 void tlb_flush_page(CPUState
*cpu
, target_ulong addr
)
317 tlb_flush_page_by_mmuidx(cpu
, addr
, ALL_MMUIDX_BITS
);
320 void tlb_flush_page_by_mmuidx_all_cpus(CPUState
*src_cpu
, target_ulong addr
,
323 const run_on_cpu_func fn
= tlb_flush_page_by_mmuidx_async_work
;
324 target_ulong addr_and_mmu_idx
;
326 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
328 /* This should already be page aligned */
329 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
330 addr_and_mmu_idx
|= idxmap
;
332 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
333 fn(src_cpu
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
336 void tlb_flush_page_all_cpus(CPUState
*src
, target_ulong addr
)
338 tlb_flush_page_by_mmuidx_all_cpus(src
, addr
, ALL_MMUIDX_BITS
);
341 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
345 const run_on_cpu_func fn
= tlb_flush_page_by_mmuidx_async_work
;
346 target_ulong addr_and_mmu_idx
;
348 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
350 /* This should already be page aligned */
351 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
352 addr_and_mmu_idx
|= idxmap
;
354 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
355 async_safe_run_on_cpu(src_cpu
, fn
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
358 void tlb_flush_page_all_cpus_synced(CPUState
*src
, target_ulong addr
)
360 tlb_flush_page_by_mmuidx_all_cpus_synced(src
, addr
, ALL_MMUIDX_BITS
);
363 /* update the TLBs so that writes to code in the virtual page 'addr'
365 void tlb_protect_code(ram_addr_t ram_addr
)
367 cpu_physical_memory_test_and_clear_dirty(ram_addr
, TARGET_PAGE_SIZE
,
371 /* update the TLB so that writes in physical page 'phys_addr' are no longer
372 tested for self modifying code */
373 void tlb_unprotect_code(ram_addr_t ram_addr
)
375 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
);
380 * Dirty write flag handling
382 * When the TCG code writes to a location it looks up the address in
383 * the TLB and uses that data to compute the final address. If any of
384 * the lower bits of the address are set then the slow path is forced.
385 * There are a number of reasons to do this but for normal RAM the
386 * most usual is detecting writes to code regions which may invalidate
389 * Other vCPUs might be reading their TLBs during guest execution, so we update
390 * te->addr_write with atomic_set. We don't need to worry about this for
391 * oversized guests as MTTCG is disabled for them.
393 * Called with tlb_c.lock held.
395 static void tlb_reset_dirty_range_locked(CPUTLBEntry
*tlb_entry
,
396 uintptr_t start
, uintptr_t length
)
398 uintptr_t addr
= tlb_entry
->addr_write
;
400 if ((addr
& (TLB_INVALID_MASK
| TLB_MMIO
| TLB_NOTDIRTY
)) == 0) {
401 addr
&= TARGET_PAGE_MASK
;
402 addr
+= tlb_entry
->addend
;
403 if ((addr
- start
) < length
) {
404 #if TCG_OVERSIZED_GUEST
405 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
407 atomic_set(&tlb_entry
->addr_write
,
408 tlb_entry
->addr_write
| TLB_NOTDIRTY
);
415 * Called with tlb_c.lock held.
416 * Called only from the vCPU context, i.e. the TLB's owner thread.
418 static inline void copy_tlb_helper_locked(CPUTLBEntry
*d
, const CPUTLBEntry
*s
)
423 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
425 * We must take tlb_c.lock to avoid racing with another vCPU update. The only
426 * thing actually updated is the target TLB entry ->addr_write flags.
428 void tlb_reset_dirty(CPUState
*cpu
, ram_addr_t start1
, ram_addr_t length
)
435 qemu_spin_lock(&env
->tlb_c
.lock
);
436 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
439 for (i
= 0; i
< CPU_TLB_SIZE
; i
++) {
440 tlb_reset_dirty_range_locked(&env
->tlb_table
[mmu_idx
][i
], start1
,
444 for (i
= 0; i
< CPU_VTLB_SIZE
; i
++) {
445 tlb_reset_dirty_range_locked(&env
->tlb_v_table
[mmu_idx
][i
], start1
,
449 qemu_spin_unlock(&env
->tlb_c
.lock
);
452 /* Called with tlb_c.lock held */
453 static inline void tlb_set_dirty1_locked(CPUTLBEntry
*tlb_entry
,
456 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
)) {
457 tlb_entry
->addr_write
= vaddr
;
461 /* update the TLB corresponding to virtual page vaddr
462 so that it is no longer dirty */
463 void tlb_set_dirty(CPUState
*cpu
, target_ulong vaddr
)
465 CPUArchState
*env
= cpu
->env_ptr
;
468 assert_cpu_is_self(cpu
);
470 vaddr
&= TARGET_PAGE_MASK
;
471 qemu_spin_lock(&env
->tlb_c
.lock
);
472 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
473 tlb_set_dirty1_locked(tlb_entry(env
, mmu_idx
, vaddr
), vaddr
);
476 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
478 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
479 tlb_set_dirty1_locked(&env
->tlb_v_table
[mmu_idx
][k
], vaddr
);
482 qemu_spin_unlock(&env
->tlb_c
.lock
);
485 /* Our TLB does not support large pages, so remember the area covered by
486 large pages and trigger a full TLB flush if these are invalidated. */
487 static void tlb_add_large_page(CPUArchState
*env
, int mmu_idx
,
488 target_ulong vaddr
, target_ulong size
)
490 target_ulong lp_addr
= env
->tlb_d
[mmu_idx
].large_page_addr
;
491 target_ulong lp_mask
= ~(size
- 1);
493 if (lp_addr
== (target_ulong
)-1) {
494 /* No previous large page. */
497 /* Extend the existing region to include the new page.
498 This is a compromise between unnecessary flushes and
499 the cost of maintaining a full variable size TLB. */
500 lp_mask
&= env
->tlb_d
[mmu_idx
].large_page_mask
;
501 while (((lp_addr
^ vaddr
) & lp_mask
) != 0) {
505 env
->tlb_d
[mmu_idx
].large_page_addr
= lp_addr
& lp_mask
;
506 env
->tlb_d
[mmu_idx
].large_page_mask
= lp_mask
;
509 /* Add a new TLB entry. At most one entry for a given virtual address
510 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
511 * supplied size is only used by tlb_flush_page.
513 * Called from TCG-generated code, which is under an RCU read-side
516 void tlb_set_page_with_attrs(CPUState
*cpu
, target_ulong vaddr
,
517 hwaddr paddr
, MemTxAttrs attrs
, int prot
,
518 int mmu_idx
, target_ulong size
)
520 CPUArchState
*env
= cpu
->env_ptr
;
521 MemoryRegionSection
*section
;
523 target_ulong address
;
524 target_ulong code_address
;
527 hwaddr iotlb
, xlat
, sz
, paddr_page
;
528 target_ulong vaddr_page
;
529 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
531 assert_cpu_is_self(cpu
);
533 if (size
<= TARGET_PAGE_SIZE
) {
534 sz
= TARGET_PAGE_SIZE
;
536 tlb_add_large_page(env
, mmu_idx
, vaddr
, size
);
539 vaddr_page
= vaddr
& TARGET_PAGE_MASK
;
540 paddr_page
= paddr
& TARGET_PAGE_MASK
;
542 section
= address_space_translate_for_iotlb(cpu
, asidx
, paddr_page
,
543 &xlat
, &sz
, attrs
, &prot
);
544 assert(sz
>= TARGET_PAGE_SIZE
);
546 tlb_debug("vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
548 vaddr
, paddr
, prot
, mmu_idx
);
550 address
= vaddr_page
;
551 if (size
< TARGET_PAGE_SIZE
) {
553 * Slow-path the TLB entries; we will repeat the MMU check and TLB
554 * fill on every access.
556 address
|= TLB_RECHECK
;
558 if (!memory_region_is_ram(section
->mr
) &&
559 !memory_region_is_romd(section
->mr
)) {
564 /* TLB_MMIO for rom/romd handled below */
565 addend
= (uintptr_t)memory_region_get_ram_ptr(section
->mr
) + xlat
;
568 code_address
= address
;
569 iotlb
= memory_region_section_get_iotlb(cpu
, section
, vaddr_page
,
570 paddr_page
, xlat
, prot
, &address
);
572 index
= tlb_index(env
, mmu_idx
, vaddr_page
);
573 te
= tlb_entry(env
, mmu_idx
, vaddr_page
);
576 * Hold the TLB lock for the rest of the function. We could acquire/release
577 * the lock several times in the function, but it is faster to amortize the
578 * acquisition cost by acquiring it just once. Note that this leads to
579 * a longer critical section, but this is not a concern since the TLB lock
580 * is unlikely to be contended.
582 qemu_spin_lock(&env
->tlb_c
.lock
);
584 /* Make sure there's no cached translation for the new page. */
585 tlb_flush_vtlb_page_locked(env
, mmu_idx
, vaddr_page
);
588 * Only evict the old entry to the victim tlb if it's for a
589 * different page; otherwise just overwrite the stale data.
591 if (!tlb_hit_page_anyprot(te
, vaddr_page
)) {
592 unsigned vidx
= env
->tlb_d
[mmu_idx
].vindex
++ % CPU_VTLB_SIZE
;
593 CPUTLBEntry
*tv
= &env
->tlb_v_table
[mmu_idx
][vidx
];
595 /* Evict the old entry into the victim tlb. */
596 copy_tlb_helper_locked(tv
, te
);
597 env
->iotlb_v
[mmu_idx
][vidx
] = env
->iotlb
[mmu_idx
][index
];
602 * At this point iotlb contains a physical section number in the lower
603 * TARGET_PAGE_BITS, and either
604 * + the ram_addr_t of the page base of the target RAM (if NOTDIRTY or ROM)
605 * + the offset within section->mr of the page base (otherwise)
606 * We subtract the vaddr_page (which is page aligned and thus won't
607 * disturb the low bits) to give an offset which can be added to the
608 * (non-page-aligned) vaddr of the eventual memory access to get
609 * the MemoryRegion offset for the access. Note that the vaddr we
610 * subtract here is that of the page base, and not the same as the
611 * vaddr we add back in io_readx()/io_writex()/get_page_addr_code().
613 env
->iotlb
[mmu_idx
][index
].addr
= iotlb
- vaddr_page
;
614 env
->iotlb
[mmu_idx
][index
].attrs
= attrs
;
616 /* Now calculate the new entry */
617 tn
.addend
= addend
- vaddr_page
;
618 if (prot
& PAGE_READ
) {
619 tn
.addr_read
= address
;
624 if (prot
& PAGE_EXEC
) {
625 tn
.addr_code
= code_address
;
631 if (prot
& PAGE_WRITE
) {
632 if ((memory_region_is_ram(section
->mr
) && section
->readonly
)
633 || memory_region_is_romd(section
->mr
)) {
634 /* Write access calls the I/O callback. */
635 tn
.addr_write
= address
| TLB_MMIO
;
636 } else if (memory_region_is_ram(section
->mr
)
637 && cpu_physical_memory_is_clean(
638 memory_region_get_ram_addr(section
->mr
) + xlat
)) {
639 tn
.addr_write
= address
| TLB_NOTDIRTY
;
641 tn
.addr_write
= address
;
643 if (prot
& PAGE_WRITE_INV
) {
644 tn
.addr_write
|= TLB_INVALID_MASK
;
648 copy_tlb_helper_locked(te
, &tn
);
649 qemu_spin_unlock(&env
->tlb_c
.lock
);
652 /* Add a new TLB entry, but without specifying the memory
653 * transaction attributes to be used.
655 void tlb_set_page(CPUState
*cpu
, target_ulong vaddr
,
656 hwaddr paddr
, int prot
,
657 int mmu_idx
, target_ulong size
)
659 tlb_set_page_with_attrs(cpu
, vaddr
, paddr
, MEMTXATTRS_UNSPECIFIED
,
660 prot
, mmu_idx
, size
);
663 static inline ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
667 ram_addr
= qemu_ram_addr_from_host(ptr
);
668 if (ram_addr
== RAM_ADDR_INVALID
) {
669 error_report("Bad ram pointer %p", ptr
);
675 static uint64_t io_readx(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
677 target_ulong addr
, uintptr_t retaddr
,
678 bool recheck
, MMUAccessType access_type
, int size
)
680 CPUState
*cpu
= ENV_GET_CPU(env
);
682 MemoryRegionSection
*section
;
690 * This is a TLB_RECHECK access, where the MMU protection
691 * covers a smaller range than a target page, and we must
692 * repeat the MMU check here. This tlb_fill() call might
693 * longjump out if this access should cause a guest exception.
696 target_ulong tlb_addr
;
698 tlb_fill(cpu
, addr
, size
, MMU_DATA_LOAD
, mmu_idx
, retaddr
);
700 entry
= tlb_entry(env
, mmu_idx
, addr
);
701 tlb_addr
= entry
->addr_read
;
702 if (!(tlb_addr
& ~(TARGET_PAGE_MASK
| TLB_RECHECK
))) {
704 uintptr_t haddr
= addr
+ entry
->addend
;
706 return ldn_p((void *)haddr
, size
);
708 /* Fall through for handling IO accesses */
711 section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
713 mr_offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
714 cpu
->mem_io_pc
= retaddr
;
715 if (mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !cpu
->can_do_io
) {
716 cpu_io_recompile(cpu
, retaddr
);
719 cpu
->mem_io_vaddr
= addr
;
720 cpu
->mem_io_access_type
= access_type
;
722 if (mr
->global_locking
&& !qemu_mutex_iothread_locked()) {
723 qemu_mutex_lock_iothread();
726 r
= memory_region_dispatch_read(mr
, mr_offset
,
727 &val
, size
, iotlbentry
->attrs
);
729 hwaddr physaddr
= mr_offset
+
730 section
->offset_within_address_space
-
731 section
->offset_within_region
;
733 cpu_transaction_failed(cpu
, physaddr
, addr
, size
, access_type
,
734 mmu_idx
, iotlbentry
->attrs
, r
, retaddr
);
737 qemu_mutex_unlock_iothread();
743 static void io_writex(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
745 uint64_t val
, target_ulong addr
,
746 uintptr_t retaddr
, bool recheck
, int size
)
748 CPUState
*cpu
= ENV_GET_CPU(env
);
750 MemoryRegionSection
*section
;
757 * This is a TLB_RECHECK access, where the MMU protection
758 * covers a smaller range than a target page, and we must
759 * repeat the MMU check here. This tlb_fill() call might
760 * longjump out if this access should cause a guest exception.
763 target_ulong tlb_addr
;
765 tlb_fill(cpu
, addr
, size
, MMU_DATA_STORE
, mmu_idx
, retaddr
);
767 entry
= tlb_entry(env
, mmu_idx
, addr
);
768 tlb_addr
= tlb_addr_write(entry
);
769 if (!(tlb_addr
& ~(TARGET_PAGE_MASK
| TLB_RECHECK
))) {
771 uintptr_t haddr
= addr
+ entry
->addend
;
773 stn_p((void *)haddr
, size
, val
);
776 /* Fall through for handling IO accesses */
779 section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
781 mr_offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
782 if (mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !cpu
->can_do_io
) {
783 cpu_io_recompile(cpu
, retaddr
);
785 cpu
->mem_io_vaddr
= addr
;
786 cpu
->mem_io_pc
= retaddr
;
788 if (mr
->global_locking
&& !qemu_mutex_iothread_locked()) {
789 qemu_mutex_lock_iothread();
792 r
= memory_region_dispatch_write(mr
, mr_offset
,
793 val
, size
, iotlbentry
->attrs
);
795 hwaddr physaddr
= mr_offset
+
796 section
->offset_within_address_space
-
797 section
->offset_within_region
;
799 cpu_transaction_failed(cpu
, physaddr
, addr
, size
, MMU_DATA_STORE
,
800 mmu_idx
, iotlbentry
->attrs
, r
, retaddr
);
803 qemu_mutex_unlock_iothread();
807 /* Return true if ADDR is present in the victim tlb, and has been copied
808 back to the main tlb. */
809 static bool victim_tlb_hit(CPUArchState
*env
, size_t mmu_idx
, size_t index
,
810 size_t elt_ofs
, target_ulong page
)
814 assert_cpu_is_self(ENV_GET_CPU(env
));
815 for (vidx
= 0; vidx
< CPU_VTLB_SIZE
; ++vidx
) {
816 CPUTLBEntry
*vtlb
= &env
->tlb_v_table
[mmu_idx
][vidx
];
819 /* elt_ofs might correspond to .addr_write, so use atomic_read */
820 #if TCG_OVERSIZED_GUEST
821 cmp
= *(target_ulong
*)((uintptr_t)vtlb
+ elt_ofs
);
823 cmp
= atomic_read((target_ulong
*)((uintptr_t)vtlb
+ elt_ofs
));
827 /* Found entry in victim tlb, swap tlb and iotlb. */
828 CPUTLBEntry tmptlb
, *tlb
= &env
->tlb_table
[mmu_idx
][index
];
830 qemu_spin_lock(&env
->tlb_c
.lock
);
831 copy_tlb_helper_locked(&tmptlb
, tlb
);
832 copy_tlb_helper_locked(tlb
, vtlb
);
833 copy_tlb_helper_locked(vtlb
, &tmptlb
);
834 qemu_spin_unlock(&env
->tlb_c
.lock
);
836 CPUIOTLBEntry tmpio
, *io
= &env
->iotlb
[mmu_idx
][index
];
837 CPUIOTLBEntry
*vio
= &env
->iotlb_v
[mmu_idx
][vidx
];
838 tmpio
= *io
; *io
= *vio
; *vio
= tmpio
;
845 /* Macro to call the above, with local variables from the use context. */
846 #define VICTIM_TLB_HIT(TY, ADDR) \
847 victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
848 (ADDR) & TARGET_PAGE_MASK)
850 /* NOTE: this function can trigger an exception */
851 /* NOTE2: the returned address is not exactly the physical address: it
852 * is actually a ram_addr_t (in system mode; the user mode emulation
853 * version of this function returns a guest virtual address).
855 tb_page_addr_t
get_page_addr_code(CPUArchState
*env
, target_ulong addr
)
857 uintptr_t mmu_idx
= cpu_mmu_index(env
, true);
858 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
859 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
862 if (unlikely(!tlb_hit(entry
->addr_code
, addr
))) {
863 if (!VICTIM_TLB_HIT(addr_code
, addr
)) {
864 tlb_fill(ENV_GET_CPU(env
), addr
, 0, MMU_INST_FETCH
, mmu_idx
, 0);
866 assert(tlb_hit(entry
->addr_code
, addr
));
869 if (unlikely(entry
->addr_code
& (TLB_RECHECK
| TLB_MMIO
))) {
871 * Return -1 if we can't translate and execute from an entire
872 * page of RAM here, which will cause us to execute by loading
873 * and translating one insn at a time, without caching:
874 * - TLB_RECHECK: means the MMU protection covers a smaller range
875 * than a target page, so we must redo the MMU check every insn
876 * - TLB_MMIO: region is not backed by RAM
881 p
= (void *)((uintptr_t)addr
+ entry
->addend
);
882 return qemu_ram_addr_from_host_nofail(p
);
885 /* Probe for whether the specified guest write access is permitted.
886 * If it is not permitted then an exception will be taken in the same
887 * way as if this were a real write access (and we will not return).
888 * Otherwise the function will return, and there will be a valid
889 * entry in the TLB for this access.
891 void probe_write(CPUArchState
*env
, target_ulong addr
, int size
, int mmu_idx
,
894 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
895 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
897 if (!tlb_hit(tlb_addr_write(entry
), addr
)) {
898 /* TLB entry is for a different page */
899 if (!VICTIM_TLB_HIT(addr_write
, addr
)) {
900 tlb_fill(ENV_GET_CPU(env
), addr
, size
, MMU_DATA_STORE
,
906 /* Probe for a read-modify-write atomic operation. Do not allow unaligned
907 * operations, or io operations to proceed. Return the host address. */
908 static void *atomic_mmu_lookup(CPUArchState
*env
, target_ulong addr
,
909 TCGMemOpIdx oi
, uintptr_t retaddr
,
912 size_t mmu_idx
= get_mmuidx(oi
);
913 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
914 CPUTLBEntry
*tlbe
= tlb_entry(env
, mmu_idx
, addr
);
915 target_ulong tlb_addr
= tlb_addr_write(tlbe
);
916 TCGMemOp mop
= get_memop(oi
);
917 int a_bits
= get_alignment_bits(mop
);
918 int s_bits
= mop
& MO_SIZE
;
921 /* Adjust the given return address. */
922 retaddr
-= GETPC_ADJ
;
924 /* Enforce guest required alignment. */
925 if (unlikely(a_bits
> 0 && (addr
& ((1 << a_bits
) - 1)))) {
926 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
927 cpu_unaligned_access(ENV_GET_CPU(env
), addr
, MMU_DATA_STORE
,
931 /* Enforce qemu required alignment. */
932 if (unlikely(addr
& ((1 << s_bits
) - 1))) {
933 /* We get here if guest alignment was not requested,
934 or was not enforced by cpu_unaligned_access above.
935 We might widen the access and emulate, but for now
936 mark an exception and exit the cpu loop. */
940 /* Check TLB entry and enforce page permissions. */
941 if (!tlb_hit(tlb_addr
, addr
)) {
942 if (!VICTIM_TLB_HIT(addr_write
, addr
)) {
943 tlb_fill(ENV_GET_CPU(env
), addr
, 1 << s_bits
, MMU_DATA_STORE
,
946 tlb_addr
= tlb_addr_write(tlbe
) & ~TLB_INVALID_MASK
;
949 /* Notice an IO access or a needs-MMU-lookup access */
950 if (unlikely(tlb_addr
& (TLB_MMIO
| TLB_RECHECK
))) {
951 /* There's really nothing that can be done to
952 support this apart from stop-the-world. */
956 /* Let the guest notice RMW on a write-only page. */
957 if (unlikely(tlbe
->addr_read
!= (tlb_addr
& ~TLB_NOTDIRTY
))) {
958 tlb_fill(ENV_GET_CPU(env
), addr
, 1 << s_bits
, MMU_DATA_LOAD
,
960 /* Since we don't support reads and writes to different addresses,
961 and we do have the proper page loaded for write, this shouldn't
962 ever return. But just in case, handle via stop-the-world. */
966 hostaddr
= (void *)((uintptr_t)addr
+ tlbe
->addend
);
969 if (unlikely(tlb_addr
& TLB_NOTDIRTY
)) {
971 memory_notdirty_write_prepare(ndi
, ENV_GET_CPU(env
), addr
,
972 qemu_ram_addr_from_host_nofail(hostaddr
),
979 cpu_loop_exit_atomic(ENV_GET_CPU(env
), retaddr
);
982 #ifdef TARGET_WORDS_BIGENDIAN
983 # define TGT_BE(X) (X)
984 # define TGT_LE(X) BSWAP(X)
986 # define TGT_BE(X) BSWAP(X)
987 # define TGT_LE(X) (X)
990 #define MMUSUFFIX _mmu
993 #include "softmmu_template.h"
996 #include "softmmu_template.h"
999 #include "softmmu_template.h"
1002 #include "softmmu_template.h"
1004 /* First set of helpers allows passing in of OI and RETADDR. This makes
1005 them callable from other helpers. */
1007 #define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr
1008 #define ATOMIC_NAME(X) \
1009 HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
1010 #define ATOMIC_MMU_DECLS NotDirtyInfo ndi
1011 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr, &ndi)
1012 #define ATOMIC_MMU_CLEANUP \
1014 if (unlikely(ndi.active)) { \
1015 memory_notdirty_write_complete(&ndi); \
1020 #include "atomic_template.h"
1023 #include "atomic_template.h"
1026 #include "atomic_template.h"
1028 #ifdef CONFIG_ATOMIC64
1030 #include "atomic_template.h"
1033 #if HAVE_CMPXCHG128 || HAVE_ATOMIC128
1034 #define DATA_SIZE 16
1035 #include "atomic_template.h"
1038 /* Second set of helpers are directly callable from TCG as helpers. */
1042 #undef ATOMIC_MMU_LOOKUP
1043 #define EXTRA_ARGS , TCGMemOpIdx oi
1044 #define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
1045 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC(), &ndi)
1048 #include "atomic_template.h"
1051 #include "atomic_template.h"
1054 #include "atomic_template.h"
1056 #ifdef CONFIG_ATOMIC64
1058 #include "atomic_template.h"
1061 /* Code access functions. */
1064 #define MMUSUFFIX _cmmu
1066 #define GETPC() ((uintptr_t)0)
1067 #define SOFTMMU_CODE_ACCESS
1070 #include "softmmu_template.h"
1073 #include "softmmu_template.h"
1076 #include "softmmu_template.h"
1079 #include "softmmu_template.h"