2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/address-spaces.h"
26 #include "exec/cpu_ldst.h"
27 #include "exec/cputlb.h"
28 #include "exec/memory-internal.h"
29 #include "exec/ram_addr.h"
31 #include "qemu/error-report.h"
33 #include "exec/helper-proto.h"
34 #include "qemu/atomic.h"
35 #include "qemu/atomic128.h"
37 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
38 /* #define DEBUG_TLB */
39 /* #define DEBUG_TLB_LOG */
42 # define DEBUG_TLB_GATE 1
44 # define DEBUG_TLB_LOG_GATE 1
46 # define DEBUG_TLB_LOG_GATE 0
49 # define DEBUG_TLB_GATE 0
50 # define DEBUG_TLB_LOG_GATE 0
53 #define tlb_debug(fmt, ...) do { \
54 if (DEBUG_TLB_LOG_GATE) { \
55 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
57 } else if (DEBUG_TLB_GATE) { \
58 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
62 #define assert_cpu_is_self(cpu) do { \
63 if (DEBUG_TLB_GATE) { \
64 g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \
68 /* run_on_cpu_data.target_ptr should always be big enough for a
69 * target_ulong even on 32 bit builds */
70 QEMU_BUILD_BUG_ON(sizeof(target_ulong
) > sizeof(run_on_cpu_data
));
72 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
74 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> 16);
75 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
77 void tlb_init(CPUState
*cpu
)
79 CPUArchState
*env
= cpu
->env_ptr
;
81 qemu_spin_init(&env
->tlb_c
.lock
);
84 /* flush_all_helper: run fn across all cpus
86 * If the wait flag is set then the src cpu's helper will be queued as
87 * "safe" work and the loop exited creating a synchronisation point
88 * where all queued work will be finished before execution starts
91 static void flush_all_helper(CPUState
*src
, run_on_cpu_func fn
,
98 async_run_on_cpu(cpu
, fn
, d
);
103 size_t tlb_flush_count(void)
109 CPUArchState
*env
= cpu
->env_ptr
;
111 count
+= atomic_read(&env
->tlb_flush_count
);
116 static void tlb_flush_one_mmuidx_locked(CPUArchState
*env
, int mmu_idx
)
118 memset(env
->tlb_table
[mmu_idx
], -1, sizeof(env
->tlb_table
[0]));
119 memset(env
->tlb_v_table
[mmu_idx
], -1, sizeof(env
->tlb_v_table
[0]));
120 env
->tlb_d
[mmu_idx
].large_page_addr
= -1;
121 env
->tlb_d
[mmu_idx
].large_page_mask
= -1;
122 env
->tlb_d
[mmu_idx
].vindex
= 0;
125 static void tlb_flush_by_mmuidx_async_work(CPUState
*cpu
, run_on_cpu_data data
)
127 CPUArchState
*env
= cpu
->env_ptr
;
128 unsigned long mmu_idx_bitmask
= data
.host_int
;
131 assert_cpu_is_self(cpu
);
133 tlb_debug("mmu_idx:0x%04lx\n", mmu_idx_bitmask
);
135 qemu_spin_lock(&env
->tlb_c
.lock
);
136 env
->tlb_c
.pending_flush
&= ~mmu_idx_bitmask
;
138 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
139 if (test_bit(mmu_idx
, &mmu_idx_bitmask
)) {
140 tlb_flush_one_mmuidx_locked(env
, mmu_idx
);
143 qemu_spin_unlock(&env
->tlb_c
.lock
);
145 cpu_tb_jmp_cache_clear(cpu
);
147 if (mmu_idx_bitmask
== ALL_MMUIDX_BITS
) {
148 atomic_set(&env
->tlb_flush_count
, env
->tlb_flush_count
+ 1);
152 void tlb_flush_by_mmuidx(CPUState
*cpu
, uint16_t idxmap
)
154 tlb_debug("mmu_idx: 0x%" PRIx16
"\n", idxmap
);
156 if (cpu
->created
&& !qemu_cpu_is_self(cpu
)) {
157 CPUArchState
*env
= cpu
->env_ptr
;
158 uint16_t pending
, to_clean
;
160 qemu_spin_lock(&env
->tlb_c
.lock
);
161 pending
= env
->tlb_c
.pending_flush
;
162 to_clean
= idxmap
& ~pending
;
163 env
->tlb_c
.pending_flush
= pending
| idxmap
;
164 qemu_spin_unlock(&env
->tlb_c
.lock
);
167 tlb_debug("reduced mmu_idx: 0x%" PRIx16
"\n", to_clean
);
168 async_run_on_cpu(cpu
, tlb_flush_by_mmuidx_async_work
,
169 RUN_ON_CPU_HOST_INT(to_clean
));
172 tlb_flush_by_mmuidx_async_work(cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
176 void tlb_flush(CPUState
*cpu
)
178 tlb_flush_by_mmuidx(cpu
, ALL_MMUIDX_BITS
);
181 void tlb_flush_by_mmuidx_all_cpus(CPUState
*src_cpu
, uint16_t idxmap
)
183 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
185 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
187 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
188 fn(src_cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
191 void tlb_flush_all_cpus(CPUState
*src_cpu
)
193 tlb_flush_by_mmuidx_all_cpus(src_cpu
, ALL_MMUIDX_BITS
);
196 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
, uint16_t idxmap
)
198 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
200 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
202 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
203 async_safe_run_on_cpu(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
206 void tlb_flush_all_cpus_synced(CPUState
*src_cpu
)
208 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu
, ALL_MMUIDX_BITS
);
211 static inline bool tlb_hit_page_anyprot(CPUTLBEntry
*tlb_entry
,
214 return tlb_hit_page(tlb_entry
->addr_read
, page
) ||
215 tlb_hit_page(tlb_addr_write(tlb_entry
), page
) ||
216 tlb_hit_page(tlb_entry
->addr_code
, page
);
219 /* Called with tlb_c.lock held */
220 static inline void tlb_flush_entry_locked(CPUTLBEntry
*tlb_entry
,
223 if (tlb_hit_page_anyprot(tlb_entry
, page
)) {
224 memset(tlb_entry
, -1, sizeof(*tlb_entry
));
228 /* Called with tlb_c.lock held */
229 static inline void tlb_flush_vtlb_page_locked(CPUArchState
*env
, int mmu_idx
,
234 assert_cpu_is_self(ENV_GET_CPU(env
));
235 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
236 tlb_flush_entry_locked(&env
->tlb_v_table
[mmu_idx
][k
], page
);
240 static void tlb_flush_page_locked(CPUArchState
*env
, int midx
,
243 target_ulong lp_addr
= env
->tlb_d
[midx
].large_page_addr
;
244 target_ulong lp_mask
= env
->tlb_d
[midx
].large_page_mask
;
246 /* Check if we need to flush due to large pages. */
247 if ((page
& lp_mask
) == lp_addr
) {
248 tlb_debug("forcing full flush midx %d ("
249 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
250 midx
, lp_addr
, lp_mask
);
251 tlb_flush_one_mmuidx_locked(env
, midx
);
253 tlb_flush_entry_locked(tlb_entry(env
, midx
, page
), page
);
254 tlb_flush_vtlb_page_locked(env
, midx
, page
);
258 static void tlb_flush_page_async_work(CPUState
*cpu
, run_on_cpu_data data
)
260 CPUArchState
*env
= cpu
->env_ptr
;
261 target_ulong addr
= (target_ulong
) data
.target_ptr
;
264 assert_cpu_is_self(cpu
);
266 tlb_debug("page addr:" TARGET_FMT_lx
"\n", addr
);
268 addr
&= TARGET_PAGE_MASK
;
269 qemu_spin_lock(&env
->tlb_c
.lock
);
270 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
271 tlb_flush_page_locked(env
, mmu_idx
, addr
);
273 qemu_spin_unlock(&env
->tlb_c
.lock
);
275 tb_flush_jmp_cache(cpu
, addr
);
278 void tlb_flush_page(CPUState
*cpu
, target_ulong addr
)
280 tlb_debug("page :" TARGET_FMT_lx
"\n", addr
);
282 if (!qemu_cpu_is_self(cpu
)) {
283 async_run_on_cpu(cpu
, tlb_flush_page_async_work
,
284 RUN_ON_CPU_TARGET_PTR(addr
));
286 tlb_flush_page_async_work(cpu
, RUN_ON_CPU_TARGET_PTR(addr
));
290 /* As we are going to hijack the bottom bits of the page address for a
291 * mmuidx bit mask we need to fail to build if we can't do that
293 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> TARGET_PAGE_BITS_MIN
);
295 static void tlb_flush_page_by_mmuidx_async_work(CPUState
*cpu
,
296 run_on_cpu_data data
)
298 CPUArchState
*env
= cpu
->env_ptr
;
299 target_ulong addr_and_mmuidx
= (target_ulong
) data
.target_ptr
;
300 target_ulong addr
= addr_and_mmuidx
& TARGET_PAGE_MASK
;
301 unsigned long mmu_idx_bitmap
= addr_and_mmuidx
& ALL_MMUIDX_BITS
;
304 assert_cpu_is_self(cpu
);
306 tlb_debug("page addr:" TARGET_FMT_lx
" mmu_map:0x%lx\n",
307 addr
, mmu_idx_bitmap
);
309 qemu_spin_lock(&env
->tlb_c
.lock
);
310 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
311 if (test_bit(mmu_idx
, &mmu_idx_bitmap
)) {
312 tlb_flush_page_locked(env
, mmu_idx
, addr
);
315 qemu_spin_unlock(&env
->tlb_c
.lock
);
317 tb_flush_jmp_cache(cpu
, addr
);
320 void tlb_flush_page_by_mmuidx(CPUState
*cpu
, target_ulong addr
, uint16_t idxmap
)
322 target_ulong addr_and_mmu_idx
;
324 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%" PRIx16
"\n", addr
, idxmap
);
326 /* This should already be page aligned */
327 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
328 addr_and_mmu_idx
|= idxmap
;
330 if (!qemu_cpu_is_self(cpu
)) {
331 async_run_on_cpu(cpu
, tlb_flush_page_by_mmuidx_async_work
,
332 RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
334 tlb_flush_page_by_mmuidx_async_work(
335 cpu
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
339 void tlb_flush_page_by_mmuidx_all_cpus(CPUState
*src_cpu
, target_ulong addr
,
342 const run_on_cpu_func fn
= tlb_flush_page_by_mmuidx_async_work
;
343 target_ulong addr_and_mmu_idx
;
345 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
347 /* This should already be page aligned */
348 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
349 addr_and_mmu_idx
|= idxmap
;
351 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
352 fn(src_cpu
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
355 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
359 const run_on_cpu_func fn
= tlb_flush_page_by_mmuidx_async_work
;
360 target_ulong addr_and_mmu_idx
;
362 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
364 /* This should already be page aligned */
365 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
366 addr_and_mmu_idx
|= idxmap
;
368 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
369 async_safe_run_on_cpu(src_cpu
, fn
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
372 void tlb_flush_page_all_cpus(CPUState
*src
, target_ulong addr
)
374 const run_on_cpu_func fn
= tlb_flush_page_async_work
;
376 flush_all_helper(src
, fn
, RUN_ON_CPU_TARGET_PTR(addr
));
377 fn(src
, RUN_ON_CPU_TARGET_PTR(addr
));
380 void tlb_flush_page_all_cpus_synced(CPUState
*src
,
383 const run_on_cpu_func fn
= tlb_flush_page_async_work
;
385 flush_all_helper(src
, fn
, RUN_ON_CPU_TARGET_PTR(addr
));
386 async_safe_run_on_cpu(src
, fn
, RUN_ON_CPU_TARGET_PTR(addr
));
389 /* update the TLBs so that writes to code in the virtual page 'addr'
391 void tlb_protect_code(ram_addr_t ram_addr
)
393 cpu_physical_memory_test_and_clear_dirty(ram_addr
, TARGET_PAGE_SIZE
,
397 /* update the TLB so that writes in physical page 'phys_addr' are no longer
398 tested for self modifying code */
399 void tlb_unprotect_code(ram_addr_t ram_addr
)
401 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
);
406 * Dirty write flag handling
408 * When the TCG code writes to a location it looks up the address in
409 * the TLB and uses that data to compute the final address. If any of
410 * the lower bits of the address are set then the slow path is forced.
411 * There are a number of reasons to do this but for normal RAM the
412 * most usual is detecting writes to code regions which may invalidate
415 * Other vCPUs might be reading their TLBs during guest execution, so we update
416 * te->addr_write with atomic_set. We don't need to worry about this for
417 * oversized guests as MTTCG is disabled for them.
419 * Called with tlb_c.lock held.
421 static void tlb_reset_dirty_range_locked(CPUTLBEntry
*tlb_entry
,
422 uintptr_t start
, uintptr_t length
)
424 uintptr_t addr
= tlb_entry
->addr_write
;
426 if ((addr
& (TLB_INVALID_MASK
| TLB_MMIO
| TLB_NOTDIRTY
)) == 0) {
427 addr
&= TARGET_PAGE_MASK
;
428 addr
+= tlb_entry
->addend
;
429 if ((addr
- start
) < length
) {
430 #if TCG_OVERSIZED_GUEST
431 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
433 atomic_set(&tlb_entry
->addr_write
,
434 tlb_entry
->addr_write
| TLB_NOTDIRTY
);
441 * Called with tlb_c.lock held.
442 * Called only from the vCPU context, i.e. the TLB's owner thread.
444 static inline void copy_tlb_helper_locked(CPUTLBEntry
*d
, const CPUTLBEntry
*s
)
449 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
451 * We must take tlb_c.lock to avoid racing with another vCPU update. The only
452 * thing actually updated is the target TLB entry ->addr_write flags.
454 void tlb_reset_dirty(CPUState
*cpu
, ram_addr_t start1
, ram_addr_t length
)
461 qemu_spin_lock(&env
->tlb_c
.lock
);
462 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
465 for (i
= 0; i
< CPU_TLB_SIZE
; i
++) {
466 tlb_reset_dirty_range_locked(&env
->tlb_table
[mmu_idx
][i
], start1
,
470 for (i
= 0; i
< CPU_VTLB_SIZE
; i
++) {
471 tlb_reset_dirty_range_locked(&env
->tlb_v_table
[mmu_idx
][i
], start1
,
475 qemu_spin_unlock(&env
->tlb_c
.lock
);
478 /* Called with tlb_c.lock held */
479 static inline void tlb_set_dirty1_locked(CPUTLBEntry
*tlb_entry
,
482 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
)) {
483 tlb_entry
->addr_write
= vaddr
;
487 /* update the TLB corresponding to virtual page vaddr
488 so that it is no longer dirty */
489 void tlb_set_dirty(CPUState
*cpu
, target_ulong vaddr
)
491 CPUArchState
*env
= cpu
->env_ptr
;
494 assert_cpu_is_self(cpu
);
496 vaddr
&= TARGET_PAGE_MASK
;
497 qemu_spin_lock(&env
->tlb_c
.lock
);
498 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
499 tlb_set_dirty1_locked(tlb_entry(env
, mmu_idx
, vaddr
), vaddr
);
502 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
504 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
505 tlb_set_dirty1_locked(&env
->tlb_v_table
[mmu_idx
][k
], vaddr
);
508 qemu_spin_unlock(&env
->tlb_c
.lock
);
511 /* Our TLB does not support large pages, so remember the area covered by
512 large pages and trigger a full TLB flush if these are invalidated. */
513 static void tlb_add_large_page(CPUArchState
*env
, int mmu_idx
,
514 target_ulong vaddr
, target_ulong size
)
516 target_ulong lp_addr
= env
->tlb_d
[mmu_idx
].large_page_addr
;
517 target_ulong lp_mask
= ~(size
- 1);
519 if (lp_addr
== (target_ulong
)-1) {
520 /* No previous large page. */
523 /* Extend the existing region to include the new page.
524 This is a compromise between unnecessary flushes and
525 the cost of maintaining a full variable size TLB. */
526 lp_mask
&= env
->tlb_d
[mmu_idx
].large_page_mask
;
527 while (((lp_addr
^ vaddr
) & lp_mask
) != 0) {
531 env
->tlb_d
[mmu_idx
].large_page_addr
= lp_addr
& lp_mask
;
532 env
->tlb_d
[mmu_idx
].large_page_mask
= lp_mask
;
535 /* Add a new TLB entry. At most one entry for a given virtual address
536 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
537 * supplied size is only used by tlb_flush_page.
539 * Called from TCG-generated code, which is under an RCU read-side
542 void tlb_set_page_with_attrs(CPUState
*cpu
, target_ulong vaddr
,
543 hwaddr paddr
, MemTxAttrs attrs
, int prot
,
544 int mmu_idx
, target_ulong size
)
546 CPUArchState
*env
= cpu
->env_ptr
;
547 MemoryRegionSection
*section
;
549 target_ulong address
;
550 target_ulong code_address
;
553 hwaddr iotlb
, xlat
, sz
, paddr_page
;
554 target_ulong vaddr_page
;
555 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
557 assert_cpu_is_self(cpu
);
559 if (size
<= TARGET_PAGE_SIZE
) {
560 sz
= TARGET_PAGE_SIZE
;
562 tlb_add_large_page(env
, mmu_idx
, vaddr
, size
);
565 vaddr_page
= vaddr
& TARGET_PAGE_MASK
;
566 paddr_page
= paddr
& TARGET_PAGE_MASK
;
568 section
= address_space_translate_for_iotlb(cpu
, asidx
, paddr_page
,
569 &xlat
, &sz
, attrs
, &prot
);
570 assert(sz
>= TARGET_PAGE_SIZE
);
572 tlb_debug("vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
574 vaddr
, paddr
, prot
, mmu_idx
);
576 address
= vaddr_page
;
577 if (size
< TARGET_PAGE_SIZE
) {
579 * Slow-path the TLB entries; we will repeat the MMU check and TLB
580 * fill on every access.
582 address
|= TLB_RECHECK
;
584 if (!memory_region_is_ram(section
->mr
) &&
585 !memory_region_is_romd(section
->mr
)) {
590 /* TLB_MMIO for rom/romd handled below */
591 addend
= (uintptr_t)memory_region_get_ram_ptr(section
->mr
) + xlat
;
594 code_address
= address
;
595 iotlb
= memory_region_section_get_iotlb(cpu
, section
, vaddr_page
,
596 paddr_page
, xlat
, prot
, &address
);
598 index
= tlb_index(env
, mmu_idx
, vaddr_page
);
599 te
= tlb_entry(env
, mmu_idx
, vaddr_page
);
602 * Hold the TLB lock for the rest of the function. We could acquire/release
603 * the lock several times in the function, but it is faster to amortize the
604 * acquisition cost by acquiring it just once. Note that this leads to
605 * a longer critical section, but this is not a concern since the TLB lock
606 * is unlikely to be contended.
608 qemu_spin_lock(&env
->tlb_c
.lock
);
610 /* Make sure there's no cached translation for the new page. */
611 tlb_flush_vtlb_page_locked(env
, mmu_idx
, vaddr_page
);
614 * Only evict the old entry to the victim tlb if it's for a
615 * different page; otherwise just overwrite the stale data.
617 if (!tlb_hit_page_anyprot(te
, vaddr_page
)) {
618 unsigned vidx
= env
->tlb_d
[mmu_idx
].vindex
++ % CPU_VTLB_SIZE
;
619 CPUTLBEntry
*tv
= &env
->tlb_v_table
[mmu_idx
][vidx
];
621 /* Evict the old entry into the victim tlb. */
622 copy_tlb_helper_locked(tv
, te
);
623 env
->iotlb_v
[mmu_idx
][vidx
] = env
->iotlb
[mmu_idx
][index
];
628 * At this point iotlb contains a physical section number in the lower
629 * TARGET_PAGE_BITS, and either
630 * + the ram_addr_t of the page base of the target RAM (if NOTDIRTY or ROM)
631 * + the offset within section->mr of the page base (otherwise)
632 * We subtract the vaddr_page (which is page aligned and thus won't
633 * disturb the low bits) to give an offset which can be added to the
634 * (non-page-aligned) vaddr of the eventual memory access to get
635 * the MemoryRegion offset for the access. Note that the vaddr we
636 * subtract here is that of the page base, and not the same as the
637 * vaddr we add back in io_readx()/io_writex()/get_page_addr_code().
639 env
->iotlb
[mmu_idx
][index
].addr
= iotlb
- vaddr_page
;
640 env
->iotlb
[mmu_idx
][index
].attrs
= attrs
;
642 /* Now calculate the new entry */
643 tn
.addend
= addend
- vaddr_page
;
644 if (prot
& PAGE_READ
) {
645 tn
.addr_read
= address
;
650 if (prot
& PAGE_EXEC
) {
651 tn
.addr_code
= code_address
;
657 if (prot
& PAGE_WRITE
) {
658 if ((memory_region_is_ram(section
->mr
) && section
->readonly
)
659 || memory_region_is_romd(section
->mr
)) {
660 /* Write access calls the I/O callback. */
661 tn
.addr_write
= address
| TLB_MMIO
;
662 } else if (memory_region_is_ram(section
->mr
)
663 && cpu_physical_memory_is_clean(
664 memory_region_get_ram_addr(section
->mr
) + xlat
)) {
665 tn
.addr_write
= address
| TLB_NOTDIRTY
;
667 tn
.addr_write
= address
;
669 if (prot
& PAGE_WRITE_INV
) {
670 tn
.addr_write
|= TLB_INVALID_MASK
;
674 copy_tlb_helper_locked(te
, &tn
);
675 qemu_spin_unlock(&env
->tlb_c
.lock
);
678 /* Add a new TLB entry, but without specifying the memory
679 * transaction attributes to be used.
681 void tlb_set_page(CPUState
*cpu
, target_ulong vaddr
,
682 hwaddr paddr
, int prot
,
683 int mmu_idx
, target_ulong size
)
685 tlb_set_page_with_attrs(cpu
, vaddr
, paddr
, MEMTXATTRS_UNSPECIFIED
,
686 prot
, mmu_idx
, size
);
689 static inline ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
693 ram_addr
= qemu_ram_addr_from_host(ptr
);
694 if (ram_addr
== RAM_ADDR_INVALID
) {
695 error_report("Bad ram pointer %p", ptr
);
701 static uint64_t io_readx(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
703 target_ulong addr
, uintptr_t retaddr
,
704 bool recheck
, MMUAccessType access_type
, int size
)
706 CPUState
*cpu
= ENV_GET_CPU(env
);
708 MemoryRegionSection
*section
;
716 * This is a TLB_RECHECK access, where the MMU protection
717 * covers a smaller range than a target page, and we must
718 * repeat the MMU check here. This tlb_fill() call might
719 * longjump out if this access should cause a guest exception.
722 target_ulong tlb_addr
;
724 tlb_fill(cpu
, addr
, size
, MMU_DATA_LOAD
, mmu_idx
, retaddr
);
726 entry
= tlb_entry(env
, mmu_idx
, addr
);
727 tlb_addr
= entry
->addr_read
;
728 if (!(tlb_addr
& ~(TARGET_PAGE_MASK
| TLB_RECHECK
))) {
730 uintptr_t haddr
= addr
+ entry
->addend
;
732 return ldn_p((void *)haddr
, size
);
734 /* Fall through for handling IO accesses */
737 section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
739 mr_offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
740 cpu
->mem_io_pc
= retaddr
;
741 if (mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !cpu
->can_do_io
) {
742 cpu_io_recompile(cpu
, retaddr
);
745 cpu
->mem_io_vaddr
= addr
;
746 cpu
->mem_io_access_type
= access_type
;
748 if (mr
->global_locking
&& !qemu_mutex_iothread_locked()) {
749 qemu_mutex_lock_iothread();
752 r
= memory_region_dispatch_read(mr
, mr_offset
,
753 &val
, size
, iotlbentry
->attrs
);
755 hwaddr physaddr
= mr_offset
+
756 section
->offset_within_address_space
-
757 section
->offset_within_region
;
759 cpu_transaction_failed(cpu
, physaddr
, addr
, size
, access_type
,
760 mmu_idx
, iotlbentry
->attrs
, r
, retaddr
);
763 qemu_mutex_unlock_iothread();
769 static void io_writex(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
771 uint64_t val
, target_ulong addr
,
772 uintptr_t retaddr
, bool recheck
, int size
)
774 CPUState
*cpu
= ENV_GET_CPU(env
);
776 MemoryRegionSection
*section
;
783 * This is a TLB_RECHECK access, where the MMU protection
784 * covers a smaller range than a target page, and we must
785 * repeat the MMU check here. This tlb_fill() call might
786 * longjump out if this access should cause a guest exception.
789 target_ulong tlb_addr
;
791 tlb_fill(cpu
, addr
, size
, MMU_DATA_STORE
, mmu_idx
, retaddr
);
793 entry
= tlb_entry(env
, mmu_idx
, addr
);
794 tlb_addr
= tlb_addr_write(entry
);
795 if (!(tlb_addr
& ~(TARGET_PAGE_MASK
| TLB_RECHECK
))) {
797 uintptr_t haddr
= addr
+ entry
->addend
;
799 stn_p((void *)haddr
, size
, val
);
802 /* Fall through for handling IO accesses */
805 section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
807 mr_offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
808 if (mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !cpu
->can_do_io
) {
809 cpu_io_recompile(cpu
, retaddr
);
811 cpu
->mem_io_vaddr
= addr
;
812 cpu
->mem_io_pc
= retaddr
;
814 if (mr
->global_locking
&& !qemu_mutex_iothread_locked()) {
815 qemu_mutex_lock_iothread();
818 r
= memory_region_dispatch_write(mr
, mr_offset
,
819 val
, size
, iotlbentry
->attrs
);
821 hwaddr physaddr
= mr_offset
+
822 section
->offset_within_address_space
-
823 section
->offset_within_region
;
825 cpu_transaction_failed(cpu
, physaddr
, addr
, size
, MMU_DATA_STORE
,
826 mmu_idx
, iotlbentry
->attrs
, r
, retaddr
);
829 qemu_mutex_unlock_iothread();
833 /* Return true if ADDR is present in the victim tlb, and has been copied
834 back to the main tlb. */
835 static bool victim_tlb_hit(CPUArchState
*env
, size_t mmu_idx
, size_t index
,
836 size_t elt_ofs
, target_ulong page
)
840 assert_cpu_is_self(ENV_GET_CPU(env
));
841 for (vidx
= 0; vidx
< CPU_VTLB_SIZE
; ++vidx
) {
842 CPUTLBEntry
*vtlb
= &env
->tlb_v_table
[mmu_idx
][vidx
];
845 /* elt_ofs might correspond to .addr_write, so use atomic_read */
846 #if TCG_OVERSIZED_GUEST
847 cmp
= *(target_ulong
*)((uintptr_t)vtlb
+ elt_ofs
);
849 cmp
= atomic_read((target_ulong
*)((uintptr_t)vtlb
+ elt_ofs
));
853 /* Found entry in victim tlb, swap tlb and iotlb. */
854 CPUTLBEntry tmptlb
, *tlb
= &env
->tlb_table
[mmu_idx
][index
];
856 qemu_spin_lock(&env
->tlb_c
.lock
);
857 copy_tlb_helper_locked(&tmptlb
, tlb
);
858 copy_tlb_helper_locked(tlb
, vtlb
);
859 copy_tlb_helper_locked(vtlb
, &tmptlb
);
860 qemu_spin_unlock(&env
->tlb_c
.lock
);
862 CPUIOTLBEntry tmpio
, *io
= &env
->iotlb
[mmu_idx
][index
];
863 CPUIOTLBEntry
*vio
= &env
->iotlb_v
[mmu_idx
][vidx
];
864 tmpio
= *io
; *io
= *vio
; *vio
= tmpio
;
871 /* Macro to call the above, with local variables from the use context. */
872 #define VICTIM_TLB_HIT(TY, ADDR) \
873 victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
874 (ADDR) & TARGET_PAGE_MASK)
876 /* NOTE: this function can trigger an exception */
877 /* NOTE2: the returned address is not exactly the physical address: it
878 * is actually a ram_addr_t (in system mode; the user mode emulation
879 * version of this function returns a guest virtual address).
881 tb_page_addr_t
get_page_addr_code(CPUArchState
*env
, target_ulong addr
)
883 uintptr_t mmu_idx
= cpu_mmu_index(env
, true);
884 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
885 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
888 if (unlikely(!tlb_hit(entry
->addr_code
, addr
))) {
889 if (!VICTIM_TLB_HIT(addr_code
, addr
)) {
890 tlb_fill(ENV_GET_CPU(env
), addr
, 0, MMU_INST_FETCH
, mmu_idx
, 0);
892 assert(tlb_hit(entry
->addr_code
, addr
));
895 if (unlikely(entry
->addr_code
& (TLB_RECHECK
| TLB_MMIO
))) {
897 * Return -1 if we can't translate and execute from an entire
898 * page of RAM here, which will cause us to execute by loading
899 * and translating one insn at a time, without caching:
900 * - TLB_RECHECK: means the MMU protection covers a smaller range
901 * than a target page, so we must redo the MMU check every insn
902 * - TLB_MMIO: region is not backed by RAM
907 p
= (void *)((uintptr_t)addr
+ entry
->addend
);
908 return qemu_ram_addr_from_host_nofail(p
);
911 /* Probe for whether the specified guest write access is permitted.
912 * If it is not permitted then an exception will be taken in the same
913 * way as if this were a real write access (and we will not return).
914 * Otherwise the function will return, and there will be a valid
915 * entry in the TLB for this access.
917 void probe_write(CPUArchState
*env
, target_ulong addr
, int size
, int mmu_idx
,
920 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
921 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
923 if (!tlb_hit(tlb_addr_write(entry
), addr
)) {
924 /* TLB entry is for a different page */
925 if (!VICTIM_TLB_HIT(addr_write
, addr
)) {
926 tlb_fill(ENV_GET_CPU(env
), addr
, size
, MMU_DATA_STORE
,
932 /* Probe for a read-modify-write atomic operation. Do not allow unaligned
933 * operations, or io operations to proceed. Return the host address. */
934 static void *atomic_mmu_lookup(CPUArchState
*env
, target_ulong addr
,
935 TCGMemOpIdx oi
, uintptr_t retaddr
,
938 size_t mmu_idx
= get_mmuidx(oi
);
939 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
940 CPUTLBEntry
*tlbe
= tlb_entry(env
, mmu_idx
, addr
);
941 target_ulong tlb_addr
= tlb_addr_write(tlbe
);
942 TCGMemOp mop
= get_memop(oi
);
943 int a_bits
= get_alignment_bits(mop
);
944 int s_bits
= mop
& MO_SIZE
;
947 /* Adjust the given return address. */
948 retaddr
-= GETPC_ADJ
;
950 /* Enforce guest required alignment. */
951 if (unlikely(a_bits
> 0 && (addr
& ((1 << a_bits
) - 1)))) {
952 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
953 cpu_unaligned_access(ENV_GET_CPU(env
), addr
, MMU_DATA_STORE
,
957 /* Enforce qemu required alignment. */
958 if (unlikely(addr
& ((1 << s_bits
) - 1))) {
959 /* We get here if guest alignment was not requested,
960 or was not enforced by cpu_unaligned_access above.
961 We might widen the access and emulate, but for now
962 mark an exception and exit the cpu loop. */
966 /* Check TLB entry and enforce page permissions. */
967 if (!tlb_hit(tlb_addr
, addr
)) {
968 if (!VICTIM_TLB_HIT(addr_write
, addr
)) {
969 tlb_fill(ENV_GET_CPU(env
), addr
, 1 << s_bits
, MMU_DATA_STORE
,
972 tlb_addr
= tlb_addr_write(tlbe
) & ~TLB_INVALID_MASK
;
975 /* Notice an IO access or a needs-MMU-lookup access */
976 if (unlikely(tlb_addr
& (TLB_MMIO
| TLB_RECHECK
))) {
977 /* There's really nothing that can be done to
978 support this apart from stop-the-world. */
982 /* Let the guest notice RMW on a write-only page. */
983 if (unlikely(tlbe
->addr_read
!= (tlb_addr
& ~TLB_NOTDIRTY
))) {
984 tlb_fill(ENV_GET_CPU(env
), addr
, 1 << s_bits
, MMU_DATA_LOAD
,
986 /* Since we don't support reads and writes to different addresses,
987 and we do have the proper page loaded for write, this shouldn't
988 ever return. But just in case, handle via stop-the-world. */
992 hostaddr
= (void *)((uintptr_t)addr
+ tlbe
->addend
);
995 if (unlikely(tlb_addr
& TLB_NOTDIRTY
)) {
997 memory_notdirty_write_prepare(ndi
, ENV_GET_CPU(env
), addr
,
998 qemu_ram_addr_from_host_nofail(hostaddr
),
1005 cpu_loop_exit_atomic(ENV_GET_CPU(env
), retaddr
);
1008 #ifdef TARGET_WORDS_BIGENDIAN
1009 # define TGT_BE(X) (X)
1010 # define TGT_LE(X) BSWAP(X)
1012 # define TGT_BE(X) BSWAP(X)
1013 # define TGT_LE(X) (X)
1016 #define MMUSUFFIX _mmu
1019 #include "softmmu_template.h"
1022 #include "softmmu_template.h"
1025 #include "softmmu_template.h"
1028 #include "softmmu_template.h"
1030 /* First set of helpers allows passing in of OI and RETADDR. This makes
1031 them callable from other helpers. */
1033 #define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr
1034 #define ATOMIC_NAME(X) \
1035 HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
1036 #define ATOMIC_MMU_DECLS NotDirtyInfo ndi
1037 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr, &ndi)
1038 #define ATOMIC_MMU_CLEANUP \
1040 if (unlikely(ndi.active)) { \
1041 memory_notdirty_write_complete(&ndi); \
1046 #include "atomic_template.h"
1049 #include "atomic_template.h"
1052 #include "atomic_template.h"
1054 #ifdef CONFIG_ATOMIC64
1056 #include "atomic_template.h"
1059 #if HAVE_CMPXCHG128 || HAVE_ATOMIC128
1060 #define DATA_SIZE 16
1061 #include "atomic_template.h"
1064 /* Second set of helpers are directly callable from TCG as helpers. */
1068 #undef ATOMIC_MMU_LOOKUP
1069 #define EXTRA_ARGS , TCGMemOpIdx oi
1070 #define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
1071 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC(), &ndi)
1074 #include "atomic_template.h"
1077 #include "atomic_template.h"
1080 #include "atomic_template.h"
1082 #ifdef CONFIG_ATOMIC64
1084 #include "atomic_template.h"
1087 /* Code access functions. */
1090 #define MMUSUFFIX _cmmu
1092 #define GETPC() ((uintptr_t)0)
1093 #define SOFTMMU_CODE_ACCESS
1096 #include "softmmu_template.h"
1099 #include "softmmu_template.h"
1102 #include "softmmu_template.h"
1105 #include "softmmu_template.h"