2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
22 #include "hw/core/tcg-cpu-ops.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/cpu_ldst.h"
26 #include "exec/cputlb.h"
27 #include "exec/memory-internal.h"
28 #include "exec/ram_addr.h"
30 #include "qemu/error-report.h"
32 #include "exec/helper-proto-common.h"
33 #include "qemu/atomic.h"
34 #include "qemu/atomic128.h"
35 #include "exec/translate-all.h"
40 #include "qemu/plugin-memory.h"
42 #include "tcg/tcg-ldst.h"
43 #include "tcg/oversized-guest.h"
45 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
46 /* #define DEBUG_TLB */
47 /* #define DEBUG_TLB_LOG */
50 # define DEBUG_TLB_GATE 1
52 # define DEBUG_TLB_LOG_GATE 1
54 # define DEBUG_TLB_LOG_GATE 0
57 # define DEBUG_TLB_GATE 0
58 # define DEBUG_TLB_LOG_GATE 0
61 #define tlb_debug(fmt, ...) do { \
62 if (DEBUG_TLB_LOG_GATE) { \
63 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
65 } else if (DEBUG_TLB_GATE) { \
66 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
70 #define assert_cpu_is_self(cpu) do { \
71 if (DEBUG_TLB_GATE) { \
72 g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \
76 /* run_on_cpu_data.target_ptr should always be big enough for a
77 * vaddr even on 32 bit builds
79 QEMU_BUILD_BUG_ON(sizeof(vaddr
) > sizeof(run_on_cpu_data
));
81 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
83 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> 16);
84 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
86 static inline size_t tlb_n_entries(CPUTLBDescFast
*fast
)
88 return (fast
->mask
>> CPU_TLB_ENTRY_BITS
) + 1;
91 static inline size_t sizeof_tlb(CPUTLBDescFast
*fast
)
93 return fast
->mask
+ (1 << CPU_TLB_ENTRY_BITS
);
96 static void tlb_window_reset(CPUTLBDesc
*desc
, int64_t ns
,
99 desc
->window_begin_ns
= ns
;
100 desc
->window_max_entries
= max_entries
;
103 static void tb_jmp_cache_clear_page(CPUState
*cpu
, vaddr page_addr
)
105 CPUJumpCache
*jc
= cpu
->tb_jmp_cache
;
112 i0
= tb_jmp_cache_hash_page(page_addr
);
113 for (i
= 0; i
< TB_JMP_PAGE_SIZE
; i
++) {
114 qatomic_set(&jc
->array
[i0
+ i
].tb
, NULL
);
119 * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary
120 * @desc: The CPUTLBDesc portion of the TLB
121 * @fast: The CPUTLBDescFast portion of the same TLB
123 * Called with tlb_lock_held.
125 * We have two main constraints when resizing a TLB: (1) we only resize it
126 * on a TLB flush (otherwise we'd have to take a perf hit by either rehashing
127 * the array or unnecessarily flushing it), which means we do not control how
128 * frequently the resizing can occur; (2) we don't have access to the guest's
129 * future scheduling decisions, and therefore have to decide the magnitude of
130 * the resize based on past observations.
132 * In general, a memory-hungry process can benefit greatly from an appropriately
133 * sized TLB, since a guest TLB miss is very expensive. This doesn't mean that
134 * we just have to make the TLB as large as possible; while an oversized TLB
135 * results in minimal TLB miss rates, it also takes longer to be flushed
136 * (flushes can be _very_ frequent), and the reduced locality can also hurt
139 * To achieve near-optimal performance for all kinds of workloads, we:
141 * 1. Aggressively increase the size of the TLB when the use rate of the
142 * TLB being flushed is high, since it is likely that in the near future this
143 * memory-hungry process will execute again, and its memory hungriness will
144 * probably be similar.
146 * 2. Slowly reduce the size of the TLB as the use rate declines over a
147 * reasonably large time window. The rationale is that if in such a time window
148 * we have not observed a high TLB use rate, it is likely that we won't observe
149 * it in the near future. In that case, once a time window expires we downsize
150 * the TLB to match the maximum use rate observed in the window.
152 * 3. Try to keep the maximum use rate in a time window in the 30-70% range,
153 * since in that range performance is likely near-optimal. Recall that the TLB
154 * is direct mapped, so we want the use rate to be low (or at least not too
155 * high), since otherwise we are likely to have a significant amount of
158 static void tlb_mmu_resize_locked(CPUTLBDesc
*desc
, CPUTLBDescFast
*fast
,
161 size_t old_size
= tlb_n_entries(fast
);
163 size_t new_size
= old_size
;
164 int64_t window_len_ms
= 100;
165 int64_t window_len_ns
= window_len_ms
* 1000 * 1000;
166 bool window_expired
= now
> desc
->window_begin_ns
+ window_len_ns
;
168 if (desc
->n_used_entries
> desc
->window_max_entries
) {
169 desc
->window_max_entries
= desc
->n_used_entries
;
171 rate
= desc
->window_max_entries
* 100 / old_size
;
174 new_size
= MIN(old_size
<< 1, 1 << CPU_TLB_DYN_MAX_BITS
);
175 } else if (rate
< 30 && window_expired
) {
176 size_t ceil
= pow2ceil(desc
->window_max_entries
);
177 size_t expected_rate
= desc
->window_max_entries
* 100 / ceil
;
180 * Avoid undersizing when the max number of entries seen is just below
181 * a pow2. For instance, if max_entries == 1025, the expected use rate
182 * would be 1025/2048==50%. However, if max_entries == 1023, we'd get
183 * 1023/1024==99.9% use rate, so we'd likely end up doubling the size
184 * later. Thus, make sure that the expected use rate remains below 70%.
185 * (and since we double the size, that means the lowest rate we'd
186 * expect to get is 35%, which is still in the 30-70% range where
187 * we consider that the size is appropriate.)
189 if (expected_rate
> 70) {
192 new_size
= MAX(ceil
, 1 << CPU_TLB_DYN_MIN_BITS
);
195 if (new_size
== old_size
) {
196 if (window_expired
) {
197 tlb_window_reset(desc
, now
, desc
->n_used_entries
);
203 g_free(desc
->fulltlb
);
205 tlb_window_reset(desc
, now
, 0);
206 /* desc->n_used_entries is cleared by the caller */
207 fast
->mask
= (new_size
- 1) << CPU_TLB_ENTRY_BITS
;
208 fast
->table
= g_try_new(CPUTLBEntry
, new_size
);
209 desc
->fulltlb
= g_try_new(CPUTLBEntryFull
, new_size
);
212 * If the allocations fail, try smaller sizes. We just freed some
213 * memory, so going back to half of new_size has a good chance of working.
214 * Increased memory pressure elsewhere in the system might cause the
215 * allocations to fail though, so we progressively reduce the allocation
216 * size, aborting if we cannot even allocate the smallest TLB we support.
218 while (fast
->table
== NULL
|| desc
->fulltlb
== NULL
) {
219 if (new_size
== (1 << CPU_TLB_DYN_MIN_BITS
)) {
220 error_report("%s: %s", __func__
, strerror(errno
));
223 new_size
= MAX(new_size
>> 1, 1 << CPU_TLB_DYN_MIN_BITS
);
224 fast
->mask
= (new_size
- 1) << CPU_TLB_ENTRY_BITS
;
227 g_free(desc
->fulltlb
);
228 fast
->table
= g_try_new(CPUTLBEntry
, new_size
);
229 desc
->fulltlb
= g_try_new(CPUTLBEntryFull
, new_size
);
233 static void tlb_mmu_flush_locked(CPUTLBDesc
*desc
, CPUTLBDescFast
*fast
)
235 desc
->n_used_entries
= 0;
236 desc
->large_page_addr
= -1;
237 desc
->large_page_mask
= -1;
239 memset(fast
->table
, -1, sizeof_tlb(fast
));
240 memset(desc
->vtable
, -1, sizeof(desc
->vtable
));
243 static void tlb_flush_one_mmuidx_locked(CPUArchState
*env
, int mmu_idx
,
246 CPUTLBDesc
*desc
= &env_tlb(env
)->d
[mmu_idx
];
247 CPUTLBDescFast
*fast
= &env_tlb(env
)->f
[mmu_idx
];
249 tlb_mmu_resize_locked(desc
, fast
, now
);
250 tlb_mmu_flush_locked(desc
, fast
);
253 static void tlb_mmu_init(CPUTLBDesc
*desc
, CPUTLBDescFast
*fast
, int64_t now
)
255 size_t n_entries
= 1 << CPU_TLB_DYN_DEFAULT_BITS
;
257 tlb_window_reset(desc
, now
, 0);
258 desc
->n_used_entries
= 0;
259 fast
->mask
= (n_entries
- 1) << CPU_TLB_ENTRY_BITS
;
260 fast
->table
= g_new(CPUTLBEntry
, n_entries
);
261 desc
->fulltlb
= g_new(CPUTLBEntryFull
, n_entries
);
262 tlb_mmu_flush_locked(desc
, fast
);
265 static inline void tlb_n_used_entries_inc(CPUArchState
*env
, uintptr_t mmu_idx
)
267 env_tlb(env
)->d
[mmu_idx
].n_used_entries
++;
270 static inline void tlb_n_used_entries_dec(CPUArchState
*env
, uintptr_t mmu_idx
)
272 env_tlb(env
)->d
[mmu_idx
].n_used_entries
--;
275 void tlb_init(CPUState
*cpu
)
277 CPUArchState
*env
= cpu
->env_ptr
;
278 int64_t now
= get_clock_realtime();
281 qemu_spin_init(&env_tlb(env
)->c
.lock
);
283 /* All tlbs are initialized flushed. */
284 env_tlb(env
)->c
.dirty
= 0;
286 for (i
= 0; i
< NB_MMU_MODES
; i
++) {
287 tlb_mmu_init(&env_tlb(env
)->d
[i
], &env_tlb(env
)->f
[i
], now
);
291 void tlb_destroy(CPUState
*cpu
)
293 CPUArchState
*env
= cpu
->env_ptr
;
296 qemu_spin_destroy(&env_tlb(env
)->c
.lock
);
297 for (i
= 0; i
< NB_MMU_MODES
; i
++) {
298 CPUTLBDesc
*desc
= &env_tlb(env
)->d
[i
];
299 CPUTLBDescFast
*fast
= &env_tlb(env
)->f
[i
];
302 g_free(desc
->fulltlb
);
306 /* flush_all_helper: run fn across all cpus
308 * If the wait flag is set then the src cpu's helper will be queued as
309 * "safe" work and the loop exited creating a synchronisation point
310 * where all queued work will be finished before execution starts
313 static void flush_all_helper(CPUState
*src
, run_on_cpu_func fn
,
320 async_run_on_cpu(cpu
, fn
, d
);
325 void tlb_flush_counts(size_t *pfull
, size_t *ppart
, size_t *pelide
)
328 size_t full
= 0, part
= 0, elide
= 0;
331 CPUArchState
*env
= cpu
->env_ptr
;
333 full
+= qatomic_read(&env_tlb(env
)->c
.full_flush_count
);
334 part
+= qatomic_read(&env_tlb(env
)->c
.part_flush_count
);
335 elide
+= qatomic_read(&env_tlb(env
)->c
.elide_flush_count
);
342 static void tlb_flush_by_mmuidx_async_work(CPUState
*cpu
, run_on_cpu_data data
)
344 CPUArchState
*env
= cpu
->env_ptr
;
345 uint16_t asked
= data
.host_int
;
346 uint16_t all_dirty
, work
, to_clean
;
347 int64_t now
= get_clock_realtime();
349 assert_cpu_is_self(cpu
);
351 tlb_debug("mmu_idx:0x%04" PRIx16
"\n", asked
);
353 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
355 all_dirty
= env_tlb(env
)->c
.dirty
;
356 to_clean
= asked
& all_dirty
;
357 all_dirty
&= ~to_clean
;
358 env_tlb(env
)->c
.dirty
= all_dirty
;
360 for (work
= to_clean
; work
!= 0; work
&= work
- 1) {
361 int mmu_idx
= ctz32(work
);
362 tlb_flush_one_mmuidx_locked(env
, mmu_idx
, now
);
365 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
367 tcg_flush_jmp_cache(cpu
);
369 if (to_clean
== ALL_MMUIDX_BITS
) {
370 qatomic_set(&env_tlb(env
)->c
.full_flush_count
,
371 env_tlb(env
)->c
.full_flush_count
+ 1);
373 qatomic_set(&env_tlb(env
)->c
.part_flush_count
,
374 env_tlb(env
)->c
.part_flush_count
+ ctpop16(to_clean
));
375 if (to_clean
!= asked
) {
376 qatomic_set(&env_tlb(env
)->c
.elide_flush_count
,
377 env_tlb(env
)->c
.elide_flush_count
+
378 ctpop16(asked
& ~to_clean
));
383 void tlb_flush_by_mmuidx(CPUState
*cpu
, uint16_t idxmap
)
385 tlb_debug("mmu_idx: 0x%" PRIx16
"\n", idxmap
);
387 if (cpu
->created
&& !qemu_cpu_is_self(cpu
)) {
388 async_run_on_cpu(cpu
, tlb_flush_by_mmuidx_async_work
,
389 RUN_ON_CPU_HOST_INT(idxmap
));
391 tlb_flush_by_mmuidx_async_work(cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
395 void tlb_flush(CPUState
*cpu
)
397 tlb_flush_by_mmuidx(cpu
, ALL_MMUIDX_BITS
);
400 void tlb_flush_by_mmuidx_all_cpus(CPUState
*src_cpu
, uint16_t idxmap
)
402 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
404 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
406 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
407 fn(src_cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
410 void tlb_flush_all_cpus(CPUState
*src_cpu
)
412 tlb_flush_by_mmuidx_all_cpus(src_cpu
, ALL_MMUIDX_BITS
);
415 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
, uint16_t idxmap
)
417 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
419 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
421 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
422 async_safe_run_on_cpu(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
425 void tlb_flush_all_cpus_synced(CPUState
*src_cpu
)
427 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu
, ALL_MMUIDX_BITS
);
430 static bool tlb_hit_page_mask_anyprot(CPUTLBEntry
*tlb_entry
,
431 vaddr page
, vaddr mask
)
434 mask
&= TARGET_PAGE_MASK
| TLB_INVALID_MASK
;
436 return (page
== (tlb_entry
->addr_read
& mask
) ||
437 page
== (tlb_addr_write(tlb_entry
) & mask
) ||
438 page
== (tlb_entry
->addr_code
& mask
));
441 static inline bool tlb_hit_page_anyprot(CPUTLBEntry
*tlb_entry
, vaddr page
)
443 return tlb_hit_page_mask_anyprot(tlb_entry
, page
, -1);
447 * tlb_entry_is_empty - return true if the entry is not in use
448 * @te: pointer to CPUTLBEntry
450 static inline bool tlb_entry_is_empty(const CPUTLBEntry
*te
)
452 return te
->addr_read
== -1 && te
->addr_write
== -1 && te
->addr_code
== -1;
455 /* Called with tlb_c.lock held */
456 static bool tlb_flush_entry_mask_locked(CPUTLBEntry
*tlb_entry
,
460 if (tlb_hit_page_mask_anyprot(tlb_entry
, page
, mask
)) {
461 memset(tlb_entry
, -1, sizeof(*tlb_entry
));
467 static inline bool tlb_flush_entry_locked(CPUTLBEntry
*tlb_entry
, vaddr page
)
469 return tlb_flush_entry_mask_locked(tlb_entry
, page
, -1);
472 /* Called with tlb_c.lock held */
473 static void tlb_flush_vtlb_page_mask_locked(CPUArchState
*env
, int mmu_idx
,
477 CPUTLBDesc
*d
= &env_tlb(env
)->d
[mmu_idx
];
480 assert_cpu_is_self(env_cpu(env
));
481 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
482 if (tlb_flush_entry_mask_locked(&d
->vtable
[k
], page
, mask
)) {
483 tlb_n_used_entries_dec(env
, mmu_idx
);
488 static inline void tlb_flush_vtlb_page_locked(CPUArchState
*env
, int mmu_idx
,
491 tlb_flush_vtlb_page_mask_locked(env
, mmu_idx
, page
, -1);
494 static void tlb_flush_page_locked(CPUArchState
*env
, int midx
, vaddr page
)
496 vaddr lp_addr
= env_tlb(env
)->d
[midx
].large_page_addr
;
497 vaddr lp_mask
= env_tlb(env
)->d
[midx
].large_page_mask
;
499 /* Check if we need to flush due to large pages. */
500 if ((page
& lp_mask
) == lp_addr
) {
501 tlb_debug("forcing full flush midx %d (%016"
502 VADDR_PRIx
"/%016" VADDR_PRIx
")\n",
503 midx
, lp_addr
, lp_mask
);
504 tlb_flush_one_mmuidx_locked(env
, midx
, get_clock_realtime());
506 if (tlb_flush_entry_locked(tlb_entry(env
, midx
, page
), page
)) {
507 tlb_n_used_entries_dec(env
, midx
);
509 tlb_flush_vtlb_page_locked(env
, midx
, page
);
514 * tlb_flush_page_by_mmuidx_async_0:
515 * @cpu: cpu on which to flush
516 * @addr: page of virtual address to flush
517 * @idxmap: set of mmu_idx to flush
519 * Helper for tlb_flush_page_by_mmuidx and friends, flush one page
520 * at @addr from the tlbs indicated by @idxmap from @cpu.
522 static void tlb_flush_page_by_mmuidx_async_0(CPUState
*cpu
,
526 CPUArchState
*env
= cpu
->env_ptr
;
529 assert_cpu_is_self(cpu
);
531 tlb_debug("page addr: %016" VADDR_PRIx
" mmu_map:0x%x\n", addr
, idxmap
);
533 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
534 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
535 if ((idxmap
>> mmu_idx
) & 1) {
536 tlb_flush_page_locked(env
, mmu_idx
, addr
);
539 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
542 * Discard jump cache entries for any tb which might potentially
543 * overlap the flushed page, which includes the previous.
545 tb_jmp_cache_clear_page(cpu
, addr
- TARGET_PAGE_SIZE
);
546 tb_jmp_cache_clear_page(cpu
, addr
);
550 * tlb_flush_page_by_mmuidx_async_1:
551 * @cpu: cpu on which to flush
552 * @data: encoded addr + idxmap
554 * Helper for tlb_flush_page_by_mmuidx and friends, called through
555 * async_run_on_cpu. The idxmap parameter is encoded in the page
556 * offset of the target_ptr field. This limits the set of mmu_idx
557 * that can be passed via this method.
559 static void tlb_flush_page_by_mmuidx_async_1(CPUState
*cpu
,
560 run_on_cpu_data data
)
562 vaddr addr_and_idxmap
= data
.target_ptr
;
563 vaddr addr
= addr_and_idxmap
& TARGET_PAGE_MASK
;
564 uint16_t idxmap
= addr_and_idxmap
& ~TARGET_PAGE_MASK
;
566 tlb_flush_page_by_mmuidx_async_0(cpu
, addr
, idxmap
);
572 } TLBFlushPageByMMUIdxData
;
575 * tlb_flush_page_by_mmuidx_async_2:
576 * @cpu: cpu on which to flush
577 * @data: allocated addr + idxmap
579 * Helper for tlb_flush_page_by_mmuidx and friends, called through
580 * async_run_on_cpu. The addr+idxmap parameters are stored in a
581 * TLBFlushPageByMMUIdxData structure that has been allocated
582 * specifically for this helper. Free the structure when done.
584 static void tlb_flush_page_by_mmuidx_async_2(CPUState
*cpu
,
585 run_on_cpu_data data
)
587 TLBFlushPageByMMUIdxData
*d
= data
.host_ptr
;
589 tlb_flush_page_by_mmuidx_async_0(cpu
, d
->addr
, d
->idxmap
);
593 void tlb_flush_page_by_mmuidx(CPUState
*cpu
, vaddr addr
, uint16_t idxmap
)
595 tlb_debug("addr: %016" VADDR_PRIx
" mmu_idx:%" PRIx16
"\n", addr
, idxmap
);
597 /* This should already be page aligned */
598 addr
&= TARGET_PAGE_MASK
;
600 if (qemu_cpu_is_self(cpu
)) {
601 tlb_flush_page_by_mmuidx_async_0(cpu
, addr
, idxmap
);
602 } else if (idxmap
< TARGET_PAGE_SIZE
) {
604 * Most targets have only a few mmu_idx. In the case where
605 * we can stuff idxmap into the low TARGET_PAGE_BITS, avoid
606 * allocating memory for this operation.
608 async_run_on_cpu(cpu
, tlb_flush_page_by_mmuidx_async_1
,
609 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
611 TLBFlushPageByMMUIdxData
*d
= g_new(TLBFlushPageByMMUIdxData
, 1);
613 /* Otherwise allocate a structure, freed by the worker. */
616 async_run_on_cpu(cpu
, tlb_flush_page_by_mmuidx_async_2
,
617 RUN_ON_CPU_HOST_PTR(d
));
621 void tlb_flush_page(CPUState
*cpu
, vaddr addr
)
623 tlb_flush_page_by_mmuidx(cpu
, addr
, ALL_MMUIDX_BITS
);
626 void tlb_flush_page_by_mmuidx_all_cpus(CPUState
*src_cpu
, vaddr addr
,
629 tlb_debug("addr: %016" VADDR_PRIx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
631 /* This should already be page aligned */
632 addr
&= TARGET_PAGE_MASK
;
635 * Allocate memory to hold addr+idxmap only when needed.
636 * See tlb_flush_page_by_mmuidx for details.
638 if (idxmap
< TARGET_PAGE_SIZE
) {
639 flush_all_helper(src_cpu
, tlb_flush_page_by_mmuidx_async_1
,
640 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
644 /* Allocate a separate data block for each destination cpu. */
645 CPU_FOREACH(dst_cpu
) {
646 if (dst_cpu
!= src_cpu
) {
647 TLBFlushPageByMMUIdxData
*d
648 = g_new(TLBFlushPageByMMUIdxData
, 1);
652 async_run_on_cpu(dst_cpu
, tlb_flush_page_by_mmuidx_async_2
,
653 RUN_ON_CPU_HOST_PTR(d
));
658 tlb_flush_page_by_mmuidx_async_0(src_cpu
, addr
, idxmap
);
661 void tlb_flush_page_all_cpus(CPUState
*src
, vaddr addr
)
663 tlb_flush_page_by_mmuidx_all_cpus(src
, addr
, ALL_MMUIDX_BITS
);
666 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
670 tlb_debug("addr: %016" VADDR_PRIx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
672 /* This should already be page aligned */
673 addr
&= TARGET_PAGE_MASK
;
676 * Allocate memory to hold addr+idxmap only when needed.
677 * See tlb_flush_page_by_mmuidx for details.
679 if (idxmap
< TARGET_PAGE_SIZE
) {
680 flush_all_helper(src_cpu
, tlb_flush_page_by_mmuidx_async_1
,
681 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
682 async_safe_run_on_cpu(src_cpu
, tlb_flush_page_by_mmuidx_async_1
,
683 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
686 TLBFlushPageByMMUIdxData
*d
;
688 /* Allocate a separate data block for each destination cpu. */
689 CPU_FOREACH(dst_cpu
) {
690 if (dst_cpu
!= src_cpu
) {
691 d
= g_new(TLBFlushPageByMMUIdxData
, 1);
694 async_run_on_cpu(dst_cpu
, tlb_flush_page_by_mmuidx_async_2
,
695 RUN_ON_CPU_HOST_PTR(d
));
699 d
= g_new(TLBFlushPageByMMUIdxData
, 1);
702 async_safe_run_on_cpu(src_cpu
, tlb_flush_page_by_mmuidx_async_2
,
703 RUN_ON_CPU_HOST_PTR(d
));
707 void tlb_flush_page_all_cpus_synced(CPUState
*src
, vaddr addr
)
709 tlb_flush_page_by_mmuidx_all_cpus_synced(src
, addr
, ALL_MMUIDX_BITS
);
712 static void tlb_flush_range_locked(CPUArchState
*env
, int midx
,
713 vaddr addr
, vaddr len
,
716 CPUTLBDesc
*d
= &env_tlb(env
)->d
[midx
];
717 CPUTLBDescFast
*f
= &env_tlb(env
)->f
[midx
];
718 vaddr mask
= MAKE_64BIT_MASK(0, bits
);
721 * If @bits is smaller than the tlb size, there may be multiple entries
722 * within the TLB; otherwise all addresses that match under @mask hit
723 * the same TLB entry.
724 * TODO: Perhaps allow bits to be a few bits less than the size.
725 * For now, just flush the entire TLB.
727 * If @len is larger than the tlb size, then it will take longer to
728 * test all of the entries in the TLB than it will to flush it all.
730 if (mask
< f
->mask
|| len
> f
->mask
) {
731 tlb_debug("forcing full flush midx %d ("
732 "%016" VADDR_PRIx
"/%016" VADDR_PRIx
"+%016" VADDR_PRIx
")\n",
733 midx
, addr
, mask
, len
);
734 tlb_flush_one_mmuidx_locked(env
, midx
, get_clock_realtime());
739 * Check if we need to flush due to large pages.
740 * Because large_page_mask contains all 1's from the msb,
741 * we only need to test the end of the range.
743 if (((addr
+ len
- 1) & d
->large_page_mask
) == d
->large_page_addr
) {
744 tlb_debug("forcing full flush midx %d ("
745 "%016" VADDR_PRIx
"/%016" VADDR_PRIx
")\n",
746 midx
, d
->large_page_addr
, d
->large_page_mask
);
747 tlb_flush_one_mmuidx_locked(env
, midx
, get_clock_realtime());
751 for (vaddr i
= 0; i
< len
; i
+= TARGET_PAGE_SIZE
) {
752 vaddr page
= addr
+ i
;
753 CPUTLBEntry
*entry
= tlb_entry(env
, midx
, page
);
755 if (tlb_flush_entry_mask_locked(entry
, page
, mask
)) {
756 tlb_n_used_entries_dec(env
, midx
);
758 tlb_flush_vtlb_page_mask_locked(env
, midx
, page
, mask
);
769 static void tlb_flush_range_by_mmuidx_async_0(CPUState
*cpu
,
772 CPUArchState
*env
= cpu
->env_ptr
;
775 assert_cpu_is_self(cpu
);
777 tlb_debug("range: %016" VADDR_PRIx
"/%u+%016" VADDR_PRIx
" mmu_map:0x%x\n",
778 d
.addr
, d
.bits
, d
.len
, d
.idxmap
);
780 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
781 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
782 if ((d
.idxmap
>> mmu_idx
) & 1) {
783 tlb_flush_range_locked(env
, mmu_idx
, d
.addr
, d
.len
, d
.bits
);
786 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
789 * If the length is larger than the jump cache size, then it will take
790 * longer to clear each entry individually than it will to clear it all.
792 if (d
.len
>= (TARGET_PAGE_SIZE
* TB_JMP_CACHE_SIZE
)) {
793 tcg_flush_jmp_cache(cpu
);
798 * Discard jump cache entries for any tb which might potentially
799 * overlap the flushed pages, which includes the previous.
801 d
.addr
-= TARGET_PAGE_SIZE
;
802 for (vaddr i
= 0, n
= d
.len
/ TARGET_PAGE_SIZE
+ 1; i
< n
; i
++) {
803 tb_jmp_cache_clear_page(cpu
, d
.addr
);
804 d
.addr
+= TARGET_PAGE_SIZE
;
808 static void tlb_flush_range_by_mmuidx_async_1(CPUState
*cpu
,
809 run_on_cpu_data data
)
811 TLBFlushRangeData
*d
= data
.host_ptr
;
812 tlb_flush_range_by_mmuidx_async_0(cpu
, *d
);
816 void tlb_flush_range_by_mmuidx(CPUState
*cpu
, vaddr addr
,
817 vaddr len
, uint16_t idxmap
,
823 * If all bits are significant, and len is small,
824 * this devolves to tlb_flush_page.
826 if (bits
>= TARGET_LONG_BITS
&& len
<= TARGET_PAGE_SIZE
) {
827 tlb_flush_page_by_mmuidx(cpu
, addr
, idxmap
);
830 /* If no page bits are significant, this devolves to tlb_flush. */
831 if (bits
< TARGET_PAGE_BITS
) {
832 tlb_flush_by_mmuidx(cpu
, idxmap
);
836 /* This should already be page aligned */
837 d
.addr
= addr
& TARGET_PAGE_MASK
;
842 if (qemu_cpu_is_self(cpu
)) {
843 tlb_flush_range_by_mmuidx_async_0(cpu
, d
);
845 /* Otherwise allocate a structure, freed by the worker. */
846 TLBFlushRangeData
*p
= g_memdup(&d
, sizeof(d
));
847 async_run_on_cpu(cpu
, tlb_flush_range_by_mmuidx_async_1
,
848 RUN_ON_CPU_HOST_PTR(p
));
852 void tlb_flush_page_bits_by_mmuidx(CPUState
*cpu
, vaddr addr
,
853 uint16_t idxmap
, unsigned bits
)
855 tlb_flush_range_by_mmuidx(cpu
, addr
, TARGET_PAGE_SIZE
, idxmap
, bits
);
858 void tlb_flush_range_by_mmuidx_all_cpus(CPUState
*src_cpu
,
859 vaddr addr
, vaddr len
,
860 uint16_t idxmap
, unsigned bits
)
866 * If all bits are significant, and len is small,
867 * this devolves to tlb_flush_page.
869 if (bits
>= TARGET_LONG_BITS
&& len
<= TARGET_PAGE_SIZE
) {
870 tlb_flush_page_by_mmuidx_all_cpus(src_cpu
, addr
, idxmap
);
873 /* If no page bits are significant, this devolves to tlb_flush. */
874 if (bits
< TARGET_PAGE_BITS
) {
875 tlb_flush_by_mmuidx_all_cpus(src_cpu
, idxmap
);
879 /* This should already be page aligned */
880 d
.addr
= addr
& TARGET_PAGE_MASK
;
885 /* Allocate a separate data block for each destination cpu. */
886 CPU_FOREACH(dst_cpu
) {
887 if (dst_cpu
!= src_cpu
) {
888 TLBFlushRangeData
*p
= g_memdup(&d
, sizeof(d
));
889 async_run_on_cpu(dst_cpu
,
890 tlb_flush_range_by_mmuidx_async_1
,
891 RUN_ON_CPU_HOST_PTR(p
));
895 tlb_flush_range_by_mmuidx_async_0(src_cpu
, d
);
898 void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState
*src_cpu
,
899 vaddr addr
, uint16_t idxmap
,
902 tlb_flush_range_by_mmuidx_all_cpus(src_cpu
, addr
, TARGET_PAGE_SIZE
,
906 void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
912 TLBFlushRangeData d
, *p
;
916 * If all bits are significant, and len is small,
917 * this devolves to tlb_flush_page.
919 if (bits
>= TARGET_LONG_BITS
&& len
<= TARGET_PAGE_SIZE
) {
920 tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu
, addr
, idxmap
);
923 /* If no page bits are significant, this devolves to tlb_flush. */
924 if (bits
< TARGET_PAGE_BITS
) {
925 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu
, idxmap
);
929 /* This should already be page aligned */
930 d
.addr
= addr
& TARGET_PAGE_MASK
;
935 /* Allocate a separate data block for each destination cpu. */
936 CPU_FOREACH(dst_cpu
) {
937 if (dst_cpu
!= src_cpu
) {
938 p
= g_memdup(&d
, sizeof(d
));
939 async_run_on_cpu(dst_cpu
, tlb_flush_range_by_mmuidx_async_1
,
940 RUN_ON_CPU_HOST_PTR(p
));
944 p
= g_memdup(&d
, sizeof(d
));
945 async_safe_run_on_cpu(src_cpu
, tlb_flush_range_by_mmuidx_async_1
,
946 RUN_ON_CPU_HOST_PTR(p
));
949 void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
954 tlb_flush_range_by_mmuidx_all_cpus_synced(src_cpu
, addr
, TARGET_PAGE_SIZE
,
958 /* update the TLBs so that writes to code in the virtual page 'addr'
960 void tlb_protect_code(ram_addr_t ram_addr
)
962 cpu_physical_memory_test_and_clear_dirty(ram_addr
& TARGET_PAGE_MASK
,
967 /* update the TLB so that writes in physical page 'phys_addr' are no longer
968 tested for self modifying code */
969 void tlb_unprotect_code(ram_addr_t ram_addr
)
971 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
);
976 * Dirty write flag handling
978 * When the TCG code writes to a location it looks up the address in
979 * the TLB and uses that data to compute the final address. If any of
980 * the lower bits of the address are set then the slow path is forced.
981 * There are a number of reasons to do this but for normal RAM the
982 * most usual is detecting writes to code regions which may invalidate
985 * Other vCPUs might be reading their TLBs during guest execution, so we update
986 * te->addr_write with qatomic_set. We don't need to worry about this for
987 * oversized guests as MTTCG is disabled for them.
989 * Called with tlb_c.lock held.
991 static void tlb_reset_dirty_range_locked(CPUTLBEntry
*tlb_entry
,
992 uintptr_t start
, uintptr_t length
)
994 uintptr_t addr
= tlb_entry
->addr_write
;
996 if ((addr
& (TLB_INVALID_MASK
| TLB_MMIO
|
997 TLB_DISCARD_WRITE
| TLB_NOTDIRTY
)) == 0) {
998 addr
&= TARGET_PAGE_MASK
;
999 addr
+= tlb_entry
->addend
;
1000 if ((addr
- start
) < length
) {
1001 #if TARGET_LONG_BITS == 32
1002 uint32_t *ptr_write
= (uint32_t *)&tlb_entry
->addr_write
;
1003 ptr_write
+= HOST_BIG_ENDIAN
;
1004 qatomic_set(ptr_write
, *ptr_write
| TLB_NOTDIRTY
);
1005 #elif TCG_OVERSIZED_GUEST
1006 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
1008 qatomic_set(&tlb_entry
->addr_write
,
1009 tlb_entry
->addr_write
| TLB_NOTDIRTY
);
1016 * Called with tlb_c.lock held.
1017 * Called only from the vCPU context, i.e. the TLB's owner thread.
1019 static inline void copy_tlb_helper_locked(CPUTLBEntry
*d
, const CPUTLBEntry
*s
)
1024 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
1026 * We must take tlb_c.lock to avoid racing with another vCPU update. The only
1027 * thing actually updated is the target TLB entry ->addr_write flags.
1029 void tlb_reset_dirty(CPUState
*cpu
, ram_addr_t start1
, ram_addr_t length
)
1036 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
1037 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1039 unsigned int n
= tlb_n_entries(&env_tlb(env
)->f
[mmu_idx
]);
1041 for (i
= 0; i
< n
; i
++) {
1042 tlb_reset_dirty_range_locked(&env_tlb(env
)->f
[mmu_idx
].table
[i
],
1046 for (i
= 0; i
< CPU_VTLB_SIZE
; i
++) {
1047 tlb_reset_dirty_range_locked(&env_tlb(env
)->d
[mmu_idx
].vtable
[i
],
1051 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
1054 /* Called with tlb_c.lock held */
1055 static inline void tlb_set_dirty1_locked(CPUTLBEntry
*tlb_entry
,
1058 if (tlb_entry
->addr_write
== (addr
| TLB_NOTDIRTY
)) {
1059 tlb_entry
->addr_write
= addr
;
1063 /* update the TLB corresponding to virtual page vaddr
1064 so that it is no longer dirty */
1065 void tlb_set_dirty(CPUState
*cpu
, vaddr addr
)
1067 CPUArchState
*env
= cpu
->env_ptr
;
1070 assert_cpu_is_self(cpu
);
1072 addr
&= TARGET_PAGE_MASK
;
1073 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
1074 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1075 tlb_set_dirty1_locked(tlb_entry(env
, mmu_idx
, addr
), addr
);
1078 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1080 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
1081 tlb_set_dirty1_locked(&env_tlb(env
)->d
[mmu_idx
].vtable
[k
], addr
);
1084 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
1087 /* Our TLB does not support large pages, so remember the area covered by
1088 large pages and trigger a full TLB flush if these are invalidated. */
1089 static void tlb_add_large_page(CPUArchState
*env
, int mmu_idx
,
1090 vaddr addr
, uint64_t size
)
1092 vaddr lp_addr
= env_tlb(env
)->d
[mmu_idx
].large_page_addr
;
1093 vaddr lp_mask
= ~(size
- 1);
1095 if (lp_addr
== (vaddr
)-1) {
1096 /* No previous large page. */
1099 /* Extend the existing region to include the new page.
1100 This is a compromise between unnecessary flushes and
1101 the cost of maintaining a full variable size TLB. */
1102 lp_mask
&= env_tlb(env
)->d
[mmu_idx
].large_page_mask
;
1103 while (((lp_addr
^ addr
) & lp_mask
) != 0) {
1107 env_tlb(env
)->d
[mmu_idx
].large_page_addr
= lp_addr
& lp_mask
;
1108 env_tlb(env
)->d
[mmu_idx
].large_page_mask
= lp_mask
;
1111 static inline void tlb_set_compare(CPUTLBEntryFull
*full
, CPUTLBEntry
*ent
,
1112 vaddr address
, int flags
,
1113 MMUAccessType access_type
, bool enable
)
1116 address
|= flags
& TLB_FLAGS_MASK
;
1117 flags
&= TLB_SLOW_FLAGS_MASK
;
1119 address
|= TLB_FORCE_SLOW
;
1125 ent
->addr_idx
[access_type
] = address
;
1126 full
->slow_flags
[access_type
] = flags
;
1130 * Add a new TLB entry. At most one entry for a given virtual address
1131 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
1132 * supplied size is only used by tlb_flush_page.
1134 * Called from TCG-generated code, which is under an RCU read-side
1137 void tlb_set_page_full(CPUState
*cpu
, int mmu_idx
,
1138 vaddr addr
, CPUTLBEntryFull
*full
)
1140 CPUArchState
*env
= cpu
->env_ptr
;
1141 CPUTLB
*tlb
= env_tlb(env
);
1142 CPUTLBDesc
*desc
= &tlb
->d
[mmu_idx
];
1143 MemoryRegionSection
*section
;
1144 unsigned int index
, read_flags
, write_flags
;
1146 CPUTLBEntry
*te
, tn
;
1147 hwaddr iotlb
, xlat
, sz
, paddr_page
;
1149 int asidx
, wp_flags
, prot
;
1150 bool is_ram
, is_romd
;
1152 assert_cpu_is_self(cpu
);
1154 if (full
->lg_page_size
<= TARGET_PAGE_BITS
) {
1155 sz
= TARGET_PAGE_SIZE
;
1157 sz
= (hwaddr
)1 << full
->lg_page_size
;
1158 tlb_add_large_page(env
, mmu_idx
, addr
, sz
);
1160 addr_page
= addr
& TARGET_PAGE_MASK
;
1161 paddr_page
= full
->phys_addr
& TARGET_PAGE_MASK
;
1164 asidx
= cpu_asidx_from_attrs(cpu
, full
->attrs
);
1165 section
= address_space_translate_for_iotlb(cpu
, asidx
, paddr_page
,
1166 &xlat
, &sz
, full
->attrs
, &prot
);
1167 assert(sz
>= TARGET_PAGE_SIZE
);
1169 tlb_debug("vaddr=%016" VADDR_PRIx
" paddr=0x" HWADDR_FMT_plx
1170 " prot=%x idx=%d\n",
1171 addr
, full
->phys_addr
, prot
, mmu_idx
);
1174 if (full
->lg_page_size
< TARGET_PAGE_BITS
) {
1175 /* Repeat the MMU check and TLB fill on every access. */
1176 read_flags
|= TLB_INVALID_MASK
;
1178 if (full
->attrs
.byte_swap
) {
1179 read_flags
|= TLB_BSWAP
;
1182 is_ram
= memory_region_is_ram(section
->mr
);
1183 is_romd
= memory_region_is_romd(section
->mr
);
1185 if (is_ram
|| is_romd
) {
1186 /* RAM and ROMD both have associated host memory. */
1187 addend
= (uintptr_t)memory_region_get_ram_ptr(section
->mr
) + xlat
;
1189 /* I/O does not; force the host address to NULL. */
1193 write_flags
= read_flags
;
1195 iotlb
= memory_region_get_ram_addr(section
->mr
) + xlat
;
1196 assert(!(iotlb
& ~TARGET_PAGE_MASK
));
1198 * Computing is_clean is expensive; avoid all that unless
1199 * the page is actually writable.
1201 if (prot
& PAGE_WRITE
) {
1202 if (section
->readonly
) {
1203 write_flags
|= TLB_DISCARD_WRITE
;
1204 } else if (cpu_physical_memory_is_clean(iotlb
)) {
1205 write_flags
|= TLB_NOTDIRTY
;
1210 iotlb
= memory_region_section_get_iotlb(cpu
, section
) + xlat
;
1212 * Writes to romd devices must go through MMIO to enable write.
1213 * Reads to romd devices go through the ram_ptr found above,
1214 * but of course reads to I/O must go through MMIO.
1216 write_flags
|= TLB_MMIO
;
1218 read_flags
= write_flags
;
1222 wp_flags
= cpu_watchpoint_address_matches(cpu
, addr_page
,
1225 index
= tlb_index(env
, mmu_idx
, addr_page
);
1226 te
= tlb_entry(env
, mmu_idx
, addr_page
);
1229 * Hold the TLB lock for the rest of the function. We could acquire/release
1230 * the lock several times in the function, but it is faster to amortize the
1231 * acquisition cost by acquiring it just once. Note that this leads to
1232 * a longer critical section, but this is not a concern since the TLB lock
1233 * is unlikely to be contended.
1235 qemu_spin_lock(&tlb
->c
.lock
);
1237 /* Note that the tlb is no longer clean. */
1238 tlb
->c
.dirty
|= 1 << mmu_idx
;
1240 /* Make sure there's no cached translation for the new page. */
1241 tlb_flush_vtlb_page_locked(env
, mmu_idx
, addr_page
);
1244 * Only evict the old entry to the victim tlb if it's for a
1245 * different page; otherwise just overwrite the stale data.
1247 if (!tlb_hit_page_anyprot(te
, addr_page
) && !tlb_entry_is_empty(te
)) {
1248 unsigned vidx
= desc
->vindex
++ % CPU_VTLB_SIZE
;
1249 CPUTLBEntry
*tv
= &desc
->vtable
[vidx
];
1251 /* Evict the old entry into the victim tlb. */
1252 copy_tlb_helper_locked(tv
, te
);
1253 desc
->vfulltlb
[vidx
] = desc
->fulltlb
[index
];
1254 tlb_n_used_entries_dec(env
, mmu_idx
);
1257 /* refill the tlb */
1259 * When memory region is ram, iotlb contains a TARGET_PAGE_BITS
1260 * aligned ram_addr_t of the page base of the target RAM.
1261 * Otherwise, iotlb contains
1262 * - a physical section number in the lower TARGET_PAGE_BITS
1263 * - the offset within section->mr of the page base (I/O, ROMD) with the
1264 * TARGET_PAGE_BITS masked off.
1265 * We subtract addr_page (which is page aligned and thus won't
1266 * disturb the low bits) to give an offset which can be added to the
1267 * (non-page-aligned) vaddr of the eventual memory access to get
1268 * the MemoryRegion offset for the access. Note that the vaddr we
1269 * subtract here is that of the page base, and not the same as the
1270 * vaddr we add back in io_prepare()/get_page_addr_code().
1272 desc
->fulltlb
[index
] = *full
;
1273 full
= &desc
->fulltlb
[index
];
1274 full
->xlat_section
= iotlb
- addr_page
;
1275 full
->phys_addr
= paddr_page
;
1277 /* Now calculate the new entry */
1278 tn
.addend
= addend
- addr_page
;
1280 tlb_set_compare(full
, &tn
, addr_page
, read_flags
,
1281 MMU_INST_FETCH
, prot
& PAGE_EXEC
);
1283 if (wp_flags
& BP_MEM_READ
) {
1284 read_flags
|= TLB_WATCHPOINT
;
1286 tlb_set_compare(full
, &tn
, addr_page
, read_flags
,
1287 MMU_DATA_LOAD
, prot
& PAGE_READ
);
1289 if (prot
& PAGE_WRITE_INV
) {
1290 write_flags
|= TLB_INVALID_MASK
;
1292 if (wp_flags
& BP_MEM_WRITE
) {
1293 write_flags
|= TLB_WATCHPOINT
;
1295 tlb_set_compare(full
, &tn
, addr_page
, write_flags
,
1296 MMU_DATA_STORE
, prot
& PAGE_WRITE
);
1298 copy_tlb_helper_locked(te
, &tn
);
1299 tlb_n_used_entries_inc(env
, mmu_idx
);
1300 qemu_spin_unlock(&tlb
->c
.lock
);
1303 void tlb_set_page_with_attrs(CPUState
*cpu
, vaddr addr
,
1304 hwaddr paddr
, MemTxAttrs attrs
, int prot
,
1305 int mmu_idx
, uint64_t size
)
1307 CPUTLBEntryFull full
= {
1311 .lg_page_size
= ctz64(size
)
1314 assert(is_power_of_2(size
));
1315 tlb_set_page_full(cpu
, mmu_idx
, addr
, &full
);
1318 void tlb_set_page(CPUState
*cpu
, vaddr addr
,
1319 hwaddr paddr
, int prot
,
1320 int mmu_idx
, uint64_t size
)
1322 tlb_set_page_with_attrs(cpu
, addr
, paddr
, MEMTXATTRS_UNSPECIFIED
,
1323 prot
, mmu_idx
, size
);
1327 * Note: tlb_fill() can trigger a resize of the TLB. This means that all of the
1328 * caller's prior references to the TLB table (e.g. CPUTLBEntry pointers) must
1329 * be discarded and looked up again (e.g. via tlb_entry()).
1331 static void tlb_fill(CPUState
*cpu
, vaddr addr
, int size
,
1332 MMUAccessType access_type
, int mmu_idx
, uintptr_t retaddr
)
1337 * This is not a probe, so only valid return is success; failure
1338 * should result in exception + longjmp to the cpu loop.
1340 ok
= cpu
->cc
->tcg_ops
->tlb_fill(cpu
, addr
, size
,
1341 access_type
, mmu_idx
, false, retaddr
);
1345 static inline void cpu_unaligned_access(CPUState
*cpu
, vaddr addr
,
1346 MMUAccessType access_type
,
1347 int mmu_idx
, uintptr_t retaddr
)
1349 cpu
->cc
->tcg_ops
->do_unaligned_access(cpu
, addr
, access_type
,
1353 static MemoryRegionSection
*
1354 io_prepare(hwaddr
*out_offset
, CPUArchState
*env
, hwaddr xlat
,
1355 MemTxAttrs attrs
, vaddr addr
, uintptr_t retaddr
)
1357 CPUState
*cpu
= env_cpu(env
);
1358 MemoryRegionSection
*section
;
1361 section
= iotlb_to_section(cpu
, xlat
, attrs
);
1362 mr_offset
= (xlat
& TARGET_PAGE_MASK
) + addr
;
1363 cpu
->mem_io_pc
= retaddr
;
1364 if (!cpu
->can_do_io
) {
1365 cpu_io_recompile(cpu
, retaddr
);
1368 *out_offset
= mr_offset
;
1372 static void io_failed(CPUArchState
*env
, CPUTLBEntryFull
*full
, vaddr addr
,
1373 unsigned size
, MMUAccessType access_type
, int mmu_idx
,
1374 MemTxResult response
, uintptr_t retaddr
)
1376 CPUState
*cpu
= env_cpu(env
);
1378 if (!cpu
->ignore_memory_transaction_failures
) {
1379 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
1381 if (cc
->tcg_ops
->do_transaction_failed
) {
1382 hwaddr physaddr
= full
->phys_addr
| (addr
& ~TARGET_PAGE_MASK
);
1384 cc
->tcg_ops
->do_transaction_failed(cpu
, physaddr
, addr
, size
,
1385 access_type
, mmu_idx
,
1386 full
->attrs
, response
, retaddr
);
1391 /* Return true if ADDR is present in the victim tlb, and has been copied
1392 back to the main tlb. */
1393 static bool victim_tlb_hit(CPUArchState
*env
, size_t mmu_idx
, size_t index
,
1394 MMUAccessType access_type
, vaddr page
)
1398 assert_cpu_is_self(env_cpu(env
));
1399 for (vidx
= 0; vidx
< CPU_VTLB_SIZE
; ++vidx
) {
1400 CPUTLBEntry
*vtlb
= &env_tlb(env
)->d
[mmu_idx
].vtable
[vidx
];
1401 uint64_t cmp
= tlb_read_idx(vtlb
, access_type
);
1404 /* Found entry in victim tlb, swap tlb and iotlb. */
1405 CPUTLBEntry tmptlb
, *tlb
= &env_tlb(env
)->f
[mmu_idx
].table
[index
];
1407 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
1408 copy_tlb_helper_locked(&tmptlb
, tlb
);
1409 copy_tlb_helper_locked(tlb
, vtlb
);
1410 copy_tlb_helper_locked(vtlb
, &tmptlb
);
1411 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
1413 CPUTLBEntryFull
*f1
= &env_tlb(env
)->d
[mmu_idx
].fulltlb
[index
];
1414 CPUTLBEntryFull
*f2
= &env_tlb(env
)->d
[mmu_idx
].vfulltlb
[vidx
];
1415 CPUTLBEntryFull tmpf
;
1416 tmpf
= *f1
; *f1
= *f2
; *f2
= tmpf
;
1423 static void notdirty_write(CPUState
*cpu
, vaddr mem_vaddr
, unsigned size
,
1424 CPUTLBEntryFull
*full
, uintptr_t retaddr
)
1426 ram_addr_t ram_addr
= mem_vaddr
+ full
->xlat_section
;
1428 trace_memory_notdirty_write_access(mem_vaddr
, ram_addr
, size
);
1430 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1431 tb_invalidate_phys_range_fast(ram_addr
, size
, retaddr
);
1435 * Set both VGA and migration bits for simplicity and to remove
1436 * the notdirty callback faster.
1438 cpu_physical_memory_set_dirty_range(ram_addr
, size
, DIRTY_CLIENTS_NOCODE
);
1440 /* We remove the notdirty callback only if the code has been flushed. */
1441 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1442 trace_memory_notdirty_set_dirty(mem_vaddr
);
1443 tlb_set_dirty(cpu
, mem_vaddr
);
1447 static int probe_access_internal(CPUArchState
*env
, vaddr addr
,
1448 int fault_size
, MMUAccessType access_type
,
1449 int mmu_idx
, bool nonfault
,
1450 void **phost
, CPUTLBEntryFull
**pfull
,
1451 uintptr_t retaddr
, bool check_mem_cbs
)
1453 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1454 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1455 uint64_t tlb_addr
= tlb_read_idx(entry
, access_type
);
1456 vaddr page_addr
= addr
& TARGET_PAGE_MASK
;
1457 int flags
= TLB_FLAGS_MASK
& ~TLB_FORCE_SLOW
;
1458 bool force_mmio
= check_mem_cbs
&& cpu_plugin_mem_cbs_enabled(env_cpu(env
));
1459 CPUTLBEntryFull
*full
;
1461 if (!tlb_hit_page(tlb_addr
, page_addr
)) {
1462 if (!victim_tlb_hit(env
, mmu_idx
, index
, access_type
, page_addr
)) {
1463 CPUState
*cs
= env_cpu(env
);
1465 if (!cs
->cc
->tcg_ops
->tlb_fill(cs
, addr
, fault_size
, access_type
,
1466 mmu_idx
, nonfault
, retaddr
)) {
1467 /* Non-faulting page table read failed. */
1470 return TLB_INVALID_MASK
;
1473 /* TLB resize via tlb_fill may have moved the entry. */
1474 index
= tlb_index(env
, mmu_idx
, addr
);
1475 entry
= tlb_entry(env
, mmu_idx
, addr
);
1478 * With PAGE_WRITE_INV, we set TLB_INVALID_MASK immediately,
1479 * to force the next access through tlb_fill. We've just
1480 * called tlb_fill, so we know that this entry *is* valid.
1482 flags
&= ~TLB_INVALID_MASK
;
1484 tlb_addr
= tlb_read_idx(entry
, access_type
);
1488 *pfull
= full
= &env_tlb(env
)->d
[mmu_idx
].fulltlb
[index
];
1489 flags
|= full
->slow_flags
[access_type
];
1491 /* Fold all "mmio-like" bits into TLB_MMIO. This is not RAM. */
1492 if (unlikely(flags
& ~(TLB_WATCHPOINT
| TLB_NOTDIRTY
))
1494 (access_type
!= MMU_INST_FETCH
&& force_mmio
)) {
1499 /* Everything else is RAM. */
1500 *phost
= (void *)((uintptr_t)addr
+ entry
->addend
);
1504 int probe_access_full(CPUArchState
*env
, vaddr addr
, int size
,
1505 MMUAccessType access_type
, int mmu_idx
,
1506 bool nonfault
, void **phost
, CPUTLBEntryFull
**pfull
,
1509 int flags
= probe_access_internal(env
, addr
, size
, access_type
, mmu_idx
,
1510 nonfault
, phost
, pfull
, retaddr
, true);
1512 /* Handle clean RAM pages. */
1513 if (unlikely(flags
& TLB_NOTDIRTY
)) {
1514 notdirty_write(env_cpu(env
), addr
, 1, *pfull
, retaddr
);
1515 flags
&= ~TLB_NOTDIRTY
;
1521 int probe_access_full_mmu(CPUArchState
*env
, vaddr addr
, int size
,
1522 MMUAccessType access_type
, int mmu_idx
,
1523 void **phost
, CPUTLBEntryFull
**pfull
)
1525 void *discard_phost
;
1526 CPUTLBEntryFull
*discard_tlb
;
1528 /* privately handle users that don't need full results */
1529 phost
= phost
? phost
: &discard_phost
;
1530 pfull
= pfull
? pfull
: &discard_tlb
;
1532 int flags
= probe_access_internal(env
, addr
, size
, access_type
, mmu_idx
,
1533 true, phost
, pfull
, 0, false);
1535 /* Handle clean RAM pages. */
1536 if (unlikely(flags
& TLB_NOTDIRTY
)) {
1537 notdirty_write(env_cpu(env
), addr
, 1, *pfull
, 0);
1538 flags
&= ~TLB_NOTDIRTY
;
1544 int probe_access_flags(CPUArchState
*env
, vaddr addr
, int size
,
1545 MMUAccessType access_type
, int mmu_idx
,
1546 bool nonfault
, void **phost
, uintptr_t retaddr
)
1548 CPUTLBEntryFull
*full
;
1551 g_assert(-(addr
| TARGET_PAGE_MASK
) >= size
);
1553 flags
= probe_access_internal(env
, addr
, size
, access_type
, mmu_idx
,
1554 nonfault
, phost
, &full
, retaddr
, true);
1556 /* Handle clean RAM pages. */
1557 if (unlikely(flags
& TLB_NOTDIRTY
)) {
1558 notdirty_write(env_cpu(env
), addr
, 1, full
, retaddr
);
1559 flags
&= ~TLB_NOTDIRTY
;
1565 void *probe_access(CPUArchState
*env
, vaddr addr
, int size
,
1566 MMUAccessType access_type
, int mmu_idx
, uintptr_t retaddr
)
1568 CPUTLBEntryFull
*full
;
1572 g_assert(-(addr
| TARGET_PAGE_MASK
) >= size
);
1574 flags
= probe_access_internal(env
, addr
, size
, access_type
, mmu_idx
,
1575 false, &host
, &full
, retaddr
, true);
1577 /* Per the interface, size == 0 merely faults the access. */
1582 if (unlikely(flags
& (TLB_NOTDIRTY
| TLB_WATCHPOINT
))) {
1583 /* Handle watchpoints. */
1584 if (flags
& TLB_WATCHPOINT
) {
1585 int wp_access
= (access_type
== MMU_DATA_STORE
1586 ? BP_MEM_WRITE
: BP_MEM_READ
);
1587 cpu_check_watchpoint(env_cpu(env
), addr
, size
,
1588 full
->attrs
, wp_access
, retaddr
);
1591 /* Handle clean RAM pages. */
1592 if (flags
& TLB_NOTDIRTY
) {
1593 notdirty_write(env_cpu(env
), addr
, 1, full
, retaddr
);
1600 void *tlb_vaddr_to_host(CPUArchState
*env
, abi_ptr addr
,
1601 MMUAccessType access_type
, int mmu_idx
)
1603 CPUTLBEntryFull
*full
;
1607 flags
= probe_access_internal(env
, addr
, 0, access_type
,
1608 mmu_idx
, true, &host
, &full
, 0, false);
1610 /* No combination of flags are expected by the caller. */
1611 return flags
? NULL
: host
;
1615 * Return a ram_addr_t for the virtual address for execution.
1617 * Return -1 if we can't translate and execute from an entire page
1618 * of RAM. This will force us to execute by loading and translating
1619 * one insn at a time, without caching.
1621 * NOTE: This function will trigger an exception if the page is
1624 tb_page_addr_t
get_page_addr_code_hostp(CPUArchState
*env
, vaddr addr
,
1627 CPUTLBEntryFull
*full
;
1630 (void)probe_access_internal(env
, addr
, 1, MMU_INST_FETCH
,
1631 cpu_mmu_index(env
, true), false,
1632 &p
, &full
, 0, false);
1637 if (full
->lg_page_size
< TARGET_PAGE_BITS
) {
1644 return qemu_ram_addr_from_host_nofail(p
);
1647 /* Load/store with atomicity primitives. */
1648 #include "ldst_atomicity.c.inc"
1650 #ifdef CONFIG_PLUGIN
1652 * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure.
1653 * This should be a hot path as we will have just looked this path up
1654 * in the softmmu lookup code (or helper). We don't handle re-fills or
1655 * checking the victim table. This is purely informational.
1657 * The one corner case is i/o write, which can cause changes to the
1658 * address space. Those changes, and the corresponding tlb flush,
1659 * should be delayed until the next TB, so even then this ought not fail.
1660 * But check, Just in Case.
1662 bool tlb_plugin_lookup(CPUState
*cpu
, vaddr addr
, int mmu_idx
,
1663 bool is_store
, struct qemu_plugin_hwaddr
*data
)
1665 CPUArchState
*env
= cpu
->env_ptr
;
1666 CPUTLBEntry
*tlbe
= tlb_entry(env
, mmu_idx
, addr
);
1667 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1668 MMUAccessType access_type
= is_store
? MMU_DATA_STORE
: MMU_DATA_LOAD
;
1669 uint64_t tlb_addr
= tlb_read_idx(tlbe
, access_type
);
1670 CPUTLBEntryFull
*full
;
1672 if (unlikely(!tlb_hit(tlb_addr
, addr
))) {
1676 full
= &env_tlb(env
)->d
[mmu_idx
].fulltlb
[index
];
1677 data
->phys_addr
= full
->phys_addr
| (addr
& ~TARGET_PAGE_MASK
);
1679 /* We must have an iotlb entry for MMIO */
1680 if (tlb_addr
& TLB_MMIO
) {
1681 MemoryRegionSection
*section
=
1682 iotlb_to_section(cpu
, full
->xlat_section
& ~TARGET_PAGE_MASK
,
1685 data
->mr
= section
->mr
;
1687 data
->is_io
= false;
1695 * Probe for a load/store operation.
1696 * Return the host address and into @flags.
1699 typedef struct MMULookupPageData
{
1700 CPUTLBEntryFull
*full
;
1705 } MMULookupPageData
;
1707 typedef struct MMULookupLocals
{
1708 MMULookupPageData page
[2];
1714 * mmu_lookup1: translate one page
1716 * @data: lookup parameters
1717 * @mmu_idx: virtual address context
1718 * @access_type: load/store/code
1719 * @ra: return address into tcg generated code, or 0
1721 * Resolve the translation for the one page at @data.addr, filling in
1722 * the rest of @data with the results. If the translation fails,
1723 * tlb_fill will longjmp out. Return true if the softmmu tlb for
1724 * @mmu_idx may have resized.
1726 static bool mmu_lookup1(CPUArchState
*env
, MMULookupPageData
*data
,
1727 int mmu_idx
, MMUAccessType access_type
, uintptr_t ra
)
1729 vaddr addr
= data
->addr
;
1730 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1731 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1732 uint64_t tlb_addr
= tlb_read_idx(entry
, access_type
);
1733 bool maybe_resized
= false;
1734 CPUTLBEntryFull
*full
;
1737 /* If the TLB entry is for a different page, reload and try again. */
1738 if (!tlb_hit(tlb_addr
, addr
)) {
1739 if (!victim_tlb_hit(env
, mmu_idx
, index
, access_type
,
1740 addr
& TARGET_PAGE_MASK
)) {
1741 tlb_fill(env_cpu(env
), addr
, data
->size
, access_type
, mmu_idx
, ra
);
1742 maybe_resized
= true;
1743 index
= tlb_index(env
, mmu_idx
, addr
);
1744 entry
= tlb_entry(env
, mmu_idx
, addr
);
1746 tlb_addr
= tlb_read_idx(entry
, access_type
) & ~TLB_INVALID_MASK
;
1749 full
= &env_tlb(env
)->d
[mmu_idx
].fulltlb
[index
];
1750 flags
= tlb_addr
& (TLB_FLAGS_MASK
& ~TLB_FORCE_SLOW
);
1751 flags
|= full
->slow_flags
[access_type
];
1754 data
->flags
= flags
;
1755 /* Compute haddr speculatively; depending on flags it might be invalid. */
1756 data
->haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
1758 return maybe_resized
;
1762 * mmu_watch_or_dirty
1764 * @data: lookup parameters
1765 * @access_type: load/store/code
1766 * @ra: return address into tcg generated code, or 0
1768 * Trigger watchpoints for @data.addr:@data.size;
1769 * record writes to protected clean pages.
1771 static void mmu_watch_or_dirty(CPUArchState
*env
, MMULookupPageData
*data
,
1772 MMUAccessType access_type
, uintptr_t ra
)
1774 CPUTLBEntryFull
*full
= data
->full
;
1775 vaddr addr
= data
->addr
;
1776 int flags
= data
->flags
;
1777 int size
= data
->size
;
1779 /* On watchpoint hit, this will longjmp out. */
1780 if (flags
& TLB_WATCHPOINT
) {
1781 int wp
= access_type
== MMU_DATA_STORE
? BP_MEM_WRITE
: BP_MEM_READ
;
1782 cpu_check_watchpoint(env_cpu(env
), addr
, size
, full
->attrs
, wp
, ra
);
1783 flags
&= ~TLB_WATCHPOINT
;
1786 /* Note that notdirty is only set for writes. */
1787 if (flags
& TLB_NOTDIRTY
) {
1788 notdirty_write(env_cpu(env
), addr
, size
, full
, ra
);
1789 flags
&= ~TLB_NOTDIRTY
;
1791 data
->flags
= flags
;
1795 * mmu_lookup: translate page(s)
1797 * @addr: virtual address
1798 * @oi: combined mmu_idx and MemOp
1799 * @ra: return address into tcg generated code, or 0
1800 * @access_type: load/store/code
1803 * Resolve the translation for the page(s) beginning at @addr, for MemOp.size
1804 * bytes. Return true if the lookup crosses a page boundary.
1806 static bool mmu_lookup(CPUArchState
*env
, vaddr addr
, MemOpIdx oi
,
1807 uintptr_t ra
, MMUAccessType type
, MMULookupLocals
*l
)
1813 l
->memop
= get_memop(oi
);
1814 l
->mmu_idx
= get_mmuidx(oi
);
1816 tcg_debug_assert(l
->mmu_idx
< NB_MMU_MODES
);
1818 /* Handle CPU specific unaligned behaviour */
1819 a_bits
= get_alignment_bits(l
->memop
);
1820 if (addr
& ((1 << a_bits
) - 1)) {
1821 cpu_unaligned_access(env_cpu(env
), addr
, type
, l
->mmu_idx
, ra
);
1824 l
->page
[0].addr
= addr
;
1825 l
->page
[0].size
= memop_size(l
->memop
);
1826 l
->page
[1].addr
= (addr
+ l
->page
[0].size
- 1) & TARGET_PAGE_MASK
;
1827 l
->page
[1].size
= 0;
1828 crosspage
= (addr
^ l
->page
[1].addr
) & TARGET_PAGE_MASK
;
1830 if (likely(!crosspage
)) {
1831 mmu_lookup1(env
, &l
->page
[0], l
->mmu_idx
, type
, ra
);
1833 flags
= l
->page
[0].flags
;
1834 if (unlikely(flags
& (TLB_WATCHPOINT
| TLB_NOTDIRTY
))) {
1835 mmu_watch_or_dirty(env
, &l
->page
[0], type
, ra
);
1837 if (unlikely(flags
& TLB_BSWAP
)) {
1838 l
->memop
^= MO_BSWAP
;
1841 /* Finish compute of page crossing. */
1842 int size0
= l
->page
[1].addr
- addr
;
1843 l
->page
[1].size
= l
->page
[0].size
- size0
;
1844 l
->page
[0].size
= size0
;
1847 * Lookup both pages, recognizing exceptions from either. If the
1848 * second lookup potentially resized, refresh first CPUTLBEntryFull.
1850 mmu_lookup1(env
, &l
->page
[0], l
->mmu_idx
, type
, ra
);
1851 if (mmu_lookup1(env
, &l
->page
[1], l
->mmu_idx
, type
, ra
)) {
1852 uintptr_t index
= tlb_index(env
, l
->mmu_idx
, addr
);
1853 l
->page
[0].full
= &env_tlb(env
)->d
[l
->mmu_idx
].fulltlb
[index
];
1856 flags
= l
->page
[0].flags
| l
->page
[1].flags
;
1857 if (unlikely(flags
& (TLB_WATCHPOINT
| TLB_NOTDIRTY
))) {
1858 mmu_watch_or_dirty(env
, &l
->page
[0], type
, ra
);
1859 mmu_watch_or_dirty(env
, &l
->page
[1], type
, ra
);
1863 * Since target/sparc is the only user of TLB_BSWAP, and all
1864 * Sparc accesses are aligned, any treatment across two pages
1865 * would be arbitrary. Refuse it until there's a use.
1867 tcg_debug_assert((flags
& TLB_BSWAP
) == 0);
1874 * Probe for an atomic operation. Do not allow unaligned operations,
1875 * or io operations to proceed. Return the host address.
1877 static void *atomic_mmu_lookup(CPUArchState
*env
, vaddr addr
, MemOpIdx oi
,
1878 int size
, uintptr_t retaddr
)
1880 uintptr_t mmu_idx
= get_mmuidx(oi
);
1881 MemOp mop
= get_memop(oi
);
1882 int a_bits
= get_alignment_bits(mop
);
1887 CPUTLBEntryFull
*full
;
1889 tcg_debug_assert(mmu_idx
< NB_MMU_MODES
);
1891 /* Adjust the given return address. */
1892 retaddr
-= GETPC_ADJ
;
1894 /* Enforce guest required alignment. */
1895 if (unlikely(a_bits
> 0 && (addr
& ((1 << a_bits
) - 1)))) {
1896 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
1897 cpu_unaligned_access(env_cpu(env
), addr
, MMU_DATA_STORE
,
1901 /* Enforce qemu required alignment. */
1902 if (unlikely(addr
& (size
- 1))) {
1903 /* We get here if guest alignment was not requested,
1904 or was not enforced by cpu_unaligned_access above.
1905 We might widen the access and emulate, but for now
1906 mark an exception and exit the cpu loop. */
1907 goto stop_the_world
;
1910 index
= tlb_index(env
, mmu_idx
, addr
);
1911 tlbe
= tlb_entry(env
, mmu_idx
, addr
);
1913 /* Check TLB entry and enforce page permissions. */
1914 tlb_addr
= tlb_addr_write(tlbe
);
1915 if (!tlb_hit(tlb_addr
, addr
)) {
1916 if (!victim_tlb_hit(env
, mmu_idx
, index
, MMU_DATA_STORE
,
1917 addr
& TARGET_PAGE_MASK
)) {
1918 tlb_fill(env_cpu(env
), addr
, size
,
1919 MMU_DATA_STORE
, mmu_idx
, retaddr
);
1920 index
= tlb_index(env
, mmu_idx
, addr
);
1921 tlbe
= tlb_entry(env
, mmu_idx
, addr
);
1923 tlb_addr
= tlb_addr_write(tlbe
) & ~TLB_INVALID_MASK
;
1927 * Let the guest notice RMW on a write-only page.
1928 * We have just verified that the page is writable.
1929 * Subpage lookups may have left TLB_INVALID_MASK set,
1930 * but addr_read will only be -1 if PAGE_READ was unset.
1932 if (unlikely(tlbe
->addr_read
== -1)) {
1933 tlb_fill(env_cpu(env
), addr
, size
, MMU_DATA_LOAD
, mmu_idx
, retaddr
);
1935 * Since we don't support reads and writes to different
1936 * addresses, and we do have the proper page loaded for
1937 * write, this shouldn't ever return. But just in case,
1938 * handle via stop-the-world.
1940 goto stop_the_world
;
1942 /* Collect tlb flags for read. */
1943 tlb_addr
|= tlbe
->addr_read
;
1945 /* Notice an IO access or a needs-MMU-lookup access */
1946 if (unlikely(tlb_addr
& (TLB_MMIO
| TLB_DISCARD_WRITE
))) {
1947 /* There's really nothing that can be done to
1948 support this apart from stop-the-world. */
1949 goto stop_the_world
;
1952 hostaddr
= (void *)((uintptr_t)addr
+ tlbe
->addend
);
1953 full
= &env_tlb(env
)->d
[mmu_idx
].fulltlb
[index
];
1955 if (unlikely(tlb_addr
& TLB_NOTDIRTY
)) {
1956 notdirty_write(env_cpu(env
), addr
, size
, full
, retaddr
);
1959 if (unlikely(tlb_addr
& TLB_FORCE_SLOW
)) {
1962 if (full
->slow_flags
[MMU_DATA_STORE
] & TLB_WATCHPOINT
) {
1963 wp_flags
|= BP_MEM_WRITE
;
1965 if (full
->slow_flags
[MMU_DATA_LOAD
] & TLB_WATCHPOINT
) {
1966 wp_flags
|= BP_MEM_READ
;
1969 cpu_check_watchpoint(env_cpu(env
), addr
, size
,
1970 full
->attrs
, wp_flags
, retaddr
);
1977 cpu_loop_exit_atomic(env_cpu(env
), retaddr
);
1983 * We support two different access types. SOFTMMU_CODE_ACCESS is
1984 * specifically for reading instructions from system memory. It is
1985 * called by the translation loop and in some helpers where the code
1986 * is disassembled. It shouldn't be called directly by guest code.
1988 * For the benefit of TCG generated code, we want to avoid the
1989 * complication of ABI-specific return type promotion and always
1990 * return a value extended to the register size of the host. This is
1991 * tcg_target_long, except in the case of a 32-bit host and 64-bit
1992 * data, and for that we always have uint64_t.
1994 * We don't bother with this widened value for SOFTMMU_CODE_ACCESS.
2000 * @full: page parameters
2001 * @ret_be: accumulated data
2002 * @addr: virtual address
2003 * @size: number of bytes
2004 * @mmu_idx: virtual address context
2005 * @ra: return address into tcg generated code, or 0
2006 * Context: iothread lock held
2008 * Load @size bytes from @addr, which is memory-mapped i/o.
2009 * The bytes are concatenated in big-endian order with @ret_be.
2011 static uint64_t do_ld_mmio_beN(CPUArchState
*env
, CPUTLBEntryFull
*full
,
2012 uint64_t ret_be
, vaddr addr
, int size
,
2013 int mmu_idx
, MMUAccessType type
, uintptr_t ra
)
2015 MemoryRegionSection
*section
;
2020 tcg_debug_assert(size
> 0 && size
<= 8);
2022 attrs
= full
->attrs
;
2023 section
= io_prepare(&mr_offset
, env
, full
->xlat_section
, attrs
, addr
, ra
);
2032 /* Read aligned pieces up to 8 bytes. */
2033 this_mop
= ctz32(size
| (int)addr
| 8);
2034 this_size
= 1 << this_mop
;
2037 r
= memory_region_dispatch_read(mr
, mr_offset
, &val
, this_mop
, attrs
);
2038 if (unlikely(r
!= MEMTX_OK
)) {
2039 io_failed(env
, full
, addr
, this_size
, type
, mmu_idx
, r
, ra
);
2041 if (this_size
== 8) {
2045 ret_be
= (ret_be
<< (this_size
* 8)) | val
;
2047 mr_offset
+= this_size
;
2056 * @p: translation parameters
2057 * @ret_be: accumulated data
2059 * Load @p->size bytes from @p->haddr, which is RAM.
2060 * The bytes to concatenated in big-endian order with @ret_be.
2062 static uint64_t do_ld_bytes_beN(MMULookupPageData
*p
, uint64_t ret_be
)
2064 uint8_t *haddr
= p
->haddr
;
2065 int i
, size
= p
->size
;
2067 for (i
= 0; i
< size
; i
++) {
2068 ret_be
= (ret_be
<< 8) | haddr
[i
];
2075 * @p: translation parameters
2076 * @ret_be: accumulated data
2078 * As do_ld_bytes_beN, but atomically on each aligned part.
2080 static uint64_t do_ld_parts_beN(MMULookupPageData
*p
, uint64_t ret_be
)
2082 void *haddr
= p
->haddr
;
2090 * Find minimum of alignment and size.
2091 * This is slightly stronger than required by MO_ATOM_SUBALIGN, which
2092 * would have only checked the low bits of addr|size once at the start,
2093 * but is just as easy.
2095 switch (((uintptr_t)haddr
| size
) & 7) {
2097 x
= cpu_to_be32(load_atomic4(haddr
));
2098 ret_be
= (ret_be
<< 32) | x
;
2103 x
= cpu_to_be16(load_atomic2(haddr
));
2104 ret_be
= (ret_be
<< 16) | x
;
2108 x
= *(uint8_t *)haddr
;
2109 ret_be
= (ret_be
<< 8) | x
;
2113 g_assert_not_reached();
2117 } while (size
!= 0);
2123 * @p: translation parameters
2124 * @ret_be: accumulated data
2126 * As do_ld_bytes_beN, but with one atomic load.
2127 * Four aligned bytes are guaranteed to cover the load.
2129 static uint64_t do_ld_whole_be4(MMULookupPageData
*p
, uint64_t ret_be
)
2131 int o
= p
->addr
& 3;
2132 uint32_t x
= load_atomic4(p
->haddr
- o
);
2136 x
>>= (4 - p
->size
) * 8;
2137 return (ret_be
<< (p
->size
* 8)) | x
;
2142 * @p: translation parameters
2143 * @ret_be: accumulated data
2145 * As do_ld_bytes_beN, but with one atomic load.
2146 * Eight aligned bytes are guaranteed to cover the load.
2148 static uint64_t do_ld_whole_be8(CPUArchState
*env
, uintptr_t ra
,
2149 MMULookupPageData
*p
, uint64_t ret_be
)
2151 int o
= p
->addr
& 7;
2152 uint64_t x
= load_atomic8_or_exit(env
, ra
, p
->haddr
- o
);
2156 x
>>= (8 - p
->size
) * 8;
2157 return (ret_be
<< (p
->size
* 8)) | x
;
2162 * @p: translation parameters
2163 * @ret_be: accumulated data
2165 * As do_ld_bytes_beN, but with one atomic load.
2166 * 16 aligned bytes are guaranteed to cover the load.
2168 static Int128
do_ld_whole_be16(CPUArchState
*env
, uintptr_t ra
,
2169 MMULookupPageData
*p
, uint64_t ret_be
)
2171 int o
= p
->addr
& 15;
2172 Int128 x
, y
= load_atomic16_or_exit(env
, ra
, p
->haddr
- o
);
2175 if (!HOST_BIG_ENDIAN
) {
2178 y
= int128_lshift(y
, o
* 8);
2179 y
= int128_urshift(y
, (16 - size
) * 8);
2180 x
= int128_make64(ret_be
);
2181 x
= int128_lshift(x
, size
* 8);
2182 return int128_or(x
, y
);
2186 * Wrapper for the above.
2188 static uint64_t do_ld_beN(CPUArchState
*env
, MMULookupPageData
*p
,
2189 uint64_t ret_be
, int mmu_idx
, MMUAccessType type
,
2190 MemOp mop
, uintptr_t ra
)
2193 unsigned tmp
, half_size
;
2195 if (unlikely(p
->flags
& TLB_MMIO
)) {
2196 QEMU_IOTHREAD_LOCK_GUARD();
2197 return do_ld_mmio_beN(env
, p
->full
, ret_be
, p
->addr
, p
->size
,
2202 * It is a given that we cross a page and therefore there is no
2203 * atomicity for the load as a whole, but subobjects may need attention.
2205 atom
= mop
& MO_ATOM_MASK
;
2207 case MO_ATOM_SUBALIGN
:
2208 return do_ld_parts_beN(p
, ret_be
);
2210 case MO_ATOM_IFALIGN_PAIR
:
2211 case MO_ATOM_WITHIN16_PAIR
:
2212 tmp
= mop
& MO_SIZE
;
2213 tmp
= tmp
? tmp
- 1 : 0;
2214 half_size
= 1 << tmp
;
2215 if (atom
== MO_ATOM_IFALIGN_PAIR
2216 ? p
->size
== half_size
2217 : p
->size
>= half_size
) {
2218 if (!HAVE_al8_fast
&& p
->size
< 4) {
2219 return do_ld_whole_be4(p
, ret_be
);
2221 return do_ld_whole_be8(env
, ra
, p
, ret_be
);
2226 case MO_ATOM_IFALIGN
:
2227 case MO_ATOM_WITHIN16
:
2229 return do_ld_bytes_beN(p
, ret_be
);
2232 g_assert_not_reached();
2237 * Wrapper for the above, for 8 < size < 16.
2239 static Int128
do_ld16_beN(CPUArchState
*env
, MMULookupPageData
*p
,
2240 uint64_t a
, int mmu_idx
, MemOp mop
, uintptr_t ra
)
2246 if (unlikely(p
->flags
& TLB_MMIO
)) {
2247 QEMU_IOTHREAD_LOCK_GUARD();
2248 a
= do_ld_mmio_beN(env
, p
->full
, a
, p
->addr
, size
- 8,
2249 mmu_idx
, MMU_DATA_LOAD
, ra
);
2250 b
= do_ld_mmio_beN(env
, p
->full
, 0, p
->addr
+ 8, 8,
2251 mmu_idx
, MMU_DATA_LOAD
, ra
);
2252 return int128_make128(b
, a
);
2256 * It is a given that we cross a page and therefore there is no
2257 * atomicity for the load as a whole, but subobjects may need attention.
2259 atom
= mop
& MO_ATOM_MASK
;
2261 case MO_ATOM_SUBALIGN
:
2263 a
= do_ld_parts_beN(p
, a
);
2264 p
->haddr
+= size
- 8;
2266 b
= do_ld_parts_beN(p
, 0);
2269 case MO_ATOM_WITHIN16_PAIR
:
2270 /* Since size > 8, this is the half that must be atomic. */
2271 return do_ld_whole_be16(env
, ra
, p
, a
);
2273 case MO_ATOM_IFALIGN_PAIR
:
2275 * Since size > 8, both halves are misaligned,
2276 * and so neither is atomic.
2278 case MO_ATOM_IFALIGN
:
2279 case MO_ATOM_WITHIN16
:
2282 a
= do_ld_bytes_beN(p
, a
);
2283 b
= ldq_be_p(p
->haddr
+ size
- 8);
2287 g_assert_not_reached();
2290 return int128_make128(b
, a
);
2293 static uint8_t do_ld_1(CPUArchState
*env
, MMULookupPageData
*p
, int mmu_idx
,
2294 MMUAccessType type
, uintptr_t ra
)
2296 if (unlikely(p
->flags
& TLB_MMIO
)) {
2297 QEMU_IOTHREAD_LOCK_GUARD();
2298 return do_ld_mmio_beN(env
, p
->full
, 0, p
->addr
, 1, mmu_idx
, type
, ra
);
2300 return *(uint8_t *)p
->haddr
;
2304 static uint16_t do_ld_2(CPUArchState
*env
, MMULookupPageData
*p
, int mmu_idx
,
2305 MMUAccessType type
, MemOp memop
, uintptr_t ra
)
2309 if (unlikely(p
->flags
& TLB_MMIO
)) {
2310 QEMU_IOTHREAD_LOCK_GUARD();
2311 ret
= do_ld_mmio_beN(env
, p
->full
, 0, p
->addr
, 2, mmu_idx
, type
, ra
);
2312 if ((memop
& MO_BSWAP
) == MO_LE
) {
2316 /* Perform the load host endian, then swap if necessary. */
2317 ret
= load_atom_2(env
, ra
, p
->haddr
, memop
);
2318 if (memop
& MO_BSWAP
) {
2325 static uint32_t do_ld_4(CPUArchState
*env
, MMULookupPageData
*p
, int mmu_idx
,
2326 MMUAccessType type
, MemOp memop
, uintptr_t ra
)
2330 if (unlikely(p
->flags
& TLB_MMIO
)) {
2331 QEMU_IOTHREAD_LOCK_GUARD();
2332 ret
= do_ld_mmio_beN(env
, p
->full
, 0, p
->addr
, 4, mmu_idx
, type
, ra
);
2333 if ((memop
& MO_BSWAP
) == MO_LE
) {
2337 /* Perform the load host endian. */
2338 ret
= load_atom_4(env
, ra
, p
->haddr
, memop
);
2339 if (memop
& MO_BSWAP
) {
2346 static uint64_t do_ld_8(CPUArchState
*env
, MMULookupPageData
*p
, int mmu_idx
,
2347 MMUAccessType type
, MemOp memop
, uintptr_t ra
)
2351 if (unlikely(p
->flags
& TLB_MMIO
)) {
2352 QEMU_IOTHREAD_LOCK_GUARD();
2353 ret
= do_ld_mmio_beN(env
, p
->full
, 0, p
->addr
, 8, mmu_idx
, type
, ra
);
2354 if ((memop
& MO_BSWAP
) == MO_LE
) {
2358 /* Perform the load host endian. */
2359 ret
= load_atom_8(env
, ra
, p
->haddr
, memop
);
2360 if (memop
& MO_BSWAP
) {
2367 static uint8_t do_ld1_mmu(CPUArchState
*env
, vaddr addr
, MemOpIdx oi
,
2368 uintptr_t ra
, MMUAccessType access_type
)
2373 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2374 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, access_type
, &l
);
2375 tcg_debug_assert(!crosspage
);
2377 return do_ld_1(env
, &l
.page
[0], l
.mmu_idx
, access_type
, ra
);
2380 tcg_target_ulong
helper_ldub_mmu(CPUArchState
*env
, uint64_t addr
,
2381 MemOpIdx oi
, uintptr_t retaddr
)
2383 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_8
);
2384 return do_ld1_mmu(env
, addr
, oi
, retaddr
, MMU_DATA_LOAD
);
2387 static uint16_t do_ld2_mmu(CPUArchState
*env
, vaddr addr
, MemOpIdx oi
,
2388 uintptr_t ra
, MMUAccessType access_type
)
2395 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2396 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, access_type
, &l
);
2397 if (likely(!crosspage
)) {
2398 return do_ld_2(env
, &l
.page
[0], l
.mmu_idx
, access_type
, l
.memop
, ra
);
2401 a
= do_ld_1(env
, &l
.page
[0], l
.mmu_idx
, access_type
, ra
);
2402 b
= do_ld_1(env
, &l
.page
[1], l
.mmu_idx
, access_type
, ra
);
2404 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2412 tcg_target_ulong
helper_lduw_mmu(CPUArchState
*env
, uint64_t addr
,
2413 MemOpIdx oi
, uintptr_t retaddr
)
2415 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_16
);
2416 return do_ld2_mmu(env
, addr
, oi
, retaddr
, MMU_DATA_LOAD
);
2419 static uint32_t do_ld4_mmu(CPUArchState
*env
, vaddr addr
, MemOpIdx oi
,
2420 uintptr_t ra
, MMUAccessType access_type
)
2426 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2427 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, access_type
, &l
);
2428 if (likely(!crosspage
)) {
2429 return do_ld_4(env
, &l
.page
[0], l
.mmu_idx
, access_type
, l
.memop
, ra
);
2432 ret
= do_ld_beN(env
, &l
.page
[0], 0, l
.mmu_idx
, access_type
, l
.memop
, ra
);
2433 ret
= do_ld_beN(env
, &l
.page
[1], ret
, l
.mmu_idx
, access_type
, l
.memop
, ra
);
2434 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2440 tcg_target_ulong
helper_ldul_mmu(CPUArchState
*env
, uint64_t addr
,
2441 MemOpIdx oi
, uintptr_t retaddr
)
2443 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_32
);
2444 return do_ld4_mmu(env
, addr
, oi
, retaddr
, MMU_DATA_LOAD
);
2447 static uint64_t do_ld8_mmu(CPUArchState
*env
, vaddr addr
, MemOpIdx oi
,
2448 uintptr_t ra
, MMUAccessType access_type
)
2454 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2455 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, access_type
, &l
);
2456 if (likely(!crosspage
)) {
2457 return do_ld_8(env
, &l
.page
[0], l
.mmu_idx
, access_type
, l
.memop
, ra
);
2460 ret
= do_ld_beN(env
, &l
.page
[0], 0, l
.mmu_idx
, access_type
, l
.memop
, ra
);
2461 ret
= do_ld_beN(env
, &l
.page
[1], ret
, l
.mmu_idx
, access_type
, l
.memop
, ra
);
2462 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2468 uint64_t helper_ldq_mmu(CPUArchState
*env
, uint64_t addr
,
2469 MemOpIdx oi
, uintptr_t retaddr
)
2471 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_64
);
2472 return do_ld8_mmu(env
, addr
, oi
, retaddr
, MMU_DATA_LOAD
);
2476 * Provide signed versions of the load routines as well. We can of course
2477 * avoid this for 64-bit data, or for 32-bit data on 32-bit host.
2480 tcg_target_ulong
helper_ldsb_mmu(CPUArchState
*env
, uint64_t addr
,
2481 MemOpIdx oi
, uintptr_t retaddr
)
2483 return (int8_t)helper_ldub_mmu(env
, addr
, oi
, retaddr
);
2486 tcg_target_ulong
helper_ldsw_mmu(CPUArchState
*env
, uint64_t addr
,
2487 MemOpIdx oi
, uintptr_t retaddr
)
2489 return (int16_t)helper_lduw_mmu(env
, addr
, oi
, retaddr
);
2492 tcg_target_ulong
helper_ldsl_mmu(CPUArchState
*env
, uint64_t addr
,
2493 MemOpIdx oi
, uintptr_t retaddr
)
2495 return (int32_t)helper_ldul_mmu(env
, addr
, oi
, retaddr
);
2498 static Int128
do_ld16_mmu(CPUArchState
*env
, vaddr addr
,
2499 MemOpIdx oi
, uintptr_t ra
)
2507 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2508 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_LOAD
, &l
);
2509 if (likely(!crosspage
)) {
2510 if (unlikely(l
.page
[0].flags
& TLB_MMIO
)) {
2511 QEMU_IOTHREAD_LOCK_GUARD();
2512 a
= do_ld_mmio_beN(env
, l
.page
[0].full
, 0, addr
, 8,
2513 l
.mmu_idx
, MMU_DATA_LOAD
, ra
);
2514 b
= do_ld_mmio_beN(env
, l
.page
[0].full
, 0, addr
+ 8, 8,
2515 l
.mmu_idx
, MMU_DATA_LOAD
, ra
);
2516 ret
= int128_make128(b
, a
);
2517 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2518 ret
= bswap128(ret
);
2521 /* Perform the load host endian. */
2522 ret
= load_atom_16(env
, ra
, l
.page
[0].haddr
, l
.memop
);
2523 if (l
.memop
& MO_BSWAP
) {
2524 ret
= bswap128(ret
);
2530 first
= l
.page
[0].size
;
2532 MemOp mop8
= (l
.memop
& ~MO_SIZE
) | MO_64
;
2534 a
= do_ld_8(env
, &l
.page
[0], l
.mmu_idx
, MMU_DATA_LOAD
, mop8
, ra
);
2535 b
= do_ld_8(env
, &l
.page
[1], l
.mmu_idx
, MMU_DATA_LOAD
, mop8
, ra
);
2536 if ((mop8
& MO_BSWAP
) == MO_LE
) {
2537 ret
= int128_make128(a
, b
);
2539 ret
= int128_make128(b
, a
);
2545 a
= do_ld_beN(env
, &l
.page
[0], 0, l
.mmu_idx
,
2546 MMU_DATA_LOAD
, l
.memop
, ra
);
2547 ret
= do_ld16_beN(env
, &l
.page
[1], a
, l
.mmu_idx
, l
.memop
, ra
);
2549 ret
= do_ld16_beN(env
, &l
.page
[0], 0, l
.mmu_idx
, l
.memop
, ra
);
2550 b
= int128_getlo(ret
);
2551 ret
= int128_lshift(ret
, l
.page
[1].size
* 8);
2552 a
= int128_gethi(ret
);
2553 b
= do_ld_beN(env
, &l
.page
[1], b
, l
.mmu_idx
,
2554 MMU_DATA_LOAD
, l
.memop
, ra
);
2555 ret
= int128_make128(b
, a
);
2557 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2558 ret
= bswap128(ret
);
2563 Int128
helper_ld16_mmu(CPUArchState
*env
, uint64_t addr
,
2564 uint32_t oi
, uintptr_t retaddr
)
2566 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_128
);
2567 return do_ld16_mmu(env
, addr
, oi
, retaddr
);
2570 Int128
helper_ld_i128(CPUArchState
*env
, uint64_t addr
, uint32_t oi
)
2572 return helper_ld16_mmu(env
, addr
, oi
, GETPC());
2576 * Load helpers for cpu_ldst.h.
2579 static void plugin_load_cb(CPUArchState
*env
, abi_ptr addr
, MemOpIdx oi
)
2581 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_R
);
2584 uint8_t cpu_ldb_mmu(CPUArchState
*env
, abi_ptr addr
, MemOpIdx oi
, uintptr_t ra
)
2588 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_UB
);
2589 ret
= do_ld1_mmu(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
2590 plugin_load_cb(env
, addr
, oi
);
2594 uint16_t cpu_ldw_mmu(CPUArchState
*env
, abi_ptr addr
,
2595 MemOpIdx oi
, uintptr_t ra
)
2599 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_16
);
2600 ret
= do_ld2_mmu(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
2601 plugin_load_cb(env
, addr
, oi
);
2605 uint32_t cpu_ldl_mmu(CPUArchState
*env
, abi_ptr addr
,
2606 MemOpIdx oi
, uintptr_t ra
)
2610 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_32
);
2611 ret
= do_ld4_mmu(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
2612 plugin_load_cb(env
, addr
, oi
);
2616 uint64_t cpu_ldq_mmu(CPUArchState
*env
, abi_ptr addr
,
2617 MemOpIdx oi
, uintptr_t ra
)
2621 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_64
);
2622 ret
= do_ld8_mmu(env
, addr
, oi
, ra
, MMU_DATA_LOAD
);
2623 plugin_load_cb(env
, addr
, oi
);
2627 Int128
cpu_ld16_mmu(CPUArchState
*env
, abi_ptr addr
,
2628 MemOpIdx oi
, uintptr_t ra
)
2632 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_128
);
2633 ret
= do_ld16_mmu(env
, addr
, oi
, ra
);
2634 plugin_load_cb(env
, addr
, oi
);
2645 * @full: page parameters
2646 * @val_le: data to store
2647 * @addr: virtual address
2648 * @size: number of bytes
2649 * @mmu_idx: virtual address context
2650 * @ra: return address into tcg generated code, or 0
2651 * Context: iothread lock held
2653 * Store @size bytes at @addr, which is memory-mapped i/o.
2654 * The bytes to store are extracted in little-endian order from @val_le;
2655 * return the bytes of @val_le beyond @p->size that have not been stored.
2657 static uint64_t do_st_mmio_leN(CPUArchState
*env
, CPUTLBEntryFull
*full
,
2658 uint64_t val_le
, vaddr addr
, int size
,
2659 int mmu_idx
, uintptr_t ra
)
2661 MemoryRegionSection
*section
;
2666 tcg_debug_assert(size
> 0 && size
<= 8);
2668 attrs
= full
->attrs
;
2669 section
= io_prepare(&mr_offset
, env
, full
->xlat_section
, attrs
, addr
, ra
);
2677 /* Store aligned pieces up to 8 bytes. */
2678 this_mop
= ctz32(size
| (int)addr
| 8);
2679 this_size
= 1 << this_mop
;
2682 r
= memory_region_dispatch_write(mr
, mr_offset
, val_le
,
2684 if (unlikely(r
!= MEMTX_OK
)) {
2685 io_failed(env
, full
, addr
, this_size
, MMU_DATA_STORE
,
2688 if (this_size
== 8) {
2692 val_le
>>= this_size
* 8;
2694 mr_offset
+= this_size
;
2702 * Wrapper for the above.
2704 static uint64_t do_st_leN(CPUArchState
*env
, MMULookupPageData
*p
,
2705 uint64_t val_le
, int mmu_idx
,
2706 MemOp mop
, uintptr_t ra
)
2709 unsigned tmp
, half_size
;
2711 if (unlikely(p
->flags
& TLB_MMIO
)) {
2712 QEMU_IOTHREAD_LOCK_GUARD();
2713 return do_st_mmio_leN(env
, p
->full
, val_le
, p
->addr
,
2714 p
->size
, mmu_idx
, ra
);
2715 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2716 return val_le
>> (p
->size
* 8);
2720 * It is a given that we cross a page and therefore there is no atomicity
2721 * for the store as a whole, but subobjects may need attention.
2723 atom
= mop
& MO_ATOM_MASK
;
2725 case MO_ATOM_SUBALIGN
:
2726 return store_parts_leN(p
->haddr
, p
->size
, val_le
);
2728 case MO_ATOM_IFALIGN_PAIR
:
2729 case MO_ATOM_WITHIN16_PAIR
:
2730 tmp
= mop
& MO_SIZE
;
2731 tmp
= tmp
? tmp
- 1 : 0;
2732 half_size
= 1 << tmp
;
2733 if (atom
== MO_ATOM_IFALIGN_PAIR
2734 ? p
->size
== half_size
2735 : p
->size
>= half_size
) {
2736 if (!HAVE_al8_fast
&& p
->size
<= 4) {
2737 return store_whole_le4(p
->haddr
, p
->size
, val_le
);
2738 } else if (HAVE_al8
) {
2739 return store_whole_le8(p
->haddr
, p
->size
, val_le
);
2741 cpu_loop_exit_atomic(env_cpu(env
), ra
);
2746 case MO_ATOM_IFALIGN
:
2747 case MO_ATOM_WITHIN16
:
2749 return store_bytes_leN(p
->haddr
, p
->size
, val_le
);
2752 g_assert_not_reached();
2757 * Wrapper for the above, for 8 < size < 16.
2759 static uint64_t do_st16_leN(CPUArchState
*env
, MMULookupPageData
*p
,
2760 Int128 val_le
, int mmu_idx
,
2761 MemOp mop
, uintptr_t ra
)
2766 if (unlikely(p
->flags
& TLB_MMIO
)) {
2767 QEMU_IOTHREAD_LOCK_GUARD();
2768 do_st_mmio_leN(env
, p
->full
, int128_getlo(val_le
),
2769 p
->addr
, 8, mmu_idx
, ra
);
2770 return do_st_mmio_leN(env
, p
->full
, int128_gethi(val_le
),
2771 p
->addr
+ 8, size
- 8, mmu_idx
, ra
);
2772 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2773 return int128_gethi(val_le
) >> ((size
- 8) * 8);
2777 * It is a given that we cross a page and therefore there is no atomicity
2778 * for the store as a whole, but subobjects may need attention.
2780 atom
= mop
& MO_ATOM_MASK
;
2782 case MO_ATOM_SUBALIGN
:
2783 store_parts_leN(p
->haddr
, 8, int128_getlo(val_le
));
2784 return store_parts_leN(p
->haddr
+ 8, p
->size
- 8,
2785 int128_gethi(val_le
));
2787 case MO_ATOM_WITHIN16_PAIR
:
2788 /* Since size > 8, this is the half that must be atomic. */
2789 if (!HAVE_ATOMIC128_RW
) {
2790 cpu_loop_exit_atomic(env_cpu(env
), ra
);
2792 return store_whole_le16(p
->haddr
, p
->size
, val_le
);
2794 case MO_ATOM_IFALIGN_PAIR
:
2796 * Since size > 8, both halves are misaligned,
2797 * and so neither is atomic.
2799 case MO_ATOM_IFALIGN
:
2800 case MO_ATOM_WITHIN16
:
2802 stq_le_p(p
->haddr
, int128_getlo(val_le
));
2803 return store_bytes_leN(p
->haddr
+ 8, p
->size
- 8,
2804 int128_gethi(val_le
));
2807 g_assert_not_reached();
2811 static void do_st_1(CPUArchState
*env
, MMULookupPageData
*p
, uint8_t val
,
2812 int mmu_idx
, uintptr_t ra
)
2814 if (unlikely(p
->flags
& TLB_MMIO
)) {
2815 QEMU_IOTHREAD_LOCK_GUARD();
2816 do_st_mmio_leN(env
, p
->full
, val
, p
->addr
, 1, mmu_idx
, ra
);
2817 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2820 *(uint8_t *)p
->haddr
= val
;
2824 static void do_st_2(CPUArchState
*env
, MMULookupPageData
*p
, uint16_t val
,
2825 int mmu_idx
, MemOp memop
, uintptr_t ra
)
2827 if (unlikely(p
->flags
& TLB_MMIO
)) {
2828 if ((memop
& MO_BSWAP
) != MO_LE
) {
2831 QEMU_IOTHREAD_LOCK_GUARD();
2832 do_st_mmio_leN(env
, p
->full
, val
, p
->addr
, 2, mmu_idx
, ra
);
2833 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2836 /* Swap to host endian if necessary, then store. */
2837 if (memop
& MO_BSWAP
) {
2840 store_atom_2(env
, ra
, p
->haddr
, memop
, val
);
2844 static void do_st_4(CPUArchState
*env
, MMULookupPageData
*p
, uint32_t val
,
2845 int mmu_idx
, MemOp memop
, uintptr_t ra
)
2847 if (unlikely(p
->flags
& TLB_MMIO
)) {
2848 if ((memop
& MO_BSWAP
) != MO_LE
) {
2851 QEMU_IOTHREAD_LOCK_GUARD();
2852 do_st_mmio_leN(env
, p
->full
, val
, p
->addr
, 4, mmu_idx
, ra
);
2853 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2856 /* Swap to host endian if necessary, then store. */
2857 if (memop
& MO_BSWAP
) {
2860 store_atom_4(env
, ra
, p
->haddr
, memop
, val
);
2864 static void do_st_8(CPUArchState
*env
, MMULookupPageData
*p
, uint64_t val
,
2865 int mmu_idx
, MemOp memop
, uintptr_t ra
)
2867 if (unlikely(p
->flags
& TLB_MMIO
)) {
2868 if ((memop
& MO_BSWAP
) != MO_LE
) {
2871 QEMU_IOTHREAD_LOCK_GUARD();
2872 do_st_mmio_leN(env
, p
->full
, val
, p
->addr
, 8, mmu_idx
, ra
);
2873 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2876 /* Swap to host endian if necessary, then store. */
2877 if (memop
& MO_BSWAP
) {
2880 store_atom_8(env
, ra
, p
->haddr
, memop
, val
);
2884 void helper_stb_mmu(CPUArchState
*env
, uint64_t addr
, uint32_t val
,
2885 MemOpIdx oi
, uintptr_t ra
)
2890 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_8
);
2891 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2892 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2893 tcg_debug_assert(!crosspage
);
2895 do_st_1(env
, &l
.page
[0], val
, l
.mmu_idx
, ra
);
2898 static void do_st2_mmu(CPUArchState
*env
, vaddr addr
, uint16_t val
,
2899 MemOpIdx oi
, uintptr_t ra
)
2905 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2906 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2907 if (likely(!crosspage
)) {
2908 do_st_2(env
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2912 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2913 a
= val
, b
= val
>> 8;
2915 b
= val
, a
= val
>> 8;
2917 do_st_1(env
, &l
.page
[0], a
, l
.mmu_idx
, ra
);
2918 do_st_1(env
, &l
.page
[1], b
, l
.mmu_idx
, ra
);
2921 void helper_stw_mmu(CPUArchState
*env
, uint64_t addr
, uint32_t val
,
2922 MemOpIdx oi
, uintptr_t retaddr
)
2924 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_16
);
2925 do_st2_mmu(env
, addr
, val
, oi
, retaddr
);
2928 static void do_st4_mmu(CPUArchState
*env
, vaddr addr
, uint32_t val
,
2929 MemOpIdx oi
, uintptr_t ra
)
2934 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2935 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2936 if (likely(!crosspage
)) {
2937 do_st_4(env
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2941 /* Swap to little endian for simplicity, then store by bytes. */
2942 if ((l
.memop
& MO_BSWAP
) != MO_LE
) {
2945 val
= do_st_leN(env
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2946 (void) do_st_leN(env
, &l
.page
[1], val
, l
.mmu_idx
, l
.memop
, ra
);
2949 void helper_stl_mmu(CPUArchState
*env
, uint64_t addr
, uint32_t val
,
2950 MemOpIdx oi
, uintptr_t retaddr
)
2952 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_32
);
2953 do_st4_mmu(env
, addr
, val
, oi
, retaddr
);
2956 static void do_st8_mmu(CPUArchState
*env
, vaddr addr
, uint64_t val
,
2957 MemOpIdx oi
, uintptr_t ra
)
2962 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2963 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2964 if (likely(!crosspage
)) {
2965 do_st_8(env
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2969 /* Swap to little endian for simplicity, then store by bytes. */
2970 if ((l
.memop
& MO_BSWAP
) != MO_LE
) {
2973 val
= do_st_leN(env
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2974 (void) do_st_leN(env
, &l
.page
[1], val
, l
.mmu_idx
, l
.memop
, ra
);
2977 void helper_stq_mmu(CPUArchState
*env
, uint64_t addr
, uint64_t val
,
2978 MemOpIdx oi
, uintptr_t retaddr
)
2980 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_64
);
2981 do_st8_mmu(env
, addr
, val
, oi
, retaddr
);
2984 static void do_st16_mmu(CPUArchState
*env
, vaddr addr
, Int128 val
,
2985 MemOpIdx oi
, uintptr_t ra
)
2992 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2993 crosspage
= mmu_lookup(env
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2994 if (likely(!crosspage
)) {
2995 if (unlikely(l
.page
[0].flags
& TLB_MMIO
)) {
2996 if ((l
.memop
& MO_BSWAP
) != MO_LE
) {
2997 val
= bswap128(val
);
2999 a
= int128_getlo(val
);
3000 b
= int128_gethi(val
);
3001 QEMU_IOTHREAD_LOCK_GUARD();
3002 do_st_mmio_leN(env
, l
.page
[0].full
, a
, addr
, 8, l
.mmu_idx
, ra
);
3003 do_st_mmio_leN(env
, l
.page
[0].full
, b
, addr
+ 8, 8, l
.mmu_idx
, ra
);
3004 } else if (unlikely(l
.page
[0].flags
& TLB_DISCARD_WRITE
)) {
3007 /* Swap to host endian if necessary, then store. */
3008 if (l
.memop
& MO_BSWAP
) {
3009 val
= bswap128(val
);
3011 store_atom_16(env
, ra
, l
.page
[0].haddr
, l
.memop
, val
);
3016 first
= l
.page
[0].size
;
3018 MemOp mop8
= (l
.memop
& ~(MO_SIZE
| MO_BSWAP
)) | MO_64
;
3020 if (l
.memop
& MO_BSWAP
) {
3021 val
= bswap128(val
);
3023 if (HOST_BIG_ENDIAN
) {
3024 b
= int128_getlo(val
), a
= int128_gethi(val
);
3026 a
= int128_getlo(val
), b
= int128_gethi(val
);
3028 do_st_8(env
, &l
.page
[0], a
, l
.mmu_idx
, mop8
, ra
);
3029 do_st_8(env
, &l
.page
[1], b
, l
.mmu_idx
, mop8
, ra
);
3033 if ((l
.memop
& MO_BSWAP
) != MO_LE
) {
3034 val
= bswap128(val
);
3037 do_st_leN(env
, &l
.page
[0], int128_getlo(val
), l
.mmu_idx
, l
.memop
, ra
);
3038 val
= int128_urshift(val
, first
* 8);
3039 do_st16_leN(env
, &l
.page
[1], val
, l
.mmu_idx
, l
.memop
, ra
);
3041 b
= do_st16_leN(env
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
3042 do_st_leN(env
, &l
.page
[1], b
, l
.mmu_idx
, l
.memop
, ra
);
3046 void helper_st16_mmu(CPUArchState
*env
, uint64_t addr
, Int128 val
,
3047 MemOpIdx oi
, uintptr_t retaddr
)
3049 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_128
);
3050 do_st16_mmu(env
, addr
, val
, oi
, retaddr
);
3053 void helper_st_i128(CPUArchState
*env
, uint64_t addr
, Int128 val
, MemOpIdx oi
)
3055 helper_st16_mmu(env
, addr
, val
, oi
, GETPC());
3059 * Store Helpers for cpu_ldst.h
3062 static void plugin_store_cb(CPUArchState
*env
, abi_ptr addr
, MemOpIdx oi
)
3064 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, oi
, QEMU_PLUGIN_MEM_W
);
3067 void cpu_stb_mmu(CPUArchState
*env
, abi_ptr addr
, uint8_t val
,
3068 MemOpIdx oi
, uintptr_t retaddr
)
3070 helper_stb_mmu(env
, addr
, val
, oi
, retaddr
);
3071 plugin_store_cb(env
, addr
, oi
);
3074 void cpu_stw_mmu(CPUArchState
*env
, abi_ptr addr
, uint16_t val
,
3075 MemOpIdx oi
, uintptr_t retaddr
)
3077 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_16
);
3078 do_st2_mmu(env
, addr
, val
, oi
, retaddr
);
3079 plugin_store_cb(env
, addr
, oi
);
3082 void cpu_stl_mmu(CPUArchState
*env
, abi_ptr addr
, uint32_t val
,
3083 MemOpIdx oi
, uintptr_t retaddr
)
3085 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_32
);
3086 do_st4_mmu(env
, addr
, val
, oi
, retaddr
);
3087 plugin_store_cb(env
, addr
, oi
);
3090 void cpu_stq_mmu(CPUArchState
*env
, abi_ptr addr
, uint64_t val
,
3091 MemOpIdx oi
, uintptr_t retaddr
)
3093 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_64
);
3094 do_st8_mmu(env
, addr
, val
, oi
, retaddr
);
3095 plugin_store_cb(env
, addr
, oi
);
3098 void cpu_st16_mmu(CPUArchState
*env
, abi_ptr addr
, Int128 val
,
3099 MemOpIdx oi
, uintptr_t retaddr
)
3101 tcg_debug_assert((get_memop(oi
) & MO_SIZE
) == MO_128
);
3102 do_st16_mmu(env
, addr
, val
, oi
, retaddr
);
3103 plugin_store_cb(env
, addr
, oi
);
3106 #include "ldst_common.c.inc"
3109 * First set of functions passes in OI and RETADDR.
3110 * This makes them callable from other helpers.
3113 #define ATOMIC_NAME(X) \
3114 glue(glue(glue(cpu_atomic_ ## X, SUFFIX), END), _mmu)
3116 #define ATOMIC_MMU_CLEANUP
3118 #include "atomic_common.c.inc"
3121 #include "atomic_template.h"
3124 #include "atomic_template.h"
3127 #include "atomic_template.h"
3129 #ifdef CONFIG_ATOMIC64
3131 #include "atomic_template.h"
3134 #if defined(CONFIG_ATOMIC128) || HAVE_CMPXCHG128
3135 #define DATA_SIZE 16
3136 #include "atomic_template.h"
3139 /* Code access functions. */
3141 uint32_t cpu_ldub_code(CPUArchState
*env
, abi_ptr addr
)
3143 MemOpIdx oi
= make_memop_idx(MO_UB
, cpu_mmu_index(env
, true));
3144 return do_ld1_mmu(env
, addr
, oi
, 0, MMU_INST_FETCH
);
3147 uint32_t cpu_lduw_code(CPUArchState
*env
, abi_ptr addr
)
3149 MemOpIdx oi
= make_memop_idx(MO_TEUW
, cpu_mmu_index(env
, true));
3150 return do_ld2_mmu(env
, addr
, oi
, 0, MMU_INST_FETCH
);
3153 uint32_t cpu_ldl_code(CPUArchState
*env
, abi_ptr addr
)
3155 MemOpIdx oi
= make_memop_idx(MO_TEUL
, cpu_mmu_index(env
, true));
3156 return do_ld4_mmu(env
, addr
, oi
, 0, MMU_INST_FETCH
);
3159 uint64_t cpu_ldq_code(CPUArchState
*env
, abi_ptr addr
)
3161 MemOpIdx oi
= make_memop_idx(MO_TEUQ
, cpu_mmu_index(env
, true));
3162 return do_ld8_mmu(env
, addr
, oi
, 0, MMU_INST_FETCH
);
3165 uint8_t cpu_ldb_code_mmu(CPUArchState
*env
, abi_ptr addr
,
3166 MemOpIdx oi
, uintptr_t retaddr
)
3168 return do_ld1_mmu(env
, addr
, oi
, retaddr
, MMU_INST_FETCH
);
3171 uint16_t cpu_ldw_code_mmu(CPUArchState
*env
, abi_ptr addr
,
3172 MemOpIdx oi
, uintptr_t retaddr
)
3174 return do_ld2_mmu(env
, addr
, oi
, retaddr
, MMU_INST_FETCH
);
3177 uint32_t cpu_ldl_code_mmu(CPUArchState
*env
, abi_ptr addr
,
3178 MemOpIdx oi
, uintptr_t retaddr
)
3180 return do_ld4_mmu(env
, addr
, oi
, retaddr
, MMU_INST_FETCH
);
3183 uint64_t cpu_ldq_code_mmu(CPUArchState
*env
, abi_ptr addr
,
3184 MemOpIdx oi
, uintptr_t retaddr
)
3186 return do_ld8_mmu(env
, addr
, oi
, retaddr
, MMU_INST_FETCH
);