2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/address-spaces.h"
26 #include "exec/cpu_ldst.h"
27 #include "exec/cputlb.h"
28 #include "exec/memory-internal.h"
29 #include "exec/ram_addr.h"
31 #include "qemu/error-report.h"
33 #include "exec/helper-proto.h"
34 #include "qemu/atomic.h"
35 #include "qemu/atomic128.h"
36 #include "translate-all.h"
37 #include "trace-root.h"
38 #include "qemu/plugin.h"
39 #include "trace/mem.h"
41 #include "qemu/plugin-memory.h"
44 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
45 /* #define DEBUG_TLB */
46 /* #define DEBUG_TLB_LOG */
49 # define DEBUG_TLB_GATE 1
51 # define DEBUG_TLB_LOG_GATE 1
53 # define DEBUG_TLB_LOG_GATE 0
56 # define DEBUG_TLB_GATE 0
57 # define DEBUG_TLB_LOG_GATE 0
60 #define tlb_debug(fmt, ...) do { \
61 if (DEBUG_TLB_LOG_GATE) { \
62 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
64 } else if (DEBUG_TLB_GATE) { \
65 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
69 #define assert_cpu_is_self(cpu) do { \
70 if (DEBUG_TLB_GATE) { \
71 g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \
75 /* run_on_cpu_data.target_ptr should always be big enough for a
76 * target_ulong even on 32 bit builds */
77 QEMU_BUILD_BUG_ON(sizeof(target_ulong
) > sizeof(run_on_cpu_data
));
79 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
81 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> 16);
82 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
84 static inline size_t sizeof_tlb(CPUArchState
*env
, uintptr_t mmu_idx
)
86 return env_tlb(env
)->f
[mmu_idx
].mask
+ (1 << CPU_TLB_ENTRY_BITS
);
89 static void tlb_window_reset(CPUTLBDesc
*desc
, int64_t ns
,
92 desc
->window_begin_ns
= ns
;
93 desc
->window_max_entries
= max_entries
;
96 static void tlb_dyn_init(CPUArchState
*env
)
100 for (i
= 0; i
< NB_MMU_MODES
; i
++) {
101 CPUTLBDesc
*desc
= &env_tlb(env
)->d
[i
];
102 size_t n_entries
= 1 << CPU_TLB_DYN_DEFAULT_BITS
;
104 tlb_window_reset(desc
, get_clock_realtime(), 0);
105 desc
->n_used_entries
= 0;
106 env_tlb(env
)->f
[i
].mask
= (n_entries
- 1) << CPU_TLB_ENTRY_BITS
;
107 env_tlb(env
)->f
[i
].table
= g_new(CPUTLBEntry
, n_entries
);
108 env_tlb(env
)->d
[i
].iotlb
= g_new(CPUIOTLBEntry
, n_entries
);
113 * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary
114 * @env: CPU that owns the TLB
115 * @mmu_idx: MMU index of the TLB
117 * Called with tlb_lock_held.
119 * We have two main constraints when resizing a TLB: (1) we only resize it
120 * on a TLB flush (otherwise we'd have to take a perf hit by either rehashing
121 * the array or unnecessarily flushing it), which means we do not control how
122 * frequently the resizing can occur; (2) we don't have access to the guest's
123 * future scheduling decisions, and therefore have to decide the magnitude of
124 * the resize based on past observations.
126 * In general, a memory-hungry process can benefit greatly from an appropriately
127 * sized TLB, since a guest TLB miss is very expensive. This doesn't mean that
128 * we just have to make the TLB as large as possible; while an oversized TLB
129 * results in minimal TLB miss rates, it also takes longer to be flushed
130 * (flushes can be _very_ frequent), and the reduced locality can also hurt
133 * To achieve near-optimal performance for all kinds of workloads, we:
135 * 1. Aggressively increase the size of the TLB when the use rate of the
136 * TLB being flushed is high, since it is likely that in the near future this
137 * memory-hungry process will execute again, and its memory hungriness will
138 * probably be similar.
140 * 2. Slowly reduce the size of the TLB as the use rate declines over a
141 * reasonably large time window. The rationale is that if in such a time window
142 * we have not observed a high TLB use rate, it is likely that we won't observe
143 * it in the near future. In that case, once a time window expires we downsize
144 * the TLB to match the maximum use rate observed in the window.
146 * 3. Try to keep the maximum use rate in a time window in the 30-70% range,
147 * since in that range performance is likely near-optimal. Recall that the TLB
148 * is direct mapped, so we want the use rate to be low (or at least not too
149 * high), since otherwise we are likely to have a significant amount of
152 static void tlb_mmu_resize_locked(CPUArchState
*env
, int mmu_idx
)
154 CPUTLBDesc
*desc
= &env_tlb(env
)->d
[mmu_idx
];
155 size_t old_size
= tlb_n_entries(env
, mmu_idx
);
157 size_t new_size
= old_size
;
158 int64_t now
= get_clock_realtime();
159 int64_t window_len_ms
= 100;
160 int64_t window_len_ns
= window_len_ms
* 1000 * 1000;
161 bool window_expired
= now
> desc
->window_begin_ns
+ window_len_ns
;
163 if (desc
->n_used_entries
> desc
->window_max_entries
) {
164 desc
->window_max_entries
= desc
->n_used_entries
;
166 rate
= desc
->window_max_entries
* 100 / old_size
;
169 new_size
= MIN(old_size
<< 1, 1 << CPU_TLB_DYN_MAX_BITS
);
170 } else if (rate
< 30 && window_expired
) {
171 size_t ceil
= pow2ceil(desc
->window_max_entries
);
172 size_t expected_rate
= desc
->window_max_entries
* 100 / ceil
;
175 * Avoid undersizing when the max number of entries seen is just below
176 * a pow2. For instance, if max_entries == 1025, the expected use rate
177 * would be 1025/2048==50%. However, if max_entries == 1023, we'd get
178 * 1023/1024==99.9% use rate, so we'd likely end up doubling the size
179 * later. Thus, make sure that the expected use rate remains below 70%.
180 * (and since we double the size, that means the lowest rate we'd
181 * expect to get is 35%, which is still in the 30-70% range where
182 * we consider that the size is appropriate.)
184 if (expected_rate
> 70) {
187 new_size
= MAX(ceil
, 1 << CPU_TLB_DYN_MIN_BITS
);
190 if (new_size
== old_size
) {
191 if (window_expired
) {
192 tlb_window_reset(desc
, now
, desc
->n_used_entries
);
197 g_free(env_tlb(env
)->f
[mmu_idx
].table
);
198 g_free(env_tlb(env
)->d
[mmu_idx
].iotlb
);
200 tlb_window_reset(desc
, now
, 0);
201 /* desc->n_used_entries is cleared by the caller */
202 env_tlb(env
)->f
[mmu_idx
].mask
= (new_size
- 1) << CPU_TLB_ENTRY_BITS
;
203 env_tlb(env
)->f
[mmu_idx
].table
= g_try_new(CPUTLBEntry
, new_size
);
204 env_tlb(env
)->d
[mmu_idx
].iotlb
= g_try_new(CPUIOTLBEntry
, new_size
);
206 * If the allocations fail, try smaller sizes. We just freed some
207 * memory, so going back to half of new_size has a good chance of working.
208 * Increased memory pressure elsewhere in the system might cause the
209 * allocations to fail though, so we progressively reduce the allocation
210 * size, aborting if we cannot even allocate the smallest TLB we support.
212 while (env_tlb(env
)->f
[mmu_idx
].table
== NULL
||
213 env_tlb(env
)->d
[mmu_idx
].iotlb
== NULL
) {
214 if (new_size
== (1 << CPU_TLB_DYN_MIN_BITS
)) {
215 error_report("%s: %s", __func__
, strerror(errno
));
218 new_size
= MAX(new_size
>> 1, 1 << CPU_TLB_DYN_MIN_BITS
);
219 env_tlb(env
)->f
[mmu_idx
].mask
= (new_size
- 1) << CPU_TLB_ENTRY_BITS
;
221 g_free(env_tlb(env
)->f
[mmu_idx
].table
);
222 g_free(env_tlb(env
)->d
[mmu_idx
].iotlb
);
223 env_tlb(env
)->f
[mmu_idx
].table
= g_try_new(CPUTLBEntry
, new_size
);
224 env_tlb(env
)->d
[mmu_idx
].iotlb
= g_try_new(CPUIOTLBEntry
, new_size
);
228 static inline void tlb_table_flush_by_mmuidx(CPUArchState
*env
, int mmu_idx
)
230 tlb_mmu_resize_locked(env
, mmu_idx
);
231 memset(env_tlb(env
)->f
[mmu_idx
].table
, -1, sizeof_tlb(env
, mmu_idx
));
232 env_tlb(env
)->d
[mmu_idx
].n_used_entries
= 0;
235 static inline void tlb_n_used_entries_inc(CPUArchState
*env
, uintptr_t mmu_idx
)
237 env_tlb(env
)->d
[mmu_idx
].n_used_entries
++;
240 static inline void tlb_n_used_entries_dec(CPUArchState
*env
, uintptr_t mmu_idx
)
242 env_tlb(env
)->d
[mmu_idx
].n_used_entries
--;
245 void tlb_init(CPUState
*cpu
)
247 CPUArchState
*env
= cpu
->env_ptr
;
249 qemu_spin_init(&env_tlb(env
)->c
.lock
);
251 /* Ensure that cpu_reset performs a full flush. */
252 env_tlb(env
)->c
.dirty
= ALL_MMUIDX_BITS
;
257 /* flush_all_helper: run fn across all cpus
259 * If the wait flag is set then the src cpu's helper will be queued as
260 * "safe" work and the loop exited creating a synchronisation point
261 * where all queued work will be finished before execution starts
264 static void flush_all_helper(CPUState
*src
, run_on_cpu_func fn
,
271 async_run_on_cpu(cpu
, fn
, d
);
276 void tlb_flush_counts(size_t *pfull
, size_t *ppart
, size_t *pelide
)
279 size_t full
= 0, part
= 0, elide
= 0;
282 CPUArchState
*env
= cpu
->env_ptr
;
284 full
+= atomic_read(&env_tlb(env
)->c
.full_flush_count
);
285 part
+= atomic_read(&env_tlb(env
)->c
.part_flush_count
);
286 elide
+= atomic_read(&env_tlb(env
)->c
.elide_flush_count
);
293 static void tlb_flush_one_mmuidx_locked(CPUArchState
*env
, int mmu_idx
)
295 tlb_table_flush_by_mmuidx(env
, mmu_idx
);
296 env_tlb(env
)->d
[mmu_idx
].large_page_addr
= -1;
297 env_tlb(env
)->d
[mmu_idx
].large_page_mask
= -1;
298 env_tlb(env
)->d
[mmu_idx
].vindex
= 0;
299 memset(env_tlb(env
)->d
[mmu_idx
].vtable
, -1,
300 sizeof(env_tlb(env
)->d
[0].vtable
));
303 static void tlb_flush_by_mmuidx_async_work(CPUState
*cpu
, run_on_cpu_data data
)
305 CPUArchState
*env
= cpu
->env_ptr
;
306 uint16_t asked
= data
.host_int
;
307 uint16_t all_dirty
, work
, to_clean
;
309 assert_cpu_is_self(cpu
);
311 tlb_debug("mmu_idx:0x%04" PRIx16
"\n", asked
);
313 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
315 all_dirty
= env_tlb(env
)->c
.dirty
;
316 to_clean
= asked
& all_dirty
;
317 all_dirty
&= ~to_clean
;
318 env_tlb(env
)->c
.dirty
= all_dirty
;
320 for (work
= to_clean
; work
!= 0; work
&= work
- 1) {
321 int mmu_idx
= ctz32(work
);
322 tlb_flush_one_mmuidx_locked(env
, mmu_idx
);
325 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
327 cpu_tb_jmp_cache_clear(cpu
);
329 if (to_clean
== ALL_MMUIDX_BITS
) {
330 atomic_set(&env_tlb(env
)->c
.full_flush_count
,
331 env_tlb(env
)->c
.full_flush_count
+ 1);
333 atomic_set(&env_tlb(env
)->c
.part_flush_count
,
334 env_tlb(env
)->c
.part_flush_count
+ ctpop16(to_clean
));
335 if (to_clean
!= asked
) {
336 atomic_set(&env_tlb(env
)->c
.elide_flush_count
,
337 env_tlb(env
)->c
.elide_flush_count
+
338 ctpop16(asked
& ~to_clean
));
343 void tlb_flush_by_mmuidx(CPUState
*cpu
, uint16_t idxmap
)
345 tlb_debug("mmu_idx: 0x%" PRIx16
"\n", idxmap
);
347 if (cpu
->created
&& !qemu_cpu_is_self(cpu
)) {
348 async_run_on_cpu(cpu
, tlb_flush_by_mmuidx_async_work
,
349 RUN_ON_CPU_HOST_INT(idxmap
));
351 tlb_flush_by_mmuidx_async_work(cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
355 void tlb_flush(CPUState
*cpu
)
357 tlb_flush_by_mmuidx(cpu
, ALL_MMUIDX_BITS
);
360 void tlb_flush_by_mmuidx_all_cpus(CPUState
*src_cpu
, uint16_t idxmap
)
362 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
364 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
366 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
367 fn(src_cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
370 void tlb_flush_all_cpus(CPUState
*src_cpu
)
372 tlb_flush_by_mmuidx_all_cpus(src_cpu
, ALL_MMUIDX_BITS
);
375 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
, uint16_t idxmap
)
377 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
379 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
381 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
382 async_safe_run_on_cpu(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
385 void tlb_flush_all_cpus_synced(CPUState
*src_cpu
)
387 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu
, ALL_MMUIDX_BITS
);
390 static inline bool tlb_hit_page_anyprot(CPUTLBEntry
*tlb_entry
,
393 return tlb_hit_page(tlb_entry
->addr_read
, page
) ||
394 tlb_hit_page(tlb_addr_write(tlb_entry
), page
) ||
395 tlb_hit_page(tlb_entry
->addr_code
, page
);
399 * tlb_entry_is_empty - return true if the entry is not in use
400 * @te: pointer to CPUTLBEntry
402 static inline bool tlb_entry_is_empty(const CPUTLBEntry
*te
)
404 return te
->addr_read
== -1 && te
->addr_write
== -1 && te
->addr_code
== -1;
407 /* Called with tlb_c.lock held */
408 static inline bool tlb_flush_entry_locked(CPUTLBEntry
*tlb_entry
,
411 if (tlb_hit_page_anyprot(tlb_entry
, page
)) {
412 memset(tlb_entry
, -1, sizeof(*tlb_entry
));
418 /* Called with tlb_c.lock held */
419 static inline void tlb_flush_vtlb_page_locked(CPUArchState
*env
, int mmu_idx
,
422 CPUTLBDesc
*d
= &env_tlb(env
)->d
[mmu_idx
];
425 assert_cpu_is_self(env_cpu(env
));
426 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
427 if (tlb_flush_entry_locked(&d
->vtable
[k
], page
)) {
428 tlb_n_used_entries_dec(env
, mmu_idx
);
433 static void tlb_flush_page_locked(CPUArchState
*env
, int midx
,
436 target_ulong lp_addr
= env_tlb(env
)->d
[midx
].large_page_addr
;
437 target_ulong lp_mask
= env_tlb(env
)->d
[midx
].large_page_mask
;
439 /* Check if we need to flush due to large pages. */
440 if ((page
& lp_mask
) == lp_addr
) {
441 tlb_debug("forcing full flush midx %d ("
442 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
443 midx
, lp_addr
, lp_mask
);
444 tlb_flush_one_mmuidx_locked(env
, midx
);
446 if (tlb_flush_entry_locked(tlb_entry(env
, midx
, page
), page
)) {
447 tlb_n_used_entries_dec(env
, midx
);
449 tlb_flush_vtlb_page_locked(env
, midx
, page
);
453 /* As we are going to hijack the bottom bits of the page address for a
454 * mmuidx bit mask we need to fail to build if we can't do that
456 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> TARGET_PAGE_BITS_MIN
);
458 static void tlb_flush_page_by_mmuidx_async_work(CPUState
*cpu
,
459 run_on_cpu_data data
)
461 CPUArchState
*env
= cpu
->env_ptr
;
462 target_ulong addr_and_mmuidx
= (target_ulong
) data
.target_ptr
;
463 target_ulong addr
= addr_and_mmuidx
& TARGET_PAGE_MASK
;
464 unsigned long mmu_idx_bitmap
= addr_and_mmuidx
& ALL_MMUIDX_BITS
;
467 assert_cpu_is_self(cpu
);
469 tlb_debug("page addr:" TARGET_FMT_lx
" mmu_map:0x%lx\n",
470 addr
, mmu_idx_bitmap
);
472 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
473 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
474 if (test_bit(mmu_idx
, &mmu_idx_bitmap
)) {
475 tlb_flush_page_locked(env
, mmu_idx
, addr
);
478 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
480 tb_flush_jmp_cache(cpu
, addr
);
483 void tlb_flush_page_by_mmuidx(CPUState
*cpu
, target_ulong addr
, uint16_t idxmap
)
485 target_ulong addr_and_mmu_idx
;
487 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%" PRIx16
"\n", addr
, idxmap
);
489 /* This should already be page aligned */
490 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
491 addr_and_mmu_idx
|= idxmap
;
493 if (!qemu_cpu_is_self(cpu
)) {
494 async_run_on_cpu(cpu
, tlb_flush_page_by_mmuidx_async_work
,
495 RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
497 tlb_flush_page_by_mmuidx_async_work(
498 cpu
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
502 void tlb_flush_page(CPUState
*cpu
, target_ulong addr
)
504 tlb_flush_page_by_mmuidx(cpu
, addr
, ALL_MMUIDX_BITS
);
507 void tlb_flush_page_by_mmuidx_all_cpus(CPUState
*src_cpu
, target_ulong addr
,
510 const run_on_cpu_func fn
= tlb_flush_page_by_mmuidx_async_work
;
511 target_ulong addr_and_mmu_idx
;
513 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
515 /* This should already be page aligned */
516 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
517 addr_and_mmu_idx
|= idxmap
;
519 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
520 fn(src_cpu
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
523 void tlb_flush_page_all_cpus(CPUState
*src
, target_ulong addr
)
525 tlb_flush_page_by_mmuidx_all_cpus(src
, addr
, ALL_MMUIDX_BITS
);
528 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
532 const run_on_cpu_func fn
= tlb_flush_page_by_mmuidx_async_work
;
533 target_ulong addr_and_mmu_idx
;
535 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
537 /* This should already be page aligned */
538 addr_and_mmu_idx
= addr
& TARGET_PAGE_MASK
;
539 addr_and_mmu_idx
|= idxmap
;
541 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
542 async_safe_run_on_cpu(src_cpu
, fn
, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx
));
545 void tlb_flush_page_all_cpus_synced(CPUState
*src
, target_ulong addr
)
547 tlb_flush_page_by_mmuidx_all_cpus_synced(src
, addr
, ALL_MMUIDX_BITS
);
550 /* update the TLBs so that writes to code in the virtual page 'addr'
552 void tlb_protect_code(ram_addr_t ram_addr
)
554 cpu_physical_memory_test_and_clear_dirty(ram_addr
, TARGET_PAGE_SIZE
,
558 /* update the TLB so that writes in physical page 'phys_addr' are no longer
559 tested for self modifying code */
560 void tlb_unprotect_code(ram_addr_t ram_addr
)
562 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
);
567 * Dirty write flag handling
569 * When the TCG code writes to a location it looks up the address in
570 * the TLB and uses that data to compute the final address. If any of
571 * the lower bits of the address are set then the slow path is forced.
572 * There are a number of reasons to do this but for normal RAM the
573 * most usual is detecting writes to code regions which may invalidate
576 * Other vCPUs might be reading their TLBs during guest execution, so we update
577 * te->addr_write with atomic_set. We don't need to worry about this for
578 * oversized guests as MTTCG is disabled for them.
580 * Called with tlb_c.lock held.
582 static void tlb_reset_dirty_range_locked(CPUTLBEntry
*tlb_entry
,
583 uintptr_t start
, uintptr_t length
)
585 uintptr_t addr
= tlb_entry
->addr_write
;
587 if ((addr
& (TLB_INVALID_MASK
| TLB_MMIO
|
588 TLB_DISCARD_WRITE
| TLB_NOTDIRTY
)) == 0) {
589 addr
&= TARGET_PAGE_MASK
;
590 addr
+= tlb_entry
->addend
;
591 if ((addr
- start
) < length
) {
592 #if TCG_OVERSIZED_GUEST
593 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
595 atomic_set(&tlb_entry
->addr_write
,
596 tlb_entry
->addr_write
| TLB_NOTDIRTY
);
603 * Called with tlb_c.lock held.
604 * Called only from the vCPU context, i.e. the TLB's owner thread.
606 static inline void copy_tlb_helper_locked(CPUTLBEntry
*d
, const CPUTLBEntry
*s
)
611 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
613 * We must take tlb_c.lock to avoid racing with another vCPU update. The only
614 * thing actually updated is the target TLB entry ->addr_write flags.
616 void tlb_reset_dirty(CPUState
*cpu
, ram_addr_t start1
, ram_addr_t length
)
623 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
624 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
626 unsigned int n
= tlb_n_entries(env
, mmu_idx
);
628 for (i
= 0; i
< n
; i
++) {
629 tlb_reset_dirty_range_locked(&env_tlb(env
)->f
[mmu_idx
].table
[i
],
633 for (i
= 0; i
< CPU_VTLB_SIZE
; i
++) {
634 tlb_reset_dirty_range_locked(&env_tlb(env
)->d
[mmu_idx
].vtable
[i
],
638 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
641 /* Called with tlb_c.lock held */
642 static inline void tlb_set_dirty1_locked(CPUTLBEntry
*tlb_entry
,
645 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
)) {
646 tlb_entry
->addr_write
= vaddr
;
650 /* update the TLB corresponding to virtual page vaddr
651 so that it is no longer dirty */
652 void tlb_set_dirty(CPUState
*cpu
, target_ulong vaddr
)
654 CPUArchState
*env
= cpu
->env_ptr
;
657 assert_cpu_is_self(cpu
);
659 vaddr
&= TARGET_PAGE_MASK
;
660 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
661 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
662 tlb_set_dirty1_locked(tlb_entry(env
, mmu_idx
, vaddr
), vaddr
);
665 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
667 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
668 tlb_set_dirty1_locked(&env_tlb(env
)->d
[mmu_idx
].vtable
[k
], vaddr
);
671 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
674 /* Our TLB does not support large pages, so remember the area covered by
675 large pages and trigger a full TLB flush if these are invalidated. */
676 static void tlb_add_large_page(CPUArchState
*env
, int mmu_idx
,
677 target_ulong vaddr
, target_ulong size
)
679 target_ulong lp_addr
= env_tlb(env
)->d
[mmu_idx
].large_page_addr
;
680 target_ulong lp_mask
= ~(size
- 1);
682 if (lp_addr
== (target_ulong
)-1) {
683 /* No previous large page. */
686 /* Extend the existing region to include the new page.
687 This is a compromise between unnecessary flushes and
688 the cost of maintaining a full variable size TLB. */
689 lp_mask
&= env_tlb(env
)->d
[mmu_idx
].large_page_mask
;
690 while (((lp_addr
^ vaddr
) & lp_mask
) != 0) {
694 env_tlb(env
)->d
[mmu_idx
].large_page_addr
= lp_addr
& lp_mask
;
695 env_tlb(env
)->d
[mmu_idx
].large_page_mask
= lp_mask
;
698 /* Add a new TLB entry. At most one entry for a given virtual address
699 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
700 * supplied size is only used by tlb_flush_page.
702 * Called from TCG-generated code, which is under an RCU read-side
705 void tlb_set_page_with_attrs(CPUState
*cpu
, target_ulong vaddr
,
706 hwaddr paddr
, MemTxAttrs attrs
, int prot
,
707 int mmu_idx
, target_ulong size
)
709 CPUArchState
*env
= cpu
->env_ptr
;
710 CPUTLB
*tlb
= env_tlb(env
);
711 CPUTLBDesc
*desc
= &tlb
->d
[mmu_idx
];
712 MemoryRegionSection
*section
;
714 target_ulong address
;
715 target_ulong write_address
;
718 hwaddr iotlb
, xlat
, sz
, paddr_page
;
719 target_ulong vaddr_page
;
720 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
722 bool is_ram
, is_romd
;
724 assert_cpu_is_self(cpu
);
726 if (size
<= TARGET_PAGE_SIZE
) {
727 sz
= TARGET_PAGE_SIZE
;
729 tlb_add_large_page(env
, mmu_idx
, vaddr
, size
);
732 vaddr_page
= vaddr
& TARGET_PAGE_MASK
;
733 paddr_page
= paddr
& TARGET_PAGE_MASK
;
735 section
= address_space_translate_for_iotlb(cpu
, asidx
, paddr_page
,
736 &xlat
, &sz
, attrs
, &prot
);
737 assert(sz
>= TARGET_PAGE_SIZE
);
739 tlb_debug("vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
741 vaddr
, paddr
, prot
, mmu_idx
);
743 address
= vaddr_page
;
744 if (size
< TARGET_PAGE_SIZE
) {
745 /* Repeat the MMU check and TLB fill on every access. */
746 address
|= TLB_INVALID_MASK
;
748 if (attrs
.byte_swap
) {
749 address
|= TLB_BSWAP
;
752 is_ram
= memory_region_is_ram(section
->mr
);
753 is_romd
= memory_region_is_romd(section
->mr
);
755 if (is_ram
|| is_romd
) {
756 /* RAM and ROMD both have associated host memory. */
757 addend
= (uintptr_t)memory_region_get_ram_ptr(section
->mr
) + xlat
;
759 /* I/O does not; force the host address to NULL. */
763 write_address
= address
;
765 iotlb
= memory_region_get_ram_addr(section
->mr
) + xlat
;
767 * Computing is_clean is expensive; avoid all that unless
768 * the page is actually writable.
770 if (prot
& PAGE_WRITE
) {
771 if (section
->readonly
) {
772 write_address
|= TLB_DISCARD_WRITE
;
773 } else if (cpu_physical_memory_is_clean(iotlb
)) {
774 write_address
|= TLB_NOTDIRTY
;
779 iotlb
= memory_region_section_get_iotlb(cpu
, section
) + xlat
;
781 * Writes to romd devices must go through MMIO to enable write.
782 * Reads to romd devices go through the ram_ptr found above,
783 * but of course reads to I/O must go through MMIO.
785 write_address
|= TLB_MMIO
;
787 address
= write_address
;
791 wp_flags
= cpu_watchpoint_address_matches(cpu
, vaddr_page
,
794 index
= tlb_index(env
, mmu_idx
, vaddr_page
);
795 te
= tlb_entry(env
, mmu_idx
, vaddr_page
);
798 * Hold the TLB lock for the rest of the function. We could acquire/release
799 * the lock several times in the function, but it is faster to amortize the
800 * acquisition cost by acquiring it just once. Note that this leads to
801 * a longer critical section, but this is not a concern since the TLB lock
802 * is unlikely to be contended.
804 qemu_spin_lock(&tlb
->c
.lock
);
806 /* Note that the tlb is no longer clean. */
807 tlb
->c
.dirty
|= 1 << mmu_idx
;
809 /* Make sure there's no cached translation for the new page. */
810 tlb_flush_vtlb_page_locked(env
, mmu_idx
, vaddr_page
);
813 * Only evict the old entry to the victim tlb if it's for a
814 * different page; otherwise just overwrite the stale data.
816 if (!tlb_hit_page_anyprot(te
, vaddr_page
) && !tlb_entry_is_empty(te
)) {
817 unsigned vidx
= desc
->vindex
++ % CPU_VTLB_SIZE
;
818 CPUTLBEntry
*tv
= &desc
->vtable
[vidx
];
820 /* Evict the old entry into the victim tlb. */
821 copy_tlb_helper_locked(tv
, te
);
822 desc
->viotlb
[vidx
] = desc
->iotlb
[index
];
823 tlb_n_used_entries_dec(env
, mmu_idx
);
828 * At this point iotlb contains a physical section number in the lower
829 * TARGET_PAGE_BITS, and either
830 * + the ram_addr_t of the page base of the target RAM (RAM)
831 * + the offset within section->mr of the page base (I/O, ROMD)
832 * We subtract the vaddr_page (which is page aligned and thus won't
833 * disturb the low bits) to give an offset which can be added to the
834 * (non-page-aligned) vaddr of the eventual memory access to get
835 * the MemoryRegion offset for the access. Note that the vaddr we
836 * subtract here is that of the page base, and not the same as the
837 * vaddr we add back in io_readx()/io_writex()/get_page_addr_code().
839 desc
->iotlb
[index
].addr
= iotlb
- vaddr_page
;
840 desc
->iotlb
[index
].attrs
= attrs
;
842 /* Now calculate the new entry */
843 tn
.addend
= addend
- vaddr_page
;
844 if (prot
& PAGE_READ
) {
845 tn
.addr_read
= address
;
846 if (wp_flags
& BP_MEM_READ
) {
847 tn
.addr_read
|= TLB_WATCHPOINT
;
853 if (prot
& PAGE_EXEC
) {
854 tn
.addr_code
= address
;
860 if (prot
& PAGE_WRITE
) {
861 tn
.addr_write
= write_address
;
862 if (prot
& PAGE_WRITE_INV
) {
863 tn
.addr_write
|= TLB_INVALID_MASK
;
865 if (wp_flags
& BP_MEM_WRITE
) {
866 tn
.addr_write
|= TLB_WATCHPOINT
;
870 copy_tlb_helper_locked(te
, &tn
);
871 tlb_n_used_entries_inc(env
, mmu_idx
);
872 qemu_spin_unlock(&tlb
->c
.lock
);
875 /* Add a new TLB entry, but without specifying the memory
876 * transaction attributes to be used.
878 void tlb_set_page(CPUState
*cpu
, target_ulong vaddr
,
879 hwaddr paddr
, int prot
,
880 int mmu_idx
, target_ulong size
)
882 tlb_set_page_with_attrs(cpu
, vaddr
, paddr
, MEMTXATTRS_UNSPECIFIED
,
883 prot
, mmu_idx
, size
);
886 static inline ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
890 ram_addr
= qemu_ram_addr_from_host(ptr
);
891 if (ram_addr
== RAM_ADDR_INVALID
) {
892 error_report("Bad ram pointer %p", ptr
);
899 * Note: tlb_fill() can trigger a resize of the TLB. This means that all of the
900 * caller's prior references to the TLB table (e.g. CPUTLBEntry pointers) must
901 * be discarded and looked up again (e.g. via tlb_entry()).
903 static void tlb_fill(CPUState
*cpu
, target_ulong addr
, int size
,
904 MMUAccessType access_type
, int mmu_idx
, uintptr_t retaddr
)
906 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
910 * This is not a probe, so only valid return is success; failure
911 * should result in exception + longjmp to the cpu loop.
913 ok
= cc
->tlb_fill(cpu
, addr
, size
, access_type
, mmu_idx
, false, retaddr
);
917 static uint64_t io_readx(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
918 int mmu_idx
, target_ulong addr
, uintptr_t retaddr
,
919 MMUAccessType access_type
, MemOp op
)
921 CPUState
*cpu
= env_cpu(env
);
923 MemoryRegionSection
*section
;
929 section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
931 mr_offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
932 cpu
->mem_io_pc
= retaddr
;
933 if (!cpu
->can_do_io
) {
934 cpu_io_recompile(cpu
, retaddr
);
937 if (mr
->global_locking
&& !qemu_mutex_iothread_locked()) {
938 qemu_mutex_lock_iothread();
941 r
= memory_region_dispatch_read(mr
, mr_offset
, &val
, op
, iotlbentry
->attrs
);
943 hwaddr physaddr
= mr_offset
+
944 section
->offset_within_address_space
-
945 section
->offset_within_region
;
947 cpu_transaction_failed(cpu
, physaddr
, addr
, memop_size(op
), access_type
,
948 mmu_idx
, iotlbentry
->attrs
, r
, retaddr
);
951 qemu_mutex_unlock_iothread();
957 static void io_writex(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
958 int mmu_idx
, uint64_t val
, target_ulong addr
,
959 uintptr_t retaddr
, MemOp op
)
961 CPUState
*cpu
= env_cpu(env
);
963 MemoryRegionSection
*section
;
968 section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
970 mr_offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
971 if (!cpu
->can_do_io
) {
972 cpu_io_recompile(cpu
, retaddr
);
974 cpu
->mem_io_pc
= retaddr
;
976 if (mr
->global_locking
&& !qemu_mutex_iothread_locked()) {
977 qemu_mutex_lock_iothread();
980 r
= memory_region_dispatch_write(mr
, mr_offset
, val
, op
, iotlbentry
->attrs
);
982 hwaddr physaddr
= mr_offset
+
983 section
->offset_within_address_space
-
984 section
->offset_within_region
;
986 cpu_transaction_failed(cpu
, physaddr
, addr
, memop_size(op
),
987 MMU_DATA_STORE
, mmu_idx
, iotlbentry
->attrs
, r
,
991 qemu_mutex_unlock_iothread();
995 static inline target_ulong
tlb_read_ofs(CPUTLBEntry
*entry
, size_t ofs
)
997 #if TCG_OVERSIZED_GUEST
998 return *(target_ulong
*)((uintptr_t)entry
+ ofs
);
1000 /* ofs might correspond to .addr_write, so use atomic_read */
1001 return atomic_read((target_ulong
*)((uintptr_t)entry
+ ofs
));
1005 /* Return true if ADDR is present in the victim tlb, and has been copied
1006 back to the main tlb. */
1007 static bool victim_tlb_hit(CPUArchState
*env
, size_t mmu_idx
, size_t index
,
1008 size_t elt_ofs
, target_ulong page
)
1012 assert_cpu_is_self(env_cpu(env
));
1013 for (vidx
= 0; vidx
< CPU_VTLB_SIZE
; ++vidx
) {
1014 CPUTLBEntry
*vtlb
= &env_tlb(env
)->d
[mmu_idx
].vtable
[vidx
];
1017 /* elt_ofs might correspond to .addr_write, so use atomic_read */
1018 #if TCG_OVERSIZED_GUEST
1019 cmp
= *(target_ulong
*)((uintptr_t)vtlb
+ elt_ofs
);
1021 cmp
= atomic_read((target_ulong
*)((uintptr_t)vtlb
+ elt_ofs
));
1025 /* Found entry in victim tlb, swap tlb and iotlb. */
1026 CPUTLBEntry tmptlb
, *tlb
= &env_tlb(env
)->f
[mmu_idx
].table
[index
];
1028 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
1029 copy_tlb_helper_locked(&tmptlb
, tlb
);
1030 copy_tlb_helper_locked(tlb
, vtlb
);
1031 copy_tlb_helper_locked(vtlb
, &tmptlb
);
1032 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
1034 CPUIOTLBEntry tmpio
, *io
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1035 CPUIOTLBEntry
*vio
= &env_tlb(env
)->d
[mmu_idx
].viotlb
[vidx
];
1036 tmpio
= *io
; *io
= *vio
; *vio
= tmpio
;
1043 /* Macro to call the above, with local variables from the use context. */
1044 #define VICTIM_TLB_HIT(TY, ADDR) \
1045 victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
1046 (ADDR) & TARGET_PAGE_MASK)
1049 * Return a ram_addr_t for the virtual address for execution.
1051 * Return -1 if we can't translate and execute from an entire page
1052 * of RAM. This will force us to execute by loading and translating
1053 * one insn at a time, without caching.
1055 * NOTE: This function will trigger an exception if the page is
1058 tb_page_addr_t
get_page_addr_code_hostp(CPUArchState
*env
, target_ulong addr
,
1061 uintptr_t mmu_idx
= cpu_mmu_index(env
, true);
1062 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1063 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1066 if (unlikely(!tlb_hit(entry
->addr_code
, addr
))) {
1067 if (!VICTIM_TLB_HIT(addr_code
, addr
)) {
1068 tlb_fill(env_cpu(env
), addr
, 0, MMU_INST_FETCH
, mmu_idx
, 0);
1069 index
= tlb_index(env
, mmu_idx
, addr
);
1070 entry
= tlb_entry(env
, mmu_idx
, addr
);
1072 if (unlikely(entry
->addr_code
& TLB_INVALID_MASK
)) {
1074 * The MMU protection covers a smaller range than a target
1075 * page, so we must redo the MMU check for every insn.
1080 assert(tlb_hit(entry
->addr_code
, addr
));
1083 if (unlikely(entry
->addr_code
& TLB_MMIO
)) {
1084 /* The region is not backed by RAM. */
1091 p
= (void *)((uintptr_t)addr
+ entry
->addend
);
1095 return qemu_ram_addr_from_host_nofail(p
);
1098 tb_page_addr_t
get_page_addr_code(CPUArchState
*env
, target_ulong addr
)
1100 return get_page_addr_code_hostp(env
, addr
, NULL
);
1103 static void notdirty_write(CPUState
*cpu
, vaddr mem_vaddr
, unsigned size
,
1104 CPUIOTLBEntry
*iotlbentry
, uintptr_t retaddr
)
1106 ram_addr_t ram_addr
= mem_vaddr
+ iotlbentry
->addr
;
1108 trace_memory_notdirty_write_access(mem_vaddr
, ram_addr
, size
);
1110 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1111 struct page_collection
*pages
1112 = page_collection_lock(ram_addr
, ram_addr
+ size
);
1113 tb_invalidate_phys_page_fast(pages
, ram_addr
, size
, retaddr
);
1114 page_collection_unlock(pages
);
1118 * Set both VGA and migration bits for simplicity and to remove
1119 * the notdirty callback faster.
1121 cpu_physical_memory_set_dirty_range(ram_addr
, size
, DIRTY_CLIENTS_NOCODE
);
1123 /* We remove the notdirty callback only if the code has been flushed. */
1124 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1125 trace_memory_notdirty_set_dirty(mem_vaddr
);
1126 tlb_set_dirty(cpu
, mem_vaddr
);
1131 * Probe for whether the specified guest access is permitted. If it is not
1132 * permitted then an exception will be taken in the same way as if this
1133 * were a real access (and we will not return).
1134 * If the size is 0 or the page requires I/O access, returns NULL; otherwise,
1135 * returns the address of the host page similar to tlb_vaddr_to_host().
1137 void *probe_access(CPUArchState
*env
, target_ulong addr
, int size
,
1138 MMUAccessType access_type
, int mmu_idx
, uintptr_t retaddr
)
1140 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1141 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1142 target_ulong tlb_addr
;
1146 g_assert(-(addr
| TARGET_PAGE_MASK
) >= size
);
1148 switch (access_type
) {
1150 elt_ofs
= offsetof(CPUTLBEntry
, addr_read
);
1151 wp_access
= BP_MEM_READ
;
1153 case MMU_DATA_STORE
:
1154 elt_ofs
= offsetof(CPUTLBEntry
, addr_write
);
1155 wp_access
= BP_MEM_WRITE
;
1157 case MMU_INST_FETCH
:
1158 elt_ofs
= offsetof(CPUTLBEntry
, addr_code
);
1159 wp_access
= BP_MEM_READ
;
1162 g_assert_not_reached();
1164 tlb_addr
= tlb_read_ofs(entry
, elt_ofs
);
1166 if (unlikely(!tlb_hit(tlb_addr
, addr
))) {
1167 if (!victim_tlb_hit(env
, mmu_idx
, index
, elt_ofs
,
1168 addr
& TARGET_PAGE_MASK
)) {
1169 tlb_fill(env_cpu(env
), addr
, size
, access_type
, mmu_idx
, retaddr
);
1170 /* TLB resize via tlb_fill may have moved the entry. */
1171 index
= tlb_index(env
, mmu_idx
, addr
);
1172 entry
= tlb_entry(env
, mmu_idx
, addr
);
1174 tlb_addr
= tlb_read_ofs(entry
, elt_ofs
);
1181 if (unlikely(tlb_addr
& TLB_FLAGS_MASK
)) {
1182 CPUIOTLBEntry
*iotlbentry
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1184 /* Reject I/O access, or other required slow-path. */
1185 if (tlb_addr
& (TLB_MMIO
| TLB_BSWAP
| TLB_DISCARD_WRITE
)) {
1189 /* Handle watchpoints. */
1190 if (tlb_addr
& TLB_WATCHPOINT
) {
1191 cpu_check_watchpoint(env_cpu(env
), addr
, size
,
1192 iotlbentry
->attrs
, wp_access
, retaddr
);
1195 /* Handle clean RAM pages. */
1196 if (tlb_addr
& TLB_NOTDIRTY
) {
1197 notdirty_write(env_cpu(env
), addr
, size
, iotlbentry
, retaddr
);
1201 return (void *)((uintptr_t)addr
+ entry
->addend
);
1204 void *tlb_vaddr_to_host(CPUArchState
*env
, abi_ptr addr
,
1205 MMUAccessType access_type
, int mmu_idx
)
1207 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1208 target_ulong tlb_addr
, page
;
1211 switch (access_type
) {
1213 elt_ofs
= offsetof(CPUTLBEntry
, addr_read
);
1215 case MMU_DATA_STORE
:
1216 elt_ofs
= offsetof(CPUTLBEntry
, addr_write
);
1218 case MMU_INST_FETCH
:
1219 elt_ofs
= offsetof(CPUTLBEntry
, addr_code
);
1222 g_assert_not_reached();
1225 page
= addr
& TARGET_PAGE_MASK
;
1226 tlb_addr
= tlb_read_ofs(entry
, elt_ofs
);
1228 if (!tlb_hit_page(tlb_addr
, page
)) {
1229 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1231 if (!victim_tlb_hit(env
, mmu_idx
, index
, elt_ofs
, page
)) {
1232 CPUState
*cs
= env_cpu(env
);
1233 CPUClass
*cc
= CPU_GET_CLASS(cs
);
1235 if (!cc
->tlb_fill(cs
, addr
, 0, access_type
, mmu_idx
, true, 0)) {
1236 /* Non-faulting page table read failed. */
1240 /* TLB resize via tlb_fill may have moved the entry. */
1241 entry
= tlb_entry(env
, mmu_idx
, addr
);
1243 tlb_addr
= tlb_read_ofs(entry
, elt_ofs
);
1246 if (tlb_addr
& ~TARGET_PAGE_MASK
) {
1251 return (void *)((uintptr_t)addr
+ entry
->addend
);
1255 #ifdef CONFIG_PLUGIN
1257 * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure.
1258 * This should be a hot path as we will have just looked this path up
1259 * in the softmmu lookup code (or helper). We don't handle re-fills or
1260 * checking the victim table. This is purely informational.
1262 * This should never fail as the memory access being instrumented
1263 * should have just filled the TLB.
1266 bool tlb_plugin_lookup(CPUState
*cpu
, target_ulong addr
, int mmu_idx
,
1267 bool is_store
, struct qemu_plugin_hwaddr
*data
)
1269 CPUArchState
*env
= cpu
->env_ptr
;
1270 CPUTLBEntry
*tlbe
= tlb_entry(env
, mmu_idx
, addr
);
1271 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1272 target_ulong tlb_addr
= is_store
? tlb_addr_write(tlbe
) : tlbe
->addr_read
;
1274 if (likely(tlb_hit(tlb_addr
, addr
))) {
1275 /* We must have an iotlb entry for MMIO */
1276 if (tlb_addr
& TLB_MMIO
) {
1277 CPUIOTLBEntry
*iotlbentry
;
1278 iotlbentry
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1280 data
->v
.io
.section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
1281 data
->v
.io
.offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
1283 data
->is_io
= false;
1284 data
->v
.ram
.hostaddr
= addr
+ tlbe
->addend
;
1293 /* Probe for a read-modify-write atomic operation. Do not allow unaligned
1294 * operations, or io operations to proceed. Return the host address. */
1295 static void *atomic_mmu_lookup(CPUArchState
*env
, target_ulong addr
,
1296 TCGMemOpIdx oi
, uintptr_t retaddr
)
1298 size_t mmu_idx
= get_mmuidx(oi
);
1299 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1300 CPUTLBEntry
*tlbe
= tlb_entry(env
, mmu_idx
, addr
);
1301 target_ulong tlb_addr
= tlb_addr_write(tlbe
);
1302 MemOp mop
= get_memop(oi
);
1303 int a_bits
= get_alignment_bits(mop
);
1304 int s_bits
= mop
& MO_SIZE
;
1307 /* Adjust the given return address. */
1308 retaddr
-= GETPC_ADJ
;
1310 /* Enforce guest required alignment. */
1311 if (unlikely(a_bits
> 0 && (addr
& ((1 << a_bits
) - 1)))) {
1312 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
1313 cpu_unaligned_access(env_cpu(env
), addr
, MMU_DATA_STORE
,
1317 /* Enforce qemu required alignment. */
1318 if (unlikely(addr
& ((1 << s_bits
) - 1))) {
1319 /* We get here if guest alignment was not requested,
1320 or was not enforced by cpu_unaligned_access above.
1321 We might widen the access and emulate, but for now
1322 mark an exception and exit the cpu loop. */
1323 goto stop_the_world
;
1326 /* Check TLB entry and enforce page permissions. */
1327 if (!tlb_hit(tlb_addr
, addr
)) {
1328 if (!VICTIM_TLB_HIT(addr_write
, addr
)) {
1329 tlb_fill(env_cpu(env
), addr
, 1 << s_bits
, MMU_DATA_STORE
,
1331 index
= tlb_index(env
, mmu_idx
, addr
);
1332 tlbe
= tlb_entry(env
, mmu_idx
, addr
);
1334 tlb_addr
= tlb_addr_write(tlbe
) & ~TLB_INVALID_MASK
;
1337 /* Notice an IO access or a needs-MMU-lookup access */
1338 if (unlikely(tlb_addr
& TLB_MMIO
)) {
1339 /* There's really nothing that can be done to
1340 support this apart from stop-the-world. */
1341 goto stop_the_world
;
1344 /* Let the guest notice RMW on a write-only page. */
1345 if (unlikely(tlbe
->addr_read
!= (tlb_addr
& ~TLB_NOTDIRTY
))) {
1346 tlb_fill(env_cpu(env
), addr
, 1 << s_bits
, MMU_DATA_LOAD
,
1348 /* Since we don't support reads and writes to different addresses,
1349 and we do have the proper page loaded for write, this shouldn't
1350 ever return. But just in case, handle via stop-the-world. */
1351 goto stop_the_world
;
1354 hostaddr
= (void *)((uintptr_t)addr
+ tlbe
->addend
);
1356 if (unlikely(tlb_addr
& TLB_NOTDIRTY
)) {
1357 notdirty_write(env_cpu(env
), addr
, 1 << s_bits
,
1358 &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
], retaddr
);
1364 cpu_loop_exit_atomic(env_cpu(env
), retaddr
);
1370 * We support two different access types. SOFTMMU_CODE_ACCESS is
1371 * specifically for reading instructions from system memory. It is
1372 * called by the translation loop and in some helpers where the code
1373 * is disassembled. It shouldn't be called directly by guest code.
1376 typedef uint64_t FullLoadHelper(CPUArchState
*env
, target_ulong addr
,
1377 TCGMemOpIdx oi
, uintptr_t retaddr
);
1379 static inline uint64_t QEMU_ALWAYS_INLINE
1380 load_memop(const void *haddr
, MemOp op
)
1384 return ldub_p(haddr
);
1386 return lduw_be_p(haddr
);
1388 return lduw_le_p(haddr
);
1390 return (uint32_t)ldl_be_p(haddr
);
1392 return (uint32_t)ldl_le_p(haddr
);
1394 return ldq_be_p(haddr
);
1396 return ldq_le_p(haddr
);
1398 qemu_build_not_reached();
1402 static inline uint64_t QEMU_ALWAYS_INLINE
1403 load_helper(CPUArchState
*env
, target_ulong addr
, TCGMemOpIdx oi
,
1404 uintptr_t retaddr
, MemOp op
, bool code_read
,
1405 FullLoadHelper
*full_load
)
1407 uintptr_t mmu_idx
= get_mmuidx(oi
);
1408 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1409 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1410 target_ulong tlb_addr
= code_read
? entry
->addr_code
: entry
->addr_read
;
1411 const size_t tlb_off
= code_read
?
1412 offsetof(CPUTLBEntry
, addr_code
) : offsetof(CPUTLBEntry
, addr_read
);
1413 const MMUAccessType access_type
=
1414 code_read
? MMU_INST_FETCH
: MMU_DATA_LOAD
;
1415 unsigned a_bits
= get_alignment_bits(get_memop(oi
));
1418 size_t size
= memop_size(op
);
1420 /* Handle CPU specific unaligned behaviour */
1421 if (addr
& ((1 << a_bits
) - 1)) {
1422 cpu_unaligned_access(env_cpu(env
), addr
, access_type
,
1426 /* If the TLB entry is for a different page, reload and try again. */
1427 if (!tlb_hit(tlb_addr
, addr
)) {
1428 if (!victim_tlb_hit(env
, mmu_idx
, index
, tlb_off
,
1429 addr
& TARGET_PAGE_MASK
)) {
1430 tlb_fill(env_cpu(env
), addr
, size
,
1431 access_type
, mmu_idx
, retaddr
);
1432 index
= tlb_index(env
, mmu_idx
, addr
);
1433 entry
= tlb_entry(env
, mmu_idx
, addr
);
1435 tlb_addr
= code_read
? entry
->addr_code
: entry
->addr_read
;
1436 tlb_addr
&= ~TLB_INVALID_MASK
;
1439 /* Handle anything that isn't just a straight memory access. */
1440 if (unlikely(tlb_addr
& ~TARGET_PAGE_MASK
)) {
1441 CPUIOTLBEntry
*iotlbentry
;
1444 /* For anything that is unaligned, recurse through full_load. */
1445 if ((addr
& (size
- 1)) != 0) {
1446 goto do_unaligned_access
;
1449 iotlbentry
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1451 /* Handle watchpoints. */
1452 if (unlikely(tlb_addr
& TLB_WATCHPOINT
)) {
1453 /* On watchpoint hit, this will longjmp out. */
1454 cpu_check_watchpoint(env_cpu(env
), addr
, size
,
1455 iotlbentry
->attrs
, BP_MEM_READ
, retaddr
);
1458 need_swap
= size
> 1 && (tlb_addr
& TLB_BSWAP
);
1460 /* Handle I/O access. */
1461 if (likely(tlb_addr
& TLB_MMIO
)) {
1462 return io_readx(env
, iotlbentry
, mmu_idx
, addr
, retaddr
,
1463 access_type
, op
^ (need_swap
* MO_BSWAP
));
1466 haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
1469 * Keep these two load_memop separate to ensure that the compiler
1470 * is able to fold the entire function to a single instruction.
1471 * There is a build-time assert inside to remind you of this. ;-)
1473 if (unlikely(need_swap
)) {
1474 return load_memop(haddr
, op
^ MO_BSWAP
);
1476 return load_memop(haddr
, op
);
1479 /* Handle slow unaligned access (it spans two pages or IO). */
1481 && unlikely((addr
& ~TARGET_PAGE_MASK
) + size
- 1
1482 >= TARGET_PAGE_SIZE
)) {
1483 target_ulong addr1
, addr2
;
1486 do_unaligned_access
:
1487 addr1
= addr
& ~((target_ulong
)size
- 1);
1488 addr2
= addr1
+ size
;
1489 r1
= full_load(env
, addr1
, oi
, retaddr
);
1490 r2
= full_load(env
, addr2
, oi
, retaddr
);
1491 shift
= (addr
& (size
- 1)) * 8;
1493 if (memop_big_endian(op
)) {
1494 /* Big-endian combine. */
1495 res
= (r1
<< shift
) | (r2
>> ((size
* 8) - shift
));
1497 /* Little-endian combine. */
1498 res
= (r1
>> shift
) | (r2
<< ((size
* 8) - shift
));
1500 return res
& MAKE_64BIT_MASK(0, size
* 8);
1503 haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
1504 return load_memop(haddr
, op
);
1508 * For the benefit of TCG generated code, we want to avoid the
1509 * complication of ABI-specific return type promotion and always
1510 * return a value extended to the register size of the host. This is
1511 * tcg_target_long, except in the case of a 32-bit host and 64-bit
1512 * data, and for that we always have uint64_t.
1514 * We don't bother with this widened value for SOFTMMU_CODE_ACCESS.
1517 static uint64_t full_ldub_mmu(CPUArchState
*env
, target_ulong addr
,
1518 TCGMemOpIdx oi
, uintptr_t retaddr
)
1520 return load_helper(env
, addr
, oi
, retaddr
, MO_UB
, false, full_ldub_mmu
);
1523 tcg_target_ulong
helper_ret_ldub_mmu(CPUArchState
*env
, target_ulong addr
,
1524 TCGMemOpIdx oi
, uintptr_t retaddr
)
1526 return full_ldub_mmu(env
, addr
, oi
, retaddr
);
1529 static uint64_t full_le_lduw_mmu(CPUArchState
*env
, target_ulong addr
,
1530 TCGMemOpIdx oi
, uintptr_t retaddr
)
1532 return load_helper(env
, addr
, oi
, retaddr
, MO_LEUW
, false,
1536 tcg_target_ulong
helper_le_lduw_mmu(CPUArchState
*env
, target_ulong addr
,
1537 TCGMemOpIdx oi
, uintptr_t retaddr
)
1539 return full_le_lduw_mmu(env
, addr
, oi
, retaddr
);
1542 static uint64_t full_be_lduw_mmu(CPUArchState
*env
, target_ulong addr
,
1543 TCGMemOpIdx oi
, uintptr_t retaddr
)
1545 return load_helper(env
, addr
, oi
, retaddr
, MO_BEUW
, false,
1549 tcg_target_ulong
helper_be_lduw_mmu(CPUArchState
*env
, target_ulong addr
,
1550 TCGMemOpIdx oi
, uintptr_t retaddr
)
1552 return full_be_lduw_mmu(env
, addr
, oi
, retaddr
);
1555 static uint64_t full_le_ldul_mmu(CPUArchState
*env
, target_ulong addr
,
1556 TCGMemOpIdx oi
, uintptr_t retaddr
)
1558 return load_helper(env
, addr
, oi
, retaddr
, MO_LEUL
, false,
1562 tcg_target_ulong
helper_le_ldul_mmu(CPUArchState
*env
, target_ulong addr
,
1563 TCGMemOpIdx oi
, uintptr_t retaddr
)
1565 return full_le_ldul_mmu(env
, addr
, oi
, retaddr
);
1568 static uint64_t full_be_ldul_mmu(CPUArchState
*env
, target_ulong addr
,
1569 TCGMemOpIdx oi
, uintptr_t retaddr
)
1571 return load_helper(env
, addr
, oi
, retaddr
, MO_BEUL
, false,
1575 tcg_target_ulong
helper_be_ldul_mmu(CPUArchState
*env
, target_ulong addr
,
1576 TCGMemOpIdx oi
, uintptr_t retaddr
)
1578 return full_be_ldul_mmu(env
, addr
, oi
, retaddr
);
1581 uint64_t helper_le_ldq_mmu(CPUArchState
*env
, target_ulong addr
,
1582 TCGMemOpIdx oi
, uintptr_t retaddr
)
1584 return load_helper(env
, addr
, oi
, retaddr
, MO_LEQ
, false,
1588 uint64_t helper_be_ldq_mmu(CPUArchState
*env
, target_ulong addr
,
1589 TCGMemOpIdx oi
, uintptr_t retaddr
)
1591 return load_helper(env
, addr
, oi
, retaddr
, MO_BEQ
, false,
1596 * Provide signed versions of the load routines as well. We can of course
1597 * avoid this for 64-bit data, or for 32-bit data on 32-bit host.
1601 tcg_target_ulong
helper_ret_ldsb_mmu(CPUArchState
*env
, target_ulong addr
,
1602 TCGMemOpIdx oi
, uintptr_t retaddr
)
1604 return (int8_t)helper_ret_ldub_mmu(env
, addr
, oi
, retaddr
);
1607 tcg_target_ulong
helper_le_ldsw_mmu(CPUArchState
*env
, target_ulong addr
,
1608 TCGMemOpIdx oi
, uintptr_t retaddr
)
1610 return (int16_t)helper_le_lduw_mmu(env
, addr
, oi
, retaddr
);
1613 tcg_target_ulong
helper_be_ldsw_mmu(CPUArchState
*env
, target_ulong addr
,
1614 TCGMemOpIdx oi
, uintptr_t retaddr
)
1616 return (int16_t)helper_be_lduw_mmu(env
, addr
, oi
, retaddr
);
1619 tcg_target_ulong
helper_le_ldsl_mmu(CPUArchState
*env
, target_ulong addr
,
1620 TCGMemOpIdx oi
, uintptr_t retaddr
)
1622 return (int32_t)helper_le_ldul_mmu(env
, addr
, oi
, retaddr
);
1625 tcg_target_ulong
helper_be_ldsl_mmu(CPUArchState
*env
, target_ulong addr
,
1626 TCGMemOpIdx oi
, uintptr_t retaddr
)
1628 return (int32_t)helper_be_ldul_mmu(env
, addr
, oi
, retaddr
);
1632 * Load helpers for cpu_ldst.h.
1635 static inline uint64_t cpu_load_helper(CPUArchState
*env
, abi_ptr addr
,
1636 int mmu_idx
, uintptr_t retaddr
,
1637 MemOp op
, FullLoadHelper
*full_load
)
1643 meminfo
= trace_mem_get_info(op
, mmu_idx
, false);
1644 trace_guest_mem_before_exec(env_cpu(env
), addr
, meminfo
);
1647 oi
= make_memop_idx(op
, mmu_idx
);
1648 ret
= full_load(env
, addr
, oi
, retaddr
);
1650 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, meminfo
);
1655 uint32_t cpu_ldub_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
1656 int mmu_idx
, uintptr_t ra
)
1658 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_UB
, full_ldub_mmu
);
1661 int cpu_ldsb_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
1662 int mmu_idx
, uintptr_t ra
)
1664 return (int8_t)cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_SB
,
1668 uint32_t cpu_lduw_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
1669 int mmu_idx
, uintptr_t ra
)
1671 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_TEUW
,
1673 ? full_le_lduw_mmu
: full_be_lduw_mmu
);
1676 int cpu_ldsw_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
1677 int mmu_idx
, uintptr_t ra
)
1679 return (int16_t)cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_TESW
,
1681 ? full_le_lduw_mmu
: full_be_lduw_mmu
);
1684 uint32_t cpu_ldl_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
1685 int mmu_idx
, uintptr_t ra
)
1687 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_TEUL
,
1689 ? full_le_ldul_mmu
: full_be_ldul_mmu
);
1692 uint64_t cpu_ldq_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
1693 int mmu_idx
, uintptr_t ra
)
1695 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_TEQ
,
1697 ? helper_le_ldq_mmu
: helper_be_ldq_mmu
);
1704 static inline void QEMU_ALWAYS_INLINE
1705 store_memop(void *haddr
, uint64_t val
, MemOp op
)
1712 stw_be_p(haddr
, val
);
1715 stw_le_p(haddr
, val
);
1718 stl_be_p(haddr
, val
);
1721 stl_le_p(haddr
, val
);
1724 stq_be_p(haddr
, val
);
1727 stq_le_p(haddr
, val
);
1730 qemu_build_not_reached();
1734 static inline void QEMU_ALWAYS_INLINE
1735 store_helper(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
1736 TCGMemOpIdx oi
, uintptr_t retaddr
, MemOp op
)
1738 uintptr_t mmu_idx
= get_mmuidx(oi
);
1739 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1740 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1741 target_ulong tlb_addr
= tlb_addr_write(entry
);
1742 const size_t tlb_off
= offsetof(CPUTLBEntry
, addr_write
);
1743 unsigned a_bits
= get_alignment_bits(get_memop(oi
));
1745 size_t size
= memop_size(op
);
1747 /* Handle CPU specific unaligned behaviour */
1748 if (addr
& ((1 << a_bits
) - 1)) {
1749 cpu_unaligned_access(env_cpu(env
), addr
, MMU_DATA_STORE
,
1753 /* If the TLB entry is for a different page, reload and try again. */
1754 if (!tlb_hit(tlb_addr
, addr
)) {
1755 if (!victim_tlb_hit(env
, mmu_idx
, index
, tlb_off
,
1756 addr
& TARGET_PAGE_MASK
)) {
1757 tlb_fill(env_cpu(env
), addr
, size
, MMU_DATA_STORE
,
1759 index
= tlb_index(env
, mmu_idx
, addr
);
1760 entry
= tlb_entry(env
, mmu_idx
, addr
);
1762 tlb_addr
= tlb_addr_write(entry
) & ~TLB_INVALID_MASK
;
1765 /* Handle anything that isn't just a straight memory access. */
1766 if (unlikely(tlb_addr
& ~TARGET_PAGE_MASK
)) {
1767 CPUIOTLBEntry
*iotlbentry
;
1770 /* For anything that is unaligned, recurse through byte stores. */
1771 if ((addr
& (size
- 1)) != 0) {
1772 goto do_unaligned_access
;
1775 iotlbentry
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1777 /* Handle watchpoints. */
1778 if (unlikely(tlb_addr
& TLB_WATCHPOINT
)) {
1779 /* On watchpoint hit, this will longjmp out. */
1780 cpu_check_watchpoint(env_cpu(env
), addr
, size
,
1781 iotlbentry
->attrs
, BP_MEM_WRITE
, retaddr
);
1784 need_swap
= size
> 1 && (tlb_addr
& TLB_BSWAP
);
1786 /* Handle I/O access. */
1787 if (tlb_addr
& TLB_MMIO
) {
1788 io_writex(env
, iotlbentry
, mmu_idx
, val
, addr
, retaddr
,
1789 op
^ (need_swap
* MO_BSWAP
));
1793 /* Ignore writes to ROM. */
1794 if (unlikely(tlb_addr
& TLB_DISCARD_WRITE
)) {
1798 /* Handle clean RAM pages. */
1799 if (tlb_addr
& TLB_NOTDIRTY
) {
1800 notdirty_write(env_cpu(env
), addr
, size
, iotlbentry
, retaddr
);
1803 haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
1806 * Keep these two store_memop separate to ensure that the compiler
1807 * is able to fold the entire function to a single instruction.
1808 * There is a build-time assert inside to remind you of this. ;-)
1810 if (unlikely(need_swap
)) {
1811 store_memop(haddr
, val
, op
^ MO_BSWAP
);
1813 store_memop(haddr
, val
, op
);
1818 /* Handle slow unaligned access (it spans two pages or IO). */
1820 && unlikely((addr
& ~TARGET_PAGE_MASK
) + size
- 1
1821 >= TARGET_PAGE_SIZE
)) {
1824 CPUTLBEntry
*entry2
;
1825 target_ulong page2
, tlb_addr2
;
1828 do_unaligned_access
:
1830 * Ensure the second page is in the TLB. Note that the first page
1831 * is already guaranteed to be filled, and that the second page
1832 * cannot evict the first.
1834 page2
= (addr
+ size
) & TARGET_PAGE_MASK
;
1835 size2
= (addr
+ size
) & ~TARGET_PAGE_MASK
;
1836 index2
= tlb_index(env
, mmu_idx
, page2
);
1837 entry2
= tlb_entry(env
, mmu_idx
, page2
);
1838 tlb_addr2
= tlb_addr_write(entry2
);
1839 if (!tlb_hit_page(tlb_addr2
, page2
)) {
1840 if (!victim_tlb_hit(env
, mmu_idx
, index2
, tlb_off
, page2
)) {
1841 tlb_fill(env_cpu(env
), page2
, size2
, MMU_DATA_STORE
,
1843 index2
= tlb_index(env
, mmu_idx
, page2
);
1844 entry2
= tlb_entry(env
, mmu_idx
, page2
);
1846 tlb_addr2
= tlb_addr_write(entry2
);
1850 * Handle watchpoints. Since this may trap, all checks
1851 * must happen before any store.
1853 if (unlikely(tlb_addr
& TLB_WATCHPOINT
)) {
1854 cpu_check_watchpoint(env_cpu(env
), addr
, size
- size2
,
1855 env_tlb(env
)->d
[mmu_idx
].iotlb
[index
].attrs
,
1856 BP_MEM_WRITE
, retaddr
);
1858 if (unlikely(tlb_addr2
& TLB_WATCHPOINT
)) {
1859 cpu_check_watchpoint(env_cpu(env
), page2
, size2
,
1860 env_tlb(env
)->d
[mmu_idx
].iotlb
[index2
].attrs
,
1861 BP_MEM_WRITE
, retaddr
);
1865 * XXX: not efficient, but simple.
1866 * This loop must go in the forward direction to avoid issues
1867 * with self-modifying code in Windows 64-bit.
1869 for (i
= 0; i
< size
; ++i
) {
1871 if (memop_big_endian(op
)) {
1872 /* Big-endian extract. */
1873 val8
= val
>> (((size
- 1) * 8) - (i
* 8));
1875 /* Little-endian extract. */
1876 val8
= val
>> (i
* 8);
1878 helper_ret_stb_mmu(env
, addr
+ i
, val8
, oi
, retaddr
);
1883 haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
1884 store_memop(haddr
, val
, op
);
1887 void helper_ret_stb_mmu(CPUArchState
*env
, target_ulong addr
, uint8_t val
,
1888 TCGMemOpIdx oi
, uintptr_t retaddr
)
1890 store_helper(env
, addr
, val
, oi
, retaddr
, MO_UB
);
1893 void helper_le_stw_mmu(CPUArchState
*env
, target_ulong addr
, uint16_t val
,
1894 TCGMemOpIdx oi
, uintptr_t retaddr
)
1896 store_helper(env
, addr
, val
, oi
, retaddr
, MO_LEUW
);
1899 void helper_be_stw_mmu(CPUArchState
*env
, target_ulong addr
, uint16_t val
,
1900 TCGMemOpIdx oi
, uintptr_t retaddr
)
1902 store_helper(env
, addr
, val
, oi
, retaddr
, MO_BEUW
);
1905 void helper_le_stl_mmu(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
1906 TCGMemOpIdx oi
, uintptr_t retaddr
)
1908 store_helper(env
, addr
, val
, oi
, retaddr
, MO_LEUL
);
1911 void helper_be_stl_mmu(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
1912 TCGMemOpIdx oi
, uintptr_t retaddr
)
1914 store_helper(env
, addr
, val
, oi
, retaddr
, MO_BEUL
);
1917 void helper_le_stq_mmu(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
1918 TCGMemOpIdx oi
, uintptr_t retaddr
)
1920 store_helper(env
, addr
, val
, oi
, retaddr
, MO_LEQ
);
1923 void helper_be_stq_mmu(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
1924 TCGMemOpIdx oi
, uintptr_t retaddr
)
1926 store_helper(env
, addr
, val
, oi
, retaddr
, MO_BEQ
);
1930 * Store Helpers for cpu_ldst.h
1933 static inline void QEMU_ALWAYS_INLINE
1934 cpu_store_helper(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
1935 int mmu_idx
, uintptr_t retaddr
, MemOp op
)
1940 meminfo
= trace_mem_get_info(op
, mmu_idx
, true);
1941 trace_guest_mem_before_exec(env_cpu(env
), addr
, meminfo
);
1943 oi
= make_memop_idx(op
, mmu_idx
);
1944 store_helper(env
, addr
, val
, oi
, retaddr
, op
);
1946 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, meminfo
);
1949 void cpu_stb_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
1950 int mmu_idx
, uintptr_t retaddr
)
1952 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_UB
);
1955 void cpu_stw_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
1956 int mmu_idx
, uintptr_t retaddr
)
1958 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_TEUW
);
1961 void cpu_stl_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
1962 int mmu_idx
, uintptr_t retaddr
)
1964 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_TEUL
);
1967 void cpu_stq_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
1968 int mmu_idx
, uintptr_t retaddr
)
1970 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_TEQ
);
1973 /* First set of helpers allows passing in of OI and RETADDR. This makes
1974 them callable from other helpers. */
1976 #define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr
1977 #define ATOMIC_NAME(X) \
1978 HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
1979 #define ATOMIC_MMU_DECLS
1980 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr)
1981 #define ATOMIC_MMU_CLEANUP
1982 #define ATOMIC_MMU_IDX get_mmuidx(oi)
1984 #include "atomic_common.inc.c"
1987 #include "atomic_template.h"
1990 #include "atomic_template.h"
1993 #include "atomic_template.h"
1995 #ifdef CONFIG_ATOMIC64
1997 #include "atomic_template.h"
2000 #if HAVE_CMPXCHG128 || HAVE_ATOMIC128
2001 #define DATA_SIZE 16
2002 #include "atomic_template.h"
2005 /* Second set of helpers are directly callable from TCG as helpers. */
2009 #undef ATOMIC_MMU_LOOKUP
2010 #define EXTRA_ARGS , TCGMemOpIdx oi
2011 #define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
2012 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC())
2015 #include "atomic_template.h"
2018 #include "atomic_template.h"
2021 #include "atomic_template.h"
2023 #ifdef CONFIG_ATOMIC64
2025 #include "atomic_template.h"
2027 #undef ATOMIC_MMU_IDX
2029 /* Code access functions. */
2031 static uint64_t full_ldub_cmmu(CPUArchState
*env
, target_ulong addr
,
2032 TCGMemOpIdx oi
, uintptr_t retaddr
)
2034 return load_helper(env
, addr
, oi
, retaddr
, MO_8
, true, full_ldub_cmmu
);
2037 uint8_t helper_ret_ldub_cmmu(CPUArchState
*env
, target_ulong addr
,
2038 TCGMemOpIdx oi
, uintptr_t retaddr
)
2040 return full_ldub_cmmu(env
, addr
, oi
, retaddr
);
2043 int8_t helper_ret_ldsb_cmmu(CPUArchState
*env
, target_ulong addr
,
2044 TCGMemOpIdx oi
, uintptr_t retaddr
)
2046 return (int8_t) full_ldub_cmmu(env
, addr
, oi
, retaddr
);
2049 static uint64_t full_le_lduw_cmmu(CPUArchState
*env
, target_ulong addr
,
2050 TCGMemOpIdx oi
, uintptr_t retaddr
)
2052 return load_helper(env
, addr
, oi
, retaddr
, MO_LEUW
, true,
2056 uint16_t helper_le_lduw_cmmu(CPUArchState
*env
, target_ulong addr
,
2057 TCGMemOpIdx oi
, uintptr_t retaddr
)
2059 return full_le_lduw_cmmu(env
, addr
, oi
, retaddr
);
2062 int16_t helper_le_ldsw_cmmu(CPUArchState
*env
, target_ulong addr
,
2063 TCGMemOpIdx oi
, uintptr_t retaddr
)
2065 return (int16_t) full_le_lduw_cmmu(env
, addr
, oi
, retaddr
);
2068 static uint64_t full_be_lduw_cmmu(CPUArchState
*env
, target_ulong addr
,
2069 TCGMemOpIdx oi
, uintptr_t retaddr
)
2071 return load_helper(env
, addr
, oi
, retaddr
, MO_BEUW
, true,
2075 uint16_t helper_be_lduw_cmmu(CPUArchState
*env
, target_ulong addr
,
2076 TCGMemOpIdx oi
, uintptr_t retaddr
)
2078 return full_be_lduw_cmmu(env
, addr
, oi
, retaddr
);
2081 int16_t helper_be_ldsw_cmmu(CPUArchState
*env
, target_ulong addr
,
2082 TCGMemOpIdx oi
, uintptr_t retaddr
)
2084 return (int16_t) full_be_lduw_cmmu(env
, addr
, oi
, retaddr
);
2087 static uint64_t full_le_ldul_cmmu(CPUArchState
*env
, target_ulong addr
,
2088 TCGMemOpIdx oi
, uintptr_t retaddr
)
2090 return load_helper(env
, addr
, oi
, retaddr
, MO_LEUL
, true,
2094 uint32_t helper_le_ldl_cmmu(CPUArchState
*env
, target_ulong addr
,
2095 TCGMemOpIdx oi
, uintptr_t retaddr
)
2097 return full_le_ldul_cmmu(env
, addr
, oi
, retaddr
);
2100 static uint64_t full_be_ldul_cmmu(CPUArchState
*env
, target_ulong addr
,
2101 TCGMemOpIdx oi
, uintptr_t retaddr
)
2103 return load_helper(env
, addr
, oi
, retaddr
, MO_BEUL
, true,
2107 uint32_t helper_be_ldl_cmmu(CPUArchState
*env
, target_ulong addr
,
2108 TCGMemOpIdx oi
, uintptr_t retaddr
)
2110 return full_be_ldul_cmmu(env
, addr
, oi
, retaddr
);
2113 uint64_t helper_le_ldq_cmmu(CPUArchState
*env
, target_ulong addr
,
2114 TCGMemOpIdx oi
, uintptr_t retaddr
)
2116 return load_helper(env
, addr
, oi
, retaddr
, MO_LEQ
, true,
2117 helper_le_ldq_cmmu
);
2120 uint64_t helper_be_ldq_cmmu(CPUArchState
*env
, target_ulong addr
,
2121 TCGMemOpIdx oi
, uintptr_t retaddr
)
2123 return load_helper(env
, addr
, oi
, retaddr
, MO_BEQ
, true,
2124 helper_be_ldq_cmmu
);