2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
22 #include "hw/core/tcg-cpu-ops.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/cpu_ldst.h"
26 #include "exec/cputlb.h"
27 #include "exec/tb-hash.h"
28 #include "exec/memory-internal.h"
29 #include "exec/ram_addr.h"
31 #include "qemu/error-report.h"
33 #include "exec/helper-proto.h"
34 #include "qemu/atomic.h"
35 #include "qemu/atomic128.h"
36 #include "exec/translate-all.h"
37 #include "trace/trace-root.h"
38 #include "trace/mem.h"
41 #include "qemu/plugin-memory.h"
44 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
45 /* #define DEBUG_TLB */
46 /* #define DEBUG_TLB_LOG */
49 # define DEBUG_TLB_GATE 1
51 # define DEBUG_TLB_LOG_GATE 1
53 # define DEBUG_TLB_LOG_GATE 0
56 # define DEBUG_TLB_GATE 0
57 # define DEBUG_TLB_LOG_GATE 0
60 #define tlb_debug(fmt, ...) do { \
61 if (DEBUG_TLB_LOG_GATE) { \
62 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
64 } else if (DEBUG_TLB_GATE) { \
65 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
69 #define assert_cpu_is_self(cpu) do { \
70 if (DEBUG_TLB_GATE) { \
71 g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \
75 /* run_on_cpu_data.target_ptr should always be big enough for a
76 * target_ulong even on 32 bit builds */
77 QEMU_BUILD_BUG_ON(sizeof(target_ulong
) > sizeof(run_on_cpu_data
));
79 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
81 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> 16);
82 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
84 static inline size_t tlb_n_entries(CPUTLBDescFast
*fast
)
86 return (fast
->mask
>> CPU_TLB_ENTRY_BITS
) + 1;
89 static inline size_t sizeof_tlb(CPUTLBDescFast
*fast
)
91 return fast
->mask
+ (1 << CPU_TLB_ENTRY_BITS
);
94 static void tlb_window_reset(CPUTLBDesc
*desc
, int64_t ns
,
97 desc
->window_begin_ns
= ns
;
98 desc
->window_max_entries
= max_entries
;
101 static void tb_jmp_cache_clear_page(CPUState
*cpu
, target_ulong page_addr
)
103 unsigned int i
, i0
= tb_jmp_cache_hash_page(page_addr
);
105 for (i
= 0; i
< TB_JMP_PAGE_SIZE
; i
++) {
106 qatomic_set(&cpu
->tb_jmp_cache
[i0
+ i
], NULL
);
110 static void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
112 /* Discard jump cache entries for any tb which might potentially
113 overlap the flushed page. */
114 tb_jmp_cache_clear_page(cpu
, addr
- TARGET_PAGE_SIZE
);
115 tb_jmp_cache_clear_page(cpu
, addr
);
119 * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary
120 * @desc: The CPUTLBDesc portion of the TLB
121 * @fast: The CPUTLBDescFast portion of the same TLB
123 * Called with tlb_lock_held.
125 * We have two main constraints when resizing a TLB: (1) we only resize it
126 * on a TLB flush (otherwise we'd have to take a perf hit by either rehashing
127 * the array or unnecessarily flushing it), which means we do not control how
128 * frequently the resizing can occur; (2) we don't have access to the guest's
129 * future scheduling decisions, and therefore have to decide the magnitude of
130 * the resize based on past observations.
132 * In general, a memory-hungry process can benefit greatly from an appropriately
133 * sized TLB, since a guest TLB miss is very expensive. This doesn't mean that
134 * we just have to make the TLB as large as possible; while an oversized TLB
135 * results in minimal TLB miss rates, it also takes longer to be flushed
136 * (flushes can be _very_ frequent), and the reduced locality can also hurt
139 * To achieve near-optimal performance for all kinds of workloads, we:
141 * 1. Aggressively increase the size of the TLB when the use rate of the
142 * TLB being flushed is high, since it is likely that in the near future this
143 * memory-hungry process will execute again, and its memory hungriness will
144 * probably be similar.
146 * 2. Slowly reduce the size of the TLB as the use rate declines over a
147 * reasonably large time window. The rationale is that if in such a time window
148 * we have not observed a high TLB use rate, it is likely that we won't observe
149 * it in the near future. In that case, once a time window expires we downsize
150 * the TLB to match the maximum use rate observed in the window.
152 * 3. Try to keep the maximum use rate in a time window in the 30-70% range,
153 * since in that range performance is likely near-optimal. Recall that the TLB
154 * is direct mapped, so we want the use rate to be low (or at least not too
155 * high), since otherwise we are likely to have a significant amount of
158 static void tlb_mmu_resize_locked(CPUTLBDesc
*desc
, CPUTLBDescFast
*fast
,
161 size_t old_size
= tlb_n_entries(fast
);
163 size_t new_size
= old_size
;
164 int64_t window_len_ms
= 100;
165 int64_t window_len_ns
= window_len_ms
* 1000 * 1000;
166 bool window_expired
= now
> desc
->window_begin_ns
+ window_len_ns
;
168 if (desc
->n_used_entries
> desc
->window_max_entries
) {
169 desc
->window_max_entries
= desc
->n_used_entries
;
171 rate
= desc
->window_max_entries
* 100 / old_size
;
174 new_size
= MIN(old_size
<< 1, 1 << CPU_TLB_DYN_MAX_BITS
);
175 } else if (rate
< 30 && window_expired
) {
176 size_t ceil
= pow2ceil(desc
->window_max_entries
);
177 size_t expected_rate
= desc
->window_max_entries
* 100 / ceil
;
180 * Avoid undersizing when the max number of entries seen is just below
181 * a pow2. For instance, if max_entries == 1025, the expected use rate
182 * would be 1025/2048==50%. However, if max_entries == 1023, we'd get
183 * 1023/1024==99.9% use rate, so we'd likely end up doubling the size
184 * later. Thus, make sure that the expected use rate remains below 70%.
185 * (and since we double the size, that means the lowest rate we'd
186 * expect to get is 35%, which is still in the 30-70% range where
187 * we consider that the size is appropriate.)
189 if (expected_rate
> 70) {
192 new_size
= MAX(ceil
, 1 << CPU_TLB_DYN_MIN_BITS
);
195 if (new_size
== old_size
) {
196 if (window_expired
) {
197 tlb_window_reset(desc
, now
, desc
->n_used_entries
);
205 tlb_window_reset(desc
, now
, 0);
206 /* desc->n_used_entries is cleared by the caller */
207 fast
->mask
= (new_size
- 1) << CPU_TLB_ENTRY_BITS
;
208 fast
->table
= g_try_new(CPUTLBEntry
, new_size
);
209 desc
->iotlb
= g_try_new(CPUIOTLBEntry
, new_size
);
212 * If the allocations fail, try smaller sizes. We just freed some
213 * memory, so going back to half of new_size has a good chance of working.
214 * Increased memory pressure elsewhere in the system might cause the
215 * allocations to fail though, so we progressively reduce the allocation
216 * size, aborting if we cannot even allocate the smallest TLB we support.
218 while (fast
->table
== NULL
|| desc
->iotlb
== NULL
) {
219 if (new_size
== (1 << CPU_TLB_DYN_MIN_BITS
)) {
220 error_report("%s: %s", __func__
, strerror(errno
));
223 new_size
= MAX(new_size
>> 1, 1 << CPU_TLB_DYN_MIN_BITS
);
224 fast
->mask
= (new_size
- 1) << CPU_TLB_ENTRY_BITS
;
228 fast
->table
= g_try_new(CPUTLBEntry
, new_size
);
229 desc
->iotlb
= g_try_new(CPUIOTLBEntry
, new_size
);
233 static void tlb_mmu_flush_locked(CPUTLBDesc
*desc
, CPUTLBDescFast
*fast
)
235 desc
->n_used_entries
= 0;
236 desc
->large_page_addr
= -1;
237 desc
->large_page_mask
= -1;
239 memset(fast
->table
, -1, sizeof_tlb(fast
));
240 memset(desc
->vtable
, -1, sizeof(desc
->vtable
));
243 static void tlb_flush_one_mmuidx_locked(CPUArchState
*env
, int mmu_idx
,
246 CPUTLBDesc
*desc
= &env_tlb(env
)->d
[mmu_idx
];
247 CPUTLBDescFast
*fast
= &env_tlb(env
)->f
[mmu_idx
];
249 tlb_mmu_resize_locked(desc
, fast
, now
);
250 tlb_mmu_flush_locked(desc
, fast
);
253 static void tlb_mmu_init(CPUTLBDesc
*desc
, CPUTLBDescFast
*fast
, int64_t now
)
255 size_t n_entries
= 1 << CPU_TLB_DYN_DEFAULT_BITS
;
257 tlb_window_reset(desc
, now
, 0);
258 desc
->n_used_entries
= 0;
259 fast
->mask
= (n_entries
- 1) << CPU_TLB_ENTRY_BITS
;
260 fast
->table
= g_new(CPUTLBEntry
, n_entries
);
261 desc
->iotlb
= g_new(CPUIOTLBEntry
, n_entries
);
262 tlb_mmu_flush_locked(desc
, fast
);
265 static inline void tlb_n_used_entries_inc(CPUArchState
*env
, uintptr_t mmu_idx
)
267 env_tlb(env
)->d
[mmu_idx
].n_used_entries
++;
270 static inline void tlb_n_used_entries_dec(CPUArchState
*env
, uintptr_t mmu_idx
)
272 env_tlb(env
)->d
[mmu_idx
].n_used_entries
--;
275 void tlb_init(CPUState
*cpu
)
277 CPUArchState
*env
= cpu
->env_ptr
;
278 int64_t now
= get_clock_realtime();
281 qemu_spin_init(&env_tlb(env
)->c
.lock
);
283 /* All tlbs are initialized flushed. */
284 env_tlb(env
)->c
.dirty
= 0;
286 for (i
= 0; i
< NB_MMU_MODES
; i
++) {
287 tlb_mmu_init(&env_tlb(env
)->d
[i
], &env_tlb(env
)->f
[i
], now
);
291 void tlb_destroy(CPUState
*cpu
)
293 CPUArchState
*env
= cpu
->env_ptr
;
296 qemu_spin_destroy(&env_tlb(env
)->c
.lock
);
297 for (i
= 0; i
< NB_MMU_MODES
; i
++) {
298 CPUTLBDesc
*desc
= &env_tlb(env
)->d
[i
];
299 CPUTLBDescFast
*fast
= &env_tlb(env
)->f
[i
];
306 /* flush_all_helper: run fn across all cpus
308 * If the wait flag is set then the src cpu's helper will be queued as
309 * "safe" work and the loop exited creating a synchronisation point
310 * where all queued work will be finished before execution starts
313 static void flush_all_helper(CPUState
*src
, run_on_cpu_func fn
,
320 async_run_on_cpu(cpu
, fn
, d
);
325 void tlb_flush_counts(size_t *pfull
, size_t *ppart
, size_t *pelide
)
328 size_t full
= 0, part
= 0, elide
= 0;
331 CPUArchState
*env
= cpu
->env_ptr
;
333 full
+= qatomic_read(&env_tlb(env
)->c
.full_flush_count
);
334 part
+= qatomic_read(&env_tlb(env
)->c
.part_flush_count
);
335 elide
+= qatomic_read(&env_tlb(env
)->c
.elide_flush_count
);
342 static void tlb_flush_by_mmuidx_async_work(CPUState
*cpu
, run_on_cpu_data data
)
344 CPUArchState
*env
= cpu
->env_ptr
;
345 uint16_t asked
= data
.host_int
;
346 uint16_t all_dirty
, work
, to_clean
;
347 int64_t now
= get_clock_realtime();
349 assert_cpu_is_self(cpu
);
351 tlb_debug("mmu_idx:0x%04" PRIx16
"\n", asked
);
353 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
355 all_dirty
= env_tlb(env
)->c
.dirty
;
356 to_clean
= asked
& all_dirty
;
357 all_dirty
&= ~to_clean
;
358 env_tlb(env
)->c
.dirty
= all_dirty
;
360 for (work
= to_clean
; work
!= 0; work
&= work
- 1) {
361 int mmu_idx
= ctz32(work
);
362 tlb_flush_one_mmuidx_locked(env
, mmu_idx
, now
);
365 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
367 cpu_tb_jmp_cache_clear(cpu
);
369 if (to_clean
== ALL_MMUIDX_BITS
) {
370 qatomic_set(&env_tlb(env
)->c
.full_flush_count
,
371 env_tlb(env
)->c
.full_flush_count
+ 1);
373 qatomic_set(&env_tlb(env
)->c
.part_flush_count
,
374 env_tlb(env
)->c
.part_flush_count
+ ctpop16(to_clean
));
375 if (to_clean
!= asked
) {
376 qatomic_set(&env_tlb(env
)->c
.elide_flush_count
,
377 env_tlb(env
)->c
.elide_flush_count
+
378 ctpop16(asked
& ~to_clean
));
383 void tlb_flush_by_mmuidx(CPUState
*cpu
, uint16_t idxmap
)
385 tlb_debug("mmu_idx: 0x%" PRIx16
"\n", idxmap
);
387 if (cpu
->created
&& !qemu_cpu_is_self(cpu
)) {
388 async_run_on_cpu(cpu
, tlb_flush_by_mmuidx_async_work
,
389 RUN_ON_CPU_HOST_INT(idxmap
));
391 tlb_flush_by_mmuidx_async_work(cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
395 void tlb_flush(CPUState
*cpu
)
397 tlb_flush_by_mmuidx(cpu
, ALL_MMUIDX_BITS
);
400 void tlb_flush_by_mmuidx_all_cpus(CPUState
*src_cpu
, uint16_t idxmap
)
402 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
404 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
406 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
407 fn(src_cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
410 void tlb_flush_all_cpus(CPUState
*src_cpu
)
412 tlb_flush_by_mmuidx_all_cpus(src_cpu
, ALL_MMUIDX_BITS
);
415 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
, uint16_t idxmap
)
417 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
419 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
421 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
422 async_safe_run_on_cpu(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
425 void tlb_flush_all_cpus_synced(CPUState
*src_cpu
)
427 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu
, ALL_MMUIDX_BITS
);
430 static bool tlb_hit_page_mask_anyprot(CPUTLBEntry
*tlb_entry
,
431 target_ulong page
, target_ulong mask
)
434 mask
&= TARGET_PAGE_MASK
| TLB_INVALID_MASK
;
436 return (page
== (tlb_entry
->addr_read
& mask
) ||
437 page
== (tlb_addr_write(tlb_entry
) & mask
) ||
438 page
== (tlb_entry
->addr_code
& mask
));
441 static inline bool tlb_hit_page_anyprot(CPUTLBEntry
*tlb_entry
,
444 return tlb_hit_page_mask_anyprot(tlb_entry
, page
, -1);
448 * tlb_entry_is_empty - return true if the entry is not in use
449 * @te: pointer to CPUTLBEntry
451 static inline bool tlb_entry_is_empty(const CPUTLBEntry
*te
)
453 return te
->addr_read
== -1 && te
->addr_write
== -1 && te
->addr_code
== -1;
456 /* Called with tlb_c.lock held */
457 static bool tlb_flush_entry_mask_locked(CPUTLBEntry
*tlb_entry
,
461 if (tlb_hit_page_mask_anyprot(tlb_entry
, page
, mask
)) {
462 memset(tlb_entry
, -1, sizeof(*tlb_entry
));
468 static inline bool tlb_flush_entry_locked(CPUTLBEntry
*tlb_entry
,
471 return tlb_flush_entry_mask_locked(tlb_entry
, page
, -1);
474 /* Called with tlb_c.lock held */
475 static void tlb_flush_vtlb_page_mask_locked(CPUArchState
*env
, int mmu_idx
,
479 CPUTLBDesc
*d
= &env_tlb(env
)->d
[mmu_idx
];
482 assert_cpu_is_self(env_cpu(env
));
483 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
484 if (tlb_flush_entry_mask_locked(&d
->vtable
[k
], page
, mask
)) {
485 tlb_n_used_entries_dec(env
, mmu_idx
);
490 static inline void tlb_flush_vtlb_page_locked(CPUArchState
*env
, int mmu_idx
,
493 tlb_flush_vtlb_page_mask_locked(env
, mmu_idx
, page
, -1);
496 static void tlb_flush_page_locked(CPUArchState
*env
, int midx
,
499 target_ulong lp_addr
= env_tlb(env
)->d
[midx
].large_page_addr
;
500 target_ulong lp_mask
= env_tlb(env
)->d
[midx
].large_page_mask
;
502 /* Check if we need to flush due to large pages. */
503 if ((page
& lp_mask
) == lp_addr
) {
504 tlb_debug("forcing full flush midx %d ("
505 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
506 midx
, lp_addr
, lp_mask
);
507 tlb_flush_one_mmuidx_locked(env
, midx
, get_clock_realtime());
509 if (tlb_flush_entry_locked(tlb_entry(env
, midx
, page
), page
)) {
510 tlb_n_used_entries_dec(env
, midx
);
512 tlb_flush_vtlb_page_locked(env
, midx
, page
);
517 * tlb_flush_page_by_mmuidx_async_0:
518 * @cpu: cpu on which to flush
519 * @addr: page of virtual address to flush
520 * @idxmap: set of mmu_idx to flush
522 * Helper for tlb_flush_page_by_mmuidx and friends, flush one page
523 * at @addr from the tlbs indicated by @idxmap from @cpu.
525 static void tlb_flush_page_by_mmuidx_async_0(CPUState
*cpu
,
529 CPUArchState
*env
= cpu
->env_ptr
;
532 assert_cpu_is_self(cpu
);
534 tlb_debug("page addr:" TARGET_FMT_lx
" mmu_map:0x%x\n", addr
, idxmap
);
536 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
537 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
538 if ((idxmap
>> mmu_idx
) & 1) {
539 tlb_flush_page_locked(env
, mmu_idx
, addr
);
542 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
544 tb_flush_jmp_cache(cpu
, addr
);
548 * tlb_flush_page_by_mmuidx_async_1:
549 * @cpu: cpu on which to flush
550 * @data: encoded addr + idxmap
552 * Helper for tlb_flush_page_by_mmuidx and friends, called through
553 * async_run_on_cpu. The idxmap parameter is encoded in the page
554 * offset of the target_ptr field. This limits the set of mmu_idx
555 * that can be passed via this method.
557 static void tlb_flush_page_by_mmuidx_async_1(CPUState
*cpu
,
558 run_on_cpu_data data
)
560 target_ulong addr_and_idxmap
= (target_ulong
) data
.target_ptr
;
561 target_ulong addr
= addr_and_idxmap
& TARGET_PAGE_MASK
;
562 uint16_t idxmap
= addr_and_idxmap
& ~TARGET_PAGE_MASK
;
564 tlb_flush_page_by_mmuidx_async_0(cpu
, addr
, idxmap
);
570 } TLBFlushPageByMMUIdxData
;
573 * tlb_flush_page_by_mmuidx_async_2:
574 * @cpu: cpu on which to flush
575 * @data: allocated addr + idxmap
577 * Helper for tlb_flush_page_by_mmuidx and friends, called through
578 * async_run_on_cpu. The addr+idxmap parameters are stored in a
579 * TLBFlushPageByMMUIdxData structure that has been allocated
580 * specifically for this helper. Free the structure when done.
582 static void tlb_flush_page_by_mmuidx_async_2(CPUState
*cpu
,
583 run_on_cpu_data data
)
585 TLBFlushPageByMMUIdxData
*d
= data
.host_ptr
;
587 tlb_flush_page_by_mmuidx_async_0(cpu
, d
->addr
, d
->idxmap
);
591 void tlb_flush_page_by_mmuidx(CPUState
*cpu
, target_ulong addr
, uint16_t idxmap
)
593 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%" PRIx16
"\n", addr
, idxmap
);
595 /* This should already be page aligned */
596 addr
&= TARGET_PAGE_MASK
;
598 if (qemu_cpu_is_self(cpu
)) {
599 tlb_flush_page_by_mmuidx_async_0(cpu
, addr
, idxmap
);
600 } else if (idxmap
< TARGET_PAGE_SIZE
) {
602 * Most targets have only a few mmu_idx. In the case where
603 * we can stuff idxmap into the low TARGET_PAGE_BITS, avoid
604 * allocating memory for this operation.
606 async_run_on_cpu(cpu
, tlb_flush_page_by_mmuidx_async_1
,
607 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
609 TLBFlushPageByMMUIdxData
*d
= g_new(TLBFlushPageByMMUIdxData
, 1);
611 /* Otherwise allocate a structure, freed by the worker. */
614 async_run_on_cpu(cpu
, tlb_flush_page_by_mmuidx_async_2
,
615 RUN_ON_CPU_HOST_PTR(d
));
619 void tlb_flush_page(CPUState
*cpu
, target_ulong addr
)
621 tlb_flush_page_by_mmuidx(cpu
, addr
, ALL_MMUIDX_BITS
);
624 void tlb_flush_page_by_mmuidx_all_cpus(CPUState
*src_cpu
, target_ulong addr
,
627 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
629 /* This should already be page aligned */
630 addr
&= TARGET_PAGE_MASK
;
633 * Allocate memory to hold addr+idxmap only when needed.
634 * See tlb_flush_page_by_mmuidx for details.
636 if (idxmap
< TARGET_PAGE_SIZE
) {
637 flush_all_helper(src_cpu
, tlb_flush_page_by_mmuidx_async_1
,
638 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
642 /* Allocate a separate data block for each destination cpu. */
643 CPU_FOREACH(dst_cpu
) {
644 if (dst_cpu
!= src_cpu
) {
645 TLBFlushPageByMMUIdxData
*d
646 = g_new(TLBFlushPageByMMUIdxData
, 1);
650 async_run_on_cpu(dst_cpu
, tlb_flush_page_by_mmuidx_async_2
,
651 RUN_ON_CPU_HOST_PTR(d
));
656 tlb_flush_page_by_mmuidx_async_0(src_cpu
, addr
, idxmap
);
659 void tlb_flush_page_all_cpus(CPUState
*src
, target_ulong addr
)
661 tlb_flush_page_by_mmuidx_all_cpus(src
, addr
, ALL_MMUIDX_BITS
);
664 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
668 tlb_debug("addr: "TARGET_FMT_lx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
670 /* This should already be page aligned */
671 addr
&= TARGET_PAGE_MASK
;
674 * Allocate memory to hold addr+idxmap only when needed.
675 * See tlb_flush_page_by_mmuidx for details.
677 if (idxmap
< TARGET_PAGE_SIZE
) {
678 flush_all_helper(src_cpu
, tlb_flush_page_by_mmuidx_async_1
,
679 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
680 async_safe_run_on_cpu(src_cpu
, tlb_flush_page_by_mmuidx_async_1
,
681 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
684 TLBFlushPageByMMUIdxData
*d
;
686 /* Allocate a separate data block for each destination cpu. */
687 CPU_FOREACH(dst_cpu
) {
688 if (dst_cpu
!= src_cpu
) {
689 d
= g_new(TLBFlushPageByMMUIdxData
, 1);
692 async_run_on_cpu(dst_cpu
, tlb_flush_page_by_mmuidx_async_2
,
693 RUN_ON_CPU_HOST_PTR(d
));
697 d
= g_new(TLBFlushPageByMMUIdxData
, 1);
700 async_safe_run_on_cpu(src_cpu
, tlb_flush_page_by_mmuidx_async_2
,
701 RUN_ON_CPU_HOST_PTR(d
));
705 void tlb_flush_page_all_cpus_synced(CPUState
*src
, target_ulong addr
)
707 tlb_flush_page_by_mmuidx_all_cpus_synced(src
, addr
, ALL_MMUIDX_BITS
);
710 static void tlb_flush_range_locked(CPUArchState
*env
, int midx
,
711 target_ulong addr
, target_ulong len
,
714 CPUTLBDesc
*d
= &env_tlb(env
)->d
[midx
];
715 CPUTLBDescFast
*f
= &env_tlb(env
)->f
[midx
];
716 target_ulong mask
= MAKE_64BIT_MASK(0, bits
);
719 * If @bits is smaller than the tlb size, there may be multiple entries
720 * within the TLB; otherwise all addresses that match under @mask hit
721 * the same TLB entry.
722 * TODO: Perhaps allow bits to be a few bits less than the size.
723 * For now, just flush the entire TLB.
725 * If @len is larger than the tlb size, then it will take longer to
726 * test all of the entries in the TLB than it will to flush it all.
728 if (mask
< f
->mask
|| len
> f
->mask
) {
729 tlb_debug("forcing full flush midx %d ("
730 TARGET_FMT_lx
"/" TARGET_FMT_lx
"+" TARGET_FMT_lx
")\n",
731 midx
, addr
, mask
, len
);
732 tlb_flush_one_mmuidx_locked(env
, midx
, get_clock_realtime());
737 * Check if we need to flush due to large pages.
738 * Because large_page_mask contains all 1's from the msb,
739 * we only need to test the end of the range.
741 if (((addr
+ len
- 1) & d
->large_page_mask
) == d
->large_page_addr
) {
742 tlb_debug("forcing full flush midx %d ("
743 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
744 midx
, d
->large_page_addr
, d
->large_page_mask
);
745 tlb_flush_one_mmuidx_locked(env
, midx
, get_clock_realtime());
749 for (target_ulong i
= 0; i
< len
; i
+= TARGET_PAGE_SIZE
) {
750 target_ulong page
= addr
+ i
;
751 CPUTLBEntry
*entry
= tlb_entry(env
, midx
, page
);
753 if (tlb_flush_entry_mask_locked(entry
, page
, mask
)) {
754 tlb_n_used_entries_dec(env
, midx
);
756 tlb_flush_vtlb_page_mask_locked(env
, midx
, page
, mask
);
768 tlb_flush_page_bits_by_mmuidx_async_0(CPUState
*cpu
,
771 CPUArchState
*env
= cpu
->env_ptr
;
774 assert_cpu_is_self(cpu
);
776 tlb_debug("range:" TARGET_FMT_lx
"/%u+" TARGET_FMT_lx
" mmu_map:0x%x\n",
777 d
.addr
, d
.bits
, d
.len
, d
.idxmap
);
779 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
780 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
781 if ((d
.idxmap
>> mmu_idx
) & 1) {
782 tlb_flush_range_locked(env
, mmu_idx
, d
.addr
, d
.len
, d
.bits
);
785 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
787 for (target_ulong i
= 0; i
< d
.len
; i
+= TARGET_PAGE_SIZE
) {
788 tb_flush_jmp_cache(cpu
, d
.addr
+ i
);
792 static void tlb_flush_page_bits_by_mmuidx_async_2(CPUState
*cpu
,
793 run_on_cpu_data data
)
795 TLBFlushRangeData
*d
= data
.host_ptr
;
796 tlb_flush_page_bits_by_mmuidx_async_0(cpu
, *d
);
800 void tlb_flush_page_bits_by_mmuidx(CPUState
*cpu
, target_ulong addr
,
801 uint16_t idxmap
, unsigned bits
)
805 /* If all bits are significant, this devolves to tlb_flush_page. */
806 if (bits
>= TARGET_LONG_BITS
) {
807 tlb_flush_page_by_mmuidx(cpu
, addr
, idxmap
);
810 /* If no page bits are significant, this devolves to tlb_flush. */
811 if (bits
< TARGET_PAGE_BITS
) {
812 tlb_flush_by_mmuidx(cpu
, idxmap
);
816 /* This should already be page aligned */
817 d
.addr
= addr
& TARGET_PAGE_MASK
;
818 d
.len
= TARGET_PAGE_SIZE
;
822 if (qemu_cpu_is_self(cpu
)) {
823 tlb_flush_page_bits_by_mmuidx_async_0(cpu
, d
);
825 /* Otherwise allocate a structure, freed by the worker. */
826 TLBFlushRangeData
*p
= g_memdup(&d
, sizeof(d
));
827 async_run_on_cpu(cpu
, tlb_flush_page_bits_by_mmuidx_async_2
,
828 RUN_ON_CPU_HOST_PTR(p
));
832 void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState
*src_cpu
,
840 /* If all bits are significant, this devolves to tlb_flush_page. */
841 if (bits
>= TARGET_LONG_BITS
) {
842 tlb_flush_page_by_mmuidx_all_cpus(src_cpu
, addr
, idxmap
);
845 /* If no page bits are significant, this devolves to tlb_flush. */
846 if (bits
< TARGET_PAGE_BITS
) {
847 tlb_flush_by_mmuidx_all_cpus(src_cpu
, idxmap
);
851 /* This should already be page aligned */
852 d
.addr
= addr
& TARGET_PAGE_MASK
;
853 d
.len
= TARGET_PAGE_SIZE
;
857 /* Allocate a separate data block for each destination cpu. */
858 CPU_FOREACH(dst_cpu
) {
859 if (dst_cpu
!= src_cpu
) {
860 TLBFlushRangeData
*p
= g_memdup(&d
, sizeof(d
));
861 async_run_on_cpu(dst_cpu
,
862 tlb_flush_page_bits_by_mmuidx_async_2
,
863 RUN_ON_CPU_HOST_PTR(p
));
867 tlb_flush_page_bits_by_mmuidx_async_0(src_cpu
, d
);
870 void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
875 TLBFlushRangeData d
, *p
;
878 /* If all bits are significant, this devolves to tlb_flush_page. */
879 if (bits
>= TARGET_LONG_BITS
) {
880 tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu
, addr
, idxmap
);
883 /* If no page bits are significant, this devolves to tlb_flush. */
884 if (bits
< TARGET_PAGE_BITS
) {
885 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu
, idxmap
);
889 /* This should already be page aligned */
890 d
.addr
= addr
& TARGET_PAGE_MASK
;
891 d
.len
= TARGET_PAGE_SIZE
;
895 /* Allocate a separate data block for each destination cpu. */
896 CPU_FOREACH(dst_cpu
) {
897 if (dst_cpu
!= src_cpu
) {
898 p
= g_memdup(&d
, sizeof(d
));
899 async_run_on_cpu(dst_cpu
, tlb_flush_page_bits_by_mmuidx_async_2
,
900 RUN_ON_CPU_HOST_PTR(p
));
904 p
= g_memdup(&d
, sizeof(d
));
905 async_safe_run_on_cpu(src_cpu
, tlb_flush_page_bits_by_mmuidx_async_2
,
906 RUN_ON_CPU_HOST_PTR(p
));
909 /* update the TLBs so that writes to code in the virtual page 'addr'
911 void tlb_protect_code(ram_addr_t ram_addr
)
913 cpu_physical_memory_test_and_clear_dirty(ram_addr
, TARGET_PAGE_SIZE
,
917 /* update the TLB so that writes in physical page 'phys_addr' are no longer
918 tested for self modifying code */
919 void tlb_unprotect_code(ram_addr_t ram_addr
)
921 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
);
926 * Dirty write flag handling
928 * When the TCG code writes to a location it looks up the address in
929 * the TLB and uses that data to compute the final address. If any of
930 * the lower bits of the address are set then the slow path is forced.
931 * There are a number of reasons to do this but for normal RAM the
932 * most usual is detecting writes to code regions which may invalidate
935 * Other vCPUs might be reading their TLBs during guest execution, so we update
936 * te->addr_write with qatomic_set. We don't need to worry about this for
937 * oversized guests as MTTCG is disabled for them.
939 * Called with tlb_c.lock held.
941 static void tlb_reset_dirty_range_locked(CPUTLBEntry
*tlb_entry
,
942 uintptr_t start
, uintptr_t length
)
944 uintptr_t addr
= tlb_entry
->addr_write
;
946 if ((addr
& (TLB_INVALID_MASK
| TLB_MMIO
|
947 TLB_DISCARD_WRITE
| TLB_NOTDIRTY
)) == 0) {
948 addr
&= TARGET_PAGE_MASK
;
949 addr
+= tlb_entry
->addend
;
950 if ((addr
- start
) < length
) {
951 #if TCG_OVERSIZED_GUEST
952 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
954 qatomic_set(&tlb_entry
->addr_write
,
955 tlb_entry
->addr_write
| TLB_NOTDIRTY
);
962 * Called with tlb_c.lock held.
963 * Called only from the vCPU context, i.e. the TLB's owner thread.
965 static inline void copy_tlb_helper_locked(CPUTLBEntry
*d
, const CPUTLBEntry
*s
)
970 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
972 * We must take tlb_c.lock to avoid racing with another vCPU update. The only
973 * thing actually updated is the target TLB entry ->addr_write flags.
975 void tlb_reset_dirty(CPUState
*cpu
, ram_addr_t start1
, ram_addr_t length
)
982 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
983 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
985 unsigned int n
= tlb_n_entries(&env_tlb(env
)->f
[mmu_idx
]);
987 for (i
= 0; i
< n
; i
++) {
988 tlb_reset_dirty_range_locked(&env_tlb(env
)->f
[mmu_idx
].table
[i
],
992 for (i
= 0; i
< CPU_VTLB_SIZE
; i
++) {
993 tlb_reset_dirty_range_locked(&env_tlb(env
)->d
[mmu_idx
].vtable
[i
],
997 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
1000 /* Called with tlb_c.lock held */
1001 static inline void tlb_set_dirty1_locked(CPUTLBEntry
*tlb_entry
,
1004 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
)) {
1005 tlb_entry
->addr_write
= vaddr
;
1009 /* update the TLB corresponding to virtual page vaddr
1010 so that it is no longer dirty */
1011 void tlb_set_dirty(CPUState
*cpu
, target_ulong vaddr
)
1013 CPUArchState
*env
= cpu
->env_ptr
;
1016 assert_cpu_is_self(cpu
);
1018 vaddr
&= TARGET_PAGE_MASK
;
1019 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
1020 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1021 tlb_set_dirty1_locked(tlb_entry(env
, mmu_idx
, vaddr
), vaddr
);
1024 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1026 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
1027 tlb_set_dirty1_locked(&env_tlb(env
)->d
[mmu_idx
].vtable
[k
], vaddr
);
1030 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
1033 /* Our TLB does not support large pages, so remember the area covered by
1034 large pages and trigger a full TLB flush if these are invalidated. */
1035 static void tlb_add_large_page(CPUArchState
*env
, int mmu_idx
,
1036 target_ulong vaddr
, target_ulong size
)
1038 target_ulong lp_addr
= env_tlb(env
)->d
[mmu_idx
].large_page_addr
;
1039 target_ulong lp_mask
= ~(size
- 1);
1041 if (lp_addr
== (target_ulong
)-1) {
1042 /* No previous large page. */
1045 /* Extend the existing region to include the new page.
1046 This is a compromise between unnecessary flushes and
1047 the cost of maintaining a full variable size TLB. */
1048 lp_mask
&= env_tlb(env
)->d
[mmu_idx
].large_page_mask
;
1049 while (((lp_addr
^ vaddr
) & lp_mask
) != 0) {
1053 env_tlb(env
)->d
[mmu_idx
].large_page_addr
= lp_addr
& lp_mask
;
1054 env_tlb(env
)->d
[mmu_idx
].large_page_mask
= lp_mask
;
1057 /* Add a new TLB entry. At most one entry for a given virtual address
1058 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
1059 * supplied size is only used by tlb_flush_page.
1061 * Called from TCG-generated code, which is under an RCU read-side
1064 void tlb_set_page_with_attrs(CPUState
*cpu
, target_ulong vaddr
,
1065 hwaddr paddr
, MemTxAttrs attrs
, int prot
,
1066 int mmu_idx
, target_ulong size
)
1068 CPUArchState
*env
= cpu
->env_ptr
;
1069 CPUTLB
*tlb
= env_tlb(env
);
1070 CPUTLBDesc
*desc
= &tlb
->d
[mmu_idx
];
1071 MemoryRegionSection
*section
;
1073 target_ulong address
;
1074 target_ulong write_address
;
1076 CPUTLBEntry
*te
, tn
;
1077 hwaddr iotlb
, xlat
, sz
, paddr_page
;
1078 target_ulong vaddr_page
;
1079 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
1081 bool is_ram
, is_romd
;
1083 assert_cpu_is_self(cpu
);
1085 if (size
<= TARGET_PAGE_SIZE
) {
1086 sz
= TARGET_PAGE_SIZE
;
1088 tlb_add_large_page(env
, mmu_idx
, vaddr
, size
);
1091 vaddr_page
= vaddr
& TARGET_PAGE_MASK
;
1092 paddr_page
= paddr
& TARGET_PAGE_MASK
;
1094 section
= address_space_translate_for_iotlb(cpu
, asidx
, paddr_page
,
1095 &xlat
, &sz
, attrs
, &prot
);
1096 assert(sz
>= TARGET_PAGE_SIZE
);
1098 tlb_debug("vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
1099 " prot=%x idx=%d\n",
1100 vaddr
, paddr
, prot
, mmu_idx
);
1102 address
= vaddr_page
;
1103 if (size
< TARGET_PAGE_SIZE
) {
1104 /* Repeat the MMU check and TLB fill on every access. */
1105 address
|= TLB_INVALID_MASK
;
1107 if (attrs
.byte_swap
) {
1108 address
|= TLB_BSWAP
;
1111 is_ram
= memory_region_is_ram(section
->mr
);
1112 is_romd
= memory_region_is_romd(section
->mr
);
1114 if (is_ram
|| is_romd
) {
1115 /* RAM and ROMD both have associated host memory. */
1116 addend
= (uintptr_t)memory_region_get_ram_ptr(section
->mr
) + xlat
;
1118 /* I/O does not; force the host address to NULL. */
1122 write_address
= address
;
1124 iotlb
= memory_region_get_ram_addr(section
->mr
) + xlat
;
1126 * Computing is_clean is expensive; avoid all that unless
1127 * the page is actually writable.
1129 if (prot
& PAGE_WRITE
) {
1130 if (section
->readonly
) {
1131 write_address
|= TLB_DISCARD_WRITE
;
1132 } else if (cpu_physical_memory_is_clean(iotlb
)) {
1133 write_address
|= TLB_NOTDIRTY
;
1138 iotlb
= memory_region_section_get_iotlb(cpu
, section
) + xlat
;
1140 * Writes to romd devices must go through MMIO to enable write.
1141 * Reads to romd devices go through the ram_ptr found above,
1142 * but of course reads to I/O must go through MMIO.
1144 write_address
|= TLB_MMIO
;
1146 address
= write_address
;
1150 wp_flags
= cpu_watchpoint_address_matches(cpu
, vaddr_page
,
1153 index
= tlb_index(env
, mmu_idx
, vaddr_page
);
1154 te
= tlb_entry(env
, mmu_idx
, vaddr_page
);
1157 * Hold the TLB lock for the rest of the function. We could acquire/release
1158 * the lock several times in the function, but it is faster to amortize the
1159 * acquisition cost by acquiring it just once. Note that this leads to
1160 * a longer critical section, but this is not a concern since the TLB lock
1161 * is unlikely to be contended.
1163 qemu_spin_lock(&tlb
->c
.lock
);
1165 /* Note that the tlb is no longer clean. */
1166 tlb
->c
.dirty
|= 1 << mmu_idx
;
1168 /* Make sure there's no cached translation for the new page. */
1169 tlb_flush_vtlb_page_locked(env
, mmu_idx
, vaddr_page
);
1172 * Only evict the old entry to the victim tlb if it's for a
1173 * different page; otherwise just overwrite the stale data.
1175 if (!tlb_hit_page_anyprot(te
, vaddr_page
) && !tlb_entry_is_empty(te
)) {
1176 unsigned vidx
= desc
->vindex
++ % CPU_VTLB_SIZE
;
1177 CPUTLBEntry
*tv
= &desc
->vtable
[vidx
];
1179 /* Evict the old entry into the victim tlb. */
1180 copy_tlb_helper_locked(tv
, te
);
1181 desc
->viotlb
[vidx
] = desc
->iotlb
[index
];
1182 tlb_n_used_entries_dec(env
, mmu_idx
);
1185 /* refill the tlb */
1187 * At this point iotlb contains a physical section number in the lower
1188 * TARGET_PAGE_BITS, and either
1189 * + the ram_addr_t of the page base of the target RAM (RAM)
1190 * + the offset within section->mr of the page base (I/O, ROMD)
1191 * We subtract the vaddr_page (which is page aligned and thus won't
1192 * disturb the low bits) to give an offset which can be added to the
1193 * (non-page-aligned) vaddr of the eventual memory access to get
1194 * the MemoryRegion offset for the access. Note that the vaddr we
1195 * subtract here is that of the page base, and not the same as the
1196 * vaddr we add back in io_readx()/io_writex()/get_page_addr_code().
1198 desc
->iotlb
[index
].addr
= iotlb
- vaddr_page
;
1199 desc
->iotlb
[index
].attrs
= attrs
;
1201 /* Now calculate the new entry */
1202 tn
.addend
= addend
- vaddr_page
;
1203 if (prot
& PAGE_READ
) {
1204 tn
.addr_read
= address
;
1205 if (wp_flags
& BP_MEM_READ
) {
1206 tn
.addr_read
|= TLB_WATCHPOINT
;
1212 if (prot
& PAGE_EXEC
) {
1213 tn
.addr_code
= address
;
1219 if (prot
& PAGE_WRITE
) {
1220 tn
.addr_write
= write_address
;
1221 if (prot
& PAGE_WRITE_INV
) {
1222 tn
.addr_write
|= TLB_INVALID_MASK
;
1224 if (wp_flags
& BP_MEM_WRITE
) {
1225 tn
.addr_write
|= TLB_WATCHPOINT
;
1229 copy_tlb_helper_locked(te
, &tn
);
1230 tlb_n_used_entries_inc(env
, mmu_idx
);
1231 qemu_spin_unlock(&tlb
->c
.lock
);
1234 /* Add a new TLB entry, but without specifying the memory
1235 * transaction attributes to be used.
1237 void tlb_set_page(CPUState
*cpu
, target_ulong vaddr
,
1238 hwaddr paddr
, int prot
,
1239 int mmu_idx
, target_ulong size
)
1241 tlb_set_page_with_attrs(cpu
, vaddr
, paddr
, MEMTXATTRS_UNSPECIFIED
,
1242 prot
, mmu_idx
, size
);
1245 static inline ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
1247 ram_addr_t ram_addr
;
1249 ram_addr
= qemu_ram_addr_from_host(ptr
);
1250 if (ram_addr
== RAM_ADDR_INVALID
) {
1251 error_report("Bad ram pointer %p", ptr
);
1258 * Note: tlb_fill() can trigger a resize of the TLB. This means that all of the
1259 * caller's prior references to the TLB table (e.g. CPUTLBEntry pointers) must
1260 * be discarded and looked up again (e.g. via tlb_entry()).
1262 static void tlb_fill(CPUState
*cpu
, target_ulong addr
, int size
,
1263 MMUAccessType access_type
, int mmu_idx
, uintptr_t retaddr
)
1265 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
1269 * This is not a probe, so only valid return is success; failure
1270 * should result in exception + longjmp to the cpu loop.
1272 ok
= cc
->tcg_ops
->tlb_fill(cpu
, addr
, size
,
1273 access_type
, mmu_idx
, false, retaddr
);
1277 static inline void cpu_unaligned_access(CPUState
*cpu
, vaddr addr
,
1278 MMUAccessType access_type
,
1279 int mmu_idx
, uintptr_t retaddr
)
1281 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
1283 cc
->tcg_ops
->do_unaligned_access(cpu
, addr
, access_type
, mmu_idx
, retaddr
);
1286 static inline void cpu_transaction_failed(CPUState
*cpu
, hwaddr physaddr
,
1287 vaddr addr
, unsigned size
,
1288 MMUAccessType access_type
,
1289 int mmu_idx
, MemTxAttrs attrs
,
1290 MemTxResult response
,
1293 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
1295 if (!cpu
->ignore_memory_transaction_failures
&&
1296 cc
->tcg_ops
->do_transaction_failed
) {
1297 cc
->tcg_ops
->do_transaction_failed(cpu
, physaddr
, addr
, size
,
1298 access_type
, mmu_idx
, attrs
,
1303 static uint64_t io_readx(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
1304 int mmu_idx
, target_ulong addr
, uintptr_t retaddr
,
1305 MMUAccessType access_type
, MemOp op
)
1307 CPUState
*cpu
= env_cpu(env
);
1309 MemoryRegionSection
*section
;
1312 bool locked
= false;
1315 section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
1317 mr_offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
1318 cpu
->mem_io_pc
= retaddr
;
1319 if (!cpu
->can_do_io
) {
1320 cpu_io_recompile(cpu
, retaddr
);
1323 if (!qemu_mutex_iothread_locked()) {
1324 qemu_mutex_lock_iothread();
1327 r
= memory_region_dispatch_read(mr
, mr_offset
, &val
, op
, iotlbentry
->attrs
);
1328 if (r
!= MEMTX_OK
) {
1329 hwaddr physaddr
= mr_offset
+
1330 section
->offset_within_address_space
-
1331 section
->offset_within_region
;
1333 cpu_transaction_failed(cpu
, physaddr
, addr
, memop_size(op
), access_type
,
1334 mmu_idx
, iotlbentry
->attrs
, r
, retaddr
);
1337 qemu_mutex_unlock_iothread();
1344 * Save a potentially trashed IOTLB entry for later lookup by plugin.
1345 * This is read by tlb_plugin_lookup if the iotlb entry doesn't match
1346 * because of the side effect of io_writex changing memory layout.
1348 static void save_iotlb_data(CPUState
*cs
, hwaddr addr
,
1349 MemoryRegionSection
*section
, hwaddr mr_offset
)
1351 #ifdef CONFIG_PLUGIN
1352 SavedIOTLB
*saved
= &cs
->saved_iotlb
;
1354 saved
->section
= section
;
1355 saved
->mr_offset
= mr_offset
;
1359 static void io_writex(CPUArchState
*env
, CPUIOTLBEntry
*iotlbentry
,
1360 int mmu_idx
, uint64_t val
, target_ulong addr
,
1361 uintptr_t retaddr
, MemOp op
)
1363 CPUState
*cpu
= env_cpu(env
);
1365 MemoryRegionSection
*section
;
1367 bool locked
= false;
1370 section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
1372 mr_offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
1373 if (!cpu
->can_do_io
) {
1374 cpu_io_recompile(cpu
, retaddr
);
1376 cpu
->mem_io_pc
= retaddr
;
1379 * The memory_region_dispatch may trigger a flush/resize
1380 * so for plugins we save the iotlb_data just in case.
1382 save_iotlb_data(cpu
, iotlbentry
->addr
, section
, mr_offset
);
1384 if (!qemu_mutex_iothread_locked()) {
1385 qemu_mutex_lock_iothread();
1388 r
= memory_region_dispatch_write(mr
, mr_offset
, val
, op
, iotlbentry
->attrs
);
1389 if (r
!= MEMTX_OK
) {
1390 hwaddr physaddr
= mr_offset
+
1391 section
->offset_within_address_space
-
1392 section
->offset_within_region
;
1394 cpu_transaction_failed(cpu
, physaddr
, addr
, memop_size(op
),
1395 MMU_DATA_STORE
, mmu_idx
, iotlbentry
->attrs
, r
,
1399 qemu_mutex_unlock_iothread();
1403 static inline target_ulong
tlb_read_ofs(CPUTLBEntry
*entry
, size_t ofs
)
1405 #if TCG_OVERSIZED_GUEST
1406 return *(target_ulong
*)((uintptr_t)entry
+ ofs
);
1408 /* ofs might correspond to .addr_write, so use qatomic_read */
1409 return qatomic_read((target_ulong
*)((uintptr_t)entry
+ ofs
));
1413 /* Return true if ADDR is present in the victim tlb, and has been copied
1414 back to the main tlb. */
1415 static bool victim_tlb_hit(CPUArchState
*env
, size_t mmu_idx
, size_t index
,
1416 size_t elt_ofs
, target_ulong page
)
1420 assert_cpu_is_self(env_cpu(env
));
1421 for (vidx
= 0; vidx
< CPU_VTLB_SIZE
; ++vidx
) {
1422 CPUTLBEntry
*vtlb
= &env_tlb(env
)->d
[mmu_idx
].vtable
[vidx
];
1425 /* elt_ofs might correspond to .addr_write, so use qatomic_read */
1426 #if TCG_OVERSIZED_GUEST
1427 cmp
= *(target_ulong
*)((uintptr_t)vtlb
+ elt_ofs
);
1429 cmp
= qatomic_read((target_ulong
*)((uintptr_t)vtlb
+ elt_ofs
));
1433 /* Found entry in victim tlb, swap tlb and iotlb. */
1434 CPUTLBEntry tmptlb
, *tlb
= &env_tlb(env
)->f
[mmu_idx
].table
[index
];
1436 qemu_spin_lock(&env_tlb(env
)->c
.lock
);
1437 copy_tlb_helper_locked(&tmptlb
, tlb
);
1438 copy_tlb_helper_locked(tlb
, vtlb
);
1439 copy_tlb_helper_locked(vtlb
, &tmptlb
);
1440 qemu_spin_unlock(&env_tlb(env
)->c
.lock
);
1442 CPUIOTLBEntry tmpio
, *io
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1443 CPUIOTLBEntry
*vio
= &env_tlb(env
)->d
[mmu_idx
].viotlb
[vidx
];
1444 tmpio
= *io
; *io
= *vio
; *vio
= tmpio
;
1451 /* Macro to call the above, with local variables from the use context. */
1452 #define VICTIM_TLB_HIT(TY, ADDR) \
1453 victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
1454 (ADDR) & TARGET_PAGE_MASK)
1457 * Return a ram_addr_t for the virtual address for execution.
1459 * Return -1 if we can't translate and execute from an entire page
1460 * of RAM. This will force us to execute by loading and translating
1461 * one insn at a time, without caching.
1463 * NOTE: This function will trigger an exception if the page is
1466 tb_page_addr_t
get_page_addr_code_hostp(CPUArchState
*env
, target_ulong addr
,
1469 uintptr_t mmu_idx
= cpu_mmu_index(env
, true);
1470 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1471 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1474 if (unlikely(!tlb_hit(entry
->addr_code
, addr
))) {
1475 if (!VICTIM_TLB_HIT(addr_code
, addr
)) {
1476 tlb_fill(env_cpu(env
), addr
, 0, MMU_INST_FETCH
, mmu_idx
, 0);
1477 index
= tlb_index(env
, mmu_idx
, addr
);
1478 entry
= tlb_entry(env
, mmu_idx
, addr
);
1480 if (unlikely(entry
->addr_code
& TLB_INVALID_MASK
)) {
1482 * The MMU protection covers a smaller range than a target
1483 * page, so we must redo the MMU check for every insn.
1488 assert(tlb_hit(entry
->addr_code
, addr
));
1491 if (unlikely(entry
->addr_code
& TLB_MMIO
)) {
1492 /* The region is not backed by RAM. */
1499 p
= (void *)((uintptr_t)addr
+ entry
->addend
);
1503 return qemu_ram_addr_from_host_nofail(p
);
1506 tb_page_addr_t
get_page_addr_code(CPUArchState
*env
, target_ulong addr
)
1508 return get_page_addr_code_hostp(env
, addr
, NULL
);
1511 static void notdirty_write(CPUState
*cpu
, vaddr mem_vaddr
, unsigned size
,
1512 CPUIOTLBEntry
*iotlbentry
, uintptr_t retaddr
)
1514 ram_addr_t ram_addr
= mem_vaddr
+ iotlbentry
->addr
;
1516 trace_memory_notdirty_write_access(mem_vaddr
, ram_addr
, size
);
1518 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1519 struct page_collection
*pages
1520 = page_collection_lock(ram_addr
, ram_addr
+ size
);
1521 tb_invalidate_phys_page_fast(pages
, ram_addr
, size
, retaddr
);
1522 page_collection_unlock(pages
);
1526 * Set both VGA and migration bits for simplicity and to remove
1527 * the notdirty callback faster.
1529 cpu_physical_memory_set_dirty_range(ram_addr
, size
, DIRTY_CLIENTS_NOCODE
);
1531 /* We remove the notdirty callback only if the code has been flushed. */
1532 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1533 trace_memory_notdirty_set_dirty(mem_vaddr
);
1534 tlb_set_dirty(cpu
, mem_vaddr
);
1538 static int probe_access_internal(CPUArchState
*env
, target_ulong addr
,
1539 int fault_size
, MMUAccessType access_type
,
1540 int mmu_idx
, bool nonfault
,
1541 void **phost
, uintptr_t retaddr
)
1543 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1544 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1545 target_ulong tlb_addr
, page_addr
;
1549 switch (access_type
) {
1551 elt_ofs
= offsetof(CPUTLBEntry
, addr_read
);
1553 case MMU_DATA_STORE
:
1554 elt_ofs
= offsetof(CPUTLBEntry
, addr_write
);
1556 case MMU_INST_FETCH
:
1557 elt_ofs
= offsetof(CPUTLBEntry
, addr_code
);
1560 g_assert_not_reached();
1562 tlb_addr
= tlb_read_ofs(entry
, elt_ofs
);
1564 page_addr
= addr
& TARGET_PAGE_MASK
;
1565 if (!tlb_hit_page(tlb_addr
, page_addr
)) {
1566 if (!victim_tlb_hit(env
, mmu_idx
, index
, elt_ofs
, page_addr
)) {
1567 CPUState
*cs
= env_cpu(env
);
1568 CPUClass
*cc
= CPU_GET_CLASS(cs
);
1570 if (!cc
->tcg_ops
->tlb_fill(cs
, addr
, fault_size
, access_type
,
1571 mmu_idx
, nonfault
, retaddr
)) {
1572 /* Non-faulting page table read failed. */
1574 return TLB_INVALID_MASK
;
1577 /* TLB resize via tlb_fill may have moved the entry. */
1578 entry
= tlb_entry(env
, mmu_idx
, addr
);
1580 tlb_addr
= tlb_read_ofs(entry
, elt_ofs
);
1582 flags
= tlb_addr
& TLB_FLAGS_MASK
;
1584 /* Fold all "mmio-like" bits into TLB_MMIO. This is not RAM. */
1585 if (unlikely(flags
& ~(TLB_WATCHPOINT
| TLB_NOTDIRTY
))) {
1590 /* Everything else is RAM. */
1591 *phost
= (void *)((uintptr_t)addr
+ entry
->addend
);
1595 int probe_access_flags(CPUArchState
*env
, target_ulong addr
,
1596 MMUAccessType access_type
, int mmu_idx
,
1597 bool nonfault
, void **phost
, uintptr_t retaddr
)
1601 flags
= probe_access_internal(env
, addr
, 0, access_type
, mmu_idx
,
1602 nonfault
, phost
, retaddr
);
1604 /* Handle clean RAM pages. */
1605 if (unlikely(flags
& TLB_NOTDIRTY
)) {
1606 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1607 CPUIOTLBEntry
*iotlbentry
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1609 notdirty_write(env_cpu(env
), addr
, 1, iotlbentry
, retaddr
);
1610 flags
&= ~TLB_NOTDIRTY
;
1616 void *probe_access(CPUArchState
*env
, target_ulong addr
, int size
,
1617 MMUAccessType access_type
, int mmu_idx
, uintptr_t retaddr
)
1622 g_assert(-(addr
| TARGET_PAGE_MASK
) >= size
);
1624 flags
= probe_access_internal(env
, addr
, size
, access_type
, mmu_idx
,
1625 false, &host
, retaddr
);
1627 /* Per the interface, size == 0 merely faults the access. */
1632 if (unlikely(flags
& (TLB_NOTDIRTY
| TLB_WATCHPOINT
))) {
1633 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1634 CPUIOTLBEntry
*iotlbentry
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1636 /* Handle watchpoints. */
1637 if (flags
& TLB_WATCHPOINT
) {
1638 int wp_access
= (access_type
== MMU_DATA_STORE
1639 ? BP_MEM_WRITE
: BP_MEM_READ
);
1640 cpu_check_watchpoint(env_cpu(env
), addr
, size
,
1641 iotlbentry
->attrs
, wp_access
, retaddr
);
1644 /* Handle clean RAM pages. */
1645 if (flags
& TLB_NOTDIRTY
) {
1646 notdirty_write(env_cpu(env
), addr
, 1, iotlbentry
, retaddr
);
1653 void *tlb_vaddr_to_host(CPUArchState
*env
, abi_ptr addr
,
1654 MMUAccessType access_type
, int mmu_idx
)
1659 flags
= probe_access_internal(env
, addr
, 0, access_type
,
1660 mmu_idx
, true, &host
, 0);
1662 /* No combination of flags are expected by the caller. */
1663 return flags
? NULL
: host
;
1666 #ifdef CONFIG_PLUGIN
1668 * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure.
1669 * This should be a hot path as we will have just looked this path up
1670 * in the softmmu lookup code (or helper). We don't handle re-fills or
1671 * checking the victim table. This is purely informational.
1673 * This almost never fails as the memory access being instrumented
1674 * should have just filled the TLB. The one corner case is io_writex
1675 * which can cause TLB flushes and potential resizing of the TLBs
1676 * losing the information we need. In those cases we need to recover
1677 * data from a copy of the iotlbentry. As long as this always occurs
1678 * from the same thread (which a mem callback will be) this is safe.
1681 bool tlb_plugin_lookup(CPUState
*cpu
, target_ulong addr
, int mmu_idx
,
1682 bool is_store
, struct qemu_plugin_hwaddr
*data
)
1684 CPUArchState
*env
= cpu
->env_ptr
;
1685 CPUTLBEntry
*tlbe
= tlb_entry(env
, mmu_idx
, addr
);
1686 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1687 target_ulong tlb_addr
= is_store
? tlb_addr_write(tlbe
) : tlbe
->addr_read
;
1689 if (likely(tlb_hit(tlb_addr
, addr
))) {
1690 /* We must have an iotlb entry for MMIO */
1691 if (tlb_addr
& TLB_MMIO
) {
1692 CPUIOTLBEntry
*iotlbentry
;
1693 iotlbentry
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1695 data
->v
.io
.section
= iotlb_to_section(cpu
, iotlbentry
->addr
, iotlbentry
->attrs
);
1696 data
->v
.io
.offset
= (iotlbentry
->addr
& TARGET_PAGE_MASK
) + addr
;
1698 data
->is_io
= false;
1699 data
->v
.ram
.hostaddr
= addr
+ tlbe
->addend
;
1703 SavedIOTLB
*saved
= &cpu
->saved_iotlb
;
1705 data
->v
.io
.section
= saved
->section
;
1706 data
->v
.io
.offset
= saved
->mr_offset
;
1713 /* Probe for a read-modify-write atomic operation. Do not allow unaligned
1714 * operations, or io operations to proceed. Return the host address. */
1715 static void *atomic_mmu_lookup(CPUArchState
*env
, target_ulong addr
,
1716 TCGMemOpIdx oi
, uintptr_t retaddr
)
1718 size_t mmu_idx
= get_mmuidx(oi
);
1719 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1720 CPUTLBEntry
*tlbe
= tlb_entry(env
, mmu_idx
, addr
);
1721 target_ulong tlb_addr
= tlb_addr_write(tlbe
);
1722 MemOp mop
= get_memop(oi
);
1723 int a_bits
= get_alignment_bits(mop
);
1724 int s_bits
= mop
& MO_SIZE
;
1727 /* Adjust the given return address. */
1728 retaddr
-= GETPC_ADJ
;
1730 /* Enforce guest required alignment. */
1731 if (unlikely(a_bits
> 0 && (addr
& ((1 << a_bits
) - 1)))) {
1732 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
1733 cpu_unaligned_access(env_cpu(env
), addr
, MMU_DATA_STORE
,
1737 /* Enforce qemu required alignment. */
1738 if (unlikely(addr
& ((1 << s_bits
) - 1))) {
1739 /* We get here if guest alignment was not requested,
1740 or was not enforced by cpu_unaligned_access above.
1741 We might widen the access and emulate, but for now
1742 mark an exception and exit the cpu loop. */
1743 goto stop_the_world
;
1746 /* Check TLB entry and enforce page permissions. */
1747 if (!tlb_hit(tlb_addr
, addr
)) {
1748 if (!VICTIM_TLB_HIT(addr_write
, addr
)) {
1749 tlb_fill(env_cpu(env
), addr
, 1 << s_bits
, MMU_DATA_STORE
,
1751 index
= tlb_index(env
, mmu_idx
, addr
);
1752 tlbe
= tlb_entry(env
, mmu_idx
, addr
);
1754 tlb_addr
= tlb_addr_write(tlbe
) & ~TLB_INVALID_MASK
;
1757 /* Notice an IO access or a needs-MMU-lookup access */
1758 if (unlikely(tlb_addr
& TLB_MMIO
)) {
1759 /* There's really nothing that can be done to
1760 support this apart from stop-the-world. */
1761 goto stop_the_world
;
1764 /* Let the guest notice RMW on a write-only page. */
1765 if (unlikely(tlbe
->addr_read
!= (tlb_addr
& ~TLB_NOTDIRTY
))) {
1766 tlb_fill(env_cpu(env
), addr
, 1 << s_bits
, MMU_DATA_LOAD
,
1768 /* Since we don't support reads and writes to different addresses,
1769 and we do have the proper page loaded for write, this shouldn't
1770 ever return. But just in case, handle via stop-the-world. */
1771 goto stop_the_world
;
1774 hostaddr
= (void *)((uintptr_t)addr
+ tlbe
->addend
);
1776 if (unlikely(tlb_addr
& TLB_NOTDIRTY
)) {
1777 notdirty_write(env_cpu(env
), addr
, 1 << s_bits
,
1778 &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
], retaddr
);
1784 cpu_loop_exit_atomic(env_cpu(env
), retaddr
);
1790 * We support two different access types. SOFTMMU_CODE_ACCESS is
1791 * specifically for reading instructions from system memory. It is
1792 * called by the translation loop and in some helpers where the code
1793 * is disassembled. It shouldn't be called directly by guest code.
1796 typedef uint64_t FullLoadHelper(CPUArchState
*env
, target_ulong addr
,
1797 TCGMemOpIdx oi
, uintptr_t retaddr
);
1799 static inline uint64_t QEMU_ALWAYS_INLINE
1800 load_memop(const void *haddr
, MemOp op
)
1804 return ldub_p(haddr
);
1806 return lduw_be_p(haddr
);
1808 return lduw_le_p(haddr
);
1810 return (uint32_t)ldl_be_p(haddr
);
1812 return (uint32_t)ldl_le_p(haddr
);
1814 return ldq_be_p(haddr
);
1816 return ldq_le_p(haddr
);
1818 qemu_build_not_reached();
1822 static inline uint64_t QEMU_ALWAYS_INLINE
1823 load_helper(CPUArchState
*env
, target_ulong addr
, TCGMemOpIdx oi
,
1824 uintptr_t retaddr
, MemOp op
, bool code_read
,
1825 FullLoadHelper
*full_load
)
1827 uintptr_t mmu_idx
= get_mmuidx(oi
);
1828 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
1829 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
1830 target_ulong tlb_addr
= code_read
? entry
->addr_code
: entry
->addr_read
;
1831 const size_t tlb_off
= code_read
?
1832 offsetof(CPUTLBEntry
, addr_code
) : offsetof(CPUTLBEntry
, addr_read
);
1833 const MMUAccessType access_type
=
1834 code_read
? MMU_INST_FETCH
: MMU_DATA_LOAD
;
1835 unsigned a_bits
= get_alignment_bits(get_memop(oi
));
1838 size_t size
= memop_size(op
);
1840 /* Handle CPU specific unaligned behaviour */
1841 if (addr
& ((1 << a_bits
) - 1)) {
1842 cpu_unaligned_access(env_cpu(env
), addr
, access_type
,
1846 /* If the TLB entry is for a different page, reload and try again. */
1847 if (!tlb_hit(tlb_addr
, addr
)) {
1848 if (!victim_tlb_hit(env
, mmu_idx
, index
, tlb_off
,
1849 addr
& TARGET_PAGE_MASK
)) {
1850 tlb_fill(env_cpu(env
), addr
, size
,
1851 access_type
, mmu_idx
, retaddr
);
1852 index
= tlb_index(env
, mmu_idx
, addr
);
1853 entry
= tlb_entry(env
, mmu_idx
, addr
);
1855 tlb_addr
= code_read
? entry
->addr_code
: entry
->addr_read
;
1856 tlb_addr
&= ~TLB_INVALID_MASK
;
1859 /* Handle anything that isn't just a straight memory access. */
1860 if (unlikely(tlb_addr
& ~TARGET_PAGE_MASK
)) {
1861 CPUIOTLBEntry
*iotlbentry
;
1864 /* For anything that is unaligned, recurse through full_load. */
1865 if ((addr
& (size
- 1)) != 0) {
1866 goto do_unaligned_access
;
1869 iotlbentry
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
1871 /* Handle watchpoints. */
1872 if (unlikely(tlb_addr
& TLB_WATCHPOINT
)) {
1873 /* On watchpoint hit, this will longjmp out. */
1874 cpu_check_watchpoint(env_cpu(env
), addr
, size
,
1875 iotlbentry
->attrs
, BP_MEM_READ
, retaddr
);
1878 need_swap
= size
> 1 && (tlb_addr
& TLB_BSWAP
);
1880 /* Handle I/O access. */
1881 if (likely(tlb_addr
& TLB_MMIO
)) {
1882 return io_readx(env
, iotlbentry
, mmu_idx
, addr
, retaddr
,
1883 access_type
, op
^ (need_swap
* MO_BSWAP
));
1886 haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
1889 * Keep these two load_memop separate to ensure that the compiler
1890 * is able to fold the entire function to a single instruction.
1891 * There is a build-time assert inside to remind you of this. ;-)
1893 if (unlikely(need_swap
)) {
1894 return load_memop(haddr
, op
^ MO_BSWAP
);
1896 return load_memop(haddr
, op
);
1899 /* Handle slow unaligned access (it spans two pages or IO). */
1901 && unlikely((addr
& ~TARGET_PAGE_MASK
) + size
- 1
1902 >= TARGET_PAGE_SIZE
)) {
1903 target_ulong addr1
, addr2
;
1906 do_unaligned_access
:
1907 addr1
= addr
& ~((target_ulong
)size
- 1);
1908 addr2
= addr1
+ size
;
1909 r1
= full_load(env
, addr1
, oi
, retaddr
);
1910 r2
= full_load(env
, addr2
, oi
, retaddr
);
1911 shift
= (addr
& (size
- 1)) * 8;
1913 if (memop_big_endian(op
)) {
1914 /* Big-endian combine. */
1915 res
= (r1
<< shift
) | (r2
>> ((size
* 8) - shift
));
1917 /* Little-endian combine. */
1918 res
= (r1
>> shift
) | (r2
<< ((size
* 8) - shift
));
1920 return res
& MAKE_64BIT_MASK(0, size
* 8);
1923 haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
1924 return load_memop(haddr
, op
);
1928 * For the benefit of TCG generated code, we want to avoid the
1929 * complication of ABI-specific return type promotion and always
1930 * return a value extended to the register size of the host. This is
1931 * tcg_target_long, except in the case of a 32-bit host and 64-bit
1932 * data, and for that we always have uint64_t.
1934 * We don't bother with this widened value for SOFTMMU_CODE_ACCESS.
1937 static uint64_t full_ldub_mmu(CPUArchState
*env
, target_ulong addr
,
1938 TCGMemOpIdx oi
, uintptr_t retaddr
)
1940 return load_helper(env
, addr
, oi
, retaddr
, MO_UB
, false, full_ldub_mmu
);
1943 tcg_target_ulong
helper_ret_ldub_mmu(CPUArchState
*env
, target_ulong addr
,
1944 TCGMemOpIdx oi
, uintptr_t retaddr
)
1946 return full_ldub_mmu(env
, addr
, oi
, retaddr
);
1949 static uint64_t full_le_lduw_mmu(CPUArchState
*env
, target_ulong addr
,
1950 TCGMemOpIdx oi
, uintptr_t retaddr
)
1952 return load_helper(env
, addr
, oi
, retaddr
, MO_LEUW
, false,
1956 tcg_target_ulong
helper_le_lduw_mmu(CPUArchState
*env
, target_ulong addr
,
1957 TCGMemOpIdx oi
, uintptr_t retaddr
)
1959 return full_le_lduw_mmu(env
, addr
, oi
, retaddr
);
1962 static uint64_t full_be_lduw_mmu(CPUArchState
*env
, target_ulong addr
,
1963 TCGMemOpIdx oi
, uintptr_t retaddr
)
1965 return load_helper(env
, addr
, oi
, retaddr
, MO_BEUW
, false,
1969 tcg_target_ulong
helper_be_lduw_mmu(CPUArchState
*env
, target_ulong addr
,
1970 TCGMemOpIdx oi
, uintptr_t retaddr
)
1972 return full_be_lduw_mmu(env
, addr
, oi
, retaddr
);
1975 static uint64_t full_le_ldul_mmu(CPUArchState
*env
, target_ulong addr
,
1976 TCGMemOpIdx oi
, uintptr_t retaddr
)
1978 return load_helper(env
, addr
, oi
, retaddr
, MO_LEUL
, false,
1982 tcg_target_ulong
helper_le_ldul_mmu(CPUArchState
*env
, target_ulong addr
,
1983 TCGMemOpIdx oi
, uintptr_t retaddr
)
1985 return full_le_ldul_mmu(env
, addr
, oi
, retaddr
);
1988 static uint64_t full_be_ldul_mmu(CPUArchState
*env
, target_ulong addr
,
1989 TCGMemOpIdx oi
, uintptr_t retaddr
)
1991 return load_helper(env
, addr
, oi
, retaddr
, MO_BEUL
, false,
1995 tcg_target_ulong
helper_be_ldul_mmu(CPUArchState
*env
, target_ulong addr
,
1996 TCGMemOpIdx oi
, uintptr_t retaddr
)
1998 return full_be_ldul_mmu(env
, addr
, oi
, retaddr
);
2001 uint64_t helper_le_ldq_mmu(CPUArchState
*env
, target_ulong addr
,
2002 TCGMemOpIdx oi
, uintptr_t retaddr
)
2004 return load_helper(env
, addr
, oi
, retaddr
, MO_LEQ
, false,
2008 uint64_t helper_be_ldq_mmu(CPUArchState
*env
, target_ulong addr
,
2009 TCGMemOpIdx oi
, uintptr_t retaddr
)
2011 return load_helper(env
, addr
, oi
, retaddr
, MO_BEQ
, false,
2016 * Provide signed versions of the load routines as well. We can of course
2017 * avoid this for 64-bit data, or for 32-bit data on 32-bit host.
2021 tcg_target_ulong
helper_ret_ldsb_mmu(CPUArchState
*env
, target_ulong addr
,
2022 TCGMemOpIdx oi
, uintptr_t retaddr
)
2024 return (int8_t)helper_ret_ldub_mmu(env
, addr
, oi
, retaddr
);
2027 tcg_target_ulong
helper_le_ldsw_mmu(CPUArchState
*env
, target_ulong addr
,
2028 TCGMemOpIdx oi
, uintptr_t retaddr
)
2030 return (int16_t)helper_le_lduw_mmu(env
, addr
, oi
, retaddr
);
2033 tcg_target_ulong
helper_be_ldsw_mmu(CPUArchState
*env
, target_ulong addr
,
2034 TCGMemOpIdx oi
, uintptr_t retaddr
)
2036 return (int16_t)helper_be_lduw_mmu(env
, addr
, oi
, retaddr
);
2039 tcg_target_ulong
helper_le_ldsl_mmu(CPUArchState
*env
, target_ulong addr
,
2040 TCGMemOpIdx oi
, uintptr_t retaddr
)
2042 return (int32_t)helper_le_ldul_mmu(env
, addr
, oi
, retaddr
);
2045 tcg_target_ulong
helper_be_ldsl_mmu(CPUArchState
*env
, target_ulong addr
,
2046 TCGMemOpIdx oi
, uintptr_t retaddr
)
2048 return (int32_t)helper_be_ldul_mmu(env
, addr
, oi
, retaddr
);
2052 * Load helpers for cpu_ldst.h.
2055 static inline uint64_t cpu_load_helper(CPUArchState
*env
, abi_ptr addr
,
2056 int mmu_idx
, uintptr_t retaddr
,
2057 MemOp op
, FullLoadHelper
*full_load
)
2063 meminfo
= trace_mem_get_info(op
, mmu_idx
, false);
2064 trace_guest_mem_before_exec(env_cpu(env
), addr
, meminfo
);
2067 oi
= make_memop_idx(op
, mmu_idx
);
2068 ret
= full_load(env
, addr
, oi
, retaddr
);
2070 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, meminfo
);
2075 uint32_t cpu_ldub_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2076 int mmu_idx
, uintptr_t ra
)
2078 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_UB
, full_ldub_mmu
);
2081 int cpu_ldsb_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2082 int mmu_idx
, uintptr_t ra
)
2084 return (int8_t)cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_SB
,
2088 uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2089 int mmu_idx
, uintptr_t ra
)
2091 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_BEUW
, full_be_lduw_mmu
);
2094 int cpu_ldsw_be_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2095 int mmu_idx
, uintptr_t ra
)
2097 return (int16_t)cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_BESW
,
2101 uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2102 int mmu_idx
, uintptr_t ra
)
2104 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_BEUL
, full_be_ldul_mmu
);
2107 uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2108 int mmu_idx
, uintptr_t ra
)
2110 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_BEQ
, helper_be_ldq_mmu
);
2113 uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2114 int mmu_idx
, uintptr_t ra
)
2116 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_LEUW
, full_le_lduw_mmu
);
2119 int cpu_ldsw_le_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2120 int mmu_idx
, uintptr_t ra
)
2122 return (int16_t)cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_LESW
,
2126 uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2127 int mmu_idx
, uintptr_t ra
)
2129 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_LEUL
, full_le_ldul_mmu
);
2132 uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState
*env
, abi_ptr addr
,
2133 int mmu_idx
, uintptr_t ra
)
2135 return cpu_load_helper(env
, addr
, mmu_idx
, ra
, MO_LEQ
, helper_le_ldq_mmu
);
2138 uint32_t cpu_ldub_data_ra(CPUArchState
*env
, target_ulong ptr
,
2141 return cpu_ldub_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2144 int cpu_ldsb_data_ra(CPUArchState
*env
, target_ulong ptr
, uintptr_t retaddr
)
2146 return cpu_ldsb_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2149 uint32_t cpu_lduw_be_data_ra(CPUArchState
*env
, target_ulong ptr
,
2152 return cpu_lduw_be_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2155 int cpu_ldsw_be_data_ra(CPUArchState
*env
, target_ulong ptr
, uintptr_t retaddr
)
2157 return cpu_ldsw_be_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2160 uint32_t cpu_ldl_be_data_ra(CPUArchState
*env
, target_ulong ptr
,
2163 return cpu_ldl_be_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2166 uint64_t cpu_ldq_be_data_ra(CPUArchState
*env
, target_ulong ptr
,
2169 return cpu_ldq_be_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2172 uint32_t cpu_lduw_le_data_ra(CPUArchState
*env
, target_ulong ptr
,
2175 return cpu_lduw_le_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2178 int cpu_ldsw_le_data_ra(CPUArchState
*env
, target_ulong ptr
, uintptr_t retaddr
)
2180 return cpu_ldsw_le_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2183 uint32_t cpu_ldl_le_data_ra(CPUArchState
*env
, target_ulong ptr
,
2186 return cpu_ldl_le_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2189 uint64_t cpu_ldq_le_data_ra(CPUArchState
*env
, target_ulong ptr
,
2192 return cpu_ldq_le_mmuidx_ra(env
, ptr
, cpu_mmu_index(env
, false), retaddr
);
2195 uint32_t cpu_ldub_data(CPUArchState
*env
, target_ulong ptr
)
2197 return cpu_ldub_data_ra(env
, ptr
, 0);
2200 int cpu_ldsb_data(CPUArchState
*env
, target_ulong ptr
)
2202 return cpu_ldsb_data_ra(env
, ptr
, 0);
2205 uint32_t cpu_lduw_be_data(CPUArchState
*env
, target_ulong ptr
)
2207 return cpu_lduw_be_data_ra(env
, ptr
, 0);
2210 int cpu_ldsw_be_data(CPUArchState
*env
, target_ulong ptr
)
2212 return cpu_ldsw_be_data_ra(env
, ptr
, 0);
2215 uint32_t cpu_ldl_be_data(CPUArchState
*env
, target_ulong ptr
)
2217 return cpu_ldl_be_data_ra(env
, ptr
, 0);
2220 uint64_t cpu_ldq_be_data(CPUArchState
*env
, target_ulong ptr
)
2222 return cpu_ldq_be_data_ra(env
, ptr
, 0);
2225 uint32_t cpu_lduw_le_data(CPUArchState
*env
, target_ulong ptr
)
2227 return cpu_lduw_le_data_ra(env
, ptr
, 0);
2230 int cpu_ldsw_le_data(CPUArchState
*env
, target_ulong ptr
)
2232 return cpu_ldsw_le_data_ra(env
, ptr
, 0);
2235 uint32_t cpu_ldl_le_data(CPUArchState
*env
, target_ulong ptr
)
2237 return cpu_ldl_le_data_ra(env
, ptr
, 0);
2240 uint64_t cpu_ldq_le_data(CPUArchState
*env
, target_ulong ptr
)
2242 return cpu_ldq_le_data_ra(env
, ptr
, 0);
2249 static inline void QEMU_ALWAYS_INLINE
2250 store_memop(void *haddr
, uint64_t val
, MemOp op
)
2257 stw_be_p(haddr
, val
);
2260 stw_le_p(haddr
, val
);
2263 stl_be_p(haddr
, val
);
2266 stl_le_p(haddr
, val
);
2269 stq_be_p(haddr
, val
);
2272 stq_le_p(haddr
, val
);
2275 qemu_build_not_reached();
2279 static void __attribute__((noinline
))
2280 store_helper_unaligned(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
2281 uintptr_t retaddr
, size_t size
, uintptr_t mmu_idx
,
2284 const size_t tlb_off
= offsetof(CPUTLBEntry
, addr_write
);
2285 uintptr_t index
, index2
;
2286 CPUTLBEntry
*entry
, *entry2
;
2287 target_ulong page2
, tlb_addr
, tlb_addr2
;
2293 * Ensure the second page is in the TLB. Note that the first page
2294 * is already guaranteed to be filled, and that the second page
2295 * cannot evict the first.
2297 page2
= (addr
+ size
) & TARGET_PAGE_MASK
;
2298 size2
= (addr
+ size
) & ~TARGET_PAGE_MASK
;
2299 index2
= tlb_index(env
, mmu_idx
, page2
);
2300 entry2
= tlb_entry(env
, mmu_idx
, page2
);
2302 tlb_addr2
= tlb_addr_write(entry2
);
2303 if (!tlb_hit_page(tlb_addr2
, page2
)) {
2304 if (!victim_tlb_hit(env
, mmu_idx
, index2
, tlb_off
, page2
)) {
2305 tlb_fill(env_cpu(env
), page2
, size2
, MMU_DATA_STORE
,
2307 index2
= tlb_index(env
, mmu_idx
, page2
);
2308 entry2
= tlb_entry(env
, mmu_idx
, page2
);
2310 tlb_addr2
= tlb_addr_write(entry2
);
2313 index
= tlb_index(env
, mmu_idx
, addr
);
2314 entry
= tlb_entry(env
, mmu_idx
, addr
);
2315 tlb_addr
= tlb_addr_write(entry
);
2318 * Handle watchpoints. Since this may trap, all checks
2319 * must happen before any store.
2321 if (unlikely(tlb_addr
& TLB_WATCHPOINT
)) {
2322 cpu_check_watchpoint(env_cpu(env
), addr
, size
- size2
,
2323 env_tlb(env
)->d
[mmu_idx
].iotlb
[index
].attrs
,
2324 BP_MEM_WRITE
, retaddr
);
2326 if (unlikely(tlb_addr2
& TLB_WATCHPOINT
)) {
2327 cpu_check_watchpoint(env_cpu(env
), page2
, size2
,
2328 env_tlb(env
)->d
[mmu_idx
].iotlb
[index2
].attrs
,
2329 BP_MEM_WRITE
, retaddr
);
2333 * XXX: not efficient, but simple.
2334 * This loop must go in the forward direction to avoid issues
2335 * with self-modifying code in Windows 64-bit.
2337 oi
= make_memop_idx(MO_UB
, mmu_idx
);
2339 for (i
= 0; i
< size
; ++i
) {
2340 /* Big-endian extract. */
2341 uint8_t val8
= val
>> (((size
- 1) * 8) - (i
* 8));
2342 helper_ret_stb_mmu(env
, addr
+ i
, val8
, oi
, retaddr
);
2345 for (i
= 0; i
< size
; ++i
) {
2346 /* Little-endian extract. */
2347 uint8_t val8
= val
>> (i
* 8);
2348 helper_ret_stb_mmu(env
, addr
+ i
, val8
, oi
, retaddr
);
2353 static inline void QEMU_ALWAYS_INLINE
2354 store_helper(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
2355 TCGMemOpIdx oi
, uintptr_t retaddr
, MemOp op
)
2357 uintptr_t mmu_idx
= get_mmuidx(oi
);
2358 uintptr_t index
= tlb_index(env
, mmu_idx
, addr
);
2359 CPUTLBEntry
*entry
= tlb_entry(env
, mmu_idx
, addr
);
2360 target_ulong tlb_addr
= tlb_addr_write(entry
);
2361 const size_t tlb_off
= offsetof(CPUTLBEntry
, addr_write
);
2362 unsigned a_bits
= get_alignment_bits(get_memop(oi
));
2364 size_t size
= memop_size(op
);
2366 /* Handle CPU specific unaligned behaviour */
2367 if (addr
& ((1 << a_bits
) - 1)) {
2368 cpu_unaligned_access(env_cpu(env
), addr
, MMU_DATA_STORE
,
2372 /* If the TLB entry is for a different page, reload and try again. */
2373 if (!tlb_hit(tlb_addr
, addr
)) {
2374 if (!victim_tlb_hit(env
, mmu_idx
, index
, tlb_off
,
2375 addr
& TARGET_PAGE_MASK
)) {
2376 tlb_fill(env_cpu(env
), addr
, size
, MMU_DATA_STORE
,
2378 index
= tlb_index(env
, mmu_idx
, addr
);
2379 entry
= tlb_entry(env
, mmu_idx
, addr
);
2381 tlb_addr
= tlb_addr_write(entry
) & ~TLB_INVALID_MASK
;
2384 /* Handle anything that isn't just a straight memory access. */
2385 if (unlikely(tlb_addr
& ~TARGET_PAGE_MASK
)) {
2386 CPUIOTLBEntry
*iotlbentry
;
2389 /* For anything that is unaligned, recurse through byte stores. */
2390 if ((addr
& (size
- 1)) != 0) {
2391 goto do_unaligned_access
;
2394 iotlbentry
= &env_tlb(env
)->d
[mmu_idx
].iotlb
[index
];
2396 /* Handle watchpoints. */
2397 if (unlikely(tlb_addr
& TLB_WATCHPOINT
)) {
2398 /* On watchpoint hit, this will longjmp out. */
2399 cpu_check_watchpoint(env_cpu(env
), addr
, size
,
2400 iotlbentry
->attrs
, BP_MEM_WRITE
, retaddr
);
2403 need_swap
= size
> 1 && (tlb_addr
& TLB_BSWAP
);
2405 /* Handle I/O access. */
2406 if (tlb_addr
& TLB_MMIO
) {
2407 io_writex(env
, iotlbentry
, mmu_idx
, val
, addr
, retaddr
,
2408 op
^ (need_swap
* MO_BSWAP
));
2412 /* Ignore writes to ROM. */
2413 if (unlikely(tlb_addr
& TLB_DISCARD_WRITE
)) {
2417 /* Handle clean RAM pages. */
2418 if (tlb_addr
& TLB_NOTDIRTY
) {
2419 notdirty_write(env_cpu(env
), addr
, size
, iotlbentry
, retaddr
);
2422 haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
2425 * Keep these two store_memop separate to ensure that the compiler
2426 * is able to fold the entire function to a single instruction.
2427 * There is a build-time assert inside to remind you of this. ;-)
2429 if (unlikely(need_swap
)) {
2430 store_memop(haddr
, val
, op
^ MO_BSWAP
);
2432 store_memop(haddr
, val
, op
);
2437 /* Handle slow unaligned access (it spans two pages or IO). */
2439 && unlikely((addr
& ~TARGET_PAGE_MASK
) + size
- 1
2440 >= TARGET_PAGE_SIZE
)) {
2441 do_unaligned_access
:
2442 store_helper_unaligned(env
, addr
, val
, retaddr
, size
,
2443 mmu_idx
, memop_big_endian(op
));
2447 haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
2448 store_memop(haddr
, val
, op
);
2451 void __attribute__((noinline
))
2452 helper_ret_stb_mmu(CPUArchState
*env
, target_ulong addr
, uint8_t val
,
2453 TCGMemOpIdx oi
, uintptr_t retaddr
)
2455 store_helper(env
, addr
, val
, oi
, retaddr
, MO_UB
);
2458 void helper_le_stw_mmu(CPUArchState
*env
, target_ulong addr
, uint16_t val
,
2459 TCGMemOpIdx oi
, uintptr_t retaddr
)
2461 store_helper(env
, addr
, val
, oi
, retaddr
, MO_LEUW
);
2464 void helper_be_stw_mmu(CPUArchState
*env
, target_ulong addr
, uint16_t val
,
2465 TCGMemOpIdx oi
, uintptr_t retaddr
)
2467 store_helper(env
, addr
, val
, oi
, retaddr
, MO_BEUW
);
2470 void helper_le_stl_mmu(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
2471 TCGMemOpIdx oi
, uintptr_t retaddr
)
2473 store_helper(env
, addr
, val
, oi
, retaddr
, MO_LEUL
);
2476 void helper_be_stl_mmu(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
2477 TCGMemOpIdx oi
, uintptr_t retaddr
)
2479 store_helper(env
, addr
, val
, oi
, retaddr
, MO_BEUL
);
2482 void helper_le_stq_mmu(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
2483 TCGMemOpIdx oi
, uintptr_t retaddr
)
2485 store_helper(env
, addr
, val
, oi
, retaddr
, MO_LEQ
);
2488 void helper_be_stq_mmu(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
2489 TCGMemOpIdx oi
, uintptr_t retaddr
)
2491 store_helper(env
, addr
, val
, oi
, retaddr
, MO_BEQ
);
2495 * Store Helpers for cpu_ldst.h
2498 static inline void QEMU_ALWAYS_INLINE
2499 cpu_store_helper(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
2500 int mmu_idx
, uintptr_t retaddr
, MemOp op
)
2505 meminfo
= trace_mem_get_info(op
, mmu_idx
, true);
2506 trace_guest_mem_before_exec(env_cpu(env
), addr
, meminfo
);
2508 oi
= make_memop_idx(op
, mmu_idx
);
2509 store_helper(env
, addr
, val
, oi
, retaddr
, op
);
2511 qemu_plugin_vcpu_mem_cb(env_cpu(env
), addr
, meminfo
);
2514 void cpu_stb_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
2515 int mmu_idx
, uintptr_t retaddr
)
2517 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_UB
);
2520 void cpu_stw_be_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
2521 int mmu_idx
, uintptr_t retaddr
)
2523 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_BEUW
);
2526 void cpu_stl_be_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
2527 int mmu_idx
, uintptr_t retaddr
)
2529 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_BEUL
);
2532 void cpu_stq_be_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
2533 int mmu_idx
, uintptr_t retaddr
)
2535 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_BEQ
);
2538 void cpu_stw_le_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
2539 int mmu_idx
, uintptr_t retaddr
)
2541 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_LEUW
);
2544 void cpu_stl_le_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint32_t val
,
2545 int mmu_idx
, uintptr_t retaddr
)
2547 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_LEUL
);
2550 void cpu_stq_le_mmuidx_ra(CPUArchState
*env
, target_ulong addr
, uint64_t val
,
2551 int mmu_idx
, uintptr_t retaddr
)
2553 cpu_store_helper(env
, addr
, val
, mmu_idx
, retaddr
, MO_LEQ
);
2556 void cpu_stb_data_ra(CPUArchState
*env
, target_ulong ptr
,
2557 uint32_t val
, uintptr_t retaddr
)
2559 cpu_stb_mmuidx_ra(env
, ptr
, val
, cpu_mmu_index(env
, false), retaddr
);
2562 void cpu_stw_be_data_ra(CPUArchState
*env
, target_ulong ptr
,
2563 uint32_t val
, uintptr_t retaddr
)
2565 cpu_stw_be_mmuidx_ra(env
, ptr
, val
, cpu_mmu_index(env
, false), retaddr
);
2568 void cpu_stl_be_data_ra(CPUArchState
*env
, target_ulong ptr
,
2569 uint32_t val
, uintptr_t retaddr
)
2571 cpu_stl_be_mmuidx_ra(env
, ptr
, val
, cpu_mmu_index(env
, false), retaddr
);
2574 void cpu_stq_be_data_ra(CPUArchState
*env
, target_ulong ptr
,
2575 uint64_t val
, uintptr_t retaddr
)
2577 cpu_stq_be_mmuidx_ra(env
, ptr
, val
, cpu_mmu_index(env
, false), retaddr
);
2580 void cpu_stw_le_data_ra(CPUArchState
*env
, target_ulong ptr
,
2581 uint32_t val
, uintptr_t retaddr
)
2583 cpu_stw_le_mmuidx_ra(env
, ptr
, val
, cpu_mmu_index(env
, false), retaddr
);
2586 void cpu_stl_le_data_ra(CPUArchState
*env
, target_ulong ptr
,
2587 uint32_t val
, uintptr_t retaddr
)
2589 cpu_stl_le_mmuidx_ra(env
, ptr
, val
, cpu_mmu_index(env
, false), retaddr
);
2592 void cpu_stq_le_data_ra(CPUArchState
*env
, target_ulong ptr
,
2593 uint64_t val
, uintptr_t retaddr
)
2595 cpu_stq_le_mmuidx_ra(env
, ptr
, val
, cpu_mmu_index(env
, false), retaddr
);
2598 void cpu_stb_data(CPUArchState
*env
, target_ulong ptr
, uint32_t val
)
2600 cpu_stb_data_ra(env
, ptr
, val
, 0);
2603 void cpu_stw_be_data(CPUArchState
*env
, target_ulong ptr
, uint32_t val
)
2605 cpu_stw_be_data_ra(env
, ptr
, val
, 0);
2608 void cpu_stl_be_data(CPUArchState
*env
, target_ulong ptr
, uint32_t val
)
2610 cpu_stl_be_data_ra(env
, ptr
, val
, 0);
2613 void cpu_stq_be_data(CPUArchState
*env
, target_ulong ptr
, uint64_t val
)
2615 cpu_stq_be_data_ra(env
, ptr
, val
, 0);
2618 void cpu_stw_le_data(CPUArchState
*env
, target_ulong ptr
, uint32_t val
)
2620 cpu_stw_le_data_ra(env
, ptr
, val
, 0);
2623 void cpu_stl_le_data(CPUArchState
*env
, target_ulong ptr
, uint32_t val
)
2625 cpu_stl_le_data_ra(env
, ptr
, val
, 0);
2628 void cpu_stq_le_data(CPUArchState
*env
, target_ulong ptr
, uint64_t val
)
2630 cpu_stq_le_data_ra(env
, ptr
, val
, 0);
2633 /* First set of helpers allows passing in of OI and RETADDR. This makes
2634 them callable from other helpers. */
2636 #define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr
2637 #define ATOMIC_NAME(X) \
2638 HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
2639 #define ATOMIC_MMU_DECLS
2640 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr)
2641 #define ATOMIC_MMU_CLEANUP
2642 #define ATOMIC_MMU_IDX get_mmuidx(oi)
2644 #include "atomic_common.c.inc"
2647 #include "atomic_template.h"
2650 #include "atomic_template.h"
2653 #include "atomic_template.h"
2655 #ifdef CONFIG_ATOMIC64
2657 #include "atomic_template.h"
2660 #if HAVE_CMPXCHG128 || HAVE_ATOMIC128
2661 #define DATA_SIZE 16
2662 #include "atomic_template.h"
2665 /* Second set of helpers are directly callable from TCG as helpers. */
2669 #undef ATOMIC_MMU_LOOKUP
2670 #define EXTRA_ARGS , TCGMemOpIdx oi
2671 #define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
2672 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC())
2675 #include "atomic_template.h"
2678 #include "atomic_template.h"
2681 #include "atomic_template.h"
2683 #ifdef CONFIG_ATOMIC64
2685 #include "atomic_template.h"
2687 #undef ATOMIC_MMU_IDX
2689 /* Code access functions. */
2691 static uint64_t full_ldub_code(CPUArchState
*env
, target_ulong addr
,
2692 TCGMemOpIdx oi
, uintptr_t retaddr
)
2694 return load_helper(env
, addr
, oi
, retaddr
, MO_8
, true, full_ldub_code
);
2697 uint32_t cpu_ldub_code(CPUArchState
*env
, abi_ptr addr
)
2699 TCGMemOpIdx oi
= make_memop_idx(MO_UB
, cpu_mmu_index(env
, true));
2700 return full_ldub_code(env
, addr
, oi
, 0);
2703 static uint64_t full_lduw_code(CPUArchState
*env
, target_ulong addr
,
2704 TCGMemOpIdx oi
, uintptr_t retaddr
)
2706 return load_helper(env
, addr
, oi
, retaddr
, MO_TEUW
, true, full_lduw_code
);
2709 uint32_t cpu_lduw_code(CPUArchState
*env
, abi_ptr addr
)
2711 TCGMemOpIdx oi
= make_memop_idx(MO_TEUW
, cpu_mmu_index(env
, true));
2712 return full_lduw_code(env
, addr
, oi
, 0);
2715 static uint64_t full_ldl_code(CPUArchState
*env
, target_ulong addr
,
2716 TCGMemOpIdx oi
, uintptr_t retaddr
)
2718 return load_helper(env
, addr
, oi
, retaddr
, MO_TEUL
, true, full_ldl_code
);
2721 uint32_t cpu_ldl_code(CPUArchState
*env
, abi_ptr addr
)
2723 TCGMemOpIdx oi
= make_memop_idx(MO_TEUL
, cpu_mmu_index(env
, true));
2724 return full_ldl_code(env
, addr
, oi
, 0);
2727 static uint64_t full_ldq_code(CPUArchState
*env
, target_ulong addr
,
2728 TCGMemOpIdx oi
, uintptr_t retaddr
)
2730 return load_helper(env
, addr
, oi
, retaddr
, MO_TEQ
, true, full_ldq_code
);
2733 uint64_t cpu_ldq_code(CPUArchState
*env
, abi_ptr addr
)
2735 TCGMemOpIdx oi
= make_memop_idx(MO_TEQ
, cpu_mmu_index(env
, true));
2736 return full_ldq_code(env
, addr
, oi
, 0);