]> git.proxmox.com Git - mirror_qemu.git/blame - exec.c
translate-all: remove tb_lock mention from cpu_restore_state_from_tb
[mirror_qemu.git] / exec.c
CommitLineData
54936004 1/*
5b6dd868 2 * Virtual page mapping
5fafdf24 3 *
54936004
FB
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
8167ee88 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
54936004 18 */
7b31bbc2 19#include "qemu/osdep.h"
da34e65c 20#include "qapi/error.h"
54936004 21
f348b6d1 22#include "qemu/cutils.h"
6180a181 23#include "cpu.h"
63c91552 24#include "exec/exec-all.h"
51180423 25#include "exec/target_page.h"
b67d9a52 26#include "tcg.h"
741da0d3 27#include "hw/qdev-core.h"
c7e002c5 28#include "hw/qdev-properties.h"
4485bd26 29#if !defined(CONFIG_USER_ONLY)
47c8ca53 30#include "hw/boards.h"
33c11879 31#include "hw/xen/xen.h"
4485bd26 32#endif
9c17d615 33#include "sysemu/kvm.h"
2ff3de68 34#include "sysemu/sysemu.h"
1de7afc9
PB
35#include "qemu/timer.h"
36#include "qemu/config-file.h"
75a34036 37#include "qemu/error-report.h"
53a5960a 38#if defined(CONFIG_USER_ONLY)
a9c94277 39#include "qemu.h"
432d268c 40#else /* !CONFIG_USER_ONLY */
741da0d3
PB
41#include "hw/hw.h"
42#include "exec/memory.h"
df43d49c 43#include "exec/ioport.h"
741da0d3 44#include "sysemu/dma.h"
9c607668 45#include "sysemu/numa.h"
79ca7a1b 46#include "sysemu/hw_accel.h"
741da0d3 47#include "exec/address-spaces.h"
9c17d615 48#include "sysemu/xen-mapcache.h"
0ab8ed18 49#include "trace-root.h"
d3a5038c 50
e2fa71f5 51#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
e2fa71f5
DDAG
52#include <linux/falloc.h>
53#endif
54
53a5960a 55#endif
0dc3f44a 56#include "qemu/rcu_queue.h"
4840f10e 57#include "qemu/main-loop.h"
5b6dd868 58#include "translate-all.h"
7615936e 59#include "sysemu/replay.h"
0cac1b66 60
022c62cb 61#include "exec/memory-internal.h"
220c3ebd 62#include "exec/ram_addr.h"
508127e2 63#include "exec/log.h"
67d95c15 64
9dfeca7c
BR
65#include "migration/vmstate.h"
66
b35ba30f 67#include "qemu/range.h"
794e8f30
MT
68#ifndef _WIN32
69#include "qemu/mmap-alloc.h"
70#endif
b35ba30f 71
be9b23c4
PX
72#include "monitor/monitor.h"
73
db7b5426 74//#define DEBUG_SUBPAGE
1196be37 75
e2eef170 76#if !defined(CONFIG_USER_ONLY)
0dc3f44a
MD
77/* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
78 * are protected by the ramlist lock.
79 */
0d53d9fe 80RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
62152b8a
AK
81
82static MemoryRegion *system_memory;
309cb471 83static MemoryRegion *system_io;
62152b8a 84
f6790af6
AK
85AddressSpace address_space_io;
86AddressSpace address_space_memory;
2673a5da 87
0844e007 88MemoryRegion io_mem_rom, io_mem_notdirty;
acc9d80b 89static MemoryRegion io_mem_unassigned;
0e0df1e2 90
7bd4f430
PB
91/* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
92#define RAM_PREALLOC (1 << 0)
93
dbcb8981
PB
94/* RAM is mmap-ed with MAP_SHARED */
95#define RAM_SHARED (1 << 1)
96
62be4e3a
MT
97/* Only a portion of RAM (used_length) is actually used, and migrated.
98 * This used_length size can change across reboots.
99 */
100#define RAM_RESIZEABLE (1 << 2)
101
2ce16640
DDAG
102/* UFFDIO_ZEROPAGE is available on this RAMBlock to atomically
103 * zero the page and wake waiting processes.
104 * (Set during postcopy)
105 */
106#define RAM_UF_ZEROPAGE (1 << 3)
b895de50
CLG
107
108/* RAM can be migrated */
109#define RAM_MIGRATABLE (1 << 4)
e2eef170 110#endif
9fa3e853 111
20bccb82
PM
112#ifdef TARGET_PAGE_BITS_VARY
113int target_page_bits;
114bool target_page_bits_decided;
115#endif
116
bdc44640 117struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
6a00d601
FB
118/* current CPU in the current thread. It is only valid inside
119 cpu_exec() */
f240eb6f 120__thread CPUState *current_cpu;
2e70f6ef 121/* 0 = Do not count executed instructions.
bf20dc07 122 1 = Precise instruction counting.
2e70f6ef 123 2 = Adaptive rate instruction counting. */
5708fc66 124int use_icount;
6a00d601 125
a0be0c58
YZ
126uintptr_t qemu_host_page_size;
127intptr_t qemu_host_page_mask;
a0be0c58 128
20bccb82
PM
129bool set_preferred_target_page_bits(int bits)
130{
131 /* The target page size is the lowest common denominator for all
132 * the CPUs in the system, so we can only make it smaller, never
133 * larger. And we can't make it smaller once we've committed to
134 * a particular size.
135 */
136#ifdef TARGET_PAGE_BITS_VARY
137 assert(bits >= TARGET_PAGE_BITS_MIN);
138 if (target_page_bits == 0 || target_page_bits > bits) {
139 if (target_page_bits_decided) {
140 return false;
141 }
142 target_page_bits = bits;
143 }
144#endif
145 return true;
146}
147
e2eef170 148#if !defined(CONFIG_USER_ONLY)
4346ae3e 149
20bccb82
PM
150static void finalize_target_page_bits(void)
151{
152#ifdef TARGET_PAGE_BITS_VARY
153 if (target_page_bits == 0) {
154 target_page_bits = TARGET_PAGE_BITS_MIN;
155 }
156 target_page_bits_decided = true;
157#endif
158}
159
1db8abb1
PB
160typedef struct PhysPageEntry PhysPageEntry;
161
162struct PhysPageEntry {
9736e55b 163 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
8b795765 164 uint32_t skip : 6;
9736e55b 165 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
8b795765 166 uint32_t ptr : 26;
1db8abb1
PB
167};
168
8b795765
MT
169#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
170
03f49957 171/* Size of the L2 (and L3, etc) page tables. */
57271d63 172#define ADDR_SPACE_BITS 64
03f49957 173
026736ce 174#define P_L2_BITS 9
03f49957
PB
175#define P_L2_SIZE (1 << P_L2_BITS)
176
177#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
178
179typedef PhysPageEntry Node[P_L2_SIZE];
0475d94f 180
53cb28cb 181typedef struct PhysPageMap {
79e2b9ae
PB
182 struct rcu_head rcu;
183
53cb28cb
MA
184 unsigned sections_nb;
185 unsigned sections_nb_alloc;
186 unsigned nodes_nb;
187 unsigned nodes_nb_alloc;
188 Node *nodes;
189 MemoryRegionSection *sections;
190} PhysPageMap;
191
1db8abb1 192struct AddressSpaceDispatch {
729633c2 193 MemoryRegionSection *mru_section;
1db8abb1
PB
194 /* This is a multi-level map on the physical address space.
195 * The bottom level has pointers to MemoryRegionSections.
196 */
197 PhysPageEntry phys_map;
53cb28cb 198 PhysPageMap map;
1db8abb1
PB
199};
200
90260c6c
JK
201#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
202typedef struct subpage_t {
203 MemoryRegion iomem;
16620684 204 FlatView *fv;
90260c6c 205 hwaddr base;
2615fabd 206 uint16_t sub_section[];
90260c6c
JK
207} subpage_t;
208
b41aac4f
LPF
209#define PHYS_SECTION_UNASSIGNED 0
210#define PHYS_SECTION_NOTDIRTY 1
211#define PHYS_SECTION_ROM 2
212#define PHYS_SECTION_WATCH 3
5312bd8b 213
e2eef170 214static void io_mem_init(void);
62152b8a 215static void memory_map_init(void);
09daed84 216static void tcg_commit(MemoryListener *listener);
e2eef170 217
1ec9b909 218static MemoryRegion io_mem_watch;
32857f4d
PM
219
220/**
221 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
222 * @cpu: the CPU whose AddressSpace this is
223 * @as: the AddressSpace itself
224 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
225 * @tcg_as_listener: listener for tracking changes to the AddressSpace
226 */
227struct CPUAddressSpace {
228 CPUState *cpu;
229 AddressSpace *as;
230 struct AddressSpaceDispatch *memory_dispatch;
231 MemoryListener tcg_as_listener;
232};
233
8deaf12c
GH
234struct DirtyBitmapSnapshot {
235 ram_addr_t start;
236 ram_addr_t end;
237 unsigned long dirty[];
238};
239
6658ffb8 240#endif
fd6ce8f6 241
6d9a1304 242#if !defined(CONFIG_USER_ONLY)
d6f2ea22 243
53cb28cb 244static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
d6f2ea22 245{
101420b8 246 static unsigned alloc_hint = 16;
53cb28cb 247 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
101420b8 248 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, alloc_hint);
53cb28cb
MA
249 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
250 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
101420b8 251 alloc_hint = map->nodes_nb_alloc;
d6f2ea22 252 }
f7bf5461
AK
253}
254
db94604b 255static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
f7bf5461
AK
256{
257 unsigned i;
8b795765 258 uint32_t ret;
db94604b
PB
259 PhysPageEntry e;
260 PhysPageEntry *p;
f7bf5461 261
53cb28cb 262 ret = map->nodes_nb++;
db94604b 263 p = map->nodes[ret];
f7bf5461 264 assert(ret != PHYS_MAP_NODE_NIL);
53cb28cb 265 assert(ret != map->nodes_nb_alloc);
db94604b
PB
266
267 e.skip = leaf ? 0 : 1;
268 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
03f49957 269 for (i = 0; i < P_L2_SIZE; ++i) {
db94604b 270 memcpy(&p[i], &e, sizeof(e));
d6f2ea22 271 }
f7bf5461 272 return ret;
d6f2ea22
AK
273}
274
53cb28cb
MA
275static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
276 hwaddr *index, hwaddr *nb, uint16_t leaf,
2999097b 277 int level)
f7bf5461
AK
278{
279 PhysPageEntry *p;
03f49957 280 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
108c49b8 281
9736e55b 282 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
db94604b 283 lp->ptr = phys_map_node_alloc(map, level == 0);
92e873b9 284 }
db94604b 285 p = map->nodes[lp->ptr];
03f49957 286 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
f7bf5461 287
03f49957 288 while (*nb && lp < &p[P_L2_SIZE]) {
07f07b31 289 if ((*index & (step - 1)) == 0 && *nb >= step) {
9736e55b 290 lp->skip = 0;
c19e8800 291 lp->ptr = leaf;
07f07b31
AK
292 *index += step;
293 *nb -= step;
2999097b 294 } else {
53cb28cb 295 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
2999097b
AK
296 }
297 ++lp;
f7bf5461
AK
298 }
299}
300
ac1970fb 301static void phys_page_set(AddressSpaceDispatch *d,
a8170e5e 302 hwaddr index, hwaddr nb,
2999097b 303 uint16_t leaf)
f7bf5461 304{
2999097b 305 /* Wildly overreserve - it doesn't matter much. */
53cb28cb 306 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
5cd2c5b6 307
53cb28cb 308 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
92e873b9
FB
309}
310
b35ba30f
MT
311/* Compact a non leaf page entry. Simply detect that the entry has a single child,
312 * and update our entry so we can skip it and go directly to the destination.
313 */
efee678d 314static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
b35ba30f
MT
315{
316 unsigned valid_ptr = P_L2_SIZE;
317 int valid = 0;
318 PhysPageEntry *p;
319 int i;
320
321 if (lp->ptr == PHYS_MAP_NODE_NIL) {
322 return;
323 }
324
325 p = nodes[lp->ptr];
326 for (i = 0; i < P_L2_SIZE; i++) {
327 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
328 continue;
329 }
330
331 valid_ptr = i;
332 valid++;
333 if (p[i].skip) {
efee678d 334 phys_page_compact(&p[i], nodes);
b35ba30f
MT
335 }
336 }
337
338 /* We can only compress if there's only one child. */
339 if (valid != 1) {
340 return;
341 }
342
343 assert(valid_ptr < P_L2_SIZE);
344
345 /* Don't compress if it won't fit in the # of bits we have. */
346 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
347 return;
348 }
349
350 lp->ptr = p[valid_ptr].ptr;
351 if (!p[valid_ptr].skip) {
352 /* If our only child is a leaf, make this a leaf. */
353 /* By design, we should have made this node a leaf to begin with so we
354 * should never reach here.
355 * But since it's so simple to handle this, let's do it just in case we
356 * change this rule.
357 */
358 lp->skip = 0;
359 } else {
360 lp->skip += p[valid_ptr].skip;
361 }
362}
363
8629d3fc 364void address_space_dispatch_compact(AddressSpaceDispatch *d)
b35ba30f 365{
b35ba30f 366 if (d->phys_map.skip) {
efee678d 367 phys_page_compact(&d->phys_map, d->map.nodes);
b35ba30f
MT
368 }
369}
370
29cb533d
FZ
371static inline bool section_covers_addr(const MemoryRegionSection *section,
372 hwaddr addr)
373{
374 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
375 * the section must cover the entire address space.
376 */
258dfaaa 377 return int128_gethi(section->size) ||
29cb533d 378 range_covers_byte(section->offset_within_address_space,
258dfaaa 379 int128_getlo(section->size), addr);
29cb533d
FZ
380}
381
003a0cf2 382static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
92e873b9 383{
003a0cf2
PX
384 PhysPageEntry lp = d->phys_map, *p;
385 Node *nodes = d->map.nodes;
386 MemoryRegionSection *sections = d->map.sections;
97115a8d 387 hwaddr index = addr >> TARGET_PAGE_BITS;
31ab2b4a 388 int i;
f1f6e3b8 389
9736e55b 390 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
c19e8800 391 if (lp.ptr == PHYS_MAP_NODE_NIL) {
9affd6fc 392 return &sections[PHYS_SECTION_UNASSIGNED];
31ab2b4a 393 }
9affd6fc 394 p = nodes[lp.ptr];
03f49957 395 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
5312bd8b 396 }
b35ba30f 397
29cb533d 398 if (section_covers_addr(&sections[lp.ptr], addr)) {
b35ba30f
MT
399 return &sections[lp.ptr];
400 } else {
401 return &sections[PHYS_SECTION_UNASSIGNED];
402 }
f3705d53
AK
403}
404
e5548617
BS
405bool memory_region_is_unassigned(MemoryRegion *mr)
406{
2a8e7499 407 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
5b6dd868 408 && mr != &io_mem_watch;
fd6ce8f6 409}
149f54b5 410
79e2b9ae 411/* Called from RCU critical section */
c7086b4a 412static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
90260c6c
JK
413 hwaddr addr,
414 bool resolve_subpage)
9f029603 415{
729633c2 416 MemoryRegionSection *section = atomic_read(&d->mru_section);
90260c6c
JK
417 subpage_t *subpage;
418
07c114bb
PB
419 if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
420 !section_covers_addr(section, addr)) {
003a0cf2 421 section = phys_page_find(d, addr);
07c114bb 422 atomic_set(&d->mru_section, section);
729633c2 423 }
90260c6c
JK
424 if (resolve_subpage && section->mr->subpage) {
425 subpage = container_of(section->mr, subpage_t, iomem);
53cb28cb 426 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
90260c6c
JK
427 }
428 return section;
9f029603
JK
429}
430
79e2b9ae 431/* Called from RCU critical section */
90260c6c 432static MemoryRegionSection *
c7086b4a 433address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
90260c6c 434 hwaddr *plen, bool resolve_subpage)
149f54b5
PB
435{
436 MemoryRegionSection *section;
965eb2fc 437 MemoryRegion *mr;
a87f3954 438 Int128 diff;
149f54b5 439
c7086b4a 440 section = address_space_lookup_region(d, addr, resolve_subpage);
149f54b5
PB
441 /* Compute offset within MemoryRegionSection */
442 addr -= section->offset_within_address_space;
443
444 /* Compute offset within MemoryRegion */
445 *xlat = addr + section->offset_within_region;
446
965eb2fc 447 mr = section->mr;
b242e0e0
PB
448
449 /* MMIO registers can be expected to perform full-width accesses based only
450 * on their address, without considering adjacent registers that could
451 * decode to completely different MemoryRegions. When such registers
452 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
453 * regions overlap wildly. For this reason we cannot clamp the accesses
454 * here.
455 *
456 * If the length is small (as is the case for address_space_ldl/stl),
457 * everything works fine. If the incoming length is large, however,
458 * the caller really has to do the clamping through memory_access_size.
459 */
965eb2fc 460 if (memory_region_is_ram(mr)) {
e4a511f8 461 diff = int128_sub(section->size, int128_make64(addr));
965eb2fc
PB
462 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
463 }
149f54b5
PB
464 return section;
465}
90260c6c 466
a411c84b
PB
467/**
468 * address_space_translate_iommu - translate an address through an IOMMU
469 * memory region and then through the target address space.
470 *
471 * @iommu_mr: the IOMMU memory region that we start the translation from
472 * @addr: the address to be translated through the MMU
473 * @xlat: the translated address offset within the destination memory region.
474 * It cannot be %NULL.
475 * @plen_out: valid read/write length of the translated address. It
476 * cannot be %NULL.
477 * @page_mask_out: page mask for the translated address. This
478 * should only be meaningful for IOMMU translated
479 * addresses, since there may be huge pages that this bit
480 * would tell. It can be %NULL if we don't care about it.
481 * @is_write: whether the translation operation is for write
482 * @is_mmio: whether this can be MMIO, set true if it can
483 * @target_as: the address space targeted by the IOMMU
2f7b009c 484 * @attrs: transaction attributes
a411c84b
PB
485 *
486 * This function is called from RCU critical section. It is the common
487 * part of flatview_do_translate and address_space_translate_cached.
488 */
489static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
490 hwaddr *xlat,
491 hwaddr *plen_out,
492 hwaddr *page_mask_out,
493 bool is_write,
494 bool is_mmio,
2f7b009c
PM
495 AddressSpace **target_as,
496 MemTxAttrs attrs)
a411c84b
PB
497{
498 MemoryRegionSection *section;
499 hwaddr page_mask = (hwaddr)-1;
500
501 do {
502 hwaddr addr = *xlat;
503 IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
2c91bcf2
PM
504 int iommu_idx = 0;
505 IOMMUTLBEntry iotlb;
506
507 if (imrc->attrs_to_index) {
508 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
509 }
510
511 iotlb = imrc->translate(iommu_mr, addr, is_write ?
512 IOMMU_WO : IOMMU_RO, iommu_idx);
a411c84b
PB
513
514 if (!(iotlb.perm & (1 << is_write))) {
515 goto unassigned;
516 }
517
518 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
519 | (addr & iotlb.addr_mask));
520 page_mask &= iotlb.addr_mask;
521 *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
522 *target_as = iotlb.target_as;
523
524 section = address_space_translate_internal(
525 address_space_to_dispatch(iotlb.target_as), addr, xlat,
526 plen_out, is_mmio);
527
528 iommu_mr = memory_region_get_iommu(section->mr);
529 } while (unlikely(iommu_mr));
530
531 if (page_mask_out) {
532 *page_mask_out = page_mask;
533 }
534 return *section;
535
536unassigned:
537 return (MemoryRegionSection) { .mr = &io_mem_unassigned };
538}
539
d5e5fafd
PX
540/**
541 * flatview_do_translate - translate an address in FlatView
542 *
543 * @fv: the flat view that we want to translate on
544 * @addr: the address to be translated in above address space
545 * @xlat: the translated address offset within memory region. It
546 * cannot be @NULL.
547 * @plen_out: valid read/write length of the translated address. It
548 * can be @NULL when we don't care about it.
549 * @page_mask_out: page mask for the translated address. This
550 * should only be meaningful for IOMMU translated
551 * addresses, since there may be huge pages that this bit
552 * would tell. It can be @NULL if we don't care about it.
553 * @is_write: whether the translation operation is for write
554 * @is_mmio: whether this can be MMIO, set true if it can
ad2804d9 555 * @target_as: the address space targeted by the IOMMU
49e14aa8 556 * @attrs: memory transaction attributes
d5e5fafd
PX
557 *
558 * This function is called from RCU critical section
559 */
16620684
AK
560static MemoryRegionSection flatview_do_translate(FlatView *fv,
561 hwaddr addr,
562 hwaddr *xlat,
d5e5fafd
PX
563 hwaddr *plen_out,
564 hwaddr *page_mask_out,
16620684
AK
565 bool is_write,
566 bool is_mmio,
49e14aa8
PM
567 AddressSpace **target_as,
568 MemTxAttrs attrs)
052c8fa9 569{
052c8fa9 570 MemoryRegionSection *section;
3df9d748 571 IOMMUMemoryRegion *iommu_mr;
d5e5fafd
PX
572 hwaddr plen = (hwaddr)(-1);
573
ad2804d9
PB
574 if (!plen_out) {
575 plen_out = &plen;
d5e5fafd 576 }
052c8fa9 577
a411c84b
PB
578 section = address_space_translate_internal(
579 flatview_to_dispatch(fv), addr, xlat,
580 plen_out, is_mmio);
052c8fa9 581
a411c84b
PB
582 iommu_mr = memory_region_get_iommu(section->mr);
583 if (unlikely(iommu_mr)) {
584 return address_space_translate_iommu(iommu_mr, xlat,
585 plen_out, page_mask_out,
586 is_write, is_mmio,
2f7b009c 587 target_as, attrs);
052c8fa9 588 }
d5e5fafd 589 if (page_mask_out) {
a411c84b
PB
590 /* Not behind an IOMMU, use default page size. */
591 *page_mask_out = ~TARGET_PAGE_MASK;
d5e5fafd
PX
592 }
593
a764040c 594 return *section;
052c8fa9
JW
595}
596
597/* Called from RCU critical section */
a764040c 598IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
7446eb07 599 bool is_write, MemTxAttrs attrs)
90260c6c 600{
a764040c 601 MemoryRegionSection section;
076a93d7 602 hwaddr xlat, page_mask;
30951157 603
076a93d7
PX
604 /*
605 * This can never be MMIO, and we don't really care about plen,
606 * but page mask.
607 */
608 section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
49e14aa8
PM
609 NULL, &page_mask, is_write, false, &as,
610 attrs);
30951157 611
a764040c
PX
612 /* Illegal translation */
613 if (section.mr == &io_mem_unassigned) {
614 goto iotlb_fail;
615 }
30951157 616
a764040c
PX
617 /* Convert memory region offset into address space offset */
618 xlat += section.offset_within_address_space -
619 section.offset_within_region;
620
a764040c 621 return (IOMMUTLBEntry) {
e76bb18f 622 .target_as = as,
076a93d7
PX
623 .iova = addr & ~page_mask,
624 .translated_addr = xlat & ~page_mask,
625 .addr_mask = page_mask,
a764040c
PX
626 /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
627 .perm = IOMMU_RW,
628 };
629
630iotlb_fail:
631 return (IOMMUTLBEntry) {0};
632}
633
634/* Called from RCU critical section */
16620684 635MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
efa99a2f
PM
636 hwaddr *plen, bool is_write,
637 MemTxAttrs attrs)
a764040c
PX
638{
639 MemoryRegion *mr;
640 MemoryRegionSection section;
16620684 641 AddressSpace *as = NULL;
a764040c
PX
642
643 /* This can be MMIO, so setup MMIO bit. */
d5e5fafd 644 section = flatview_do_translate(fv, addr, xlat, plen, NULL,
49e14aa8 645 is_write, true, &as, attrs);
a764040c
PX
646 mr = section.mr;
647
fe680d0d 648 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
a87f3954 649 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
23820dbf 650 *plen = MIN(page, *plen);
a87f3954
PB
651 }
652
30951157 653 return mr;
90260c6c
JK
654}
655
1f871c5e
PM
656typedef struct TCGIOMMUNotifier {
657 IOMMUNotifier n;
658 MemoryRegion *mr;
659 CPUState *cpu;
660 int iommu_idx;
661 bool active;
662} TCGIOMMUNotifier;
663
664static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
665{
666 TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
667
668 if (!notifier->active) {
669 return;
670 }
671 tlb_flush(notifier->cpu);
672 notifier->active = false;
673 /* We leave the notifier struct on the list to avoid reallocating it later.
674 * Generally the number of IOMMUs a CPU deals with will be small.
675 * In any case we can't unregister the iommu notifier from a notify
676 * callback.
677 */
678}
679
680static void tcg_register_iommu_notifier(CPUState *cpu,
681 IOMMUMemoryRegion *iommu_mr,
682 int iommu_idx)
683{
684 /* Make sure this CPU has an IOMMU notifier registered for this
685 * IOMMU/IOMMU index combination, so that we can flush its TLB
686 * when the IOMMU tells us the mappings we've cached have changed.
687 */
688 MemoryRegion *mr = MEMORY_REGION(iommu_mr);
689 TCGIOMMUNotifier *notifier;
690 int i;
691
692 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
693 notifier = &g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier, i);
694 if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
695 break;
696 }
697 }
698 if (i == cpu->iommu_notifiers->len) {
699 /* Not found, add a new entry at the end of the array */
700 cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
701 notifier = &g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier, i);
702
703 notifier->mr = mr;
704 notifier->iommu_idx = iommu_idx;
705 notifier->cpu = cpu;
706 /* Rather than trying to register interest in the specific part
707 * of the iommu's address space that we've accessed and then
708 * expand it later as subsequent accesses touch more of it, we
709 * just register interest in the whole thing, on the assumption
710 * that iommu reconfiguration will be rare.
711 */
712 iommu_notifier_init(&notifier->n,
713 tcg_iommu_unmap_notify,
714 IOMMU_NOTIFIER_UNMAP,
715 0,
716 HWADDR_MAX,
717 iommu_idx);
718 memory_region_register_iommu_notifier(notifier->mr, &notifier->n);
719 }
720
721 if (!notifier->active) {
722 notifier->active = true;
723 }
724}
725
726static void tcg_iommu_free_notifier_list(CPUState *cpu)
727{
728 /* Destroy the CPU's notifier list */
729 int i;
730 TCGIOMMUNotifier *notifier;
731
732 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
733 notifier = &g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier, i);
734 memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
735 }
736 g_array_free(cpu->iommu_notifiers, true);
737}
738
79e2b9ae 739/* Called from RCU critical section */
90260c6c 740MemoryRegionSection *
d7898cda 741address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
1f871c5e
PM
742 hwaddr *xlat, hwaddr *plen,
743 MemTxAttrs attrs, int *prot)
90260c6c 744{
30951157 745 MemoryRegionSection *section;
1f871c5e
PM
746 IOMMUMemoryRegion *iommu_mr;
747 IOMMUMemoryRegionClass *imrc;
748 IOMMUTLBEntry iotlb;
749 int iommu_idx;
f35e44e7 750 AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
d7898cda 751
1f871c5e
PM
752 for (;;) {
753 section = address_space_translate_internal(d, addr, &addr, plen, false);
754
755 iommu_mr = memory_region_get_iommu(section->mr);
756 if (!iommu_mr) {
757 break;
758 }
759
760 imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
761
762 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
763 tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
764 /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
765 * doesn't short-cut its translation table walk.
766 */
767 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
768 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
769 | (addr & iotlb.addr_mask));
770 /* Update the caller's prot bits to remove permissions the IOMMU
771 * is giving us a failure response for. If we get down to no
772 * permissions left at all we can give up now.
773 */
774 if (!(iotlb.perm & IOMMU_RO)) {
775 *prot &= ~(PAGE_READ | PAGE_EXEC);
776 }
777 if (!(iotlb.perm & IOMMU_WO)) {
778 *prot &= ~PAGE_WRITE;
779 }
780
781 if (!*prot) {
782 goto translate_fail;
783 }
784
785 d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
786 }
30951157 787
3df9d748 788 assert(!memory_region_is_iommu(section->mr));
1f871c5e 789 *xlat = addr;
30951157 790 return section;
1f871c5e
PM
791
792translate_fail:
793 return &d->map.sections[PHYS_SECTION_UNASSIGNED];
90260c6c 794}
5b6dd868 795#endif
fd6ce8f6 796
b170fce3 797#if !defined(CONFIG_USER_ONLY)
5b6dd868
BS
798
799static int cpu_common_post_load(void *opaque, int version_id)
fd6ce8f6 800{
259186a7 801 CPUState *cpu = opaque;
a513fe19 802
5b6dd868
BS
803 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
804 version_id is increased. */
259186a7 805 cpu->interrupt_request &= ~0x01;
d10eb08f 806 tlb_flush(cpu);
5b6dd868 807
15a356c4
PD
808 /* loadvm has just updated the content of RAM, bypassing the
809 * usual mechanisms that ensure we flush TBs for writes to
810 * memory we've translated code from. So we must flush all TBs,
811 * which will now be stale.
812 */
813 tb_flush(cpu);
814
5b6dd868 815 return 0;
a513fe19 816}
7501267e 817
6c3bff0e
PD
818static int cpu_common_pre_load(void *opaque)
819{
820 CPUState *cpu = opaque;
821
adee6424 822 cpu->exception_index = -1;
6c3bff0e
PD
823
824 return 0;
825}
826
827static bool cpu_common_exception_index_needed(void *opaque)
828{
829 CPUState *cpu = opaque;
830
adee6424 831 return tcg_enabled() && cpu->exception_index != -1;
6c3bff0e
PD
832}
833
834static const VMStateDescription vmstate_cpu_common_exception_index = {
835 .name = "cpu_common/exception_index",
836 .version_id = 1,
837 .minimum_version_id = 1,
5cd8cada 838 .needed = cpu_common_exception_index_needed,
6c3bff0e
PD
839 .fields = (VMStateField[]) {
840 VMSTATE_INT32(exception_index, CPUState),
841 VMSTATE_END_OF_LIST()
842 }
843};
844
bac05aa9
AS
845static bool cpu_common_crash_occurred_needed(void *opaque)
846{
847 CPUState *cpu = opaque;
848
849 return cpu->crash_occurred;
850}
851
852static const VMStateDescription vmstate_cpu_common_crash_occurred = {
853 .name = "cpu_common/crash_occurred",
854 .version_id = 1,
855 .minimum_version_id = 1,
856 .needed = cpu_common_crash_occurred_needed,
857 .fields = (VMStateField[]) {
858 VMSTATE_BOOL(crash_occurred, CPUState),
859 VMSTATE_END_OF_LIST()
860 }
861};
862
1a1562f5 863const VMStateDescription vmstate_cpu_common = {
5b6dd868
BS
864 .name = "cpu_common",
865 .version_id = 1,
866 .minimum_version_id = 1,
6c3bff0e 867 .pre_load = cpu_common_pre_load,
5b6dd868 868 .post_load = cpu_common_post_load,
35d08458 869 .fields = (VMStateField[]) {
259186a7
AF
870 VMSTATE_UINT32(halted, CPUState),
871 VMSTATE_UINT32(interrupt_request, CPUState),
5b6dd868 872 VMSTATE_END_OF_LIST()
6c3bff0e 873 },
5cd8cada
JQ
874 .subsections = (const VMStateDescription*[]) {
875 &vmstate_cpu_common_exception_index,
bac05aa9 876 &vmstate_cpu_common_crash_occurred,
5cd8cada 877 NULL
5b6dd868
BS
878 }
879};
1a1562f5 880
5b6dd868 881#endif
ea041c0e 882
38d8f5c8 883CPUState *qemu_get_cpu(int index)
ea041c0e 884{
bdc44640 885 CPUState *cpu;
ea041c0e 886
bdc44640 887 CPU_FOREACH(cpu) {
55e5c285 888 if (cpu->cpu_index == index) {
bdc44640 889 return cpu;
55e5c285 890 }
ea041c0e 891 }
5b6dd868 892
bdc44640 893 return NULL;
ea041c0e
FB
894}
895
09daed84 896#if !defined(CONFIG_USER_ONLY)
80ceb07a
PX
897void cpu_address_space_init(CPUState *cpu, int asidx,
898 const char *prefix, MemoryRegion *mr)
09daed84 899{
12ebc9a7 900 CPUAddressSpace *newas;
80ceb07a 901 AddressSpace *as = g_new0(AddressSpace, 1);
87a621d8 902 char *as_name;
80ceb07a
PX
903
904 assert(mr);
87a621d8
PX
905 as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
906 address_space_init(as, mr, as_name);
907 g_free(as_name);
12ebc9a7
PM
908
909 /* Target code should have set num_ases before calling us */
910 assert(asidx < cpu->num_ases);
911
56943e8c
PM
912 if (asidx == 0) {
913 /* address space 0 gets the convenience alias */
914 cpu->as = as;
915 }
916
12ebc9a7
PM
917 /* KVM cannot currently support multiple address spaces. */
918 assert(asidx == 0 || !kvm_enabled());
09daed84 919
12ebc9a7
PM
920 if (!cpu->cpu_ases) {
921 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
09daed84 922 }
32857f4d 923
12ebc9a7
PM
924 newas = &cpu->cpu_ases[asidx];
925 newas->cpu = cpu;
926 newas->as = as;
56943e8c 927 if (tcg_enabled()) {
12ebc9a7
PM
928 newas->tcg_as_listener.commit = tcg_commit;
929 memory_listener_register(&newas->tcg_as_listener, as);
56943e8c 930 }
09daed84 931}
651a5bc0
PM
932
933AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
934{
935 /* Return the AddressSpace corresponding to the specified index */
936 return cpu->cpu_ases[asidx].as;
937}
09daed84
EI
938#endif
939
7bbc124e 940void cpu_exec_unrealizefn(CPUState *cpu)
1c59eb39 941{
9dfeca7c
BR
942 CPUClass *cc = CPU_GET_CLASS(cpu);
943
267f685b 944 cpu_list_remove(cpu);
9dfeca7c
BR
945
946 if (cc->vmsd != NULL) {
947 vmstate_unregister(NULL, cc->vmsd, cpu);
948 }
949 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
950 vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
951 }
1f871c5e
PM
952#ifndef CONFIG_USER_ONLY
953 tcg_iommu_free_notifier_list(cpu);
954#endif
1c59eb39
BR
955}
956
c7e002c5
FZ
957Property cpu_common_props[] = {
958#ifndef CONFIG_USER_ONLY
959 /* Create a memory property for softmmu CPU object,
960 * so users can wire up its memory. (This can't go in qom/cpu.c
961 * because that file is compiled only once for both user-mode
962 * and system builds.) The default if no link is set up is to use
963 * the system address space.
964 */
965 DEFINE_PROP_LINK("memory", CPUState, memory, TYPE_MEMORY_REGION,
966 MemoryRegion *),
967#endif
968 DEFINE_PROP_END_OF_LIST(),
969};
970
39e329e3 971void cpu_exec_initfn(CPUState *cpu)
ea041c0e 972{
56943e8c 973 cpu->as = NULL;
12ebc9a7 974 cpu->num_ases = 0;
56943e8c 975
291135b5 976#ifndef CONFIG_USER_ONLY
291135b5 977 cpu->thread_id = qemu_get_thread_id();
6731d864
PC
978 cpu->memory = system_memory;
979 object_ref(OBJECT(cpu->memory));
291135b5 980#endif
39e329e3
LV
981}
982
ce5b1bbf 983void cpu_exec_realizefn(CPUState *cpu, Error **errp)
39e329e3 984{
55c3ceef 985 CPUClass *cc = CPU_GET_CLASS(cpu);
2dda6354 986 static bool tcg_target_initialized;
291135b5 987
267f685b 988 cpu_list_add(cpu);
1bc7e522 989
2dda6354
EC
990 if (tcg_enabled() && !tcg_target_initialized) {
991 tcg_target_initialized = true;
55c3ceef
RH
992 cc->tcg_initialize();
993 }
994
1bc7e522 995#ifndef CONFIG_USER_ONLY
e0d47944 996 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
741da0d3 997 vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
e0d47944 998 }
b170fce3 999 if (cc->vmsd != NULL) {
741da0d3 1000 vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
b170fce3 1001 }
1f871c5e
PM
1002
1003 cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier));
741da0d3 1004#endif
ea041c0e
FB
1005}
1006
2278b939
IM
1007const char *parse_cpu_model(const char *cpu_model)
1008{
1009 ObjectClass *oc;
1010 CPUClass *cc;
1011 gchar **model_pieces;
1012 const char *cpu_type;
1013
1014 model_pieces = g_strsplit(cpu_model, ",", 2);
1015
1016 oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);
1017 if (oc == NULL) {
1018 error_report("unable to find CPU model '%s'", model_pieces[0]);
1019 g_strfreev(model_pieces);
1020 exit(EXIT_FAILURE);
1021 }
1022
1023 cpu_type = object_class_get_name(oc);
1024 cc = CPU_CLASS(oc);
1025 cc->parse_features(cpu_type, model_pieces[1], &error_fatal);
1026 g_strfreev(model_pieces);
1027 return cpu_type;
1028}
1029
406bc339 1030#if defined(CONFIG_USER_ONLY)
00b941e5 1031static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
1e7855a5 1032{
406bc339
PK
1033 mmap_lock();
1034 tb_lock();
1035 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1036 tb_unlock();
1037 mmap_unlock();
1038}
1039#else
1040static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
1041{
1042 MemTxAttrs attrs;
1043 hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
1044 int asidx = cpu_asidx_from_attrs(cpu, attrs);
1045 if (phys != -1) {
1046 /* Locks grabbed by tb_invalidate_phys_addr */
1047 tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
c874dc4f 1048 phys | (pc & ~TARGET_PAGE_MASK), attrs);
406bc339 1049 }
1e7855a5 1050}
406bc339 1051#endif
d720b93d 1052
c527ee8f 1053#if defined(CONFIG_USER_ONLY)
75a34036 1054void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
c527ee8f
PB
1055
1056{
1057}
1058
3ee887e8
PM
1059int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
1060 int flags)
1061{
1062 return -ENOSYS;
1063}
1064
1065void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
1066{
1067}
1068
75a34036 1069int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
c527ee8f
PB
1070 int flags, CPUWatchpoint **watchpoint)
1071{
1072 return -ENOSYS;
1073}
1074#else
6658ffb8 1075/* Add a watchpoint. */
75a34036 1076int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
a1d1bb31 1077 int flags, CPUWatchpoint **watchpoint)
6658ffb8 1078{
c0ce998e 1079 CPUWatchpoint *wp;
6658ffb8 1080
05068c0d 1081 /* forbid ranges which are empty or run off the end of the address space */
07e2863d 1082 if (len == 0 || (addr + len - 1) < addr) {
75a34036
AF
1083 error_report("tried to set invalid watchpoint at %"
1084 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
b4051334
AL
1085 return -EINVAL;
1086 }
7267c094 1087 wp = g_malloc(sizeof(*wp));
a1d1bb31
AL
1088
1089 wp->vaddr = addr;
05068c0d 1090 wp->len = len;
a1d1bb31
AL
1091 wp->flags = flags;
1092
2dc9f411 1093 /* keep all GDB-injected watchpoints in front */
ff4700b0
AF
1094 if (flags & BP_GDB) {
1095 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
1096 } else {
1097 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
1098 }
6658ffb8 1099
31b030d4 1100 tlb_flush_page(cpu, addr);
a1d1bb31
AL
1101
1102 if (watchpoint)
1103 *watchpoint = wp;
1104 return 0;
6658ffb8
PB
1105}
1106
a1d1bb31 1107/* Remove a specific watchpoint. */
75a34036 1108int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
a1d1bb31 1109 int flags)
6658ffb8 1110{
a1d1bb31 1111 CPUWatchpoint *wp;
6658ffb8 1112
ff4700b0 1113 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
05068c0d 1114 if (addr == wp->vaddr && len == wp->len
6e140f28 1115 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
75a34036 1116 cpu_watchpoint_remove_by_ref(cpu, wp);
6658ffb8
PB
1117 return 0;
1118 }
1119 }
a1d1bb31 1120 return -ENOENT;
6658ffb8
PB
1121}
1122
a1d1bb31 1123/* Remove a specific watchpoint by reference. */
75a34036 1124void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
a1d1bb31 1125{
ff4700b0 1126 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
7d03f82f 1127
31b030d4 1128 tlb_flush_page(cpu, watchpoint->vaddr);
a1d1bb31 1129
7267c094 1130 g_free(watchpoint);
a1d1bb31
AL
1131}
1132
1133/* Remove all matching watchpoints. */
75a34036 1134void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
a1d1bb31 1135{
c0ce998e 1136 CPUWatchpoint *wp, *next;
a1d1bb31 1137
ff4700b0 1138 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
75a34036
AF
1139 if (wp->flags & mask) {
1140 cpu_watchpoint_remove_by_ref(cpu, wp);
1141 }
c0ce998e 1142 }
7d03f82f 1143}
05068c0d
PM
1144
1145/* Return true if this watchpoint address matches the specified
1146 * access (ie the address range covered by the watchpoint overlaps
1147 * partially or completely with the address range covered by the
1148 * access).
1149 */
1150static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
1151 vaddr addr,
1152 vaddr len)
1153{
1154 /* We know the lengths are non-zero, but a little caution is
1155 * required to avoid errors in the case where the range ends
1156 * exactly at the top of the address space and so addr + len
1157 * wraps round to zero.
1158 */
1159 vaddr wpend = wp->vaddr + wp->len - 1;
1160 vaddr addrend = addr + len - 1;
1161
1162 return !(addr > wpend || wp->vaddr > addrend);
1163}
1164
c527ee8f 1165#endif
7d03f82f 1166
a1d1bb31 1167/* Add a breakpoint. */
b3310ab3 1168int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
a1d1bb31 1169 CPUBreakpoint **breakpoint)
4c3a88a2 1170{
c0ce998e 1171 CPUBreakpoint *bp;
3b46e624 1172
7267c094 1173 bp = g_malloc(sizeof(*bp));
4c3a88a2 1174
a1d1bb31
AL
1175 bp->pc = pc;
1176 bp->flags = flags;
1177
2dc9f411 1178 /* keep all GDB-injected breakpoints in front */
00b941e5 1179 if (flags & BP_GDB) {
f0c3c505 1180 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
00b941e5 1181 } else {
f0c3c505 1182 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
00b941e5 1183 }
3b46e624 1184
f0c3c505 1185 breakpoint_invalidate(cpu, pc);
a1d1bb31 1186
00b941e5 1187 if (breakpoint) {
a1d1bb31 1188 *breakpoint = bp;
00b941e5 1189 }
4c3a88a2 1190 return 0;
4c3a88a2
FB
1191}
1192
a1d1bb31 1193/* Remove a specific breakpoint. */
b3310ab3 1194int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
a1d1bb31 1195{
a1d1bb31
AL
1196 CPUBreakpoint *bp;
1197
f0c3c505 1198 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
a1d1bb31 1199 if (bp->pc == pc && bp->flags == flags) {
b3310ab3 1200 cpu_breakpoint_remove_by_ref(cpu, bp);
a1d1bb31
AL
1201 return 0;
1202 }
7d03f82f 1203 }
a1d1bb31 1204 return -ENOENT;
7d03f82f
EI
1205}
1206
a1d1bb31 1207/* Remove a specific breakpoint by reference. */
b3310ab3 1208void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
4c3a88a2 1209{
f0c3c505
AF
1210 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
1211
1212 breakpoint_invalidate(cpu, breakpoint->pc);
a1d1bb31 1213
7267c094 1214 g_free(breakpoint);
a1d1bb31
AL
1215}
1216
1217/* Remove all matching breakpoints. */
b3310ab3 1218void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
a1d1bb31 1219{
c0ce998e 1220 CPUBreakpoint *bp, *next;
a1d1bb31 1221
f0c3c505 1222 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
b3310ab3
AF
1223 if (bp->flags & mask) {
1224 cpu_breakpoint_remove_by_ref(cpu, bp);
1225 }
c0ce998e 1226 }
4c3a88a2
FB
1227}
1228
c33a346e
FB
1229/* enable or disable single step mode. EXCP_DEBUG is returned by the
1230 CPU loop after each instruction */
3825b28f 1231void cpu_single_step(CPUState *cpu, int enabled)
c33a346e 1232{
ed2803da
AF
1233 if (cpu->singlestep_enabled != enabled) {
1234 cpu->singlestep_enabled = enabled;
1235 if (kvm_enabled()) {
38e478ec 1236 kvm_update_guest_debug(cpu, 0);
ed2803da 1237 } else {
ccbb4d44 1238 /* must flush all the translated code to avoid inconsistencies */
e22a25c9 1239 /* XXX: only flush what is necessary */
bbd77c18 1240 tb_flush(cpu);
e22a25c9 1241 }
c33a346e 1242 }
c33a346e
FB
1243}
1244
a47dddd7 1245void cpu_abort(CPUState *cpu, const char *fmt, ...)
7501267e
FB
1246{
1247 va_list ap;
493ae1f0 1248 va_list ap2;
7501267e
FB
1249
1250 va_start(ap, fmt);
493ae1f0 1251 va_copy(ap2, ap);
7501267e
FB
1252 fprintf(stderr, "qemu: fatal: ");
1253 vfprintf(stderr, fmt, ap);
1254 fprintf(stderr, "\n");
878096ee 1255 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
013a2942 1256 if (qemu_log_separate()) {
1ee73216 1257 qemu_log_lock();
93fcfe39
AL
1258 qemu_log("qemu: fatal: ");
1259 qemu_log_vprintf(fmt, ap2);
1260 qemu_log("\n");
a0762859 1261 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
31b1a7b4 1262 qemu_log_flush();
1ee73216 1263 qemu_log_unlock();
93fcfe39 1264 qemu_log_close();
924edcae 1265 }
493ae1f0 1266 va_end(ap2);
f9373291 1267 va_end(ap);
7615936e 1268 replay_finish();
fd052bf6
RV
1269#if defined(CONFIG_USER_ONLY)
1270 {
1271 struct sigaction act;
1272 sigfillset(&act.sa_mask);
1273 act.sa_handler = SIG_DFL;
8347c185 1274 act.sa_flags = 0;
fd052bf6
RV
1275 sigaction(SIGABRT, &act, NULL);
1276 }
1277#endif
7501267e
FB
1278 abort();
1279}
1280
0124311e 1281#if !defined(CONFIG_USER_ONLY)
0dc3f44a 1282/* Called from RCU critical section */
041603fe
PB
1283static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1284{
1285 RAMBlock *block;
1286
43771539 1287 block = atomic_rcu_read(&ram_list.mru_block);
9b8424d5 1288 if (block && addr - block->offset < block->max_length) {
68851b98 1289 return block;
041603fe 1290 }
99e15582 1291 RAMBLOCK_FOREACH(block) {
9b8424d5 1292 if (addr - block->offset < block->max_length) {
041603fe
PB
1293 goto found;
1294 }
1295 }
1296
1297 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1298 abort();
1299
1300found:
43771539
PB
1301 /* It is safe to write mru_block outside the iothread lock. This
1302 * is what happens:
1303 *
1304 * mru_block = xxx
1305 * rcu_read_unlock()
1306 * xxx removed from list
1307 * rcu_read_lock()
1308 * read mru_block
1309 * mru_block = NULL;
1310 * call_rcu(reclaim_ramblock, xxx);
1311 * rcu_read_unlock()
1312 *
1313 * atomic_rcu_set is not needed here. The block was already published
1314 * when it was placed into the list. Here we're just making an extra
1315 * copy of the pointer.
1316 */
041603fe
PB
1317 ram_list.mru_block = block;
1318 return block;
1319}
1320
a2f4d5be 1321static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
d24981d3 1322{
9a13565d 1323 CPUState *cpu;
041603fe 1324 ram_addr_t start1;
a2f4d5be
JQ
1325 RAMBlock *block;
1326 ram_addr_t end;
1327
1328 end = TARGET_PAGE_ALIGN(start + length);
1329 start &= TARGET_PAGE_MASK;
d24981d3 1330
0dc3f44a 1331 rcu_read_lock();
041603fe
PB
1332 block = qemu_get_ram_block(start);
1333 assert(block == qemu_get_ram_block(end - 1));
1240be24 1334 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
9a13565d
PC
1335 CPU_FOREACH(cpu) {
1336 tlb_reset_dirty(cpu, start1, length);
1337 }
0dc3f44a 1338 rcu_read_unlock();
d24981d3
JQ
1339}
1340
5579c7f3 1341/* Note: start and end must be within the same ram block. */
03eebc9e
SH
1342bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
1343 ram_addr_t length,
1344 unsigned client)
1ccde1cb 1345{
5b82b703 1346 DirtyMemoryBlocks *blocks;
03eebc9e 1347 unsigned long end, page;
5b82b703 1348 bool dirty = false;
03eebc9e
SH
1349
1350 if (length == 0) {
1351 return false;
1352 }
f23db169 1353
03eebc9e
SH
1354 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
1355 page = start >> TARGET_PAGE_BITS;
5b82b703
SH
1356
1357 rcu_read_lock();
1358
1359 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
1360
1361 while (page < end) {
1362 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1363 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1364 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
1365
1366 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
1367 offset, num);
1368 page += num;
1369 }
1370
1371 rcu_read_unlock();
03eebc9e
SH
1372
1373 if (dirty && tcg_enabled()) {
a2f4d5be 1374 tlb_reset_dirty_range_all(start, length);
5579c7f3 1375 }
03eebc9e
SH
1376
1377 return dirty;
1ccde1cb
FB
1378}
1379
8deaf12c
GH
1380DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
1381 (ram_addr_t start, ram_addr_t length, unsigned client)
1382{
1383 DirtyMemoryBlocks *blocks;
1384 unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
1385 ram_addr_t first = QEMU_ALIGN_DOWN(start, align);
1386 ram_addr_t last = QEMU_ALIGN_UP(start + length, align);
1387 DirtyBitmapSnapshot *snap;
1388 unsigned long page, end, dest;
1389
1390 snap = g_malloc0(sizeof(*snap) +
1391 ((last - first) >> (TARGET_PAGE_BITS + 3)));
1392 snap->start = first;
1393 snap->end = last;
1394
1395 page = first >> TARGET_PAGE_BITS;
1396 end = last >> TARGET_PAGE_BITS;
1397 dest = 0;
1398
1399 rcu_read_lock();
1400
1401 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
1402
1403 while (page < end) {
1404 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1405 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1406 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
1407
1408 assert(QEMU_IS_ALIGNED(offset, (1 << BITS_PER_LEVEL)));
1409 assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
1410 offset >>= BITS_PER_LEVEL;
1411
1412 bitmap_copy_and_clear_atomic(snap->dirty + dest,
1413 blocks->blocks[idx] + offset,
1414 num);
1415 page += num;
1416 dest += num >> BITS_PER_LEVEL;
1417 }
1418
1419 rcu_read_unlock();
1420
1421 if (tcg_enabled()) {
1422 tlb_reset_dirty_range_all(start, length);
1423 }
1424
1425 return snap;
1426}
1427
1428bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
1429 ram_addr_t start,
1430 ram_addr_t length)
1431{
1432 unsigned long page, end;
1433
1434 assert(start >= snap->start);
1435 assert(start + length <= snap->end);
1436
1437 end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
1438 page = (start - snap->start) >> TARGET_PAGE_BITS;
1439
1440 while (page < end) {
1441 if (test_bit(page, snap->dirty)) {
1442 return true;
1443 }
1444 page++;
1445 }
1446 return false;
1447}
1448
79e2b9ae 1449/* Called from RCU critical section */
bb0e627a 1450hwaddr memory_region_section_get_iotlb(CPUState *cpu,
149f54b5
PB
1451 MemoryRegionSection *section,
1452 target_ulong vaddr,
1453 hwaddr paddr, hwaddr xlat,
1454 int prot,
1455 target_ulong *address)
e5548617 1456{
a8170e5e 1457 hwaddr iotlb;
e5548617
BS
1458 CPUWatchpoint *wp;
1459
cc5bea60 1460 if (memory_region_is_ram(section->mr)) {
e5548617 1461 /* Normal RAM. */
e4e69794 1462 iotlb = memory_region_get_ram_addr(section->mr) + xlat;
e5548617 1463 if (!section->readonly) {
b41aac4f 1464 iotlb |= PHYS_SECTION_NOTDIRTY;
e5548617 1465 } else {
b41aac4f 1466 iotlb |= PHYS_SECTION_ROM;
e5548617
BS
1467 }
1468 } else {
0b8e2c10
PM
1469 AddressSpaceDispatch *d;
1470
16620684 1471 d = flatview_to_dispatch(section->fv);
0b8e2c10 1472 iotlb = section - d->map.sections;
149f54b5 1473 iotlb += xlat;
e5548617
BS
1474 }
1475
1476 /* Make accesses to pages with watchpoints go via the
1477 watchpoint trap routines. */
ff4700b0 1478 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
05068c0d 1479 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
e5548617
BS
1480 /* Avoid trapping reads of pages with a write breakpoint. */
1481 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
b41aac4f 1482 iotlb = PHYS_SECTION_WATCH + paddr;
e5548617
BS
1483 *address |= TLB_MMIO;
1484 break;
1485 }
1486 }
1487 }
1488
1489 return iotlb;
1490}
9fa3e853
FB
1491#endif /* defined(CONFIG_USER_ONLY) */
1492
e2eef170 1493#if !defined(CONFIG_USER_ONLY)
8da3ff18 1494
c227f099 1495static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
5312bd8b 1496 uint16_t section);
16620684 1497static subpage_t *subpage_init(FlatView *fv, hwaddr base);
54688b1e 1498
06329cce 1499static void *(*phys_mem_alloc)(size_t size, uint64_t *align, bool shared) =
a2b257d6 1500 qemu_anon_ram_alloc;
91138037
MA
1501
1502/*
1503 * Set a custom physical guest memory alloator.
1504 * Accelerators with unusual needs may need this. Hopefully, we can
1505 * get rid of it eventually.
1506 */
06329cce 1507void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align, bool shared))
91138037
MA
1508{
1509 phys_mem_alloc = alloc;
1510}
1511
53cb28cb
MA
1512static uint16_t phys_section_add(PhysPageMap *map,
1513 MemoryRegionSection *section)
5312bd8b 1514{
68f3f65b
PB
1515 /* The physical section number is ORed with a page-aligned
1516 * pointer to produce the iotlb entries. Thus it should
1517 * never overflow into the page-aligned value.
1518 */
53cb28cb 1519 assert(map->sections_nb < TARGET_PAGE_SIZE);
68f3f65b 1520
53cb28cb
MA
1521 if (map->sections_nb == map->sections_nb_alloc) {
1522 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1523 map->sections = g_renew(MemoryRegionSection, map->sections,
1524 map->sections_nb_alloc);
5312bd8b 1525 }
53cb28cb 1526 map->sections[map->sections_nb] = *section;
dfde4e6e 1527 memory_region_ref(section->mr);
53cb28cb 1528 return map->sections_nb++;
5312bd8b
AK
1529}
1530
058bc4b5
PB
1531static void phys_section_destroy(MemoryRegion *mr)
1532{
55b4e80b
DS
1533 bool have_sub_page = mr->subpage;
1534
dfde4e6e
PB
1535 memory_region_unref(mr);
1536
55b4e80b 1537 if (have_sub_page) {
058bc4b5 1538 subpage_t *subpage = container_of(mr, subpage_t, iomem);
b4fefef9 1539 object_unref(OBJECT(&subpage->iomem));
058bc4b5
PB
1540 g_free(subpage);
1541 }
1542}
1543
6092666e 1544static void phys_sections_free(PhysPageMap *map)
5312bd8b 1545{
9affd6fc
PB
1546 while (map->sections_nb > 0) {
1547 MemoryRegionSection *section = &map->sections[--map->sections_nb];
058bc4b5
PB
1548 phys_section_destroy(section->mr);
1549 }
9affd6fc
PB
1550 g_free(map->sections);
1551 g_free(map->nodes);
5312bd8b
AK
1552}
1553
9950322a 1554static void register_subpage(FlatView *fv, MemoryRegionSection *section)
0f0cb164 1555{
9950322a 1556 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
0f0cb164 1557 subpage_t *subpage;
a8170e5e 1558 hwaddr base = section->offset_within_address_space
0f0cb164 1559 & TARGET_PAGE_MASK;
003a0cf2 1560 MemoryRegionSection *existing = phys_page_find(d, base);
0f0cb164
AK
1561 MemoryRegionSection subsection = {
1562 .offset_within_address_space = base,
052e87b0 1563 .size = int128_make64(TARGET_PAGE_SIZE),
0f0cb164 1564 };
a8170e5e 1565 hwaddr start, end;
0f0cb164 1566
f3705d53 1567 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
0f0cb164 1568
f3705d53 1569 if (!(existing->mr->subpage)) {
16620684
AK
1570 subpage = subpage_init(fv, base);
1571 subsection.fv = fv;
0f0cb164 1572 subsection.mr = &subpage->iomem;
ac1970fb 1573 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
53cb28cb 1574 phys_section_add(&d->map, &subsection));
0f0cb164 1575 } else {
f3705d53 1576 subpage = container_of(existing->mr, subpage_t, iomem);
0f0cb164
AK
1577 }
1578 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
052e87b0 1579 end = start + int128_get64(section->size) - 1;
53cb28cb
MA
1580 subpage_register(subpage, start, end,
1581 phys_section_add(&d->map, section));
0f0cb164
AK
1582}
1583
1584
9950322a 1585static void register_multipage(FlatView *fv,
052e87b0 1586 MemoryRegionSection *section)
33417e70 1587{
9950322a 1588 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
a8170e5e 1589 hwaddr start_addr = section->offset_within_address_space;
53cb28cb 1590 uint16_t section_index = phys_section_add(&d->map, section);
052e87b0
PB
1591 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1592 TARGET_PAGE_BITS));
dd81124b 1593
733d5ef5
PB
1594 assert(num_pages);
1595 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
33417e70
FB
1596}
1597
8629d3fc 1598void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
0f0cb164 1599{
99b9cc06 1600 MemoryRegionSection now = *section, remain = *section;
052e87b0 1601 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
0f0cb164 1602
733d5ef5
PB
1603 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1604 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1605 - now.offset_within_address_space;
1606
052e87b0 1607 now.size = int128_min(int128_make64(left), now.size);
9950322a 1608 register_subpage(fv, &now);
733d5ef5 1609 } else {
052e87b0 1610 now.size = int128_zero();
733d5ef5 1611 }
052e87b0
PB
1612 while (int128_ne(remain.size, now.size)) {
1613 remain.size = int128_sub(remain.size, now.size);
1614 remain.offset_within_address_space += int128_get64(now.size);
1615 remain.offset_within_region += int128_get64(now.size);
69b67646 1616 now = remain;
052e87b0 1617 if (int128_lt(remain.size, page_size)) {
9950322a 1618 register_subpage(fv, &now);
88266249 1619 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
052e87b0 1620 now.size = page_size;
9950322a 1621 register_subpage(fv, &now);
69b67646 1622 } else {
052e87b0 1623 now.size = int128_and(now.size, int128_neg(page_size));
9950322a 1624 register_multipage(fv, &now);
69b67646 1625 }
0f0cb164
AK
1626 }
1627}
1628
62a2744c
SY
1629void qemu_flush_coalesced_mmio_buffer(void)
1630{
1631 if (kvm_enabled())
1632 kvm_flush_coalesced_mmio_buffer();
1633}
1634
b2a8658e
UD
1635void qemu_mutex_lock_ramlist(void)
1636{
1637 qemu_mutex_lock(&ram_list.mutex);
1638}
1639
1640void qemu_mutex_unlock_ramlist(void)
1641{
1642 qemu_mutex_unlock(&ram_list.mutex);
1643}
1644
be9b23c4
PX
1645void ram_block_dump(Monitor *mon)
1646{
1647 RAMBlock *block;
1648 char *psize;
1649
1650 rcu_read_lock();
1651 monitor_printf(mon, "%24s %8s %18s %18s %18s\n",
1652 "Block Name", "PSize", "Offset", "Used", "Total");
1653 RAMBLOCK_FOREACH(block) {
1654 psize = size_to_str(block->page_size);
1655 monitor_printf(mon, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
1656 " 0x%016" PRIx64 "\n", block->idstr, psize,
1657 (uint64_t)block->offset,
1658 (uint64_t)block->used_length,
1659 (uint64_t)block->max_length);
1660 g_free(psize);
1661 }
1662 rcu_read_unlock();
1663}
1664
9c607668
AK
1665#ifdef __linux__
1666/*
1667 * FIXME TOCTTOU: this iterates over memory backends' mem-path, which
1668 * may or may not name the same files / on the same filesystem now as
1669 * when we actually open and map them. Iterate over the file
1670 * descriptors instead, and use qemu_fd_getpagesize().
1671 */
1672static int find_max_supported_pagesize(Object *obj, void *opaque)
1673{
9c607668
AK
1674 long *hpsize_min = opaque;
1675
1676 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
2b108085
DG
1677 long hpsize = host_memory_backend_pagesize(MEMORY_BACKEND(obj));
1678
0de6e2a3
DG
1679 if (hpsize < *hpsize_min) {
1680 *hpsize_min = hpsize;
9c607668
AK
1681 }
1682 }
1683
1684 return 0;
1685}
1686
1687long qemu_getrampagesize(void)
1688{
1689 long hpsize = LONG_MAX;
1690 long mainrampagesize;
1691 Object *memdev_root;
1692
0de6e2a3 1693 mainrampagesize = qemu_mempath_getpagesize(mem_path);
9c607668
AK
1694
1695 /* it's possible we have memory-backend objects with
1696 * hugepage-backed RAM. these may get mapped into system
1697 * address space via -numa parameters or memory hotplug
1698 * hooks. we want to take these into account, but we
1699 * also want to make sure these supported hugepage
1700 * sizes are applicable across the entire range of memory
1701 * we may boot from, so we take the min across all
1702 * backends, and assume normal pages in cases where a
1703 * backend isn't backed by hugepages.
1704 */
1705 memdev_root = object_resolve_path("/objects", NULL);
1706 if (memdev_root) {
1707 object_child_foreach(memdev_root, find_max_supported_pagesize, &hpsize);
1708 }
1709 if (hpsize == LONG_MAX) {
1710 /* No additional memory regions found ==> Report main RAM page size */
1711 return mainrampagesize;
1712 }
1713
1714 /* If NUMA is disabled or the NUMA nodes are not backed with a
1715 * memory-backend, then there is at least one node using "normal" RAM,
1716 * so if its page size is smaller we have got to report that size instead.
1717 */
1718 if (hpsize > mainrampagesize &&
1719 (nb_numa_nodes == 0 || numa_info[0].node_memdev == NULL)) {
1720 static bool warned;
1721 if (!warned) {
1722 error_report("Huge page support disabled (n/a for main memory).");
1723 warned = true;
1724 }
1725 return mainrampagesize;
1726 }
1727
1728 return hpsize;
1729}
1730#else
1731long qemu_getrampagesize(void)
1732{
1733 return getpagesize();
1734}
1735#endif
1736
e1e84ba0 1737#ifdef __linux__
d6af99c9
HZ
1738static int64_t get_file_size(int fd)
1739{
1740 int64_t size = lseek(fd, 0, SEEK_END);
1741 if (size < 0) {
1742 return -errno;
1743 }
1744 return size;
1745}
1746
8d37b030
MAL
1747static int file_ram_open(const char *path,
1748 const char *region_name,
1749 bool *created,
1750 Error **errp)
c902760f
MT
1751{
1752 char *filename;
8ca761f6
PF
1753 char *sanitized_name;
1754 char *c;
5c3ece79 1755 int fd = -1;
c902760f 1756
8d37b030 1757 *created = false;
fd97fd44
MA
1758 for (;;) {
1759 fd = open(path, O_RDWR);
1760 if (fd >= 0) {
1761 /* @path names an existing file, use it */
1762 break;
8d31d6b6 1763 }
fd97fd44
MA
1764 if (errno == ENOENT) {
1765 /* @path names a file that doesn't exist, create it */
1766 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1767 if (fd >= 0) {
8d37b030 1768 *created = true;
fd97fd44
MA
1769 break;
1770 }
1771 } else if (errno == EISDIR) {
1772 /* @path names a directory, create a file there */
1773 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
8d37b030 1774 sanitized_name = g_strdup(region_name);
fd97fd44
MA
1775 for (c = sanitized_name; *c != '\0'; c++) {
1776 if (*c == '/') {
1777 *c = '_';
1778 }
1779 }
8ca761f6 1780
fd97fd44
MA
1781 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1782 sanitized_name);
1783 g_free(sanitized_name);
8d31d6b6 1784
fd97fd44
MA
1785 fd = mkstemp(filename);
1786 if (fd >= 0) {
1787 unlink(filename);
1788 g_free(filename);
1789 break;
1790 }
1791 g_free(filename);
8d31d6b6 1792 }
fd97fd44
MA
1793 if (errno != EEXIST && errno != EINTR) {
1794 error_setg_errno(errp, errno,
1795 "can't open backing store %s for guest RAM",
1796 path);
8d37b030 1797 return -1;
fd97fd44
MA
1798 }
1799 /*
1800 * Try again on EINTR and EEXIST. The latter happens when
1801 * something else creates the file between our two open().
1802 */
8d31d6b6 1803 }
c902760f 1804
8d37b030
MAL
1805 return fd;
1806}
1807
1808static void *file_ram_alloc(RAMBlock *block,
1809 ram_addr_t memory,
1810 int fd,
1811 bool truncate,
1812 Error **errp)
1813{
1814 void *area;
1815
863e9621 1816 block->page_size = qemu_fd_getpagesize(fd);
98376843
HZ
1817 if (block->mr->align % block->page_size) {
1818 error_setg(errp, "alignment 0x%" PRIx64
1819 " must be multiples of page size 0x%zx",
1820 block->mr->align, block->page_size);
1821 return NULL;
1822 }
1823 block->mr->align = MAX(block->page_size, block->mr->align);
8360668e
HZ
1824#if defined(__s390x__)
1825 if (kvm_enabled()) {
1826 block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
1827 }
1828#endif
fd97fd44 1829
863e9621 1830 if (memory < block->page_size) {
fd97fd44 1831 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
863e9621
DDAG
1832 "or larger than page size 0x%zx",
1833 memory, block->page_size);
8d37b030 1834 return NULL;
1775f111
HZ
1835 }
1836
863e9621 1837 memory = ROUND_UP(memory, block->page_size);
c902760f
MT
1838
1839 /*
1840 * ftruncate is not supported by hugetlbfs in older
1841 * hosts, so don't bother bailing out on errors.
1842 * If anything goes wrong with it under other filesystems,
1843 * mmap will fail.
d6af99c9
HZ
1844 *
1845 * Do not truncate the non-empty backend file to avoid corrupting
1846 * the existing data in the file. Disabling shrinking is not
1847 * enough. For example, the current vNVDIMM implementation stores
1848 * the guest NVDIMM labels at the end of the backend file. If the
1849 * backend file is later extended, QEMU will not be able to find
1850 * those labels. Therefore, extending the non-empty backend file
1851 * is disabled as well.
c902760f 1852 */
8d37b030 1853 if (truncate && ftruncate(fd, memory)) {
9742bf26 1854 perror("ftruncate");
7f56e740 1855 }
c902760f 1856
d2f39add
DD
1857 area = qemu_ram_mmap(fd, memory, block->mr->align,
1858 block->flags & RAM_SHARED);
c902760f 1859 if (area == MAP_FAILED) {
7f56e740 1860 error_setg_errno(errp, errno,
fd97fd44 1861 "unable to map backing store for guest RAM");
8d37b030 1862 return NULL;
c902760f 1863 }
ef36fa14
MT
1864
1865 if (mem_prealloc) {
1e356fc1 1866 os_mem_prealloc(fd, area, memory, smp_cpus, errp);
056b68af 1867 if (errp && *errp) {
8d37b030
MAL
1868 qemu_ram_munmap(area, memory);
1869 return NULL;
056b68af 1870 }
ef36fa14
MT
1871 }
1872
04b16653 1873 block->fd = fd;
c902760f
MT
1874 return area;
1875}
1876#endif
1877
154cc9ea
DDAG
1878/* Allocate space within the ram_addr_t space that governs the
1879 * dirty bitmaps.
1880 * Called with the ramlist lock held.
1881 */
d17b5288 1882static ram_addr_t find_ram_offset(ram_addr_t size)
04b16653
AW
1883{
1884 RAMBlock *block, *next_block;
3e837b2c 1885 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
04b16653 1886
49cd9ac6
SH
1887 assert(size != 0); /* it would hand out same offset multiple times */
1888
0dc3f44a 1889 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
04b16653 1890 return 0;
0d53d9fe 1891 }
04b16653 1892
99e15582 1893 RAMBLOCK_FOREACH(block) {
154cc9ea 1894 ram_addr_t candidate, next = RAM_ADDR_MAX;
04b16653 1895
801110ab
DDAG
1896 /* Align blocks to start on a 'long' in the bitmap
1897 * which makes the bitmap sync'ing take the fast path.
1898 */
154cc9ea 1899 candidate = block->offset + block->max_length;
801110ab 1900 candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
04b16653 1901
154cc9ea
DDAG
1902 /* Search for the closest following block
1903 * and find the gap.
1904 */
99e15582 1905 RAMBLOCK_FOREACH(next_block) {
154cc9ea 1906 if (next_block->offset >= candidate) {
04b16653
AW
1907 next = MIN(next, next_block->offset);
1908 }
1909 }
154cc9ea
DDAG
1910
1911 /* If it fits remember our place and remember the size
1912 * of gap, but keep going so that we might find a smaller
1913 * gap to fill so avoiding fragmentation.
1914 */
1915 if (next - candidate >= size && next - candidate < mingap) {
1916 offset = candidate;
1917 mingap = next - candidate;
04b16653 1918 }
154cc9ea
DDAG
1919
1920 trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
04b16653 1921 }
3e837b2c
AW
1922
1923 if (offset == RAM_ADDR_MAX) {
1924 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1925 (uint64_t)size);
1926 abort();
1927 }
1928
154cc9ea
DDAG
1929 trace_find_ram_offset(size, offset);
1930
04b16653
AW
1931 return offset;
1932}
1933
b8c48993 1934unsigned long last_ram_page(void)
d17b5288
AW
1935{
1936 RAMBlock *block;
1937 ram_addr_t last = 0;
1938
0dc3f44a 1939 rcu_read_lock();
99e15582 1940 RAMBLOCK_FOREACH(block) {
62be4e3a 1941 last = MAX(last, block->offset + block->max_length);
0d53d9fe 1942 }
0dc3f44a 1943 rcu_read_unlock();
b8c48993 1944 return last >> TARGET_PAGE_BITS;
d17b5288
AW
1945}
1946
ddb97f1d
JB
1947static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1948{
1949 int ret;
ddb97f1d
JB
1950
1951 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
47c8ca53 1952 if (!machine_dump_guest_core(current_machine)) {
ddb97f1d
JB
1953 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1954 if (ret) {
1955 perror("qemu_madvise");
1956 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1957 "but dump_guest_core=off specified\n");
1958 }
1959 }
1960}
1961
422148d3
DDAG
1962const char *qemu_ram_get_idstr(RAMBlock *rb)
1963{
1964 return rb->idstr;
1965}
1966
463a4ac2
DDAG
1967bool qemu_ram_is_shared(RAMBlock *rb)
1968{
1969 return rb->flags & RAM_SHARED;
1970}
1971
2ce16640
DDAG
1972/* Note: Only set at the start of postcopy */
1973bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
1974{
1975 return rb->flags & RAM_UF_ZEROPAGE;
1976}
1977
1978void qemu_ram_set_uf_zeroable(RAMBlock *rb)
1979{
1980 rb->flags |= RAM_UF_ZEROPAGE;
1981}
1982
b895de50
CLG
1983bool qemu_ram_is_migratable(RAMBlock *rb)
1984{
1985 return rb->flags & RAM_MIGRATABLE;
1986}
1987
1988void qemu_ram_set_migratable(RAMBlock *rb)
1989{
1990 rb->flags |= RAM_MIGRATABLE;
1991}
1992
1993void qemu_ram_unset_migratable(RAMBlock *rb)
1994{
1995 rb->flags &= ~RAM_MIGRATABLE;
1996}
1997
ae3a7047 1998/* Called with iothread lock held. */
fa53a0e5 1999void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
20cfe881 2000{
fa53a0e5 2001 RAMBlock *block;
20cfe881 2002
c5705a77
AK
2003 assert(new_block);
2004 assert(!new_block->idstr[0]);
84b89d78 2005
09e5ab63
AL
2006 if (dev) {
2007 char *id = qdev_get_dev_path(dev);
84b89d78
CM
2008 if (id) {
2009 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
7267c094 2010 g_free(id);
84b89d78
CM
2011 }
2012 }
2013 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2014
ab0a9956 2015 rcu_read_lock();
99e15582 2016 RAMBLOCK_FOREACH(block) {
fa53a0e5
GA
2017 if (block != new_block &&
2018 !strcmp(block->idstr, new_block->idstr)) {
84b89d78
CM
2019 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2020 new_block->idstr);
2021 abort();
2022 }
2023 }
0dc3f44a 2024 rcu_read_unlock();
c5705a77
AK
2025}
2026
ae3a7047 2027/* Called with iothread lock held. */
fa53a0e5 2028void qemu_ram_unset_idstr(RAMBlock *block)
20cfe881 2029{
ae3a7047
MD
2030 /* FIXME: arch_init.c assumes that this is not called throughout
2031 * migration. Ignore the problem since hot-unplug during migration
2032 * does not work anyway.
2033 */
20cfe881
HT
2034 if (block) {
2035 memset(block->idstr, 0, sizeof(block->idstr));
2036 }
2037}
2038
863e9621
DDAG
2039size_t qemu_ram_pagesize(RAMBlock *rb)
2040{
2041 return rb->page_size;
2042}
2043
67f11b5c
DDAG
2044/* Returns the largest size of page in use */
2045size_t qemu_ram_pagesize_largest(void)
2046{
2047 RAMBlock *block;
2048 size_t largest = 0;
2049
99e15582 2050 RAMBLOCK_FOREACH(block) {
67f11b5c
DDAG
2051 largest = MAX(largest, qemu_ram_pagesize(block));
2052 }
2053
2054 return largest;
2055}
2056
8490fc78
LC
2057static int memory_try_enable_merging(void *addr, size_t len)
2058{
75cc7f01 2059 if (!machine_mem_merge(current_machine)) {
8490fc78
LC
2060 /* disabled by the user */
2061 return 0;
2062 }
2063
2064 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
2065}
2066
62be4e3a
MT
2067/* Only legal before guest might have detected the memory size: e.g. on
2068 * incoming migration, or right after reset.
2069 *
2070 * As memory core doesn't know how is memory accessed, it is up to
2071 * resize callback to update device state and/or add assertions to detect
2072 * misuse, if necessary.
2073 */
fa53a0e5 2074int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
62be4e3a 2075{
62be4e3a
MT
2076 assert(block);
2077
4ed023ce 2078 newsize = HOST_PAGE_ALIGN(newsize);
129ddaf3 2079
62be4e3a
MT
2080 if (block->used_length == newsize) {
2081 return 0;
2082 }
2083
2084 if (!(block->flags & RAM_RESIZEABLE)) {
2085 error_setg_errno(errp, EINVAL,
2086 "Length mismatch: %s: 0x" RAM_ADDR_FMT
2087 " in != 0x" RAM_ADDR_FMT, block->idstr,
2088 newsize, block->used_length);
2089 return -EINVAL;
2090 }
2091
2092 if (block->max_length < newsize) {
2093 error_setg_errno(errp, EINVAL,
2094 "Length too large: %s: 0x" RAM_ADDR_FMT
2095 " > 0x" RAM_ADDR_FMT, block->idstr,
2096 newsize, block->max_length);
2097 return -EINVAL;
2098 }
2099
2100 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
2101 block->used_length = newsize;
58d2707e
PB
2102 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
2103 DIRTY_CLIENTS_ALL);
62be4e3a
MT
2104 memory_region_set_size(block->mr, newsize);
2105 if (block->resized) {
2106 block->resized(block->idstr, newsize, block->host);
2107 }
2108 return 0;
2109}
2110
5b82b703
SH
2111/* Called with ram_list.mutex held */
2112static void dirty_memory_extend(ram_addr_t old_ram_size,
2113 ram_addr_t new_ram_size)
2114{
2115 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
2116 DIRTY_MEMORY_BLOCK_SIZE);
2117 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
2118 DIRTY_MEMORY_BLOCK_SIZE);
2119 int i;
2120
2121 /* Only need to extend if block count increased */
2122 if (new_num_blocks <= old_num_blocks) {
2123 return;
2124 }
2125
2126 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
2127 DirtyMemoryBlocks *old_blocks;
2128 DirtyMemoryBlocks *new_blocks;
2129 int j;
2130
2131 old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
2132 new_blocks = g_malloc(sizeof(*new_blocks) +
2133 sizeof(new_blocks->blocks[0]) * new_num_blocks);
2134
2135 if (old_num_blocks) {
2136 memcpy(new_blocks->blocks, old_blocks->blocks,
2137 old_num_blocks * sizeof(old_blocks->blocks[0]));
2138 }
2139
2140 for (j = old_num_blocks; j < new_num_blocks; j++) {
2141 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
2142 }
2143
2144 atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
2145
2146 if (old_blocks) {
2147 g_free_rcu(old_blocks, rcu);
2148 }
2149 }
2150}
2151
06329cce 2152static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)
c5705a77 2153{
e1c57ab8 2154 RAMBlock *block;
0d53d9fe 2155 RAMBlock *last_block = NULL;
2152f5ca 2156 ram_addr_t old_ram_size, new_ram_size;
37aa7a0e 2157 Error *err = NULL;
2152f5ca 2158
b8c48993 2159 old_ram_size = last_ram_page();
c5705a77 2160
b2a8658e 2161 qemu_mutex_lock_ramlist();
9b8424d5 2162 new_block->offset = find_ram_offset(new_block->max_length);
e1c57ab8
PB
2163
2164 if (!new_block->host) {
2165 if (xen_enabled()) {
9b8424d5 2166 xen_ram_alloc(new_block->offset, new_block->max_length,
37aa7a0e
MA
2167 new_block->mr, &err);
2168 if (err) {
2169 error_propagate(errp, err);
2170 qemu_mutex_unlock_ramlist();
39c350ee 2171 return;
37aa7a0e 2172 }
e1c57ab8 2173 } else {
9b8424d5 2174 new_block->host = phys_mem_alloc(new_block->max_length,
06329cce 2175 &new_block->mr->align, shared);
39228250 2176 if (!new_block->host) {
ef701d7b
HT
2177 error_setg_errno(errp, errno,
2178 "cannot set up guest memory '%s'",
2179 memory_region_name(new_block->mr));
2180 qemu_mutex_unlock_ramlist();
39c350ee 2181 return;
39228250 2182 }
9b8424d5 2183 memory_try_enable_merging(new_block->host, new_block->max_length);
6977dfe6 2184 }
c902760f 2185 }
94a6b54f 2186
dd631697
LZ
2187 new_ram_size = MAX(old_ram_size,
2188 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
2189 if (new_ram_size > old_ram_size) {
5b82b703 2190 dirty_memory_extend(old_ram_size, new_ram_size);
dd631697 2191 }
0d53d9fe
MD
2192 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
2193 * QLIST (which has an RCU-friendly variant) does not have insertion at
2194 * tail, so save the last element in last_block.
2195 */
99e15582 2196 RAMBLOCK_FOREACH(block) {
0d53d9fe 2197 last_block = block;
9b8424d5 2198 if (block->max_length < new_block->max_length) {
abb26d63
PB
2199 break;
2200 }
2201 }
2202 if (block) {
0dc3f44a 2203 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
0d53d9fe 2204 } else if (last_block) {
0dc3f44a 2205 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
0d53d9fe 2206 } else { /* list is empty */
0dc3f44a 2207 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
abb26d63 2208 }
0d6d3c87 2209 ram_list.mru_block = NULL;
94a6b54f 2210
0dc3f44a
MD
2211 /* Write list before version */
2212 smp_wmb();
f798b07f 2213 ram_list.version++;
b2a8658e 2214 qemu_mutex_unlock_ramlist();
f798b07f 2215
9b8424d5 2216 cpu_physical_memory_set_dirty_range(new_block->offset,
58d2707e
PB
2217 new_block->used_length,
2218 DIRTY_CLIENTS_ALL);
94a6b54f 2219
a904c911
PB
2220 if (new_block->host) {
2221 qemu_ram_setup_dump(new_block->host, new_block->max_length);
2222 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
c2cd627d 2223 /* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU */
a904c911 2224 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
0987d735 2225 ram_block_notify_add(new_block->host, new_block->max_length);
e1c57ab8 2226 }
94a6b54f 2227}
e9a1ab19 2228
0b183fc8 2229#ifdef __linux__
38b3362d
MAL
2230RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
2231 bool share, int fd,
2232 Error **errp)
e1c57ab8
PB
2233{
2234 RAMBlock *new_block;
ef701d7b 2235 Error *local_err = NULL;
8d37b030 2236 int64_t file_size;
e1c57ab8
PB
2237
2238 if (xen_enabled()) {
7f56e740 2239 error_setg(errp, "-mem-path not supported with Xen");
528f46af 2240 return NULL;
e1c57ab8
PB
2241 }
2242
e45e7ae2
MAL
2243 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2244 error_setg(errp,
2245 "host lacks kvm mmu notifiers, -mem-path unsupported");
2246 return NULL;
2247 }
2248
e1c57ab8
PB
2249 if (phys_mem_alloc != qemu_anon_ram_alloc) {
2250 /*
2251 * file_ram_alloc() needs to allocate just like
2252 * phys_mem_alloc, but we haven't bothered to provide
2253 * a hook there.
2254 */
7f56e740
PB
2255 error_setg(errp,
2256 "-mem-path not supported with this accelerator");
528f46af 2257 return NULL;
e1c57ab8
PB
2258 }
2259
4ed023ce 2260 size = HOST_PAGE_ALIGN(size);
8d37b030
MAL
2261 file_size = get_file_size(fd);
2262 if (file_size > 0 && file_size < size) {
2263 error_setg(errp, "backing store %s size 0x%" PRIx64
2264 " does not match 'size' option 0x" RAM_ADDR_FMT,
2265 mem_path, file_size, size);
8d37b030
MAL
2266 return NULL;
2267 }
2268
e1c57ab8
PB
2269 new_block = g_malloc0(sizeof(*new_block));
2270 new_block->mr = mr;
9b8424d5
MT
2271 new_block->used_length = size;
2272 new_block->max_length = size;
dbcb8981 2273 new_block->flags = share ? RAM_SHARED : 0;
8d37b030 2274 new_block->host = file_ram_alloc(new_block, size, fd, !file_size, errp);
7f56e740
PB
2275 if (!new_block->host) {
2276 g_free(new_block);
528f46af 2277 return NULL;
7f56e740
PB
2278 }
2279
06329cce 2280 ram_block_add(new_block, &local_err, share);
ef701d7b
HT
2281 if (local_err) {
2282 g_free(new_block);
2283 error_propagate(errp, local_err);
528f46af 2284 return NULL;
ef701d7b 2285 }
528f46af 2286 return new_block;
38b3362d
MAL
2287
2288}
2289
2290
2291RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
2292 bool share, const char *mem_path,
2293 Error **errp)
2294{
2295 int fd;
2296 bool created;
2297 RAMBlock *block;
2298
2299 fd = file_ram_open(mem_path, memory_region_name(mr), &created, errp);
2300 if (fd < 0) {
2301 return NULL;
2302 }
2303
2304 block = qemu_ram_alloc_from_fd(size, mr, share, fd, errp);
2305 if (!block) {
2306 if (created) {
2307 unlink(mem_path);
2308 }
2309 close(fd);
2310 return NULL;
2311 }
2312
2313 return block;
e1c57ab8 2314}
0b183fc8 2315#endif
e1c57ab8 2316
62be4e3a 2317static
528f46af
FZ
2318RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
2319 void (*resized)(const char*,
2320 uint64_t length,
2321 void *host),
06329cce 2322 void *host, bool resizeable, bool share,
528f46af 2323 MemoryRegion *mr, Error **errp)
e1c57ab8
PB
2324{
2325 RAMBlock *new_block;
ef701d7b 2326 Error *local_err = NULL;
e1c57ab8 2327
4ed023ce
DDAG
2328 size = HOST_PAGE_ALIGN(size);
2329 max_size = HOST_PAGE_ALIGN(max_size);
e1c57ab8
PB
2330 new_block = g_malloc0(sizeof(*new_block));
2331 new_block->mr = mr;
62be4e3a 2332 new_block->resized = resized;
9b8424d5
MT
2333 new_block->used_length = size;
2334 new_block->max_length = max_size;
62be4e3a 2335 assert(max_size >= size);
e1c57ab8 2336 new_block->fd = -1;
863e9621 2337 new_block->page_size = getpagesize();
e1c57ab8
PB
2338 new_block->host = host;
2339 if (host) {
7bd4f430 2340 new_block->flags |= RAM_PREALLOC;
e1c57ab8 2341 }
62be4e3a
MT
2342 if (resizeable) {
2343 new_block->flags |= RAM_RESIZEABLE;
2344 }
06329cce 2345 ram_block_add(new_block, &local_err, share);
ef701d7b
HT
2346 if (local_err) {
2347 g_free(new_block);
2348 error_propagate(errp, local_err);
528f46af 2349 return NULL;
ef701d7b 2350 }
528f46af 2351 return new_block;
e1c57ab8
PB
2352}
2353
528f46af 2354RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
62be4e3a
MT
2355 MemoryRegion *mr, Error **errp)
2356{
06329cce
MA
2357 return qemu_ram_alloc_internal(size, size, NULL, host, false,
2358 false, mr, errp);
62be4e3a
MT
2359}
2360
06329cce
MA
2361RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share,
2362 MemoryRegion *mr, Error **errp)
6977dfe6 2363{
06329cce
MA
2364 return qemu_ram_alloc_internal(size, size, NULL, NULL, false,
2365 share, mr, errp);
62be4e3a
MT
2366}
2367
528f46af 2368RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
62be4e3a
MT
2369 void (*resized)(const char*,
2370 uint64_t length,
2371 void *host),
2372 MemoryRegion *mr, Error **errp)
2373{
06329cce
MA
2374 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true,
2375 false, mr, errp);
6977dfe6
YT
2376}
2377
43771539
PB
2378static void reclaim_ramblock(RAMBlock *block)
2379{
2380 if (block->flags & RAM_PREALLOC) {
2381 ;
2382 } else if (xen_enabled()) {
2383 xen_invalidate_map_cache_entry(block->host);
2384#ifndef _WIN32
2385 } else if (block->fd >= 0) {
2f3a2bb1 2386 qemu_ram_munmap(block->host, block->max_length);
43771539
PB
2387 close(block->fd);
2388#endif
2389 } else {
2390 qemu_anon_ram_free(block->host, block->max_length);
2391 }
2392 g_free(block);
2393}
2394
f1060c55 2395void qemu_ram_free(RAMBlock *block)
e9a1ab19 2396{
85bc2a15
MAL
2397 if (!block) {
2398 return;
2399 }
2400
0987d735
PB
2401 if (block->host) {
2402 ram_block_notify_remove(block->host, block->max_length);
2403 }
2404
b2a8658e 2405 qemu_mutex_lock_ramlist();
f1060c55
FZ
2406 QLIST_REMOVE_RCU(block, next);
2407 ram_list.mru_block = NULL;
2408 /* Write list before version */
2409 smp_wmb();
2410 ram_list.version++;
2411 call_rcu(block, reclaim_ramblock, rcu);
b2a8658e 2412 qemu_mutex_unlock_ramlist();
e9a1ab19
FB
2413}
2414
cd19cfa2
HY
2415#ifndef _WIN32
2416void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2417{
2418 RAMBlock *block;
2419 ram_addr_t offset;
2420 int flags;
2421 void *area, *vaddr;
2422
99e15582 2423 RAMBLOCK_FOREACH(block) {
cd19cfa2 2424 offset = addr - block->offset;
9b8424d5 2425 if (offset < block->max_length) {
1240be24 2426 vaddr = ramblock_ptr(block, offset);
7bd4f430 2427 if (block->flags & RAM_PREALLOC) {
cd19cfa2 2428 ;
dfeaf2ab
MA
2429 } else if (xen_enabled()) {
2430 abort();
cd19cfa2
HY
2431 } else {
2432 flags = MAP_FIXED;
3435f395 2433 if (block->fd >= 0) {
dbcb8981
PB
2434 flags |= (block->flags & RAM_SHARED ?
2435 MAP_SHARED : MAP_PRIVATE);
3435f395
MA
2436 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
2437 flags, block->fd, offset);
cd19cfa2 2438 } else {
2eb9fbaa
MA
2439 /*
2440 * Remap needs to match alloc. Accelerators that
2441 * set phys_mem_alloc never remap. If they did,
2442 * we'd need a remap hook here.
2443 */
2444 assert(phys_mem_alloc == qemu_anon_ram_alloc);
2445
cd19cfa2
HY
2446 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
2447 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
2448 flags, -1, 0);
cd19cfa2
HY
2449 }
2450 if (area != vaddr) {
493d89bf
AF
2451 error_report("Could not remap addr: "
2452 RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
2453 length, addr);
cd19cfa2
HY
2454 exit(1);
2455 }
8490fc78 2456 memory_try_enable_merging(vaddr, length);
ddb97f1d 2457 qemu_ram_setup_dump(vaddr, length);
cd19cfa2 2458 }
cd19cfa2
HY
2459 }
2460 }
2461}
2462#endif /* !_WIN32 */
2463
1b5ec234 2464/* Return a host pointer to ram allocated with qemu_ram_alloc.
ae3a7047
MD
2465 * This should not be used for general purpose DMA. Use address_space_map
2466 * or address_space_rw instead. For local memory (e.g. video ram) that the
2467 * device owns, use memory_region_get_ram_ptr.
0dc3f44a 2468 *
49b24afc 2469 * Called within RCU critical section.
1b5ec234 2470 */
0878d0e1 2471void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
1b5ec234 2472{
3655cb9c
GA
2473 RAMBlock *block = ram_block;
2474
2475 if (block == NULL) {
2476 block = qemu_get_ram_block(addr);
0878d0e1 2477 addr -= block->offset;
3655cb9c 2478 }
ae3a7047
MD
2479
2480 if (xen_enabled() && block->host == NULL) {
0d6d3c87
PB
2481 /* We need to check if the requested address is in the RAM
2482 * because we don't want to map the entire memory in QEMU.
2483 * In that case just map until the end of the page.
2484 */
2485 if (block->offset == 0) {
1ff7c598 2486 return xen_map_cache(addr, 0, 0, false);
0d6d3c87 2487 }
ae3a7047 2488
1ff7c598 2489 block->host = xen_map_cache(block->offset, block->max_length, 1, false);
0d6d3c87 2490 }
0878d0e1 2491 return ramblock_ptr(block, addr);
dc828ca1
PB
2492}
2493
0878d0e1 2494/* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
ae3a7047 2495 * but takes a size argument.
0dc3f44a 2496 *
e81bcda5 2497 * Called within RCU critical section.
ae3a7047 2498 */
3655cb9c 2499static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
f5aa69bd 2500 hwaddr *size, bool lock)
38bee5dc 2501{
3655cb9c 2502 RAMBlock *block = ram_block;
8ab934f9
SS
2503 if (*size == 0) {
2504 return NULL;
2505 }
e81bcda5 2506
3655cb9c
GA
2507 if (block == NULL) {
2508 block = qemu_get_ram_block(addr);
0878d0e1 2509 addr -= block->offset;
3655cb9c 2510 }
0878d0e1 2511 *size = MIN(*size, block->max_length - addr);
e81bcda5
PB
2512
2513 if (xen_enabled() && block->host == NULL) {
2514 /* We need to check if the requested address is in the RAM
2515 * because we don't want to map the entire memory in QEMU.
2516 * In that case just map the requested area.
2517 */
2518 if (block->offset == 0) {
f5aa69bd 2519 return xen_map_cache(addr, *size, lock, lock);
38bee5dc
SS
2520 }
2521
f5aa69bd 2522 block->host = xen_map_cache(block->offset, block->max_length, 1, lock);
38bee5dc 2523 }
e81bcda5 2524
0878d0e1 2525 return ramblock_ptr(block, addr);
38bee5dc
SS
2526}
2527
f90bb71b
DDAG
2528/* Return the offset of a hostpointer within a ramblock */
2529ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
2530{
2531 ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
2532 assert((uintptr_t)host >= (uintptr_t)rb->host);
2533 assert(res < rb->max_length);
2534
2535 return res;
2536}
2537
422148d3
DDAG
2538/*
2539 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
2540 * in that RAMBlock.
2541 *
2542 * ptr: Host pointer to look up
2543 * round_offset: If true round the result offset down to a page boundary
2544 * *ram_addr: set to result ram_addr
2545 * *offset: set to result offset within the RAMBlock
2546 *
2547 * Returns: RAMBlock (or NULL if not found)
ae3a7047
MD
2548 *
2549 * By the time this function returns, the returned pointer is not protected
2550 * by RCU anymore. If the caller is not within an RCU critical section and
2551 * does not hold the iothread lock, it must have other means of protecting the
2552 * pointer, such as a reference to the region that includes the incoming
2553 * ram_addr_t.
2554 */
422148d3 2555RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
422148d3 2556 ram_addr_t *offset)
5579c7f3 2557{
94a6b54f
PB
2558 RAMBlock *block;
2559 uint8_t *host = ptr;
2560
868bb33f 2561 if (xen_enabled()) {
f615f396 2562 ram_addr_t ram_addr;
0dc3f44a 2563 rcu_read_lock();
f615f396
PB
2564 ram_addr = xen_ram_addr_from_mapcache(ptr);
2565 block = qemu_get_ram_block(ram_addr);
422148d3 2566 if (block) {
d6b6aec4 2567 *offset = ram_addr - block->offset;
422148d3 2568 }
0dc3f44a 2569 rcu_read_unlock();
422148d3 2570 return block;
712c2b41
SS
2571 }
2572
0dc3f44a
MD
2573 rcu_read_lock();
2574 block = atomic_rcu_read(&ram_list.mru_block);
9b8424d5 2575 if (block && block->host && host - block->host < block->max_length) {
23887b79
PB
2576 goto found;
2577 }
2578
99e15582 2579 RAMBLOCK_FOREACH(block) {
432d268c
JN
2580 /* This case append when the block is not mapped. */
2581 if (block->host == NULL) {
2582 continue;
2583 }
9b8424d5 2584 if (host - block->host < block->max_length) {
23887b79 2585 goto found;
f471a17e 2586 }
94a6b54f 2587 }
432d268c 2588
0dc3f44a 2589 rcu_read_unlock();
1b5ec234 2590 return NULL;
23887b79
PB
2591
2592found:
422148d3
DDAG
2593 *offset = (host - block->host);
2594 if (round_offset) {
2595 *offset &= TARGET_PAGE_MASK;
2596 }
0dc3f44a 2597 rcu_read_unlock();
422148d3
DDAG
2598 return block;
2599}
2600
e3dd7493
DDAG
2601/*
2602 * Finds the named RAMBlock
2603 *
2604 * name: The name of RAMBlock to find
2605 *
2606 * Returns: RAMBlock (or NULL if not found)
2607 */
2608RAMBlock *qemu_ram_block_by_name(const char *name)
2609{
2610 RAMBlock *block;
2611
99e15582 2612 RAMBLOCK_FOREACH(block) {
e3dd7493
DDAG
2613 if (!strcmp(name, block->idstr)) {
2614 return block;
2615 }
2616 }
2617
2618 return NULL;
2619}
2620
422148d3
DDAG
2621/* Some of the softmmu routines need to translate from a host pointer
2622 (typically a TLB entry) back to a ram offset. */
07bdaa41 2623ram_addr_t qemu_ram_addr_from_host(void *ptr)
422148d3
DDAG
2624{
2625 RAMBlock *block;
f615f396 2626 ram_addr_t offset;
422148d3 2627
f615f396 2628 block = qemu_ram_block_from_host(ptr, false, &offset);
422148d3 2629 if (!block) {
07bdaa41 2630 return RAM_ADDR_INVALID;
422148d3
DDAG
2631 }
2632
07bdaa41 2633 return block->offset + offset;
e890261f 2634}
f471a17e 2635
27266271
PM
2636/* Called within RCU critical section. */
2637void memory_notdirty_write_prepare(NotDirtyInfo *ndi,
2638 CPUState *cpu,
2639 vaddr mem_vaddr,
2640 ram_addr_t ram_addr,
2641 unsigned size)
2642{
2643 ndi->cpu = cpu;
2644 ndi->ram_addr = ram_addr;
2645 ndi->mem_vaddr = mem_vaddr;
2646 ndi->size = size;
2647 ndi->locked = false;
ba051fb5 2648
5aa1ef71 2649 assert(tcg_enabled());
52159192 2650 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
27266271 2651 ndi->locked = true;
ba051fb5 2652 tb_lock();
0e0df1e2 2653 tb_invalidate_phys_page_fast(ram_addr, size);
3a7d929e 2654 }
27266271
PM
2655}
2656
2657/* Called within RCU critical section. */
2658void memory_notdirty_write_complete(NotDirtyInfo *ndi)
2659{
2660 if (ndi->locked) {
2661 tb_unlock();
2662 }
2663
2664 /* Set both VGA and migration bits for simplicity and to remove
2665 * the notdirty callback faster.
2666 */
2667 cpu_physical_memory_set_dirty_range(ndi->ram_addr, ndi->size,
2668 DIRTY_CLIENTS_NOCODE);
2669 /* we remove the notdirty callback only if the code has been
2670 flushed */
2671 if (!cpu_physical_memory_is_clean(ndi->ram_addr)) {
2672 tlb_set_dirty(ndi->cpu, ndi->mem_vaddr);
2673 }
2674}
2675
2676/* Called within RCU critical section. */
2677static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
2678 uint64_t val, unsigned size)
2679{
2680 NotDirtyInfo ndi;
2681
2682 memory_notdirty_write_prepare(&ndi, current_cpu, current_cpu->mem_io_vaddr,
2683 ram_addr, size);
2684
6d3ede54 2685 stn_p(qemu_map_ram_ptr(NULL, ram_addr), size, val);
27266271 2686 memory_notdirty_write_complete(&ndi);
9fa3e853
FB
2687}
2688
b018ddf6 2689static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
8372d383
PM
2690 unsigned size, bool is_write,
2691 MemTxAttrs attrs)
b018ddf6
PB
2692{
2693 return is_write;
2694}
2695
0e0df1e2 2696static const MemoryRegionOps notdirty_mem_ops = {
0e0df1e2 2697 .write = notdirty_mem_write,
b018ddf6 2698 .valid.accepts = notdirty_mem_accepts,
0e0df1e2 2699 .endianness = DEVICE_NATIVE_ENDIAN,
ad52878f
AB
2700 .valid = {
2701 .min_access_size = 1,
2702 .max_access_size = 8,
2703 .unaligned = false,
2704 },
2705 .impl = {
2706 .min_access_size = 1,
2707 .max_access_size = 8,
2708 .unaligned = false,
2709 },
1ccde1cb
FB
2710};
2711
0f459d16 2712/* Generate a debug exception if a watchpoint has been hit. */
66b9b43c 2713static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
0f459d16 2714{
93afeade 2715 CPUState *cpu = current_cpu;
568496c0 2716 CPUClass *cc = CPU_GET_CLASS(cpu);
0f459d16 2717 target_ulong vaddr;
a1d1bb31 2718 CPUWatchpoint *wp;
0f459d16 2719
5aa1ef71 2720 assert(tcg_enabled());
ff4700b0 2721 if (cpu->watchpoint_hit) {
06d55cc1
AL
2722 /* We re-entered the check after replacing the TB. Now raise
2723 * the debug interrupt so that is will trigger after the
2724 * current instruction. */
93afeade 2725 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
06d55cc1
AL
2726 return;
2727 }
93afeade 2728 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
40612000 2729 vaddr = cc->adjust_watchpoint_address(cpu, vaddr, len);
ff4700b0 2730 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
05068c0d
PM
2731 if (cpu_watchpoint_address_matches(wp, vaddr, len)
2732 && (wp->flags & flags)) {
08225676
PM
2733 if (flags == BP_MEM_READ) {
2734 wp->flags |= BP_WATCHPOINT_HIT_READ;
2735 } else {
2736 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
2737 }
2738 wp->hitaddr = vaddr;
66b9b43c 2739 wp->hitattrs = attrs;
ff4700b0 2740 if (!cpu->watchpoint_hit) {
568496c0
SF
2741 if (wp->flags & BP_CPU &&
2742 !cc->debug_check_watchpoint(cpu, wp)) {
2743 wp->flags &= ~BP_WATCHPOINT_HIT;
2744 continue;
2745 }
ff4700b0 2746 cpu->watchpoint_hit = wp;
a5e99826 2747
8d04fb55
JK
2748 /* Both tb_lock and iothread_mutex will be reset when
2749 * cpu_loop_exit or cpu_loop_exit_noexc longjmp
2750 * back into the cpu_exec main loop.
a5e99826
FK
2751 */
2752 tb_lock();
239c51a5 2753 tb_check_watchpoint(cpu);
6e140f28 2754 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
27103424 2755 cpu->exception_index = EXCP_DEBUG;
5638d180 2756 cpu_loop_exit(cpu);
6e140f28 2757 } else {
9b990ee5
RH
2758 /* Force execution of one insn next time. */
2759 cpu->cflags_next_tb = 1 | curr_cflags();
6886b980 2760 cpu_loop_exit_noexc(cpu);
6e140f28 2761 }
06d55cc1 2762 }
6e140f28
AL
2763 } else {
2764 wp->flags &= ~BP_WATCHPOINT_HIT;
0f459d16
PB
2765 }
2766 }
2767}
2768
6658ffb8
PB
2769/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2770 so these check for a hit then pass through to the normal out-of-line
2771 phys routines. */
66b9b43c
PM
2772static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2773 unsigned size, MemTxAttrs attrs)
6658ffb8 2774{
66b9b43c
PM
2775 MemTxResult res;
2776 uint64_t data;
79ed0416
PM
2777 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2778 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
66b9b43c
PM
2779
2780 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
1ec9b909 2781 switch (size) {
66b9b43c 2782 case 1:
79ed0416 2783 data = address_space_ldub(as, addr, attrs, &res);
66b9b43c
PM
2784 break;
2785 case 2:
79ed0416 2786 data = address_space_lduw(as, addr, attrs, &res);
66b9b43c
PM
2787 break;
2788 case 4:
79ed0416 2789 data = address_space_ldl(as, addr, attrs, &res);
66b9b43c 2790 break;
306526b5
PB
2791 case 8:
2792 data = address_space_ldq(as, addr, attrs, &res);
2793 break;
1ec9b909
AK
2794 default: abort();
2795 }
66b9b43c
PM
2796 *pdata = data;
2797 return res;
6658ffb8
PB
2798}
2799
66b9b43c
PM
2800static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2801 uint64_t val, unsigned size,
2802 MemTxAttrs attrs)
6658ffb8 2803{
66b9b43c 2804 MemTxResult res;
79ed0416
PM
2805 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2806 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
66b9b43c
PM
2807
2808 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
1ec9b909 2809 switch (size) {
67364150 2810 case 1:
79ed0416 2811 address_space_stb(as, addr, val, attrs, &res);
67364150
MF
2812 break;
2813 case 2:
79ed0416 2814 address_space_stw(as, addr, val, attrs, &res);
67364150
MF
2815 break;
2816 case 4:
79ed0416 2817 address_space_stl(as, addr, val, attrs, &res);
67364150 2818 break;
306526b5
PB
2819 case 8:
2820 address_space_stq(as, addr, val, attrs, &res);
2821 break;
1ec9b909
AK
2822 default: abort();
2823 }
66b9b43c 2824 return res;
6658ffb8
PB
2825}
2826
1ec9b909 2827static const MemoryRegionOps watch_mem_ops = {
66b9b43c
PM
2828 .read_with_attrs = watch_mem_read,
2829 .write_with_attrs = watch_mem_write,
1ec9b909 2830 .endianness = DEVICE_NATIVE_ENDIAN,
306526b5
PB
2831 .valid = {
2832 .min_access_size = 1,
2833 .max_access_size = 8,
2834 .unaligned = false,
2835 },
2836 .impl = {
2837 .min_access_size = 1,
2838 .max_access_size = 8,
2839 .unaligned = false,
2840 },
6658ffb8 2841};
6658ffb8 2842
b2a44fca
PB
2843static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
2844 MemTxAttrs attrs, uint8_t *buf, int len);
16620684
AK
2845static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
2846 const uint8_t *buf, int len);
2847static bool flatview_access_valid(FlatView *fv, hwaddr addr, int len,
eace72b7 2848 bool is_write, MemTxAttrs attrs);
16620684 2849
f25a49e0
PM
2850static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2851 unsigned len, MemTxAttrs attrs)
db7b5426 2852{
acc9d80b 2853 subpage_t *subpage = opaque;
ff6cff75 2854 uint8_t buf[8];
5c9eb028 2855 MemTxResult res;
791af8c8 2856
db7b5426 2857#if defined(DEBUG_SUBPAGE)
016e9d62 2858 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
acc9d80b 2859 subpage, len, addr);
db7b5426 2860#endif
16620684 2861 res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
5c9eb028
PM
2862 if (res) {
2863 return res;
f25a49e0 2864 }
6d3ede54
PM
2865 *data = ldn_p(buf, len);
2866 return MEMTX_OK;
db7b5426
BS
2867}
2868
f25a49e0
PM
2869static MemTxResult subpage_write(void *opaque, hwaddr addr,
2870 uint64_t value, unsigned len, MemTxAttrs attrs)
db7b5426 2871{
acc9d80b 2872 subpage_t *subpage = opaque;
ff6cff75 2873 uint8_t buf[8];
acc9d80b 2874
db7b5426 2875#if defined(DEBUG_SUBPAGE)
016e9d62 2876 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
acc9d80b
JK
2877 " value %"PRIx64"\n",
2878 __func__, subpage, len, addr, value);
db7b5426 2879#endif
6d3ede54 2880 stn_p(buf, len, value);
16620684 2881 return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
db7b5426
BS
2882}
2883
c353e4cc 2884static bool subpage_accepts(void *opaque, hwaddr addr,
8372d383
PM
2885 unsigned len, bool is_write,
2886 MemTxAttrs attrs)
c353e4cc 2887{
acc9d80b 2888 subpage_t *subpage = opaque;
c353e4cc 2889#if defined(DEBUG_SUBPAGE)
016e9d62 2890 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
acc9d80b 2891 __func__, subpage, is_write ? 'w' : 'r', len, addr);
c353e4cc
PB
2892#endif
2893
16620684 2894 return flatview_access_valid(subpage->fv, addr + subpage->base,
eace72b7 2895 len, is_write, attrs);
c353e4cc
PB
2896}
2897
70c68e44 2898static const MemoryRegionOps subpage_ops = {
f25a49e0
PM
2899 .read_with_attrs = subpage_read,
2900 .write_with_attrs = subpage_write,
ff6cff75
PB
2901 .impl.min_access_size = 1,
2902 .impl.max_access_size = 8,
2903 .valid.min_access_size = 1,
2904 .valid.max_access_size = 8,
c353e4cc 2905 .valid.accepts = subpage_accepts,
70c68e44 2906 .endianness = DEVICE_NATIVE_ENDIAN,
db7b5426
BS
2907};
2908
c227f099 2909static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
5312bd8b 2910 uint16_t section)
db7b5426
BS
2911{
2912 int idx, eidx;
2913
2914 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2915 return -1;
2916 idx = SUBPAGE_IDX(start);
2917 eidx = SUBPAGE_IDX(end);
2918#if defined(DEBUG_SUBPAGE)
016e9d62
AK
2919 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2920 __func__, mmio, start, end, idx, eidx, section);
db7b5426 2921#endif
db7b5426 2922 for (; idx <= eidx; idx++) {
5312bd8b 2923 mmio->sub_section[idx] = section;
db7b5426
BS
2924 }
2925
2926 return 0;
2927}
2928
16620684 2929static subpage_t *subpage_init(FlatView *fv, hwaddr base)
db7b5426 2930{
c227f099 2931 subpage_t *mmio;
db7b5426 2932
2615fabd 2933 mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
16620684 2934 mmio->fv = fv;
1eec614b 2935 mmio->base = base;
2c9b15ca 2936 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
b4fefef9 2937 NULL, TARGET_PAGE_SIZE);
b3b00c78 2938 mmio->iomem.subpage = true;
db7b5426 2939#if defined(DEBUG_SUBPAGE)
016e9d62
AK
2940 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2941 mmio, base, TARGET_PAGE_SIZE);
db7b5426 2942#endif
b41aac4f 2943 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
db7b5426
BS
2944
2945 return mmio;
2946}
2947
16620684 2948static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
5312bd8b 2949{
16620684 2950 assert(fv);
5312bd8b 2951 MemoryRegionSection section = {
16620684 2952 .fv = fv,
5312bd8b
AK
2953 .mr = mr,
2954 .offset_within_address_space = 0,
2955 .offset_within_region = 0,
052e87b0 2956 .size = int128_2_64(),
5312bd8b
AK
2957 };
2958
53cb28cb 2959 return phys_section_add(map, &section);
5312bd8b
AK
2960}
2961
8af36743
PM
2962static void readonly_mem_write(void *opaque, hwaddr addr,
2963 uint64_t val, unsigned size)
2964{
2965 /* Ignore any write to ROM. */
2966}
2967
2968static bool readonly_mem_accepts(void *opaque, hwaddr addr,
8372d383
PM
2969 unsigned size, bool is_write,
2970 MemTxAttrs attrs)
8af36743
PM
2971{
2972 return is_write;
2973}
2974
2975/* This will only be used for writes, because reads are special cased
2976 * to directly access the underlying host ram.
2977 */
2978static const MemoryRegionOps readonly_mem_ops = {
2979 .write = readonly_mem_write,
2980 .valid.accepts = readonly_mem_accepts,
2981 .endianness = DEVICE_NATIVE_ENDIAN,
2982 .valid = {
2983 .min_access_size = 1,
2984 .max_access_size = 8,
2985 .unaligned = false,
2986 },
2987 .impl = {
2988 .min_access_size = 1,
2989 .max_access_size = 8,
2990 .unaligned = false,
2991 },
2992};
2993
2d54f194
PM
2994MemoryRegionSection *iotlb_to_section(CPUState *cpu,
2995 hwaddr index, MemTxAttrs attrs)
aa102231 2996{
a54c87b6
PM
2997 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2998 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
32857f4d 2999 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
79e2b9ae 3000 MemoryRegionSection *sections = d->map.sections;
9d82b5a7 3001
2d54f194 3002 return &sections[index & ~TARGET_PAGE_MASK];
aa102231
AK
3003}
3004
e9179ce1
AK
3005static void io_mem_init(void)
3006{
8af36743
PM
3007 memory_region_init_io(&io_mem_rom, NULL, &readonly_mem_ops,
3008 NULL, NULL, UINT64_MAX);
2c9b15ca 3009 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1f6245e5 3010 NULL, UINT64_MAX);
8d04fb55
JK
3011
3012 /* io_mem_notdirty calls tb_invalidate_phys_page_fast,
3013 * which can be called without the iothread mutex.
3014 */
2c9b15ca 3015 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1f6245e5 3016 NULL, UINT64_MAX);
8d04fb55
JK
3017 memory_region_clear_global_locking(&io_mem_notdirty);
3018
2c9b15ca 3019 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1f6245e5 3020 NULL, UINT64_MAX);
e9179ce1
AK
3021}
3022
8629d3fc 3023AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
00752703 3024{
53cb28cb
MA
3025 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
3026 uint16_t n;
3027
16620684 3028 n = dummy_section(&d->map, fv, &io_mem_unassigned);
53cb28cb 3029 assert(n == PHYS_SECTION_UNASSIGNED);
16620684 3030 n = dummy_section(&d->map, fv, &io_mem_notdirty);
53cb28cb 3031 assert(n == PHYS_SECTION_NOTDIRTY);
16620684 3032 n = dummy_section(&d->map, fv, &io_mem_rom);
53cb28cb 3033 assert(n == PHYS_SECTION_ROM);
16620684 3034 n = dummy_section(&d->map, fv, &io_mem_watch);
53cb28cb 3035 assert(n == PHYS_SECTION_WATCH);
00752703 3036
9736e55b 3037 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
66a6df1d
AK
3038
3039 return d;
00752703
PB
3040}
3041
66a6df1d 3042void address_space_dispatch_free(AddressSpaceDispatch *d)
79e2b9ae
PB
3043{
3044 phys_sections_free(&d->map);
3045 g_free(d);
3046}
3047
1d71148e 3048static void tcg_commit(MemoryListener *listener)
50c1e149 3049{
32857f4d
PM
3050 CPUAddressSpace *cpuas;
3051 AddressSpaceDispatch *d;
117712c3
AK
3052
3053 /* since each CPU stores ram addresses in its TLB cache, we must
3054 reset the modified entries */
32857f4d
PM
3055 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
3056 cpu_reloading_memory_map();
3057 /* The CPU and TLB are protected by the iothread lock.
3058 * We reload the dispatch pointer now because cpu_reloading_memory_map()
3059 * may have split the RCU critical section.
3060 */
66a6df1d 3061 d = address_space_to_dispatch(cpuas->as);
f35e44e7 3062 atomic_rcu_set(&cpuas->memory_dispatch, d);
d10eb08f 3063 tlb_flush(cpuas->cpu);
50c1e149
AK
3064}
3065
62152b8a
AK
3066static void memory_map_init(void)
3067{
7267c094 3068 system_memory = g_malloc(sizeof(*system_memory));
03f49957 3069
57271d63 3070 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
7dca8043 3071 address_space_init(&address_space_memory, system_memory, "memory");
309cb471 3072
7267c094 3073 system_io = g_malloc(sizeof(*system_io));
3bb28b72
JK
3074 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
3075 65536);
7dca8043 3076 address_space_init(&address_space_io, system_io, "I/O");
62152b8a
AK
3077}
3078
3079MemoryRegion *get_system_memory(void)
3080{
3081 return system_memory;
3082}
3083
309cb471
AK
3084MemoryRegion *get_system_io(void)
3085{
3086 return system_io;
3087}
3088
e2eef170
PB
3089#endif /* !defined(CONFIG_USER_ONLY) */
3090
13eb76e0
FB
3091/* physical memory access (slow version, mainly for debug) */
3092#if defined(CONFIG_USER_ONLY)
f17ec444 3093int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
a68fe89c 3094 uint8_t *buf, int len, int is_write)
13eb76e0
FB
3095{
3096 int l, flags;
3097 target_ulong page;
53a5960a 3098 void * p;
13eb76e0
FB
3099
3100 while (len > 0) {
3101 page = addr & TARGET_PAGE_MASK;
3102 l = (page + TARGET_PAGE_SIZE) - addr;
3103 if (l > len)
3104 l = len;
3105 flags = page_get_flags(page);
3106 if (!(flags & PAGE_VALID))
a68fe89c 3107 return -1;
13eb76e0
FB
3108 if (is_write) {
3109 if (!(flags & PAGE_WRITE))
a68fe89c 3110 return -1;
579a97f7 3111 /* XXX: this code should not depend on lock_user */
72fb7daa 3112 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
a68fe89c 3113 return -1;
72fb7daa
AJ
3114 memcpy(p, buf, l);
3115 unlock_user(p, addr, l);
13eb76e0
FB
3116 } else {
3117 if (!(flags & PAGE_READ))
a68fe89c 3118 return -1;
579a97f7 3119 /* XXX: this code should not depend on lock_user */
72fb7daa 3120 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
a68fe89c 3121 return -1;
72fb7daa 3122 memcpy(buf, p, l);
5b257578 3123 unlock_user(p, addr, 0);
13eb76e0
FB
3124 }
3125 len -= l;
3126 buf += l;
3127 addr += l;
3128 }
a68fe89c 3129 return 0;
13eb76e0 3130}
8df1cd07 3131
13eb76e0 3132#else
51d7a9eb 3133
845b6214 3134static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
a8170e5e 3135 hwaddr length)
51d7a9eb 3136{
e87f7778 3137 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
0878d0e1
PB
3138 addr += memory_region_get_ram_addr(mr);
3139
e87f7778
PB
3140 /* No early return if dirty_log_mask is or becomes 0, because
3141 * cpu_physical_memory_set_dirty_range will still call
3142 * xen_modified_memory.
3143 */
3144 if (dirty_log_mask) {
3145 dirty_log_mask =
3146 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
3147 }
3148 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
5aa1ef71 3149 assert(tcg_enabled());
ba051fb5 3150 tb_lock();
e87f7778 3151 tb_invalidate_phys_range(addr, addr + length);
ba051fb5 3152 tb_unlock();
e87f7778 3153 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
51d7a9eb 3154 }
e87f7778 3155 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
51d7a9eb
AP
3156}
3157
23326164 3158static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
82f2563f 3159{
e1622f4b 3160 unsigned access_size_max = mr->ops->valid.max_access_size;
23326164
RH
3161
3162 /* Regions are assumed to support 1-4 byte accesses unless
3163 otherwise specified. */
23326164
RH
3164 if (access_size_max == 0) {
3165 access_size_max = 4;
3166 }
3167
3168 /* Bound the maximum access by the alignment of the address. */
3169 if (!mr->ops->impl.unaligned) {
3170 unsigned align_size_max = addr & -addr;
3171 if (align_size_max != 0 && align_size_max < access_size_max) {
3172 access_size_max = align_size_max;
3173 }
82f2563f 3174 }
23326164
RH
3175
3176 /* Don't attempt accesses larger than the maximum. */
3177 if (l > access_size_max) {
3178 l = access_size_max;
82f2563f 3179 }
6554f5c0 3180 l = pow2floor(l);
23326164
RH
3181
3182 return l;
82f2563f
PB
3183}
3184
4840f10e 3185static bool prepare_mmio_access(MemoryRegion *mr)
125b3806 3186{
4840f10e
JK
3187 bool unlocked = !qemu_mutex_iothread_locked();
3188 bool release_lock = false;
3189
3190 if (unlocked && mr->global_locking) {
3191 qemu_mutex_lock_iothread();
3192 unlocked = false;
3193 release_lock = true;
3194 }
125b3806 3195 if (mr->flush_coalesced_mmio) {
4840f10e
JK
3196 if (unlocked) {
3197 qemu_mutex_lock_iothread();
3198 }
125b3806 3199 qemu_flush_coalesced_mmio_buffer();
4840f10e
JK
3200 if (unlocked) {
3201 qemu_mutex_unlock_iothread();
3202 }
125b3806 3203 }
4840f10e
JK
3204
3205 return release_lock;
125b3806
PB
3206}
3207
a203ac70 3208/* Called within RCU critical section. */
16620684
AK
3209static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
3210 MemTxAttrs attrs,
3211 const uint8_t *buf,
3212 int len, hwaddr addr1,
3213 hwaddr l, MemoryRegion *mr)
13eb76e0 3214{
13eb76e0 3215 uint8_t *ptr;
791af8c8 3216 uint64_t val;
3b643495 3217 MemTxResult result = MEMTX_OK;
4840f10e 3218 bool release_lock = false;
3b46e624 3219
a203ac70 3220 for (;;) {
eb7eeb88
PB
3221 if (!memory_access_is_direct(mr, true)) {
3222 release_lock |= prepare_mmio_access(mr);
3223 l = memory_access_size(mr, l, addr1);
3224 /* XXX: could force current_cpu to NULL to avoid
3225 potential bugs */
6d3ede54
PM
3226 val = ldn_p(buf, l);
3227 result |= memory_region_dispatch_write(mr, addr1, val, l, attrs);
13eb76e0 3228 } else {
eb7eeb88 3229 /* RAM case */
f5aa69bd 3230 ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
eb7eeb88
PB
3231 memcpy(ptr, buf, l);
3232 invalidate_and_set_dirty(mr, addr1, l);
13eb76e0 3233 }
4840f10e
JK
3234
3235 if (release_lock) {
3236 qemu_mutex_unlock_iothread();
3237 release_lock = false;
3238 }
3239
13eb76e0
FB
3240 len -= l;
3241 buf += l;
3242 addr += l;
a203ac70
PB
3243
3244 if (!len) {
3245 break;
3246 }
3247
3248 l = len;
efa99a2f 3249 mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
13eb76e0 3250 }
fd8aaa76 3251
3b643495 3252 return result;
13eb76e0 3253}
8df1cd07 3254
4c6ebbb3 3255/* Called from RCU critical section. */
16620684
AK
3256static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
3257 const uint8_t *buf, int len)
ac1970fb 3258{
eb7eeb88 3259 hwaddr l;
eb7eeb88
PB
3260 hwaddr addr1;
3261 MemoryRegion *mr;
3262 MemTxResult result = MEMTX_OK;
eb7eeb88 3263
4c6ebbb3 3264 l = len;
efa99a2f 3265 mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
4c6ebbb3
PB
3266 result = flatview_write_continue(fv, addr, attrs, buf, len,
3267 addr1, l, mr);
a203ac70
PB
3268
3269 return result;
3270}
3271
3272/* Called within RCU critical section. */
16620684
AK
3273MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
3274 MemTxAttrs attrs, uint8_t *buf,
3275 int len, hwaddr addr1, hwaddr l,
3276 MemoryRegion *mr)
a203ac70
PB
3277{
3278 uint8_t *ptr;
3279 uint64_t val;
3280 MemTxResult result = MEMTX_OK;
3281 bool release_lock = false;
eb7eeb88 3282
a203ac70 3283 for (;;) {
eb7eeb88
PB
3284 if (!memory_access_is_direct(mr, false)) {
3285 /* I/O case */
3286 release_lock |= prepare_mmio_access(mr);
3287 l = memory_access_size(mr, l, addr1);
6d3ede54
PM
3288 result |= memory_region_dispatch_read(mr, addr1, &val, l, attrs);
3289 stn_p(buf, l, val);
eb7eeb88
PB
3290 } else {
3291 /* RAM case */
f5aa69bd 3292 ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
eb7eeb88
PB
3293 memcpy(buf, ptr, l);
3294 }
3295
3296 if (release_lock) {
3297 qemu_mutex_unlock_iothread();
3298 release_lock = false;
3299 }
3300
3301 len -= l;
3302 buf += l;
3303 addr += l;
a203ac70
PB
3304
3305 if (!len) {
3306 break;
3307 }
3308
3309 l = len;
efa99a2f 3310 mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
a203ac70
PB
3311 }
3312
3313 return result;
3314}
3315
b2a44fca
PB
3316/* Called from RCU critical section. */
3317static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
3318 MemTxAttrs attrs, uint8_t *buf, int len)
a203ac70
PB
3319{
3320 hwaddr l;
3321 hwaddr addr1;
3322 MemoryRegion *mr;
eb7eeb88 3323
b2a44fca 3324 l = len;
efa99a2f 3325 mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
b2a44fca
PB
3326 return flatview_read_continue(fv, addr, attrs, buf, len,
3327 addr1, l, mr);
ac1970fb
AK
3328}
3329
b2a44fca
PB
3330MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
3331 MemTxAttrs attrs, uint8_t *buf, int len)
3332{
3333 MemTxResult result = MEMTX_OK;
3334 FlatView *fv;
3335
3336 if (len > 0) {
3337 rcu_read_lock();
3338 fv = address_space_to_flatview(as);
3339 result = flatview_read(fv, addr, attrs, buf, len);
3340 rcu_read_unlock();
3341 }
3342
3343 return result;
3344}
3345
4c6ebbb3
PB
3346MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
3347 MemTxAttrs attrs,
3348 const uint8_t *buf, int len)
3349{
3350 MemTxResult result = MEMTX_OK;
3351 FlatView *fv;
3352
3353 if (len > 0) {
3354 rcu_read_lock();
3355 fv = address_space_to_flatview(as);
3356 result = flatview_write(fv, addr, attrs, buf, len);
3357 rcu_read_unlock();
3358 }
3359
3360 return result;
3361}
3362
db84fd97
PB
3363MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
3364 uint8_t *buf, int len, bool is_write)
3365{
3366 if (is_write) {
3367 return address_space_write(as, addr, attrs, buf, len);
3368 } else {
3369 return address_space_read_full(as, addr, attrs, buf, len);
3370 }
3371}
3372
a8170e5e 3373void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
ac1970fb
AK
3374 int len, int is_write)
3375{
5c9eb028
PM
3376 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
3377 buf, len, is_write);
ac1970fb
AK
3378}
3379
582b55a9
AG
3380enum write_rom_type {
3381 WRITE_DATA,
3382 FLUSH_CACHE,
3383};
3384
2a221651 3385static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
582b55a9 3386 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
d0ecd2aa 3387{
149f54b5 3388 hwaddr l;
d0ecd2aa 3389 uint8_t *ptr;
149f54b5 3390 hwaddr addr1;
5c8a00ce 3391 MemoryRegion *mr;
3b46e624 3392
41063e1e 3393 rcu_read_lock();
d0ecd2aa 3394 while (len > 0) {
149f54b5 3395 l = len;
bc6b1cec
PM
3396 mr = address_space_translate(as, addr, &addr1, &l, true,
3397 MEMTXATTRS_UNSPECIFIED);
3b46e624 3398
5c8a00ce
PB
3399 if (!(memory_region_is_ram(mr) ||
3400 memory_region_is_romd(mr))) {
b242e0e0 3401 l = memory_access_size(mr, l, addr1);
d0ecd2aa 3402 } else {
d0ecd2aa 3403 /* ROM/RAM case */
0878d0e1 3404 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
582b55a9
AG
3405 switch (type) {
3406 case WRITE_DATA:
3407 memcpy(ptr, buf, l);
845b6214 3408 invalidate_and_set_dirty(mr, addr1, l);
582b55a9
AG
3409 break;
3410 case FLUSH_CACHE:
3411 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
3412 break;
3413 }
d0ecd2aa
FB
3414 }
3415 len -= l;
3416 buf += l;
3417 addr += l;
3418 }
41063e1e 3419 rcu_read_unlock();
d0ecd2aa
FB
3420}
3421
582b55a9 3422/* used for ROM loading : can write in RAM and ROM */
2a221651 3423void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
582b55a9
AG
3424 const uint8_t *buf, int len)
3425{
2a221651 3426 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
582b55a9
AG
3427}
3428
3429void cpu_flush_icache_range(hwaddr start, int len)
3430{
3431 /*
3432 * This function should do the same thing as an icache flush that was
3433 * triggered from within the guest. For TCG we are always cache coherent,
3434 * so there is no need to flush anything. For KVM / Xen we need to flush
3435 * the host's instruction cache at least.
3436 */
3437 if (tcg_enabled()) {
3438 return;
3439 }
3440
2a221651
EI
3441 cpu_physical_memory_write_rom_internal(&address_space_memory,
3442 start, NULL, len, FLUSH_CACHE);
582b55a9
AG
3443}
3444
6d16c2f8 3445typedef struct {
d3e71559 3446 MemoryRegion *mr;
6d16c2f8 3447 void *buffer;
a8170e5e
AK
3448 hwaddr addr;
3449 hwaddr len;
c2cba0ff 3450 bool in_use;
6d16c2f8
AL
3451} BounceBuffer;
3452
3453static BounceBuffer bounce;
3454
ba223c29 3455typedef struct MapClient {
e95205e1 3456 QEMUBH *bh;
72cf2d4f 3457 QLIST_ENTRY(MapClient) link;
ba223c29
AL
3458} MapClient;
3459
38e047b5 3460QemuMutex map_client_list_lock;
72cf2d4f
BS
3461static QLIST_HEAD(map_client_list, MapClient) map_client_list
3462 = QLIST_HEAD_INITIALIZER(map_client_list);
ba223c29 3463
e95205e1
FZ
3464static void cpu_unregister_map_client_do(MapClient *client)
3465{
3466 QLIST_REMOVE(client, link);
3467 g_free(client);
3468}
3469
33b6c2ed
FZ
3470static void cpu_notify_map_clients_locked(void)
3471{
3472 MapClient *client;
3473
3474 while (!QLIST_EMPTY(&map_client_list)) {
3475 client = QLIST_FIRST(&map_client_list);
e95205e1
FZ
3476 qemu_bh_schedule(client->bh);
3477 cpu_unregister_map_client_do(client);
33b6c2ed
FZ
3478 }
3479}
3480
e95205e1 3481void cpu_register_map_client(QEMUBH *bh)
ba223c29 3482{
7267c094 3483 MapClient *client = g_malloc(sizeof(*client));
ba223c29 3484
38e047b5 3485 qemu_mutex_lock(&map_client_list_lock);
e95205e1 3486 client->bh = bh;
72cf2d4f 3487 QLIST_INSERT_HEAD(&map_client_list, client, link);
33b6c2ed
FZ
3488 if (!atomic_read(&bounce.in_use)) {
3489 cpu_notify_map_clients_locked();
3490 }
38e047b5 3491 qemu_mutex_unlock(&map_client_list_lock);
ba223c29
AL
3492}
3493
38e047b5 3494void cpu_exec_init_all(void)
ba223c29 3495{
38e047b5 3496 qemu_mutex_init(&ram_list.mutex);
20bccb82
PM
3497 /* The data structures we set up here depend on knowing the page size,
3498 * so no more changes can be made after this point.
3499 * In an ideal world, nothing we did before we had finished the
3500 * machine setup would care about the target page size, and we could
3501 * do this much later, rather than requiring board models to state
3502 * up front what their requirements are.
3503 */
3504 finalize_target_page_bits();
38e047b5 3505 io_mem_init();
680a4783 3506 memory_map_init();
38e047b5 3507 qemu_mutex_init(&map_client_list_lock);
ba223c29
AL
3508}
3509
e95205e1 3510void cpu_unregister_map_client(QEMUBH *bh)
ba223c29
AL
3511{
3512 MapClient *client;
3513
e95205e1
FZ
3514 qemu_mutex_lock(&map_client_list_lock);
3515 QLIST_FOREACH(client, &map_client_list, link) {
3516 if (client->bh == bh) {
3517 cpu_unregister_map_client_do(client);
3518 break;
3519 }
ba223c29 3520 }
e95205e1 3521 qemu_mutex_unlock(&map_client_list_lock);
ba223c29
AL
3522}
3523
3524static void cpu_notify_map_clients(void)
3525{
38e047b5 3526 qemu_mutex_lock(&map_client_list_lock);
33b6c2ed 3527 cpu_notify_map_clients_locked();
38e047b5 3528 qemu_mutex_unlock(&map_client_list_lock);
ba223c29
AL
3529}
3530
16620684 3531static bool flatview_access_valid(FlatView *fv, hwaddr addr, int len,
eace72b7 3532 bool is_write, MemTxAttrs attrs)
51644ab7 3533{
5c8a00ce 3534 MemoryRegion *mr;
51644ab7
PB
3535 hwaddr l, xlat;
3536
3537 while (len > 0) {
3538 l = len;
efa99a2f 3539 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
5c8a00ce
PB
3540 if (!memory_access_is_direct(mr, is_write)) {
3541 l = memory_access_size(mr, l, addr);
eace72b7 3542 if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
51644ab7
PB
3543 return false;
3544 }
3545 }
3546
3547 len -= l;
3548 addr += l;
3549 }
3550 return true;
3551}
3552
16620684 3553bool address_space_access_valid(AddressSpace *as, hwaddr addr,
fddffa42
PM
3554 int len, bool is_write,
3555 MemTxAttrs attrs)
16620684 3556{
11e732a5
PB
3557 FlatView *fv;
3558 bool result;
3559
3560 rcu_read_lock();
3561 fv = address_space_to_flatview(as);
eace72b7 3562 result = flatview_access_valid(fv, addr, len, is_write, attrs);
11e732a5
PB
3563 rcu_read_unlock();
3564 return result;
16620684
AK
3565}
3566
715c31ec 3567static hwaddr
16620684 3568flatview_extend_translation(FlatView *fv, hwaddr addr,
53d0790d
PM
3569 hwaddr target_len,
3570 MemoryRegion *mr, hwaddr base, hwaddr len,
3571 bool is_write, MemTxAttrs attrs)
715c31ec
PB
3572{
3573 hwaddr done = 0;
3574 hwaddr xlat;
3575 MemoryRegion *this_mr;
3576
3577 for (;;) {
3578 target_len -= len;
3579 addr += len;
3580 done += len;
3581 if (target_len == 0) {
3582 return done;
3583 }
3584
3585 len = target_len;
16620684 3586 this_mr = flatview_translate(fv, addr, &xlat,
efa99a2f 3587 &len, is_write, attrs);
715c31ec
PB
3588 if (this_mr != mr || xlat != base + done) {
3589 return done;
3590 }
3591 }
3592}
3593
6d16c2f8
AL
3594/* Map a physical memory region into a host virtual address.
3595 * May map a subset of the requested range, given by and returned in *plen.
3596 * May return NULL if resources needed to perform the mapping are exhausted.
3597 * Use only for reads OR writes - not for read-modify-write operations.
ba223c29
AL
3598 * Use cpu_register_map_client() to know when retrying the map operation is
3599 * likely to succeed.
6d16c2f8 3600 */
ac1970fb 3601void *address_space_map(AddressSpace *as,
a8170e5e
AK
3602 hwaddr addr,
3603 hwaddr *plen,
f26404fb
PM
3604 bool is_write,
3605 MemTxAttrs attrs)
6d16c2f8 3606{
a8170e5e 3607 hwaddr len = *plen;
715c31ec
PB
3608 hwaddr l, xlat;
3609 MemoryRegion *mr;
e81bcda5 3610 void *ptr;
ad0c60fa 3611 FlatView *fv;
6d16c2f8 3612
e3127ae0
PB
3613 if (len == 0) {
3614 return NULL;
3615 }
38bee5dc 3616
e3127ae0 3617 l = len;
41063e1e 3618 rcu_read_lock();
ad0c60fa 3619 fv = address_space_to_flatview(as);
efa99a2f 3620 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
41063e1e 3621
e3127ae0 3622 if (!memory_access_is_direct(mr, is_write)) {
c2cba0ff 3623 if (atomic_xchg(&bounce.in_use, true)) {
41063e1e 3624 rcu_read_unlock();
e3127ae0 3625 return NULL;
6d16c2f8 3626 }
e85d9db5
KW
3627 /* Avoid unbounded allocations */
3628 l = MIN(l, TARGET_PAGE_SIZE);
3629 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
e3127ae0
PB
3630 bounce.addr = addr;
3631 bounce.len = l;
d3e71559
PB
3632
3633 memory_region_ref(mr);
3634 bounce.mr = mr;
e3127ae0 3635 if (!is_write) {
16620684 3636 flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED,
5c9eb028 3637 bounce.buffer, l);
8ab934f9 3638 }
6d16c2f8 3639
41063e1e 3640 rcu_read_unlock();
e3127ae0
PB
3641 *plen = l;
3642 return bounce.buffer;
3643 }
3644
e3127ae0 3645
d3e71559 3646 memory_region_ref(mr);
16620684 3647 *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
53d0790d 3648 l, is_write, attrs);
f5aa69bd 3649 ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen, true);
e81bcda5
PB
3650 rcu_read_unlock();
3651
3652 return ptr;
6d16c2f8
AL
3653}
3654
ac1970fb 3655/* Unmaps a memory region previously mapped by address_space_map().
6d16c2f8
AL
3656 * Will also mark the memory as dirty if is_write == 1. access_len gives
3657 * the amount of memory that was actually read or written by the caller.
3658 */
a8170e5e
AK
3659void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
3660 int is_write, hwaddr access_len)
6d16c2f8
AL
3661{
3662 if (buffer != bounce.buffer) {
d3e71559
PB
3663 MemoryRegion *mr;
3664 ram_addr_t addr1;
3665
07bdaa41 3666 mr = memory_region_from_host(buffer, &addr1);
d3e71559 3667 assert(mr != NULL);
6d16c2f8 3668 if (is_write) {
845b6214 3669 invalidate_and_set_dirty(mr, addr1, access_len);
6d16c2f8 3670 }
868bb33f 3671 if (xen_enabled()) {
e41d7c69 3672 xen_invalidate_map_cache_entry(buffer);
050a0ddf 3673 }
d3e71559 3674 memory_region_unref(mr);
6d16c2f8
AL
3675 return;
3676 }
3677 if (is_write) {
5c9eb028
PM
3678 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
3679 bounce.buffer, access_len);
6d16c2f8 3680 }
f8a83245 3681 qemu_vfree(bounce.buffer);
6d16c2f8 3682 bounce.buffer = NULL;
d3e71559 3683 memory_region_unref(bounce.mr);
c2cba0ff 3684 atomic_mb_set(&bounce.in_use, false);
ba223c29 3685 cpu_notify_map_clients();
6d16c2f8 3686}
d0ecd2aa 3687
a8170e5e
AK
3688void *cpu_physical_memory_map(hwaddr addr,
3689 hwaddr *plen,
ac1970fb
AK
3690 int is_write)
3691{
f26404fb
PM
3692 return address_space_map(&address_space_memory, addr, plen, is_write,
3693 MEMTXATTRS_UNSPECIFIED);
ac1970fb
AK
3694}
3695
a8170e5e
AK
3696void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3697 int is_write, hwaddr access_len)
ac1970fb
AK
3698{
3699 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3700}
3701
0ce265ff
PB
3702#define ARG1_DECL AddressSpace *as
3703#define ARG1 as
3704#define SUFFIX
3705#define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
3706#define IS_DIRECT(mr, is_write) memory_access_is_direct(mr, is_write)
3707#define MAP_RAM(mr, ofs) qemu_map_ram_ptr((mr)->ram_block, ofs)
3708#define INVALIDATE(mr, ofs, len) invalidate_and_set_dirty(mr, ofs, len)
3709#define RCU_READ_LOCK(...) rcu_read_lock()
3710#define RCU_READ_UNLOCK(...) rcu_read_unlock()
3711#include "memory_ldst.inc.c"
1e78bcc1 3712
1f4e496e
PB
3713int64_t address_space_cache_init(MemoryRegionCache *cache,
3714 AddressSpace *as,
3715 hwaddr addr,
3716 hwaddr len,
3717 bool is_write)
3718{
48564041
PB
3719 AddressSpaceDispatch *d;
3720 hwaddr l;
3721 MemoryRegion *mr;
3722
3723 assert(len > 0);
3724
3725 l = len;
3726 cache->fv = address_space_get_flatview(as);
3727 d = flatview_to_dispatch(cache->fv);
3728 cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
3729
3730 mr = cache->mrs.mr;
3731 memory_region_ref(mr);
3732 if (memory_access_is_direct(mr, is_write)) {
53d0790d
PM
3733 /* We don't care about the memory attributes here as we're only
3734 * doing this if we found actual RAM, which behaves the same
3735 * regardless of attributes; so UNSPECIFIED is fine.
3736 */
48564041 3737 l = flatview_extend_translation(cache->fv, addr, len, mr,
53d0790d
PM
3738 cache->xlat, l, is_write,
3739 MEMTXATTRS_UNSPECIFIED);
48564041
PB
3740 cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true);
3741 } else {
3742 cache->ptr = NULL;
3743 }
3744
3745 cache->len = l;
3746 cache->is_write = is_write;
3747 return l;
1f4e496e
PB
3748}
3749
3750void address_space_cache_invalidate(MemoryRegionCache *cache,
3751 hwaddr addr,
3752 hwaddr access_len)
3753{
48564041
PB
3754 assert(cache->is_write);
3755 if (likely(cache->ptr)) {
3756 invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
3757 }
1f4e496e
PB
3758}
3759
3760void address_space_cache_destroy(MemoryRegionCache *cache)
3761{
48564041
PB
3762 if (!cache->mrs.mr) {
3763 return;
3764 }
3765
3766 if (xen_enabled()) {
3767 xen_invalidate_map_cache_entry(cache->ptr);
3768 }
3769 memory_region_unref(cache->mrs.mr);
3770 flatview_unref(cache->fv);
3771 cache->mrs.mr = NULL;
3772 cache->fv = NULL;
3773}
3774
3775/* Called from RCU critical section. This function has the same
3776 * semantics as address_space_translate, but it only works on a
3777 * predefined range of a MemoryRegion that was mapped with
3778 * address_space_cache_init.
3779 */
3780static inline MemoryRegion *address_space_translate_cached(
3781 MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
bc6b1cec 3782 hwaddr *plen, bool is_write, MemTxAttrs attrs)
48564041
PB
3783{
3784 MemoryRegionSection section;
3785 MemoryRegion *mr;
3786 IOMMUMemoryRegion *iommu_mr;
3787 AddressSpace *target_as;
3788
3789 assert(!cache->ptr);
3790 *xlat = addr + cache->xlat;
3791
3792 mr = cache->mrs.mr;
3793 iommu_mr = memory_region_get_iommu(mr);
3794 if (!iommu_mr) {
3795 /* MMIO region. */
3796 return mr;
3797 }
3798
3799 section = address_space_translate_iommu(iommu_mr, xlat, plen,
3800 NULL, is_write, true,
2f7b009c 3801 &target_as, attrs);
48564041
PB
3802 return section.mr;
3803}
3804
3805/* Called from RCU critical section. address_space_read_cached uses this
3806 * out of line function when the target is an MMIO or IOMMU region.
3807 */
3808void
3809address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3810 void *buf, int len)
3811{
3812 hwaddr addr1, l;
3813 MemoryRegion *mr;
3814
3815 l = len;
bc6b1cec
PM
3816 mr = address_space_translate_cached(cache, addr, &addr1, &l, false,
3817 MEMTXATTRS_UNSPECIFIED);
48564041
PB
3818 flatview_read_continue(cache->fv,
3819 addr, MEMTXATTRS_UNSPECIFIED, buf, len,
3820 addr1, l, mr);
3821}
3822
3823/* Called from RCU critical section. address_space_write_cached uses this
3824 * out of line function when the target is an MMIO or IOMMU region.
3825 */
3826void
3827address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3828 const void *buf, int len)
3829{
3830 hwaddr addr1, l;
3831 MemoryRegion *mr;
3832
3833 l = len;
bc6b1cec
PM
3834 mr = address_space_translate_cached(cache, addr, &addr1, &l, true,
3835 MEMTXATTRS_UNSPECIFIED);
48564041
PB
3836 flatview_write_continue(cache->fv,
3837 addr, MEMTXATTRS_UNSPECIFIED, buf, len,
3838 addr1, l, mr);
1f4e496e
PB
3839}
3840
3841#define ARG1_DECL MemoryRegionCache *cache
3842#define ARG1 cache
48564041
PB
3843#define SUFFIX _cached_slow
3844#define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
3845#define IS_DIRECT(mr, is_write) memory_access_is_direct(mr, is_write)
3846#define MAP_RAM(mr, ofs) (cache->ptr + (ofs - cache->xlat))
90c4fe5f 3847#define INVALIDATE(mr, ofs, len) invalidate_and_set_dirty(mr, ofs, len)
48564041
PB
3848#define RCU_READ_LOCK() ((void)0)
3849#define RCU_READ_UNLOCK() ((void)0)
1f4e496e
PB
3850#include "memory_ldst.inc.c"
3851
5e2972fd 3852/* virtual memory access for debug (includes writing to ROM) */
f17ec444 3853int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
b448f2f3 3854 uint8_t *buf, int len, int is_write)
13eb76e0
FB
3855{
3856 int l;
a8170e5e 3857 hwaddr phys_addr;
9b3c35e0 3858 target_ulong page;
13eb76e0 3859
79ca7a1b 3860 cpu_synchronize_state(cpu);
13eb76e0 3861 while (len > 0) {
5232e4c7
PM
3862 int asidx;
3863 MemTxAttrs attrs;
3864
13eb76e0 3865 page = addr & TARGET_PAGE_MASK;
5232e4c7
PM
3866 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3867 asidx = cpu_asidx_from_attrs(cpu, attrs);
13eb76e0
FB
3868 /* if no physical page mapped, return an error */
3869 if (phys_addr == -1)
3870 return -1;
3871 l = (page + TARGET_PAGE_SIZE) - addr;
3872 if (l > len)
3873 l = len;
5e2972fd 3874 phys_addr += (addr & ~TARGET_PAGE_MASK);
2e38847b 3875 if (is_write) {
5232e4c7
PM
3876 cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
3877 phys_addr, buf, l);
2e38847b 3878 } else {
5232e4c7
PM
3879 address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
3880 MEMTXATTRS_UNSPECIFIED,
5c9eb028 3881 buf, l, 0);
2e38847b 3882 }
13eb76e0
FB
3883 len -= l;
3884 buf += l;
3885 addr += l;
3886 }
3887 return 0;
3888}
038629a6
DDAG
3889
3890/*
3891 * Allows code that needs to deal with migration bitmaps etc to still be built
3892 * target independent.
3893 */
20afaed9 3894size_t qemu_target_page_size(void)
038629a6 3895{
20afaed9 3896 return TARGET_PAGE_SIZE;
038629a6
DDAG
3897}
3898
46d702b1
JQ
3899int qemu_target_page_bits(void)
3900{
3901 return TARGET_PAGE_BITS;
3902}
3903
3904int qemu_target_page_bits_min(void)
3905{
3906 return TARGET_PAGE_BITS_MIN;
3907}
a68fe89c 3908#endif
13eb76e0 3909
8e4a424b
BS
3910/*
3911 * A helper function for the _utterly broken_ virtio device model to find out if
3912 * it's running on a big endian machine. Don't do this at home kids!
3913 */
98ed8ecf
GK
3914bool target_words_bigendian(void);
3915bool target_words_bigendian(void)
8e4a424b
BS
3916{
3917#if defined(TARGET_WORDS_BIGENDIAN)
3918 return true;
3919#else
3920 return false;
3921#endif
3922}
3923
76f35538 3924#ifndef CONFIG_USER_ONLY
a8170e5e 3925bool cpu_physical_memory_is_io(hwaddr phys_addr)
76f35538 3926{
5c8a00ce 3927 MemoryRegion*mr;
149f54b5 3928 hwaddr l = 1;
41063e1e 3929 bool res;
76f35538 3930
41063e1e 3931 rcu_read_lock();
5c8a00ce 3932 mr = address_space_translate(&address_space_memory,
bc6b1cec
PM
3933 phys_addr, &phys_addr, &l, false,
3934 MEMTXATTRS_UNSPECIFIED);
76f35538 3935
41063e1e
PB
3936 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3937 rcu_read_unlock();
3938 return res;
76f35538 3939}
bd2fa51f 3940
e3807054 3941int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
bd2fa51f
MH
3942{
3943 RAMBlock *block;
e3807054 3944 int ret = 0;
bd2fa51f 3945
0dc3f44a 3946 rcu_read_lock();
99e15582 3947 RAMBLOCK_FOREACH(block) {
e3807054
DDAG
3948 ret = func(block->idstr, block->host, block->offset,
3949 block->used_length, opaque);
3950 if (ret) {
3951 break;
3952 }
bd2fa51f 3953 }
0dc3f44a 3954 rcu_read_unlock();
e3807054 3955 return ret;
bd2fa51f 3956}
d3a5038c 3957
b895de50
CLG
3958int qemu_ram_foreach_migratable_block(RAMBlockIterFunc func, void *opaque)
3959{
3960 RAMBlock *block;
3961 int ret = 0;
3962
3963 rcu_read_lock();
3964 RAMBLOCK_FOREACH(block) {
3965 if (!qemu_ram_is_migratable(block)) {
3966 continue;
3967 }
3968 ret = func(block->idstr, block->host, block->offset,
3969 block->used_length, opaque);
3970 if (ret) {
3971 break;
3972 }
3973 }
3974 rcu_read_unlock();
3975 return ret;
3976}
3977
d3a5038c
DDAG
3978/*
3979 * Unmap pages of memory from start to start+length such that
3980 * they a) read as 0, b) Trigger whatever fault mechanism
3981 * the OS provides for postcopy.
3982 * The pages must be unmapped by the end of the function.
3983 * Returns: 0 on success, none-0 on failure
3984 *
3985 */
3986int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
3987{
3988 int ret = -1;
3989
3990 uint8_t *host_startaddr = rb->host + start;
3991
3992 if ((uintptr_t)host_startaddr & (rb->page_size - 1)) {
3993 error_report("ram_block_discard_range: Unaligned start address: %p",
3994 host_startaddr);
3995 goto err;
3996 }
3997
3998 if ((start + length) <= rb->used_length) {
db144f70 3999 bool need_madvise, need_fallocate;
d3a5038c
DDAG
4000 uint8_t *host_endaddr = host_startaddr + length;
4001 if ((uintptr_t)host_endaddr & (rb->page_size - 1)) {
4002 error_report("ram_block_discard_range: Unaligned end address: %p",
4003 host_endaddr);
4004 goto err;
4005 }
4006
4007 errno = ENOTSUP; /* If we are missing MADVISE etc */
4008
db144f70
DDAG
4009 /* The logic here is messy;
4010 * madvise DONTNEED fails for hugepages
4011 * fallocate works on hugepages and shmem
4012 */
4013 need_madvise = (rb->page_size == qemu_host_page_size);
4014 need_fallocate = rb->fd != -1;
4015 if (need_fallocate) {
4016 /* For a file, this causes the area of the file to be zero'd
4017 * if read, and for hugetlbfs also causes it to be unmapped
4018 * so a userfault will trigger.
e2fa71f5
DDAG
4019 */
4020#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
4021 ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
4022 start, length);
db144f70
DDAG
4023 if (ret) {
4024 ret = -errno;
4025 error_report("ram_block_discard_range: Failed to fallocate "
4026 "%s:%" PRIx64 " +%zx (%d)",
4027 rb->idstr, start, length, ret);
4028 goto err;
4029 }
4030#else
4031 ret = -ENOSYS;
4032 error_report("ram_block_discard_range: fallocate not available/file"
4033 "%s:%" PRIx64 " +%zx (%d)",
4034 rb->idstr, start, length, ret);
4035 goto err;
e2fa71f5
DDAG
4036#endif
4037 }
db144f70
DDAG
4038 if (need_madvise) {
4039 /* For normal RAM this causes it to be unmapped,
4040 * for shared memory it causes the local mapping to disappear
4041 * and to fall back on the file contents (which we just
4042 * fallocate'd away).
4043 */
4044#if defined(CONFIG_MADVISE)
4045 ret = madvise(host_startaddr, length, MADV_DONTNEED);
4046 if (ret) {
4047 ret = -errno;
4048 error_report("ram_block_discard_range: Failed to discard range "
4049 "%s:%" PRIx64 " +%zx (%d)",
4050 rb->idstr, start, length, ret);
4051 goto err;
4052 }
4053#else
4054 ret = -ENOSYS;
4055 error_report("ram_block_discard_range: MADVISE not available"
d3a5038c
DDAG
4056 "%s:%" PRIx64 " +%zx (%d)",
4057 rb->idstr, start, length, ret);
db144f70
DDAG
4058 goto err;
4059#endif
d3a5038c 4060 }
db144f70
DDAG
4061 trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
4062 need_madvise, need_fallocate, ret);
d3a5038c
DDAG
4063 } else {
4064 error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64
4065 "/%zx/" RAM_ADDR_FMT")",
4066 rb->idstr, start, length, rb->used_length);
4067 }
4068
4069err:
4070 return ret;
4071}
4072
ec3f8c99 4073#endif
a0be0c58
YZ
4074
4075void page_size_init(void)
4076{
4077 /* NOTE: we can always suppose that qemu_host_page_size >=
4078 TARGET_PAGE_SIZE */
a0be0c58
YZ
4079 if (qemu_host_page_size == 0) {
4080 qemu_host_page_size = qemu_real_host_page_size;
4081 }
4082 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
4083 qemu_host_page_size = TARGET_PAGE_SIZE;
4084 }
4085 qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
4086}
5e8fd947
AK
4087
4088#if !defined(CONFIG_USER_ONLY)
4089
4090static void mtree_print_phys_entries(fprintf_function mon, void *f,
4091 int start, int end, int skip, int ptr)
4092{
4093 if (start == end - 1) {
4094 mon(f, "\t%3d ", start);
4095 } else {
4096 mon(f, "\t%3d..%-3d ", start, end - 1);
4097 }
4098 mon(f, " skip=%d ", skip);
4099 if (ptr == PHYS_MAP_NODE_NIL) {
4100 mon(f, " ptr=NIL");
4101 } else if (!skip) {
4102 mon(f, " ptr=#%d", ptr);
4103 } else {
4104 mon(f, " ptr=[%d]", ptr);
4105 }
4106 mon(f, "\n");
4107}
4108
4109#define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
4110 int128_sub((size), int128_one())) : 0)
4111
4112void mtree_print_dispatch(fprintf_function mon, void *f,
4113 AddressSpaceDispatch *d, MemoryRegion *root)
4114{
4115 int i;
4116
4117 mon(f, " Dispatch\n");
4118 mon(f, " Physical sections\n");
4119
4120 for (i = 0; i < d->map.sections_nb; ++i) {
4121 MemoryRegionSection *s = d->map.sections + i;
4122 const char *names[] = { " [unassigned]", " [not dirty]",
4123 " [ROM]", " [watch]" };
4124
4125 mon(f, " #%d @" TARGET_FMT_plx ".." TARGET_FMT_plx " %s%s%s%s%s",
4126 i,
4127 s->offset_within_address_space,
4128 s->offset_within_address_space + MR_SIZE(s->mr->size),
4129 s->mr->name ? s->mr->name : "(noname)",
4130 i < ARRAY_SIZE(names) ? names[i] : "",
4131 s->mr == root ? " [ROOT]" : "",
4132 s == d->mru_section ? " [MRU]" : "",
4133 s->mr->is_iommu ? " [iommu]" : "");
4134
4135 if (s->mr->alias) {
4136 mon(f, " alias=%s", s->mr->alias->name ?
4137 s->mr->alias->name : "noname");
4138 }
4139 mon(f, "\n");
4140 }
4141
4142 mon(f, " Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
4143 P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
4144 for (i = 0; i < d->map.nodes_nb; ++i) {
4145 int j, jprev;
4146 PhysPageEntry prev;
4147 Node *n = d->map.nodes + i;
4148
4149 mon(f, " [%d]\n", i);
4150
4151 for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
4152 PhysPageEntry *pe = *n + j;
4153
4154 if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
4155 continue;
4156 }
4157
4158 mtree_print_phys_entries(mon, f, jprev, j, prev.skip, prev.ptr);
4159
4160 jprev = j;
4161 prev = *pe;
4162 }
4163
4164 if (jprev != ARRAY_SIZE(*n)) {
4165 mtree_print_phys_entries(mon, f, jprev, j, prev.skip, prev.ptr);
4166 }
4167 }
4168}
4169
4170#endif