include/exec/ram_addr.h

   1 /*
   2  * Declarations for cpu physical memory functions
   3  *
   4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
   5  *
   6  * Authors:
   7  *  Avi Kivity <avi@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * later.  See the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 /*
  15  * This header is for use by exec.c and memory.c ONLY.  Do not include it.
  16  * The functions declared here will be removed soon.
  17  */
  18
  19 #ifndef RAM_ADDR_H
  20 #define RAM_ADDR_H
  21
  22 #ifndef CONFIG_USER_ONLY
  23 #include "hw/xen/xen.h"
  24
  25 struct RAMBlock {
  26     struct rcu_head rcu;
  27     struct MemoryRegion *mr;
  28     uint8_t *host;
  29     ram_addr_t offset;
  30     ram_addr_t used_length;
  31     ram_addr_t max_length;
  32     void (*resized)(const char*, uint64_t length, void *host);
  33     uint32_t flags;
  34     /* Protected by iothread lock.  */
  35     char idstr[256];
  36     /* RCU-enabled, writes protected by the ramlist lock */
  37     QLIST_ENTRY(RAMBlock) next;
  38     int fd;
  39     size_t page_size;
  40 };
  41
  42 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
  43 {
  44     return (b && b->host && offset < b->used_length) ? true : false;
  45 }
  46
  47 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
  48 {
  49     assert(offset_in_ramblock(block, offset));
  50     return (char *)block->host + offset;
  51 }
  52
  53 /* The dirty memory bitmap is split into fixed-size blocks to allow growth
  54  * under RCU.  The bitmap for a block can be accessed as follows:
  55  *
  56  *   rcu_read_lock();
  57  *
  58  *   DirtyMemoryBlocks *blocks =
  59  *       atomic_rcu_read(&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]);
  60  *
  61  *   ram_addr_t idx = (addr >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
  62  *   unsigned long *block = blocks.blocks[idx];
  63  *   ...access block bitmap...
  64  *
  65  *   rcu_read_unlock();
  66  *
  67  * Remember to check for the end of the block when accessing a range of
  68  * addresses.  Move on to the next block if you reach the end.
  69  *
  70  * Organization into blocks allows dirty memory to grow (but not shrink) under
  71  * RCU.  When adding new RAMBlocks requires the dirty memory to grow, a new
  72  * DirtyMemoryBlocks array is allocated with pointers to existing blocks kept
  73  * the same.  Other threads can safely access existing blocks while dirty
  74  * memory is being grown.  When no threads are using the old DirtyMemoryBlocks
  75  * anymore it is freed by RCU (but the underlying blocks stay because they are
  76  * pointed to from the new DirtyMemoryBlocks).
  77  */
  78 #define DIRTY_MEMORY_BLOCK_SIZE ((ram_addr_t)256 * 1024 * 8)
  79 typedef struct {
  80     struct rcu_head rcu;
  81     unsigned long *blocks[];
  82 } DirtyMemoryBlocks;
  83
  84 typedef struct RAMList {
  85     QemuMutex mutex;
  86     RAMBlock *mru_block;
  87     /* RCU-enabled, writes protected by the ramlist lock. */
  88     QLIST_HEAD(, RAMBlock) blocks;
  89     DirtyMemoryBlocks *dirty_memory[DIRTY_MEMORY_NUM];
  90     uint32_t version;
  91 } RAMList;
  92 extern RAMList ram_list;
  93
  94 ram_addr_t last_ram_offset(void);
  95 void qemu_mutex_lock_ramlist(void);
  96 void qemu_mutex_unlock_ramlist(void);
  97
  98 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  99                                    bool share, const char *mem_path,
 100                                    Error **errp);
 101 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
 102                                   MemoryRegion *mr, Error **errp);
 103 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
 104 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
 105                                     void (*resized)(const char*,
 106                                                     uint64_t length,
 107                                                     void *host),
 108                                     MemoryRegion *mr, Error **errp);
 109 void qemu_ram_free(RAMBlock *block);
 110
 111 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
 112
 113 #define DIRTY_CLIENTS_ALL     ((1 << DIRTY_MEMORY_NUM) - 1)
 114 #define DIRTY_CLIENTS_NOCODE  (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
 115
 116 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
 117                                                  ram_addr_t length,
 118                                                  unsigned client)
 119 {
 120     DirtyMemoryBlocks *blocks;
 121     unsigned long end, page;
 122     unsigned long idx, offset, base;
 123     bool dirty = false;
 124
 125     assert(client < DIRTY_MEMORY_NUM);
 126
 127     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 128     page = start >> TARGET_PAGE_BITS;
 129
 130     rcu_read_lock();
 131
 132     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 133
 134     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 135     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 136     base = page - offset;
 137     while (page < end) {
 138         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 139         unsigned long num = next - base;
 140         unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
 141         if (found < num) {
 142             dirty = true;
 143             break;
 144         }
 145
 146         page = next;
 147         idx++;
 148         offset = 0;
 149         base += DIRTY_MEMORY_BLOCK_SIZE;
 150     }
 151
 152     rcu_read_unlock();
 153
 154     return dirty;
 155 }
 156
 157 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
 158                                                  ram_addr_t length,
 159                                                  unsigned client)
 160 {
 161     DirtyMemoryBlocks *blocks;
 162     unsigned long end, page;
 163     unsigned long idx, offset, base;
 164     bool dirty = true;
 165
 166     assert(client < DIRTY_MEMORY_NUM);
 167
 168     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 169     page = start >> TARGET_PAGE_BITS;
 170
 171     rcu_read_lock();
 172
 173     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 174
 175     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 176     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 177     base = page - offset;
 178     while (page < end) {
 179         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 180         unsigned long num = next - base;
 181         unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
 182         if (found < num) {
 183             dirty = false;
 184             break;
 185         }
 186
 187         page = next;
 188         idx++;
 189         offset = 0;
 190         base += DIRTY_MEMORY_BLOCK_SIZE;
 191     }
 192
 193     rcu_read_unlock();
 194
 195     return dirty;
 196 }
 197
 198 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
 199                                                       unsigned client)
 200 {
 201     return cpu_physical_memory_get_dirty(addr, 1, client);
 202 }
 203
 204 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
 205 {
 206     bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
 207     bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
 208     bool migration =
 209         cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
 210     return !(vga && code && migration);
 211 }
 212
 213 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
 214                                                                ram_addr_t length,
 215                                                                uint8_t mask)
 216 {
 217     uint8_t ret = 0;
 218
 219     if (mask & (1 << DIRTY_MEMORY_VGA) &&
 220         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
 221         ret |= (1 << DIRTY_MEMORY_VGA);
 222     }
 223     if (mask & (1 << DIRTY_MEMORY_CODE) &&
 224         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
 225         ret |= (1 << DIRTY_MEMORY_CODE);
 226     }
 227     if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
 228         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
 229         ret |= (1 << DIRTY_MEMORY_MIGRATION);
 230     }
 231     return ret;
 232 }
 233
 234 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
 235                                                       unsigned client)
 236 {
 237     unsigned long page, idx, offset;
 238     DirtyMemoryBlocks *blocks;
 239
 240     assert(client < DIRTY_MEMORY_NUM);
 241
 242     page = addr >> TARGET_PAGE_BITS;
 243     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 244     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 245
 246     rcu_read_lock();
 247
 248     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 249
 250     set_bit_atomic(offset, blocks->blocks[idx]);
 251
 252     rcu_read_unlock();
 253 }
 254
 255 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
 256                                                        ram_addr_t length,
 257                                                        uint8_t mask)
 258 {
 259     DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
 260     unsigned long end, page;
 261     unsigned long idx, offset, base;
 262     int i;
 263
 264     if (!mask && !xen_enabled()) {
 265         return;
 266     }
 267
 268     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 269     page = start >> TARGET_PAGE_BITS;
 270
 271     rcu_read_lock();
 272
 273     for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 274         blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
 275     }
 276
 277     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 278     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 279     base = page - offset;
 280     while (page < end) {
 281         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 282
 283         if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
 284             bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
 285                               offset, next - page);
 286         }
 287         if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
 288             bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
 289                               offset, next - page);
 290         }
 291         if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
 292             bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
 293                               offset, next - page);
 294         }
 295
 296         page = next;
 297         idx++;
 298         offset = 0;
 299         base += DIRTY_MEMORY_BLOCK_SIZE;
 300     }
 301
 302     rcu_read_unlock();
 303
 304     xen_modified_memory(start, length);
 305 }
 306
 307 #if !defined(_WIN32)
 308 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
 309                                                           ram_addr_t start,
 310                                                           ram_addr_t pages)
 311 {
 312     unsigned long i, j;
 313     unsigned long page_number, c;
 314     hwaddr addr;
 315     ram_addr_t ram_addr;
 316     unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
 317     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 318     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 319
 320     /* start address is aligned at the start of a word? */
 321     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
 322         (hpratio == 1)) {
 323         unsigned long **blocks[DIRTY_MEMORY_NUM];
 324         unsigned long idx;
 325         unsigned long offset;
 326         long k;
 327         long nr = BITS_TO_LONGS(pages);
 328
 329         idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
 330         offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
 331                           DIRTY_MEMORY_BLOCK_SIZE);
 332
 333         rcu_read_lock();
 334
 335         for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 336             blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
 337         }
 338
 339         for (k = 0; k < nr; k++) {
 340             if (bitmap[k]) {
 341                 unsigned long temp = leul_to_cpu(bitmap[k]);
 342
 343                 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
 344                 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
 345                 if (tcg_enabled()) {
 346                     atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
 347                 }
 348             }
 349
 350             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 351                 offset = 0;
 352                 idx++;
 353             }
 354         }
 355
 356         rcu_read_unlock();
 357
 358         xen_modified_memory(start, pages << TARGET_PAGE_BITS);
 359     } else {
 360         uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
 361         /*
 362          * bitmap-traveling is faster than memory-traveling (for addr...)
 363          * especially when most of the memory is not dirty.
 364          */
 365         for (i = 0; i < len; i++) {
 366             if (bitmap[i] != 0) {
 367                 c = leul_to_cpu(bitmap[i]);
 368                 do {
 369                     j = ctzl(c);
 370                     c &= ~(1ul << j);
 371                     page_number = (i * HOST_LONG_BITS + j) * hpratio;
 372                     addr = page_number * TARGET_PAGE_SIZE;
 373                     ram_addr = start + addr;
 374                     cpu_physical_memory_set_dirty_range(ram_addr,
 375                                        TARGET_PAGE_SIZE * hpratio, clients);
 376                 } while (c != 0);
 377             }
 378         }
 379     }
 380 }
 381 #endif /* not _WIN32 */
 382
 383 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
 384                                               ram_addr_t length,
 385                                               unsigned client);
 386
 387 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
 388                                                          ram_addr_t length)
 389 {
 390     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
 391     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
 392     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
 393 }
 394
 395
 396 static inline
 397 uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
 398                                                ram_addr_t start,
 399                                                ram_addr_t length)
 400 {
 401     ram_addr_t addr;
 402     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 403     uint64_t num_dirty = 0;
 404
 405     /* start address is aligned at the start of a word? */
 406     if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
 407         int k;
 408         int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
 409         unsigned long * const *src;
 410         unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
 411         unsigned long offset = BIT_WORD((page * BITS_PER_LONG) %
 412                                         DIRTY_MEMORY_BLOCK_SIZE);
 413
 414         rcu_read_lock();
 415
 416         src = atomic_rcu_read(
 417                 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
 418
 419         for (k = page; k < page + nr; k++) {
 420             if (src[idx][offset]) {
 421                 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
 422                 unsigned long new_dirty;
 423                 new_dirty = ~dest[k];
 424                 dest[k] |= bits;
 425                 new_dirty &= bits;
 426                 num_dirty += ctpopl(new_dirty);
 427             }
 428
 429             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 430                 offset = 0;
 431                 idx++;
 432             }
 433         }
 434
 435         rcu_read_unlock();
 436     } else {
 437         for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
 438             if (cpu_physical_memory_test_and_clear_dirty(
 439                         start + addr,
 440                         TARGET_PAGE_SIZE,
 441                         DIRTY_MEMORY_MIGRATION)) {
 442                 long k = (start + addr) >> TARGET_PAGE_BITS;
 443                 if (!test_and_set_bit(k, dest)) {
 444                     num_dirty++;
 445                 }
 446             }
 447         }
 448     }
 449
 450     return num_dirty;
 451 }
 452
 453 void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
 454 #endif
 455 #endif