include/exec/ram_addr.h

   1 /*
   2  * Declarations for cpu physical memory functions
   3  *
   4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
   5  *
   6  * Authors:
   7  *  Avi Kivity <avi@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * later.  See the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 /*
  15  * This header is for use by exec.c and memory.c ONLY.  Do not include it.
  16  * The functions declared here will be removed soon.
  17  */
  18
  19 #ifndef RAM_ADDR_H
  20 #define RAM_ADDR_H
  21
  22 #ifndef CONFIG_USER_ONLY
  23 #include "hw/xen/xen.h"
  24 #include "exec/ramlist.h"
  25
  26 struct RAMBlock {
  27     struct rcu_head rcu;
  28     struct MemoryRegion *mr;
  29     uint8_t *host;
  30     ram_addr_t offset;
  31     ram_addr_t used_length;
  32     ram_addr_t max_length;
  33     void (*resized)(const char*, uint64_t length, void *host);
  34     uint32_t flags;
  35     /* Protected by iothread lock.  */
  36     char idstr[256];
  37     /* RCU-enabled, writes protected by the ramlist lock */
  38     QLIST_ENTRY(RAMBlock) next;
  39     QLIST_HEAD(, RAMBlockNotifier) ramblock_notifiers;
  40     int fd;
  41     size_t page_size;
  42 };
  43
  44 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
  45 {
  46     return (b && b->host && offset < b->used_length) ? true : false;
  47 }
  48
  49 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
  50 {
  51     assert(offset_in_ramblock(block, offset));
  52     return (char *)block->host + offset;
  53 }
  54
  55 ram_addr_t last_ram_offset(void);
  56 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  57                                    bool share, const char *mem_path,
  58                                    Error **errp);
  59 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
  60                                   MemoryRegion *mr, Error **errp);
  61 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
  62 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
  63                                     void (*resized)(const char*,
  64                                                     uint64_t length,
  65                                                     void *host),
  66                                     MemoryRegion *mr, Error **errp);
  67 void qemu_ram_free(RAMBlock *block);
  68
  69 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
  70
  71 #define DIRTY_CLIENTS_ALL     ((1 << DIRTY_MEMORY_NUM) - 1)
  72 #define DIRTY_CLIENTS_NOCODE  (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
  73
  74 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
  75                                                  ram_addr_t length,
  76                                                  unsigned client)
  77 {
  78     DirtyMemoryBlocks *blocks;
  79     unsigned long end, page;
  80     unsigned long idx, offset, base;
  81     bool dirty = false;
  82
  83     assert(client < DIRTY_MEMORY_NUM);
  84
  85     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
  86     page = start >> TARGET_PAGE_BITS;
  87
  88     rcu_read_lock();
  89
  90     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  91
  92     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  93     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  94     base = page - offset;
  95     while (page < end) {
  96         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
  97         unsigned long num = next - base;
  98         unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
  99         if (found < num) {
 100             dirty = true;
 101             break;
 102         }
 103
 104         page = next;
 105         idx++;
 106         offset = 0;
 107         base += DIRTY_MEMORY_BLOCK_SIZE;
 108     }
 109
 110     rcu_read_unlock();
 111
 112     return dirty;
 113 }
 114
 115 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
 116                                                  ram_addr_t length,
 117                                                  unsigned client)
 118 {
 119     DirtyMemoryBlocks *blocks;
 120     unsigned long end, page;
 121     unsigned long idx, offset, base;
 122     bool dirty = true;
 123
 124     assert(client < DIRTY_MEMORY_NUM);
 125
 126     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 127     page = start >> TARGET_PAGE_BITS;
 128
 129     rcu_read_lock();
 130
 131     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 132
 133     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 134     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 135     base = page - offset;
 136     while (page < end) {
 137         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 138         unsigned long num = next - base;
 139         unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
 140         if (found < num) {
 141             dirty = false;
 142             break;
 143         }
 144
 145         page = next;
 146         idx++;
 147         offset = 0;
 148         base += DIRTY_MEMORY_BLOCK_SIZE;
 149     }
 150
 151     rcu_read_unlock();
 152
 153     return dirty;
 154 }
 155
 156 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
 157                                                       unsigned client)
 158 {
 159     return cpu_physical_memory_get_dirty(addr, 1, client);
 160 }
 161
 162 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
 163 {
 164     bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
 165     bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
 166     bool migration =
 167         cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
 168     return !(vga && code && migration);
 169 }
 170
 171 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
 172                                                                ram_addr_t length,
 173                                                                uint8_t mask)
 174 {
 175     uint8_t ret = 0;
 176
 177     if (mask & (1 << DIRTY_MEMORY_VGA) &&
 178         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
 179         ret |= (1 << DIRTY_MEMORY_VGA);
 180     }
 181     if (mask & (1 << DIRTY_MEMORY_CODE) &&
 182         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
 183         ret |= (1 << DIRTY_MEMORY_CODE);
 184     }
 185     if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
 186         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
 187         ret |= (1 << DIRTY_MEMORY_MIGRATION);
 188     }
 189     return ret;
 190 }
 191
 192 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
 193                                                       unsigned client)
 194 {
 195     unsigned long page, idx, offset;
 196     DirtyMemoryBlocks *blocks;
 197
 198     assert(client < DIRTY_MEMORY_NUM);
 199
 200     page = addr >> TARGET_PAGE_BITS;
 201     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 202     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 203
 204     rcu_read_lock();
 205
 206     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 207
 208     set_bit_atomic(offset, blocks->blocks[idx]);
 209
 210     rcu_read_unlock();
 211 }
 212
 213 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
 214                                                        ram_addr_t length,
 215                                                        uint8_t mask)
 216 {
 217     DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
 218     unsigned long end, page;
 219     unsigned long idx, offset, base;
 220     int i;
 221
 222     if (!mask && !xen_enabled()) {
 223         return;
 224     }
 225
 226     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 227     page = start >> TARGET_PAGE_BITS;
 228
 229     rcu_read_lock();
 230
 231     for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 232         blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
 233     }
 234
 235     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 236     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 237     base = page - offset;
 238     while (page < end) {
 239         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 240
 241         if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
 242             bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
 243                               offset, next - page);
 244         }
 245         if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
 246             bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
 247                               offset, next - page);
 248         }
 249         if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
 250             bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
 251                               offset, next - page);
 252         }
 253
 254         page = next;
 255         idx++;
 256         offset = 0;
 257         base += DIRTY_MEMORY_BLOCK_SIZE;
 258     }
 259
 260     rcu_read_unlock();
 261
 262     xen_modified_memory(start, length);
 263 }
 264
 265 #if !defined(_WIN32)
 266 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
 267                                                           ram_addr_t start,
 268                                                           ram_addr_t pages)
 269 {
 270     unsigned long i, j;
 271     unsigned long page_number, c;
 272     hwaddr addr;
 273     ram_addr_t ram_addr;
 274     unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
 275     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 276     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 277
 278     /* start address is aligned at the start of a word? */
 279     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
 280         (hpratio == 1)) {
 281         unsigned long **blocks[DIRTY_MEMORY_NUM];
 282         unsigned long idx;
 283         unsigned long offset;
 284         long k;
 285         long nr = BITS_TO_LONGS(pages);
 286
 287         idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
 288         offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
 289                           DIRTY_MEMORY_BLOCK_SIZE);
 290
 291         rcu_read_lock();
 292
 293         for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 294             blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
 295         }
 296
 297         for (k = 0; k < nr; k++) {
 298             if (bitmap[k]) {
 299                 unsigned long temp = leul_to_cpu(bitmap[k]);
 300
 301                 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
 302                 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
 303                 if (tcg_enabled()) {
 304                     atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
 305                 }
 306             }
 307
 308             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 309                 offset = 0;
 310                 idx++;
 311             }
 312         }
 313
 314         rcu_read_unlock();
 315
 316         xen_modified_memory(start, pages << TARGET_PAGE_BITS);
 317     } else {
 318         uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
 319         /*
 320          * bitmap-traveling is faster than memory-traveling (for addr...)
 321          * especially when most of the memory is not dirty.
 322          */
 323         for (i = 0; i < len; i++) {
 324             if (bitmap[i] != 0) {
 325                 c = leul_to_cpu(bitmap[i]);
 326                 do {
 327                     j = ctzl(c);
 328                     c &= ~(1ul << j);
 329                     page_number = (i * HOST_LONG_BITS + j) * hpratio;
 330                     addr = page_number * TARGET_PAGE_SIZE;
 331                     ram_addr = start + addr;
 332                     cpu_physical_memory_set_dirty_range(ram_addr,
 333                                        TARGET_PAGE_SIZE * hpratio, clients);
 334                 } while (c != 0);
 335             }
 336         }
 337     }
 338 }
 339 #endif /* not _WIN32 */
 340
 341 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
 342                                               ram_addr_t length,
 343                                               unsigned client);
 344
 345 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
 346                                                          ram_addr_t length)
 347 {
 348     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
 349     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
 350     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
 351 }
 352
 353
 354 static inline
 355 uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
 356                                                ram_addr_t start,
 357                                                ram_addr_t length)
 358 {
 359     ram_addr_t addr;
 360     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 361     uint64_t num_dirty = 0;
 362
 363     /* start address is aligned at the start of a word? */
 364     if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
 365         int k;
 366         int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
 367         unsigned long * const *src;
 368         unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
 369         unsigned long offset = BIT_WORD((page * BITS_PER_LONG) %
 370                                         DIRTY_MEMORY_BLOCK_SIZE);
 371
 372         rcu_read_lock();
 373
 374         src = atomic_rcu_read(
 375                 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
 376
 377         for (k = page; k < page + nr; k++) {
 378             if (src[idx][offset]) {
 379                 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
 380                 unsigned long new_dirty;
 381                 new_dirty = ~dest[k];
 382                 dest[k] |= bits;
 383                 new_dirty &= bits;
 384                 num_dirty += ctpopl(new_dirty);
 385             }
 386
 387             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 388                 offset = 0;
 389                 idx++;
 390             }
 391         }
 392
 393         rcu_read_unlock();
 394     } else {
 395         for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
 396             if (cpu_physical_memory_test_and_clear_dirty(
 397                         start + addr,
 398                         TARGET_PAGE_SIZE,
 399                         DIRTY_MEMORY_MIGRATION)) {
 400                 long k = (start + addr) >> TARGET_PAGE_BITS;
 401                 if (!test_and_set_bit(k, dest)) {
 402                     num_dirty++;
 403                 }
 404             }
 405         }
 406     }
 407
 408     return num_dirty;
 409 }
 410
 411 void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
 412 #endif
 413 #endif