2 * Postcopy migration for RAM
4 * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates
7 * Dave Gilbert <dgilbert@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
15 * Postcopy is a migration technique where the execution flips from the
16 * source to the destination before all the data has been copied.
23 #include "qemu-common.h"
24 #include "migration/migration.h"
25 #include "migration/postcopy-ram.h"
26 #include "sysemu/sysemu.h"
27 #include "qemu/error-report.h"
30 /* Arbitrary limit on size of each discard command,
31 * keeps them around ~200 bytes
33 #define MAX_DISCARDS_PER_COMMAND 12
35 struct PostcopyDiscardState
{
36 const char *ramblock_name
;
37 uint64_t offset
; /* Bitmap entry for the 1st bit of this RAMBlock */
40 * Start and length of a discard range (bytes)
42 uint64_t start_list
[MAX_DISCARDS_PER_COMMAND
];
43 uint64_t length_list
[MAX_DISCARDS_PER_COMMAND
];
44 unsigned int nsentwords
;
45 unsigned int nsentcmds
;
48 /* Postcopy needs to detect accesses to pages that haven't yet been copied
49 * across, and efficiently map new pages in, the techniques for doing this
50 * are target OS specific.
52 #if defined(__linux__)
55 #include <sys/ioctl.h>
56 #include <sys/syscall.h>
57 #include <sys/types.h>
58 #include <asm/types.h> /* for __u64 */
61 #if defined(__linux__) && defined(__NR_userfaultfd)
62 #include <linux/userfaultfd.h>
64 static bool ufd_version_check(int ufd
)
66 struct uffdio_api api_struct
;
69 api_struct
.api
= UFFD_API
;
70 api_struct
.features
= 0;
71 if (ioctl(ufd
, UFFDIO_API
, &api_struct
)) {
72 error_report("postcopy_ram_supported_by_host: UFFDIO_API failed: %s",
77 ioctl_mask
= (__u64
)1 << _UFFDIO_REGISTER
|
78 (__u64
)1 << _UFFDIO_UNREGISTER
;
79 if ((api_struct
.ioctls
& ioctl_mask
) != ioctl_mask
) {
80 error_report("Missing userfault features: %" PRIx64
,
81 (uint64_t)(~api_struct
.ioctls
& ioctl_mask
));
88 bool postcopy_ram_supported_by_host(void)
90 long pagesize
= getpagesize();
92 bool ret
= false; /* Error unless we change it */
93 void *testarea
= NULL
;
94 struct uffdio_register reg_struct
;
95 struct uffdio_range range_struct
;
96 uint64_t feature_mask
;
98 if ((1ul << qemu_target_page_bits()) > pagesize
) {
99 error_report("Target page size bigger than host page size");
103 ufd
= syscall(__NR_userfaultfd
, O_CLOEXEC
);
105 error_report("%s: userfaultfd not available: %s", __func__
,
110 /* Version and features check */
111 if (!ufd_version_check(ufd
)) {
116 * We need to check that the ops we need are supported on anon memory
117 * To do that we need to register a chunk and see the flags that
120 testarea
= mmap(NULL
, pagesize
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
|
121 MAP_ANONYMOUS
, -1, 0);
122 if (testarea
== MAP_FAILED
) {
123 error_report("%s: Failed to map test area: %s", __func__
,
127 g_assert(((size_t)testarea
& (pagesize
-1)) == 0);
129 reg_struct
.range
.start
= (uintptr_t)testarea
;
130 reg_struct
.range
.len
= pagesize
;
131 reg_struct
.mode
= UFFDIO_REGISTER_MODE_MISSING
;
133 if (ioctl(ufd
, UFFDIO_REGISTER
, ®_struct
)) {
134 error_report("%s userfault register: %s", __func__
, strerror(errno
));
138 range_struct
.start
= (uintptr_t)testarea
;
139 range_struct
.len
= pagesize
;
140 if (ioctl(ufd
, UFFDIO_UNREGISTER
, &range_struct
)) {
141 error_report("%s userfault unregister: %s", __func__
, strerror(errno
));
145 feature_mask
= (__u64
)1 << _UFFDIO_WAKE
|
146 (__u64
)1 << _UFFDIO_COPY
|
147 (__u64
)1 << _UFFDIO_ZEROPAGE
;
148 if ((reg_struct
.ioctls
& feature_mask
) != feature_mask
) {
149 error_report("Missing userfault map features: %" PRIx64
,
150 (uint64_t)(~reg_struct
.ioctls
& feature_mask
));
158 munmap(testarea
, pagesize
);
167 * postcopy_ram_discard_range: Discard a range of memory.
168 * We can assume that if we've been called postcopy_ram_hosttest returned true.
170 * @mis: Current incoming migration state.
171 * @start, @length: range of memory to discard.
173 * returns: 0 on success.
175 int postcopy_ram_discard_range(MigrationIncomingState
*mis
, uint8_t *start
,
178 trace_postcopy_ram_discard_range(start
, length
);
179 if (madvise(start
, length
, MADV_DONTNEED
)) {
180 error_report("%s MADV_DONTNEED: %s", __func__
, strerror(errno
));
188 * Setup an area of RAM so that it *can* be used for postcopy later; this
189 * must be done right at the start prior to pre-copy.
190 * opaque should be the MIS.
192 static int init_range(const char *block_name
, void *host_addr
,
193 ram_addr_t offset
, ram_addr_t length
, void *opaque
)
195 MigrationIncomingState
*mis
= opaque
;
197 trace_postcopy_init_range(block_name
, host_addr
, offset
, length
);
200 * We need the whole of RAM to be truly empty for postcopy, so things
201 * like ROMs and any data tables built during init must be zero'd
202 * - we're going to get the copy from the source anyway.
203 * (Precopy will just overwrite this data, so doesn't need the discard)
205 if (postcopy_ram_discard_range(mis
, host_addr
, length
)) {
213 * At the end of migration, undo the effects of init_range
214 * opaque should be the MIS.
216 static int cleanup_range(const char *block_name
, void *host_addr
,
217 ram_addr_t offset
, ram_addr_t length
, void *opaque
)
219 MigrationIncomingState
*mis
= opaque
;
220 struct uffdio_range range_struct
;
221 trace_postcopy_cleanup_range(block_name
, host_addr
, offset
, length
);
224 * We turned off hugepage for the precopy stage with postcopy enabled
225 * we can turn it back on now.
228 if (madvise(host_addr
, length
, MADV_HUGEPAGE
)) {
229 error_report("%s HUGEPAGE: %s", __func__
, strerror(errno
));
235 * We can also turn off userfault now since we should have all the
236 * pages. It can be useful to leave it on to debug postcopy
237 * if you're not sure it's always getting every page.
239 range_struct
.start
= (uintptr_t)host_addr
;
240 range_struct
.len
= length
;
242 if (ioctl(mis
->userfault_fd
, UFFDIO_UNREGISTER
, &range_struct
)) {
243 error_report("%s: userfault unregister %s", __func__
, strerror(errno
));
252 * Initialise postcopy-ram, setting the RAM to a state where we can go into
253 * postcopy later; must be called prior to any precopy.
254 * called from arch_init's similarly named ram_postcopy_incoming_init
256 int postcopy_ram_incoming_init(MigrationIncomingState
*mis
, size_t ram_pages
)
258 if (qemu_ram_foreach_block(init_range
, mis
)) {
266 * At the end of a migration where postcopy_ram_incoming_init was called.
268 int postcopy_ram_incoming_cleanup(MigrationIncomingState
*mis
)
270 /* TODO: Join the fault thread once we're sure it will exit */
271 if (qemu_ram_foreach_block(cleanup_range
, mis
)) {
275 if (mis
->postcopy_tmp_page
) {
276 munmap(mis
->postcopy_tmp_page
, getpagesize());
277 mis
->postcopy_tmp_page
= NULL
;
283 * Mark the given area of RAM as requiring notification to unwritten areas
284 * Used as a callback on qemu_ram_foreach_block.
285 * host_addr: Base of area to mark
286 * offset: Offset in the whole ram arena
287 * length: Length of the section
288 * opaque: MigrationIncomingState pointer
289 * Returns 0 on success
291 static int ram_block_enable_notify(const char *block_name
, void *host_addr
,
292 ram_addr_t offset
, ram_addr_t length
,
295 MigrationIncomingState
*mis
= opaque
;
296 struct uffdio_register reg_struct
;
298 reg_struct
.range
.start
= (uintptr_t)host_addr
;
299 reg_struct
.range
.len
= length
;
300 reg_struct
.mode
= UFFDIO_REGISTER_MODE_MISSING
;
302 /* Now tell our userfault_fd that it's responsible for this area */
303 if (ioctl(mis
->userfault_fd
, UFFDIO_REGISTER
, ®_struct
)) {
304 error_report("%s userfault register: %s", __func__
, strerror(errno
));
312 * Handle faults detected by the USERFAULT markings
314 static void *postcopy_ram_fault_thread(void *opaque
)
316 MigrationIncomingState
*mis
= opaque
;
318 fprintf(stderr
, "postcopy_ram_fault_thread\n");
319 /* TODO: In later patch */
320 qemu_sem_post(&mis
->fault_thread_sem
);
322 /* TODO: In later patch */
328 int postcopy_ram_enable_notify(MigrationIncomingState
*mis
)
330 /* Create the fault handler thread and wait for it to be ready */
331 qemu_sem_init(&mis
->fault_thread_sem
, 0);
332 qemu_thread_create(&mis
->fault_thread
, "postcopy/fault",
333 postcopy_ram_fault_thread
, mis
, QEMU_THREAD_JOINABLE
);
334 qemu_sem_wait(&mis
->fault_thread_sem
);
335 qemu_sem_destroy(&mis
->fault_thread_sem
);
337 /* Mark so that we get notified of accesses to unwritten areas */
338 if (qemu_ram_foreach_block(ram_block_enable_notify
, mis
)) {
346 * Place a host page (from) at (host) atomically
347 * returns 0 on success
349 int postcopy_place_page(MigrationIncomingState
*mis
, void *host
, void *from
)
351 struct uffdio_copy copy_struct
;
353 copy_struct
.dst
= (uint64_t)(uintptr_t)host
;
354 copy_struct
.src
= (uint64_t)(uintptr_t)from
;
355 copy_struct
.len
= getpagesize();
356 copy_struct
.mode
= 0;
358 /* copy also acks to the kernel waking the stalled thread up
359 * TODO: We can inhibit that ack and only do it if it was requested
360 * which would be slightly cheaper, but we'd have to be careful
361 * of the order of updating our page state.
363 if (ioctl(mis
->userfault_fd
, UFFDIO_COPY
, ©_struct
)) {
365 error_report("%s: %s copy host: %p from: %p",
366 __func__
, strerror(e
), host
, from
);
371 trace_postcopy_place_page(host
);
376 * Place a zero page at (host) atomically
377 * returns 0 on success
379 int postcopy_place_page_zero(MigrationIncomingState
*mis
, void *host
)
381 struct uffdio_zeropage zero_struct
;
383 zero_struct
.range
.start
= (uint64_t)(uintptr_t)host
;
384 zero_struct
.range
.len
= getpagesize();
385 zero_struct
.mode
= 0;
387 if (ioctl(mis
->userfault_fd
, UFFDIO_ZEROPAGE
, &zero_struct
)) {
389 error_report("%s: %s zero host: %p",
390 __func__
, strerror(e
), host
);
395 trace_postcopy_place_page_zero(host
);
400 * Returns a target page of memory that can be mapped at a later point in time
401 * using postcopy_place_page
402 * The same address is used repeatedly, postcopy_place_page just takes the
404 * Returns: Pointer to allocated page
407 void *postcopy_get_tmp_page(MigrationIncomingState
*mis
)
409 if (!mis
->postcopy_tmp_page
) {
410 mis
->postcopy_tmp_page
= mmap(NULL
, getpagesize(),
411 PROT_READ
| PROT_WRITE
, MAP_PRIVATE
|
412 MAP_ANONYMOUS
, -1, 0);
413 if (!mis
->postcopy_tmp_page
) {
414 error_report("%s: %s", __func__
, strerror(errno
));
419 return mis
->postcopy_tmp_page
;
423 /* No target OS support, stubs just fail */
424 bool postcopy_ram_supported_by_host(void)
426 error_report("%s: No OS support", __func__
);
430 int postcopy_ram_incoming_init(MigrationIncomingState
*mis
, size_t ram_pages
)
432 error_report("postcopy_ram_incoming_init: No OS support");
436 int postcopy_ram_incoming_cleanup(MigrationIncomingState
*mis
)
442 int postcopy_ram_discard_range(MigrationIncomingState
*mis
, uint8_t *start
,
449 int postcopy_ram_enable_notify(MigrationIncomingState
*mis
)
455 int postcopy_place_page(MigrationIncomingState
*mis
, void *host
, void *from
)
461 int postcopy_place_page_zero(MigrationIncomingState
*mis
, void *host
)
467 void *postcopy_get_tmp_page(MigrationIncomingState
*mis
)
475 /* ------------------------------------------------------------------------- */
478 * postcopy_discard_send_init: Called at the start of each RAMBlock before
479 * asking to discard individual ranges.
481 * @ms: The current migration state.
482 * @offset: the bitmap offset of the named RAMBlock in the migration
484 * @name: RAMBlock that discards will operate on.
486 * returns: a new PDS.
488 PostcopyDiscardState
*postcopy_discard_send_init(MigrationState
*ms
,
489 unsigned long offset
,
492 PostcopyDiscardState
*res
= g_malloc0(sizeof(PostcopyDiscardState
));
495 res
->ramblock_name
= name
;
496 res
->offset
= offset
;
503 * postcopy_discard_send_range: Called by the bitmap code for each chunk to
504 * discard. May send a discard message, may just leave it queued to
507 * @ms: Current migration state.
508 * @pds: Structure initialised by postcopy_discard_send_init().
509 * @start,@length: a range of pages in the migration bitmap in the
510 * RAM block passed to postcopy_discard_send_init() (length=1 is one page)
512 void postcopy_discard_send_range(MigrationState
*ms
, PostcopyDiscardState
*pds
,
513 unsigned long start
, unsigned long length
)
515 size_t tp_bits
= qemu_target_page_bits();
516 /* Convert to byte offsets within the RAM block */
517 pds
->start_list
[pds
->cur_entry
] = (start
- pds
->offset
) << tp_bits
;
518 pds
->length_list
[pds
->cur_entry
] = length
<< tp_bits
;
519 trace_postcopy_discard_send_range(pds
->ramblock_name
, start
, length
);
523 if (pds
->cur_entry
== MAX_DISCARDS_PER_COMMAND
) {
524 /* Full set, ship it! */
525 qemu_savevm_send_postcopy_ram_discard(ms
->file
, pds
->ramblock_name
,
535 * postcopy_discard_send_finish: Called at the end of each RAMBlock by the
536 * bitmap code. Sends any outstanding discard messages, frees the PDS
538 * @ms: Current migration state.
539 * @pds: Structure initialised by postcopy_discard_send_init().
541 void postcopy_discard_send_finish(MigrationState
*ms
, PostcopyDiscardState
*pds
)
543 /* Anything unsent? */
544 if (pds
->cur_entry
) {
545 qemu_savevm_send_postcopy_ram_discard(ms
->file
, pds
->ramblock_name
,
552 trace_postcopy_discard_send_finish(pds
->ramblock_name
, pds
->nsentwords
,