4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu/osdep.h"
30 #include "qemu/cutils.h"
31 #include "qemu/bitops.h"
32 #include "qemu/bitmap.h"
33 #include "qemu/main-loop.h"
36 #include "migration.h"
37 #include "migration/register.h"
38 #include "migration/misc.h"
39 #include "qemu-file.h"
40 #include "postcopy-ram.h"
41 #include "page_cache.h"
42 #include "qemu/error-report.h"
43 #include "qapi/error.h"
44 #include "qapi/qapi-types-migration.h"
45 #include "qapi/qapi-events-migration.h"
46 #include "qapi/qmp/qerror.h"
48 #include "exec/ram_addr.h"
49 #include "exec/target_page.h"
50 #include "qemu/rcu_queue.h"
51 #include "migration/colo.h"
53 #include "sysemu/cpu-throttle.h"
57 #include "sysemu/runstate.h"
59 #if defined(__linux__)
60 #include "qemu/userfaultfd.h"
61 #endif /* defined(__linux__) */
63 /***********************************************************/
64 /* ram save/restore */
66 /* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
67 * worked for pages that where filled with the same char. We switched
68 * it to only search for the zero value. And to avoid confusion with
69 * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it.
72 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
73 #define RAM_SAVE_FLAG_ZERO 0x02
74 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
75 #define RAM_SAVE_FLAG_PAGE 0x08
76 #define RAM_SAVE_FLAG_EOS 0x10
77 #define RAM_SAVE_FLAG_CONTINUE 0x20
78 #define RAM_SAVE_FLAG_XBZRLE 0x40
79 /* 0x80 is reserved in migration.h start with 0x100 next */
80 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
82 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
84 return buffer_is_zero(p
, size
);
87 XBZRLECacheStats xbzrle_counters
;
89 /* struct contains XBZRLE cache and a static page
90 used by the compression */
92 /* buffer used for XBZRLE encoding */
94 /* buffer for storing page content */
96 /* Cache for XBZRLE, Protected by lock. */
99 /* it will store a page full of zeros */
100 uint8_t *zero_target_page
;
101 /* buffer used for XBZRLE decoding */
102 uint8_t *decoded_buf
;
105 static void XBZRLE_cache_lock(void)
107 if (migrate_use_xbzrle()) {
108 qemu_mutex_lock(&XBZRLE
.lock
);
112 static void XBZRLE_cache_unlock(void)
114 if (migrate_use_xbzrle()) {
115 qemu_mutex_unlock(&XBZRLE
.lock
);
120 * xbzrle_cache_resize: resize the xbzrle cache
122 * This function is called from migrate_params_apply in main
123 * thread, possibly while a migration is in progress. A running
124 * migration may be using the cache and might finish during this call,
125 * hence changes to the cache are protected by XBZRLE.lock().
127 * Returns 0 for success or -1 for error
129 * @new_size: new cache size
130 * @errp: set *errp if the check failed, with reason
132 int xbzrle_cache_resize(uint64_t new_size
, Error
**errp
)
134 PageCache
*new_cache
;
137 /* Check for truncation */
138 if (new_size
!= (size_t)new_size
) {
139 error_setg(errp
, QERR_INVALID_PARAMETER_VALUE
, "cache size",
140 "exceeding address space");
144 if (new_size
== migrate_xbzrle_cache_size()) {
151 if (XBZRLE
.cache
!= NULL
) {
152 new_cache
= cache_init(new_size
, TARGET_PAGE_SIZE
, errp
);
158 cache_fini(XBZRLE
.cache
);
159 XBZRLE
.cache
= new_cache
;
162 XBZRLE_cache_unlock();
166 bool ramblock_is_ignored(RAMBlock
*block
)
168 return !qemu_ram_is_migratable(block
) ||
169 (migrate_ignore_shared() && qemu_ram_is_shared(block
));
172 #undef RAMBLOCK_FOREACH
174 int foreach_not_ignored_block(RAMBlockIterFunc func
, void *opaque
)
179 RCU_READ_LOCK_GUARD();
181 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
182 ret
= func(block
, opaque
);
190 static void ramblock_recv_map_init(void)
194 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
195 assert(!rb
->receivedmap
);
196 rb
->receivedmap
= bitmap_new(rb
->max_length
>> qemu_target_page_bits());
200 int ramblock_recv_bitmap_test(RAMBlock
*rb
, void *host_addr
)
202 return test_bit(ramblock_recv_bitmap_offset(host_addr
, rb
),
206 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock
*rb
, uint64_t byte_offset
)
208 return test_bit(byte_offset
>> TARGET_PAGE_BITS
, rb
->receivedmap
);
211 void ramblock_recv_bitmap_set(RAMBlock
*rb
, void *host_addr
)
213 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr
, rb
), rb
->receivedmap
);
216 void ramblock_recv_bitmap_set_range(RAMBlock
*rb
, void *host_addr
,
219 bitmap_set_atomic(rb
->receivedmap
,
220 ramblock_recv_bitmap_offset(host_addr
, rb
),
224 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
227 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
229 * Returns >0 if success with sent bytes, or <0 if error.
231 int64_t ramblock_recv_bitmap_send(QEMUFile
*file
,
232 const char *block_name
)
234 RAMBlock
*block
= qemu_ram_block_by_name(block_name
);
235 unsigned long *le_bitmap
, nbits
;
239 error_report("%s: invalid block name: %s", __func__
, block_name
);
243 nbits
= block
->postcopy_length
>> TARGET_PAGE_BITS
;
246 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
247 * machines we may need 4 more bytes for padding (see below
248 * comment). So extend it a bit before hand.
250 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
253 * Always use little endian when sending the bitmap. This is
254 * required that when source and destination VMs are not using the
255 * same endianness. (Note: big endian won't work.)
257 bitmap_to_le(le_bitmap
, block
->receivedmap
, nbits
);
259 /* Size of the bitmap, in bytes */
260 size
= DIV_ROUND_UP(nbits
, 8);
263 * size is always aligned to 8 bytes for 64bit machines, but it
264 * may not be true for 32bit machines. We need this padding to
265 * make sure the migration can survive even between 32bit and
268 size
= ROUND_UP(size
, 8);
270 qemu_put_be64(file
, size
);
271 qemu_put_buffer(file
, (const uint8_t *)le_bitmap
, size
);
273 * Mark as an end, in case the middle part is screwed up due to
274 * some "mysterious" reason.
276 qemu_put_be64(file
, RAMBLOCK_RECV_BITMAP_ENDING
);
281 if (qemu_file_get_error(file
)) {
282 return qemu_file_get_error(file
);
285 return size
+ sizeof(size
);
289 * An outstanding page request, on the source, having been received
292 struct RAMSrcPageRequest
{
297 QSIMPLEQ_ENTRY(RAMSrcPageRequest
) next_req
;
300 /* State of RAM for migration */
302 /* QEMUFile used for this migration */
304 /* UFFD file descriptor, used in 'write-tracking' migration */
306 /* Last block that we have visited searching for dirty pages */
307 RAMBlock
*last_seen_block
;
308 /* Last block from where we have sent data */
309 RAMBlock
*last_sent_block
;
310 /* Last dirty target page we have sent */
311 ram_addr_t last_page
;
312 /* last ram version we have seen */
313 uint32_t last_version
;
314 /* How many times we have dirty too many pages */
315 int dirty_rate_high_cnt
;
316 /* these variables are used for bitmap sync */
317 /* last time we did a full bitmap_sync */
318 int64_t time_last_bitmap_sync
;
319 /* bytes transferred at start_time */
320 uint64_t bytes_xfer_prev
;
321 /* number of dirty pages since start_time */
322 uint64_t num_dirty_pages_period
;
323 /* xbzrle misses since the beginning of the period */
324 uint64_t xbzrle_cache_miss_prev
;
325 /* Amount of xbzrle pages since the beginning of the period */
326 uint64_t xbzrle_pages_prev
;
327 /* Amount of xbzrle encoded bytes since the beginning of the period */
328 uint64_t xbzrle_bytes_prev
;
329 /* Start using XBZRLE (e.g., after the first round). */
332 /* compression statistics since the beginning of the period */
333 /* amount of count that no free thread to compress data */
334 uint64_t compress_thread_busy_prev
;
335 /* amount bytes after compression */
336 uint64_t compressed_size_prev
;
337 /* amount of compressed pages */
338 uint64_t compress_pages_prev
;
340 /* total handled target pages at the beginning of period */
341 uint64_t target_page_count_prev
;
342 /* total handled target pages since start */
343 uint64_t target_page_count
;
344 /* number of dirty bits in the bitmap */
345 uint64_t migration_dirty_pages
;
346 /* Protects modification of the bitmap and migration dirty pages */
347 QemuMutex bitmap_mutex
;
348 /* The RAMBlock used in the last src_page_requests */
349 RAMBlock
*last_req_rb
;
350 /* Queue of outstanding page requests from the destination */
351 QemuMutex src_page_req_mutex
;
352 QSIMPLEQ_HEAD(, RAMSrcPageRequest
) src_page_requests
;
354 typedef struct RAMState RAMState
;
356 static RAMState
*ram_state
;
358 static NotifierWithReturnList precopy_notifier_list
;
360 void precopy_infrastructure_init(void)
362 notifier_with_return_list_init(&precopy_notifier_list
);
365 void precopy_add_notifier(NotifierWithReturn
*n
)
367 notifier_with_return_list_add(&precopy_notifier_list
, n
);
370 void precopy_remove_notifier(NotifierWithReturn
*n
)
372 notifier_with_return_remove(n
);
375 int precopy_notify(PrecopyNotifyReason reason
, Error
**errp
)
377 PrecopyNotifyData pnd
;
381 return notifier_with_return_list_notify(&precopy_notifier_list
, &pnd
);
384 uint64_t ram_bytes_remaining(void)
386 return ram_state
? (ram_state
->migration_dirty_pages
* TARGET_PAGE_SIZE
) :
390 MigrationStats ram_counters
;
392 /* used by the search for pages to send */
393 struct PageSearchStatus
{
394 /* Current block being searched */
396 /* Current page to search from */
398 /* Set once we wrap around */
401 typedef struct PageSearchStatus PageSearchStatus
;
403 CompressionStats compression_counters
;
405 struct CompressParam
{
415 /* internally used fields */
419 typedef struct CompressParam CompressParam
;
421 struct DecompressParam
{
431 typedef struct DecompressParam DecompressParam
;
433 static CompressParam
*comp_param
;
434 static QemuThread
*compress_threads
;
435 /* comp_done_cond is used to wake up the migration thread when
436 * one of the compression threads has finished the compression.
437 * comp_done_lock is used to co-work with comp_done_cond.
439 static QemuMutex comp_done_lock
;
440 static QemuCond comp_done_cond
;
441 /* The empty QEMUFileOps will be used by file in CompressParam */
442 static const QEMUFileOps empty_ops
= { };
444 static QEMUFile
*decomp_file
;
445 static DecompressParam
*decomp_param
;
446 static QemuThread
*decompress_threads
;
447 static QemuMutex decomp_done_lock
;
448 static QemuCond decomp_done_cond
;
450 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
451 ram_addr_t offset
, uint8_t *source_buf
);
453 static void *do_data_compress(void *opaque
)
455 CompressParam
*param
= opaque
;
460 qemu_mutex_lock(¶m
->mutex
);
461 while (!param
->quit
) {
463 block
= param
->block
;
464 offset
= param
->offset
;
466 qemu_mutex_unlock(¶m
->mutex
);
468 zero_page
= do_compress_ram_page(param
->file
, ¶m
->stream
,
469 block
, offset
, param
->originbuf
);
471 qemu_mutex_lock(&comp_done_lock
);
473 param
->zero_page
= zero_page
;
474 qemu_cond_signal(&comp_done_cond
);
475 qemu_mutex_unlock(&comp_done_lock
);
477 qemu_mutex_lock(¶m
->mutex
);
479 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
482 qemu_mutex_unlock(¶m
->mutex
);
487 static void compress_threads_save_cleanup(void)
491 if (!migrate_use_compression() || !comp_param
) {
495 thread_count
= migrate_compress_threads();
496 for (i
= 0; i
< thread_count
; i
++) {
498 * we use it as a indicator which shows if the thread is
499 * properly init'd or not
501 if (!comp_param
[i
].file
) {
505 qemu_mutex_lock(&comp_param
[i
].mutex
);
506 comp_param
[i
].quit
= true;
507 qemu_cond_signal(&comp_param
[i
].cond
);
508 qemu_mutex_unlock(&comp_param
[i
].mutex
);
510 qemu_thread_join(compress_threads
+ i
);
511 qemu_mutex_destroy(&comp_param
[i
].mutex
);
512 qemu_cond_destroy(&comp_param
[i
].cond
);
513 deflateEnd(&comp_param
[i
].stream
);
514 g_free(comp_param
[i
].originbuf
);
515 qemu_fclose(comp_param
[i
].file
);
516 comp_param
[i
].file
= NULL
;
518 qemu_mutex_destroy(&comp_done_lock
);
519 qemu_cond_destroy(&comp_done_cond
);
520 g_free(compress_threads
);
522 compress_threads
= NULL
;
526 static int compress_threads_save_setup(void)
530 if (!migrate_use_compression()) {
533 thread_count
= migrate_compress_threads();
534 compress_threads
= g_new0(QemuThread
, thread_count
);
535 comp_param
= g_new0(CompressParam
, thread_count
);
536 qemu_cond_init(&comp_done_cond
);
537 qemu_mutex_init(&comp_done_lock
);
538 for (i
= 0; i
< thread_count
; i
++) {
539 comp_param
[i
].originbuf
= g_try_malloc(TARGET_PAGE_SIZE
);
540 if (!comp_param
[i
].originbuf
) {
544 if (deflateInit(&comp_param
[i
].stream
,
545 migrate_compress_level()) != Z_OK
) {
546 g_free(comp_param
[i
].originbuf
);
550 /* comp_param[i].file is just used as a dummy buffer to save data,
551 * set its ops to empty.
553 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
554 comp_param
[i
].done
= true;
555 comp_param
[i
].quit
= false;
556 qemu_mutex_init(&comp_param
[i
].mutex
);
557 qemu_cond_init(&comp_param
[i
].cond
);
558 qemu_thread_create(compress_threads
+ i
, "compress",
559 do_data_compress
, comp_param
+ i
,
560 QEMU_THREAD_JOINABLE
);
565 compress_threads_save_cleanup();
570 * save_page_header: write page header to wire
572 * If this is the 1st block, it also writes the block identification
574 * Returns the number of bytes written
576 * @f: QEMUFile where to send the data
577 * @block: block that contains the page we want to send
578 * @offset: offset inside the block for the page
579 * in the lower bits, it contains flags
581 static size_t save_page_header(RAMState
*rs
, QEMUFile
*f
, RAMBlock
*block
,
586 if (block
== rs
->last_sent_block
) {
587 offset
|= RAM_SAVE_FLAG_CONTINUE
;
589 qemu_put_be64(f
, offset
);
592 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
593 len
= strlen(block
->idstr
);
594 qemu_put_byte(f
, len
);
595 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
597 rs
->last_sent_block
= block
;
603 * mig_throttle_guest_down: throttle down the guest
605 * Reduce amount of guest cpu execution to hopefully slow down memory
606 * writes. If guest dirty memory rate is reduced below the rate at
607 * which we can transfer pages to the destination then we should be
608 * able to complete migration. Some workloads dirty memory way too
609 * fast and will not effectively converge, even with auto-converge.
611 static void mig_throttle_guest_down(uint64_t bytes_dirty_period
,
612 uint64_t bytes_dirty_threshold
)
614 MigrationState
*s
= migrate_get_current();
615 uint64_t pct_initial
= s
->parameters
.cpu_throttle_initial
;
616 uint64_t pct_increment
= s
->parameters
.cpu_throttle_increment
;
617 bool pct_tailslow
= s
->parameters
.cpu_throttle_tailslow
;
618 int pct_max
= s
->parameters
.max_cpu_throttle
;
620 uint64_t throttle_now
= cpu_throttle_get_percentage();
621 uint64_t cpu_now
, cpu_ideal
, throttle_inc
;
623 /* We have not started throttling yet. Let's start it. */
624 if (!cpu_throttle_active()) {
625 cpu_throttle_set(pct_initial
);
627 /* Throttling already on, just increase the rate */
629 throttle_inc
= pct_increment
;
631 /* Compute the ideal CPU percentage used by Guest, which may
632 * make the dirty rate match the dirty rate threshold. */
633 cpu_now
= 100 - throttle_now
;
634 cpu_ideal
= cpu_now
* (bytes_dirty_threshold
* 1.0 /
636 throttle_inc
= MIN(cpu_now
- cpu_ideal
, pct_increment
);
638 cpu_throttle_set(MIN(throttle_now
+ throttle_inc
, pct_max
));
643 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
645 * @rs: current RAM state
646 * @current_addr: address for the zero page
648 * Update the xbzrle cache to reflect a page that's been sent as all 0.
649 * The important thing is that a stale (not-yet-0'd) page be replaced
651 * As a bonus, if the page wasn't in the cache it gets added so that
652 * when a small write is made into the 0'd page it gets XBZRLE sent.
654 static void xbzrle_cache_zero_page(RAMState
*rs
, ram_addr_t current_addr
)
656 if (!rs
->xbzrle_enabled
) {
660 /* We don't care if this fails to allocate a new cache page
661 * as long as it updated an old one */
662 cache_insert(XBZRLE
.cache
, current_addr
, XBZRLE
.zero_target_page
,
663 ram_counters
.dirty_sync_count
);
666 #define ENCODING_FLAG_XBZRLE 0x1
669 * save_xbzrle_page: compress and send current page
671 * Returns: 1 means that we wrote the page
672 * 0 means that page is identical to the one already sent
673 * -1 means that xbzrle would be longer than normal
675 * @rs: current RAM state
676 * @current_data: pointer to the address of the page contents
677 * @current_addr: addr of the page
678 * @block: block that contains the page we want to send
679 * @offset: offset inside the block for the page
680 * @last_stage: if we are at the completion stage
682 static int save_xbzrle_page(RAMState
*rs
, uint8_t **current_data
,
683 ram_addr_t current_addr
, RAMBlock
*block
,
684 ram_addr_t offset
, bool last_stage
)
686 int encoded_len
= 0, bytes_xbzrle
;
687 uint8_t *prev_cached_page
;
689 if (!cache_is_cached(XBZRLE
.cache
, current_addr
,
690 ram_counters
.dirty_sync_count
)) {
691 xbzrle_counters
.cache_miss
++;
693 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
694 ram_counters
.dirty_sync_count
) == -1) {
697 /* update *current_data when the page has been
698 inserted into cache */
699 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
706 * Reaching here means the page has hit the xbzrle cache, no matter what
707 * encoding result it is (normal encoding, overflow or skipping the page),
708 * count the page as encoded. This is used to calculate the encoding rate.
710 * Example: 2 pages (8KB) being encoded, first page encoding generates 2KB,
711 * 2nd page turns out to be skipped (i.e. no new bytes written to the
712 * page), the overall encoding rate will be 8KB / 2KB = 4, which has the
713 * skipped page included. In this way, the encoding rate can tell if the
714 * guest page is good for xbzrle encoding.
716 xbzrle_counters
.pages
++;
717 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
719 /* save current buffer into memory */
720 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
722 /* XBZRLE encoding (if there is no overflow) */
723 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
724 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
728 * Update the cache contents, so that it corresponds to the data
729 * sent, in all cases except where we skip the page.
731 if (!last_stage
&& encoded_len
!= 0) {
732 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
734 * In the case where we couldn't compress, ensure that the caller
735 * sends the data from the cache, since the guest might have
736 * changed the RAM since we copied it.
738 *current_data
= prev_cached_page
;
741 if (encoded_len
== 0) {
742 trace_save_xbzrle_page_skipping();
744 } else if (encoded_len
== -1) {
745 trace_save_xbzrle_page_overflow();
746 xbzrle_counters
.overflow
++;
747 xbzrle_counters
.bytes
+= TARGET_PAGE_SIZE
;
751 /* Send XBZRLE based compressed page */
752 bytes_xbzrle
= save_page_header(rs
, rs
->f
, block
,
753 offset
| RAM_SAVE_FLAG_XBZRLE
);
754 qemu_put_byte(rs
->f
, ENCODING_FLAG_XBZRLE
);
755 qemu_put_be16(rs
->f
, encoded_len
);
756 qemu_put_buffer(rs
->f
, XBZRLE
.encoded_buf
, encoded_len
);
757 bytes_xbzrle
+= encoded_len
+ 1 + 2;
759 * Like compressed_size (please see update_compress_thread_counts),
760 * the xbzrle encoded bytes don't count the 8 byte header with
761 * RAM_SAVE_FLAG_CONTINUE.
763 xbzrle_counters
.bytes
+= bytes_xbzrle
- 8;
764 ram_counters
.transferred
+= bytes_xbzrle
;
770 * migration_bitmap_find_dirty: find the next dirty page from start
772 * Returns the page offset within memory region of the start of a dirty page
774 * @rs: current RAM state
775 * @rb: RAMBlock where to search for dirty pages
776 * @start: page where we start the search
779 unsigned long migration_bitmap_find_dirty(RAMState
*rs
, RAMBlock
*rb
,
782 unsigned long size
= rb
->used_length
>> TARGET_PAGE_BITS
;
783 unsigned long *bitmap
= rb
->bmap
;
785 if (ramblock_is_ignored(rb
)) {
789 return find_next_bit(bitmap
, size
, start
);
792 static inline bool migration_bitmap_clear_dirty(RAMState
*rs
,
799 * Clear dirty bitmap if needed. This _must_ be called before we
800 * send any of the page in the chunk because we need to make sure
801 * we can capture further page content changes when we sync dirty
802 * log the next time. So as long as we are going to send any of
803 * the page in the chunk we clear the remote dirty bitmap for all.
804 * Clearing it earlier won't be a problem, but too late will.
806 if (rb
->clear_bmap
&& clear_bmap_test_and_clear(rb
, page
)) {
807 uint8_t shift
= rb
->clear_bmap_shift
;
808 hwaddr size
= 1ULL << (TARGET_PAGE_BITS
+ shift
);
809 hwaddr start
= (((ram_addr_t
)page
) << TARGET_PAGE_BITS
) & (-size
);
812 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
813 * can make things easier sometimes since then start address
814 * of the small chunk will always be 64 pages aligned so the
815 * bitmap will always be aligned to unsigned long. We should
816 * even be able to remove this restriction but I'm simply
820 trace_migration_bitmap_clear_dirty(rb
->idstr
, start
, size
, page
);
821 memory_region_clear_dirty_bitmap(rb
->mr
, start
, size
);
824 ret
= test_and_clear_bit(page
, rb
->bmap
);
827 rs
->migration_dirty_pages
--;
833 /* Called with RCU critical section */
834 static void ramblock_sync_dirty_bitmap(RAMState
*rs
, RAMBlock
*rb
)
836 uint64_t new_dirty_pages
=
837 cpu_physical_memory_sync_dirty_bitmap(rb
, 0, rb
->used_length
);
839 rs
->migration_dirty_pages
+= new_dirty_pages
;
840 rs
->num_dirty_pages_period
+= new_dirty_pages
;
844 * ram_pagesize_summary: calculate all the pagesizes of a VM
846 * Returns a summary bitmap of the page sizes of all RAMBlocks
848 * For VMs with just normal pages this is equivalent to the host page
849 * size. If it's got some huge pages then it's the OR of all the
850 * different page sizes.
852 uint64_t ram_pagesize_summary(void)
855 uint64_t summary
= 0;
857 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
858 summary
|= block
->page_size
;
864 uint64_t ram_get_total_transferred_pages(void)
866 return ram_counters
.normal
+ ram_counters
.duplicate
+
867 compression_counters
.pages
+ xbzrle_counters
.pages
;
870 static void migration_update_rates(RAMState
*rs
, int64_t end_time
)
872 uint64_t page_count
= rs
->target_page_count
- rs
->target_page_count_prev
;
873 double compressed_size
;
875 /* calculate period counters */
876 ram_counters
.dirty_pages_rate
= rs
->num_dirty_pages_period
* 1000
877 / (end_time
- rs
->time_last_bitmap_sync
);
883 if (migrate_use_xbzrle()) {
884 double encoded_size
, unencoded_size
;
886 xbzrle_counters
.cache_miss_rate
= (double)(xbzrle_counters
.cache_miss
-
887 rs
->xbzrle_cache_miss_prev
) / page_count
;
888 rs
->xbzrle_cache_miss_prev
= xbzrle_counters
.cache_miss
;
889 unencoded_size
= (xbzrle_counters
.pages
- rs
->xbzrle_pages_prev
) *
891 encoded_size
= xbzrle_counters
.bytes
- rs
->xbzrle_bytes_prev
;
892 if (xbzrle_counters
.pages
== rs
->xbzrle_pages_prev
|| !encoded_size
) {
893 xbzrle_counters
.encoding_rate
= 0;
895 xbzrle_counters
.encoding_rate
= unencoded_size
/ encoded_size
;
897 rs
->xbzrle_pages_prev
= xbzrle_counters
.pages
;
898 rs
->xbzrle_bytes_prev
= xbzrle_counters
.bytes
;
901 if (migrate_use_compression()) {
902 compression_counters
.busy_rate
= (double)(compression_counters
.busy
-
903 rs
->compress_thread_busy_prev
) / page_count
;
904 rs
->compress_thread_busy_prev
= compression_counters
.busy
;
906 compressed_size
= compression_counters
.compressed_size
-
907 rs
->compressed_size_prev
;
908 if (compressed_size
) {
909 double uncompressed_size
= (compression_counters
.pages
-
910 rs
->compress_pages_prev
) * TARGET_PAGE_SIZE
;
912 /* Compression-Ratio = Uncompressed-size / Compressed-size */
913 compression_counters
.compression_rate
=
914 uncompressed_size
/ compressed_size
;
916 rs
->compress_pages_prev
= compression_counters
.pages
;
917 rs
->compressed_size_prev
= compression_counters
.compressed_size
;
922 static void migration_trigger_throttle(RAMState
*rs
)
924 MigrationState
*s
= migrate_get_current();
925 uint64_t threshold
= s
->parameters
.throttle_trigger_threshold
;
927 uint64_t bytes_xfer_period
= ram_counters
.transferred
- rs
->bytes_xfer_prev
;
928 uint64_t bytes_dirty_period
= rs
->num_dirty_pages_period
* TARGET_PAGE_SIZE
;
929 uint64_t bytes_dirty_threshold
= bytes_xfer_period
* threshold
/ 100;
931 /* During block migration the auto-converge logic incorrectly detects
932 * that ram migration makes no progress. Avoid this by disabling the
933 * throttling logic during the bulk phase of block migration. */
934 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
935 /* The following detection logic can be refined later. For now:
936 Check to see if the ratio between dirtied bytes and the approx.
937 amount of bytes that just got transferred since the last time
938 we were in this routine reaches the threshold. If that happens
939 twice, start or increase throttling. */
941 if ((bytes_dirty_period
> bytes_dirty_threshold
) &&
942 (++rs
->dirty_rate_high_cnt
>= 2)) {
943 trace_migration_throttle();
944 rs
->dirty_rate_high_cnt
= 0;
945 mig_throttle_guest_down(bytes_dirty_period
,
946 bytes_dirty_threshold
);
951 static void migration_bitmap_sync(RAMState
*rs
)
956 ram_counters
.dirty_sync_count
++;
958 if (!rs
->time_last_bitmap_sync
) {
959 rs
->time_last_bitmap_sync
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
962 trace_migration_bitmap_sync_start();
963 memory_global_dirty_log_sync();
965 qemu_mutex_lock(&rs
->bitmap_mutex
);
966 WITH_RCU_READ_LOCK_GUARD() {
967 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
968 ramblock_sync_dirty_bitmap(rs
, block
);
970 ram_counters
.remaining
= ram_bytes_remaining();
972 qemu_mutex_unlock(&rs
->bitmap_mutex
);
974 memory_global_after_dirty_log_sync();
975 trace_migration_bitmap_sync_end(rs
->num_dirty_pages_period
);
977 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
979 /* more than 1 second = 1000 millisecons */
980 if (end_time
> rs
->time_last_bitmap_sync
+ 1000) {
981 migration_trigger_throttle(rs
);
983 migration_update_rates(rs
, end_time
);
985 rs
->target_page_count_prev
= rs
->target_page_count
;
987 /* reset period counters */
988 rs
->time_last_bitmap_sync
= end_time
;
989 rs
->num_dirty_pages_period
= 0;
990 rs
->bytes_xfer_prev
= ram_counters
.transferred
;
992 if (migrate_use_events()) {
993 qapi_event_send_migration_pass(ram_counters
.dirty_sync_count
);
997 static void migration_bitmap_sync_precopy(RAMState
*rs
)
999 Error
*local_err
= NULL
;
1002 * The current notifier usage is just an optimization to migration, so we
1003 * don't stop the normal migration process in the error case.
1005 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC
, &local_err
)) {
1006 error_report_err(local_err
);
1010 migration_bitmap_sync(rs
);
1012 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC
, &local_err
)) {
1013 error_report_err(local_err
);
1018 * save_zero_page_to_file: send the zero page to the file
1020 * Returns the size of data written to the file, 0 means the page is not
1023 * @rs: current RAM state
1024 * @file: the file where the data is saved
1025 * @block: block that contains the page we want to send
1026 * @offset: offset inside the block for the page
1028 static int save_zero_page_to_file(RAMState
*rs
, QEMUFile
*file
,
1029 RAMBlock
*block
, ram_addr_t offset
)
1031 uint8_t *p
= block
->host
+ offset
;
1034 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
1035 len
+= save_page_header(rs
, file
, block
, offset
| RAM_SAVE_FLAG_ZERO
);
1036 qemu_put_byte(file
, 0);
1043 * save_zero_page: send the zero page to the stream
1045 * Returns the number of pages written.
1047 * @rs: current RAM state
1048 * @block: block that contains the page we want to send
1049 * @offset: offset inside the block for the page
1051 static int save_zero_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
1053 int len
= save_zero_page_to_file(rs
, rs
->f
, block
, offset
);
1056 ram_counters
.duplicate
++;
1057 ram_counters
.transferred
+= len
;
1063 static void ram_release_pages(const char *rbname
, uint64_t offset
, int pages
)
1065 if (!migrate_release_ram() || !migration_in_postcopy()) {
1069 ram_discard_range(rbname
, offset
, ((ram_addr_t
)pages
) << TARGET_PAGE_BITS
);
1073 * @pages: the number of pages written by the control path,
1075 * > 0 - number of pages written
1077 * Return true if the pages has been saved, otherwise false is returned.
1079 static bool control_save_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
1082 uint64_t bytes_xmit
= 0;
1086 ret
= ram_control_save_page(rs
->f
, block
->offset
, offset
, TARGET_PAGE_SIZE
,
1088 if (ret
== RAM_SAVE_CONTROL_NOT_SUPP
) {
1093 ram_counters
.transferred
+= bytes_xmit
;
1097 if (ret
== RAM_SAVE_CONTROL_DELAYED
) {
1101 if (bytes_xmit
> 0) {
1102 ram_counters
.normal
++;
1103 } else if (bytes_xmit
== 0) {
1104 ram_counters
.duplicate
++;
1111 * directly send the page to the stream
1113 * Returns the number of pages written.
1115 * @rs: current RAM state
1116 * @block: block that contains the page we want to send
1117 * @offset: offset inside the block for the page
1118 * @buf: the page to be sent
1119 * @async: send to page asyncly
1121 static int save_normal_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
1122 uint8_t *buf
, bool async
)
1124 ram_counters
.transferred
+= save_page_header(rs
, rs
->f
, block
,
1125 offset
| RAM_SAVE_FLAG_PAGE
);
1127 qemu_put_buffer_async(rs
->f
, buf
, TARGET_PAGE_SIZE
,
1128 migrate_release_ram() &
1129 migration_in_postcopy());
1131 qemu_put_buffer(rs
->f
, buf
, TARGET_PAGE_SIZE
);
1133 ram_counters
.transferred
+= TARGET_PAGE_SIZE
;
1134 ram_counters
.normal
++;
1139 * ram_save_page: send the given page to the stream
1141 * Returns the number of pages written.
1143 * >=0 - Number of pages written - this might legally be 0
1144 * if xbzrle noticed the page was the same.
1146 * @rs: current RAM state
1147 * @block: block that contains the page we want to send
1148 * @offset: offset inside the block for the page
1149 * @last_stage: if we are at the completion stage
1151 static int ram_save_page(RAMState
*rs
, PageSearchStatus
*pss
, bool last_stage
)
1155 bool send_async
= true;
1156 RAMBlock
*block
= pss
->block
;
1157 ram_addr_t offset
= ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
;
1158 ram_addr_t current_addr
= block
->offset
+ offset
;
1160 p
= block
->host
+ offset
;
1161 trace_ram_save_page(block
->idstr
, (uint64_t)offset
, p
);
1163 XBZRLE_cache_lock();
1164 if (rs
->xbzrle_enabled
&& !migration_in_postcopy()) {
1165 pages
= save_xbzrle_page(rs
, &p
, current_addr
, block
,
1166 offset
, last_stage
);
1168 /* Can't send this cached data async, since the cache page
1169 * might get updated before it gets to the wire
1175 /* XBZRLE overflow or normal page */
1177 pages
= save_normal_page(rs
, block
, offset
, p
, send_async
);
1180 XBZRLE_cache_unlock();
1185 static int ram_save_multifd_page(RAMState
*rs
, RAMBlock
*block
,
1188 if (multifd_queue_page(rs
->f
, block
, offset
) < 0) {
1191 ram_counters
.normal
++;
1196 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
1197 ram_addr_t offset
, uint8_t *source_buf
)
1199 RAMState
*rs
= ram_state
;
1200 uint8_t *p
= block
->host
+ (offset
& TARGET_PAGE_MASK
);
1201 bool zero_page
= false;
1204 if (save_zero_page_to_file(rs
, f
, block
, offset
)) {
1209 save_page_header(rs
, f
, block
, offset
| RAM_SAVE_FLAG_COMPRESS_PAGE
);
1212 * copy it to a internal buffer to avoid it being modified by VM
1213 * so that we can catch up the error during compression and
1216 memcpy(source_buf
, p
, TARGET_PAGE_SIZE
);
1217 ret
= qemu_put_compression_data(f
, stream
, source_buf
, TARGET_PAGE_SIZE
);
1219 qemu_file_set_error(migrate_get_current()->to_dst_file
, ret
);
1220 error_report("compressed data failed!");
1225 ram_release_pages(block
->idstr
, offset
& TARGET_PAGE_MASK
, 1);
1230 update_compress_thread_counts(const CompressParam
*param
, int bytes_xmit
)
1232 ram_counters
.transferred
+= bytes_xmit
;
1234 if (param
->zero_page
) {
1235 ram_counters
.duplicate
++;
1239 /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
1240 compression_counters
.compressed_size
+= bytes_xmit
- 8;
1241 compression_counters
.pages
++;
1244 static bool save_page_use_compression(RAMState
*rs
);
1246 static void flush_compressed_data(RAMState
*rs
)
1248 int idx
, len
, thread_count
;
1250 if (!save_page_use_compression(rs
)) {
1253 thread_count
= migrate_compress_threads();
1255 qemu_mutex_lock(&comp_done_lock
);
1256 for (idx
= 0; idx
< thread_count
; idx
++) {
1257 while (!comp_param
[idx
].done
) {
1258 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
1261 qemu_mutex_unlock(&comp_done_lock
);
1263 for (idx
= 0; idx
< thread_count
; idx
++) {
1264 qemu_mutex_lock(&comp_param
[idx
].mutex
);
1265 if (!comp_param
[idx
].quit
) {
1266 len
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
1268 * it's safe to fetch zero_page without holding comp_done_lock
1269 * as there is no further request submitted to the thread,
1270 * i.e, the thread should be waiting for a request at this point.
1272 update_compress_thread_counts(&comp_param
[idx
], len
);
1274 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
1278 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
1281 param
->block
= block
;
1282 param
->offset
= offset
;
1285 static int compress_page_with_multi_thread(RAMState
*rs
, RAMBlock
*block
,
1288 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
1289 bool wait
= migrate_compress_wait_thread();
1291 thread_count
= migrate_compress_threads();
1292 qemu_mutex_lock(&comp_done_lock
);
1294 for (idx
= 0; idx
< thread_count
; idx
++) {
1295 if (comp_param
[idx
].done
) {
1296 comp_param
[idx
].done
= false;
1297 bytes_xmit
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
1298 qemu_mutex_lock(&comp_param
[idx
].mutex
);
1299 set_compress_params(&comp_param
[idx
], block
, offset
);
1300 qemu_cond_signal(&comp_param
[idx
].cond
);
1301 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
1303 update_compress_thread_counts(&comp_param
[idx
], bytes_xmit
);
1309 * wait for the free thread if the user specifies 'compress-wait-thread',
1310 * otherwise we will post the page out in the main thread as normal page.
1312 if (pages
< 0 && wait
) {
1313 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
1316 qemu_mutex_unlock(&comp_done_lock
);
1322 * find_dirty_block: find the next dirty page and update any state
1323 * associated with the search process.
1325 * Returns true if a page is found
1327 * @rs: current RAM state
1328 * @pss: data about the state of the current dirty page scan
1329 * @again: set to false if the search has scanned the whole of RAM
1331 static bool find_dirty_block(RAMState
*rs
, PageSearchStatus
*pss
, bool *again
)
1333 pss
->page
= migration_bitmap_find_dirty(rs
, pss
->block
, pss
->page
);
1334 if (pss
->complete_round
&& pss
->block
== rs
->last_seen_block
&&
1335 pss
->page
>= rs
->last_page
) {
1337 * We've been once around the RAM and haven't found anything.
1343 if (!offset_in_ramblock(pss
->block
,
1344 ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
)) {
1345 /* Didn't find anything in this RAM Block */
1347 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
1350 * If memory migration starts over, we will meet a dirtied page
1351 * which may still exists in compression threads's ring, so we
1352 * should flush the compressed data to make sure the new page
1353 * is not overwritten by the old one in the destination.
1355 * Also If xbzrle is on, stop using the data compression at this
1356 * point. In theory, xbzrle can do better than compression.
1358 flush_compressed_data(rs
);
1360 /* Hit the end of the list */
1361 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1362 /* Flag that we've looped */
1363 pss
->complete_round
= true;
1364 /* After the first round, enable XBZRLE. */
1365 if (migrate_use_xbzrle()) {
1366 rs
->xbzrle_enabled
= true;
1369 /* Didn't find anything this time, but try again on the new block */
1373 /* Can go around again, but... */
1375 /* We've found something so probably don't need to */
1381 * unqueue_page: gets a page of the queue
1383 * Helper for 'get_queued_page' - gets a page off the queue
1385 * Returns the block of the page (or NULL if none available)
1387 * @rs: current RAM state
1388 * @offset: used to return the offset within the RAMBlock
1390 static RAMBlock
*unqueue_page(RAMState
*rs
, ram_addr_t
*offset
)
1392 RAMBlock
*block
= NULL
;
1394 if (QSIMPLEQ_EMPTY_ATOMIC(&rs
->src_page_requests
)) {
1398 QEMU_LOCK_GUARD(&rs
->src_page_req_mutex
);
1399 if (!QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
1400 struct RAMSrcPageRequest
*entry
=
1401 QSIMPLEQ_FIRST(&rs
->src_page_requests
);
1403 *offset
= entry
->offset
;
1405 if (entry
->len
> TARGET_PAGE_SIZE
) {
1406 entry
->len
-= TARGET_PAGE_SIZE
;
1407 entry
->offset
+= TARGET_PAGE_SIZE
;
1409 memory_region_unref(block
->mr
);
1410 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
1412 migration_consume_urgent_request();
1419 #if defined(__linux__)
1421 * poll_fault_page: try to get next UFFD write fault page and, if pending fault
1422 * is found, return RAM block pointer and page offset
1424 * Returns pointer to the RAMBlock containing faulting page,
1425 * NULL if no write faults are pending
1427 * @rs: current RAM state
1428 * @offset: page offset from the beginning of the block
1430 static RAMBlock
*poll_fault_page(RAMState
*rs
, ram_addr_t
*offset
)
1432 struct uffd_msg uffd_msg
;
1437 if (!migrate_background_snapshot()) {
1441 res
= uffd_read_events(rs
->uffdio_fd
, &uffd_msg
, 1);
1446 page_address
= (void *)(uintptr_t) uffd_msg
.arg
.pagefault
.address
;
1447 block
= qemu_ram_block_from_host(page_address
, false, offset
);
1448 assert(block
&& (block
->flags
& RAM_UF_WRITEPROTECT
) != 0);
1453 * ram_save_release_protection: release UFFD write protection after
1454 * a range of pages has been saved
1456 * @rs: current RAM state
1457 * @pss: page-search-status structure
1458 * @start_page: index of the first page in the range relative to pss->block
1460 * Returns 0 on success, negative value in case of an error
1462 static int ram_save_release_protection(RAMState
*rs
, PageSearchStatus
*pss
,
1463 unsigned long start_page
)
1467 /* Check if page is from UFFD-managed region. */
1468 if (pss
->block
->flags
& RAM_UF_WRITEPROTECT
) {
1469 void *page_address
= pss
->block
->host
+ (start_page
<< TARGET_PAGE_BITS
);
1470 uint64_t run_length
= (pss
->page
- start_page
+ 1) << TARGET_PAGE_BITS
;
1472 /* Flush async buffers before un-protect. */
1474 /* Un-protect memory range. */
1475 res
= uffd_change_protection(rs
->uffdio_fd
, page_address
, run_length
,
1482 /* ram_write_tracking_available: check if kernel supports required UFFD features
1484 * Returns true if supports, false otherwise
1486 bool ram_write_tracking_available(void)
1488 uint64_t uffd_features
;
1491 res
= uffd_query_features(&uffd_features
);
1493 (uffd_features
& UFFD_FEATURE_PAGEFAULT_FLAG_WP
) != 0);
1496 /* ram_write_tracking_compatible: check if guest configuration is
1497 * compatible with 'write-tracking'
1499 * Returns true if compatible, false otherwise
1501 bool ram_write_tracking_compatible(void)
1503 const uint64_t uffd_ioctls_mask
= BIT(_UFFDIO_WRITEPROTECT
);
1508 /* Open UFFD file descriptor */
1509 uffd_fd
= uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP
, false);
1514 RCU_READ_LOCK_GUARD();
1516 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1517 uint64_t uffd_ioctls
;
1519 /* Nothing to do with read-only and MMIO-writable regions */
1520 if (block
->mr
->readonly
|| block
->mr
->rom_device
) {
1523 /* Try to register block memory via UFFD-IO to track writes */
1524 if (uffd_register_memory(uffd_fd
, block
->host
, block
->max_length
,
1525 UFFDIO_REGISTER_MODE_WP
, &uffd_ioctls
)) {
1528 if ((uffd_ioctls
& uffd_ioctls_mask
) != uffd_ioctls_mask
) {
1535 uffd_close_fd(uffd_fd
);
1540 * ram_block_populate_pages: populate memory in the RAM block by reading
1541 * an integer from the beginning of each page.
1543 * Since it's solely used for userfault_fd WP feature, here we just
1544 * hardcode page size to qemu_real_host_page_size.
1546 * @block: RAM block to populate
1548 static void ram_block_populate_pages(RAMBlock
*block
)
1550 char *ptr
= (char *) block
->host
;
1552 for (ram_addr_t offset
= 0; offset
< block
->used_length
;
1553 offset
+= qemu_real_host_page_size
) {
1554 char tmp
= *(ptr
+ offset
);
1556 /* Don't optimize the read out */
1557 asm volatile("" : "+r" (tmp
));
1562 * ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
1564 void ram_write_tracking_prepare(void)
1568 RCU_READ_LOCK_GUARD();
1570 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1571 /* Nothing to do with read-only and MMIO-writable regions */
1572 if (block
->mr
->readonly
|| block
->mr
->rom_device
) {
1577 * Populate pages of the RAM block before enabling userfault_fd
1580 * This stage is required since ioctl(UFFDIO_WRITEPROTECT) with
1581 * UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
1582 * pages with pte_none() entries in page table.
1584 ram_block_populate_pages(block
);
1589 * ram_write_tracking_start: start UFFD-WP memory tracking
1591 * Returns 0 for success or negative value in case of error
1593 int ram_write_tracking_start(void)
1596 RAMState
*rs
= ram_state
;
1599 /* Open UFFD file descriptor */
1600 uffd_fd
= uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP
, true);
1604 rs
->uffdio_fd
= uffd_fd
;
1606 RCU_READ_LOCK_GUARD();
1608 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1609 /* Nothing to do with read-only and MMIO-writable regions */
1610 if (block
->mr
->readonly
|| block
->mr
->rom_device
) {
1614 /* Register block memory with UFFD to track writes */
1615 if (uffd_register_memory(rs
->uffdio_fd
, block
->host
,
1616 block
->max_length
, UFFDIO_REGISTER_MODE_WP
, NULL
)) {
1619 /* Apply UFFD write protection to the block memory range */
1620 if (uffd_change_protection(rs
->uffdio_fd
, block
->host
,
1621 block
->max_length
, true, false)) {
1624 block
->flags
|= RAM_UF_WRITEPROTECT
;
1625 memory_region_ref(block
->mr
);
1627 trace_ram_write_tracking_ramblock_start(block
->idstr
, block
->page_size
,
1628 block
->host
, block
->max_length
);
1634 error_report("ram_write_tracking_start() failed: restoring initial memory state");
1636 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1637 if ((block
->flags
& RAM_UF_WRITEPROTECT
) == 0) {
1641 * In case some memory block failed to be write-protected
1642 * remove protection and unregister all succeeded RAM blocks
1644 uffd_change_protection(rs
->uffdio_fd
, block
->host
, block
->max_length
,
1646 uffd_unregister_memory(rs
->uffdio_fd
, block
->host
, block
->max_length
);
1647 /* Cleanup flags and remove reference */
1648 block
->flags
&= ~RAM_UF_WRITEPROTECT
;
1649 memory_region_unref(block
->mr
);
1652 uffd_close_fd(uffd_fd
);
1658 * ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
1660 void ram_write_tracking_stop(void)
1662 RAMState
*rs
= ram_state
;
1665 RCU_READ_LOCK_GUARD();
1667 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1668 if ((block
->flags
& RAM_UF_WRITEPROTECT
) == 0) {
1671 /* Remove protection and unregister all affected RAM blocks */
1672 uffd_change_protection(rs
->uffdio_fd
, block
->host
, block
->max_length
,
1674 uffd_unregister_memory(rs
->uffdio_fd
, block
->host
, block
->max_length
);
1676 trace_ram_write_tracking_ramblock_stop(block
->idstr
, block
->page_size
,
1677 block
->host
, block
->max_length
);
1679 /* Cleanup flags and remove reference */
1680 block
->flags
&= ~RAM_UF_WRITEPROTECT
;
1681 memory_region_unref(block
->mr
);
1684 /* Finally close UFFD file descriptor */
1685 uffd_close_fd(rs
->uffdio_fd
);
1690 /* No target OS support, stubs just fail or ignore */
1692 static RAMBlock
*poll_fault_page(RAMState
*rs
, ram_addr_t
*offset
)
1700 static int ram_save_release_protection(RAMState
*rs
, PageSearchStatus
*pss
,
1701 unsigned long start_page
)
1710 bool ram_write_tracking_available(void)
1715 bool ram_write_tracking_compatible(void)
1721 int ram_write_tracking_start(void)
1727 void ram_write_tracking_stop(void)
1731 #endif /* defined(__linux__) */
1734 * get_queued_page: unqueue a page from the postcopy requests
1736 * Skips pages that are already sent (!dirty)
1738 * Returns true if a queued page is found
1740 * @rs: current RAM state
1741 * @pss: data about the state of the current dirty page scan
1743 static bool get_queued_page(RAMState
*rs
, PageSearchStatus
*pss
)
1750 block
= unqueue_page(rs
, &offset
);
1752 * We're sending this page, and since it's postcopy nothing else
1753 * will dirty it, and we must make sure it doesn't get sent again
1754 * even if this queue request was received after the background
1755 * search already sent it.
1760 page
= offset
>> TARGET_PAGE_BITS
;
1761 dirty
= test_bit(page
, block
->bmap
);
1763 trace_get_queued_page_not_dirty(block
->idstr
, (uint64_t)offset
,
1766 trace_get_queued_page(block
->idstr
, (uint64_t)offset
, page
);
1770 } while (block
&& !dirty
);
1774 * Poll write faults too if background snapshot is enabled; that's
1775 * when we have vcpus got blocked by the write protected pages.
1777 block
= poll_fault_page(rs
, &offset
);
1782 * We want the background search to continue from the queued page
1783 * since the guest is likely to want other pages near to the page
1784 * it just requested.
1787 pss
->page
= offset
>> TARGET_PAGE_BITS
;
1790 * This unqueued page would break the "one round" check, even is
1793 pss
->complete_round
= false;
1800 * migration_page_queue_free: drop any remaining pages in the ram
1803 * It should be empty at the end anyway, but in error cases there may
1804 * be some left. in case that there is any page left, we drop it.
1807 static void migration_page_queue_free(RAMState
*rs
)
1809 struct RAMSrcPageRequest
*mspr
, *next_mspr
;
1810 /* This queue generally should be empty - but in the case of a failed
1811 * migration might have some droppings in.
1813 RCU_READ_LOCK_GUARD();
1814 QSIMPLEQ_FOREACH_SAFE(mspr
, &rs
->src_page_requests
, next_req
, next_mspr
) {
1815 memory_region_unref(mspr
->rb
->mr
);
1816 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
1822 * ram_save_queue_pages: queue the page for transmission
1824 * A request from postcopy destination for example.
1826 * Returns zero on success or negative on error
1828 * @rbname: Name of the RAMBLock of the request. NULL means the
1829 * same that last one.
1830 * @start: starting address from the start of the RAMBlock
1831 * @len: length (in bytes) to send
1833 int ram_save_queue_pages(const char *rbname
, ram_addr_t start
, ram_addr_t len
)
1836 RAMState
*rs
= ram_state
;
1838 ram_counters
.postcopy_requests
++;
1839 RCU_READ_LOCK_GUARD();
1842 /* Reuse last RAMBlock */
1843 ramblock
= rs
->last_req_rb
;
1847 * Shouldn't happen, we can't reuse the last RAMBlock if
1848 * it's the 1st request.
1850 error_report("ram_save_queue_pages no previous block");
1854 ramblock
= qemu_ram_block_by_name(rbname
);
1857 /* We shouldn't be asked for a non-existent RAMBlock */
1858 error_report("ram_save_queue_pages no block '%s'", rbname
);
1861 rs
->last_req_rb
= ramblock
;
1863 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
1864 if (!offset_in_ramblock(ramblock
, start
+ len
- 1)) {
1865 error_report("%s request overrun start=" RAM_ADDR_FMT
" len="
1866 RAM_ADDR_FMT
" blocklen=" RAM_ADDR_FMT
,
1867 __func__
, start
, len
, ramblock
->used_length
);
1871 struct RAMSrcPageRequest
*new_entry
=
1872 g_malloc0(sizeof(struct RAMSrcPageRequest
));
1873 new_entry
->rb
= ramblock
;
1874 new_entry
->offset
= start
;
1875 new_entry
->len
= len
;
1877 memory_region_ref(ramblock
->mr
);
1878 qemu_mutex_lock(&rs
->src_page_req_mutex
);
1879 QSIMPLEQ_INSERT_TAIL(&rs
->src_page_requests
, new_entry
, next_req
);
1880 migration_make_urgent_request();
1881 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
1886 static bool save_page_use_compression(RAMState
*rs
)
1888 if (!migrate_use_compression()) {
1893 * If xbzrle is enabled (e.g., after first round of migration), stop
1894 * using the data compression. In theory, xbzrle can do better than
1897 if (rs
->xbzrle_enabled
) {
1905 * try to compress the page before posting it out, return true if the page
1906 * has been properly handled by compression, otherwise needs other
1907 * paths to handle it
1909 static bool save_compress_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
1911 if (!save_page_use_compression(rs
)) {
1916 * When starting the process of a new block, the first page of
1917 * the block should be sent out before other pages in the same
1918 * block, and all the pages in last block should have been sent
1919 * out, keeping this order is important, because the 'cont' flag
1920 * is used to avoid resending the block name.
1922 * We post the fist page as normal page as compression will take
1923 * much CPU resource.
1925 if (block
!= rs
->last_sent_block
) {
1926 flush_compressed_data(rs
);
1930 if (compress_page_with_multi_thread(rs
, block
, offset
) > 0) {
1934 compression_counters
.busy
++;
1939 * ram_save_target_page: save one target page
1941 * Returns the number of pages written
1943 * @rs: current RAM state
1944 * @pss: data about the page we want to send
1945 * @last_stage: if we are at the completion stage
1947 static int ram_save_target_page(RAMState
*rs
, PageSearchStatus
*pss
,
1950 RAMBlock
*block
= pss
->block
;
1951 ram_addr_t offset
= ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
;
1954 if (control_save_page(rs
, block
, offset
, &res
)) {
1958 if (save_compress_page(rs
, block
, offset
)) {
1962 res
= save_zero_page(rs
, block
, offset
);
1964 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
1965 * page would be stale
1967 if (!save_page_use_compression(rs
)) {
1968 XBZRLE_cache_lock();
1969 xbzrle_cache_zero_page(rs
, block
->offset
+ offset
);
1970 XBZRLE_cache_unlock();
1972 ram_release_pages(block
->idstr
, offset
, res
);
1977 * Do not use multifd for:
1978 * 1. Compression as the first page in the new block should be posted out
1979 * before sending the compressed page
1980 * 2. In postcopy as one whole host page should be placed
1982 if (!save_page_use_compression(rs
) && migrate_use_multifd()
1983 && !migration_in_postcopy()) {
1984 return ram_save_multifd_page(rs
, block
, offset
);
1987 return ram_save_page(rs
, pss
, last_stage
);
1991 * ram_save_host_page: save a whole host page
1993 * Starting at *offset send pages up to the end of the current host
1994 * page. It's valid for the initial offset to point into the middle of
1995 * a host page in which case the remainder of the hostpage is sent.
1996 * Only dirty target pages are sent. Note that the host page size may
1997 * be a huge page for this block.
1998 * The saving stops at the boundary of the used_length of the block
1999 * if the RAMBlock isn't a multiple of the host page size.
2001 * Returns the number of pages written or negative on error
2003 * @rs: current RAM state
2004 * @ms: current migration state
2005 * @pss: data about the page we want to send
2006 * @last_stage: if we are at the completion stage
2008 static int ram_save_host_page(RAMState
*rs
, PageSearchStatus
*pss
,
2011 int tmppages
, pages
= 0;
2012 size_t pagesize_bits
=
2013 qemu_ram_pagesize(pss
->block
) >> TARGET_PAGE_BITS
;
2014 unsigned long hostpage_boundary
=
2015 QEMU_ALIGN_UP(pss
->page
+ 1, pagesize_bits
);
2016 unsigned long start_page
= pss
->page
;
2019 if (ramblock_is_ignored(pss
->block
)) {
2020 error_report("block %s should not be migrated !", pss
->block
->idstr
);
2025 /* Check the pages is dirty and if it is send it */
2026 if (migration_bitmap_clear_dirty(rs
, pss
->block
, pss
->page
)) {
2027 tmppages
= ram_save_target_page(rs
, pss
, last_stage
);
2034 * Allow rate limiting to happen in the middle of huge pages if
2035 * something is sent in the current iteration.
2037 if (pagesize_bits
> 1 && tmppages
> 0) {
2038 migration_rate_limit();
2041 pss
->page
= migration_bitmap_find_dirty(rs
, pss
->block
, pss
->page
);
2042 } while ((pss
->page
< hostpage_boundary
) &&
2043 offset_in_ramblock(pss
->block
,
2044 ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
));
2045 /* The offset we leave with is the min boundary of host page and block */
2046 pss
->page
= MIN(pss
->page
, hostpage_boundary
) - 1;
2048 res
= ram_save_release_protection(rs
, pss
, start_page
);
2049 return (res
< 0 ? res
: pages
);
2053 * ram_find_and_save_block: finds a dirty page and sends it to f
2055 * Called within an RCU critical section.
2057 * Returns the number of pages written where zero means no dirty pages,
2058 * or negative on error
2060 * @rs: current RAM state
2061 * @last_stage: if we are at the completion stage
2063 * On systems where host-page-size > target-page-size it will send all the
2064 * pages in a host page that are dirty.
2067 static int ram_find_and_save_block(RAMState
*rs
, bool last_stage
)
2069 PageSearchStatus pss
;
2073 /* No dirty page as there is zero RAM */
2074 if (!ram_bytes_total()) {
2078 pss
.block
= rs
->last_seen_block
;
2079 pss
.page
= rs
->last_page
;
2080 pss
.complete_round
= false;
2083 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2088 found
= get_queued_page(rs
, &pss
);
2091 /* priority queue empty, so just search for something dirty */
2092 found
= find_dirty_block(rs
, &pss
, &again
);
2096 pages
= ram_save_host_page(rs
, &pss
, last_stage
);
2098 } while (!pages
&& again
);
2100 rs
->last_seen_block
= pss
.block
;
2101 rs
->last_page
= pss
.page
;
2106 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
2108 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
2111 ram_counters
.duplicate
+= pages
;
2113 ram_counters
.normal
+= pages
;
2114 ram_counters
.transferred
+= size
;
2115 qemu_update_position(f
, size
);
2119 static uint64_t ram_bytes_total_common(bool count_ignored
)
2124 RCU_READ_LOCK_GUARD();
2126 if (count_ignored
) {
2127 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
2128 total
+= block
->used_length
;
2131 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2132 total
+= block
->used_length
;
2138 uint64_t ram_bytes_total(void)
2140 return ram_bytes_total_common(false);
2143 static void xbzrle_load_setup(void)
2145 XBZRLE
.decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
2148 static void xbzrle_load_cleanup(void)
2150 g_free(XBZRLE
.decoded_buf
);
2151 XBZRLE
.decoded_buf
= NULL
;
2154 static void ram_state_cleanup(RAMState
**rsp
)
2157 migration_page_queue_free(*rsp
);
2158 qemu_mutex_destroy(&(*rsp
)->bitmap_mutex
);
2159 qemu_mutex_destroy(&(*rsp
)->src_page_req_mutex
);
2165 static void xbzrle_cleanup(void)
2167 XBZRLE_cache_lock();
2169 cache_fini(XBZRLE
.cache
);
2170 g_free(XBZRLE
.encoded_buf
);
2171 g_free(XBZRLE
.current_buf
);
2172 g_free(XBZRLE
.zero_target_page
);
2173 XBZRLE
.cache
= NULL
;
2174 XBZRLE
.encoded_buf
= NULL
;
2175 XBZRLE
.current_buf
= NULL
;
2176 XBZRLE
.zero_target_page
= NULL
;
2178 XBZRLE_cache_unlock();
2181 static void ram_save_cleanup(void *opaque
)
2183 RAMState
**rsp
= opaque
;
2186 /* We don't use dirty log with background snapshots */
2187 if (!migrate_background_snapshot()) {
2188 /* caller have hold iothread lock or is in a bh, so there is
2189 * no writing race against the migration bitmap
2191 memory_global_dirty_log_stop();
2194 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2195 g_free(block
->clear_bmap
);
2196 block
->clear_bmap
= NULL
;
2197 g_free(block
->bmap
);
2202 compress_threads_save_cleanup();
2203 ram_state_cleanup(rsp
);
2206 static void ram_state_reset(RAMState
*rs
)
2208 rs
->last_seen_block
= NULL
;
2209 rs
->last_sent_block
= NULL
;
2211 rs
->last_version
= ram_list
.version
;
2212 rs
->xbzrle_enabled
= false;
2215 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2218 * 'expected' is the value you expect the bitmap mostly to be full
2219 * of; it won't bother printing lines that are all this value.
2220 * If 'todump' is null the migration bitmap is dumped.
2222 void ram_debug_dump_bitmap(unsigned long *todump
, bool expected
,
2223 unsigned long pages
)
2226 int64_t linelen
= 128;
2229 for (cur
= 0; cur
< pages
; cur
+= linelen
) {
2233 * Last line; catch the case where the line length
2234 * is longer than remaining ram
2236 if (cur
+ linelen
> pages
) {
2237 linelen
= pages
- cur
;
2239 for (curb
= 0; curb
< linelen
; curb
++) {
2240 bool thisbit
= test_bit(cur
+ curb
, todump
);
2241 linebuf
[curb
] = thisbit
? '1' : '.';
2242 found
= found
|| (thisbit
!= expected
);
2245 linebuf
[curb
] = '\0';
2246 fprintf(stderr
, "0x%08" PRIx64
" : %s\n", cur
, linebuf
);
2251 /* **** functions for postcopy ***** */
2253 void ram_postcopy_migrated_memory_release(MigrationState
*ms
)
2255 struct RAMBlock
*block
;
2257 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2258 unsigned long *bitmap
= block
->bmap
;
2259 unsigned long range
= block
->used_length
>> TARGET_PAGE_BITS
;
2260 unsigned long run_start
= find_next_zero_bit(bitmap
, range
, 0);
2262 while (run_start
< range
) {
2263 unsigned long run_end
= find_next_bit(bitmap
, range
, run_start
+ 1);
2264 ram_discard_range(block
->idstr
,
2265 ((ram_addr_t
)run_start
) << TARGET_PAGE_BITS
,
2266 ((ram_addr_t
)(run_end
- run_start
))
2267 << TARGET_PAGE_BITS
);
2268 run_start
= find_next_zero_bit(bitmap
, range
, run_end
+ 1);
2274 * postcopy_send_discard_bm_ram: discard a RAMBlock
2276 * Returns zero on success
2278 * Callback from postcopy_each_ram_send_discard for each RAMBlock
2280 * @ms: current migration state
2281 * @block: RAMBlock to discard
2283 static int postcopy_send_discard_bm_ram(MigrationState
*ms
, RAMBlock
*block
)
2285 unsigned long end
= block
->used_length
>> TARGET_PAGE_BITS
;
2286 unsigned long current
;
2287 unsigned long *bitmap
= block
->bmap
;
2289 for (current
= 0; current
< end
; ) {
2290 unsigned long one
= find_next_bit(bitmap
, end
, current
);
2291 unsigned long zero
, discard_length
;
2297 zero
= find_next_zero_bit(bitmap
, end
, one
+ 1);
2300 discard_length
= end
- one
;
2302 discard_length
= zero
- one
;
2304 postcopy_discard_send_range(ms
, one
, discard_length
);
2305 current
= one
+ discard_length
;
2312 * postcopy_each_ram_send_discard: discard all RAMBlocks
2314 * Returns 0 for success or negative for error
2316 * Utility for the outgoing postcopy code.
2317 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2318 * passing it bitmap indexes and name.
2319 * (qemu_ram_foreach_block ends up passing unscaled lengths
2320 * which would mean postcopy code would have to deal with target page)
2322 * @ms: current migration state
2324 static int postcopy_each_ram_send_discard(MigrationState
*ms
)
2326 struct RAMBlock
*block
;
2329 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2330 postcopy_discard_send_init(ms
, block
->idstr
);
2333 * Postcopy sends chunks of bitmap over the wire, but it
2334 * just needs indexes at this point, avoids it having
2335 * target page specific code.
2337 ret
= postcopy_send_discard_bm_ram(ms
, block
);
2338 postcopy_discard_send_finish(ms
);
2348 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2350 * Helper for postcopy_chunk_hostpages; it's called twice to
2351 * canonicalize the two bitmaps, that are similar, but one is
2354 * Postcopy requires that all target pages in a hostpage are dirty or
2355 * clean, not a mix. This function canonicalizes the bitmaps.
2357 * @ms: current migration state
2358 * @block: block that contains the page we want to canonicalize
2360 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, RAMBlock
*block
)
2362 RAMState
*rs
= ram_state
;
2363 unsigned long *bitmap
= block
->bmap
;
2364 unsigned int host_ratio
= block
->page_size
/ TARGET_PAGE_SIZE
;
2365 unsigned long pages
= block
->used_length
>> TARGET_PAGE_BITS
;
2366 unsigned long run_start
;
2368 if (block
->page_size
== TARGET_PAGE_SIZE
) {
2369 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2373 /* Find a dirty page */
2374 run_start
= find_next_bit(bitmap
, pages
, 0);
2376 while (run_start
< pages
) {
2379 * If the start of this run of pages is in the middle of a host
2380 * page, then we need to fixup this host page.
2382 if (QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2383 /* Find the end of this run */
2384 run_start
= find_next_zero_bit(bitmap
, pages
, run_start
+ 1);
2386 * If the end isn't at the start of a host page, then the
2387 * run doesn't finish at the end of a host page
2388 * and we need to discard.
2392 if (!QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2394 unsigned long fixup_start_addr
= QEMU_ALIGN_DOWN(run_start
,
2396 run_start
= QEMU_ALIGN_UP(run_start
, host_ratio
);
2398 /* Clean up the bitmap */
2399 for (page
= fixup_start_addr
;
2400 page
< fixup_start_addr
+ host_ratio
; page
++) {
2402 * Remark them as dirty, updating the count for any pages
2403 * that weren't previously dirty.
2405 rs
->migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
2409 /* Find the next dirty page for the next iteration */
2410 run_start
= find_next_bit(bitmap
, pages
, run_start
);
2415 * postcopy_chunk_hostpages: discard any partially sent host page
2417 * Utility for the outgoing postcopy code.
2419 * Discard any partially sent host-page size chunks, mark any partially
2420 * dirty host-page size chunks as all dirty. In this case the host-page
2421 * is the host-page for the particular RAMBlock, i.e. it might be a huge page
2423 * Returns zero on success
2425 * @ms: current migration state
2426 * @block: block we want to work with
2428 static int postcopy_chunk_hostpages(MigrationState
*ms
, RAMBlock
*block
)
2430 postcopy_discard_send_init(ms
, block
->idstr
);
2433 * Ensure that all partially dirty host pages are made fully dirty.
2435 postcopy_chunk_hostpages_pass(ms
, block
);
2437 postcopy_discard_send_finish(ms
);
2442 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2444 * Returns zero on success
2446 * Transmit the set of pages to be discarded after precopy to the target
2447 * these are pages that:
2448 * a) Have been previously transmitted but are now dirty again
2449 * b) Pages that have never been transmitted, this ensures that
2450 * any pages on the destination that have been mapped by background
2451 * tasks get discarded (transparent huge pages is the specific concern)
2452 * Hopefully this is pretty sparse
2454 * @ms: current migration state
2456 int ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
2458 RAMState
*rs
= ram_state
;
2462 RCU_READ_LOCK_GUARD();
2464 /* This should be our last sync, the src is now paused */
2465 migration_bitmap_sync(rs
);
2467 /* Easiest way to make sure we don't resume in the middle of a host-page */
2468 rs
->last_seen_block
= NULL
;
2469 rs
->last_sent_block
= NULL
;
2472 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2473 /* Deal with TPS != HPS and huge pages */
2474 ret
= postcopy_chunk_hostpages(ms
, block
);
2479 #ifdef DEBUG_POSTCOPY
2480 ram_debug_dump_bitmap(block
->bmap
, true,
2481 block
->used_length
>> TARGET_PAGE_BITS
);
2484 trace_ram_postcopy_send_discard_bitmap();
2486 return postcopy_each_ram_send_discard(ms
);
2490 * ram_discard_range: discard dirtied pages at the beginning of postcopy
2492 * Returns zero on success
2494 * @rbname: name of the RAMBlock of the request. NULL means the
2495 * same that last one.
2496 * @start: RAMBlock starting page
2497 * @length: RAMBlock size
2499 int ram_discard_range(const char *rbname
, uint64_t start
, size_t length
)
2501 trace_ram_discard_range(rbname
, start
, length
);
2503 RCU_READ_LOCK_GUARD();
2504 RAMBlock
*rb
= qemu_ram_block_by_name(rbname
);
2507 error_report("ram_discard_range: Failed to find block '%s'", rbname
);
2512 * On source VM, we don't need to update the received bitmap since
2513 * we don't even have one.
2515 if (rb
->receivedmap
) {
2516 bitmap_clear(rb
->receivedmap
, start
>> qemu_target_page_bits(),
2517 length
>> qemu_target_page_bits());
2520 return ram_block_discard_range(rb
, start
, length
);
2524 * For every allocation, we will try not to crash the VM if the
2525 * allocation failed.
2527 static int xbzrle_init(void)
2529 Error
*local_err
= NULL
;
2531 if (!migrate_use_xbzrle()) {
2535 XBZRLE_cache_lock();
2537 XBZRLE
.zero_target_page
= g_try_malloc0(TARGET_PAGE_SIZE
);
2538 if (!XBZRLE
.zero_target_page
) {
2539 error_report("%s: Error allocating zero page", __func__
);
2543 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size(),
2544 TARGET_PAGE_SIZE
, &local_err
);
2545 if (!XBZRLE
.cache
) {
2546 error_report_err(local_err
);
2547 goto free_zero_page
;
2550 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
2551 if (!XBZRLE
.encoded_buf
) {
2552 error_report("%s: Error allocating encoded_buf", __func__
);
2556 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
2557 if (!XBZRLE
.current_buf
) {
2558 error_report("%s: Error allocating current_buf", __func__
);
2559 goto free_encoded_buf
;
2562 /* We are all good */
2563 XBZRLE_cache_unlock();
2567 g_free(XBZRLE
.encoded_buf
);
2568 XBZRLE
.encoded_buf
= NULL
;
2570 cache_fini(XBZRLE
.cache
);
2571 XBZRLE
.cache
= NULL
;
2573 g_free(XBZRLE
.zero_target_page
);
2574 XBZRLE
.zero_target_page
= NULL
;
2576 XBZRLE_cache_unlock();
2580 static int ram_state_init(RAMState
**rsp
)
2582 *rsp
= g_try_new0(RAMState
, 1);
2585 error_report("%s: Init ramstate fail", __func__
);
2589 qemu_mutex_init(&(*rsp
)->bitmap_mutex
);
2590 qemu_mutex_init(&(*rsp
)->src_page_req_mutex
);
2591 QSIMPLEQ_INIT(&(*rsp
)->src_page_requests
);
2594 * Count the total number of pages used by ram blocks not including any
2595 * gaps due to alignment or unplugs.
2596 * This must match with the initial values of dirty bitmap.
2598 (*rsp
)->migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
2599 ram_state_reset(*rsp
);
2604 static void ram_list_init_bitmaps(void)
2606 MigrationState
*ms
= migrate_get_current();
2608 unsigned long pages
;
2611 /* Skip setting bitmap if there is no RAM */
2612 if (ram_bytes_total()) {
2613 shift
= ms
->clear_bitmap_shift
;
2614 if (shift
> CLEAR_BITMAP_SHIFT_MAX
) {
2615 error_report("clear_bitmap_shift (%u) too big, using "
2616 "max value (%u)", shift
, CLEAR_BITMAP_SHIFT_MAX
);
2617 shift
= CLEAR_BITMAP_SHIFT_MAX
;
2618 } else if (shift
< CLEAR_BITMAP_SHIFT_MIN
) {
2619 error_report("clear_bitmap_shift (%u) too small, using "
2620 "min value (%u)", shift
, CLEAR_BITMAP_SHIFT_MIN
);
2621 shift
= CLEAR_BITMAP_SHIFT_MIN
;
2624 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2625 pages
= block
->max_length
>> TARGET_PAGE_BITS
;
2627 * The initial dirty bitmap for migration must be set with all
2628 * ones to make sure we'll migrate every guest RAM page to
2630 * Here we set RAMBlock.bmap all to 1 because when rebegin a
2631 * new migration after a failed migration, ram_list.
2632 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
2635 block
->bmap
= bitmap_new(pages
);
2636 bitmap_set(block
->bmap
, 0, pages
);
2637 block
->clear_bmap_shift
= shift
;
2638 block
->clear_bmap
= bitmap_new(clear_bmap_size(pages
, shift
));
2643 static void ram_init_bitmaps(RAMState
*rs
)
2645 /* For memory_global_dirty_log_start below. */
2646 qemu_mutex_lock_iothread();
2647 qemu_mutex_lock_ramlist();
2649 WITH_RCU_READ_LOCK_GUARD() {
2650 ram_list_init_bitmaps();
2651 /* We don't use dirty log with background snapshots */
2652 if (!migrate_background_snapshot()) {
2653 memory_global_dirty_log_start();
2654 migration_bitmap_sync_precopy(rs
);
2657 qemu_mutex_unlock_ramlist();
2658 qemu_mutex_unlock_iothread();
2661 static int ram_init_all(RAMState
**rsp
)
2663 if (ram_state_init(rsp
)) {
2667 if (xbzrle_init()) {
2668 ram_state_cleanup(rsp
);
2672 ram_init_bitmaps(*rsp
);
2677 static void ram_state_resume_prepare(RAMState
*rs
, QEMUFile
*out
)
2683 * Postcopy is not using xbzrle/compression, so no need for that.
2684 * Also, since source are already halted, we don't need to care
2685 * about dirty page logging as well.
2688 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2689 pages
+= bitmap_count_one(block
->bmap
,
2690 block
->used_length
>> TARGET_PAGE_BITS
);
2693 /* This may not be aligned with current bitmaps. Recalculate. */
2694 rs
->migration_dirty_pages
= pages
;
2696 ram_state_reset(rs
);
2698 /* Update RAMState cache of output QEMUFile */
2701 trace_ram_state_resume_prepare(pages
);
2705 * This function clears bits of the free pages reported by the caller from the
2706 * migration dirty bitmap. @addr is the host address corresponding to the
2707 * start of the continuous guest free pages, and @len is the total bytes of
2710 void qemu_guest_free_page_hint(void *addr
, size_t len
)
2714 size_t used_len
, start
, npages
;
2715 MigrationState
*s
= migrate_get_current();
2717 /* This function is currently expected to be used during live migration */
2718 if (!migration_is_setup_or_active(s
->state
)) {
2722 for (; len
> 0; len
-= used_len
, addr
+= used_len
) {
2723 block
= qemu_ram_block_from_host(addr
, false, &offset
);
2724 if (unlikely(!block
|| offset
>= block
->used_length
)) {
2726 * The implementation might not support RAMBlock resize during
2727 * live migration, but it could happen in theory with future
2728 * updates. So we add a check here to capture that case.
2730 error_report_once("%s unexpected error", __func__
);
2734 if (len
<= block
->used_length
- offset
) {
2737 used_len
= block
->used_length
- offset
;
2740 start
= offset
>> TARGET_PAGE_BITS
;
2741 npages
= used_len
>> TARGET_PAGE_BITS
;
2743 qemu_mutex_lock(&ram_state
->bitmap_mutex
);
2744 ram_state
->migration_dirty_pages
-=
2745 bitmap_count_one_with_offset(block
->bmap
, start
, npages
);
2746 bitmap_clear(block
->bmap
, start
, npages
);
2747 qemu_mutex_unlock(&ram_state
->bitmap_mutex
);
2752 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
2753 * long-running RCU critical section. When rcu-reclaims in the code
2754 * start to become numerous it will be necessary to reduce the
2755 * granularity of these critical sections.
2759 * ram_save_setup: Setup RAM for migration
2761 * Returns zero to indicate success and negative for error
2763 * @f: QEMUFile where to send the data
2764 * @opaque: RAMState pointer
2766 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
2768 RAMState
**rsp
= opaque
;
2771 if (compress_threads_save_setup()) {
2775 /* migration has already setup the bitmap, reuse it. */
2776 if (!migration_in_colo_state()) {
2777 if (ram_init_all(rsp
) != 0) {
2778 compress_threads_save_cleanup();
2784 WITH_RCU_READ_LOCK_GUARD() {
2785 qemu_put_be64(f
, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE
);
2787 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
2788 qemu_put_byte(f
, strlen(block
->idstr
));
2789 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
2790 qemu_put_be64(f
, block
->used_length
);
2791 if (migrate_postcopy_ram() && block
->page_size
!=
2792 qemu_host_page_size
) {
2793 qemu_put_be64(f
, block
->page_size
);
2795 if (migrate_ignore_shared()) {
2796 qemu_put_be64(f
, block
->mr
->addr
);
2801 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
2802 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
2804 multifd_send_sync_main(f
);
2805 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2812 * ram_save_iterate: iterative stage for migration
2814 * Returns zero to indicate success and negative for error
2816 * @f: QEMUFile where to send the data
2817 * @opaque: RAMState pointer
2819 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
2821 RAMState
**temp
= opaque
;
2822 RAMState
*rs
= *temp
;
2828 if (blk_mig_bulk_active()) {
2829 /* Avoid transferring ram during bulk phase of block migration as
2830 * the bulk phase will usually take a long time and transferring
2831 * ram updates during that time is pointless. */
2836 * We'll take this lock a little bit long, but it's okay for two reasons.
2837 * Firstly, the only possible other thread to take it is who calls
2838 * qemu_guest_free_page_hint(), which should be rare; secondly, see
2839 * MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which
2840 * guarantees that we'll at least released it in a regular basis.
2842 qemu_mutex_lock(&rs
->bitmap_mutex
);
2843 WITH_RCU_READ_LOCK_GUARD() {
2844 if (ram_list
.version
!= rs
->last_version
) {
2845 ram_state_reset(rs
);
2848 /* Read version before ram_list.blocks */
2851 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
2853 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
2855 while ((ret
= qemu_file_rate_limit(f
)) == 0 ||
2856 !QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
2859 if (qemu_file_get_error(f
)) {
2863 pages
= ram_find_and_save_block(rs
, false);
2864 /* no more pages to sent */
2871 qemu_file_set_error(f
, pages
);
2875 rs
->target_page_count
+= pages
;
2878 * During postcopy, it is necessary to make sure one whole host
2879 * page is sent in one chunk.
2881 if (migrate_postcopy_ram()) {
2882 flush_compressed_data(rs
);
2886 * we want to check in the 1st loop, just in case it was the 1st
2887 * time and we had to sync the dirty bitmap.
2888 * qemu_clock_get_ns() is a bit expensive, so we only check each
2891 if ((i
& 63) == 0) {
2892 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) /
2894 if (t1
> MAX_WAIT
) {
2895 trace_ram_save_iterate_big_wait(t1
, i
);
2902 qemu_mutex_unlock(&rs
->bitmap_mutex
);
2905 * Must occur before EOS (or any QEMUFile operation)
2906 * because of RDMA protocol.
2908 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
2912 && migration_is_setup_or_active(migrate_get_current()->state
)) {
2913 multifd_send_sync_main(rs
->f
);
2914 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2916 ram_counters
.transferred
+= 8;
2918 ret
= qemu_file_get_error(f
);
2928 * ram_save_complete: function called to send the remaining amount of ram
2930 * Returns zero to indicate success or negative on error
2932 * Called with iothread lock
2934 * @f: QEMUFile where to send the data
2935 * @opaque: RAMState pointer
2937 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
2939 RAMState
**temp
= opaque
;
2940 RAMState
*rs
= *temp
;
2943 WITH_RCU_READ_LOCK_GUARD() {
2944 if (!migration_in_postcopy()) {
2945 migration_bitmap_sync_precopy(rs
);
2948 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
2950 /* try transferring iterative blocks of memory */
2952 /* flush all remaining blocks regardless of rate limiting */
2956 pages
= ram_find_and_save_block(rs
, !migration_in_colo_state());
2957 /* no more blocks to sent */
2967 flush_compressed_data(rs
);
2968 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
2972 multifd_send_sync_main(rs
->f
);
2973 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2980 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
2981 uint64_t *res_precopy_only
,
2982 uint64_t *res_compatible
,
2983 uint64_t *res_postcopy_only
)
2985 RAMState
**temp
= opaque
;
2986 RAMState
*rs
= *temp
;
2987 uint64_t remaining_size
;
2989 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
2991 if (!migration_in_postcopy() &&
2992 remaining_size
< max_size
) {
2993 qemu_mutex_lock_iothread();
2994 WITH_RCU_READ_LOCK_GUARD() {
2995 migration_bitmap_sync_precopy(rs
);
2997 qemu_mutex_unlock_iothread();
2998 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3001 if (migrate_postcopy_ram()) {
3002 /* We can do postcopy, and all the data is postcopiable */
3003 *res_compatible
+= remaining_size
;
3005 *res_precopy_only
+= remaining_size
;
3009 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
3011 unsigned int xh_len
;
3013 uint8_t *loaded_data
;
3015 /* extract RLE header */
3016 xh_flags
= qemu_get_byte(f
);
3017 xh_len
= qemu_get_be16(f
);
3019 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
3020 error_report("Failed to load XBZRLE page - wrong compression!");
3024 if (xh_len
> TARGET_PAGE_SIZE
) {
3025 error_report("Failed to load XBZRLE page - len overflow!");
3028 loaded_data
= XBZRLE
.decoded_buf
;
3029 /* load data and decode */
3030 /* it can change loaded_data to point to an internal buffer */
3031 qemu_get_buffer_in_place(f
, &loaded_data
, xh_len
);
3034 if (xbzrle_decode_buffer(loaded_data
, xh_len
, host
,
3035 TARGET_PAGE_SIZE
) == -1) {
3036 error_report("Failed to load XBZRLE page - decode error!");
3044 * ram_block_from_stream: read a RAMBlock id from the migration stream
3046 * Must be called from within a rcu critical section.
3048 * Returns a pointer from within the RCU-protected ram_list.
3050 * @f: QEMUFile where to read the data from
3051 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3053 static inline RAMBlock
*ram_block_from_stream(QEMUFile
*f
, int flags
)
3055 static RAMBlock
*block
;
3059 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
3061 error_report("Ack, bad migration stream!");
3067 len
= qemu_get_byte(f
);
3068 qemu_get_buffer(f
, (uint8_t *)id
, len
);
3071 block
= qemu_ram_block_by_name(id
);
3073 error_report("Can't find block %s", id
);
3077 if (ramblock_is_ignored(block
)) {
3078 error_report("block %s should not be migrated !", id
);
3085 static inline void *host_from_ram_block_offset(RAMBlock
*block
,
3088 if (!offset_in_ramblock(block
, offset
)) {
3092 return block
->host
+ offset
;
3095 static void *host_page_from_ram_block_offset(RAMBlock
*block
,
3098 /* Note: Explicitly no check against offset_in_ramblock(). */
3099 return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block
->host
+ offset
),
3103 static ram_addr_t
host_page_offset_from_ram_block_offset(RAMBlock
*block
,
3106 return ((uintptr_t)block
->host
+ offset
) & (block
->page_size
- 1);
3109 static inline void *colo_cache_from_block_offset(RAMBlock
*block
,
3110 ram_addr_t offset
, bool record_bitmap
)
3112 if (!offset_in_ramblock(block
, offset
)) {
3115 if (!block
->colo_cache
) {
3116 error_report("%s: colo_cache is NULL in block :%s",
3117 __func__
, block
->idstr
);
3122 * During colo checkpoint, we need bitmap of these migrated pages.
3123 * It help us to decide which pages in ram cache should be flushed
3124 * into VM's RAM later.
3126 if (record_bitmap
&&
3127 !test_and_set_bit(offset
>> TARGET_PAGE_BITS
, block
->bmap
)) {
3128 ram_state
->migration_dirty_pages
++;
3130 return block
->colo_cache
+ offset
;
3134 * ram_handle_compressed: handle the zero page case
3136 * If a page (or a whole RDMA chunk) has been
3137 * determined to be zero, then zap it.
3139 * @host: host address for the zero page
3140 * @ch: what the page is filled from. We only support zero
3141 * @size: size of the zero page
3143 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
3145 if (ch
!= 0 || !is_zero_range(host
, size
)) {
3146 memset(host
, ch
, size
);
3150 /* return the size after decompression, or negative value on error */
3152 qemu_uncompress_data(z_stream
*stream
, uint8_t *dest
, size_t dest_len
,
3153 const uint8_t *source
, size_t source_len
)
3157 err
= inflateReset(stream
);
3162 stream
->avail_in
= source_len
;
3163 stream
->next_in
= (uint8_t *)source
;
3164 stream
->avail_out
= dest_len
;
3165 stream
->next_out
= dest
;
3167 err
= inflate(stream
, Z_NO_FLUSH
);
3168 if (err
!= Z_STREAM_END
) {
3172 return stream
->total_out
;
3175 static void *do_data_decompress(void *opaque
)
3177 DecompressParam
*param
= opaque
;
3178 unsigned long pagesize
;
3182 qemu_mutex_lock(¶m
->mutex
);
3183 while (!param
->quit
) {
3188 qemu_mutex_unlock(¶m
->mutex
);
3190 pagesize
= TARGET_PAGE_SIZE
;
3192 ret
= qemu_uncompress_data(¶m
->stream
, des
, pagesize
,
3193 param
->compbuf
, len
);
3194 if (ret
< 0 && migrate_get_current()->decompress_error_check
) {
3195 error_report("decompress data failed");
3196 qemu_file_set_error(decomp_file
, ret
);
3199 qemu_mutex_lock(&decomp_done_lock
);
3201 qemu_cond_signal(&decomp_done_cond
);
3202 qemu_mutex_unlock(&decomp_done_lock
);
3204 qemu_mutex_lock(¶m
->mutex
);
3206 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
3209 qemu_mutex_unlock(¶m
->mutex
);
3214 static int wait_for_decompress_done(void)
3216 int idx
, thread_count
;
3218 if (!migrate_use_compression()) {
3222 thread_count
= migrate_decompress_threads();
3223 qemu_mutex_lock(&decomp_done_lock
);
3224 for (idx
= 0; idx
< thread_count
; idx
++) {
3225 while (!decomp_param
[idx
].done
) {
3226 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3229 qemu_mutex_unlock(&decomp_done_lock
);
3230 return qemu_file_get_error(decomp_file
);
3233 static void compress_threads_load_cleanup(void)
3235 int i
, thread_count
;
3237 if (!migrate_use_compression()) {
3240 thread_count
= migrate_decompress_threads();
3241 for (i
= 0; i
< thread_count
; i
++) {
3243 * we use it as a indicator which shows if the thread is
3244 * properly init'd or not
3246 if (!decomp_param
[i
].compbuf
) {
3250 qemu_mutex_lock(&decomp_param
[i
].mutex
);
3251 decomp_param
[i
].quit
= true;
3252 qemu_cond_signal(&decomp_param
[i
].cond
);
3253 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
3255 for (i
= 0; i
< thread_count
; i
++) {
3256 if (!decomp_param
[i
].compbuf
) {
3260 qemu_thread_join(decompress_threads
+ i
);
3261 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
3262 qemu_cond_destroy(&decomp_param
[i
].cond
);
3263 inflateEnd(&decomp_param
[i
].stream
);
3264 g_free(decomp_param
[i
].compbuf
);
3265 decomp_param
[i
].compbuf
= NULL
;
3267 g_free(decompress_threads
);
3268 g_free(decomp_param
);
3269 decompress_threads
= NULL
;
3270 decomp_param
= NULL
;
3274 static int compress_threads_load_setup(QEMUFile
*f
)
3276 int i
, thread_count
;
3278 if (!migrate_use_compression()) {
3282 thread_count
= migrate_decompress_threads();
3283 decompress_threads
= g_new0(QemuThread
, thread_count
);
3284 decomp_param
= g_new0(DecompressParam
, thread_count
);
3285 qemu_mutex_init(&decomp_done_lock
);
3286 qemu_cond_init(&decomp_done_cond
);
3288 for (i
= 0; i
< thread_count
; i
++) {
3289 if (inflateInit(&decomp_param
[i
].stream
) != Z_OK
) {
3293 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
3294 qemu_mutex_init(&decomp_param
[i
].mutex
);
3295 qemu_cond_init(&decomp_param
[i
].cond
);
3296 decomp_param
[i
].done
= true;
3297 decomp_param
[i
].quit
= false;
3298 qemu_thread_create(decompress_threads
+ i
, "decompress",
3299 do_data_decompress
, decomp_param
+ i
,
3300 QEMU_THREAD_JOINABLE
);
3304 compress_threads_load_cleanup();
3308 static void decompress_data_with_multi_threads(QEMUFile
*f
,
3309 void *host
, int len
)
3311 int idx
, thread_count
;
3313 thread_count
= migrate_decompress_threads();
3314 QEMU_LOCK_GUARD(&decomp_done_lock
);
3316 for (idx
= 0; idx
< thread_count
; idx
++) {
3317 if (decomp_param
[idx
].done
) {
3318 decomp_param
[idx
].done
= false;
3319 qemu_mutex_lock(&decomp_param
[idx
].mutex
);
3320 qemu_get_buffer(f
, decomp_param
[idx
].compbuf
, len
);
3321 decomp_param
[idx
].des
= host
;
3322 decomp_param
[idx
].len
= len
;
3323 qemu_cond_signal(&decomp_param
[idx
].cond
);
3324 qemu_mutex_unlock(&decomp_param
[idx
].mutex
);
3328 if (idx
< thread_count
) {
3331 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3336 static void colo_init_ram_state(void)
3338 ram_state_init(&ram_state
);
3342 * colo cache: this is for secondary VM, we cache the whole
3343 * memory of the secondary VM, it is need to hold the global lock
3344 * to call this helper.
3346 int colo_init_ram_cache(void)
3350 WITH_RCU_READ_LOCK_GUARD() {
3351 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3352 block
->colo_cache
= qemu_anon_ram_alloc(block
->used_length
,
3353 NULL
, false, false);
3354 if (!block
->colo_cache
) {
3355 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3356 "size 0x" RAM_ADDR_FMT
, __func__
, block
->idstr
,
3357 block
->used_length
);
3358 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3359 if (block
->colo_cache
) {
3360 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3361 block
->colo_cache
= NULL
;
3370 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3371 * with to decide which page in cache should be flushed into SVM's RAM. Here
3372 * we use the same name 'ram_bitmap' as for migration.
3374 if (ram_bytes_total()) {
3377 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3378 unsigned long pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3379 block
->bmap
= bitmap_new(pages
);
3383 colo_init_ram_state();
3387 /* TODO: duplicated with ram_init_bitmaps */
3388 void colo_incoming_start_dirty_log(void)
3390 RAMBlock
*block
= NULL
;
3391 /* For memory_global_dirty_log_start below. */
3392 qemu_mutex_lock_iothread();
3393 qemu_mutex_lock_ramlist();
3395 memory_global_dirty_log_sync();
3396 WITH_RCU_READ_LOCK_GUARD() {
3397 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3398 ramblock_sync_dirty_bitmap(ram_state
, block
);
3399 /* Discard this dirty bitmap record */
3400 bitmap_zero(block
->bmap
, block
->max_length
>> TARGET_PAGE_BITS
);
3402 memory_global_dirty_log_start();
3404 ram_state
->migration_dirty_pages
= 0;
3405 qemu_mutex_unlock_ramlist();
3406 qemu_mutex_unlock_iothread();
3409 /* It is need to hold the global lock to call this helper */
3410 void colo_release_ram_cache(void)
3414 memory_global_dirty_log_stop();
3415 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3416 g_free(block
->bmap
);
3420 WITH_RCU_READ_LOCK_GUARD() {
3421 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3422 if (block
->colo_cache
) {
3423 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3424 block
->colo_cache
= NULL
;
3428 ram_state_cleanup(&ram_state
);
3432 * ram_load_setup: Setup RAM for migration incoming side
3434 * Returns zero to indicate success and negative for error
3436 * @f: QEMUFile where to receive the data
3437 * @opaque: RAMState pointer
3439 static int ram_load_setup(QEMUFile
*f
, void *opaque
)
3441 if (compress_threads_load_setup(f
)) {
3445 xbzrle_load_setup();
3446 ramblock_recv_map_init();
3451 static int ram_load_cleanup(void *opaque
)
3455 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3456 qemu_ram_block_writeback(rb
);
3459 xbzrle_load_cleanup();
3460 compress_threads_load_cleanup();
3462 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3463 g_free(rb
->receivedmap
);
3464 rb
->receivedmap
= NULL
;
3471 * ram_postcopy_incoming_init: allocate postcopy data structures
3473 * Returns 0 for success and negative if there was one error
3475 * @mis: current migration incoming state
3477 * Allocate data structures etc needed by incoming migration with
3478 * postcopy-ram. postcopy-ram's similarly names
3479 * postcopy_ram_incoming_init does the work.
3481 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
3483 return postcopy_ram_incoming_init(mis
);
3487 * ram_load_postcopy: load a page in postcopy case
3489 * Returns 0 for success or -errno in case of error
3491 * Called in postcopy mode by ram_load().
3492 * rcu_read_lock is taken prior to this being called.
3494 * @f: QEMUFile where to send the data
3496 static int ram_load_postcopy(QEMUFile
*f
)
3498 int flags
= 0, ret
= 0;
3499 bool place_needed
= false;
3500 bool matches_target_page_size
= false;
3501 MigrationIncomingState
*mis
= migration_incoming_get_current();
3502 /* Temporary page that is later 'placed' */
3503 void *postcopy_host_page
= mis
->postcopy_tmp_page
;
3504 void *host_page
= NULL
;
3505 bool all_zero
= true;
3506 int target_pages
= 0;
3508 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
3510 void *page_buffer
= NULL
;
3511 void *place_source
= NULL
;
3512 RAMBlock
*block
= NULL
;
3516 addr
= qemu_get_be64(f
);
3519 * If qemu file error, we should stop here, and then "addr"
3522 ret
= qemu_file_get_error(f
);
3527 flags
= addr
& ~TARGET_PAGE_MASK
;
3528 addr
&= TARGET_PAGE_MASK
;
3530 trace_ram_load_postcopy_loop((uint64_t)addr
, flags
);
3531 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
3532 RAM_SAVE_FLAG_COMPRESS_PAGE
)) {
3533 block
= ram_block_from_stream(f
, flags
);
3540 * Relying on used_length is racy and can result in false positives.
3541 * We might place pages beyond used_length in case RAM was shrunk
3542 * while in postcopy, which is fine - trying to place via
3543 * UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault.
3545 if (!block
->host
|| addr
>= block
->postcopy_length
) {
3546 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
3551 matches_target_page_size
= block
->page_size
== TARGET_PAGE_SIZE
;
3553 * Postcopy requires that we place whole host pages atomically;
3554 * these may be huge pages for RAMBlocks that are backed by
3556 * To make it atomic, the data is read into a temporary page
3557 * that's moved into place later.
3558 * The migration protocol uses, possibly smaller, target-pages
3559 * however the source ensures it always sends all the components
3560 * of a host page in one chunk.
3562 page_buffer
= postcopy_host_page
+
3563 host_page_offset_from_ram_block_offset(block
, addr
);
3564 /* If all TP are zero then we can optimise the place */
3565 if (target_pages
== 1) {
3566 host_page
= host_page_from_ram_block_offset(block
, addr
);
3567 } else if (host_page
!= host_page_from_ram_block_offset(block
,
3569 /* not the 1st TP within the HP */
3570 error_report("Non-same host page %p/%p", host_page
,
3571 host_page_from_ram_block_offset(block
, addr
));
3577 * If it's the last part of a host page then we place the host
3580 if (target_pages
== (block
->page_size
/ TARGET_PAGE_SIZE
)) {
3581 place_needed
= true;
3583 place_source
= postcopy_host_page
;
3586 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
3587 case RAM_SAVE_FLAG_ZERO
:
3588 ch
= qemu_get_byte(f
);
3590 * Can skip to set page_buffer when
3591 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
3593 if (ch
|| !matches_target_page_size
) {
3594 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
3601 case RAM_SAVE_FLAG_PAGE
:
3603 if (!matches_target_page_size
) {
3604 /* For huge pages, we always use temporary buffer */
3605 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
3608 * For small pages that matches target page size, we
3609 * avoid the qemu_file copy. Instead we directly use
3610 * the buffer of QEMUFile to place the page. Note: we
3611 * cannot do any QEMUFile operation before using that
3612 * buffer to make sure the buffer is valid when
3615 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
3619 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
3621 len
= qemu_get_be32(f
);
3622 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
3623 error_report("Invalid compressed data length: %d", len
);
3627 decompress_data_with_multi_threads(f
, page_buffer
, len
);
3630 case RAM_SAVE_FLAG_EOS
:
3632 multifd_recv_sync_main();
3635 error_report("Unknown combination of migration flags: 0x%x"
3636 " (postcopy mode)", flags
);
3641 /* Got the whole host page, wait for decompress before placing. */
3643 ret
|= wait_for_decompress_done();
3646 /* Detect for any possible file errors */
3647 if (!ret
&& qemu_file_get_error(f
)) {
3648 ret
= qemu_file_get_error(f
);
3651 if (!ret
&& place_needed
) {
3653 ret
= postcopy_place_page_zero(mis
, host_page
, block
);
3655 ret
= postcopy_place_page(mis
, host_page
, place_source
,
3658 place_needed
= false;
3660 /* Assume we have a zero page until we detect something different */
3668 static bool postcopy_is_advised(void)
3670 PostcopyState ps
= postcopy_state_get();
3671 return ps
>= POSTCOPY_INCOMING_ADVISE
&& ps
< POSTCOPY_INCOMING_END
;
3674 static bool postcopy_is_running(void)
3676 PostcopyState ps
= postcopy_state_get();
3677 return ps
>= POSTCOPY_INCOMING_LISTENING
&& ps
< POSTCOPY_INCOMING_END
;
3681 * Flush content of RAM cache into SVM's memory.
3682 * Only flush the pages that be dirtied by PVM or SVM or both.
3684 void colo_flush_ram_cache(void)
3686 RAMBlock
*block
= NULL
;
3689 unsigned long offset
= 0;
3691 memory_global_dirty_log_sync();
3692 qemu_mutex_lock(&ram_state
->bitmap_mutex
);
3693 WITH_RCU_READ_LOCK_GUARD() {
3694 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3695 ramblock_sync_dirty_bitmap(ram_state
, block
);
3699 trace_colo_flush_ram_cache_begin(ram_state
->migration_dirty_pages
);
3700 WITH_RCU_READ_LOCK_GUARD() {
3701 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
3704 offset
= migration_bitmap_find_dirty(ram_state
, block
, offset
);
3706 if (!offset_in_ramblock(block
,
3707 ((ram_addr_t
)offset
) << TARGET_PAGE_BITS
)) {
3709 block
= QLIST_NEXT_RCU(block
, next
);
3711 migration_bitmap_clear_dirty(ram_state
, block
, offset
);
3712 dst_host
= block
->host
3713 + (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
);
3714 src_host
= block
->colo_cache
3715 + (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
);
3716 memcpy(dst_host
, src_host
, TARGET_PAGE_SIZE
);
3720 trace_colo_flush_ram_cache_end();
3721 qemu_mutex_unlock(&ram_state
->bitmap_mutex
);
3725 * ram_load_precopy: load pages in precopy case
3727 * Returns 0 for success or -errno in case of error
3729 * Called in precopy mode by ram_load().
3730 * rcu_read_lock is taken prior to this being called.
3732 * @f: QEMUFile where to send the data
3734 static int ram_load_precopy(QEMUFile
*f
)
3736 int flags
= 0, ret
= 0, invalid_flags
= 0, len
= 0, i
= 0;
3737 /* ADVISE is earlier, it shows the source has the postcopy capability on */
3738 bool postcopy_advised
= postcopy_is_advised();
3739 if (!migrate_use_compression()) {
3740 invalid_flags
|= RAM_SAVE_FLAG_COMPRESS_PAGE
;
3743 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
3744 ram_addr_t addr
, total_ram_bytes
;
3745 void *host
= NULL
, *host_bak
= NULL
;
3749 * Yield periodically to let main loop run, but an iteration of
3750 * the main loop is expensive, so do it each some iterations
3752 if ((i
& 32767) == 0 && qemu_in_coroutine()) {
3753 aio_co_schedule(qemu_get_current_aio_context(),
3754 qemu_coroutine_self());
3755 qemu_coroutine_yield();
3759 addr
= qemu_get_be64(f
);
3760 flags
= addr
& ~TARGET_PAGE_MASK
;
3761 addr
&= TARGET_PAGE_MASK
;
3763 if (flags
& invalid_flags
) {
3764 if (flags
& invalid_flags
& RAM_SAVE_FLAG_COMPRESS_PAGE
) {
3765 error_report("Received an unexpected compressed page");
3772 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
3773 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
3774 RAMBlock
*block
= ram_block_from_stream(f
, flags
);
3776 host
= host_from_ram_block_offset(block
, addr
);
3778 * After going into COLO stage, we should not load the page
3779 * into SVM's memory directly, we put them into colo_cache firstly.
3780 * NOTE: We need to keep a copy of SVM's ram in colo_cache.
3781 * Previously, we copied all these memory in preparing stage of COLO
3782 * while we need to stop VM, which is a time-consuming process.
3783 * Here we optimize it by a trick, back-up every page while in
3784 * migration process while COLO is enabled, though it affects the
3785 * speed of the migration, but it obviously reduce the downtime of
3786 * back-up all SVM'S memory in COLO preparing stage.
3788 if (migration_incoming_colo_enabled()) {
3789 if (migration_incoming_in_colo_state()) {
3790 /* In COLO stage, put all pages into cache temporarily */
3791 host
= colo_cache_from_block_offset(block
, addr
, true);
3794 * In migration stage but before COLO stage,
3795 * Put all pages into both cache and SVM's memory.
3797 host_bak
= colo_cache_from_block_offset(block
, addr
, false);
3801 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
3805 if (!migration_incoming_in_colo_state()) {
3806 ramblock_recv_bitmap_set(block
, host
);
3809 trace_ram_load_loop(block
->idstr
, (uint64_t)addr
, flags
, host
);
3812 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
3813 case RAM_SAVE_FLAG_MEM_SIZE
:
3814 /* Synchronize RAM block list */
3815 total_ram_bytes
= addr
;
3816 while (!ret
&& total_ram_bytes
) {
3821 len
= qemu_get_byte(f
);
3822 qemu_get_buffer(f
, (uint8_t *)id
, len
);
3824 length
= qemu_get_be64(f
);
3826 block
= qemu_ram_block_by_name(id
);
3827 if (block
&& !qemu_ram_is_migratable(block
)) {
3828 error_report("block %s should not be migrated !", id
);
3831 if (length
!= block
->used_length
) {
3832 Error
*local_err
= NULL
;
3834 ret
= qemu_ram_resize(block
, length
,
3837 error_report_err(local_err
);
3840 /* For postcopy we need to check hugepage sizes match */
3841 if (postcopy_advised
&& migrate_postcopy_ram() &&
3842 block
->page_size
!= qemu_host_page_size
) {
3843 uint64_t remote_page_size
= qemu_get_be64(f
);
3844 if (remote_page_size
!= block
->page_size
) {
3845 error_report("Mismatched RAM page size %s "
3846 "(local) %zd != %" PRId64
,
3847 id
, block
->page_size
,
3852 if (migrate_ignore_shared()) {
3853 hwaddr addr
= qemu_get_be64(f
);
3854 if (ramblock_is_ignored(block
) &&
3855 block
->mr
->addr
!= addr
) {
3856 error_report("Mismatched GPAs for block %s "
3857 "%" PRId64
"!= %" PRId64
,
3859 (uint64_t)block
->mr
->addr
);
3863 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
3866 error_report("Unknown ramblock \"%s\", cannot "
3867 "accept migration", id
);
3871 total_ram_bytes
-= length
;
3875 case RAM_SAVE_FLAG_ZERO
:
3876 ch
= qemu_get_byte(f
);
3877 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
3880 case RAM_SAVE_FLAG_PAGE
:
3881 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
3884 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
3885 len
= qemu_get_be32(f
);
3886 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
3887 error_report("Invalid compressed data length: %d", len
);
3891 decompress_data_with_multi_threads(f
, host
, len
);
3894 case RAM_SAVE_FLAG_XBZRLE
:
3895 if (load_xbzrle(f
, addr
, host
) < 0) {
3896 error_report("Failed to decompress XBZRLE page at "
3897 RAM_ADDR_FMT
, addr
);
3902 case RAM_SAVE_FLAG_EOS
:
3904 multifd_recv_sync_main();
3907 if (flags
& RAM_SAVE_FLAG_HOOK
) {
3908 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
3910 error_report("Unknown combination of migration flags: 0x%x",
3916 ret
= qemu_file_get_error(f
);
3918 if (!ret
&& host_bak
) {
3919 memcpy(host_bak
, host
, TARGET_PAGE_SIZE
);
3923 ret
|= wait_for_decompress_done();
3927 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
3930 static uint64_t seq_iter
;
3932 * If system is running in postcopy mode, page inserts to host memory must
3935 bool postcopy_running
= postcopy_is_running();
3939 if (version_id
!= 4) {
3944 * This RCU critical section can be very long running.
3945 * When RCU reclaims in the code start to become numerous,
3946 * it will be necessary to reduce the granularity of this
3949 WITH_RCU_READ_LOCK_GUARD() {
3950 if (postcopy_running
) {
3951 ret
= ram_load_postcopy(f
);
3953 ret
= ram_load_precopy(f
);
3956 trace_ram_load_complete(ret
, seq_iter
);
3961 static bool ram_has_postcopy(void *opaque
)
3964 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3965 if (ramblock_is_pmem(rb
)) {
3966 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
3967 "is not supported now!", rb
->idstr
, rb
->host
);
3972 return migrate_postcopy_ram();
3975 /* Sync all the dirty bitmap with destination VM. */
3976 static int ram_dirty_bitmap_sync_all(MigrationState
*s
, RAMState
*rs
)
3979 QEMUFile
*file
= s
->to_dst_file
;
3980 int ramblock_count
= 0;
3982 trace_ram_dirty_bitmap_sync_start();
3984 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3985 qemu_savevm_send_recv_bitmap(file
, block
->idstr
);
3986 trace_ram_dirty_bitmap_request(block
->idstr
);
3990 trace_ram_dirty_bitmap_sync_wait();
3992 /* Wait until all the ramblocks' dirty bitmap synced */
3993 while (ramblock_count
--) {
3994 qemu_sem_wait(&s
->rp_state
.rp_sem
);
3997 trace_ram_dirty_bitmap_sync_complete();
4002 static void ram_dirty_bitmap_reload_notify(MigrationState
*s
)
4004 qemu_sem_post(&s
->rp_state
.rp_sem
);
4008 * Read the received bitmap, revert it as the initial dirty bitmap.
4009 * This is only used when the postcopy migration is paused but wants
4010 * to resume from a middle point.
4012 int ram_dirty_bitmap_reload(MigrationState
*s
, RAMBlock
*block
)
4015 QEMUFile
*file
= s
->rp_state
.from_dst_file
;
4016 unsigned long *le_bitmap
, nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
4017 uint64_t local_size
= DIV_ROUND_UP(nbits
, 8);
4018 uint64_t size
, end_mark
;
4020 trace_ram_dirty_bitmap_reload_begin(block
->idstr
);
4022 if (s
->state
!= MIGRATION_STATUS_POSTCOPY_RECOVER
) {
4023 error_report("%s: incorrect state %s", __func__
,
4024 MigrationStatus_str(s
->state
));
4029 * Note: see comments in ramblock_recv_bitmap_send() on why we
4030 * need the endianness conversion, and the paddings.
4032 local_size
= ROUND_UP(local_size
, 8);
4035 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
4037 size
= qemu_get_be64(file
);
4039 /* The size of the bitmap should match with our ramblock */
4040 if (size
!= local_size
) {
4041 error_report("%s: ramblock '%s' bitmap size mismatch "
4042 "(0x%"PRIx64
" != 0x%"PRIx64
")", __func__
,
4043 block
->idstr
, size
, local_size
);
4048 size
= qemu_get_buffer(file
, (uint8_t *)le_bitmap
, local_size
);
4049 end_mark
= qemu_get_be64(file
);
4051 ret
= qemu_file_get_error(file
);
4052 if (ret
|| size
!= local_size
) {
4053 error_report("%s: read bitmap failed for ramblock '%s': %d"
4054 " (size 0x%"PRIx64
", got: 0x%"PRIx64
")",
4055 __func__
, block
->idstr
, ret
, local_size
, size
);
4060 if (end_mark
!= RAMBLOCK_RECV_BITMAP_ENDING
) {
4061 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIx64
,
4062 __func__
, block
->idstr
, end_mark
);
4068 * Endianness conversion. We are during postcopy (though paused).
4069 * The dirty bitmap won't change. We can directly modify it.
4071 bitmap_from_le(block
->bmap
, le_bitmap
, nbits
);
4074 * What we received is "received bitmap". Revert it as the initial
4075 * dirty bitmap for this ramblock.
4077 bitmap_complement(block
->bmap
, block
->bmap
, nbits
);
4079 trace_ram_dirty_bitmap_reload_complete(block
->idstr
);
4082 * We succeeded to sync bitmap for current ramblock. If this is
4083 * the last one to sync, we need to notify the main send thread.
4085 ram_dirty_bitmap_reload_notify(s
);
4093 static int ram_resume_prepare(MigrationState
*s
, void *opaque
)
4095 RAMState
*rs
= *(RAMState
**)opaque
;
4098 ret
= ram_dirty_bitmap_sync_all(s
, rs
);
4103 ram_state_resume_prepare(rs
, s
->to_dst_file
);
4108 static SaveVMHandlers savevm_ram_handlers
= {
4109 .save_setup
= ram_save_setup
,
4110 .save_live_iterate
= ram_save_iterate
,
4111 .save_live_complete_postcopy
= ram_save_complete
,
4112 .save_live_complete_precopy
= ram_save_complete
,
4113 .has_postcopy
= ram_has_postcopy
,
4114 .save_live_pending
= ram_save_pending
,
4115 .load_state
= ram_load
,
4116 .save_cleanup
= ram_save_cleanup
,
4117 .load_setup
= ram_load_setup
,
4118 .load_cleanup
= ram_load_cleanup
,
4119 .resume_prepare
= ram_resume_prepare
,
4122 static void ram_mig_ram_block_resized(RAMBlockNotifier
*n
, void *host
,
4123 size_t old_size
, size_t new_size
)
4125 PostcopyState ps
= postcopy_state_get();
4127 RAMBlock
*rb
= qemu_ram_block_from_host(host
, false, &offset
);
4130 if (ramblock_is_ignored(rb
)) {
4134 if (!migration_is_idle()) {
4136 * Precopy code on the source cannot deal with the size of RAM blocks
4137 * changing at random points in time - especially after sending the
4138 * RAM block sizes in the migration stream, they must no longer change.
4139 * Abort and indicate a proper reason.
4141 error_setg(&err
, "RAM block '%s' resized during precopy.", rb
->idstr
);
4142 migrate_set_error(migrate_get_current(), err
);
4148 case POSTCOPY_INCOMING_ADVISE
:
4150 * Update what ram_postcopy_incoming_init()->init_range() does at the
4151 * time postcopy was advised. Syncing RAM blocks with the source will
4152 * result in RAM resizes.
4154 if (old_size
< new_size
) {
4155 if (ram_discard_range(rb
->idstr
, old_size
, new_size
- old_size
)) {
4156 error_report("RAM block '%s' discard of resized RAM failed",
4160 rb
->postcopy_length
= new_size
;
4162 case POSTCOPY_INCOMING_NONE
:
4163 case POSTCOPY_INCOMING_RUNNING
:
4164 case POSTCOPY_INCOMING_END
:
4166 * Once our guest is running, postcopy does no longer care about
4167 * resizes. When growing, the new memory was not available on the
4168 * source, no handler needed.
4172 error_report("RAM block '%s' resized during postcopy state: %d",
4178 static RAMBlockNotifier ram_mig_ram_notifier
= {
4179 .ram_block_resized
= ram_mig_ram_block_resized
,
4182 void ram_mig_init(void)
4184 qemu_mutex_init(&XBZRLE
.lock
);
4185 register_savevm_live("ram", 0, 4, &savevm_ram_handlers
, &ram_state
);
4186 ram_block_notifier_add(&ram_mig_ram_notifier
);