4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu/osdep.h"
32 #include "qemu/cutils.h"
33 #include "qemu/bitops.h"
34 #include "qemu/bitmap.h"
35 #include "qemu/main-loop.h"
38 #include "migration.h"
40 #include "migration/register.h"
41 #include "migration/misc.h"
42 #include "qemu-file.h"
43 #include "postcopy-ram.h"
44 #include "page_cache.h"
45 #include "qemu/error-report.h"
46 #include "qapi/error.h"
47 #include "qapi/qapi-events-migration.h"
48 #include "qapi/qmp/qerror.h"
50 #include "exec/ram_addr.h"
51 #include "exec/target_page.h"
52 #include "qemu/rcu_queue.h"
53 #include "migration/colo.h"
55 #include "sysemu/sysemu.h"
56 #include "qemu/uuid.h"
60 /***********************************************************/
61 /* ram save/restore */
63 /* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
64 * worked for pages that where filled with the same char. We switched
65 * it to only search for the zero value. And to avoid confusion with
66 * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it.
69 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
70 #define RAM_SAVE_FLAG_ZERO 0x02
71 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
72 #define RAM_SAVE_FLAG_PAGE 0x08
73 #define RAM_SAVE_FLAG_EOS 0x10
74 #define RAM_SAVE_FLAG_CONTINUE 0x20
75 #define RAM_SAVE_FLAG_XBZRLE 0x40
76 /* 0x80 is reserved in migration.h start with 0x100 next */
77 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
79 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
81 return buffer_is_zero(p
, size
);
84 XBZRLECacheStats xbzrle_counters
;
86 /* struct contains XBZRLE cache and a static page
87 used by the compression */
89 /* buffer used for XBZRLE encoding */
91 /* buffer for storing page content */
93 /* Cache for XBZRLE, Protected by lock. */
96 /* it will store a page full of zeros */
97 uint8_t *zero_target_page
;
98 /* buffer used for XBZRLE decoding */
102 static void XBZRLE_cache_lock(void)
104 if (migrate_use_xbzrle())
105 qemu_mutex_lock(&XBZRLE
.lock
);
108 static void XBZRLE_cache_unlock(void)
110 if (migrate_use_xbzrle())
111 qemu_mutex_unlock(&XBZRLE
.lock
);
115 * xbzrle_cache_resize: resize the xbzrle cache
117 * This function is called from qmp_migrate_set_cache_size in main
118 * thread, possibly while a migration is in progress. A running
119 * migration may be using the cache and might finish during this call,
120 * hence changes to the cache are protected by XBZRLE.lock().
122 * Returns 0 for success or -1 for error
124 * @new_size: new cache size
125 * @errp: set *errp if the check failed, with reason
127 int xbzrle_cache_resize(int64_t new_size
, Error
**errp
)
129 PageCache
*new_cache
;
132 /* Check for truncation */
133 if (new_size
!= (size_t)new_size
) {
134 error_setg(errp
, QERR_INVALID_PARAMETER_VALUE
, "cache size",
135 "exceeding address space");
139 if (new_size
== migrate_xbzrle_cache_size()) {
146 if (XBZRLE
.cache
!= NULL
) {
147 new_cache
= cache_init(new_size
, TARGET_PAGE_SIZE
, errp
);
153 cache_fini(XBZRLE
.cache
);
154 XBZRLE
.cache
= new_cache
;
157 XBZRLE_cache_unlock();
161 static bool ramblock_is_ignored(RAMBlock
*block
)
163 return !qemu_ram_is_migratable(block
) ||
164 (migrate_ignore_shared() && qemu_ram_is_shared(block
));
167 /* Should be holding either ram_list.mutex, or the RCU lock. */
168 #define RAMBLOCK_FOREACH_NOT_IGNORED(block) \
169 INTERNAL_RAMBLOCK_FOREACH(block) \
170 if (ramblock_is_ignored(block)) {} else
172 #define RAMBLOCK_FOREACH_MIGRATABLE(block) \
173 INTERNAL_RAMBLOCK_FOREACH(block) \
174 if (!qemu_ram_is_migratable(block)) {} else
176 #undef RAMBLOCK_FOREACH
178 int foreach_not_ignored_block(RAMBlockIterFunc func
, void *opaque
)
183 RCU_READ_LOCK_GUARD();
185 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
186 ret
= func(block
, opaque
);
194 static void ramblock_recv_map_init(void)
198 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
199 assert(!rb
->receivedmap
);
200 rb
->receivedmap
= bitmap_new(rb
->max_length
>> qemu_target_page_bits());
204 int ramblock_recv_bitmap_test(RAMBlock
*rb
, void *host_addr
)
206 return test_bit(ramblock_recv_bitmap_offset(host_addr
, rb
),
210 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock
*rb
, uint64_t byte_offset
)
212 return test_bit(byte_offset
>> TARGET_PAGE_BITS
, rb
->receivedmap
);
215 void ramblock_recv_bitmap_set(RAMBlock
*rb
, void *host_addr
)
217 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr
, rb
), rb
->receivedmap
);
220 void ramblock_recv_bitmap_set_range(RAMBlock
*rb
, void *host_addr
,
223 bitmap_set_atomic(rb
->receivedmap
,
224 ramblock_recv_bitmap_offset(host_addr
, rb
),
228 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
231 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
233 * Returns >0 if success with sent bytes, or <0 if error.
235 int64_t ramblock_recv_bitmap_send(QEMUFile
*file
,
236 const char *block_name
)
238 RAMBlock
*block
= qemu_ram_block_by_name(block_name
);
239 unsigned long *le_bitmap
, nbits
;
243 error_report("%s: invalid block name: %s", __func__
, block_name
);
247 nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
250 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
251 * machines we may need 4 more bytes for padding (see below
252 * comment). So extend it a bit before hand.
254 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
257 * Always use little endian when sending the bitmap. This is
258 * required that when source and destination VMs are not using the
259 * same endianess. (Note: big endian won't work.)
261 bitmap_to_le(le_bitmap
, block
->receivedmap
, nbits
);
263 /* Size of the bitmap, in bytes */
264 size
= DIV_ROUND_UP(nbits
, 8);
267 * size is always aligned to 8 bytes for 64bit machines, but it
268 * may not be true for 32bit machines. We need this padding to
269 * make sure the migration can survive even between 32bit and
272 size
= ROUND_UP(size
, 8);
274 qemu_put_be64(file
, size
);
275 qemu_put_buffer(file
, (const uint8_t *)le_bitmap
, size
);
277 * Mark as an end, in case the middle part is screwed up due to
278 * some "misterious" reason.
280 qemu_put_be64(file
, RAMBLOCK_RECV_BITMAP_ENDING
);
285 if (qemu_file_get_error(file
)) {
286 return qemu_file_get_error(file
);
289 return size
+ sizeof(size
);
293 * An outstanding page request, on the source, having been received
296 struct RAMSrcPageRequest
{
301 QSIMPLEQ_ENTRY(RAMSrcPageRequest
) next_req
;
304 /* State of RAM for migration */
306 /* QEMUFile used for this migration */
308 /* Last block that we have visited searching for dirty pages */
309 RAMBlock
*last_seen_block
;
310 /* Last block from where we have sent data */
311 RAMBlock
*last_sent_block
;
312 /* Last dirty target page we have sent */
313 ram_addr_t last_page
;
314 /* last ram version we have seen */
315 uint32_t last_version
;
316 /* We are in the first round */
318 /* The free page optimization is enabled */
320 /* How many times we have dirty too many pages */
321 int dirty_rate_high_cnt
;
322 /* these variables are used for bitmap sync */
323 /* last time we did a full bitmap_sync */
324 int64_t time_last_bitmap_sync
;
325 /* bytes transferred at start_time */
326 uint64_t bytes_xfer_prev
;
327 /* number of dirty pages since start_time */
328 uint64_t num_dirty_pages_period
;
329 /* xbzrle misses since the beginning of the period */
330 uint64_t xbzrle_cache_miss_prev
;
332 /* compression statistics since the beginning of the period */
333 /* amount of count that no free thread to compress data */
334 uint64_t compress_thread_busy_prev
;
335 /* amount bytes after compression */
336 uint64_t compressed_size_prev
;
337 /* amount of compressed pages */
338 uint64_t compress_pages_prev
;
340 /* total handled target pages at the beginning of period */
341 uint64_t target_page_count_prev
;
342 /* total handled target pages since start */
343 uint64_t target_page_count
;
344 /* number of dirty bits in the bitmap */
345 uint64_t migration_dirty_pages
;
346 /* Protects modification of the bitmap and migration dirty pages */
347 QemuMutex bitmap_mutex
;
348 /* The RAMBlock used in the last src_page_requests */
349 RAMBlock
*last_req_rb
;
350 /* Queue of outstanding page requests from the destination */
351 QemuMutex src_page_req_mutex
;
352 QSIMPLEQ_HEAD(, RAMSrcPageRequest
) src_page_requests
;
354 typedef struct RAMState RAMState
;
356 static RAMState
*ram_state
;
358 static NotifierWithReturnList precopy_notifier_list
;
360 void precopy_infrastructure_init(void)
362 notifier_with_return_list_init(&precopy_notifier_list
);
365 void precopy_add_notifier(NotifierWithReturn
*n
)
367 notifier_with_return_list_add(&precopy_notifier_list
, n
);
370 void precopy_remove_notifier(NotifierWithReturn
*n
)
372 notifier_with_return_remove(n
);
375 int precopy_notify(PrecopyNotifyReason reason
, Error
**errp
)
377 PrecopyNotifyData pnd
;
381 return notifier_with_return_list_notify(&precopy_notifier_list
, &pnd
);
384 void precopy_enable_free_page_optimization(void)
390 ram_state
->fpo_enabled
= true;
393 uint64_t ram_bytes_remaining(void)
395 return ram_state
? (ram_state
->migration_dirty_pages
* TARGET_PAGE_SIZE
) :
399 MigrationStats ram_counters
;
401 /* used by the search for pages to send */
402 struct PageSearchStatus
{
403 /* Current block being searched */
405 /* Current page to search from */
407 /* Set once we wrap around */
410 typedef struct PageSearchStatus PageSearchStatus
;
412 CompressionStats compression_counters
;
414 struct CompressParam
{
424 /* internally used fields */
428 typedef struct CompressParam CompressParam
;
430 struct DecompressParam
{
440 typedef struct DecompressParam DecompressParam
;
442 static CompressParam
*comp_param
;
443 static QemuThread
*compress_threads
;
444 /* comp_done_cond is used to wake up the migration thread when
445 * one of the compression threads has finished the compression.
446 * comp_done_lock is used to co-work with comp_done_cond.
448 static QemuMutex comp_done_lock
;
449 static QemuCond comp_done_cond
;
450 /* The empty QEMUFileOps will be used by file in CompressParam */
451 static const QEMUFileOps empty_ops
= { };
453 static QEMUFile
*decomp_file
;
454 static DecompressParam
*decomp_param
;
455 static QemuThread
*decompress_threads
;
456 static QemuMutex decomp_done_lock
;
457 static QemuCond decomp_done_cond
;
459 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
460 ram_addr_t offset
, uint8_t *source_buf
);
462 static void *do_data_compress(void *opaque
)
464 CompressParam
*param
= opaque
;
469 qemu_mutex_lock(¶m
->mutex
);
470 while (!param
->quit
) {
472 block
= param
->block
;
473 offset
= param
->offset
;
475 qemu_mutex_unlock(¶m
->mutex
);
477 zero_page
= do_compress_ram_page(param
->file
, ¶m
->stream
,
478 block
, offset
, param
->originbuf
);
480 qemu_mutex_lock(&comp_done_lock
);
482 param
->zero_page
= zero_page
;
483 qemu_cond_signal(&comp_done_cond
);
484 qemu_mutex_unlock(&comp_done_lock
);
486 qemu_mutex_lock(¶m
->mutex
);
488 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
491 qemu_mutex_unlock(¶m
->mutex
);
496 static void compress_threads_save_cleanup(void)
500 if (!migrate_use_compression() || !comp_param
) {
504 thread_count
= migrate_compress_threads();
505 for (i
= 0; i
< thread_count
; i
++) {
507 * we use it as a indicator which shows if the thread is
508 * properly init'd or not
510 if (!comp_param
[i
].file
) {
514 qemu_mutex_lock(&comp_param
[i
].mutex
);
515 comp_param
[i
].quit
= true;
516 qemu_cond_signal(&comp_param
[i
].cond
);
517 qemu_mutex_unlock(&comp_param
[i
].mutex
);
519 qemu_thread_join(compress_threads
+ i
);
520 qemu_mutex_destroy(&comp_param
[i
].mutex
);
521 qemu_cond_destroy(&comp_param
[i
].cond
);
522 deflateEnd(&comp_param
[i
].stream
);
523 g_free(comp_param
[i
].originbuf
);
524 qemu_fclose(comp_param
[i
].file
);
525 comp_param
[i
].file
= NULL
;
527 qemu_mutex_destroy(&comp_done_lock
);
528 qemu_cond_destroy(&comp_done_cond
);
529 g_free(compress_threads
);
531 compress_threads
= NULL
;
535 static int compress_threads_save_setup(void)
539 if (!migrate_use_compression()) {
542 thread_count
= migrate_compress_threads();
543 compress_threads
= g_new0(QemuThread
, thread_count
);
544 comp_param
= g_new0(CompressParam
, thread_count
);
545 qemu_cond_init(&comp_done_cond
);
546 qemu_mutex_init(&comp_done_lock
);
547 for (i
= 0; i
< thread_count
; i
++) {
548 comp_param
[i
].originbuf
= g_try_malloc(TARGET_PAGE_SIZE
);
549 if (!comp_param
[i
].originbuf
) {
553 if (deflateInit(&comp_param
[i
].stream
,
554 migrate_compress_level()) != Z_OK
) {
555 g_free(comp_param
[i
].originbuf
);
559 /* comp_param[i].file is just used as a dummy buffer to save data,
560 * set its ops to empty.
562 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
563 comp_param
[i
].done
= true;
564 comp_param
[i
].quit
= false;
565 qemu_mutex_init(&comp_param
[i
].mutex
);
566 qemu_cond_init(&comp_param
[i
].cond
);
567 qemu_thread_create(compress_threads
+ i
, "compress",
568 do_data_compress
, comp_param
+ i
,
569 QEMU_THREAD_JOINABLE
);
574 compress_threads_save_cleanup();
580 #define MULTIFD_MAGIC 0x11223344U
581 #define MULTIFD_VERSION 1
583 #define MULTIFD_FLAG_SYNC (1 << 0)
585 /* This value needs to be a multiple of qemu_target_page_size() */
586 #define MULTIFD_PACKET_SIZE (512 * 1024)
591 unsigned char uuid
[16]; /* QemuUUID */
593 uint8_t unused1
[7]; /* Reserved for future use */
594 uint64_t unused2
[4]; /* Reserved for future use */
595 } __attribute__((packed
)) MultiFDInit_t
;
601 /* maximum number of allocated pages */
602 uint32_t pages_alloc
;
604 /* size of the next packet that contains pages */
605 uint32_t next_packet_size
;
607 uint64_t unused
[4]; /* Reserved for future use */
610 } __attribute__((packed
)) MultiFDPacket_t
;
613 /* number of used pages */
615 /* number of allocated pages */
617 /* global number of generated multifd packets */
619 /* offset of each page */
621 /* pointer to each page */
627 /* this fields are not changed once the thread is created */
630 /* channel thread name */
632 /* channel thread id */
634 /* communication channel */
636 /* sem where to wait for more work */
638 /* this mutex protects the following parameters */
640 /* is this channel thread running */
642 /* should this thread finish */
644 /* thread has work to do */
646 /* array of pages to sent */
647 MultiFDPages_t
*pages
;
648 /* packet allocated len */
650 /* pointer to the packet */
651 MultiFDPacket_t
*packet
;
652 /* multifd flags for each packet */
654 /* size of the next packet that contains pages */
655 uint32_t next_packet_size
;
656 /* global number of generated multifd packets */
658 /* thread local variables */
659 /* packets sent through this channel */
660 uint64_t num_packets
;
661 /* pages sent through this channel */
663 /* syncs main thread and channels */
664 QemuSemaphore sem_sync
;
668 /* this fields are not changed once the thread is created */
671 /* channel thread name */
673 /* channel thread id */
675 /* communication channel */
677 /* this mutex protects the following parameters */
679 /* is this channel thread running */
681 /* should this thread finish */
683 /* array of pages to receive */
684 MultiFDPages_t
*pages
;
685 /* packet allocated len */
687 /* pointer to the packet */
688 MultiFDPacket_t
*packet
;
689 /* multifd flags for each packet */
691 /* global number of generated multifd packets */
693 /* thread local variables */
694 /* size of the next packet that contains pages */
695 uint32_t next_packet_size
;
696 /* packets sent through this channel */
697 uint64_t num_packets
;
698 /* pages sent through this channel */
700 /* syncs main thread and channels */
701 QemuSemaphore sem_sync
;
704 static int multifd_send_initial_packet(MultiFDSendParams
*p
, Error
**errp
)
706 MultiFDInit_t msg
= {};
709 msg
.magic
= cpu_to_be32(MULTIFD_MAGIC
);
710 msg
.version
= cpu_to_be32(MULTIFD_VERSION
);
712 memcpy(msg
.uuid
, &qemu_uuid
.data
, sizeof(msg
.uuid
));
714 ret
= qio_channel_write_all(p
->c
, (char *)&msg
, sizeof(msg
), errp
);
721 static int multifd_recv_initial_packet(QIOChannel
*c
, Error
**errp
)
726 ret
= qio_channel_read_all(c
, (char *)&msg
, sizeof(msg
), errp
);
731 msg
.magic
= be32_to_cpu(msg
.magic
);
732 msg
.version
= be32_to_cpu(msg
.version
);
734 if (msg
.magic
!= MULTIFD_MAGIC
) {
735 error_setg(errp
, "multifd: received packet magic %x "
736 "expected %x", msg
.magic
, MULTIFD_MAGIC
);
740 if (msg
.version
!= MULTIFD_VERSION
) {
741 error_setg(errp
, "multifd: received packet version %d "
742 "expected %d", msg
.version
, MULTIFD_VERSION
);
746 if (memcmp(msg
.uuid
, &qemu_uuid
, sizeof(qemu_uuid
))) {
747 char *uuid
= qemu_uuid_unparse_strdup(&qemu_uuid
);
748 char *msg_uuid
= qemu_uuid_unparse_strdup((const QemuUUID
*)msg
.uuid
);
750 error_setg(errp
, "multifd: received uuid '%s' and expected "
751 "uuid '%s' for channel %hhd", msg_uuid
, uuid
, msg
.id
);
757 if (msg
.id
> migrate_multifd_channels()) {
758 error_setg(errp
, "multifd: received channel version %d "
759 "expected %d", msg
.version
, MULTIFD_VERSION
);
766 static MultiFDPages_t
*multifd_pages_init(size_t size
)
768 MultiFDPages_t
*pages
= g_new0(MultiFDPages_t
, 1);
770 pages
->allocated
= size
;
771 pages
->iov
= g_new0(struct iovec
, size
);
772 pages
->offset
= g_new0(ram_addr_t
, size
);
777 static void multifd_pages_clear(MultiFDPages_t
*pages
)
780 pages
->allocated
= 0;
781 pages
->packet_num
= 0;
785 g_free(pages
->offset
);
786 pages
->offset
= NULL
;
790 static void multifd_send_fill_packet(MultiFDSendParams
*p
)
792 MultiFDPacket_t
*packet
= p
->packet
;
795 packet
->flags
= cpu_to_be32(p
->flags
);
796 packet
->pages_alloc
= cpu_to_be32(p
->pages
->allocated
);
797 packet
->pages_used
= cpu_to_be32(p
->pages
->used
);
798 packet
->next_packet_size
= cpu_to_be32(p
->next_packet_size
);
799 packet
->packet_num
= cpu_to_be64(p
->packet_num
);
801 if (p
->pages
->block
) {
802 strncpy(packet
->ramblock
, p
->pages
->block
->idstr
, 256);
805 for (i
= 0; i
< p
->pages
->used
; i
++) {
806 /* there are architectures where ram_addr_t is 32 bit */
807 uint64_t temp
= p
->pages
->offset
[i
];
809 packet
->offset
[i
] = cpu_to_be64(temp
);
813 static int multifd_recv_unfill_packet(MultiFDRecvParams
*p
, Error
**errp
)
815 MultiFDPacket_t
*packet
= p
->packet
;
816 uint32_t pages_max
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
820 packet
->magic
= be32_to_cpu(packet
->magic
);
821 if (packet
->magic
!= MULTIFD_MAGIC
) {
822 error_setg(errp
, "multifd: received packet "
823 "magic %x and expected magic %x",
824 packet
->magic
, MULTIFD_MAGIC
);
828 packet
->version
= be32_to_cpu(packet
->version
);
829 if (packet
->version
!= MULTIFD_VERSION
) {
830 error_setg(errp
, "multifd: received packet "
831 "version %d and expected version %d",
832 packet
->version
, MULTIFD_VERSION
);
836 p
->flags
= be32_to_cpu(packet
->flags
);
838 packet
->pages_alloc
= be32_to_cpu(packet
->pages_alloc
);
840 * If we received a packet that is 100 times bigger than expected
841 * just stop migration. It is a magic number.
843 if (packet
->pages_alloc
> pages_max
* 100) {
844 error_setg(errp
, "multifd: received packet "
845 "with size %d and expected a maximum size of %d",
846 packet
->pages_alloc
, pages_max
* 100) ;
850 * We received a packet that is bigger than expected but inside
851 * reasonable limits (see previous comment). Just reallocate.
853 if (packet
->pages_alloc
> p
->pages
->allocated
) {
854 multifd_pages_clear(p
->pages
);
855 p
->pages
= multifd_pages_init(packet
->pages_alloc
);
858 p
->pages
->used
= be32_to_cpu(packet
->pages_used
);
859 if (p
->pages
->used
> packet
->pages_alloc
) {
860 error_setg(errp
, "multifd: received packet "
861 "with %d pages and expected maximum pages are %d",
862 p
->pages
->used
, packet
->pages_alloc
) ;
866 p
->next_packet_size
= be32_to_cpu(packet
->next_packet_size
);
867 p
->packet_num
= be64_to_cpu(packet
->packet_num
);
869 if (p
->pages
->used
== 0) {
873 /* make sure that ramblock is 0 terminated */
874 packet
->ramblock
[255] = 0;
875 block
= qemu_ram_block_by_name(packet
->ramblock
);
877 error_setg(errp
, "multifd: unknown ram block %s",
882 for (i
= 0; i
< p
->pages
->used
; i
++) {
883 uint64_t offset
= be64_to_cpu(packet
->offset
[i
]);
885 if (offset
> (block
->used_length
- qemu_target_page_size())) {
886 error_setg(errp
, "multifd: offset too long %" PRIu64
887 " (max " RAM_ADDR_FMT
")",
888 offset
, block
->max_length
);
891 p
->pages
->iov
[i
].iov_base
= block
->host
+ offset
;
892 p
->pages
->iov
[i
].iov_len
= qemu_target_page_size();
899 MultiFDSendParams
*params
;
900 /* array of pages to sent */
901 MultiFDPages_t
*pages
;
902 /* global number of generated multifd packets */
904 /* send channels ready */
905 QemuSemaphore channels_ready
;
907 * Have we already run terminate threads. There is a race when it
908 * happens that we got one error while we are exiting.
909 * We will use atomic operations. Only valid values are 0 and 1.
912 } *multifd_send_state
;
915 * How we use multifd_send_state->pages and channel->pages?
917 * We create a pages for each channel, and a main one. Each time that
918 * we need to send a batch of pages we interchange the ones between
919 * multifd_send_state and the channel that is sending it. There are
920 * two reasons for that:
921 * - to not have to do so many mallocs during migration
922 * - to make easier to know what to free at the end of migration
924 * This way we always know who is the owner of each "pages" struct,
925 * and we don't need any locking. It belongs to the migration thread
926 * or to the channel thread. Switching is safe because the migration
927 * thread is using the channel mutex when changing it, and the channel
928 * have to had finish with its own, otherwise pending_job can't be
932 static int multifd_send_pages(QEMUFile
*f
)
935 static int next_channel
;
936 MultiFDSendParams
*p
= NULL
; /* make happy gcc */
937 MultiFDPages_t
*pages
= multifd_send_state
->pages
;
938 uint64_t transferred
;
940 if (atomic_read(&multifd_send_state
->exiting
)) {
944 qemu_sem_wait(&multifd_send_state
->channels_ready
);
945 for (i
= next_channel
;; i
= (i
+ 1) % migrate_multifd_channels()) {
946 p
= &multifd_send_state
->params
[i
];
948 qemu_mutex_lock(&p
->mutex
);
950 error_report("%s: channel %d has already quit!", __func__
, i
);
951 qemu_mutex_unlock(&p
->mutex
);
954 if (!p
->pending_job
) {
956 next_channel
= (i
+ 1) % migrate_multifd_channels();
959 qemu_mutex_unlock(&p
->mutex
);
961 assert(!p
->pages
->used
);
962 assert(!p
->pages
->block
);
964 p
->packet_num
= multifd_send_state
->packet_num
++;
965 multifd_send_state
->pages
= p
->pages
;
967 transferred
= ((uint64_t) pages
->used
) * qemu_target_page_size()
969 qemu_file_update_transfer(f
, transferred
);
970 ram_counters
.multifd_bytes
+= transferred
;
971 ram_counters
.transferred
+= transferred
;;
972 qemu_mutex_unlock(&p
->mutex
);
973 qemu_sem_post(&p
->sem
);
978 static int multifd_queue_page(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
)
980 MultiFDPages_t
*pages
= multifd_send_state
->pages
;
983 pages
->block
= block
;
986 if (pages
->block
== block
) {
987 pages
->offset
[pages
->used
] = offset
;
988 pages
->iov
[pages
->used
].iov_base
= block
->host
+ offset
;
989 pages
->iov
[pages
->used
].iov_len
= qemu_target_page_size();
992 if (pages
->used
< pages
->allocated
) {
997 if (multifd_send_pages(f
) < 0) {
1001 if (pages
->block
!= block
) {
1002 return multifd_queue_page(f
, block
, offset
);
1008 static void multifd_send_terminate_threads(Error
*err
)
1012 trace_multifd_send_terminate_threads(err
!= NULL
);
1015 MigrationState
*s
= migrate_get_current();
1016 migrate_set_error(s
, err
);
1017 if (s
->state
== MIGRATION_STATUS_SETUP
||
1018 s
->state
== MIGRATION_STATUS_PRE_SWITCHOVER
||
1019 s
->state
== MIGRATION_STATUS_DEVICE
||
1020 s
->state
== MIGRATION_STATUS_ACTIVE
) {
1021 migrate_set_state(&s
->state
, s
->state
,
1022 MIGRATION_STATUS_FAILED
);
1027 * We don't want to exit each threads twice. Depending on where
1028 * we get the error, or if there are two independent errors in two
1029 * threads at the same time, we can end calling this function
1032 if (atomic_xchg(&multifd_send_state
->exiting
, 1)) {
1036 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1037 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1039 qemu_mutex_lock(&p
->mutex
);
1041 qemu_sem_post(&p
->sem
);
1042 qemu_mutex_unlock(&p
->mutex
);
1046 void multifd_save_cleanup(void)
1050 if (!migrate_use_multifd()) {
1053 multifd_send_terminate_threads(NULL
);
1054 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1055 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1058 qemu_thread_join(&p
->thread
);
1061 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1062 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1064 socket_send_channel_destroy(p
->c
);
1066 qemu_mutex_destroy(&p
->mutex
);
1067 qemu_sem_destroy(&p
->sem
);
1068 qemu_sem_destroy(&p
->sem_sync
);
1071 multifd_pages_clear(p
->pages
);
1077 qemu_sem_destroy(&multifd_send_state
->channels_ready
);
1078 g_free(multifd_send_state
->params
);
1079 multifd_send_state
->params
= NULL
;
1080 multifd_pages_clear(multifd_send_state
->pages
);
1081 multifd_send_state
->pages
= NULL
;
1082 g_free(multifd_send_state
);
1083 multifd_send_state
= NULL
;
1086 static void multifd_send_sync_main(QEMUFile
*f
)
1090 if (!migrate_use_multifd()) {
1093 if (multifd_send_state
->pages
->used
) {
1094 if (multifd_send_pages(f
) < 0) {
1095 error_report("%s: multifd_send_pages fail", __func__
);
1099 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1100 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1102 trace_multifd_send_sync_main_signal(p
->id
);
1104 qemu_mutex_lock(&p
->mutex
);
1107 error_report("%s: channel %d has already quit", __func__
, i
);
1108 qemu_mutex_unlock(&p
->mutex
);
1112 p
->packet_num
= multifd_send_state
->packet_num
++;
1113 p
->flags
|= MULTIFD_FLAG_SYNC
;
1115 qemu_file_update_transfer(f
, p
->packet_len
);
1116 ram_counters
.multifd_bytes
+= p
->packet_len
;
1117 ram_counters
.transferred
+= p
->packet_len
;
1118 qemu_mutex_unlock(&p
->mutex
);
1119 qemu_sem_post(&p
->sem
);
1121 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1122 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1124 trace_multifd_send_sync_main_wait(p
->id
);
1125 qemu_sem_wait(&p
->sem_sync
);
1127 trace_multifd_send_sync_main(multifd_send_state
->packet_num
);
1130 static void *multifd_send_thread(void *opaque
)
1132 MultiFDSendParams
*p
= opaque
;
1133 Error
*local_err
= NULL
;
1137 trace_multifd_send_thread_start(p
->id
);
1138 rcu_register_thread();
1140 if (multifd_send_initial_packet(p
, &local_err
) < 0) {
1144 /* initial packet */
1148 qemu_sem_wait(&p
->sem
);
1150 if (atomic_read(&multifd_send_state
->exiting
)) {
1153 qemu_mutex_lock(&p
->mutex
);
1155 if (p
->pending_job
) {
1156 uint32_t used
= p
->pages
->used
;
1157 uint64_t packet_num
= p
->packet_num
;
1160 p
->next_packet_size
= used
* qemu_target_page_size();
1161 multifd_send_fill_packet(p
);
1164 p
->num_pages
+= used
;
1166 p
->pages
->block
= NULL
;
1167 qemu_mutex_unlock(&p
->mutex
);
1169 trace_multifd_send(p
->id
, packet_num
, used
, flags
,
1170 p
->next_packet_size
);
1172 ret
= qio_channel_write_all(p
->c
, (void *)p
->packet
,
1173 p
->packet_len
, &local_err
);
1179 ret
= qio_channel_writev_all(p
->c
, p
->pages
->iov
,
1186 qemu_mutex_lock(&p
->mutex
);
1188 qemu_mutex_unlock(&p
->mutex
);
1190 if (flags
& MULTIFD_FLAG_SYNC
) {
1191 qemu_sem_post(&p
->sem_sync
);
1193 qemu_sem_post(&multifd_send_state
->channels_ready
);
1194 } else if (p
->quit
) {
1195 qemu_mutex_unlock(&p
->mutex
);
1198 qemu_mutex_unlock(&p
->mutex
);
1199 /* sometimes there are spurious wakeups */
1205 trace_multifd_send_error(p
->id
);
1206 multifd_send_terminate_threads(local_err
);
1210 * Error happen, I will exit, but I can't just leave, tell
1211 * who pay attention to me.
1214 qemu_sem_post(&p
->sem_sync
);
1215 qemu_sem_post(&multifd_send_state
->channels_ready
);
1218 qemu_mutex_lock(&p
->mutex
);
1220 qemu_mutex_unlock(&p
->mutex
);
1222 rcu_unregister_thread();
1223 trace_multifd_send_thread_end(p
->id
, p
->num_packets
, p
->num_pages
);
1228 static void multifd_new_send_channel_async(QIOTask
*task
, gpointer opaque
)
1230 MultiFDSendParams
*p
= opaque
;
1231 QIOChannel
*sioc
= QIO_CHANNEL(qio_task_get_source(task
));
1232 Error
*local_err
= NULL
;
1234 trace_multifd_new_send_channel_async(p
->id
);
1235 if (qio_task_propagate_error(task
, &local_err
)) {
1236 migrate_set_error(migrate_get_current(), local_err
);
1237 /* Error happen, we need to tell who pay attention to me */
1238 qemu_sem_post(&multifd_send_state
->channels_ready
);
1239 qemu_sem_post(&p
->sem_sync
);
1241 * Although multifd_send_thread is not created, but main migration
1242 * thread neet to judge whether it is running, so we need to mark
1247 p
->c
= QIO_CHANNEL(sioc
);
1248 qio_channel_set_delay(p
->c
, false);
1250 qemu_thread_create(&p
->thread
, p
->name
, multifd_send_thread
, p
,
1251 QEMU_THREAD_JOINABLE
);
1255 int multifd_save_setup(Error
**errp
)
1258 uint32_t page_count
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
1261 if (!migrate_use_multifd()) {
1264 thread_count
= migrate_multifd_channels();
1265 multifd_send_state
= g_malloc0(sizeof(*multifd_send_state
));
1266 multifd_send_state
->params
= g_new0(MultiFDSendParams
, thread_count
);
1267 multifd_send_state
->pages
= multifd_pages_init(page_count
);
1268 qemu_sem_init(&multifd_send_state
->channels_ready
, 0);
1269 atomic_set(&multifd_send_state
->exiting
, 0);
1271 for (i
= 0; i
< thread_count
; i
++) {
1272 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1274 qemu_mutex_init(&p
->mutex
);
1275 qemu_sem_init(&p
->sem
, 0);
1276 qemu_sem_init(&p
->sem_sync
, 0);
1280 p
->pages
= multifd_pages_init(page_count
);
1281 p
->packet_len
= sizeof(MultiFDPacket_t
)
1282 + sizeof(uint64_t) * page_count
;
1283 p
->packet
= g_malloc0(p
->packet_len
);
1284 p
->packet
->magic
= cpu_to_be32(MULTIFD_MAGIC
);
1285 p
->packet
->version
= cpu_to_be32(MULTIFD_VERSION
);
1286 p
->name
= g_strdup_printf("multifdsend_%d", i
);
1287 socket_send_channel_create(multifd_new_send_channel_async
, p
);
1293 MultiFDRecvParams
*params
;
1294 /* number of created threads */
1296 /* syncs main thread and channels */
1297 QemuSemaphore sem_sync
;
1298 /* global number of generated multifd packets */
1299 uint64_t packet_num
;
1300 } *multifd_recv_state
;
1302 static void multifd_recv_terminate_threads(Error
*err
)
1306 trace_multifd_recv_terminate_threads(err
!= NULL
);
1309 MigrationState
*s
= migrate_get_current();
1310 migrate_set_error(s
, err
);
1311 if (s
->state
== MIGRATION_STATUS_SETUP
||
1312 s
->state
== MIGRATION_STATUS_ACTIVE
) {
1313 migrate_set_state(&s
->state
, s
->state
,
1314 MIGRATION_STATUS_FAILED
);
1318 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1319 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1321 qemu_mutex_lock(&p
->mutex
);
1324 * We could arrive here for two reasons:
1325 * - normal quit, i.e. everything went fine, just finished
1326 * - error quit: We close the channels so the channel threads
1327 * finish the qio_channel_read_all_eof()
1330 qio_channel_shutdown(p
->c
, QIO_CHANNEL_SHUTDOWN_BOTH
, NULL
);
1332 qemu_mutex_unlock(&p
->mutex
);
1336 int multifd_load_cleanup(Error
**errp
)
1341 if (!migrate_use_multifd()) {
1344 multifd_recv_terminate_threads(NULL
);
1345 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1346 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1351 * multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code,
1352 * however try to wakeup it without harm in cleanup phase.
1354 qemu_sem_post(&p
->sem_sync
);
1355 qemu_thread_join(&p
->thread
);
1358 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1359 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1361 object_unref(OBJECT(p
->c
));
1363 qemu_mutex_destroy(&p
->mutex
);
1364 qemu_sem_destroy(&p
->sem_sync
);
1367 multifd_pages_clear(p
->pages
);
1373 qemu_sem_destroy(&multifd_recv_state
->sem_sync
);
1374 g_free(multifd_recv_state
->params
);
1375 multifd_recv_state
->params
= NULL
;
1376 g_free(multifd_recv_state
);
1377 multifd_recv_state
= NULL
;
1382 static void multifd_recv_sync_main(void)
1386 if (!migrate_use_multifd()) {
1389 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1390 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1392 trace_multifd_recv_sync_main_wait(p
->id
);
1393 qemu_sem_wait(&multifd_recv_state
->sem_sync
);
1395 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1396 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1398 qemu_mutex_lock(&p
->mutex
);
1399 if (multifd_recv_state
->packet_num
< p
->packet_num
) {
1400 multifd_recv_state
->packet_num
= p
->packet_num
;
1402 qemu_mutex_unlock(&p
->mutex
);
1403 trace_multifd_recv_sync_main_signal(p
->id
);
1404 qemu_sem_post(&p
->sem_sync
);
1406 trace_multifd_recv_sync_main(multifd_recv_state
->packet_num
);
1409 static void *multifd_recv_thread(void *opaque
)
1411 MultiFDRecvParams
*p
= opaque
;
1412 Error
*local_err
= NULL
;
1415 trace_multifd_recv_thread_start(p
->id
);
1416 rcu_register_thread();
1426 ret
= qio_channel_read_all_eof(p
->c
, (void *)p
->packet
,
1427 p
->packet_len
, &local_err
);
1428 if (ret
== 0) { /* EOF */
1431 if (ret
== -1) { /* Error */
1435 qemu_mutex_lock(&p
->mutex
);
1436 ret
= multifd_recv_unfill_packet(p
, &local_err
);
1438 qemu_mutex_unlock(&p
->mutex
);
1442 used
= p
->pages
->used
;
1444 trace_multifd_recv(p
->id
, p
->packet_num
, used
, flags
,
1445 p
->next_packet_size
);
1447 p
->num_pages
+= used
;
1448 qemu_mutex_unlock(&p
->mutex
);
1451 ret
= qio_channel_readv_all(p
->c
, p
->pages
->iov
,
1458 if (flags
& MULTIFD_FLAG_SYNC
) {
1459 qemu_sem_post(&multifd_recv_state
->sem_sync
);
1460 qemu_sem_wait(&p
->sem_sync
);
1465 multifd_recv_terminate_threads(local_err
);
1467 qemu_mutex_lock(&p
->mutex
);
1469 qemu_mutex_unlock(&p
->mutex
);
1471 rcu_unregister_thread();
1472 trace_multifd_recv_thread_end(p
->id
, p
->num_packets
, p
->num_pages
);
1477 int multifd_load_setup(Error
**errp
)
1480 uint32_t page_count
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
1483 if (!migrate_use_multifd()) {
1486 thread_count
= migrate_multifd_channels();
1487 multifd_recv_state
= g_malloc0(sizeof(*multifd_recv_state
));
1488 multifd_recv_state
->params
= g_new0(MultiFDRecvParams
, thread_count
);
1489 atomic_set(&multifd_recv_state
->count
, 0);
1490 qemu_sem_init(&multifd_recv_state
->sem_sync
, 0);
1492 for (i
= 0; i
< thread_count
; i
++) {
1493 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1495 qemu_mutex_init(&p
->mutex
);
1496 qemu_sem_init(&p
->sem_sync
, 0);
1499 p
->pages
= multifd_pages_init(page_count
);
1500 p
->packet_len
= sizeof(MultiFDPacket_t
)
1501 + sizeof(uint64_t) * page_count
;
1502 p
->packet
= g_malloc0(p
->packet_len
);
1503 p
->name
= g_strdup_printf("multifdrecv_%d", i
);
1508 bool multifd_recv_all_channels_created(void)
1510 int thread_count
= migrate_multifd_channels();
1512 if (!migrate_use_multifd()) {
1516 return thread_count
== atomic_read(&multifd_recv_state
->count
);
1520 * Try to receive all multifd channels to get ready for the migration.
1521 * - Return true and do not set @errp when correctly receving all channels;
1522 * - Return false and do not set @errp when correctly receiving the current one;
1523 * - Return false and set @errp when failing to receive the current channel.
1525 bool multifd_recv_new_channel(QIOChannel
*ioc
, Error
**errp
)
1527 MultiFDRecvParams
*p
;
1528 Error
*local_err
= NULL
;
1531 id
= multifd_recv_initial_packet(ioc
, &local_err
);
1533 multifd_recv_terminate_threads(local_err
);
1534 error_propagate_prepend(errp
, local_err
,
1535 "failed to receive packet"
1536 " via multifd channel %d: ",
1537 atomic_read(&multifd_recv_state
->count
));
1540 trace_multifd_recv_new_channel(id
);
1542 p
= &multifd_recv_state
->params
[id
];
1544 error_setg(&local_err
, "multifd: received id '%d' already setup'",
1546 multifd_recv_terminate_threads(local_err
);
1547 error_propagate(errp
, local_err
);
1551 object_ref(OBJECT(ioc
));
1552 /* initial packet */
1556 qemu_thread_create(&p
->thread
, p
->name
, multifd_recv_thread
, p
,
1557 QEMU_THREAD_JOINABLE
);
1558 atomic_inc(&multifd_recv_state
->count
);
1559 return atomic_read(&multifd_recv_state
->count
) ==
1560 migrate_multifd_channels();
1564 * save_page_header: write page header to wire
1566 * If this is the 1st block, it also writes the block identification
1568 * Returns the number of bytes written
1570 * @f: QEMUFile where to send the data
1571 * @block: block that contains the page we want to send
1572 * @offset: offset inside the block for the page
1573 * in the lower bits, it contains flags
1575 static size_t save_page_header(RAMState
*rs
, QEMUFile
*f
, RAMBlock
*block
,
1580 if (block
== rs
->last_sent_block
) {
1581 offset
|= RAM_SAVE_FLAG_CONTINUE
;
1583 qemu_put_be64(f
, offset
);
1586 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
1587 len
= strlen(block
->idstr
);
1588 qemu_put_byte(f
, len
);
1589 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
1591 rs
->last_sent_block
= block
;
1597 * mig_throttle_guest_down: throotle down the guest
1599 * Reduce amount of guest cpu execution to hopefully slow down memory
1600 * writes. If guest dirty memory rate is reduced below the rate at
1601 * which we can transfer pages to the destination then we should be
1602 * able to complete migration. Some workloads dirty memory way too
1603 * fast and will not effectively converge, even with auto-converge.
1605 static void mig_throttle_guest_down(void)
1607 MigrationState
*s
= migrate_get_current();
1608 uint64_t pct_initial
= s
->parameters
.cpu_throttle_initial
;
1609 uint64_t pct_icrement
= s
->parameters
.cpu_throttle_increment
;
1610 int pct_max
= s
->parameters
.max_cpu_throttle
;
1612 /* We have not started throttling yet. Let's start it. */
1613 if (!cpu_throttle_active()) {
1614 cpu_throttle_set(pct_initial
);
1616 /* Throttling already on, just increase the rate */
1617 cpu_throttle_set(MIN(cpu_throttle_get_percentage() + pct_icrement
,
1623 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
1625 * @rs: current RAM state
1626 * @current_addr: address for the zero page
1628 * Update the xbzrle cache to reflect a page that's been sent as all 0.
1629 * The important thing is that a stale (not-yet-0'd) page be replaced
1631 * As a bonus, if the page wasn't in the cache it gets added so that
1632 * when a small write is made into the 0'd page it gets XBZRLE sent.
1634 static void xbzrle_cache_zero_page(RAMState
*rs
, ram_addr_t current_addr
)
1636 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
1640 /* We don't care if this fails to allocate a new cache page
1641 * as long as it updated an old one */
1642 cache_insert(XBZRLE
.cache
, current_addr
, XBZRLE
.zero_target_page
,
1643 ram_counters
.dirty_sync_count
);
1646 #define ENCODING_FLAG_XBZRLE 0x1
1649 * save_xbzrle_page: compress and send current page
1651 * Returns: 1 means that we wrote the page
1652 * 0 means that page is identical to the one already sent
1653 * -1 means that xbzrle would be longer than normal
1655 * @rs: current RAM state
1656 * @current_data: pointer to the address of the page contents
1657 * @current_addr: addr of the page
1658 * @block: block that contains the page we want to send
1659 * @offset: offset inside the block for the page
1660 * @last_stage: if we are at the completion stage
1662 static int save_xbzrle_page(RAMState
*rs
, uint8_t **current_data
,
1663 ram_addr_t current_addr
, RAMBlock
*block
,
1664 ram_addr_t offset
, bool last_stage
)
1666 int encoded_len
= 0, bytes_xbzrle
;
1667 uint8_t *prev_cached_page
;
1669 if (!cache_is_cached(XBZRLE
.cache
, current_addr
,
1670 ram_counters
.dirty_sync_count
)) {
1671 xbzrle_counters
.cache_miss
++;
1673 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
1674 ram_counters
.dirty_sync_count
) == -1) {
1677 /* update *current_data when the page has been
1678 inserted into cache */
1679 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
1685 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
1687 /* save current buffer into memory */
1688 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
1690 /* XBZRLE encoding (if there is no overflow) */
1691 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
1692 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
1696 * Update the cache contents, so that it corresponds to the data
1697 * sent, in all cases except where we skip the page.
1699 if (!last_stage
&& encoded_len
!= 0) {
1700 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
1702 * In the case where we couldn't compress, ensure that the caller
1703 * sends the data from the cache, since the guest might have
1704 * changed the RAM since we copied it.
1706 *current_data
= prev_cached_page
;
1709 if (encoded_len
== 0) {
1710 trace_save_xbzrle_page_skipping();
1712 } else if (encoded_len
== -1) {
1713 trace_save_xbzrle_page_overflow();
1714 xbzrle_counters
.overflow
++;
1718 /* Send XBZRLE based compressed page */
1719 bytes_xbzrle
= save_page_header(rs
, rs
->f
, block
,
1720 offset
| RAM_SAVE_FLAG_XBZRLE
);
1721 qemu_put_byte(rs
->f
, ENCODING_FLAG_XBZRLE
);
1722 qemu_put_be16(rs
->f
, encoded_len
);
1723 qemu_put_buffer(rs
->f
, XBZRLE
.encoded_buf
, encoded_len
);
1724 bytes_xbzrle
+= encoded_len
+ 1 + 2;
1725 xbzrle_counters
.pages
++;
1726 xbzrle_counters
.bytes
+= bytes_xbzrle
;
1727 ram_counters
.transferred
+= bytes_xbzrle
;
1733 * migration_bitmap_find_dirty: find the next dirty page from start
1735 * Returns the page offset within memory region of the start of a dirty page
1737 * @rs: current RAM state
1738 * @rb: RAMBlock where to search for dirty pages
1739 * @start: page where we start the search
1742 unsigned long migration_bitmap_find_dirty(RAMState
*rs
, RAMBlock
*rb
,
1743 unsigned long start
)
1745 unsigned long size
= rb
->used_length
>> TARGET_PAGE_BITS
;
1746 unsigned long *bitmap
= rb
->bmap
;
1749 if (ramblock_is_ignored(rb
)) {
1754 * When the free page optimization is enabled, we need to check the bitmap
1755 * to send the non-free pages rather than all the pages in the bulk stage.
1757 if (!rs
->fpo_enabled
&& rs
->ram_bulk_stage
&& start
> 0) {
1760 next
= find_next_bit(bitmap
, size
, start
);
1766 static inline bool migration_bitmap_clear_dirty(RAMState
*rs
,
1772 qemu_mutex_lock(&rs
->bitmap_mutex
);
1775 * Clear dirty bitmap if needed. This _must_ be called before we
1776 * send any of the page in the chunk because we need to make sure
1777 * we can capture further page content changes when we sync dirty
1778 * log the next time. So as long as we are going to send any of
1779 * the page in the chunk we clear the remote dirty bitmap for all.
1780 * Clearing it earlier won't be a problem, but too late will.
1782 if (rb
->clear_bmap
&& clear_bmap_test_and_clear(rb
, page
)) {
1783 uint8_t shift
= rb
->clear_bmap_shift
;
1784 hwaddr size
= 1ULL << (TARGET_PAGE_BITS
+ shift
);
1785 hwaddr start
= (((ram_addr_t
)page
) << TARGET_PAGE_BITS
) & (-size
);
1788 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
1789 * can make things easier sometimes since then start address
1790 * of the small chunk will always be 64 pages aligned so the
1791 * bitmap will always be aligned to unsigned long. We should
1792 * even be able to remove this restriction but I'm simply
1796 trace_migration_bitmap_clear_dirty(rb
->idstr
, start
, size
, page
);
1797 memory_region_clear_dirty_bitmap(rb
->mr
, start
, size
);
1800 ret
= test_and_clear_bit(page
, rb
->bmap
);
1803 rs
->migration_dirty_pages
--;
1805 qemu_mutex_unlock(&rs
->bitmap_mutex
);
1810 /* Called with RCU critical section */
1811 static void ramblock_sync_dirty_bitmap(RAMState
*rs
, RAMBlock
*rb
)
1813 rs
->migration_dirty_pages
+=
1814 cpu_physical_memory_sync_dirty_bitmap(rb
, 0, rb
->used_length
,
1815 &rs
->num_dirty_pages_period
);
1819 * ram_pagesize_summary: calculate all the pagesizes of a VM
1821 * Returns a summary bitmap of the page sizes of all RAMBlocks
1823 * For VMs with just normal pages this is equivalent to the host page
1824 * size. If it's got some huge pages then it's the OR of all the
1825 * different page sizes.
1827 uint64_t ram_pagesize_summary(void)
1830 uint64_t summary
= 0;
1832 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1833 summary
|= block
->page_size
;
1839 uint64_t ram_get_total_transferred_pages(void)
1841 return ram_counters
.normal
+ ram_counters
.duplicate
+
1842 compression_counters
.pages
+ xbzrle_counters
.pages
;
1845 static void migration_update_rates(RAMState
*rs
, int64_t end_time
)
1847 uint64_t page_count
= rs
->target_page_count
- rs
->target_page_count_prev
;
1848 double compressed_size
;
1850 /* calculate period counters */
1851 ram_counters
.dirty_pages_rate
= rs
->num_dirty_pages_period
* 1000
1852 / (end_time
- rs
->time_last_bitmap_sync
);
1858 if (migrate_use_xbzrle()) {
1859 xbzrle_counters
.cache_miss_rate
= (double)(xbzrle_counters
.cache_miss
-
1860 rs
->xbzrle_cache_miss_prev
) / page_count
;
1861 rs
->xbzrle_cache_miss_prev
= xbzrle_counters
.cache_miss
;
1864 if (migrate_use_compression()) {
1865 compression_counters
.busy_rate
= (double)(compression_counters
.busy
-
1866 rs
->compress_thread_busy_prev
) / page_count
;
1867 rs
->compress_thread_busy_prev
= compression_counters
.busy
;
1869 compressed_size
= compression_counters
.compressed_size
-
1870 rs
->compressed_size_prev
;
1871 if (compressed_size
) {
1872 double uncompressed_size
= (compression_counters
.pages
-
1873 rs
->compress_pages_prev
) * TARGET_PAGE_SIZE
;
1875 /* Compression-Ratio = Uncompressed-size / Compressed-size */
1876 compression_counters
.compression_rate
=
1877 uncompressed_size
/ compressed_size
;
1879 rs
->compress_pages_prev
= compression_counters
.pages
;
1880 rs
->compressed_size_prev
= compression_counters
.compressed_size
;
1885 static void migration_bitmap_sync(RAMState
*rs
)
1889 uint64_t bytes_xfer_now
;
1891 ram_counters
.dirty_sync_count
++;
1893 if (!rs
->time_last_bitmap_sync
) {
1894 rs
->time_last_bitmap_sync
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1897 trace_migration_bitmap_sync_start();
1898 memory_global_dirty_log_sync();
1900 qemu_mutex_lock(&rs
->bitmap_mutex
);
1901 WITH_RCU_READ_LOCK_GUARD() {
1902 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1903 ramblock_sync_dirty_bitmap(rs
, block
);
1905 ram_counters
.remaining
= ram_bytes_remaining();
1907 qemu_mutex_unlock(&rs
->bitmap_mutex
);
1909 memory_global_after_dirty_log_sync();
1910 trace_migration_bitmap_sync_end(rs
->num_dirty_pages_period
);
1912 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1914 /* more than 1 second = 1000 millisecons */
1915 if (end_time
> rs
->time_last_bitmap_sync
+ 1000) {
1916 bytes_xfer_now
= ram_counters
.transferred
;
1918 /* During block migration the auto-converge logic incorrectly detects
1919 * that ram migration makes no progress. Avoid this by disabling the
1920 * throttling logic during the bulk phase of block migration. */
1921 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
1922 /* The following detection logic can be refined later. For now:
1923 Check to see if the dirtied bytes is 50% more than the approx.
1924 amount of bytes that just got transferred since the last time we
1925 were in this routine. If that happens twice, start or increase
1928 if ((rs
->num_dirty_pages_period
* TARGET_PAGE_SIZE
>
1929 (bytes_xfer_now
- rs
->bytes_xfer_prev
) / 2) &&
1930 (++rs
->dirty_rate_high_cnt
>= 2)) {
1931 trace_migration_throttle();
1932 rs
->dirty_rate_high_cnt
= 0;
1933 mig_throttle_guest_down();
1937 migration_update_rates(rs
, end_time
);
1939 rs
->target_page_count_prev
= rs
->target_page_count
;
1941 /* reset period counters */
1942 rs
->time_last_bitmap_sync
= end_time
;
1943 rs
->num_dirty_pages_period
= 0;
1944 rs
->bytes_xfer_prev
= bytes_xfer_now
;
1946 if (migrate_use_events()) {
1947 qapi_event_send_migration_pass(ram_counters
.dirty_sync_count
);
1951 static void migration_bitmap_sync_precopy(RAMState
*rs
)
1953 Error
*local_err
= NULL
;
1956 * The current notifier usage is just an optimization to migration, so we
1957 * don't stop the normal migration process in the error case.
1959 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC
, &local_err
)) {
1960 error_report_err(local_err
);
1963 migration_bitmap_sync(rs
);
1965 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC
, &local_err
)) {
1966 error_report_err(local_err
);
1971 * save_zero_page_to_file: send the zero page to the file
1973 * Returns the size of data written to the file, 0 means the page is not
1976 * @rs: current RAM state
1977 * @file: the file where the data is saved
1978 * @block: block that contains the page we want to send
1979 * @offset: offset inside the block for the page
1981 static int save_zero_page_to_file(RAMState
*rs
, QEMUFile
*file
,
1982 RAMBlock
*block
, ram_addr_t offset
)
1984 uint8_t *p
= block
->host
+ offset
;
1987 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
1988 len
+= save_page_header(rs
, file
, block
, offset
| RAM_SAVE_FLAG_ZERO
);
1989 qemu_put_byte(file
, 0);
1996 * save_zero_page: send the zero page to the stream
1998 * Returns the number of pages written.
2000 * @rs: current RAM state
2001 * @block: block that contains the page we want to send
2002 * @offset: offset inside the block for the page
2004 static int save_zero_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
2006 int len
= save_zero_page_to_file(rs
, rs
->f
, block
, offset
);
2009 ram_counters
.duplicate
++;
2010 ram_counters
.transferred
+= len
;
2016 static void ram_release_pages(const char *rbname
, uint64_t offset
, int pages
)
2018 if (!migrate_release_ram() || !migration_in_postcopy()) {
2022 ram_discard_range(rbname
, offset
, ((ram_addr_t
)pages
) << TARGET_PAGE_BITS
);
2026 * @pages: the number of pages written by the control path,
2028 * > 0 - number of pages written
2030 * Return true if the pages has been saved, otherwise false is returned.
2032 static bool control_save_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
2035 uint64_t bytes_xmit
= 0;
2039 ret
= ram_control_save_page(rs
->f
, block
->offset
, offset
, TARGET_PAGE_SIZE
,
2041 if (ret
== RAM_SAVE_CONTROL_NOT_SUPP
) {
2046 ram_counters
.transferred
+= bytes_xmit
;
2050 if (ret
== RAM_SAVE_CONTROL_DELAYED
) {
2054 if (bytes_xmit
> 0) {
2055 ram_counters
.normal
++;
2056 } else if (bytes_xmit
== 0) {
2057 ram_counters
.duplicate
++;
2064 * directly send the page to the stream
2066 * Returns the number of pages written.
2068 * @rs: current RAM state
2069 * @block: block that contains the page we want to send
2070 * @offset: offset inside the block for the page
2071 * @buf: the page to be sent
2072 * @async: send to page asyncly
2074 static int save_normal_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
2075 uint8_t *buf
, bool async
)
2077 ram_counters
.transferred
+= save_page_header(rs
, rs
->f
, block
,
2078 offset
| RAM_SAVE_FLAG_PAGE
);
2080 qemu_put_buffer_async(rs
->f
, buf
, TARGET_PAGE_SIZE
,
2081 migrate_release_ram() &
2082 migration_in_postcopy());
2084 qemu_put_buffer(rs
->f
, buf
, TARGET_PAGE_SIZE
);
2086 ram_counters
.transferred
+= TARGET_PAGE_SIZE
;
2087 ram_counters
.normal
++;
2092 * ram_save_page: send the given page to the stream
2094 * Returns the number of pages written.
2096 * >=0 - Number of pages written - this might legally be 0
2097 * if xbzrle noticed the page was the same.
2099 * @rs: current RAM state
2100 * @block: block that contains the page we want to send
2101 * @offset: offset inside the block for the page
2102 * @last_stage: if we are at the completion stage
2104 static int ram_save_page(RAMState
*rs
, PageSearchStatus
*pss
, bool last_stage
)
2108 bool send_async
= true;
2109 RAMBlock
*block
= pss
->block
;
2110 ram_addr_t offset
= ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
;
2111 ram_addr_t current_addr
= block
->offset
+ offset
;
2113 p
= block
->host
+ offset
;
2114 trace_ram_save_page(block
->idstr
, (uint64_t)offset
, p
);
2116 XBZRLE_cache_lock();
2117 if (!rs
->ram_bulk_stage
&& !migration_in_postcopy() &&
2118 migrate_use_xbzrle()) {
2119 pages
= save_xbzrle_page(rs
, &p
, current_addr
, block
,
2120 offset
, last_stage
);
2122 /* Can't send this cached data async, since the cache page
2123 * might get updated before it gets to the wire
2129 /* XBZRLE overflow or normal page */
2131 pages
= save_normal_page(rs
, block
, offset
, p
, send_async
);
2134 XBZRLE_cache_unlock();
2139 static int ram_save_multifd_page(RAMState
*rs
, RAMBlock
*block
,
2142 if (multifd_queue_page(rs
->f
, block
, offset
) < 0) {
2145 ram_counters
.normal
++;
2150 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
2151 ram_addr_t offset
, uint8_t *source_buf
)
2153 RAMState
*rs
= ram_state
;
2154 uint8_t *p
= block
->host
+ (offset
& TARGET_PAGE_MASK
);
2155 bool zero_page
= false;
2158 if (save_zero_page_to_file(rs
, f
, block
, offset
)) {
2163 save_page_header(rs
, f
, block
, offset
| RAM_SAVE_FLAG_COMPRESS_PAGE
);
2166 * copy it to a internal buffer to avoid it being modified by VM
2167 * so that we can catch up the error during compression and
2170 memcpy(source_buf
, p
, TARGET_PAGE_SIZE
);
2171 ret
= qemu_put_compression_data(f
, stream
, source_buf
, TARGET_PAGE_SIZE
);
2173 qemu_file_set_error(migrate_get_current()->to_dst_file
, ret
);
2174 error_report("compressed data failed!");
2179 ram_release_pages(block
->idstr
, offset
& TARGET_PAGE_MASK
, 1);
2184 update_compress_thread_counts(const CompressParam
*param
, int bytes_xmit
)
2186 ram_counters
.transferred
+= bytes_xmit
;
2188 if (param
->zero_page
) {
2189 ram_counters
.duplicate
++;
2193 /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
2194 compression_counters
.compressed_size
+= bytes_xmit
- 8;
2195 compression_counters
.pages
++;
2198 static bool save_page_use_compression(RAMState
*rs
);
2200 static void flush_compressed_data(RAMState
*rs
)
2202 int idx
, len
, thread_count
;
2204 if (!save_page_use_compression(rs
)) {
2207 thread_count
= migrate_compress_threads();
2209 qemu_mutex_lock(&comp_done_lock
);
2210 for (idx
= 0; idx
< thread_count
; idx
++) {
2211 while (!comp_param
[idx
].done
) {
2212 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
2215 qemu_mutex_unlock(&comp_done_lock
);
2217 for (idx
= 0; idx
< thread_count
; idx
++) {
2218 qemu_mutex_lock(&comp_param
[idx
].mutex
);
2219 if (!comp_param
[idx
].quit
) {
2220 len
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
2222 * it's safe to fetch zero_page without holding comp_done_lock
2223 * as there is no further request submitted to the thread,
2224 * i.e, the thread should be waiting for a request at this point.
2226 update_compress_thread_counts(&comp_param
[idx
], len
);
2228 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
2232 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
2235 param
->block
= block
;
2236 param
->offset
= offset
;
2239 static int compress_page_with_multi_thread(RAMState
*rs
, RAMBlock
*block
,
2242 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
2243 bool wait
= migrate_compress_wait_thread();
2245 thread_count
= migrate_compress_threads();
2246 qemu_mutex_lock(&comp_done_lock
);
2248 for (idx
= 0; idx
< thread_count
; idx
++) {
2249 if (comp_param
[idx
].done
) {
2250 comp_param
[idx
].done
= false;
2251 bytes_xmit
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
2252 qemu_mutex_lock(&comp_param
[idx
].mutex
);
2253 set_compress_params(&comp_param
[idx
], block
, offset
);
2254 qemu_cond_signal(&comp_param
[idx
].cond
);
2255 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
2257 update_compress_thread_counts(&comp_param
[idx
], bytes_xmit
);
2263 * wait for the free thread if the user specifies 'compress-wait-thread',
2264 * otherwise we will post the page out in the main thread as normal page.
2266 if (pages
< 0 && wait
) {
2267 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
2270 qemu_mutex_unlock(&comp_done_lock
);
2276 * find_dirty_block: find the next dirty page and update any state
2277 * associated with the search process.
2279 * Returns true if a page is found
2281 * @rs: current RAM state
2282 * @pss: data about the state of the current dirty page scan
2283 * @again: set to false if the search has scanned the whole of RAM
2285 static bool find_dirty_block(RAMState
*rs
, PageSearchStatus
*pss
, bool *again
)
2287 pss
->page
= migration_bitmap_find_dirty(rs
, pss
->block
, pss
->page
);
2288 if (pss
->complete_round
&& pss
->block
== rs
->last_seen_block
&&
2289 pss
->page
>= rs
->last_page
) {
2291 * We've been once around the RAM and haven't found anything.
2297 if ((((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
)
2298 >= pss
->block
->used_length
) {
2299 /* Didn't find anything in this RAM Block */
2301 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
2304 * If memory migration starts over, we will meet a dirtied page
2305 * which may still exists in compression threads's ring, so we
2306 * should flush the compressed data to make sure the new page
2307 * is not overwritten by the old one in the destination.
2309 * Also If xbzrle is on, stop using the data compression at this
2310 * point. In theory, xbzrle can do better than compression.
2312 flush_compressed_data(rs
);
2314 /* Hit the end of the list */
2315 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2316 /* Flag that we've looped */
2317 pss
->complete_round
= true;
2318 rs
->ram_bulk_stage
= false;
2320 /* Didn't find anything this time, but try again on the new block */
2324 /* Can go around again, but... */
2326 /* We've found something so probably don't need to */
2332 * unqueue_page: gets a page of the queue
2334 * Helper for 'get_queued_page' - gets a page off the queue
2336 * Returns the block of the page (or NULL if none available)
2338 * @rs: current RAM state
2339 * @offset: used to return the offset within the RAMBlock
2341 static RAMBlock
*unqueue_page(RAMState
*rs
, ram_addr_t
*offset
)
2343 RAMBlock
*block
= NULL
;
2345 if (QSIMPLEQ_EMPTY_ATOMIC(&rs
->src_page_requests
)) {
2349 qemu_mutex_lock(&rs
->src_page_req_mutex
);
2350 if (!QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
2351 struct RAMSrcPageRequest
*entry
=
2352 QSIMPLEQ_FIRST(&rs
->src_page_requests
);
2354 *offset
= entry
->offset
;
2356 if (entry
->len
> TARGET_PAGE_SIZE
) {
2357 entry
->len
-= TARGET_PAGE_SIZE
;
2358 entry
->offset
+= TARGET_PAGE_SIZE
;
2360 memory_region_unref(block
->mr
);
2361 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
2363 migration_consume_urgent_request();
2366 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
2372 * get_queued_page: unqueue a page from the postcopy requests
2374 * Skips pages that are already sent (!dirty)
2376 * Returns true if a queued page is found
2378 * @rs: current RAM state
2379 * @pss: data about the state of the current dirty page scan
2381 static bool get_queued_page(RAMState
*rs
, PageSearchStatus
*pss
)
2388 block
= unqueue_page(rs
, &offset
);
2390 * We're sending this page, and since it's postcopy nothing else
2391 * will dirty it, and we must make sure it doesn't get sent again
2392 * even if this queue request was received after the background
2393 * search already sent it.
2398 page
= offset
>> TARGET_PAGE_BITS
;
2399 dirty
= test_bit(page
, block
->bmap
);
2401 trace_get_queued_page_not_dirty(block
->idstr
, (uint64_t)offset
,
2404 trace_get_queued_page(block
->idstr
, (uint64_t)offset
, page
);
2408 } while (block
&& !dirty
);
2412 * As soon as we start servicing pages out of order, then we have
2413 * to kill the bulk stage, since the bulk stage assumes
2414 * in (migration_bitmap_find_and_reset_dirty) that every page is
2415 * dirty, that's no longer true.
2417 rs
->ram_bulk_stage
= false;
2420 * We want the background search to continue from the queued page
2421 * since the guest is likely to want other pages near to the page
2422 * it just requested.
2425 pss
->page
= offset
>> TARGET_PAGE_BITS
;
2428 * This unqueued page would break the "one round" check, even is
2431 pss
->complete_round
= false;
2438 * migration_page_queue_free: drop any remaining pages in the ram
2441 * It should be empty at the end anyway, but in error cases there may
2442 * be some left. in case that there is any page left, we drop it.
2445 static void migration_page_queue_free(RAMState
*rs
)
2447 struct RAMSrcPageRequest
*mspr
, *next_mspr
;
2448 /* This queue generally should be empty - but in the case of a failed
2449 * migration might have some droppings in.
2451 RCU_READ_LOCK_GUARD();
2452 QSIMPLEQ_FOREACH_SAFE(mspr
, &rs
->src_page_requests
, next_req
, next_mspr
) {
2453 memory_region_unref(mspr
->rb
->mr
);
2454 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
2460 * ram_save_queue_pages: queue the page for transmission
2462 * A request from postcopy destination for example.
2464 * Returns zero on success or negative on error
2466 * @rbname: Name of the RAMBLock of the request. NULL means the
2467 * same that last one.
2468 * @start: starting address from the start of the RAMBlock
2469 * @len: length (in bytes) to send
2471 int ram_save_queue_pages(const char *rbname
, ram_addr_t start
, ram_addr_t len
)
2474 RAMState
*rs
= ram_state
;
2476 ram_counters
.postcopy_requests
++;
2477 RCU_READ_LOCK_GUARD();
2480 /* Reuse last RAMBlock */
2481 ramblock
= rs
->last_req_rb
;
2485 * Shouldn't happen, we can't reuse the last RAMBlock if
2486 * it's the 1st request.
2488 error_report("ram_save_queue_pages no previous block");
2492 ramblock
= qemu_ram_block_by_name(rbname
);
2495 /* We shouldn't be asked for a non-existent RAMBlock */
2496 error_report("ram_save_queue_pages no block '%s'", rbname
);
2499 rs
->last_req_rb
= ramblock
;
2501 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
2502 if (start
+len
> ramblock
->used_length
) {
2503 error_report("%s request overrun start=" RAM_ADDR_FMT
" len="
2504 RAM_ADDR_FMT
" blocklen=" RAM_ADDR_FMT
,
2505 __func__
, start
, len
, ramblock
->used_length
);
2509 struct RAMSrcPageRequest
*new_entry
=
2510 g_malloc0(sizeof(struct RAMSrcPageRequest
));
2511 new_entry
->rb
= ramblock
;
2512 new_entry
->offset
= start
;
2513 new_entry
->len
= len
;
2515 memory_region_ref(ramblock
->mr
);
2516 qemu_mutex_lock(&rs
->src_page_req_mutex
);
2517 QSIMPLEQ_INSERT_TAIL(&rs
->src_page_requests
, new_entry
, next_req
);
2518 migration_make_urgent_request();
2519 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
2524 static bool save_page_use_compression(RAMState
*rs
)
2526 if (!migrate_use_compression()) {
2531 * If xbzrle is on, stop using the data compression after first
2532 * round of migration even if compression is enabled. In theory,
2533 * xbzrle can do better than compression.
2535 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
2543 * try to compress the page before posting it out, return true if the page
2544 * has been properly handled by compression, otherwise needs other
2545 * paths to handle it
2547 static bool save_compress_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
2549 if (!save_page_use_compression(rs
)) {
2554 * When starting the process of a new block, the first page of
2555 * the block should be sent out before other pages in the same
2556 * block, and all the pages in last block should have been sent
2557 * out, keeping this order is important, because the 'cont' flag
2558 * is used to avoid resending the block name.
2560 * We post the fist page as normal page as compression will take
2561 * much CPU resource.
2563 if (block
!= rs
->last_sent_block
) {
2564 flush_compressed_data(rs
);
2568 if (compress_page_with_multi_thread(rs
, block
, offset
) > 0) {
2572 compression_counters
.busy
++;
2577 * ram_save_target_page: save one target page
2579 * Returns the number of pages written
2581 * @rs: current RAM state
2582 * @pss: data about the page we want to send
2583 * @last_stage: if we are at the completion stage
2585 static int ram_save_target_page(RAMState
*rs
, PageSearchStatus
*pss
,
2588 RAMBlock
*block
= pss
->block
;
2589 ram_addr_t offset
= ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
;
2592 if (control_save_page(rs
, block
, offset
, &res
)) {
2596 if (save_compress_page(rs
, block
, offset
)) {
2600 res
= save_zero_page(rs
, block
, offset
);
2602 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
2603 * page would be stale
2605 if (!save_page_use_compression(rs
)) {
2606 XBZRLE_cache_lock();
2607 xbzrle_cache_zero_page(rs
, block
->offset
+ offset
);
2608 XBZRLE_cache_unlock();
2610 ram_release_pages(block
->idstr
, offset
, res
);
2615 * Do not use multifd for:
2616 * 1. Compression as the first page in the new block should be posted out
2617 * before sending the compressed page
2618 * 2. In postcopy as one whole host page should be placed
2620 if (!save_page_use_compression(rs
) && migrate_use_multifd()
2621 && !migration_in_postcopy()) {
2622 return ram_save_multifd_page(rs
, block
, offset
);
2625 return ram_save_page(rs
, pss
, last_stage
);
2629 * ram_save_host_page: save a whole host page
2631 * Starting at *offset send pages up to the end of the current host
2632 * page. It's valid for the initial offset to point into the middle of
2633 * a host page in which case the remainder of the hostpage is sent.
2634 * Only dirty target pages are sent. Note that the host page size may
2635 * be a huge page for this block.
2636 * The saving stops at the boundary of the used_length of the block
2637 * if the RAMBlock isn't a multiple of the host page size.
2639 * Returns the number of pages written or negative on error
2641 * @rs: current RAM state
2642 * @ms: current migration state
2643 * @pss: data about the page we want to send
2644 * @last_stage: if we are at the completion stage
2646 static int ram_save_host_page(RAMState
*rs
, PageSearchStatus
*pss
,
2649 int tmppages
, pages
= 0;
2650 size_t pagesize_bits
=
2651 qemu_ram_pagesize(pss
->block
) >> TARGET_PAGE_BITS
;
2653 if (ramblock_is_ignored(pss
->block
)) {
2654 error_report("block %s should not be migrated !", pss
->block
->idstr
);
2659 /* Check the pages is dirty and if it is send it */
2660 if (!migration_bitmap_clear_dirty(rs
, pss
->block
, pss
->page
)) {
2665 tmppages
= ram_save_target_page(rs
, pss
, last_stage
);
2672 /* Allow rate limiting to happen in the middle of huge pages */
2673 migration_rate_limit();
2674 } while ((pss
->page
& (pagesize_bits
- 1)) &&
2675 offset_in_ramblock(pss
->block
,
2676 ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
));
2678 /* The offset we leave with is the last one we looked at */
2684 * ram_find_and_save_block: finds a dirty page and sends it to f
2686 * Called within an RCU critical section.
2688 * Returns the number of pages written where zero means no dirty pages,
2689 * or negative on error
2691 * @rs: current RAM state
2692 * @last_stage: if we are at the completion stage
2694 * On systems where host-page-size > target-page-size it will send all the
2695 * pages in a host page that are dirty.
2698 static int ram_find_and_save_block(RAMState
*rs
, bool last_stage
)
2700 PageSearchStatus pss
;
2704 /* No dirty page as there is zero RAM */
2705 if (!ram_bytes_total()) {
2709 pss
.block
= rs
->last_seen_block
;
2710 pss
.page
= rs
->last_page
;
2711 pss
.complete_round
= false;
2714 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2719 found
= get_queued_page(rs
, &pss
);
2722 /* priority queue empty, so just search for something dirty */
2723 found
= find_dirty_block(rs
, &pss
, &again
);
2727 pages
= ram_save_host_page(rs
, &pss
, last_stage
);
2729 } while (!pages
&& again
);
2731 rs
->last_seen_block
= pss
.block
;
2732 rs
->last_page
= pss
.page
;
2737 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
2739 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
2742 ram_counters
.duplicate
+= pages
;
2744 ram_counters
.normal
+= pages
;
2745 ram_counters
.transferred
+= size
;
2746 qemu_update_position(f
, size
);
2750 static uint64_t ram_bytes_total_common(bool count_ignored
)
2755 RCU_READ_LOCK_GUARD();
2757 if (count_ignored
) {
2758 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
2759 total
+= block
->used_length
;
2762 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2763 total
+= block
->used_length
;
2769 uint64_t ram_bytes_total(void)
2771 return ram_bytes_total_common(false);
2774 static void xbzrle_load_setup(void)
2776 XBZRLE
.decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
2779 static void xbzrle_load_cleanup(void)
2781 g_free(XBZRLE
.decoded_buf
);
2782 XBZRLE
.decoded_buf
= NULL
;
2785 static void ram_state_cleanup(RAMState
**rsp
)
2788 migration_page_queue_free(*rsp
);
2789 qemu_mutex_destroy(&(*rsp
)->bitmap_mutex
);
2790 qemu_mutex_destroy(&(*rsp
)->src_page_req_mutex
);
2796 static void xbzrle_cleanup(void)
2798 XBZRLE_cache_lock();
2800 cache_fini(XBZRLE
.cache
);
2801 g_free(XBZRLE
.encoded_buf
);
2802 g_free(XBZRLE
.current_buf
);
2803 g_free(XBZRLE
.zero_target_page
);
2804 XBZRLE
.cache
= NULL
;
2805 XBZRLE
.encoded_buf
= NULL
;
2806 XBZRLE
.current_buf
= NULL
;
2807 XBZRLE
.zero_target_page
= NULL
;
2809 XBZRLE_cache_unlock();
2812 static void ram_save_cleanup(void *opaque
)
2814 RAMState
**rsp
= opaque
;
2817 /* caller have hold iothread lock or is in a bh, so there is
2818 * no writing race against the migration bitmap
2820 memory_global_dirty_log_stop();
2822 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2823 g_free(block
->clear_bmap
);
2824 block
->clear_bmap
= NULL
;
2825 g_free(block
->bmap
);
2830 compress_threads_save_cleanup();
2831 ram_state_cleanup(rsp
);
2834 static void ram_state_reset(RAMState
*rs
)
2836 rs
->last_seen_block
= NULL
;
2837 rs
->last_sent_block
= NULL
;
2839 rs
->last_version
= ram_list
.version
;
2840 rs
->ram_bulk_stage
= true;
2841 rs
->fpo_enabled
= false;
2844 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2847 * 'expected' is the value you expect the bitmap mostly to be full
2848 * of; it won't bother printing lines that are all this value.
2849 * If 'todump' is null the migration bitmap is dumped.
2851 void ram_debug_dump_bitmap(unsigned long *todump
, bool expected
,
2852 unsigned long pages
)
2855 int64_t linelen
= 128;
2858 for (cur
= 0; cur
< pages
; cur
+= linelen
) {
2862 * Last line; catch the case where the line length
2863 * is longer than remaining ram
2865 if (cur
+ linelen
> pages
) {
2866 linelen
= pages
- cur
;
2868 for (curb
= 0; curb
< linelen
; curb
++) {
2869 bool thisbit
= test_bit(cur
+ curb
, todump
);
2870 linebuf
[curb
] = thisbit
? '1' : '.';
2871 found
= found
|| (thisbit
!= expected
);
2874 linebuf
[curb
] = '\0';
2875 fprintf(stderr
, "0x%08" PRIx64
" : %s\n", cur
, linebuf
);
2880 /* **** functions for postcopy ***** */
2882 void ram_postcopy_migrated_memory_release(MigrationState
*ms
)
2884 struct RAMBlock
*block
;
2886 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2887 unsigned long *bitmap
= block
->bmap
;
2888 unsigned long range
= block
->used_length
>> TARGET_PAGE_BITS
;
2889 unsigned long run_start
= find_next_zero_bit(bitmap
, range
, 0);
2891 while (run_start
< range
) {
2892 unsigned long run_end
= find_next_bit(bitmap
, range
, run_start
+ 1);
2893 ram_discard_range(block
->idstr
,
2894 ((ram_addr_t
)run_start
) << TARGET_PAGE_BITS
,
2895 ((ram_addr_t
)(run_end
- run_start
))
2896 << TARGET_PAGE_BITS
);
2897 run_start
= find_next_zero_bit(bitmap
, range
, run_end
+ 1);
2903 * postcopy_send_discard_bm_ram: discard a RAMBlock
2905 * Returns zero on success
2907 * Callback from postcopy_each_ram_send_discard for each RAMBlock
2909 * @ms: current migration state
2910 * @block: RAMBlock to discard
2912 static int postcopy_send_discard_bm_ram(MigrationState
*ms
, RAMBlock
*block
)
2914 unsigned long end
= block
->used_length
>> TARGET_PAGE_BITS
;
2915 unsigned long current
;
2916 unsigned long *bitmap
= block
->bmap
;
2918 for (current
= 0; current
< end
; ) {
2919 unsigned long one
= find_next_bit(bitmap
, end
, current
);
2920 unsigned long zero
, discard_length
;
2926 zero
= find_next_zero_bit(bitmap
, end
, one
+ 1);
2929 discard_length
= end
- one
;
2931 discard_length
= zero
- one
;
2933 postcopy_discard_send_range(ms
, one
, discard_length
);
2934 current
= one
+ discard_length
;
2941 * postcopy_each_ram_send_discard: discard all RAMBlocks
2943 * Returns 0 for success or negative for error
2945 * Utility for the outgoing postcopy code.
2946 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2947 * passing it bitmap indexes and name.
2948 * (qemu_ram_foreach_block ends up passing unscaled lengths
2949 * which would mean postcopy code would have to deal with target page)
2951 * @ms: current migration state
2953 static int postcopy_each_ram_send_discard(MigrationState
*ms
)
2955 struct RAMBlock
*block
;
2958 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2959 postcopy_discard_send_init(ms
, block
->idstr
);
2962 * Postcopy sends chunks of bitmap over the wire, but it
2963 * just needs indexes at this point, avoids it having
2964 * target page specific code.
2966 ret
= postcopy_send_discard_bm_ram(ms
, block
);
2967 postcopy_discard_send_finish(ms
);
2977 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2979 * Helper for postcopy_chunk_hostpages; it's called twice to
2980 * canonicalize the two bitmaps, that are similar, but one is
2983 * Postcopy requires that all target pages in a hostpage are dirty or
2984 * clean, not a mix. This function canonicalizes the bitmaps.
2986 * @ms: current migration state
2987 * @block: block that contains the page we want to canonicalize
2989 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, RAMBlock
*block
)
2991 RAMState
*rs
= ram_state
;
2992 unsigned long *bitmap
= block
->bmap
;
2993 unsigned int host_ratio
= block
->page_size
/ TARGET_PAGE_SIZE
;
2994 unsigned long pages
= block
->used_length
>> TARGET_PAGE_BITS
;
2995 unsigned long run_start
;
2997 if (block
->page_size
== TARGET_PAGE_SIZE
) {
2998 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
3002 /* Find a dirty page */
3003 run_start
= find_next_bit(bitmap
, pages
, 0);
3005 while (run_start
< pages
) {
3008 * If the start of this run of pages is in the middle of a host
3009 * page, then we need to fixup this host page.
3011 if (QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
3012 /* Find the end of this run */
3013 run_start
= find_next_zero_bit(bitmap
, pages
, run_start
+ 1);
3015 * If the end isn't at the start of a host page, then the
3016 * run doesn't finish at the end of a host page
3017 * and we need to discard.
3021 if (!QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
3023 unsigned long fixup_start_addr
= QEMU_ALIGN_DOWN(run_start
,
3025 run_start
= QEMU_ALIGN_UP(run_start
, host_ratio
);
3027 /* Clean up the bitmap */
3028 for (page
= fixup_start_addr
;
3029 page
< fixup_start_addr
+ host_ratio
; page
++) {
3031 * Remark them as dirty, updating the count for any pages
3032 * that weren't previously dirty.
3034 rs
->migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
3038 /* Find the next dirty page for the next iteration */
3039 run_start
= find_next_bit(bitmap
, pages
, run_start
);
3044 * postcopy_chunk_hostpages: discard any partially sent host page
3046 * Utility for the outgoing postcopy code.
3048 * Discard any partially sent host-page size chunks, mark any partially
3049 * dirty host-page size chunks as all dirty. In this case the host-page
3050 * is the host-page for the particular RAMBlock, i.e. it might be a huge page
3052 * Returns zero on success
3054 * @ms: current migration state
3055 * @block: block we want to work with
3057 static int postcopy_chunk_hostpages(MigrationState
*ms
, RAMBlock
*block
)
3059 postcopy_discard_send_init(ms
, block
->idstr
);
3062 * Ensure that all partially dirty host pages are made fully dirty.
3064 postcopy_chunk_hostpages_pass(ms
, block
);
3066 postcopy_discard_send_finish(ms
);
3071 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
3073 * Returns zero on success
3075 * Transmit the set of pages to be discarded after precopy to the target
3076 * these are pages that:
3077 * a) Have been previously transmitted but are now dirty again
3078 * b) Pages that have never been transmitted, this ensures that
3079 * any pages on the destination that have been mapped by background
3080 * tasks get discarded (transparent huge pages is the specific concern)
3081 * Hopefully this is pretty sparse
3083 * @ms: current migration state
3085 int ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
3087 RAMState
*rs
= ram_state
;
3091 RCU_READ_LOCK_GUARD();
3093 /* This should be our last sync, the src is now paused */
3094 migration_bitmap_sync(rs
);
3096 /* Easiest way to make sure we don't resume in the middle of a host-page */
3097 rs
->last_seen_block
= NULL
;
3098 rs
->last_sent_block
= NULL
;
3101 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3102 /* Deal with TPS != HPS and huge pages */
3103 ret
= postcopy_chunk_hostpages(ms
, block
);
3108 #ifdef DEBUG_POSTCOPY
3109 ram_debug_dump_bitmap(block
->bmap
, true,
3110 block
->used_length
>> TARGET_PAGE_BITS
);
3113 trace_ram_postcopy_send_discard_bitmap();
3115 ret
= postcopy_each_ram_send_discard(ms
);
3121 * ram_discard_range: discard dirtied pages at the beginning of postcopy
3123 * Returns zero on success
3125 * @rbname: name of the RAMBlock of the request. NULL means the
3126 * same that last one.
3127 * @start: RAMBlock starting page
3128 * @length: RAMBlock size
3130 int ram_discard_range(const char *rbname
, uint64_t start
, size_t length
)
3132 trace_ram_discard_range(rbname
, start
, length
);
3134 RCU_READ_LOCK_GUARD();
3135 RAMBlock
*rb
= qemu_ram_block_by_name(rbname
);
3138 error_report("ram_discard_range: Failed to find block '%s'", rbname
);
3143 * On source VM, we don't need to update the received bitmap since
3144 * we don't even have one.
3146 if (rb
->receivedmap
) {
3147 bitmap_clear(rb
->receivedmap
, start
>> qemu_target_page_bits(),
3148 length
>> qemu_target_page_bits());
3151 return ram_block_discard_range(rb
, start
, length
);
3155 * For every allocation, we will try not to crash the VM if the
3156 * allocation failed.
3158 static int xbzrle_init(void)
3160 Error
*local_err
= NULL
;
3162 if (!migrate_use_xbzrle()) {
3166 XBZRLE_cache_lock();
3168 XBZRLE
.zero_target_page
= g_try_malloc0(TARGET_PAGE_SIZE
);
3169 if (!XBZRLE
.zero_target_page
) {
3170 error_report("%s: Error allocating zero page", __func__
);
3174 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size(),
3175 TARGET_PAGE_SIZE
, &local_err
);
3176 if (!XBZRLE
.cache
) {
3177 error_report_err(local_err
);
3178 goto free_zero_page
;
3181 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
3182 if (!XBZRLE
.encoded_buf
) {
3183 error_report("%s: Error allocating encoded_buf", __func__
);
3187 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
3188 if (!XBZRLE
.current_buf
) {
3189 error_report("%s: Error allocating current_buf", __func__
);
3190 goto free_encoded_buf
;
3193 /* We are all good */
3194 XBZRLE_cache_unlock();
3198 g_free(XBZRLE
.encoded_buf
);
3199 XBZRLE
.encoded_buf
= NULL
;
3201 cache_fini(XBZRLE
.cache
);
3202 XBZRLE
.cache
= NULL
;
3204 g_free(XBZRLE
.zero_target_page
);
3205 XBZRLE
.zero_target_page
= NULL
;
3207 XBZRLE_cache_unlock();
3211 static int ram_state_init(RAMState
**rsp
)
3213 *rsp
= g_try_new0(RAMState
, 1);
3216 error_report("%s: Init ramstate fail", __func__
);
3220 qemu_mutex_init(&(*rsp
)->bitmap_mutex
);
3221 qemu_mutex_init(&(*rsp
)->src_page_req_mutex
);
3222 QSIMPLEQ_INIT(&(*rsp
)->src_page_requests
);
3225 * Count the total number of pages used by ram blocks not including any
3226 * gaps due to alignment or unplugs.
3227 * This must match with the initial values of dirty bitmap.
3229 (*rsp
)->migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
3230 ram_state_reset(*rsp
);
3235 static void ram_list_init_bitmaps(void)
3237 MigrationState
*ms
= migrate_get_current();
3239 unsigned long pages
;
3242 /* Skip setting bitmap if there is no RAM */
3243 if (ram_bytes_total()) {
3244 shift
= ms
->clear_bitmap_shift
;
3245 if (shift
> CLEAR_BITMAP_SHIFT_MAX
) {
3246 error_report("clear_bitmap_shift (%u) too big, using "
3247 "max value (%u)", shift
, CLEAR_BITMAP_SHIFT_MAX
);
3248 shift
= CLEAR_BITMAP_SHIFT_MAX
;
3249 } else if (shift
< CLEAR_BITMAP_SHIFT_MIN
) {
3250 error_report("clear_bitmap_shift (%u) too small, using "
3251 "min value (%u)", shift
, CLEAR_BITMAP_SHIFT_MIN
);
3252 shift
= CLEAR_BITMAP_SHIFT_MIN
;
3255 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3256 pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3258 * The initial dirty bitmap for migration must be set with all
3259 * ones to make sure we'll migrate every guest RAM page to
3261 * Here we set RAMBlock.bmap all to 1 because when rebegin a
3262 * new migration after a failed migration, ram_list.
3263 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
3266 block
->bmap
= bitmap_new(pages
);
3267 bitmap_set(block
->bmap
, 0, pages
);
3268 block
->clear_bmap_shift
= shift
;
3269 block
->clear_bmap
= bitmap_new(clear_bmap_size(pages
, shift
));
3274 static void ram_init_bitmaps(RAMState
*rs
)
3276 /* For memory_global_dirty_log_start below. */
3277 qemu_mutex_lock_iothread();
3278 qemu_mutex_lock_ramlist();
3280 WITH_RCU_READ_LOCK_GUARD() {
3281 ram_list_init_bitmaps();
3282 memory_global_dirty_log_start();
3283 migration_bitmap_sync_precopy(rs
);
3285 qemu_mutex_unlock_ramlist();
3286 qemu_mutex_unlock_iothread();
3289 static int ram_init_all(RAMState
**rsp
)
3291 if (ram_state_init(rsp
)) {
3295 if (xbzrle_init()) {
3296 ram_state_cleanup(rsp
);
3300 ram_init_bitmaps(*rsp
);
3305 static void ram_state_resume_prepare(RAMState
*rs
, QEMUFile
*out
)
3311 * Postcopy is not using xbzrle/compression, so no need for that.
3312 * Also, since source are already halted, we don't need to care
3313 * about dirty page logging as well.
3316 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3317 pages
+= bitmap_count_one(block
->bmap
,
3318 block
->used_length
>> TARGET_PAGE_BITS
);
3321 /* This may not be aligned with current bitmaps. Recalculate. */
3322 rs
->migration_dirty_pages
= pages
;
3324 rs
->last_seen_block
= NULL
;
3325 rs
->last_sent_block
= NULL
;
3327 rs
->last_version
= ram_list
.version
;
3329 * Disable the bulk stage, otherwise we'll resend the whole RAM no
3330 * matter what we have sent.
3332 rs
->ram_bulk_stage
= false;
3334 /* Update RAMState cache of output QEMUFile */
3337 trace_ram_state_resume_prepare(pages
);
3341 * This function clears bits of the free pages reported by the caller from the
3342 * migration dirty bitmap. @addr is the host address corresponding to the
3343 * start of the continuous guest free pages, and @len is the total bytes of
3346 void qemu_guest_free_page_hint(void *addr
, size_t len
)
3350 size_t used_len
, start
, npages
;
3351 MigrationState
*s
= migrate_get_current();
3353 /* This function is currently expected to be used during live migration */
3354 if (!migration_is_setup_or_active(s
->state
)) {
3358 for (; len
> 0; len
-= used_len
, addr
+= used_len
) {
3359 block
= qemu_ram_block_from_host(addr
, false, &offset
);
3360 if (unlikely(!block
|| offset
>= block
->used_length
)) {
3362 * The implementation might not support RAMBlock resize during
3363 * live migration, but it could happen in theory with future
3364 * updates. So we add a check here to capture that case.
3366 error_report_once("%s unexpected error", __func__
);
3370 if (len
<= block
->used_length
- offset
) {
3373 used_len
= block
->used_length
- offset
;
3376 start
= offset
>> TARGET_PAGE_BITS
;
3377 npages
= used_len
>> TARGET_PAGE_BITS
;
3379 qemu_mutex_lock(&ram_state
->bitmap_mutex
);
3380 ram_state
->migration_dirty_pages
-=
3381 bitmap_count_one_with_offset(block
->bmap
, start
, npages
);
3382 bitmap_clear(block
->bmap
, start
, npages
);
3383 qemu_mutex_unlock(&ram_state
->bitmap_mutex
);
3388 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
3389 * long-running RCU critical section. When rcu-reclaims in the code
3390 * start to become numerous it will be necessary to reduce the
3391 * granularity of these critical sections.
3395 * ram_save_setup: Setup RAM for migration
3397 * Returns zero to indicate success and negative for error
3399 * @f: QEMUFile where to send the data
3400 * @opaque: RAMState pointer
3402 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
3404 RAMState
**rsp
= opaque
;
3407 if (compress_threads_save_setup()) {
3411 /* migration has already setup the bitmap, reuse it. */
3412 if (!migration_in_colo_state()) {
3413 if (ram_init_all(rsp
) != 0) {
3414 compress_threads_save_cleanup();
3420 WITH_RCU_READ_LOCK_GUARD() {
3421 qemu_put_be64(f
, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE
);
3423 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
3424 qemu_put_byte(f
, strlen(block
->idstr
));
3425 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
3426 qemu_put_be64(f
, block
->used_length
);
3427 if (migrate_postcopy_ram() && block
->page_size
!=
3428 qemu_host_page_size
) {
3429 qemu_put_be64(f
, block
->page_size
);
3431 if (migrate_ignore_shared()) {
3432 qemu_put_be64(f
, block
->mr
->addr
);
3437 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
3438 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
3440 multifd_send_sync_main(f
);
3441 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3448 * ram_save_iterate: iterative stage for migration
3450 * Returns zero to indicate success and negative for error
3452 * @f: QEMUFile where to send the data
3453 * @opaque: RAMState pointer
3455 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
3457 RAMState
**temp
= opaque
;
3458 RAMState
*rs
= *temp
;
3464 if (blk_mig_bulk_active()) {
3465 /* Avoid transferring ram during bulk phase of block migration as
3466 * the bulk phase will usually take a long time and transferring
3467 * ram updates during that time is pointless. */
3471 WITH_RCU_READ_LOCK_GUARD() {
3472 if (ram_list
.version
!= rs
->last_version
) {
3473 ram_state_reset(rs
);
3476 /* Read version before ram_list.blocks */
3479 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
3481 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
3483 while ((ret
= qemu_file_rate_limit(f
)) == 0 ||
3484 !QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
3487 if (qemu_file_get_error(f
)) {
3491 pages
= ram_find_and_save_block(rs
, false);
3492 /* no more pages to sent */
3499 qemu_file_set_error(f
, pages
);
3503 rs
->target_page_count
+= pages
;
3506 * During postcopy, it is necessary to make sure one whole host
3507 * page is sent in one chunk.
3509 if (migrate_postcopy_ram()) {
3510 flush_compressed_data(rs
);
3514 * we want to check in the 1st loop, just in case it was the 1st
3515 * time and we had to sync the dirty bitmap.
3516 * qemu_clock_get_ns() is a bit expensive, so we only check each
3519 if ((i
& 63) == 0) {
3520 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) /
3522 if (t1
> MAX_WAIT
) {
3523 trace_ram_save_iterate_big_wait(t1
, i
);
3532 * Must occur before EOS (or any QEMUFile operation)
3533 * because of RDMA protocol.
3535 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
3539 && migration_is_setup_or_active(migrate_get_current()->state
)) {
3540 multifd_send_sync_main(rs
->f
);
3541 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3543 ram_counters
.transferred
+= 8;
3545 ret
= qemu_file_get_error(f
);
3555 * ram_save_complete: function called to send the remaining amount of ram
3557 * Returns zero to indicate success or negative on error
3559 * Called with iothread lock
3561 * @f: QEMUFile where to send the data
3562 * @opaque: RAMState pointer
3564 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
3566 RAMState
**temp
= opaque
;
3567 RAMState
*rs
= *temp
;
3570 WITH_RCU_READ_LOCK_GUARD() {
3571 if (!migration_in_postcopy()) {
3572 migration_bitmap_sync_precopy(rs
);
3575 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
3577 /* try transferring iterative blocks of memory */
3579 /* flush all remaining blocks regardless of rate limiting */
3583 pages
= ram_find_and_save_block(rs
, !migration_in_colo_state());
3584 /* no more blocks to sent */
3594 flush_compressed_data(rs
);
3595 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
3599 multifd_send_sync_main(rs
->f
);
3600 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3607 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
3608 uint64_t *res_precopy_only
,
3609 uint64_t *res_compatible
,
3610 uint64_t *res_postcopy_only
)
3612 RAMState
**temp
= opaque
;
3613 RAMState
*rs
= *temp
;
3614 uint64_t remaining_size
;
3616 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3618 if (!migration_in_postcopy() &&
3619 remaining_size
< max_size
) {
3620 qemu_mutex_lock_iothread();
3621 WITH_RCU_READ_LOCK_GUARD() {
3622 migration_bitmap_sync_precopy(rs
);
3624 qemu_mutex_unlock_iothread();
3625 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3628 if (migrate_postcopy_ram()) {
3629 /* We can do postcopy, and all the data is postcopiable */
3630 *res_compatible
+= remaining_size
;
3632 *res_precopy_only
+= remaining_size
;
3636 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
3638 unsigned int xh_len
;
3640 uint8_t *loaded_data
;
3642 /* extract RLE header */
3643 xh_flags
= qemu_get_byte(f
);
3644 xh_len
= qemu_get_be16(f
);
3646 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
3647 error_report("Failed to load XBZRLE page - wrong compression!");
3651 if (xh_len
> TARGET_PAGE_SIZE
) {
3652 error_report("Failed to load XBZRLE page - len overflow!");
3655 loaded_data
= XBZRLE
.decoded_buf
;
3656 /* load data and decode */
3657 /* it can change loaded_data to point to an internal buffer */
3658 qemu_get_buffer_in_place(f
, &loaded_data
, xh_len
);
3661 if (xbzrle_decode_buffer(loaded_data
, xh_len
, host
,
3662 TARGET_PAGE_SIZE
) == -1) {
3663 error_report("Failed to load XBZRLE page - decode error!");
3671 * ram_block_from_stream: read a RAMBlock id from the migration stream
3673 * Must be called from within a rcu critical section.
3675 * Returns a pointer from within the RCU-protected ram_list.
3677 * @f: QEMUFile where to read the data from
3678 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3680 static inline RAMBlock
*ram_block_from_stream(QEMUFile
*f
, int flags
)
3682 static RAMBlock
*block
= NULL
;
3686 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
3688 error_report("Ack, bad migration stream!");
3694 len
= qemu_get_byte(f
);
3695 qemu_get_buffer(f
, (uint8_t *)id
, len
);
3698 block
= qemu_ram_block_by_name(id
);
3700 error_report("Can't find block %s", id
);
3704 if (ramblock_is_ignored(block
)) {
3705 error_report("block %s should not be migrated !", id
);
3712 static inline void *host_from_ram_block_offset(RAMBlock
*block
,
3715 if (!offset_in_ramblock(block
, offset
)) {
3719 return block
->host
+ offset
;
3722 static inline void *colo_cache_from_block_offset(RAMBlock
*block
,
3725 if (!offset_in_ramblock(block
, offset
)) {
3728 if (!block
->colo_cache
) {
3729 error_report("%s: colo_cache is NULL in block :%s",
3730 __func__
, block
->idstr
);
3735 * During colo checkpoint, we need bitmap of these migrated pages.
3736 * It help us to decide which pages in ram cache should be flushed
3737 * into VM's RAM later.
3739 if (!test_and_set_bit(offset
>> TARGET_PAGE_BITS
, block
->bmap
)) {
3740 ram_state
->migration_dirty_pages
++;
3742 return block
->colo_cache
+ offset
;
3746 * ram_handle_compressed: handle the zero page case
3748 * If a page (or a whole RDMA chunk) has been
3749 * determined to be zero, then zap it.
3751 * @host: host address for the zero page
3752 * @ch: what the page is filled from. We only support zero
3753 * @size: size of the zero page
3755 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
3757 if (ch
!= 0 || !is_zero_range(host
, size
)) {
3758 memset(host
, ch
, size
);
3762 /* return the size after decompression, or negative value on error */
3764 qemu_uncompress_data(z_stream
*stream
, uint8_t *dest
, size_t dest_len
,
3765 const uint8_t *source
, size_t source_len
)
3769 err
= inflateReset(stream
);
3774 stream
->avail_in
= source_len
;
3775 stream
->next_in
= (uint8_t *)source
;
3776 stream
->avail_out
= dest_len
;
3777 stream
->next_out
= dest
;
3779 err
= inflate(stream
, Z_NO_FLUSH
);
3780 if (err
!= Z_STREAM_END
) {
3784 return stream
->total_out
;
3787 static void *do_data_decompress(void *opaque
)
3789 DecompressParam
*param
= opaque
;
3790 unsigned long pagesize
;
3794 qemu_mutex_lock(¶m
->mutex
);
3795 while (!param
->quit
) {
3800 qemu_mutex_unlock(¶m
->mutex
);
3802 pagesize
= TARGET_PAGE_SIZE
;
3804 ret
= qemu_uncompress_data(¶m
->stream
, des
, pagesize
,
3805 param
->compbuf
, len
);
3806 if (ret
< 0 && migrate_get_current()->decompress_error_check
) {
3807 error_report("decompress data failed");
3808 qemu_file_set_error(decomp_file
, ret
);
3811 qemu_mutex_lock(&decomp_done_lock
);
3813 qemu_cond_signal(&decomp_done_cond
);
3814 qemu_mutex_unlock(&decomp_done_lock
);
3816 qemu_mutex_lock(¶m
->mutex
);
3818 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
3821 qemu_mutex_unlock(¶m
->mutex
);
3826 static int wait_for_decompress_done(void)
3828 int idx
, thread_count
;
3830 if (!migrate_use_compression()) {
3834 thread_count
= migrate_decompress_threads();
3835 qemu_mutex_lock(&decomp_done_lock
);
3836 for (idx
= 0; idx
< thread_count
; idx
++) {
3837 while (!decomp_param
[idx
].done
) {
3838 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3841 qemu_mutex_unlock(&decomp_done_lock
);
3842 return qemu_file_get_error(decomp_file
);
3845 static void compress_threads_load_cleanup(void)
3847 int i
, thread_count
;
3849 if (!migrate_use_compression()) {
3852 thread_count
= migrate_decompress_threads();
3853 for (i
= 0; i
< thread_count
; i
++) {
3855 * we use it as a indicator which shows if the thread is
3856 * properly init'd or not
3858 if (!decomp_param
[i
].compbuf
) {
3862 qemu_mutex_lock(&decomp_param
[i
].mutex
);
3863 decomp_param
[i
].quit
= true;
3864 qemu_cond_signal(&decomp_param
[i
].cond
);
3865 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
3867 for (i
= 0; i
< thread_count
; i
++) {
3868 if (!decomp_param
[i
].compbuf
) {
3872 qemu_thread_join(decompress_threads
+ i
);
3873 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
3874 qemu_cond_destroy(&decomp_param
[i
].cond
);
3875 inflateEnd(&decomp_param
[i
].stream
);
3876 g_free(decomp_param
[i
].compbuf
);
3877 decomp_param
[i
].compbuf
= NULL
;
3879 g_free(decompress_threads
);
3880 g_free(decomp_param
);
3881 decompress_threads
= NULL
;
3882 decomp_param
= NULL
;
3886 static int compress_threads_load_setup(QEMUFile
*f
)
3888 int i
, thread_count
;
3890 if (!migrate_use_compression()) {
3894 thread_count
= migrate_decompress_threads();
3895 decompress_threads
= g_new0(QemuThread
, thread_count
);
3896 decomp_param
= g_new0(DecompressParam
, thread_count
);
3897 qemu_mutex_init(&decomp_done_lock
);
3898 qemu_cond_init(&decomp_done_cond
);
3900 for (i
= 0; i
< thread_count
; i
++) {
3901 if (inflateInit(&decomp_param
[i
].stream
) != Z_OK
) {
3905 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
3906 qemu_mutex_init(&decomp_param
[i
].mutex
);
3907 qemu_cond_init(&decomp_param
[i
].cond
);
3908 decomp_param
[i
].done
= true;
3909 decomp_param
[i
].quit
= false;
3910 qemu_thread_create(decompress_threads
+ i
, "decompress",
3911 do_data_decompress
, decomp_param
+ i
,
3912 QEMU_THREAD_JOINABLE
);
3916 compress_threads_load_cleanup();
3920 static void decompress_data_with_multi_threads(QEMUFile
*f
,
3921 void *host
, int len
)
3923 int idx
, thread_count
;
3925 thread_count
= migrate_decompress_threads();
3926 qemu_mutex_lock(&decomp_done_lock
);
3928 for (idx
= 0; idx
< thread_count
; idx
++) {
3929 if (decomp_param
[idx
].done
) {
3930 decomp_param
[idx
].done
= false;
3931 qemu_mutex_lock(&decomp_param
[idx
].mutex
);
3932 qemu_get_buffer(f
, decomp_param
[idx
].compbuf
, len
);
3933 decomp_param
[idx
].des
= host
;
3934 decomp_param
[idx
].len
= len
;
3935 qemu_cond_signal(&decomp_param
[idx
].cond
);
3936 qemu_mutex_unlock(&decomp_param
[idx
].mutex
);
3940 if (idx
< thread_count
) {
3943 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3946 qemu_mutex_unlock(&decomp_done_lock
);
3950 * colo cache: this is for secondary VM, we cache the whole
3951 * memory of the secondary VM, it is need to hold the global lock
3952 * to call this helper.
3954 int colo_init_ram_cache(void)
3958 WITH_RCU_READ_LOCK_GUARD() {
3959 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3960 block
->colo_cache
= qemu_anon_ram_alloc(block
->used_length
,
3963 if (!block
->colo_cache
) {
3964 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3965 "size 0x" RAM_ADDR_FMT
, __func__
, block
->idstr
,
3966 block
->used_length
);
3967 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3968 if (block
->colo_cache
) {
3969 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3970 block
->colo_cache
= NULL
;
3975 memcpy(block
->colo_cache
, block
->host
, block
->used_length
);
3980 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3981 * with to decide which page in cache should be flushed into SVM's RAM. Here
3982 * we use the same name 'ram_bitmap' as for migration.
3984 if (ram_bytes_total()) {
3987 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3988 unsigned long pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3990 block
->bmap
= bitmap_new(pages
);
3991 bitmap_set(block
->bmap
, 0, pages
);
3994 ram_state
= g_new0(RAMState
, 1);
3995 ram_state
->migration_dirty_pages
= 0;
3996 qemu_mutex_init(&ram_state
->bitmap_mutex
);
3997 memory_global_dirty_log_start();
4002 /* It is need to hold the global lock to call this helper */
4003 void colo_release_ram_cache(void)
4007 memory_global_dirty_log_stop();
4008 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4009 g_free(block
->bmap
);
4013 WITH_RCU_READ_LOCK_GUARD() {
4014 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4015 if (block
->colo_cache
) {
4016 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
4017 block
->colo_cache
= NULL
;
4021 qemu_mutex_destroy(&ram_state
->bitmap_mutex
);
4027 * ram_load_setup: Setup RAM for migration incoming side
4029 * Returns zero to indicate success and negative for error
4031 * @f: QEMUFile where to receive the data
4032 * @opaque: RAMState pointer
4034 static int ram_load_setup(QEMUFile
*f
, void *opaque
)
4036 if (compress_threads_load_setup(f
)) {
4040 xbzrle_load_setup();
4041 ramblock_recv_map_init();
4046 static int ram_load_cleanup(void *opaque
)
4050 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4051 qemu_ram_block_writeback(rb
);
4054 xbzrle_load_cleanup();
4055 compress_threads_load_cleanup();
4057 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4058 g_free(rb
->receivedmap
);
4059 rb
->receivedmap
= NULL
;
4066 * ram_postcopy_incoming_init: allocate postcopy data structures
4068 * Returns 0 for success and negative if there was one error
4070 * @mis: current migration incoming state
4072 * Allocate data structures etc needed by incoming migration with
4073 * postcopy-ram. postcopy-ram's similarly names
4074 * postcopy_ram_incoming_init does the work.
4076 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
4078 return postcopy_ram_incoming_init(mis
);
4082 * ram_load_postcopy: load a page in postcopy case
4084 * Returns 0 for success or -errno in case of error
4086 * Called in postcopy mode by ram_load().
4087 * rcu_read_lock is taken prior to this being called.
4089 * @f: QEMUFile where to send the data
4091 static int ram_load_postcopy(QEMUFile
*f
)
4093 int flags
= 0, ret
= 0;
4094 bool place_needed
= false;
4095 bool matches_target_page_size
= false;
4096 MigrationIncomingState
*mis
= migration_incoming_get_current();
4097 /* Temporary page that is later 'placed' */
4098 void *postcopy_host_page
= mis
->postcopy_tmp_page
;
4099 void *this_host
= NULL
;
4100 bool all_zero
= false;
4101 int target_pages
= 0;
4103 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
4106 void *page_buffer
= NULL
;
4107 void *place_source
= NULL
;
4108 RAMBlock
*block
= NULL
;
4112 addr
= qemu_get_be64(f
);
4115 * If qemu file error, we should stop here, and then "addr"
4118 ret
= qemu_file_get_error(f
);
4123 flags
= addr
& ~TARGET_PAGE_MASK
;
4124 addr
&= TARGET_PAGE_MASK
;
4126 trace_ram_load_postcopy_loop((uint64_t)addr
, flags
);
4127 place_needed
= false;
4128 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
4129 RAM_SAVE_FLAG_COMPRESS_PAGE
)) {
4130 block
= ram_block_from_stream(f
, flags
);
4132 host
= host_from_ram_block_offset(block
, addr
);
4134 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
4139 matches_target_page_size
= block
->page_size
== TARGET_PAGE_SIZE
;
4141 * Postcopy requires that we place whole host pages atomically;
4142 * these may be huge pages for RAMBlocks that are backed by
4144 * To make it atomic, the data is read into a temporary page
4145 * that's moved into place later.
4146 * The migration protocol uses, possibly smaller, target-pages
4147 * however the source ensures it always sends all the components
4148 * of a host page in one chunk.
4150 page_buffer
= postcopy_host_page
+
4151 ((uintptr_t)host
& (block
->page_size
- 1));
4152 /* If all TP are zero then we can optimise the place */
4153 if (target_pages
== 1) {
4155 this_host
= (void *)QEMU_ALIGN_DOWN((uintptr_t)host
,
4158 /* not the 1st TP within the HP */
4159 if (QEMU_ALIGN_DOWN((uintptr_t)host
, block
->page_size
) !=
4160 (uintptr_t)this_host
) {
4161 error_report("Non-same host page %p/%p",
4169 * If it's the last part of a host page then we place the host
4172 if (target_pages
== (block
->page_size
/ TARGET_PAGE_SIZE
)) {
4173 place_needed
= true;
4176 place_source
= postcopy_host_page
;
4179 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4180 case RAM_SAVE_FLAG_ZERO
:
4181 ch
= qemu_get_byte(f
);
4183 * Can skip to set page_buffer when
4184 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
4186 if (ch
|| !matches_target_page_size
) {
4187 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
4194 case RAM_SAVE_FLAG_PAGE
:
4196 if (!matches_target_page_size
) {
4197 /* For huge pages, we always use temporary buffer */
4198 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
4201 * For small pages that matches target page size, we
4202 * avoid the qemu_file copy. Instead we directly use
4203 * the buffer of QEMUFile to place the page. Note: we
4204 * cannot do any QEMUFile operation before using that
4205 * buffer to make sure the buffer is valid when
4208 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
4212 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
4214 len
= qemu_get_be32(f
);
4215 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
4216 error_report("Invalid compressed data length: %d", len
);
4220 decompress_data_with_multi_threads(f
, page_buffer
, len
);
4223 case RAM_SAVE_FLAG_EOS
:
4225 multifd_recv_sync_main();
4228 error_report("Unknown combination of migration flags: %#x"
4229 " (postcopy mode)", flags
);
4234 /* Got the whole host page, wait for decompress before placing. */
4236 ret
|= wait_for_decompress_done();
4239 /* Detect for any possible file errors */
4240 if (!ret
&& qemu_file_get_error(f
)) {
4241 ret
= qemu_file_get_error(f
);
4244 if (!ret
&& place_needed
) {
4245 /* This gets called at the last target page in the host page */
4246 void *place_dest
= (void *)QEMU_ALIGN_DOWN((uintptr_t)host
,
4250 ret
= postcopy_place_page_zero(mis
, place_dest
,
4253 ret
= postcopy_place_page(mis
, place_dest
,
4254 place_source
, block
);
4262 static bool postcopy_is_advised(void)
4264 PostcopyState ps
= postcopy_state_get();
4265 return ps
>= POSTCOPY_INCOMING_ADVISE
&& ps
< POSTCOPY_INCOMING_END
;
4268 static bool postcopy_is_running(void)
4270 PostcopyState ps
= postcopy_state_get();
4271 return ps
>= POSTCOPY_INCOMING_LISTENING
&& ps
< POSTCOPY_INCOMING_END
;
4275 * Flush content of RAM cache into SVM's memory.
4276 * Only flush the pages that be dirtied by PVM or SVM or both.
4278 static void colo_flush_ram_cache(void)
4280 RAMBlock
*block
= NULL
;
4283 unsigned long offset
= 0;
4285 memory_global_dirty_log_sync();
4286 WITH_RCU_READ_LOCK_GUARD() {
4287 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4288 ramblock_sync_dirty_bitmap(ram_state
, block
);
4292 trace_colo_flush_ram_cache_begin(ram_state
->migration_dirty_pages
);
4293 WITH_RCU_READ_LOCK_GUARD() {
4294 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
4297 offset
= migration_bitmap_find_dirty(ram_state
, block
, offset
);
4299 if (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
4300 >= block
->used_length
) {
4302 block
= QLIST_NEXT_RCU(block
, next
);
4304 migration_bitmap_clear_dirty(ram_state
, block
, offset
);
4305 dst_host
= block
->host
4306 + (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
);
4307 src_host
= block
->colo_cache
4308 + (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
);
4309 memcpy(dst_host
, src_host
, TARGET_PAGE_SIZE
);
4313 trace_colo_flush_ram_cache_end();
4317 * ram_load_precopy: load pages in precopy case
4319 * Returns 0 for success or -errno in case of error
4321 * Called in precopy mode by ram_load().
4322 * rcu_read_lock is taken prior to this being called.
4324 * @f: QEMUFile where to send the data
4326 static int ram_load_precopy(QEMUFile
*f
)
4328 int flags
= 0, ret
= 0, invalid_flags
= 0, len
= 0, i
= 0;
4329 /* ADVISE is earlier, it shows the source has the postcopy capability on */
4330 bool postcopy_advised
= postcopy_is_advised();
4331 if (!migrate_use_compression()) {
4332 invalid_flags
|= RAM_SAVE_FLAG_COMPRESS_PAGE
;
4335 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
4336 ram_addr_t addr
, total_ram_bytes
;
4341 * Yield periodically to let main loop run, but an iteration of
4342 * the main loop is expensive, so do it each some iterations
4344 if ((i
& 32767) == 0 && qemu_in_coroutine()) {
4345 aio_co_schedule(qemu_get_current_aio_context(),
4346 qemu_coroutine_self());
4347 qemu_coroutine_yield();
4351 addr
= qemu_get_be64(f
);
4352 flags
= addr
& ~TARGET_PAGE_MASK
;
4353 addr
&= TARGET_PAGE_MASK
;
4355 if (flags
& invalid_flags
) {
4356 if (flags
& invalid_flags
& RAM_SAVE_FLAG_COMPRESS_PAGE
) {
4357 error_report("Received an unexpected compressed page");
4364 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
4365 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
4366 RAMBlock
*block
= ram_block_from_stream(f
, flags
);
4369 * After going into COLO, we should load the Page into colo_cache.
4371 if (migration_incoming_in_colo_state()) {
4372 host
= colo_cache_from_block_offset(block
, addr
);
4374 host
= host_from_ram_block_offset(block
, addr
);
4377 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
4382 if (!migration_incoming_in_colo_state()) {
4383 ramblock_recv_bitmap_set(block
, host
);
4386 trace_ram_load_loop(block
->idstr
, (uint64_t)addr
, flags
, host
);
4389 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4390 case RAM_SAVE_FLAG_MEM_SIZE
:
4391 /* Synchronize RAM block list */
4392 total_ram_bytes
= addr
;
4393 while (!ret
&& total_ram_bytes
) {
4398 len
= qemu_get_byte(f
);
4399 qemu_get_buffer(f
, (uint8_t *)id
, len
);
4401 length
= qemu_get_be64(f
);
4403 block
= qemu_ram_block_by_name(id
);
4404 if (block
&& !qemu_ram_is_migratable(block
)) {
4405 error_report("block %s should not be migrated !", id
);
4408 if (length
!= block
->used_length
) {
4409 Error
*local_err
= NULL
;
4411 ret
= qemu_ram_resize(block
, length
,
4414 error_report_err(local_err
);
4417 /* For postcopy we need to check hugepage sizes match */
4418 if (postcopy_advised
&&
4419 block
->page_size
!= qemu_host_page_size
) {
4420 uint64_t remote_page_size
= qemu_get_be64(f
);
4421 if (remote_page_size
!= block
->page_size
) {
4422 error_report("Mismatched RAM page size %s "
4423 "(local) %zd != %" PRId64
,
4424 id
, block
->page_size
,
4429 if (migrate_ignore_shared()) {
4430 hwaddr addr
= qemu_get_be64(f
);
4431 if (ramblock_is_ignored(block
) &&
4432 block
->mr
->addr
!= addr
) {
4433 error_report("Mismatched GPAs for block %s "
4434 "%" PRId64
"!= %" PRId64
,
4436 (uint64_t)block
->mr
->addr
);
4440 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
4443 error_report("Unknown ramblock \"%s\", cannot "
4444 "accept migration", id
);
4448 total_ram_bytes
-= length
;
4452 case RAM_SAVE_FLAG_ZERO
:
4453 ch
= qemu_get_byte(f
);
4454 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
4457 case RAM_SAVE_FLAG_PAGE
:
4458 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
4461 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
4462 len
= qemu_get_be32(f
);
4463 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
4464 error_report("Invalid compressed data length: %d", len
);
4468 decompress_data_with_multi_threads(f
, host
, len
);
4471 case RAM_SAVE_FLAG_XBZRLE
:
4472 if (load_xbzrle(f
, addr
, host
) < 0) {
4473 error_report("Failed to decompress XBZRLE page at "
4474 RAM_ADDR_FMT
, addr
);
4479 case RAM_SAVE_FLAG_EOS
:
4481 multifd_recv_sync_main();
4484 if (flags
& RAM_SAVE_FLAG_HOOK
) {
4485 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
4487 error_report("Unknown combination of migration flags: %#x",
4493 ret
= qemu_file_get_error(f
);
4497 ret
|= wait_for_decompress_done();
4501 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
4504 static uint64_t seq_iter
;
4506 * If system is running in postcopy mode, page inserts to host memory must
4509 bool postcopy_running
= postcopy_is_running();
4513 if (version_id
!= 4) {
4518 * This RCU critical section can be very long running.
4519 * When RCU reclaims in the code start to become numerous,
4520 * it will be necessary to reduce the granularity of this
4523 WITH_RCU_READ_LOCK_GUARD() {
4524 if (postcopy_running
) {
4525 ret
= ram_load_postcopy(f
);
4527 ret
= ram_load_precopy(f
);
4530 trace_ram_load_complete(ret
, seq_iter
);
4532 if (!ret
&& migration_incoming_in_colo_state()) {
4533 colo_flush_ram_cache();
4538 static bool ram_has_postcopy(void *opaque
)
4541 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4542 if (ramblock_is_pmem(rb
)) {
4543 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4544 "is not supported now!", rb
->idstr
, rb
->host
);
4549 return migrate_postcopy_ram();
4552 /* Sync all the dirty bitmap with destination VM. */
4553 static int ram_dirty_bitmap_sync_all(MigrationState
*s
, RAMState
*rs
)
4556 QEMUFile
*file
= s
->to_dst_file
;
4557 int ramblock_count
= 0;
4559 trace_ram_dirty_bitmap_sync_start();
4561 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4562 qemu_savevm_send_recv_bitmap(file
, block
->idstr
);
4563 trace_ram_dirty_bitmap_request(block
->idstr
);
4567 trace_ram_dirty_bitmap_sync_wait();
4569 /* Wait until all the ramblocks' dirty bitmap synced */
4570 while (ramblock_count
--) {
4571 qemu_sem_wait(&s
->rp_state
.rp_sem
);
4574 trace_ram_dirty_bitmap_sync_complete();
4579 static void ram_dirty_bitmap_reload_notify(MigrationState
*s
)
4581 qemu_sem_post(&s
->rp_state
.rp_sem
);
4585 * Read the received bitmap, revert it as the initial dirty bitmap.
4586 * This is only used when the postcopy migration is paused but wants
4587 * to resume from a middle point.
4589 int ram_dirty_bitmap_reload(MigrationState
*s
, RAMBlock
*block
)
4592 QEMUFile
*file
= s
->rp_state
.from_dst_file
;
4593 unsigned long *le_bitmap
, nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
4594 uint64_t local_size
= DIV_ROUND_UP(nbits
, 8);
4595 uint64_t size
, end_mark
;
4597 trace_ram_dirty_bitmap_reload_begin(block
->idstr
);
4599 if (s
->state
!= MIGRATION_STATUS_POSTCOPY_RECOVER
) {
4600 error_report("%s: incorrect state %s", __func__
,
4601 MigrationStatus_str(s
->state
));
4606 * Note: see comments in ramblock_recv_bitmap_send() on why we
4607 * need the endianess convertion, and the paddings.
4609 local_size
= ROUND_UP(local_size
, 8);
4612 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
4614 size
= qemu_get_be64(file
);
4616 /* The size of the bitmap should match with our ramblock */
4617 if (size
!= local_size
) {
4618 error_report("%s: ramblock '%s' bitmap size mismatch "
4619 "(0x%"PRIx64
" != 0x%"PRIx64
")", __func__
,
4620 block
->idstr
, size
, local_size
);
4625 size
= qemu_get_buffer(file
, (uint8_t *)le_bitmap
, local_size
);
4626 end_mark
= qemu_get_be64(file
);
4628 ret
= qemu_file_get_error(file
);
4629 if (ret
|| size
!= local_size
) {
4630 error_report("%s: read bitmap failed for ramblock '%s': %d"
4631 " (size 0x%"PRIx64
", got: 0x%"PRIx64
")",
4632 __func__
, block
->idstr
, ret
, local_size
, size
);
4637 if (end_mark
!= RAMBLOCK_RECV_BITMAP_ENDING
) {
4638 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIu64
,
4639 __func__
, block
->idstr
, end_mark
);
4645 * Endianess convertion. We are during postcopy (though paused).
4646 * The dirty bitmap won't change. We can directly modify it.
4648 bitmap_from_le(block
->bmap
, le_bitmap
, nbits
);
4651 * What we received is "received bitmap". Revert it as the initial
4652 * dirty bitmap for this ramblock.
4654 bitmap_complement(block
->bmap
, block
->bmap
, nbits
);
4656 trace_ram_dirty_bitmap_reload_complete(block
->idstr
);
4659 * We succeeded to sync bitmap for current ramblock. If this is
4660 * the last one to sync, we need to notify the main send thread.
4662 ram_dirty_bitmap_reload_notify(s
);
4670 static int ram_resume_prepare(MigrationState
*s
, void *opaque
)
4672 RAMState
*rs
= *(RAMState
**)opaque
;
4675 ret
= ram_dirty_bitmap_sync_all(s
, rs
);
4680 ram_state_resume_prepare(rs
, s
->to_dst_file
);
4685 static SaveVMHandlers savevm_ram_handlers
= {
4686 .save_setup
= ram_save_setup
,
4687 .save_live_iterate
= ram_save_iterate
,
4688 .save_live_complete_postcopy
= ram_save_complete
,
4689 .save_live_complete_precopy
= ram_save_complete
,
4690 .has_postcopy
= ram_has_postcopy
,
4691 .save_live_pending
= ram_save_pending
,
4692 .load_state
= ram_load
,
4693 .save_cleanup
= ram_save_cleanup
,
4694 .load_setup
= ram_load_setup
,
4695 .load_cleanup
= ram_load_cleanup
,
4696 .resume_prepare
= ram_resume_prepare
,
4699 void ram_mig_init(void)
4701 qemu_mutex_init(&XBZRLE
.lock
);
4702 register_savevm_live("ram", 0, 4, &savevm_ram_handlers
, &ram_state
);