4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu/osdep.h"
32 #include "qemu/cutils.h"
33 #include "qemu/bitops.h"
34 #include "qemu/bitmap.h"
35 #include "qemu/main-loop.h"
38 #include "migration.h"
40 #include "migration/register.h"
41 #include "migration/misc.h"
42 #include "qemu-file.h"
43 #include "postcopy-ram.h"
44 #include "page_cache.h"
45 #include "qemu/error-report.h"
46 #include "qapi/error.h"
47 #include "qapi/qapi-events-migration.h"
48 #include "qapi/qmp/qerror.h"
50 #include "exec/ram_addr.h"
51 #include "exec/target_page.h"
52 #include "qemu/rcu_queue.h"
53 #include "migration/colo.h"
55 #include "sysemu/sysemu.h"
56 #include "qemu/uuid.h"
60 /***********************************************************/
61 /* ram save/restore */
63 /* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
64 * worked for pages that where filled with the same char. We switched
65 * it to only search for the zero value. And to avoid confusion with
66 * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it.
69 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
70 #define RAM_SAVE_FLAG_ZERO 0x02
71 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
72 #define RAM_SAVE_FLAG_PAGE 0x08
73 #define RAM_SAVE_FLAG_EOS 0x10
74 #define RAM_SAVE_FLAG_CONTINUE 0x20
75 #define RAM_SAVE_FLAG_XBZRLE 0x40
76 /* 0x80 is reserved in migration.h start with 0x100 next */
77 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
79 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
81 return buffer_is_zero(p
, size
);
84 XBZRLECacheStats xbzrle_counters
;
86 /* struct contains XBZRLE cache and a static page
87 used by the compression */
89 /* buffer used for XBZRLE encoding */
91 /* buffer for storing page content */
93 /* Cache for XBZRLE, Protected by lock. */
96 /* it will store a page full of zeros */
97 uint8_t *zero_target_page
;
98 /* buffer used for XBZRLE decoding */
102 static void XBZRLE_cache_lock(void)
104 if (migrate_use_xbzrle())
105 qemu_mutex_lock(&XBZRLE
.lock
);
108 static void XBZRLE_cache_unlock(void)
110 if (migrate_use_xbzrle())
111 qemu_mutex_unlock(&XBZRLE
.lock
);
115 * xbzrle_cache_resize: resize the xbzrle cache
117 * This function is called from qmp_migrate_set_cache_size in main
118 * thread, possibly while a migration is in progress. A running
119 * migration may be using the cache and might finish during this call,
120 * hence changes to the cache are protected by XBZRLE.lock().
122 * Returns 0 for success or -1 for error
124 * @new_size: new cache size
125 * @errp: set *errp if the check failed, with reason
127 int xbzrle_cache_resize(int64_t new_size
, Error
**errp
)
129 PageCache
*new_cache
;
132 /* Check for truncation */
133 if (new_size
!= (size_t)new_size
) {
134 error_setg(errp
, QERR_INVALID_PARAMETER_VALUE
, "cache size",
135 "exceeding address space");
139 if (new_size
== migrate_xbzrle_cache_size()) {
146 if (XBZRLE
.cache
!= NULL
) {
147 new_cache
= cache_init(new_size
, TARGET_PAGE_SIZE
, errp
);
153 cache_fini(XBZRLE
.cache
);
154 XBZRLE
.cache
= new_cache
;
157 XBZRLE_cache_unlock();
161 static bool ramblock_is_ignored(RAMBlock
*block
)
163 return !qemu_ram_is_migratable(block
) ||
164 (migrate_ignore_shared() && qemu_ram_is_shared(block
));
167 /* Should be holding either ram_list.mutex, or the RCU lock. */
168 #define RAMBLOCK_FOREACH_NOT_IGNORED(block) \
169 INTERNAL_RAMBLOCK_FOREACH(block) \
170 if (ramblock_is_ignored(block)) {} else
172 #define RAMBLOCK_FOREACH_MIGRATABLE(block) \
173 INTERNAL_RAMBLOCK_FOREACH(block) \
174 if (!qemu_ram_is_migratable(block)) {} else
176 #undef RAMBLOCK_FOREACH
178 int foreach_not_ignored_block(RAMBlockIterFunc func
, void *opaque
)
183 RCU_READ_LOCK_GUARD();
185 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
186 ret
= func(block
, opaque
);
194 static void ramblock_recv_map_init(void)
198 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
199 assert(!rb
->receivedmap
);
200 rb
->receivedmap
= bitmap_new(rb
->max_length
>> qemu_target_page_bits());
204 int ramblock_recv_bitmap_test(RAMBlock
*rb
, void *host_addr
)
206 return test_bit(ramblock_recv_bitmap_offset(host_addr
, rb
),
210 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock
*rb
, uint64_t byte_offset
)
212 return test_bit(byte_offset
>> TARGET_PAGE_BITS
, rb
->receivedmap
);
215 void ramblock_recv_bitmap_set(RAMBlock
*rb
, void *host_addr
)
217 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr
, rb
), rb
->receivedmap
);
220 void ramblock_recv_bitmap_set_range(RAMBlock
*rb
, void *host_addr
,
223 bitmap_set_atomic(rb
->receivedmap
,
224 ramblock_recv_bitmap_offset(host_addr
, rb
),
228 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
231 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
233 * Returns >0 if success with sent bytes, or <0 if error.
235 int64_t ramblock_recv_bitmap_send(QEMUFile
*file
,
236 const char *block_name
)
238 RAMBlock
*block
= qemu_ram_block_by_name(block_name
);
239 unsigned long *le_bitmap
, nbits
;
243 error_report("%s: invalid block name: %s", __func__
, block_name
);
247 nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
250 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
251 * machines we may need 4 more bytes for padding (see below
252 * comment). So extend it a bit before hand.
254 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
257 * Always use little endian when sending the bitmap. This is
258 * required that when source and destination VMs are not using the
259 * same endianess. (Note: big endian won't work.)
261 bitmap_to_le(le_bitmap
, block
->receivedmap
, nbits
);
263 /* Size of the bitmap, in bytes */
264 size
= DIV_ROUND_UP(nbits
, 8);
267 * size is always aligned to 8 bytes for 64bit machines, but it
268 * may not be true for 32bit machines. We need this padding to
269 * make sure the migration can survive even between 32bit and
272 size
= ROUND_UP(size
, 8);
274 qemu_put_be64(file
, size
);
275 qemu_put_buffer(file
, (const uint8_t *)le_bitmap
, size
);
277 * Mark as an end, in case the middle part is screwed up due to
278 * some "misterious" reason.
280 qemu_put_be64(file
, RAMBLOCK_RECV_BITMAP_ENDING
);
285 if (qemu_file_get_error(file
)) {
286 return qemu_file_get_error(file
);
289 return size
+ sizeof(size
);
293 * An outstanding page request, on the source, having been received
296 struct RAMSrcPageRequest
{
301 QSIMPLEQ_ENTRY(RAMSrcPageRequest
) next_req
;
304 /* State of RAM for migration */
306 /* QEMUFile used for this migration */
308 /* Last block that we have visited searching for dirty pages */
309 RAMBlock
*last_seen_block
;
310 /* Last block from where we have sent data */
311 RAMBlock
*last_sent_block
;
312 /* Last dirty target page we have sent */
313 ram_addr_t last_page
;
314 /* last ram version we have seen */
315 uint32_t last_version
;
316 /* We are in the first round */
318 /* The free page optimization is enabled */
320 /* How many times we have dirty too many pages */
321 int dirty_rate_high_cnt
;
322 /* these variables are used for bitmap sync */
323 /* last time we did a full bitmap_sync */
324 int64_t time_last_bitmap_sync
;
325 /* bytes transferred at start_time */
326 uint64_t bytes_xfer_prev
;
327 /* number of dirty pages since start_time */
328 uint64_t num_dirty_pages_period
;
329 /* xbzrle misses since the beginning of the period */
330 uint64_t xbzrle_cache_miss_prev
;
332 /* compression statistics since the beginning of the period */
333 /* amount of count that no free thread to compress data */
334 uint64_t compress_thread_busy_prev
;
335 /* amount bytes after compression */
336 uint64_t compressed_size_prev
;
337 /* amount of compressed pages */
338 uint64_t compress_pages_prev
;
340 /* total handled target pages at the beginning of period */
341 uint64_t target_page_count_prev
;
342 /* total handled target pages since start */
343 uint64_t target_page_count
;
344 /* number of dirty bits in the bitmap */
345 uint64_t migration_dirty_pages
;
346 /* Protects modification of the bitmap and migration dirty pages */
347 QemuMutex bitmap_mutex
;
348 /* The RAMBlock used in the last src_page_requests */
349 RAMBlock
*last_req_rb
;
350 /* Queue of outstanding page requests from the destination */
351 QemuMutex src_page_req_mutex
;
352 QSIMPLEQ_HEAD(, RAMSrcPageRequest
) src_page_requests
;
354 typedef struct RAMState RAMState
;
356 static RAMState
*ram_state
;
358 static NotifierWithReturnList precopy_notifier_list
;
360 void precopy_infrastructure_init(void)
362 notifier_with_return_list_init(&precopy_notifier_list
);
365 void precopy_add_notifier(NotifierWithReturn
*n
)
367 notifier_with_return_list_add(&precopy_notifier_list
, n
);
370 void precopy_remove_notifier(NotifierWithReturn
*n
)
372 notifier_with_return_remove(n
);
375 int precopy_notify(PrecopyNotifyReason reason
, Error
**errp
)
377 PrecopyNotifyData pnd
;
381 return notifier_with_return_list_notify(&precopy_notifier_list
, &pnd
);
384 void precopy_enable_free_page_optimization(void)
390 ram_state
->fpo_enabled
= true;
393 uint64_t ram_bytes_remaining(void)
395 return ram_state
? (ram_state
->migration_dirty_pages
* TARGET_PAGE_SIZE
) :
399 MigrationStats ram_counters
;
401 /* used by the search for pages to send */
402 struct PageSearchStatus
{
403 /* Current block being searched */
405 /* Current page to search from */
407 /* Set once we wrap around */
410 typedef struct PageSearchStatus PageSearchStatus
;
412 CompressionStats compression_counters
;
414 struct CompressParam
{
424 /* internally used fields */
428 typedef struct CompressParam CompressParam
;
430 struct DecompressParam
{
440 typedef struct DecompressParam DecompressParam
;
442 static CompressParam
*comp_param
;
443 static QemuThread
*compress_threads
;
444 /* comp_done_cond is used to wake up the migration thread when
445 * one of the compression threads has finished the compression.
446 * comp_done_lock is used to co-work with comp_done_cond.
448 static QemuMutex comp_done_lock
;
449 static QemuCond comp_done_cond
;
450 /* The empty QEMUFileOps will be used by file in CompressParam */
451 static const QEMUFileOps empty_ops
= { };
453 static QEMUFile
*decomp_file
;
454 static DecompressParam
*decomp_param
;
455 static QemuThread
*decompress_threads
;
456 static QemuMutex decomp_done_lock
;
457 static QemuCond decomp_done_cond
;
459 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
460 ram_addr_t offset
, uint8_t *source_buf
);
462 static void *do_data_compress(void *opaque
)
464 CompressParam
*param
= opaque
;
469 qemu_mutex_lock(¶m
->mutex
);
470 while (!param
->quit
) {
472 block
= param
->block
;
473 offset
= param
->offset
;
475 qemu_mutex_unlock(¶m
->mutex
);
477 zero_page
= do_compress_ram_page(param
->file
, ¶m
->stream
,
478 block
, offset
, param
->originbuf
);
480 qemu_mutex_lock(&comp_done_lock
);
482 param
->zero_page
= zero_page
;
483 qemu_cond_signal(&comp_done_cond
);
484 qemu_mutex_unlock(&comp_done_lock
);
486 qemu_mutex_lock(¶m
->mutex
);
488 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
491 qemu_mutex_unlock(¶m
->mutex
);
496 static void compress_threads_save_cleanup(void)
500 if (!migrate_use_compression() || !comp_param
) {
504 thread_count
= migrate_compress_threads();
505 for (i
= 0; i
< thread_count
; i
++) {
507 * we use it as a indicator which shows if the thread is
508 * properly init'd or not
510 if (!comp_param
[i
].file
) {
514 qemu_mutex_lock(&comp_param
[i
].mutex
);
515 comp_param
[i
].quit
= true;
516 qemu_cond_signal(&comp_param
[i
].cond
);
517 qemu_mutex_unlock(&comp_param
[i
].mutex
);
519 qemu_thread_join(compress_threads
+ i
);
520 qemu_mutex_destroy(&comp_param
[i
].mutex
);
521 qemu_cond_destroy(&comp_param
[i
].cond
);
522 deflateEnd(&comp_param
[i
].stream
);
523 g_free(comp_param
[i
].originbuf
);
524 qemu_fclose(comp_param
[i
].file
);
525 comp_param
[i
].file
= NULL
;
527 qemu_mutex_destroy(&comp_done_lock
);
528 qemu_cond_destroy(&comp_done_cond
);
529 g_free(compress_threads
);
531 compress_threads
= NULL
;
535 static int compress_threads_save_setup(void)
539 if (!migrate_use_compression()) {
542 thread_count
= migrate_compress_threads();
543 compress_threads
= g_new0(QemuThread
, thread_count
);
544 comp_param
= g_new0(CompressParam
, thread_count
);
545 qemu_cond_init(&comp_done_cond
);
546 qemu_mutex_init(&comp_done_lock
);
547 for (i
= 0; i
< thread_count
; i
++) {
548 comp_param
[i
].originbuf
= g_try_malloc(TARGET_PAGE_SIZE
);
549 if (!comp_param
[i
].originbuf
) {
553 if (deflateInit(&comp_param
[i
].stream
,
554 migrate_compress_level()) != Z_OK
) {
555 g_free(comp_param
[i
].originbuf
);
559 /* comp_param[i].file is just used as a dummy buffer to save data,
560 * set its ops to empty.
562 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
563 comp_param
[i
].done
= true;
564 comp_param
[i
].quit
= false;
565 qemu_mutex_init(&comp_param
[i
].mutex
);
566 qemu_cond_init(&comp_param
[i
].cond
);
567 qemu_thread_create(compress_threads
+ i
, "compress",
568 do_data_compress
, comp_param
+ i
,
569 QEMU_THREAD_JOINABLE
);
574 compress_threads_save_cleanup();
580 #define MULTIFD_MAGIC 0x11223344U
581 #define MULTIFD_VERSION 1
583 #define MULTIFD_FLAG_SYNC (1 << 0)
585 /* This value needs to be a multiple of qemu_target_page_size() */
586 #define MULTIFD_PACKET_SIZE (512 * 1024)
591 unsigned char uuid
[16]; /* QemuUUID */
593 uint8_t unused1
[7]; /* Reserved for future use */
594 uint64_t unused2
[4]; /* Reserved for future use */
595 } __attribute__((packed
)) MultiFDInit_t
;
601 /* maximum number of allocated pages */
602 uint32_t pages_alloc
;
604 /* size of the next packet that contains pages */
605 uint32_t next_packet_size
;
607 uint64_t unused
[4]; /* Reserved for future use */
610 } __attribute__((packed
)) MultiFDPacket_t
;
613 /* number of used pages */
615 /* number of allocated pages */
617 /* global number of generated multifd packets */
619 /* offset of each page */
621 /* pointer to each page */
627 /* this fields are not changed once the thread is created */
630 /* channel thread name */
632 /* channel thread id */
634 /* communication channel */
636 /* sem where to wait for more work */
638 /* this mutex protects the following parameters */
640 /* is this channel thread running */
642 /* should this thread finish */
644 /* thread has work to do */
646 /* array of pages to sent */
647 MultiFDPages_t
*pages
;
648 /* packet allocated len */
650 /* pointer to the packet */
651 MultiFDPacket_t
*packet
;
652 /* multifd flags for each packet */
654 /* size of the next packet that contains pages */
655 uint32_t next_packet_size
;
656 /* global number of generated multifd packets */
658 /* thread local variables */
659 /* packets sent through this channel */
660 uint64_t num_packets
;
661 /* pages sent through this channel */
663 /* syncs main thread and channels */
664 QemuSemaphore sem_sync
;
668 /* this fields are not changed once the thread is created */
671 /* channel thread name */
673 /* channel thread id */
675 /* communication channel */
677 /* this mutex protects the following parameters */
679 /* is this channel thread running */
681 /* should this thread finish */
683 /* array of pages to receive */
684 MultiFDPages_t
*pages
;
685 /* packet allocated len */
687 /* pointer to the packet */
688 MultiFDPacket_t
*packet
;
689 /* multifd flags for each packet */
691 /* global number of generated multifd packets */
693 /* thread local variables */
694 /* size of the next packet that contains pages */
695 uint32_t next_packet_size
;
696 /* packets sent through this channel */
697 uint64_t num_packets
;
698 /* pages sent through this channel */
700 /* syncs main thread and channels */
701 QemuSemaphore sem_sync
;
704 static int multifd_send_initial_packet(MultiFDSendParams
*p
, Error
**errp
)
706 MultiFDInit_t msg
= {};
709 msg
.magic
= cpu_to_be32(MULTIFD_MAGIC
);
710 msg
.version
= cpu_to_be32(MULTIFD_VERSION
);
712 memcpy(msg
.uuid
, &qemu_uuid
.data
, sizeof(msg
.uuid
));
714 ret
= qio_channel_write_all(p
->c
, (char *)&msg
, sizeof(msg
), errp
);
721 static int multifd_recv_initial_packet(QIOChannel
*c
, Error
**errp
)
726 ret
= qio_channel_read_all(c
, (char *)&msg
, sizeof(msg
), errp
);
731 msg
.magic
= be32_to_cpu(msg
.magic
);
732 msg
.version
= be32_to_cpu(msg
.version
);
734 if (msg
.magic
!= MULTIFD_MAGIC
) {
735 error_setg(errp
, "multifd: received packet magic %x "
736 "expected %x", msg
.magic
, MULTIFD_MAGIC
);
740 if (msg
.version
!= MULTIFD_VERSION
) {
741 error_setg(errp
, "multifd: received packet version %d "
742 "expected %d", msg
.version
, MULTIFD_VERSION
);
746 if (memcmp(msg
.uuid
, &qemu_uuid
, sizeof(qemu_uuid
))) {
747 char *uuid
= qemu_uuid_unparse_strdup(&qemu_uuid
);
748 char *msg_uuid
= qemu_uuid_unparse_strdup((const QemuUUID
*)msg
.uuid
);
750 error_setg(errp
, "multifd: received uuid '%s' and expected "
751 "uuid '%s' for channel %hhd", msg_uuid
, uuid
, msg
.id
);
757 if (msg
.id
> migrate_multifd_channels()) {
758 error_setg(errp
, "multifd: received channel version %d "
759 "expected %d", msg
.version
, MULTIFD_VERSION
);
766 static MultiFDPages_t
*multifd_pages_init(size_t size
)
768 MultiFDPages_t
*pages
= g_new0(MultiFDPages_t
, 1);
770 pages
->allocated
= size
;
771 pages
->iov
= g_new0(struct iovec
, size
);
772 pages
->offset
= g_new0(ram_addr_t
, size
);
777 static void multifd_pages_clear(MultiFDPages_t
*pages
)
780 pages
->allocated
= 0;
781 pages
->packet_num
= 0;
785 g_free(pages
->offset
);
786 pages
->offset
= NULL
;
790 static void multifd_send_fill_packet(MultiFDSendParams
*p
)
792 MultiFDPacket_t
*packet
= p
->packet
;
795 packet
->flags
= cpu_to_be32(p
->flags
);
796 packet
->pages_alloc
= cpu_to_be32(p
->pages
->allocated
);
797 packet
->pages_used
= cpu_to_be32(p
->pages
->used
);
798 packet
->next_packet_size
= cpu_to_be32(p
->next_packet_size
);
799 packet
->packet_num
= cpu_to_be64(p
->packet_num
);
801 if (p
->pages
->block
) {
802 strncpy(packet
->ramblock
, p
->pages
->block
->idstr
, 256);
805 for (i
= 0; i
< p
->pages
->used
; i
++) {
806 packet
->offset
[i
] = cpu_to_be64(p
->pages
->offset
[i
]);
810 static int multifd_recv_unfill_packet(MultiFDRecvParams
*p
, Error
**errp
)
812 MultiFDPacket_t
*packet
= p
->packet
;
813 uint32_t pages_max
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
817 packet
->magic
= be32_to_cpu(packet
->magic
);
818 if (packet
->magic
!= MULTIFD_MAGIC
) {
819 error_setg(errp
, "multifd: received packet "
820 "magic %x and expected magic %x",
821 packet
->magic
, MULTIFD_MAGIC
);
825 packet
->version
= be32_to_cpu(packet
->version
);
826 if (packet
->version
!= MULTIFD_VERSION
) {
827 error_setg(errp
, "multifd: received packet "
828 "version %d and expected version %d",
829 packet
->version
, MULTIFD_VERSION
);
833 p
->flags
= be32_to_cpu(packet
->flags
);
835 packet
->pages_alloc
= be32_to_cpu(packet
->pages_alloc
);
837 * If we received a packet that is 100 times bigger than expected
838 * just stop migration. It is a magic number.
840 if (packet
->pages_alloc
> pages_max
* 100) {
841 error_setg(errp
, "multifd: received packet "
842 "with size %d and expected a maximum size of %d",
843 packet
->pages_alloc
, pages_max
* 100) ;
847 * We received a packet that is bigger than expected but inside
848 * reasonable limits (see previous comment). Just reallocate.
850 if (packet
->pages_alloc
> p
->pages
->allocated
) {
851 multifd_pages_clear(p
->pages
);
852 p
->pages
= multifd_pages_init(packet
->pages_alloc
);
855 p
->pages
->used
= be32_to_cpu(packet
->pages_used
);
856 if (p
->pages
->used
> packet
->pages_alloc
) {
857 error_setg(errp
, "multifd: received packet "
858 "with %d pages and expected maximum pages are %d",
859 p
->pages
->used
, packet
->pages_alloc
) ;
863 p
->next_packet_size
= be32_to_cpu(packet
->next_packet_size
);
864 p
->packet_num
= be64_to_cpu(packet
->packet_num
);
866 if (p
->pages
->used
== 0) {
870 /* make sure that ramblock is 0 terminated */
871 packet
->ramblock
[255] = 0;
872 block
= qemu_ram_block_by_name(packet
->ramblock
);
874 error_setg(errp
, "multifd: unknown ram block %s",
879 for (i
= 0; i
< p
->pages
->used
; i
++) {
880 ram_addr_t offset
= be64_to_cpu(packet
->offset
[i
]);
882 if (offset
> (block
->used_length
- TARGET_PAGE_SIZE
)) {
883 error_setg(errp
, "multifd: offset too long " RAM_ADDR_FMT
884 " (max " RAM_ADDR_FMT
")",
885 offset
, block
->max_length
);
888 p
->pages
->iov
[i
].iov_base
= block
->host
+ offset
;
889 p
->pages
->iov
[i
].iov_len
= TARGET_PAGE_SIZE
;
896 MultiFDSendParams
*params
;
897 /* array of pages to sent */
898 MultiFDPages_t
*pages
;
899 /* global number of generated multifd packets */
901 /* send channels ready */
902 QemuSemaphore channels_ready
;
904 * Have we already run terminate threads. There is a race when it
905 * happens that we got one error while we are exiting.
906 * We will use atomic operations. Only valid values are 0 and 1.
909 } *multifd_send_state
;
912 * How we use multifd_send_state->pages and channel->pages?
914 * We create a pages for each channel, and a main one. Each time that
915 * we need to send a batch of pages we interchange the ones between
916 * multifd_send_state and the channel that is sending it. There are
917 * two reasons for that:
918 * - to not have to do so many mallocs during migration
919 * - to make easier to know what to free at the end of migration
921 * This way we always know who is the owner of each "pages" struct,
922 * and we don't need any locking. It belongs to the migration thread
923 * or to the channel thread. Switching is safe because the migration
924 * thread is using the channel mutex when changing it, and the channel
925 * have to had finish with its own, otherwise pending_job can't be
929 static int multifd_send_pages(RAMState
*rs
)
932 static int next_channel
;
933 MultiFDSendParams
*p
= NULL
; /* make happy gcc */
934 MultiFDPages_t
*pages
= multifd_send_state
->pages
;
935 uint64_t transferred
;
937 if (atomic_read(&multifd_send_state
->exiting
)) {
941 qemu_sem_wait(&multifd_send_state
->channels_ready
);
942 for (i
= next_channel
;; i
= (i
+ 1) % migrate_multifd_channels()) {
943 p
= &multifd_send_state
->params
[i
];
945 qemu_mutex_lock(&p
->mutex
);
947 error_report("%s: channel %d has already quit!", __func__
, i
);
948 qemu_mutex_unlock(&p
->mutex
);
951 if (!p
->pending_job
) {
953 next_channel
= (i
+ 1) % migrate_multifd_channels();
956 qemu_mutex_unlock(&p
->mutex
);
960 p
->packet_num
= multifd_send_state
->packet_num
++;
961 p
->pages
->block
= NULL
;
962 multifd_send_state
->pages
= p
->pages
;
964 transferred
= ((uint64_t) pages
->used
) * TARGET_PAGE_SIZE
+ p
->packet_len
;
965 qemu_file_update_transfer(rs
->f
, transferred
);
966 ram_counters
.multifd_bytes
+= transferred
;
967 ram_counters
.transferred
+= transferred
;;
968 qemu_mutex_unlock(&p
->mutex
);
969 qemu_sem_post(&p
->sem
);
974 static int multifd_queue_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
976 MultiFDPages_t
*pages
= multifd_send_state
->pages
;
979 pages
->block
= block
;
982 if (pages
->block
== block
) {
983 pages
->offset
[pages
->used
] = offset
;
984 pages
->iov
[pages
->used
].iov_base
= block
->host
+ offset
;
985 pages
->iov
[pages
->used
].iov_len
= TARGET_PAGE_SIZE
;
988 if (pages
->used
< pages
->allocated
) {
993 if (multifd_send_pages(rs
) < 0) {
997 if (pages
->block
!= block
) {
998 return multifd_queue_page(rs
, block
, offset
);
1004 static void multifd_send_terminate_threads(Error
*err
)
1008 trace_multifd_send_terminate_threads(err
!= NULL
);
1011 MigrationState
*s
= migrate_get_current();
1012 migrate_set_error(s
, err
);
1013 if (s
->state
== MIGRATION_STATUS_SETUP
||
1014 s
->state
== MIGRATION_STATUS_PRE_SWITCHOVER
||
1015 s
->state
== MIGRATION_STATUS_DEVICE
||
1016 s
->state
== MIGRATION_STATUS_ACTIVE
) {
1017 migrate_set_state(&s
->state
, s
->state
,
1018 MIGRATION_STATUS_FAILED
);
1023 * We don't want to exit each threads twice. Depending on where
1024 * we get the error, or if there are two independent errors in two
1025 * threads at the same time, we can end calling this function
1028 if (atomic_xchg(&multifd_send_state
->exiting
, 1)) {
1032 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1033 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1035 qemu_mutex_lock(&p
->mutex
);
1037 qemu_sem_post(&p
->sem
);
1038 qemu_mutex_unlock(&p
->mutex
);
1042 void multifd_save_cleanup(void)
1046 if (!migrate_use_multifd()) {
1049 multifd_send_terminate_threads(NULL
);
1050 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1051 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1054 qemu_thread_join(&p
->thread
);
1056 socket_send_channel_destroy(p
->c
);
1058 qemu_mutex_destroy(&p
->mutex
);
1059 qemu_sem_destroy(&p
->sem
);
1060 qemu_sem_destroy(&p
->sem_sync
);
1063 multifd_pages_clear(p
->pages
);
1069 qemu_sem_destroy(&multifd_send_state
->channels_ready
);
1070 g_free(multifd_send_state
->params
);
1071 multifd_send_state
->params
= NULL
;
1072 multifd_pages_clear(multifd_send_state
->pages
);
1073 multifd_send_state
->pages
= NULL
;
1074 g_free(multifd_send_state
);
1075 multifd_send_state
= NULL
;
1078 static void multifd_send_sync_main(RAMState
*rs
)
1082 if (!migrate_use_multifd()) {
1085 if (multifd_send_state
->pages
->used
) {
1086 if (multifd_send_pages(rs
) < 0) {
1087 error_report("%s: multifd_send_pages fail", __func__
);
1091 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1092 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1094 trace_multifd_send_sync_main_signal(p
->id
);
1096 qemu_mutex_lock(&p
->mutex
);
1099 error_report("%s: channel %d has already quit", __func__
, i
);
1100 qemu_mutex_unlock(&p
->mutex
);
1104 p
->packet_num
= multifd_send_state
->packet_num
++;
1105 p
->flags
|= MULTIFD_FLAG_SYNC
;
1107 qemu_file_update_transfer(rs
->f
, p
->packet_len
);
1108 ram_counters
.multifd_bytes
+= p
->packet_len
;
1109 ram_counters
.transferred
+= p
->packet_len
;
1110 qemu_mutex_unlock(&p
->mutex
);
1111 qemu_sem_post(&p
->sem
);
1113 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1114 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1116 trace_multifd_send_sync_main_wait(p
->id
);
1117 qemu_sem_wait(&p
->sem_sync
);
1119 trace_multifd_send_sync_main(multifd_send_state
->packet_num
);
1122 static void *multifd_send_thread(void *opaque
)
1124 MultiFDSendParams
*p
= opaque
;
1125 Error
*local_err
= NULL
;
1129 trace_multifd_send_thread_start(p
->id
);
1130 rcu_register_thread();
1132 if (multifd_send_initial_packet(p
, &local_err
) < 0) {
1136 /* initial packet */
1140 qemu_sem_wait(&p
->sem
);
1142 if (atomic_read(&multifd_send_state
->exiting
)) {
1145 qemu_mutex_lock(&p
->mutex
);
1147 if (p
->pending_job
) {
1148 uint32_t used
= p
->pages
->used
;
1149 uint64_t packet_num
= p
->packet_num
;
1152 p
->next_packet_size
= used
* qemu_target_page_size();
1153 multifd_send_fill_packet(p
);
1156 p
->num_pages
+= used
;
1157 qemu_mutex_unlock(&p
->mutex
);
1159 trace_multifd_send(p
->id
, packet_num
, used
, flags
,
1160 p
->next_packet_size
);
1162 ret
= qio_channel_write_all(p
->c
, (void *)p
->packet
,
1163 p
->packet_len
, &local_err
);
1169 ret
= qio_channel_writev_all(p
->c
, p
->pages
->iov
,
1176 qemu_mutex_lock(&p
->mutex
);
1178 qemu_mutex_unlock(&p
->mutex
);
1180 if (flags
& MULTIFD_FLAG_SYNC
) {
1181 qemu_sem_post(&p
->sem_sync
);
1183 qemu_sem_post(&multifd_send_state
->channels_ready
);
1184 } else if (p
->quit
) {
1185 qemu_mutex_unlock(&p
->mutex
);
1188 qemu_mutex_unlock(&p
->mutex
);
1189 /* sometimes there are spurious wakeups */
1195 trace_multifd_send_error(p
->id
);
1196 multifd_send_terminate_threads(local_err
);
1200 * Error happen, I will exit, but I can't just leave, tell
1201 * who pay attention to me.
1204 qemu_sem_post(&p
->sem_sync
);
1205 qemu_sem_post(&multifd_send_state
->channels_ready
);
1208 qemu_mutex_lock(&p
->mutex
);
1210 qemu_mutex_unlock(&p
->mutex
);
1212 rcu_unregister_thread();
1213 trace_multifd_send_thread_end(p
->id
, p
->num_packets
, p
->num_pages
);
1218 static void multifd_new_send_channel_async(QIOTask
*task
, gpointer opaque
)
1220 MultiFDSendParams
*p
= opaque
;
1221 QIOChannel
*sioc
= QIO_CHANNEL(qio_task_get_source(task
));
1222 Error
*local_err
= NULL
;
1224 trace_multifd_new_send_channel_async(p
->id
);
1225 if (qio_task_propagate_error(task
, &local_err
)) {
1226 migrate_set_error(migrate_get_current(), local_err
);
1227 multifd_save_cleanup();
1229 p
->c
= QIO_CHANNEL(sioc
);
1230 qio_channel_set_delay(p
->c
, false);
1232 qemu_thread_create(&p
->thread
, p
->name
, multifd_send_thread
, p
,
1233 QEMU_THREAD_JOINABLE
);
1237 int multifd_save_setup(void)
1240 uint32_t page_count
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
1243 if (!migrate_use_multifd()) {
1246 thread_count
= migrate_multifd_channels();
1247 multifd_send_state
= g_malloc0(sizeof(*multifd_send_state
));
1248 multifd_send_state
->params
= g_new0(MultiFDSendParams
, thread_count
);
1249 multifd_send_state
->pages
= multifd_pages_init(page_count
);
1250 qemu_sem_init(&multifd_send_state
->channels_ready
, 0);
1251 atomic_set(&multifd_send_state
->exiting
, 0);
1253 for (i
= 0; i
< thread_count
; i
++) {
1254 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1256 qemu_mutex_init(&p
->mutex
);
1257 qemu_sem_init(&p
->sem
, 0);
1258 qemu_sem_init(&p
->sem_sync
, 0);
1262 p
->pages
= multifd_pages_init(page_count
);
1263 p
->packet_len
= sizeof(MultiFDPacket_t
)
1264 + sizeof(ram_addr_t
) * page_count
;
1265 p
->packet
= g_malloc0(p
->packet_len
);
1266 p
->packet
->magic
= cpu_to_be32(MULTIFD_MAGIC
);
1267 p
->packet
->version
= cpu_to_be32(MULTIFD_VERSION
);
1268 p
->name
= g_strdup_printf("multifdsend_%d", i
);
1269 socket_send_channel_create(multifd_new_send_channel_async
, p
);
1275 MultiFDRecvParams
*params
;
1276 /* number of created threads */
1278 /* syncs main thread and channels */
1279 QemuSemaphore sem_sync
;
1280 /* global number of generated multifd packets */
1281 uint64_t packet_num
;
1282 } *multifd_recv_state
;
1284 static void multifd_recv_terminate_threads(Error
*err
)
1288 trace_multifd_recv_terminate_threads(err
!= NULL
);
1291 MigrationState
*s
= migrate_get_current();
1292 migrate_set_error(s
, err
);
1293 if (s
->state
== MIGRATION_STATUS_SETUP
||
1294 s
->state
== MIGRATION_STATUS_ACTIVE
) {
1295 migrate_set_state(&s
->state
, s
->state
,
1296 MIGRATION_STATUS_FAILED
);
1300 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1301 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1303 qemu_mutex_lock(&p
->mutex
);
1305 /* We could arrive here for two reasons:
1306 - normal quit, i.e. everything went fine, just finished
1307 - error quit: We close the channels so the channel threads
1308 finish the qio_channel_read_all_eof() */
1309 qio_channel_shutdown(p
->c
, QIO_CHANNEL_SHUTDOWN_BOTH
, NULL
);
1310 qemu_mutex_unlock(&p
->mutex
);
1314 int multifd_load_cleanup(Error
**errp
)
1319 if (!migrate_use_multifd()) {
1322 multifd_recv_terminate_threads(NULL
);
1323 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1324 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1329 * multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code,
1330 * however try to wakeup it without harm in cleanup phase.
1332 qemu_sem_post(&p
->sem_sync
);
1333 qemu_thread_join(&p
->thread
);
1335 object_unref(OBJECT(p
->c
));
1337 qemu_mutex_destroy(&p
->mutex
);
1338 qemu_sem_destroy(&p
->sem_sync
);
1341 multifd_pages_clear(p
->pages
);
1347 qemu_sem_destroy(&multifd_recv_state
->sem_sync
);
1348 g_free(multifd_recv_state
->params
);
1349 multifd_recv_state
->params
= NULL
;
1350 g_free(multifd_recv_state
);
1351 multifd_recv_state
= NULL
;
1356 static void multifd_recv_sync_main(void)
1360 if (!migrate_use_multifd()) {
1363 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1364 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1366 trace_multifd_recv_sync_main_wait(p
->id
);
1367 qemu_sem_wait(&multifd_recv_state
->sem_sync
);
1369 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1370 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1372 qemu_mutex_lock(&p
->mutex
);
1373 if (multifd_recv_state
->packet_num
< p
->packet_num
) {
1374 multifd_recv_state
->packet_num
= p
->packet_num
;
1376 qemu_mutex_unlock(&p
->mutex
);
1377 trace_multifd_recv_sync_main_signal(p
->id
);
1378 qemu_sem_post(&p
->sem_sync
);
1380 trace_multifd_recv_sync_main(multifd_recv_state
->packet_num
);
1383 static void *multifd_recv_thread(void *opaque
)
1385 MultiFDRecvParams
*p
= opaque
;
1386 Error
*local_err
= NULL
;
1389 trace_multifd_recv_thread_start(p
->id
);
1390 rcu_register_thread();
1400 ret
= qio_channel_read_all_eof(p
->c
, (void *)p
->packet
,
1401 p
->packet_len
, &local_err
);
1402 if (ret
== 0) { /* EOF */
1405 if (ret
== -1) { /* Error */
1409 qemu_mutex_lock(&p
->mutex
);
1410 ret
= multifd_recv_unfill_packet(p
, &local_err
);
1412 qemu_mutex_unlock(&p
->mutex
);
1416 used
= p
->pages
->used
;
1418 trace_multifd_recv(p
->id
, p
->packet_num
, used
, flags
,
1419 p
->next_packet_size
);
1421 p
->num_pages
+= used
;
1422 qemu_mutex_unlock(&p
->mutex
);
1425 ret
= qio_channel_readv_all(p
->c
, p
->pages
->iov
,
1432 if (flags
& MULTIFD_FLAG_SYNC
) {
1433 qemu_sem_post(&multifd_recv_state
->sem_sync
);
1434 qemu_sem_wait(&p
->sem_sync
);
1439 multifd_recv_terminate_threads(local_err
);
1441 qemu_mutex_lock(&p
->mutex
);
1443 qemu_mutex_unlock(&p
->mutex
);
1445 rcu_unregister_thread();
1446 trace_multifd_recv_thread_end(p
->id
, p
->num_packets
, p
->num_pages
);
1451 int multifd_load_setup(void)
1454 uint32_t page_count
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
1457 if (!migrate_use_multifd()) {
1460 thread_count
= migrate_multifd_channels();
1461 multifd_recv_state
= g_malloc0(sizeof(*multifd_recv_state
));
1462 multifd_recv_state
->params
= g_new0(MultiFDRecvParams
, thread_count
);
1463 atomic_set(&multifd_recv_state
->count
, 0);
1464 qemu_sem_init(&multifd_recv_state
->sem_sync
, 0);
1466 for (i
= 0; i
< thread_count
; i
++) {
1467 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1469 qemu_mutex_init(&p
->mutex
);
1470 qemu_sem_init(&p
->sem_sync
, 0);
1473 p
->pages
= multifd_pages_init(page_count
);
1474 p
->packet_len
= sizeof(MultiFDPacket_t
)
1475 + sizeof(ram_addr_t
) * page_count
;
1476 p
->packet
= g_malloc0(p
->packet_len
);
1477 p
->name
= g_strdup_printf("multifdrecv_%d", i
);
1482 bool multifd_recv_all_channels_created(void)
1484 int thread_count
= migrate_multifd_channels();
1486 if (!migrate_use_multifd()) {
1490 return thread_count
== atomic_read(&multifd_recv_state
->count
);
1494 * Try to receive all multifd channels to get ready for the migration.
1495 * - Return true and do not set @errp when correctly receving all channels;
1496 * - Return false and do not set @errp when correctly receiving the current one;
1497 * - Return false and set @errp when failing to receive the current channel.
1499 bool multifd_recv_new_channel(QIOChannel
*ioc
, Error
**errp
)
1501 MultiFDRecvParams
*p
;
1502 Error
*local_err
= NULL
;
1505 id
= multifd_recv_initial_packet(ioc
, &local_err
);
1507 multifd_recv_terminate_threads(local_err
);
1508 error_propagate_prepend(errp
, local_err
,
1509 "failed to receive packet"
1510 " via multifd channel %d: ",
1511 atomic_read(&multifd_recv_state
->count
));
1514 trace_multifd_recv_new_channel(id
);
1516 p
= &multifd_recv_state
->params
[id
];
1518 error_setg(&local_err
, "multifd: received id '%d' already setup'",
1520 multifd_recv_terminate_threads(local_err
);
1521 error_propagate(errp
, local_err
);
1525 object_ref(OBJECT(ioc
));
1526 /* initial packet */
1530 qemu_thread_create(&p
->thread
, p
->name
, multifd_recv_thread
, p
,
1531 QEMU_THREAD_JOINABLE
);
1532 atomic_inc(&multifd_recv_state
->count
);
1533 return atomic_read(&multifd_recv_state
->count
) ==
1534 migrate_multifd_channels();
1538 * save_page_header: write page header to wire
1540 * If this is the 1st block, it also writes the block identification
1542 * Returns the number of bytes written
1544 * @f: QEMUFile where to send the data
1545 * @block: block that contains the page we want to send
1546 * @offset: offset inside the block for the page
1547 * in the lower bits, it contains flags
1549 static size_t save_page_header(RAMState
*rs
, QEMUFile
*f
, RAMBlock
*block
,
1554 if (block
== rs
->last_sent_block
) {
1555 offset
|= RAM_SAVE_FLAG_CONTINUE
;
1557 qemu_put_be64(f
, offset
);
1560 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
1561 len
= strlen(block
->idstr
);
1562 qemu_put_byte(f
, len
);
1563 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
1565 rs
->last_sent_block
= block
;
1571 * mig_throttle_guest_down: throotle down the guest
1573 * Reduce amount of guest cpu execution to hopefully slow down memory
1574 * writes. If guest dirty memory rate is reduced below the rate at
1575 * which we can transfer pages to the destination then we should be
1576 * able to complete migration. Some workloads dirty memory way too
1577 * fast and will not effectively converge, even with auto-converge.
1579 static void mig_throttle_guest_down(void)
1581 MigrationState
*s
= migrate_get_current();
1582 uint64_t pct_initial
= s
->parameters
.cpu_throttle_initial
;
1583 uint64_t pct_icrement
= s
->parameters
.cpu_throttle_increment
;
1584 int pct_max
= s
->parameters
.max_cpu_throttle
;
1586 /* We have not started throttling yet. Let's start it. */
1587 if (!cpu_throttle_active()) {
1588 cpu_throttle_set(pct_initial
);
1590 /* Throttling already on, just increase the rate */
1591 cpu_throttle_set(MIN(cpu_throttle_get_percentage() + pct_icrement
,
1597 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
1599 * @rs: current RAM state
1600 * @current_addr: address for the zero page
1602 * Update the xbzrle cache to reflect a page that's been sent as all 0.
1603 * The important thing is that a stale (not-yet-0'd) page be replaced
1605 * As a bonus, if the page wasn't in the cache it gets added so that
1606 * when a small write is made into the 0'd page it gets XBZRLE sent.
1608 static void xbzrle_cache_zero_page(RAMState
*rs
, ram_addr_t current_addr
)
1610 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
1614 /* We don't care if this fails to allocate a new cache page
1615 * as long as it updated an old one */
1616 cache_insert(XBZRLE
.cache
, current_addr
, XBZRLE
.zero_target_page
,
1617 ram_counters
.dirty_sync_count
);
1620 #define ENCODING_FLAG_XBZRLE 0x1
1623 * save_xbzrle_page: compress and send current page
1625 * Returns: 1 means that we wrote the page
1626 * 0 means that page is identical to the one already sent
1627 * -1 means that xbzrle would be longer than normal
1629 * @rs: current RAM state
1630 * @current_data: pointer to the address of the page contents
1631 * @current_addr: addr of the page
1632 * @block: block that contains the page we want to send
1633 * @offset: offset inside the block for the page
1634 * @last_stage: if we are at the completion stage
1636 static int save_xbzrle_page(RAMState
*rs
, uint8_t **current_data
,
1637 ram_addr_t current_addr
, RAMBlock
*block
,
1638 ram_addr_t offset
, bool last_stage
)
1640 int encoded_len
= 0, bytes_xbzrle
;
1641 uint8_t *prev_cached_page
;
1643 if (!cache_is_cached(XBZRLE
.cache
, current_addr
,
1644 ram_counters
.dirty_sync_count
)) {
1645 xbzrle_counters
.cache_miss
++;
1647 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
1648 ram_counters
.dirty_sync_count
) == -1) {
1651 /* update *current_data when the page has been
1652 inserted into cache */
1653 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
1659 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
1661 /* save current buffer into memory */
1662 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
1664 /* XBZRLE encoding (if there is no overflow) */
1665 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
1666 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
1670 * Update the cache contents, so that it corresponds to the data
1671 * sent, in all cases except where we skip the page.
1673 if (!last_stage
&& encoded_len
!= 0) {
1674 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
1676 * In the case where we couldn't compress, ensure that the caller
1677 * sends the data from the cache, since the guest might have
1678 * changed the RAM since we copied it.
1680 *current_data
= prev_cached_page
;
1683 if (encoded_len
== 0) {
1684 trace_save_xbzrle_page_skipping();
1686 } else if (encoded_len
== -1) {
1687 trace_save_xbzrle_page_overflow();
1688 xbzrle_counters
.overflow
++;
1692 /* Send XBZRLE based compressed page */
1693 bytes_xbzrle
= save_page_header(rs
, rs
->f
, block
,
1694 offset
| RAM_SAVE_FLAG_XBZRLE
);
1695 qemu_put_byte(rs
->f
, ENCODING_FLAG_XBZRLE
);
1696 qemu_put_be16(rs
->f
, encoded_len
);
1697 qemu_put_buffer(rs
->f
, XBZRLE
.encoded_buf
, encoded_len
);
1698 bytes_xbzrle
+= encoded_len
+ 1 + 2;
1699 xbzrle_counters
.pages
++;
1700 xbzrle_counters
.bytes
+= bytes_xbzrle
;
1701 ram_counters
.transferred
+= bytes_xbzrle
;
1707 * migration_bitmap_find_dirty: find the next dirty page from start
1709 * Returns the page offset within memory region of the start of a dirty page
1711 * @rs: current RAM state
1712 * @rb: RAMBlock where to search for dirty pages
1713 * @start: page where we start the search
1716 unsigned long migration_bitmap_find_dirty(RAMState
*rs
, RAMBlock
*rb
,
1717 unsigned long start
)
1719 unsigned long size
= rb
->used_length
>> TARGET_PAGE_BITS
;
1720 unsigned long *bitmap
= rb
->bmap
;
1723 if (ramblock_is_ignored(rb
)) {
1728 * When the free page optimization is enabled, we need to check the bitmap
1729 * to send the non-free pages rather than all the pages in the bulk stage.
1731 if (!rs
->fpo_enabled
&& rs
->ram_bulk_stage
&& start
> 0) {
1734 next
= find_next_bit(bitmap
, size
, start
);
1740 static inline bool migration_bitmap_clear_dirty(RAMState
*rs
,
1746 qemu_mutex_lock(&rs
->bitmap_mutex
);
1749 * Clear dirty bitmap if needed. This _must_ be called before we
1750 * send any of the page in the chunk because we need to make sure
1751 * we can capture further page content changes when we sync dirty
1752 * log the next time. So as long as we are going to send any of
1753 * the page in the chunk we clear the remote dirty bitmap for all.
1754 * Clearing it earlier won't be a problem, but too late will.
1756 if (rb
->clear_bmap
&& clear_bmap_test_and_clear(rb
, page
)) {
1757 uint8_t shift
= rb
->clear_bmap_shift
;
1758 hwaddr size
= 1ULL << (TARGET_PAGE_BITS
+ shift
);
1759 hwaddr start
= (page
<< TARGET_PAGE_BITS
) & (-size
);
1762 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
1763 * can make things easier sometimes since then start address
1764 * of the small chunk will always be 64 pages aligned so the
1765 * bitmap will always be aligned to unsigned long. We should
1766 * even be able to remove this restriction but I'm simply
1770 trace_migration_bitmap_clear_dirty(rb
->idstr
, start
, size
, page
);
1771 memory_region_clear_dirty_bitmap(rb
->mr
, start
, size
);
1774 ret
= test_and_clear_bit(page
, rb
->bmap
);
1777 rs
->migration_dirty_pages
--;
1779 qemu_mutex_unlock(&rs
->bitmap_mutex
);
1784 /* Called with RCU critical section */
1785 static void ramblock_sync_dirty_bitmap(RAMState
*rs
, RAMBlock
*rb
)
1787 rs
->migration_dirty_pages
+=
1788 cpu_physical_memory_sync_dirty_bitmap(rb
, 0, rb
->used_length
,
1789 &rs
->num_dirty_pages_period
);
1793 * ram_pagesize_summary: calculate all the pagesizes of a VM
1795 * Returns a summary bitmap of the page sizes of all RAMBlocks
1797 * For VMs with just normal pages this is equivalent to the host page
1798 * size. If it's got some huge pages then it's the OR of all the
1799 * different page sizes.
1801 uint64_t ram_pagesize_summary(void)
1804 uint64_t summary
= 0;
1806 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1807 summary
|= block
->page_size
;
1813 uint64_t ram_get_total_transferred_pages(void)
1815 return ram_counters
.normal
+ ram_counters
.duplicate
+
1816 compression_counters
.pages
+ xbzrle_counters
.pages
;
1819 static void migration_update_rates(RAMState
*rs
, int64_t end_time
)
1821 uint64_t page_count
= rs
->target_page_count
- rs
->target_page_count_prev
;
1822 double compressed_size
;
1824 /* calculate period counters */
1825 ram_counters
.dirty_pages_rate
= rs
->num_dirty_pages_period
* 1000
1826 / (end_time
- rs
->time_last_bitmap_sync
);
1832 if (migrate_use_xbzrle()) {
1833 xbzrle_counters
.cache_miss_rate
= (double)(xbzrle_counters
.cache_miss
-
1834 rs
->xbzrle_cache_miss_prev
) / page_count
;
1835 rs
->xbzrle_cache_miss_prev
= xbzrle_counters
.cache_miss
;
1838 if (migrate_use_compression()) {
1839 compression_counters
.busy_rate
= (double)(compression_counters
.busy
-
1840 rs
->compress_thread_busy_prev
) / page_count
;
1841 rs
->compress_thread_busy_prev
= compression_counters
.busy
;
1843 compressed_size
= compression_counters
.compressed_size
-
1844 rs
->compressed_size_prev
;
1845 if (compressed_size
) {
1846 double uncompressed_size
= (compression_counters
.pages
-
1847 rs
->compress_pages_prev
) * TARGET_PAGE_SIZE
;
1849 /* Compression-Ratio = Uncompressed-size / Compressed-size */
1850 compression_counters
.compression_rate
=
1851 uncompressed_size
/ compressed_size
;
1853 rs
->compress_pages_prev
= compression_counters
.pages
;
1854 rs
->compressed_size_prev
= compression_counters
.compressed_size
;
1859 static void migration_bitmap_sync(RAMState
*rs
)
1863 uint64_t bytes_xfer_now
;
1865 ram_counters
.dirty_sync_count
++;
1867 if (!rs
->time_last_bitmap_sync
) {
1868 rs
->time_last_bitmap_sync
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1871 trace_migration_bitmap_sync_start();
1872 memory_global_dirty_log_sync();
1874 qemu_mutex_lock(&rs
->bitmap_mutex
);
1875 WITH_RCU_READ_LOCK_GUARD() {
1876 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1877 ramblock_sync_dirty_bitmap(rs
, block
);
1879 ram_counters
.remaining
= ram_bytes_remaining();
1881 qemu_mutex_unlock(&rs
->bitmap_mutex
);
1883 memory_global_after_dirty_log_sync();
1884 trace_migration_bitmap_sync_end(rs
->num_dirty_pages_period
);
1886 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1888 /* more than 1 second = 1000 millisecons */
1889 if (end_time
> rs
->time_last_bitmap_sync
+ 1000) {
1890 bytes_xfer_now
= ram_counters
.transferred
;
1892 /* During block migration the auto-converge logic incorrectly detects
1893 * that ram migration makes no progress. Avoid this by disabling the
1894 * throttling logic during the bulk phase of block migration. */
1895 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
1896 /* The following detection logic can be refined later. For now:
1897 Check to see if the dirtied bytes is 50% more than the approx.
1898 amount of bytes that just got transferred since the last time we
1899 were in this routine. If that happens twice, start or increase
1902 if ((rs
->num_dirty_pages_period
* TARGET_PAGE_SIZE
>
1903 (bytes_xfer_now
- rs
->bytes_xfer_prev
) / 2) &&
1904 (++rs
->dirty_rate_high_cnt
>= 2)) {
1905 trace_migration_throttle();
1906 rs
->dirty_rate_high_cnt
= 0;
1907 mig_throttle_guest_down();
1911 migration_update_rates(rs
, end_time
);
1913 rs
->target_page_count_prev
= rs
->target_page_count
;
1915 /* reset period counters */
1916 rs
->time_last_bitmap_sync
= end_time
;
1917 rs
->num_dirty_pages_period
= 0;
1918 rs
->bytes_xfer_prev
= bytes_xfer_now
;
1920 if (migrate_use_events()) {
1921 qapi_event_send_migration_pass(ram_counters
.dirty_sync_count
);
1925 static void migration_bitmap_sync_precopy(RAMState
*rs
)
1927 Error
*local_err
= NULL
;
1930 * The current notifier usage is just an optimization to migration, so we
1931 * don't stop the normal migration process in the error case.
1933 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC
, &local_err
)) {
1934 error_report_err(local_err
);
1937 migration_bitmap_sync(rs
);
1939 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC
, &local_err
)) {
1940 error_report_err(local_err
);
1945 * save_zero_page_to_file: send the zero page to the file
1947 * Returns the size of data written to the file, 0 means the page is not
1950 * @rs: current RAM state
1951 * @file: the file where the data is saved
1952 * @block: block that contains the page we want to send
1953 * @offset: offset inside the block for the page
1955 static int save_zero_page_to_file(RAMState
*rs
, QEMUFile
*file
,
1956 RAMBlock
*block
, ram_addr_t offset
)
1958 uint8_t *p
= block
->host
+ offset
;
1961 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
1962 len
+= save_page_header(rs
, file
, block
, offset
| RAM_SAVE_FLAG_ZERO
);
1963 qemu_put_byte(file
, 0);
1970 * save_zero_page: send the zero page to the stream
1972 * Returns the number of pages written.
1974 * @rs: current RAM state
1975 * @block: block that contains the page we want to send
1976 * @offset: offset inside the block for the page
1978 static int save_zero_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
1980 int len
= save_zero_page_to_file(rs
, rs
->f
, block
, offset
);
1983 ram_counters
.duplicate
++;
1984 ram_counters
.transferred
+= len
;
1990 static void ram_release_pages(const char *rbname
, uint64_t offset
, int pages
)
1992 if (!migrate_release_ram() || !migration_in_postcopy()) {
1996 ram_discard_range(rbname
, offset
, pages
<< TARGET_PAGE_BITS
);
2000 * @pages: the number of pages written by the control path,
2002 * > 0 - number of pages written
2004 * Return true if the pages has been saved, otherwise false is returned.
2006 static bool control_save_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
2009 uint64_t bytes_xmit
= 0;
2013 ret
= ram_control_save_page(rs
->f
, block
->offset
, offset
, TARGET_PAGE_SIZE
,
2015 if (ret
== RAM_SAVE_CONTROL_NOT_SUPP
) {
2020 ram_counters
.transferred
+= bytes_xmit
;
2024 if (ret
== RAM_SAVE_CONTROL_DELAYED
) {
2028 if (bytes_xmit
> 0) {
2029 ram_counters
.normal
++;
2030 } else if (bytes_xmit
== 0) {
2031 ram_counters
.duplicate
++;
2038 * directly send the page to the stream
2040 * Returns the number of pages written.
2042 * @rs: current RAM state
2043 * @block: block that contains the page we want to send
2044 * @offset: offset inside the block for the page
2045 * @buf: the page to be sent
2046 * @async: send to page asyncly
2048 static int save_normal_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
2049 uint8_t *buf
, bool async
)
2051 ram_counters
.transferred
+= save_page_header(rs
, rs
->f
, block
,
2052 offset
| RAM_SAVE_FLAG_PAGE
);
2054 qemu_put_buffer_async(rs
->f
, buf
, TARGET_PAGE_SIZE
,
2055 migrate_release_ram() &
2056 migration_in_postcopy());
2058 qemu_put_buffer(rs
->f
, buf
, TARGET_PAGE_SIZE
);
2060 ram_counters
.transferred
+= TARGET_PAGE_SIZE
;
2061 ram_counters
.normal
++;
2066 * ram_save_page: send the given page to the stream
2068 * Returns the number of pages written.
2070 * >=0 - Number of pages written - this might legally be 0
2071 * if xbzrle noticed the page was the same.
2073 * @rs: current RAM state
2074 * @block: block that contains the page we want to send
2075 * @offset: offset inside the block for the page
2076 * @last_stage: if we are at the completion stage
2078 static int ram_save_page(RAMState
*rs
, PageSearchStatus
*pss
, bool last_stage
)
2082 bool send_async
= true;
2083 RAMBlock
*block
= pss
->block
;
2084 ram_addr_t offset
= pss
->page
<< TARGET_PAGE_BITS
;
2085 ram_addr_t current_addr
= block
->offset
+ offset
;
2087 p
= block
->host
+ offset
;
2088 trace_ram_save_page(block
->idstr
, (uint64_t)offset
, p
);
2090 XBZRLE_cache_lock();
2091 if (!rs
->ram_bulk_stage
&& !migration_in_postcopy() &&
2092 migrate_use_xbzrle()) {
2093 pages
= save_xbzrle_page(rs
, &p
, current_addr
, block
,
2094 offset
, last_stage
);
2096 /* Can't send this cached data async, since the cache page
2097 * might get updated before it gets to the wire
2103 /* XBZRLE overflow or normal page */
2105 pages
= save_normal_page(rs
, block
, offset
, p
, send_async
);
2108 XBZRLE_cache_unlock();
2113 static int ram_save_multifd_page(RAMState
*rs
, RAMBlock
*block
,
2116 if (multifd_queue_page(rs
, block
, offset
) < 0) {
2119 ram_counters
.normal
++;
2124 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
2125 ram_addr_t offset
, uint8_t *source_buf
)
2127 RAMState
*rs
= ram_state
;
2128 uint8_t *p
= block
->host
+ (offset
& TARGET_PAGE_MASK
);
2129 bool zero_page
= false;
2132 if (save_zero_page_to_file(rs
, f
, block
, offset
)) {
2137 save_page_header(rs
, f
, block
, offset
| RAM_SAVE_FLAG_COMPRESS_PAGE
);
2140 * copy it to a internal buffer to avoid it being modified by VM
2141 * so that we can catch up the error during compression and
2144 memcpy(source_buf
, p
, TARGET_PAGE_SIZE
);
2145 ret
= qemu_put_compression_data(f
, stream
, source_buf
, TARGET_PAGE_SIZE
);
2147 qemu_file_set_error(migrate_get_current()->to_dst_file
, ret
);
2148 error_report("compressed data failed!");
2153 ram_release_pages(block
->idstr
, offset
& TARGET_PAGE_MASK
, 1);
2158 update_compress_thread_counts(const CompressParam
*param
, int bytes_xmit
)
2160 ram_counters
.transferred
+= bytes_xmit
;
2162 if (param
->zero_page
) {
2163 ram_counters
.duplicate
++;
2167 /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
2168 compression_counters
.compressed_size
+= bytes_xmit
- 8;
2169 compression_counters
.pages
++;
2172 static bool save_page_use_compression(RAMState
*rs
);
2174 static void flush_compressed_data(RAMState
*rs
)
2176 int idx
, len
, thread_count
;
2178 if (!save_page_use_compression(rs
)) {
2181 thread_count
= migrate_compress_threads();
2183 qemu_mutex_lock(&comp_done_lock
);
2184 for (idx
= 0; idx
< thread_count
; idx
++) {
2185 while (!comp_param
[idx
].done
) {
2186 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
2189 qemu_mutex_unlock(&comp_done_lock
);
2191 for (idx
= 0; idx
< thread_count
; idx
++) {
2192 qemu_mutex_lock(&comp_param
[idx
].mutex
);
2193 if (!comp_param
[idx
].quit
) {
2194 len
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
2196 * it's safe to fetch zero_page without holding comp_done_lock
2197 * as there is no further request submitted to the thread,
2198 * i.e, the thread should be waiting for a request at this point.
2200 update_compress_thread_counts(&comp_param
[idx
], len
);
2202 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
2206 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
2209 param
->block
= block
;
2210 param
->offset
= offset
;
2213 static int compress_page_with_multi_thread(RAMState
*rs
, RAMBlock
*block
,
2216 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
2217 bool wait
= migrate_compress_wait_thread();
2219 thread_count
= migrate_compress_threads();
2220 qemu_mutex_lock(&comp_done_lock
);
2222 for (idx
= 0; idx
< thread_count
; idx
++) {
2223 if (comp_param
[idx
].done
) {
2224 comp_param
[idx
].done
= false;
2225 bytes_xmit
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
2226 qemu_mutex_lock(&comp_param
[idx
].mutex
);
2227 set_compress_params(&comp_param
[idx
], block
, offset
);
2228 qemu_cond_signal(&comp_param
[idx
].cond
);
2229 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
2231 update_compress_thread_counts(&comp_param
[idx
], bytes_xmit
);
2237 * wait for the free thread if the user specifies 'compress-wait-thread',
2238 * otherwise we will post the page out in the main thread as normal page.
2240 if (pages
< 0 && wait
) {
2241 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
2244 qemu_mutex_unlock(&comp_done_lock
);
2250 * find_dirty_block: find the next dirty page and update any state
2251 * associated with the search process.
2253 * Returns true if a page is found
2255 * @rs: current RAM state
2256 * @pss: data about the state of the current dirty page scan
2257 * @again: set to false if the search has scanned the whole of RAM
2259 static bool find_dirty_block(RAMState
*rs
, PageSearchStatus
*pss
, bool *again
)
2261 pss
->page
= migration_bitmap_find_dirty(rs
, pss
->block
, pss
->page
);
2262 if (pss
->complete_round
&& pss
->block
== rs
->last_seen_block
&&
2263 pss
->page
>= rs
->last_page
) {
2265 * We've been once around the RAM and haven't found anything.
2271 if ((pss
->page
<< TARGET_PAGE_BITS
) >= pss
->block
->used_length
) {
2272 /* Didn't find anything in this RAM Block */
2274 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
2277 * If memory migration starts over, we will meet a dirtied page
2278 * which may still exists in compression threads's ring, so we
2279 * should flush the compressed data to make sure the new page
2280 * is not overwritten by the old one in the destination.
2282 * Also If xbzrle is on, stop using the data compression at this
2283 * point. In theory, xbzrle can do better than compression.
2285 flush_compressed_data(rs
);
2287 /* Hit the end of the list */
2288 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2289 /* Flag that we've looped */
2290 pss
->complete_round
= true;
2291 rs
->ram_bulk_stage
= false;
2293 /* Didn't find anything this time, but try again on the new block */
2297 /* Can go around again, but... */
2299 /* We've found something so probably don't need to */
2305 * unqueue_page: gets a page of the queue
2307 * Helper for 'get_queued_page' - gets a page off the queue
2309 * Returns the block of the page (or NULL if none available)
2311 * @rs: current RAM state
2312 * @offset: used to return the offset within the RAMBlock
2314 static RAMBlock
*unqueue_page(RAMState
*rs
, ram_addr_t
*offset
)
2316 RAMBlock
*block
= NULL
;
2318 if (QSIMPLEQ_EMPTY_ATOMIC(&rs
->src_page_requests
)) {
2322 qemu_mutex_lock(&rs
->src_page_req_mutex
);
2323 if (!QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
2324 struct RAMSrcPageRequest
*entry
=
2325 QSIMPLEQ_FIRST(&rs
->src_page_requests
);
2327 *offset
= entry
->offset
;
2329 if (entry
->len
> TARGET_PAGE_SIZE
) {
2330 entry
->len
-= TARGET_PAGE_SIZE
;
2331 entry
->offset
+= TARGET_PAGE_SIZE
;
2333 memory_region_unref(block
->mr
);
2334 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
2336 migration_consume_urgent_request();
2339 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
2345 * get_queued_page: unqueue a page from the postcopy requests
2347 * Skips pages that are already sent (!dirty)
2349 * Returns true if a queued page is found
2351 * @rs: current RAM state
2352 * @pss: data about the state of the current dirty page scan
2354 static bool get_queued_page(RAMState
*rs
, PageSearchStatus
*pss
)
2361 block
= unqueue_page(rs
, &offset
);
2363 * We're sending this page, and since it's postcopy nothing else
2364 * will dirty it, and we must make sure it doesn't get sent again
2365 * even if this queue request was received after the background
2366 * search already sent it.
2371 page
= offset
>> TARGET_PAGE_BITS
;
2372 dirty
= test_bit(page
, block
->bmap
);
2374 trace_get_queued_page_not_dirty(block
->idstr
, (uint64_t)offset
,
2377 trace_get_queued_page(block
->idstr
, (uint64_t)offset
, page
);
2381 } while (block
&& !dirty
);
2385 * As soon as we start servicing pages out of order, then we have
2386 * to kill the bulk stage, since the bulk stage assumes
2387 * in (migration_bitmap_find_and_reset_dirty) that every page is
2388 * dirty, that's no longer true.
2390 rs
->ram_bulk_stage
= false;
2393 * We want the background search to continue from the queued page
2394 * since the guest is likely to want other pages near to the page
2395 * it just requested.
2398 pss
->page
= offset
>> TARGET_PAGE_BITS
;
2401 * This unqueued page would break the "one round" check, even is
2404 pss
->complete_round
= false;
2411 * migration_page_queue_free: drop any remaining pages in the ram
2414 * It should be empty at the end anyway, but in error cases there may
2415 * be some left. in case that there is any page left, we drop it.
2418 static void migration_page_queue_free(RAMState
*rs
)
2420 struct RAMSrcPageRequest
*mspr
, *next_mspr
;
2421 /* This queue generally should be empty - but in the case of a failed
2422 * migration might have some droppings in.
2424 RCU_READ_LOCK_GUARD();
2425 QSIMPLEQ_FOREACH_SAFE(mspr
, &rs
->src_page_requests
, next_req
, next_mspr
) {
2426 memory_region_unref(mspr
->rb
->mr
);
2427 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
2433 * ram_save_queue_pages: queue the page for transmission
2435 * A request from postcopy destination for example.
2437 * Returns zero on success or negative on error
2439 * @rbname: Name of the RAMBLock of the request. NULL means the
2440 * same that last one.
2441 * @start: starting address from the start of the RAMBlock
2442 * @len: length (in bytes) to send
2444 int ram_save_queue_pages(const char *rbname
, ram_addr_t start
, ram_addr_t len
)
2447 RAMState
*rs
= ram_state
;
2449 ram_counters
.postcopy_requests
++;
2450 RCU_READ_LOCK_GUARD();
2453 /* Reuse last RAMBlock */
2454 ramblock
= rs
->last_req_rb
;
2458 * Shouldn't happen, we can't reuse the last RAMBlock if
2459 * it's the 1st request.
2461 error_report("ram_save_queue_pages no previous block");
2465 ramblock
= qemu_ram_block_by_name(rbname
);
2468 /* We shouldn't be asked for a non-existent RAMBlock */
2469 error_report("ram_save_queue_pages no block '%s'", rbname
);
2472 rs
->last_req_rb
= ramblock
;
2474 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
2475 if (start
+len
> ramblock
->used_length
) {
2476 error_report("%s request overrun start=" RAM_ADDR_FMT
" len="
2477 RAM_ADDR_FMT
" blocklen=" RAM_ADDR_FMT
,
2478 __func__
, start
, len
, ramblock
->used_length
);
2482 struct RAMSrcPageRequest
*new_entry
=
2483 g_malloc0(sizeof(struct RAMSrcPageRequest
));
2484 new_entry
->rb
= ramblock
;
2485 new_entry
->offset
= start
;
2486 new_entry
->len
= len
;
2488 memory_region_ref(ramblock
->mr
);
2489 qemu_mutex_lock(&rs
->src_page_req_mutex
);
2490 QSIMPLEQ_INSERT_TAIL(&rs
->src_page_requests
, new_entry
, next_req
);
2491 migration_make_urgent_request();
2492 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
2497 static bool save_page_use_compression(RAMState
*rs
)
2499 if (!migrate_use_compression()) {
2504 * If xbzrle is on, stop using the data compression after first
2505 * round of migration even if compression is enabled. In theory,
2506 * xbzrle can do better than compression.
2508 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
2516 * try to compress the page before posting it out, return true if the page
2517 * has been properly handled by compression, otherwise needs other
2518 * paths to handle it
2520 static bool save_compress_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
2522 if (!save_page_use_compression(rs
)) {
2527 * When starting the process of a new block, the first page of
2528 * the block should be sent out before other pages in the same
2529 * block, and all the pages in last block should have been sent
2530 * out, keeping this order is important, because the 'cont' flag
2531 * is used to avoid resending the block name.
2533 * We post the fist page as normal page as compression will take
2534 * much CPU resource.
2536 if (block
!= rs
->last_sent_block
) {
2537 flush_compressed_data(rs
);
2541 if (compress_page_with_multi_thread(rs
, block
, offset
) > 0) {
2545 compression_counters
.busy
++;
2550 * ram_save_target_page: save one target page
2552 * Returns the number of pages written
2554 * @rs: current RAM state
2555 * @pss: data about the page we want to send
2556 * @last_stage: if we are at the completion stage
2558 static int ram_save_target_page(RAMState
*rs
, PageSearchStatus
*pss
,
2561 RAMBlock
*block
= pss
->block
;
2562 ram_addr_t offset
= pss
->page
<< TARGET_PAGE_BITS
;
2565 if (control_save_page(rs
, block
, offset
, &res
)) {
2569 if (save_compress_page(rs
, block
, offset
)) {
2573 res
= save_zero_page(rs
, block
, offset
);
2575 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
2576 * page would be stale
2578 if (!save_page_use_compression(rs
)) {
2579 XBZRLE_cache_lock();
2580 xbzrle_cache_zero_page(rs
, block
->offset
+ offset
);
2581 XBZRLE_cache_unlock();
2583 ram_release_pages(block
->idstr
, offset
, res
);
2588 * do not use multifd for compression as the first page in the new
2589 * block should be posted out before sending the compressed page
2591 if (!save_page_use_compression(rs
) && migrate_use_multifd()) {
2592 return ram_save_multifd_page(rs
, block
, offset
);
2595 return ram_save_page(rs
, pss
, last_stage
);
2599 * ram_save_host_page: save a whole host page
2601 * Starting at *offset send pages up to the end of the current host
2602 * page. It's valid for the initial offset to point into the middle of
2603 * a host page in which case the remainder of the hostpage is sent.
2604 * Only dirty target pages are sent. Note that the host page size may
2605 * be a huge page for this block.
2606 * The saving stops at the boundary of the used_length of the block
2607 * if the RAMBlock isn't a multiple of the host page size.
2609 * Returns the number of pages written or negative on error
2611 * @rs: current RAM state
2612 * @ms: current migration state
2613 * @pss: data about the page we want to send
2614 * @last_stage: if we are at the completion stage
2616 static int ram_save_host_page(RAMState
*rs
, PageSearchStatus
*pss
,
2619 int tmppages
, pages
= 0;
2620 size_t pagesize_bits
=
2621 qemu_ram_pagesize(pss
->block
) >> TARGET_PAGE_BITS
;
2623 if (ramblock_is_ignored(pss
->block
)) {
2624 error_report("block %s should not be migrated !", pss
->block
->idstr
);
2629 /* Check the pages is dirty and if it is send it */
2630 if (!migration_bitmap_clear_dirty(rs
, pss
->block
, pss
->page
)) {
2635 tmppages
= ram_save_target_page(rs
, pss
, last_stage
);
2642 /* Allow rate limiting to happen in the middle of huge pages */
2643 migration_rate_limit();
2644 } while ((pss
->page
& (pagesize_bits
- 1)) &&
2645 offset_in_ramblock(pss
->block
, pss
->page
<< TARGET_PAGE_BITS
));
2647 /* The offset we leave with is the last one we looked at */
2653 * ram_find_and_save_block: finds a dirty page and sends it to f
2655 * Called within an RCU critical section.
2657 * Returns the number of pages written where zero means no dirty pages,
2658 * or negative on error
2660 * @rs: current RAM state
2661 * @last_stage: if we are at the completion stage
2663 * On systems where host-page-size > target-page-size it will send all the
2664 * pages in a host page that are dirty.
2667 static int ram_find_and_save_block(RAMState
*rs
, bool last_stage
)
2669 PageSearchStatus pss
;
2673 /* No dirty page as there is zero RAM */
2674 if (!ram_bytes_total()) {
2678 pss
.block
= rs
->last_seen_block
;
2679 pss
.page
= rs
->last_page
;
2680 pss
.complete_round
= false;
2683 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2688 found
= get_queued_page(rs
, &pss
);
2691 /* priority queue empty, so just search for something dirty */
2692 found
= find_dirty_block(rs
, &pss
, &again
);
2696 pages
= ram_save_host_page(rs
, &pss
, last_stage
);
2698 } while (!pages
&& again
);
2700 rs
->last_seen_block
= pss
.block
;
2701 rs
->last_page
= pss
.page
;
2706 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
2708 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
2711 ram_counters
.duplicate
+= pages
;
2713 ram_counters
.normal
+= pages
;
2714 ram_counters
.transferred
+= size
;
2715 qemu_update_position(f
, size
);
2719 static uint64_t ram_bytes_total_common(bool count_ignored
)
2724 RCU_READ_LOCK_GUARD();
2726 if (count_ignored
) {
2727 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
2728 total
+= block
->used_length
;
2731 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2732 total
+= block
->used_length
;
2738 uint64_t ram_bytes_total(void)
2740 return ram_bytes_total_common(false);
2743 static void xbzrle_load_setup(void)
2745 XBZRLE
.decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
2748 static void xbzrle_load_cleanup(void)
2750 g_free(XBZRLE
.decoded_buf
);
2751 XBZRLE
.decoded_buf
= NULL
;
2754 static void ram_state_cleanup(RAMState
**rsp
)
2757 migration_page_queue_free(*rsp
);
2758 qemu_mutex_destroy(&(*rsp
)->bitmap_mutex
);
2759 qemu_mutex_destroy(&(*rsp
)->src_page_req_mutex
);
2765 static void xbzrle_cleanup(void)
2767 XBZRLE_cache_lock();
2769 cache_fini(XBZRLE
.cache
);
2770 g_free(XBZRLE
.encoded_buf
);
2771 g_free(XBZRLE
.current_buf
);
2772 g_free(XBZRLE
.zero_target_page
);
2773 XBZRLE
.cache
= NULL
;
2774 XBZRLE
.encoded_buf
= NULL
;
2775 XBZRLE
.current_buf
= NULL
;
2776 XBZRLE
.zero_target_page
= NULL
;
2778 XBZRLE_cache_unlock();
2781 static void ram_save_cleanup(void *opaque
)
2783 RAMState
**rsp
= opaque
;
2786 /* caller have hold iothread lock or is in a bh, so there is
2787 * no writing race against the migration bitmap
2789 memory_global_dirty_log_stop();
2791 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2792 g_free(block
->clear_bmap
);
2793 block
->clear_bmap
= NULL
;
2794 g_free(block
->bmap
);
2799 compress_threads_save_cleanup();
2800 ram_state_cleanup(rsp
);
2803 static void ram_state_reset(RAMState
*rs
)
2805 rs
->last_seen_block
= NULL
;
2806 rs
->last_sent_block
= NULL
;
2808 rs
->last_version
= ram_list
.version
;
2809 rs
->ram_bulk_stage
= true;
2810 rs
->fpo_enabled
= false;
2813 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2816 * 'expected' is the value you expect the bitmap mostly to be full
2817 * of; it won't bother printing lines that are all this value.
2818 * If 'todump' is null the migration bitmap is dumped.
2820 void ram_debug_dump_bitmap(unsigned long *todump
, bool expected
,
2821 unsigned long pages
)
2824 int64_t linelen
= 128;
2827 for (cur
= 0; cur
< pages
; cur
+= linelen
) {
2831 * Last line; catch the case where the line length
2832 * is longer than remaining ram
2834 if (cur
+ linelen
> pages
) {
2835 linelen
= pages
- cur
;
2837 for (curb
= 0; curb
< linelen
; curb
++) {
2838 bool thisbit
= test_bit(cur
+ curb
, todump
);
2839 linebuf
[curb
] = thisbit
? '1' : '.';
2840 found
= found
|| (thisbit
!= expected
);
2843 linebuf
[curb
] = '\0';
2844 fprintf(stderr
, "0x%08" PRIx64
" : %s\n", cur
, linebuf
);
2849 /* **** functions for postcopy ***** */
2851 void ram_postcopy_migrated_memory_release(MigrationState
*ms
)
2853 struct RAMBlock
*block
;
2855 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2856 unsigned long *bitmap
= block
->bmap
;
2857 unsigned long range
= block
->used_length
>> TARGET_PAGE_BITS
;
2858 unsigned long run_start
= find_next_zero_bit(bitmap
, range
, 0);
2860 while (run_start
< range
) {
2861 unsigned long run_end
= find_next_bit(bitmap
, range
, run_start
+ 1);
2862 ram_discard_range(block
->idstr
, run_start
<< TARGET_PAGE_BITS
,
2863 (run_end
- run_start
) << TARGET_PAGE_BITS
);
2864 run_start
= find_next_zero_bit(bitmap
, range
, run_end
+ 1);
2870 * postcopy_send_discard_bm_ram: discard a RAMBlock
2872 * Returns zero on success
2874 * Callback from postcopy_each_ram_send_discard for each RAMBlock
2876 * @ms: current migration state
2877 * @block: RAMBlock to discard
2879 static int postcopy_send_discard_bm_ram(MigrationState
*ms
, RAMBlock
*block
)
2881 unsigned long end
= block
->used_length
>> TARGET_PAGE_BITS
;
2882 unsigned long current
;
2883 unsigned long *bitmap
= block
->bmap
;
2885 for (current
= 0; current
< end
; ) {
2886 unsigned long one
= find_next_bit(bitmap
, end
, current
);
2887 unsigned long zero
, discard_length
;
2893 zero
= find_next_zero_bit(bitmap
, end
, one
+ 1);
2896 discard_length
= end
- one
;
2898 discard_length
= zero
- one
;
2900 postcopy_discard_send_range(ms
, one
, discard_length
);
2901 current
= one
+ discard_length
;
2908 * postcopy_each_ram_send_discard: discard all RAMBlocks
2910 * Returns 0 for success or negative for error
2912 * Utility for the outgoing postcopy code.
2913 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2914 * passing it bitmap indexes and name.
2915 * (qemu_ram_foreach_block ends up passing unscaled lengths
2916 * which would mean postcopy code would have to deal with target page)
2918 * @ms: current migration state
2920 static int postcopy_each_ram_send_discard(MigrationState
*ms
)
2922 struct RAMBlock
*block
;
2925 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2926 postcopy_discard_send_init(ms
, block
->idstr
);
2929 * Postcopy sends chunks of bitmap over the wire, but it
2930 * just needs indexes at this point, avoids it having
2931 * target page specific code.
2933 ret
= postcopy_send_discard_bm_ram(ms
, block
);
2934 postcopy_discard_send_finish(ms
);
2944 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2946 * Helper for postcopy_chunk_hostpages; it's called twice to
2947 * canonicalize the two bitmaps, that are similar, but one is
2950 * Postcopy requires that all target pages in a hostpage are dirty or
2951 * clean, not a mix. This function canonicalizes the bitmaps.
2953 * @ms: current migration state
2954 * @block: block that contains the page we want to canonicalize
2956 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, RAMBlock
*block
)
2958 RAMState
*rs
= ram_state
;
2959 unsigned long *bitmap
= block
->bmap
;
2960 unsigned int host_ratio
= block
->page_size
/ TARGET_PAGE_SIZE
;
2961 unsigned long pages
= block
->used_length
>> TARGET_PAGE_BITS
;
2962 unsigned long run_start
;
2964 if (block
->page_size
== TARGET_PAGE_SIZE
) {
2965 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2969 /* Find a dirty page */
2970 run_start
= find_next_bit(bitmap
, pages
, 0);
2972 while (run_start
< pages
) {
2975 * If the start of this run of pages is in the middle of a host
2976 * page, then we need to fixup this host page.
2978 if (QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2979 /* Find the end of this run */
2980 run_start
= find_next_zero_bit(bitmap
, pages
, run_start
+ 1);
2982 * If the end isn't at the start of a host page, then the
2983 * run doesn't finish at the end of a host page
2984 * and we need to discard.
2988 if (!QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2990 unsigned long fixup_start_addr
= QEMU_ALIGN_DOWN(run_start
,
2992 run_start
= QEMU_ALIGN_UP(run_start
, host_ratio
);
2994 /* Clean up the bitmap */
2995 for (page
= fixup_start_addr
;
2996 page
< fixup_start_addr
+ host_ratio
; page
++) {
2998 * Remark them as dirty, updating the count for any pages
2999 * that weren't previously dirty.
3001 rs
->migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
3005 /* Find the next dirty page for the next iteration */
3006 run_start
= find_next_bit(bitmap
, pages
, run_start
);
3011 * postcopy_chunk_hostpages: discard any partially sent host page
3013 * Utility for the outgoing postcopy code.
3015 * Discard any partially sent host-page size chunks, mark any partially
3016 * dirty host-page size chunks as all dirty. In this case the host-page
3017 * is the host-page for the particular RAMBlock, i.e. it might be a huge page
3019 * Returns zero on success
3021 * @ms: current migration state
3022 * @block: block we want to work with
3024 static int postcopy_chunk_hostpages(MigrationState
*ms
, RAMBlock
*block
)
3026 postcopy_discard_send_init(ms
, block
->idstr
);
3029 * Ensure that all partially dirty host pages are made fully dirty.
3031 postcopy_chunk_hostpages_pass(ms
, block
);
3033 postcopy_discard_send_finish(ms
);
3038 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
3040 * Returns zero on success
3042 * Transmit the set of pages to be discarded after precopy to the target
3043 * these are pages that:
3044 * a) Have been previously transmitted but are now dirty again
3045 * b) Pages that have never been transmitted, this ensures that
3046 * any pages on the destination that have been mapped by background
3047 * tasks get discarded (transparent huge pages is the specific concern)
3048 * Hopefully this is pretty sparse
3050 * @ms: current migration state
3052 int ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
3054 RAMState
*rs
= ram_state
;
3058 RCU_READ_LOCK_GUARD();
3060 /* This should be our last sync, the src is now paused */
3061 migration_bitmap_sync(rs
);
3063 /* Easiest way to make sure we don't resume in the middle of a host-page */
3064 rs
->last_seen_block
= NULL
;
3065 rs
->last_sent_block
= NULL
;
3068 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3069 /* Deal with TPS != HPS and huge pages */
3070 ret
= postcopy_chunk_hostpages(ms
, block
);
3075 #ifdef DEBUG_POSTCOPY
3076 ram_debug_dump_bitmap(block
->bmap
, true,
3077 block
->used_length
>> TARGET_PAGE_BITS
);
3080 trace_ram_postcopy_send_discard_bitmap();
3082 ret
= postcopy_each_ram_send_discard(ms
);
3088 * ram_discard_range: discard dirtied pages at the beginning of postcopy
3090 * Returns zero on success
3092 * @rbname: name of the RAMBlock of the request. NULL means the
3093 * same that last one.
3094 * @start: RAMBlock starting page
3095 * @length: RAMBlock size
3097 int ram_discard_range(const char *rbname
, uint64_t start
, size_t length
)
3099 trace_ram_discard_range(rbname
, start
, length
);
3101 RCU_READ_LOCK_GUARD();
3102 RAMBlock
*rb
= qemu_ram_block_by_name(rbname
);
3105 error_report("ram_discard_range: Failed to find block '%s'", rbname
);
3110 * On source VM, we don't need to update the received bitmap since
3111 * we don't even have one.
3113 if (rb
->receivedmap
) {
3114 bitmap_clear(rb
->receivedmap
, start
>> qemu_target_page_bits(),
3115 length
>> qemu_target_page_bits());
3118 return ram_block_discard_range(rb
, start
, length
);
3122 * For every allocation, we will try not to crash the VM if the
3123 * allocation failed.
3125 static int xbzrle_init(void)
3127 Error
*local_err
= NULL
;
3129 if (!migrate_use_xbzrle()) {
3133 XBZRLE_cache_lock();
3135 XBZRLE
.zero_target_page
= g_try_malloc0(TARGET_PAGE_SIZE
);
3136 if (!XBZRLE
.zero_target_page
) {
3137 error_report("%s: Error allocating zero page", __func__
);
3141 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size(),
3142 TARGET_PAGE_SIZE
, &local_err
);
3143 if (!XBZRLE
.cache
) {
3144 error_report_err(local_err
);
3145 goto free_zero_page
;
3148 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
3149 if (!XBZRLE
.encoded_buf
) {
3150 error_report("%s: Error allocating encoded_buf", __func__
);
3154 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
3155 if (!XBZRLE
.current_buf
) {
3156 error_report("%s: Error allocating current_buf", __func__
);
3157 goto free_encoded_buf
;
3160 /* We are all good */
3161 XBZRLE_cache_unlock();
3165 g_free(XBZRLE
.encoded_buf
);
3166 XBZRLE
.encoded_buf
= NULL
;
3168 cache_fini(XBZRLE
.cache
);
3169 XBZRLE
.cache
= NULL
;
3171 g_free(XBZRLE
.zero_target_page
);
3172 XBZRLE
.zero_target_page
= NULL
;
3174 XBZRLE_cache_unlock();
3178 static int ram_state_init(RAMState
**rsp
)
3180 *rsp
= g_try_new0(RAMState
, 1);
3183 error_report("%s: Init ramstate fail", __func__
);
3187 qemu_mutex_init(&(*rsp
)->bitmap_mutex
);
3188 qemu_mutex_init(&(*rsp
)->src_page_req_mutex
);
3189 QSIMPLEQ_INIT(&(*rsp
)->src_page_requests
);
3192 * Count the total number of pages used by ram blocks not including any
3193 * gaps due to alignment or unplugs.
3194 * This must match with the initial values of dirty bitmap.
3196 (*rsp
)->migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
3197 ram_state_reset(*rsp
);
3202 static void ram_list_init_bitmaps(void)
3204 MigrationState
*ms
= migrate_get_current();
3206 unsigned long pages
;
3209 /* Skip setting bitmap if there is no RAM */
3210 if (ram_bytes_total()) {
3211 shift
= ms
->clear_bitmap_shift
;
3212 if (shift
> CLEAR_BITMAP_SHIFT_MAX
) {
3213 error_report("clear_bitmap_shift (%u) too big, using "
3214 "max value (%u)", shift
, CLEAR_BITMAP_SHIFT_MAX
);
3215 shift
= CLEAR_BITMAP_SHIFT_MAX
;
3216 } else if (shift
< CLEAR_BITMAP_SHIFT_MIN
) {
3217 error_report("clear_bitmap_shift (%u) too small, using "
3218 "min value (%u)", shift
, CLEAR_BITMAP_SHIFT_MIN
);
3219 shift
= CLEAR_BITMAP_SHIFT_MIN
;
3222 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3223 pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3225 * The initial dirty bitmap for migration must be set with all
3226 * ones to make sure we'll migrate every guest RAM page to
3228 * Here we set RAMBlock.bmap all to 1 because when rebegin a
3229 * new migration after a failed migration, ram_list.
3230 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
3233 block
->bmap
= bitmap_new(pages
);
3234 bitmap_set(block
->bmap
, 0, pages
);
3235 block
->clear_bmap_shift
= shift
;
3236 block
->clear_bmap
= bitmap_new(clear_bmap_size(pages
, shift
));
3241 static void ram_init_bitmaps(RAMState
*rs
)
3243 /* For memory_global_dirty_log_start below. */
3244 qemu_mutex_lock_iothread();
3245 qemu_mutex_lock_ramlist();
3247 WITH_RCU_READ_LOCK_GUARD() {
3248 ram_list_init_bitmaps();
3249 memory_global_dirty_log_start();
3250 migration_bitmap_sync_precopy(rs
);
3252 qemu_mutex_unlock_ramlist();
3253 qemu_mutex_unlock_iothread();
3256 static int ram_init_all(RAMState
**rsp
)
3258 if (ram_state_init(rsp
)) {
3262 if (xbzrle_init()) {
3263 ram_state_cleanup(rsp
);
3267 ram_init_bitmaps(*rsp
);
3272 static void ram_state_resume_prepare(RAMState
*rs
, QEMUFile
*out
)
3278 * Postcopy is not using xbzrle/compression, so no need for that.
3279 * Also, since source are already halted, we don't need to care
3280 * about dirty page logging as well.
3283 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3284 pages
+= bitmap_count_one(block
->bmap
,
3285 block
->used_length
>> TARGET_PAGE_BITS
);
3288 /* This may not be aligned with current bitmaps. Recalculate. */
3289 rs
->migration_dirty_pages
= pages
;
3291 rs
->last_seen_block
= NULL
;
3292 rs
->last_sent_block
= NULL
;
3294 rs
->last_version
= ram_list
.version
;
3296 * Disable the bulk stage, otherwise we'll resend the whole RAM no
3297 * matter what we have sent.
3299 rs
->ram_bulk_stage
= false;
3301 /* Update RAMState cache of output QEMUFile */
3304 trace_ram_state_resume_prepare(pages
);
3308 * This function clears bits of the free pages reported by the caller from the
3309 * migration dirty bitmap. @addr is the host address corresponding to the
3310 * start of the continuous guest free pages, and @len is the total bytes of
3313 void qemu_guest_free_page_hint(void *addr
, size_t len
)
3317 size_t used_len
, start
, npages
;
3318 MigrationState
*s
= migrate_get_current();
3320 /* This function is currently expected to be used during live migration */
3321 if (!migration_is_setup_or_active(s
->state
)) {
3325 for (; len
> 0; len
-= used_len
, addr
+= used_len
) {
3326 block
= qemu_ram_block_from_host(addr
, false, &offset
);
3327 if (unlikely(!block
|| offset
>= block
->used_length
)) {
3329 * The implementation might not support RAMBlock resize during
3330 * live migration, but it could happen in theory with future
3331 * updates. So we add a check here to capture that case.
3333 error_report_once("%s unexpected error", __func__
);
3337 if (len
<= block
->used_length
- offset
) {
3340 used_len
= block
->used_length
- offset
;
3343 start
= offset
>> TARGET_PAGE_BITS
;
3344 npages
= used_len
>> TARGET_PAGE_BITS
;
3346 qemu_mutex_lock(&ram_state
->bitmap_mutex
);
3347 ram_state
->migration_dirty_pages
-=
3348 bitmap_count_one_with_offset(block
->bmap
, start
, npages
);
3349 bitmap_clear(block
->bmap
, start
, npages
);
3350 qemu_mutex_unlock(&ram_state
->bitmap_mutex
);
3355 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
3356 * long-running RCU critical section. When rcu-reclaims in the code
3357 * start to become numerous it will be necessary to reduce the
3358 * granularity of these critical sections.
3362 * ram_save_setup: Setup RAM for migration
3364 * Returns zero to indicate success and negative for error
3366 * @f: QEMUFile where to send the data
3367 * @opaque: RAMState pointer
3369 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
3371 RAMState
**rsp
= opaque
;
3374 if (compress_threads_save_setup()) {
3378 /* migration has already setup the bitmap, reuse it. */
3379 if (!migration_in_colo_state()) {
3380 if (ram_init_all(rsp
) != 0) {
3381 compress_threads_save_cleanup();
3387 WITH_RCU_READ_LOCK_GUARD() {
3388 qemu_put_be64(f
, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE
);
3390 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
3391 qemu_put_byte(f
, strlen(block
->idstr
));
3392 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
3393 qemu_put_be64(f
, block
->used_length
);
3394 if (migrate_postcopy_ram() && block
->page_size
!=
3395 qemu_host_page_size
) {
3396 qemu_put_be64(f
, block
->page_size
);
3398 if (migrate_ignore_shared()) {
3399 qemu_put_be64(f
, block
->mr
->addr
);
3404 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
3405 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
3407 multifd_send_sync_main(*rsp
);
3408 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3415 * ram_save_iterate: iterative stage for migration
3417 * Returns zero to indicate success and negative for error
3419 * @f: QEMUFile where to send the data
3420 * @opaque: RAMState pointer
3422 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
3424 RAMState
**temp
= opaque
;
3425 RAMState
*rs
= *temp
;
3431 if (blk_mig_bulk_active()) {
3432 /* Avoid transferring ram during bulk phase of block migration as
3433 * the bulk phase will usually take a long time and transferring
3434 * ram updates during that time is pointless. */
3438 WITH_RCU_READ_LOCK_GUARD() {
3439 if (ram_list
.version
!= rs
->last_version
) {
3440 ram_state_reset(rs
);
3443 /* Read version before ram_list.blocks */
3446 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
3448 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
3450 while ((ret
= qemu_file_rate_limit(f
)) == 0 ||
3451 !QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
3454 if (qemu_file_get_error(f
)) {
3458 pages
= ram_find_and_save_block(rs
, false);
3459 /* no more pages to sent */
3466 qemu_file_set_error(f
, pages
);
3470 rs
->target_page_count
+= pages
;
3473 * we want to check in the 1st loop, just in case it was the 1st
3474 * time and we had to sync the dirty bitmap.
3475 * qemu_clock_get_ns() is a bit expensive, so we only check each
3478 if ((i
& 63) == 0) {
3479 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) /
3481 if (t1
> MAX_WAIT
) {
3482 trace_ram_save_iterate_big_wait(t1
, i
);
3491 * Must occur before EOS (or any QEMUFile operation)
3492 * because of RDMA protocol.
3494 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
3497 multifd_send_sync_main(rs
);
3498 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3500 ram_counters
.transferred
+= 8;
3502 ret
= qemu_file_get_error(f
);
3511 * ram_save_complete: function called to send the remaining amount of ram
3513 * Returns zero to indicate success or negative on error
3515 * Called with iothread lock
3517 * @f: QEMUFile where to send the data
3518 * @opaque: RAMState pointer
3520 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
3522 RAMState
**temp
= opaque
;
3523 RAMState
*rs
= *temp
;
3526 WITH_RCU_READ_LOCK_GUARD() {
3527 if (!migration_in_postcopy()) {
3528 migration_bitmap_sync_precopy(rs
);
3531 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
3533 /* try transferring iterative blocks of memory */
3535 /* flush all remaining blocks regardless of rate limiting */
3539 pages
= ram_find_and_save_block(rs
, !migration_in_colo_state());
3540 /* no more blocks to sent */
3550 flush_compressed_data(rs
);
3551 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
3554 multifd_send_sync_main(rs
);
3555 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3561 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
3562 uint64_t *res_precopy_only
,
3563 uint64_t *res_compatible
,
3564 uint64_t *res_postcopy_only
)
3566 RAMState
**temp
= opaque
;
3567 RAMState
*rs
= *temp
;
3568 uint64_t remaining_size
;
3570 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3572 if (!migration_in_postcopy() &&
3573 remaining_size
< max_size
) {
3574 qemu_mutex_lock_iothread();
3575 WITH_RCU_READ_LOCK_GUARD() {
3576 migration_bitmap_sync_precopy(rs
);
3578 qemu_mutex_unlock_iothread();
3579 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3582 if (migrate_postcopy_ram()) {
3583 /* We can do postcopy, and all the data is postcopiable */
3584 *res_compatible
+= remaining_size
;
3586 *res_precopy_only
+= remaining_size
;
3590 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
3592 unsigned int xh_len
;
3594 uint8_t *loaded_data
;
3596 /* extract RLE header */
3597 xh_flags
= qemu_get_byte(f
);
3598 xh_len
= qemu_get_be16(f
);
3600 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
3601 error_report("Failed to load XBZRLE page - wrong compression!");
3605 if (xh_len
> TARGET_PAGE_SIZE
) {
3606 error_report("Failed to load XBZRLE page - len overflow!");
3609 loaded_data
= XBZRLE
.decoded_buf
;
3610 /* load data and decode */
3611 /* it can change loaded_data to point to an internal buffer */
3612 qemu_get_buffer_in_place(f
, &loaded_data
, xh_len
);
3615 if (xbzrle_decode_buffer(loaded_data
, xh_len
, host
,
3616 TARGET_PAGE_SIZE
) == -1) {
3617 error_report("Failed to load XBZRLE page - decode error!");
3625 * ram_block_from_stream: read a RAMBlock id from the migration stream
3627 * Must be called from within a rcu critical section.
3629 * Returns a pointer from within the RCU-protected ram_list.
3631 * @f: QEMUFile where to read the data from
3632 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3634 static inline RAMBlock
*ram_block_from_stream(QEMUFile
*f
, int flags
)
3636 static RAMBlock
*block
= NULL
;
3640 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
3642 error_report("Ack, bad migration stream!");
3648 len
= qemu_get_byte(f
);
3649 qemu_get_buffer(f
, (uint8_t *)id
, len
);
3652 block
= qemu_ram_block_by_name(id
);
3654 error_report("Can't find block %s", id
);
3658 if (ramblock_is_ignored(block
)) {
3659 error_report("block %s should not be migrated !", id
);
3666 static inline void *host_from_ram_block_offset(RAMBlock
*block
,
3669 if (!offset_in_ramblock(block
, offset
)) {
3673 return block
->host
+ offset
;
3676 static inline void *colo_cache_from_block_offset(RAMBlock
*block
,
3679 if (!offset_in_ramblock(block
, offset
)) {
3682 if (!block
->colo_cache
) {
3683 error_report("%s: colo_cache is NULL in block :%s",
3684 __func__
, block
->idstr
);
3689 * During colo checkpoint, we need bitmap of these migrated pages.
3690 * It help us to decide which pages in ram cache should be flushed
3691 * into VM's RAM later.
3693 if (!test_and_set_bit(offset
>> TARGET_PAGE_BITS
, block
->bmap
)) {
3694 ram_state
->migration_dirty_pages
++;
3696 return block
->colo_cache
+ offset
;
3700 * ram_handle_compressed: handle the zero page case
3702 * If a page (or a whole RDMA chunk) has been
3703 * determined to be zero, then zap it.
3705 * @host: host address for the zero page
3706 * @ch: what the page is filled from. We only support zero
3707 * @size: size of the zero page
3709 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
3711 if (ch
!= 0 || !is_zero_range(host
, size
)) {
3712 memset(host
, ch
, size
);
3716 /* return the size after decompression, or negative value on error */
3718 qemu_uncompress_data(z_stream
*stream
, uint8_t *dest
, size_t dest_len
,
3719 const uint8_t *source
, size_t source_len
)
3723 err
= inflateReset(stream
);
3728 stream
->avail_in
= source_len
;
3729 stream
->next_in
= (uint8_t *)source
;
3730 stream
->avail_out
= dest_len
;
3731 stream
->next_out
= dest
;
3733 err
= inflate(stream
, Z_NO_FLUSH
);
3734 if (err
!= Z_STREAM_END
) {
3738 return stream
->total_out
;
3741 static void *do_data_decompress(void *opaque
)
3743 DecompressParam
*param
= opaque
;
3744 unsigned long pagesize
;
3748 qemu_mutex_lock(¶m
->mutex
);
3749 while (!param
->quit
) {
3754 qemu_mutex_unlock(¶m
->mutex
);
3756 pagesize
= TARGET_PAGE_SIZE
;
3758 ret
= qemu_uncompress_data(¶m
->stream
, des
, pagesize
,
3759 param
->compbuf
, len
);
3760 if (ret
< 0 && migrate_get_current()->decompress_error_check
) {
3761 error_report("decompress data failed");
3762 qemu_file_set_error(decomp_file
, ret
);
3765 qemu_mutex_lock(&decomp_done_lock
);
3767 qemu_cond_signal(&decomp_done_cond
);
3768 qemu_mutex_unlock(&decomp_done_lock
);
3770 qemu_mutex_lock(¶m
->mutex
);
3772 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
3775 qemu_mutex_unlock(¶m
->mutex
);
3780 static int wait_for_decompress_done(void)
3782 int idx
, thread_count
;
3784 if (!migrate_use_compression()) {
3788 thread_count
= migrate_decompress_threads();
3789 qemu_mutex_lock(&decomp_done_lock
);
3790 for (idx
= 0; idx
< thread_count
; idx
++) {
3791 while (!decomp_param
[idx
].done
) {
3792 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3795 qemu_mutex_unlock(&decomp_done_lock
);
3796 return qemu_file_get_error(decomp_file
);
3799 static void compress_threads_load_cleanup(void)
3801 int i
, thread_count
;
3803 if (!migrate_use_compression()) {
3806 thread_count
= migrate_decompress_threads();
3807 for (i
= 0; i
< thread_count
; i
++) {
3809 * we use it as a indicator which shows if the thread is
3810 * properly init'd or not
3812 if (!decomp_param
[i
].compbuf
) {
3816 qemu_mutex_lock(&decomp_param
[i
].mutex
);
3817 decomp_param
[i
].quit
= true;
3818 qemu_cond_signal(&decomp_param
[i
].cond
);
3819 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
3821 for (i
= 0; i
< thread_count
; i
++) {
3822 if (!decomp_param
[i
].compbuf
) {
3826 qemu_thread_join(decompress_threads
+ i
);
3827 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
3828 qemu_cond_destroy(&decomp_param
[i
].cond
);
3829 inflateEnd(&decomp_param
[i
].stream
);
3830 g_free(decomp_param
[i
].compbuf
);
3831 decomp_param
[i
].compbuf
= NULL
;
3833 g_free(decompress_threads
);
3834 g_free(decomp_param
);
3835 decompress_threads
= NULL
;
3836 decomp_param
= NULL
;
3840 static int compress_threads_load_setup(QEMUFile
*f
)
3842 int i
, thread_count
;
3844 if (!migrate_use_compression()) {
3848 thread_count
= migrate_decompress_threads();
3849 decompress_threads
= g_new0(QemuThread
, thread_count
);
3850 decomp_param
= g_new0(DecompressParam
, thread_count
);
3851 qemu_mutex_init(&decomp_done_lock
);
3852 qemu_cond_init(&decomp_done_cond
);
3854 for (i
= 0; i
< thread_count
; i
++) {
3855 if (inflateInit(&decomp_param
[i
].stream
) != Z_OK
) {
3859 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
3860 qemu_mutex_init(&decomp_param
[i
].mutex
);
3861 qemu_cond_init(&decomp_param
[i
].cond
);
3862 decomp_param
[i
].done
= true;
3863 decomp_param
[i
].quit
= false;
3864 qemu_thread_create(decompress_threads
+ i
, "decompress",
3865 do_data_decompress
, decomp_param
+ i
,
3866 QEMU_THREAD_JOINABLE
);
3870 compress_threads_load_cleanup();
3874 static void decompress_data_with_multi_threads(QEMUFile
*f
,
3875 void *host
, int len
)
3877 int idx
, thread_count
;
3879 thread_count
= migrate_decompress_threads();
3880 qemu_mutex_lock(&decomp_done_lock
);
3882 for (idx
= 0; idx
< thread_count
; idx
++) {
3883 if (decomp_param
[idx
].done
) {
3884 decomp_param
[idx
].done
= false;
3885 qemu_mutex_lock(&decomp_param
[idx
].mutex
);
3886 qemu_get_buffer(f
, decomp_param
[idx
].compbuf
, len
);
3887 decomp_param
[idx
].des
= host
;
3888 decomp_param
[idx
].len
= len
;
3889 qemu_cond_signal(&decomp_param
[idx
].cond
);
3890 qemu_mutex_unlock(&decomp_param
[idx
].mutex
);
3894 if (idx
< thread_count
) {
3897 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3900 qemu_mutex_unlock(&decomp_done_lock
);
3904 * colo cache: this is for secondary VM, we cache the whole
3905 * memory of the secondary VM, it is need to hold the global lock
3906 * to call this helper.
3908 int colo_init_ram_cache(void)
3912 WITH_RCU_READ_LOCK_GUARD() {
3913 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3914 block
->colo_cache
= qemu_anon_ram_alloc(block
->used_length
,
3917 if (!block
->colo_cache
) {
3918 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3919 "size 0x" RAM_ADDR_FMT
, __func__
, block
->idstr
,
3920 block
->used_length
);
3921 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3922 if (block
->colo_cache
) {
3923 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3924 block
->colo_cache
= NULL
;
3929 memcpy(block
->colo_cache
, block
->host
, block
->used_length
);
3934 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3935 * with to decide which page in cache should be flushed into SVM's RAM. Here
3936 * we use the same name 'ram_bitmap' as for migration.
3938 if (ram_bytes_total()) {
3941 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3942 unsigned long pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3944 block
->bmap
= bitmap_new(pages
);
3945 bitmap_set(block
->bmap
, 0, pages
);
3948 ram_state
= g_new0(RAMState
, 1);
3949 ram_state
->migration_dirty_pages
= 0;
3950 qemu_mutex_init(&ram_state
->bitmap_mutex
);
3951 memory_global_dirty_log_start();
3956 /* It is need to hold the global lock to call this helper */
3957 void colo_release_ram_cache(void)
3961 memory_global_dirty_log_stop();
3962 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3963 g_free(block
->bmap
);
3967 WITH_RCU_READ_LOCK_GUARD() {
3968 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3969 if (block
->colo_cache
) {
3970 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3971 block
->colo_cache
= NULL
;
3975 qemu_mutex_destroy(&ram_state
->bitmap_mutex
);
3981 * ram_load_setup: Setup RAM for migration incoming side
3983 * Returns zero to indicate success and negative for error
3985 * @f: QEMUFile where to receive the data
3986 * @opaque: RAMState pointer
3988 static int ram_load_setup(QEMUFile
*f
, void *opaque
)
3990 if (compress_threads_load_setup(f
)) {
3994 xbzrle_load_setup();
3995 ramblock_recv_map_init();
4000 static int ram_load_cleanup(void *opaque
)
4004 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4005 qemu_ram_block_writeback(rb
);
4008 xbzrle_load_cleanup();
4009 compress_threads_load_cleanup();
4011 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4012 g_free(rb
->receivedmap
);
4013 rb
->receivedmap
= NULL
;
4020 * ram_postcopy_incoming_init: allocate postcopy data structures
4022 * Returns 0 for success and negative if there was one error
4024 * @mis: current migration incoming state
4026 * Allocate data structures etc needed by incoming migration with
4027 * postcopy-ram. postcopy-ram's similarly names
4028 * postcopy_ram_incoming_init does the work.
4030 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
4032 return postcopy_ram_incoming_init(mis
);
4036 * ram_load_postcopy: load a page in postcopy case
4038 * Returns 0 for success or -errno in case of error
4040 * Called in postcopy mode by ram_load().
4041 * rcu_read_lock is taken prior to this being called.
4043 * @f: QEMUFile where to send the data
4045 static int ram_load_postcopy(QEMUFile
*f
)
4047 int flags
= 0, ret
= 0;
4048 bool place_needed
= false;
4049 bool matches_target_page_size
= false;
4050 MigrationIncomingState
*mis
= migration_incoming_get_current();
4051 /* Temporary page that is later 'placed' */
4052 void *postcopy_host_page
= mis
->postcopy_tmp_page
;
4053 void *last_host
= NULL
;
4054 bool all_zero
= false;
4056 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
4059 void *page_buffer
= NULL
;
4060 void *place_source
= NULL
;
4061 RAMBlock
*block
= NULL
;
4064 addr
= qemu_get_be64(f
);
4067 * If qemu file error, we should stop here, and then "addr"
4070 ret
= qemu_file_get_error(f
);
4075 flags
= addr
& ~TARGET_PAGE_MASK
;
4076 addr
&= TARGET_PAGE_MASK
;
4078 trace_ram_load_postcopy_loop((uint64_t)addr
, flags
);
4079 place_needed
= false;
4080 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
)) {
4081 block
= ram_block_from_stream(f
, flags
);
4083 host
= host_from_ram_block_offset(block
, addr
);
4085 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
4089 matches_target_page_size
= block
->page_size
== TARGET_PAGE_SIZE
;
4091 * Postcopy requires that we place whole host pages atomically;
4092 * these may be huge pages for RAMBlocks that are backed by
4094 * To make it atomic, the data is read into a temporary page
4095 * that's moved into place later.
4096 * The migration protocol uses, possibly smaller, target-pages
4097 * however the source ensures it always sends all the components
4098 * of a host page in order.
4100 page_buffer
= postcopy_host_page
+
4101 ((uintptr_t)host
& (block
->page_size
- 1));
4102 /* If all TP are zero then we can optimise the place */
4103 if (!((uintptr_t)host
& (block
->page_size
- 1))) {
4106 /* not the 1st TP within the HP */
4107 if (host
!= (last_host
+ TARGET_PAGE_SIZE
)) {
4108 error_report("Non-sequential target page %p/%p",
4117 * If it's the last part of a host page then we place the host
4120 place_needed
= (((uintptr_t)host
+ TARGET_PAGE_SIZE
) &
4121 (block
->page_size
- 1)) == 0;
4122 place_source
= postcopy_host_page
;
4126 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4127 case RAM_SAVE_FLAG_ZERO
:
4128 ch
= qemu_get_byte(f
);
4130 * Can skip to set page_buffer when
4131 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
4133 if (ch
|| !matches_target_page_size
) {
4134 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
4141 case RAM_SAVE_FLAG_PAGE
:
4143 if (!matches_target_page_size
) {
4144 /* For huge pages, we always use temporary buffer */
4145 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
4148 * For small pages that matches target page size, we
4149 * avoid the qemu_file copy. Instead we directly use
4150 * the buffer of QEMUFile to place the page. Note: we
4151 * cannot do any QEMUFile operation before using that
4152 * buffer to make sure the buffer is valid when
4155 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
4159 case RAM_SAVE_FLAG_EOS
:
4161 multifd_recv_sync_main();
4164 error_report("Unknown combination of migration flags: %#x"
4165 " (postcopy mode)", flags
);
4170 /* Detect for any possible file errors */
4171 if (!ret
&& qemu_file_get_error(f
)) {
4172 ret
= qemu_file_get_error(f
);
4175 if (!ret
&& place_needed
) {
4176 /* This gets called at the last target page in the host page */
4177 void *place_dest
= host
+ TARGET_PAGE_SIZE
- block
->page_size
;
4180 ret
= postcopy_place_page_zero(mis
, place_dest
,
4183 ret
= postcopy_place_page(mis
, place_dest
,
4184 place_source
, block
);
4192 static bool postcopy_is_advised(void)
4194 PostcopyState ps
= postcopy_state_get();
4195 return ps
>= POSTCOPY_INCOMING_ADVISE
&& ps
< POSTCOPY_INCOMING_END
;
4198 static bool postcopy_is_running(void)
4200 PostcopyState ps
= postcopy_state_get();
4201 return ps
>= POSTCOPY_INCOMING_LISTENING
&& ps
< POSTCOPY_INCOMING_END
;
4205 * Flush content of RAM cache into SVM's memory.
4206 * Only flush the pages that be dirtied by PVM or SVM or both.
4208 static void colo_flush_ram_cache(void)
4210 RAMBlock
*block
= NULL
;
4213 unsigned long offset
= 0;
4215 memory_global_dirty_log_sync();
4216 WITH_RCU_READ_LOCK_GUARD() {
4217 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4218 ramblock_sync_dirty_bitmap(ram_state
, block
);
4222 trace_colo_flush_ram_cache_begin(ram_state
->migration_dirty_pages
);
4223 WITH_RCU_READ_LOCK_GUARD() {
4224 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
4227 offset
= migration_bitmap_find_dirty(ram_state
, block
, offset
);
4229 if (offset
<< TARGET_PAGE_BITS
>= block
->used_length
) {
4231 block
= QLIST_NEXT_RCU(block
, next
);
4233 migration_bitmap_clear_dirty(ram_state
, block
, offset
);
4234 dst_host
= block
->host
+ (offset
<< TARGET_PAGE_BITS
);
4235 src_host
= block
->colo_cache
+ (offset
<< TARGET_PAGE_BITS
);
4236 memcpy(dst_host
, src_host
, TARGET_PAGE_SIZE
);
4240 trace_colo_flush_ram_cache_end();
4244 * ram_load_precopy: load pages in precopy case
4246 * Returns 0 for success or -errno in case of error
4248 * Called in precopy mode by ram_load().
4249 * rcu_read_lock is taken prior to this being called.
4251 * @f: QEMUFile where to send the data
4253 static int ram_load_precopy(QEMUFile
*f
)
4255 int flags
= 0, ret
= 0, invalid_flags
= 0, len
= 0, i
= 0;
4256 /* ADVISE is earlier, it shows the source has the postcopy capability on */
4257 bool postcopy_advised
= postcopy_is_advised();
4258 if (!migrate_use_compression()) {
4259 invalid_flags
|= RAM_SAVE_FLAG_COMPRESS_PAGE
;
4262 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
4263 ram_addr_t addr
, total_ram_bytes
;
4268 * Yield periodically to let main loop run, but an iteration of
4269 * the main loop is expensive, so do it each some iterations
4271 if ((i
& 32767) == 0 && qemu_in_coroutine()) {
4272 aio_co_schedule(qemu_get_current_aio_context(),
4273 qemu_coroutine_self());
4274 qemu_coroutine_yield();
4278 addr
= qemu_get_be64(f
);
4279 flags
= addr
& ~TARGET_PAGE_MASK
;
4280 addr
&= TARGET_PAGE_MASK
;
4282 if (flags
& invalid_flags
) {
4283 if (flags
& invalid_flags
& RAM_SAVE_FLAG_COMPRESS_PAGE
) {
4284 error_report("Received an unexpected compressed page");
4291 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
4292 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
4293 RAMBlock
*block
= ram_block_from_stream(f
, flags
);
4296 * After going into COLO, we should load the Page into colo_cache.
4298 if (migration_incoming_in_colo_state()) {
4299 host
= colo_cache_from_block_offset(block
, addr
);
4301 host
= host_from_ram_block_offset(block
, addr
);
4304 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
4309 if (!migration_incoming_in_colo_state()) {
4310 ramblock_recv_bitmap_set(block
, host
);
4313 trace_ram_load_loop(block
->idstr
, (uint64_t)addr
, flags
, host
);
4316 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4317 case RAM_SAVE_FLAG_MEM_SIZE
:
4318 /* Synchronize RAM block list */
4319 total_ram_bytes
= addr
;
4320 while (!ret
&& total_ram_bytes
) {
4325 len
= qemu_get_byte(f
);
4326 qemu_get_buffer(f
, (uint8_t *)id
, len
);
4328 length
= qemu_get_be64(f
);
4330 block
= qemu_ram_block_by_name(id
);
4331 if (block
&& !qemu_ram_is_migratable(block
)) {
4332 error_report("block %s should not be migrated !", id
);
4335 if (length
!= block
->used_length
) {
4336 Error
*local_err
= NULL
;
4338 ret
= qemu_ram_resize(block
, length
,
4341 error_report_err(local_err
);
4344 /* For postcopy we need to check hugepage sizes match */
4345 if (postcopy_advised
&&
4346 block
->page_size
!= qemu_host_page_size
) {
4347 uint64_t remote_page_size
= qemu_get_be64(f
);
4348 if (remote_page_size
!= block
->page_size
) {
4349 error_report("Mismatched RAM page size %s "
4350 "(local) %zd != %" PRId64
,
4351 id
, block
->page_size
,
4356 if (migrate_ignore_shared()) {
4357 hwaddr addr
= qemu_get_be64(f
);
4358 if (ramblock_is_ignored(block
) &&
4359 block
->mr
->addr
!= addr
) {
4360 error_report("Mismatched GPAs for block %s "
4361 "%" PRId64
"!= %" PRId64
,
4363 (uint64_t)block
->mr
->addr
);
4367 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
4370 error_report("Unknown ramblock \"%s\", cannot "
4371 "accept migration", id
);
4375 total_ram_bytes
-= length
;
4379 case RAM_SAVE_FLAG_ZERO
:
4380 ch
= qemu_get_byte(f
);
4381 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
4384 case RAM_SAVE_FLAG_PAGE
:
4385 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
4388 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
4389 len
= qemu_get_be32(f
);
4390 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
4391 error_report("Invalid compressed data length: %d", len
);
4395 decompress_data_with_multi_threads(f
, host
, len
);
4398 case RAM_SAVE_FLAG_XBZRLE
:
4399 if (load_xbzrle(f
, addr
, host
) < 0) {
4400 error_report("Failed to decompress XBZRLE page at "
4401 RAM_ADDR_FMT
, addr
);
4406 case RAM_SAVE_FLAG_EOS
:
4408 multifd_recv_sync_main();
4411 if (flags
& RAM_SAVE_FLAG_HOOK
) {
4412 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
4414 error_report("Unknown combination of migration flags: %#x",
4420 ret
= qemu_file_get_error(f
);
4424 ret
|= wait_for_decompress_done();
4428 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
4431 static uint64_t seq_iter
;
4433 * If system is running in postcopy mode, page inserts to host memory must
4436 bool postcopy_running
= postcopy_is_running();
4440 if (version_id
!= 4) {
4445 * This RCU critical section can be very long running.
4446 * When RCU reclaims in the code start to become numerous,
4447 * it will be necessary to reduce the granularity of this
4450 WITH_RCU_READ_LOCK_GUARD() {
4451 if (postcopy_running
) {
4452 ret
= ram_load_postcopy(f
);
4454 ret
= ram_load_precopy(f
);
4457 trace_ram_load_complete(ret
, seq_iter
);
4459 if (!ret
&& migration_incoming_in_colo_state()) {
4460 colo_flush_ram_cache();
4465 static bool ram_has_postcopy(void *opaque
)
4468 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4469 if (ramblock_is_pmem(rb
)) {
4470 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4471 "is not supported now!", rb
->idstr
, rb
->host
);
4476 return migrate_postcopy_ram();
4479 /* Sync all the dirty bitmap with destination VM. */
4480 static int ram_dirty_bitmap_sync_all(MigrationState
*s
, RAMState
*rs
)
4483 QEMUFile
*file
= s
->to_dst_file
;
4484 int ramblock_count
= 0;
4486 trace_ram_dirty_bitmap_sync_start();
4488 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4489 qemu_savevm_send_recv_bitmap(file
, block
->idstr
);
4490 trace_ram_dirty_bitmap_request(block
->idstr
);
4494 trace_ram_dirty_bitmap_sync_wait();
4496 /* Wait until all the ramblocks' dirty bitmap synced */
4497 while (ramblock_count
--) {
4498 qemu_sem_wait(&s
->rp_state
.rp_sem
);
4501 trace_ram_dirty_bitmap_sync_complete();
4506 static void ram_dirty_bitmap_reload_notify(MigrationState
*s
)
4508 qemu_sem_post(&s
->rp_state
.rp_sem
);
4512 * Read the received bitmap, revert it as the initial dirty bitmap.
4513 * This is only used when the postcopy migration is paused but wants
4514 * to resume from a middle point.
4516 int ram_dirty_bitmap_reload(MigrationState
*s
, RAMBlock
*block
)
4519 QEMUFile
*file
= s
->rp_state
.from_dst_file
;
4520 unsigned long *le_bitmap
, nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
4521 uint64_t local_size
= DIV_ROUND_UP(nbits
, 8);
4522 uint64_t size
, end_mark
;
4524 trace_ram_dirty_bitmap_reload_begin(block
->idstr
);
4526 if (s
->state
!= MIGRATION_STATUS_POSTCOPY_RECOVER
) {
4527 error_report("%s: incorrect state %s", __func__
,
4528 MigrationStatus_str(s
->state
));
4533 * Note: see comments in ramblock_recv_bitmap_send() on why we
4534 * need the endianess convertion, and the paddings.
4536 local_size
= ROUND_UP(local_size
, 8);
4539 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
4541 size
= qemu_get_be64(file
);
4543 /* The size of the bitmap should match with our ramblock */
4544 if (size
!= local_size
) {
4545 error_report("%s: ramblock '%s' bitmap size mismatch "
4546 "(0x%"PRIx64
" != 0x%"PRIx64
")", __func__
,
4547 block
->idstr
, size
, local_size
);
4552 size
= qemu_get_buffer(file
, (uint8_t *)le_bitmap
, local_size
);
4553 end_mark
= qemu_get_be64(file
);
4555 ret
= qemu_file_get_error(file
);
4556 if (ret
|| size
!= local_size
) {
4557 error_report("%s: read bitmap failed for ramblock '%s': %d"
4558 " (size 0x%"PRIx64
", got: 0x%"PRIx64
")",
4559 __func__
, block
->idstr
, ret
, local_size
, size
);
4564 if (end_mark
!= RAMBLOCK_RECV_BITMAP_ENDING
) {
4565 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIu64
,
4566 __func__
, block
->idstr
, end_mark
);
4572 * Endianess convertion. We are during postcopy (though paused).
4573 * The dirty bitmap won't change. We can directly modify it.
4575 bitmap_from_le(block
->bmap
, le_bitmap
, nbits
);
4578 * What we received is "received bitmap". Revert it as the initial
4579 * dirty bitmap for this ramblock.
4581 bitmap_complement(block
->bmap
, block
->bmap
, nbits
);
4583 trace_ram_dirty_bitmap_reload_complete(block
->idstr
);
4586 * We succeeded to sync bitmap for current ramblock. If this is
4587 * the last one to sync, we need to notify the main send thread.
4589 ram_dirty_bitmap_reload_notify(s
);
4597 static int ram_resume_prepare(MigrationState
*s
, void *opaque
)
4599 RAMState
*rs
= *(RAMState
**)opaque
;
4602 ret
= ram_dirty_bitmap_sync_all(s
, rs
);
4607 ram_state_resume_prepare(rs
, s
->to_dst_file
);
4612 static SaveVMHandlers savevm_ram_handlers
= {
4613 .save_setup
= ram_save_setup
,
4614 .save_live_iterate
= ram_save_iterate
,
4615 .save_live_complete_postcopy
= ram_save_complete
,
4616 .save_live_complete_precopy
= ram_save_complete
,
4617 .has_postcopy
= ram_has_postcopy
,
4618 .save_live_pending
= ram_save_pending
,
4619 .load_state
= ram_load
,
4620 .save_cleanup
= ram_save_cleanup
,
4621 .load_setup
= ram_load_setup
,
4622 .load_cleanup
= ram_load_cleanup
,
4623 .resume_prepare
= ram_resume_prepare
,
4626 void ram_mig_init(void)
4628 qemu_mutex_init(&XBZRLE
.lock
);
4629 register_savevm_live("ram", 0, 4, &savevm_ram_handlers
, &ram_state
);