4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
28 #include "qemu/osdep.h"
29 #include "qemu-common.h"
32 #include "qapi-event.h"
33 #include "qemu/cutils.h"
34 #include "qemu/bitops.h"
35 #include "qemu/bitmap.h"
36 #include "qemu/timer.h"
37 #include "qemu/main-loop.h"
38 #include "migration/migration.h"
39 #include "migration/postcopy-ram.h"
40 #include "exec/address-spaces.h"
41 #include "migration/page_cache.h"
42 #include "qemu/error-report.h"
44 #include "exec/ram_addr.h"
45 #include "qemu/rcu_queue.h"
46 #include "migration/colo.h"
48 static int dirty_rate_high_cnt
;
50 static uint64_t bitmap_sync_count
;
52 /***********************************************************/
53 /* ram save/restore */
55 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
56 #define RAM_SAVE_FLAG_COMPRESS 0x02
57 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
58 #define RAM_SAVE_FLAG_PAGE 0x08
59 #define RAM_SAVE_FLAG_EOS 0x10
60 #define RAM_SAVE_FLAG_CONTINUE 0x20
61 #define RAM_SAVE_FLAG_XBZRLE 0x40
62 /* 0x80 is reserved in migration.h start with 0x100 next */
63 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
65 static uint8_t *ZERO_TARGET_PAGE
;
67 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
69 return buffer_is_zero(p
, size
);
72 /* struct contains XBZRLE cache and a static page
73 used by the compression */
75 /* buffer used for XBZRLE encoding */
77 /* buffer for storing page content */
79 /* Cache for XBZRLE, Protected by lock. */
84 /* buffer used for XBZRLE decoding */
85 static uint8_t *xbzrle_decoded_buf
;
87 static void XBZRLE_cache_lock(void)
89 if (migrate_use_xbzrle())
90 qemu_mutex_lock(&XBZRLE
.lock
);
93 static void XBZRLE_cache_unlock(void)
95 if (migrate_use_xbzrle())
96 qemu_mutex_unlock(&XBZRLE
.lock
);
100 * called from qmp_migrate_set_cache_size in main thread, possibly while
101 * a migration is in progress.
102 * A running migration maybe using the cache and might finish during this
103 * call, hence changes to the cache are protected by XBZRLE.lock().
105 int64_t xbzrle_cache_resize(int64_t new_size
)
107 PageCache
*new_cache
;
110 if (new_size
< TARGET_PAGE_SIZE
) {
116 if (XBZRLE
.cache
!= NULL
) {
117 if (pow2floor(new_size
) == migrate_xbzrle_cache_size()) {
120 new_cache
= cache_init(new_size
/ TARGET_PAGE_SIZE
,
123 error_report("Error creating cache");
128 cache_fini(XBZRLE
.cache
);
129 XBZRLE
.cache
= new_cache
;
133 ret
= pow2floor(new_size
);
135 XBZRLE_cache_unlock();
139 /* accounting for migration statistics */
140 typedef struct AccountingInfo
{
142 uint64_t skipped_pages
;
145 uint64_t xbzrle_bytes
;
146 uint64_t xbzrle_pages
;
147 uint64_t xbzrle_cache_miss
;
148 double xbzrle_cache_miss_rate
;
149 uint64_t xbzrle_overflows
;
152 static AccountingInfo acct_info
;
154 static void acct_clear(void)
156 memset(&acct_info
, 0, sizeof(acct_info
));
159 uint64_t dup_mig_bytes_transferred(void)
161 return acct_info
.dup_pages
* TARGET_PAGE_SIZE
;
164 uint64_t dup_mig_pages_transferred(void)
166 return acct_info
.dup_pages
;
169 uint64_t skipped_mig_bytes_transferred(void)
171 return acct_info
.skipped_pages
* TARGET_PAGE_SIZE
;
174 uint64_t skipped_mig_pages_transferred(void)
176 return acct_info
.skipped_pages
;
179 uint64_t norm_mig_bytes_transferred(void)
181 return acct_info
.norm_pages
* TARGET_PAGE_SIZE
;
184 uint64_t norm_mig_pages_transferred(void)
186 return acct_info
.norm_pages
;
189 uint64_t xbzrle_mig_bytes_transferred(void)
191 return acct_info
.xbzrle_bytes
;
194 uint64_t xbzrle_mig_pages_transferred(void)
196 return acct_info
.xbzrle_pages
;
199 uint64_t xbzrle_mig_pages_cache_miss(void)
201 return acct_info
.xbzrle_cache_miss
;
204 double xbzrle_mig_cache_miss_rate(void)
206 return acct_info
.xbzrle_cache_miss_rate
;
209 uint64_t xbzrle_mig_pages_overflow(void)
211 return acct_info
.xbzrle_overflows
;
214 /* This is the last block that we have visited serching for dirty pages
216 static RAMBlock
*last_seen_block
;
217 /* This is the last block from where we have sent data */
218 static RAMBlock
*last_sent_block
;
219 static ram_addr_t last_offset
;
220 static QemuMutex migration_bitmap_mutex
;
221 static uint64_t migration_dirty_pages
;
222 static uint32_t last_version
;
223 static bool ram_bulk_stage
;
225 /* used by the search for pages to send */
226 struct PageSearchStatus
{
227 /* Current block being searched */
229 /* Current offset to search from */
231 /* Set once we wrap around */
234 typedef struct PageSearchStatus PageSearchStatus
;
236 static struct BitmapRcu
{
238 /* Main migration bitmap */
240 /* bitmap of pages that haven't been sent even once
241 * only maintained and used in postcopy at the moment
242 * where it's used to send the dirtymap at the start
243 * of the postcopy phase
245 unsigned long *unsentmap
;
246 } *migration_bitmap_rcu
;
248 struct CompressParam
{
257 typedef struct CompressParam CompressParam
;
259 struct DecompressParam
{
268 typedef struct DecompressParam DecompressParam
;
270 static CompressParam
*comp_param
;
271 static QemuThread
*compress_threads
;
272 /* comp_done_cond is used to wake up the migration thread when
273 * one of the compression threads has finished the compression.
274 * comp_done_lock is used to co-work with comp_done_cond.
276 static QemuMutex comp_done_lock
;
277 static QemuCond comp_done_cond
;
278 /* The empty QEMUFileOps will be used by file in CompressParam */
279 static const QEMUFileOps empty_ops
= { };
281 static bool compression_switch
;
282 static DecompressParam
*decomp_param
;
283 static QemuThread
*decompress_threads
;
284 static QemuMutex decomp_done_lock
;
285 static QemuCond decomp_done_cond
;
287 static int do_compress_ram_page(QEMUFile
*f
, RAMBlock
*block
,
290 static void *do_data_compress(void *opaque
)
292 CompressParam
*param
= opaque
;
296 qemu_mutex_lock(¶m
->mutex
);
297 while (!param
->quit
) {
299 block
= param
->block
;
300 offset
= param
->offset
;
302 qemu_mutex_unlock(¶m
->mutex
);
304 do_compress_ram_page(param
->file
, block
, offset
);
306 qemu_mutex_lock(&comp_done_lock
);
308 qemu_cond_signal(&comp_done_cond
);
309 qemu_mutex_unlock(&comp_done_lock
);
311 qemu_mutex_lock(¶m
->mutex
);
313 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
316 qemu_mutex_unlock(¶m
->mutex
);
321 static inline void terminate_compression_threads(void)
323 int idx
, thread_count
;
325 thread_count
= migrate_compress_threads();
326 for (idx
= 0; idx
< thread_count
; idx
++) {
327 qemu_mutex_lock(&comp_param
[idx
].mutex
);
328 comp_param
[idx
].quit
= true;
329 qemu_cond_signal(&comp_param
[idx
].cond
);
330 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
334 void migrate_compress_threads_join(void)
338 if (!migrate_use_compression()) {
341 terminate_compression_threads();
342 thread_count
= migrate_compress_threads();
343 for (i
= 0; i
< thread_count
; i
++) {
344 qemu_thread_join(compress_threads
+ i
);
345 qemu_fclose(comp_param
[i
].file
);
346 qemu_mutex_destroy(&comp_param
[i
].mutex
);
347 qemu_cond_destroy(&comp_param
[i
].cond
);
349 qemu_mutex_destroy(&comp_done_lock
);
350 qemu_cond_destroy(&comp_done_cond
);
351 g_free(compress_threads
);
353 compress_threads
= NULL
;
357 void migrate_compress_threads_create(void)
361 if (!migrate_use_compression()) {
364 compression_switch
= true;
365 thread_count
= migrate_compress_threads();
366 compress_threads
= g_new0(QemuThread
, thread_count
);
367 comp_param
= g_new0(CompressParam
, thread_count
);
368 qemu_cond_init(&comp_done_cond
);
369 qemu_mutex_init(&comp_done_lock
);
370 for (i
= 0; i
< thread_count
; i
++) {
371 /* comp_param[i].file is just used as a dummy buffer to save data,
372 * set its ops to empty.
374 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
375 comp_param
[i
].done
= true;
376 comp_param
[i
].quit
= false;
377 qemu_mutex_init(&comp_param
[i
].mutex
);
378 qemu_cond_init(&comp_param
[i
].cond
);
379 qemu_thread_create(compress_threads
+ i
, "compress",
380 do_data_compress
, comp_param
+ i
,
381 QEMU_THREAD_JOINABLE
);
386 * save_page_header: Write page header to wire
388 * If this is the 1st block, it also writes the block identification
390 * Returns: Number of bytes written
392 * @f: QEMUFile where to send the data
393 * @block: block that contains the page we want to send
394 * @offset: offset inside the block for the page
395 * in the lower bits, it contains flags
397 static size_t save_page_header(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
)
401 qemu_put_be64(f
, offset
);
404 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
405 len
= strlen(block
->idstr
);
406 qemu_put_byte(f
, len
);
407 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
413 /* Reduce amount of guest cpu execution to hopefully slow down memory writes.
414 * If guest dirty memory rate is reduced below the rate at which we can
415 * transfer pages to the destination then we should be able to complete
416 * migration. Some workloads dirty memory way too fast and will not effectively
417 * converge, even with auto-converge.
419 static void mig_throttle_guest_down(void)
421 MigrationState
*s
= migrate_get_current();
422 uint64_t pct_initial
= s
->parameters
.cpu_throttle_initial
;
423 uint64_t pct_icrement
= s
->parameters
.cpu_throttle_increment
;
425 /* We have not started throttling yet. Let's start it. */
426 if (!cpu_throttle_active()) {
427 cpu_throttle_set(pct_initial
);
429 /* Throttling already on, just increase the rate */
430 cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement
);
434 /* Update the xbzrle cache to reflect a page that's been sent as all 0.
435 * The important thing is that a stale (not-yet-0'd) page be replaced
437 * As a bonus, if the page wasn't in the cache it gets added so that
438 * when a small write is made into the 0'd page it gets XBZRLE sent
440 static void xbzrle_cache_zero_page(ram_addr_t current_addr
)
442 if (ram_bulk_stage
|| !migrate_use_xbzrle()) {
446 /* We don't care if this fails to allocate a new cache page
447 * as long as it updated an old one */
448 cache_insert(XBZRLE
.cache
, current_addr
, ZERO_TARGET_PAGE
,
452 #define ENCODING_FLAG_XBZRLE 0x1
455 * save_xbzrle_page: compress and send current page
457 * Returns: 1 means that we wrote the page
458 * 0 means that page is identical to the one already sent
459 * -1 means that xbzrle would be longer than normal
461 * @f: QEMUFile where to send the data
464 * @block: block that contains the page we want to send
465 * @offset: offset inside the block for the page
466 * @last_stage: if we are at the completion stage
467 * @bytes_transferred: increase it with the number of transferred bytes
469 static int save_xbzrle_page(QEMUFile
*f
, uint8_t **current_data
,
470 ram_addr_t current_addr
, RAMBlock
*block
,
471 ram_addr_t offset
, bool last_stage
,
472 uint64_t *bytes_transferred
)
474 int encoded_len
= 0, bytes_xbzrle
;
475 uint8_t *prev_cached_page
;
477 if (!cache_is_cached(XBZRLE
.cache
, current_addr
, bitmap_sync_count
)) {
478 acct_info
.xbzrle_cache_miss
++;
480 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
481 bitmap_sync_count
) == -1) {
484 /* update *current_data when the page has been
485 inserted into cache */
486 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
492 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
494 /* save current buffer into memory */
495 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
497 /* XBZRLE encoding (if there is no overflow) */
498 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
499 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
501 if (encoded_len
== 0) {
502 trace_save_xbzrle_page_skipping();
504 } else if (encoded_len
== -1) {
505 trace_save_xbzrle_page_overflow();
506 acct_info
.xbzrle_overflows
++;
507 /* update data in the cache */
509 memcpy(prev_cached_page
, *current_data
, TARGET_PAGE_SIZE
);
510 *current_data
= prev_cached_page
;
515 /* we need to update the data in the cache, in order to get the same data */
517 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
520 /* Send XBZRLE based compressed page */
521 bytes_xbzrle
= save_page_header(f
, block
, offset
| RAM_SAVE_FLAG_XBZRLE
);
522 qemu_put_byte(f
, ENCODING_FLAG_XBZRLE
);
523 qemu_put_be16(f
, encoded_len
);
524 qemu_put_buffer(f
, XBZRLE
.encoded_buf
, encoded_len
);
525 bytes_xbzrle
+= encoded_len
+ 1 + 2;
526 acct_info
.xbzrle_pages
++;
527 acct_info
.xbzrle_bytes
+= bytes_xbzrle
;
528 *bytes_transferred
+= bytes_xbzrle
;
533 /* Called with rcu_read_lock() to protect migration_bitmap
534 * rb: The RAMBlock to search for dirty pages in
535 * start: Start address (typically so we can continue from previous page)
536 * ram_addr_abs: Pointer into which to store the address of the dirty page
537 * within the global ram_addr space
539 * Returns: byte offset within memory region of the start of a dirty page
542 ram_addr_t
migration_bitmap_find_dirty(RAMBlock
*rb
,
544 ram_addr_t
*ram_addr_abs
)
546 unsigned long base
= rb
->offset
>> TARGET_PAGE_BITS
;
547 unsigned long nr
= base
+ (start
>> TARGET_PAGE_BITS
);
548 uint64_t rb_size
= rb
->used_length
;
549 unsigned long size
= base
+ (rb_size
>> TARGET_PAGE_BITS
);
550 unsigned long *bitmap
;
554 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
555 if (ram_bulk_stage
&& nr
> base
) {
558 next
= find_next_bit(bitmap
, size
, nr
);
561 *ram_addr_abs
= next
<< TARGET_PAGE_BITS
;
562 return (next
- base
) << TARGET_PAGE_BITS
;
565 static inline bool migration_bitmap_clear_dirty(ram_addr_t addr
)
568 int nr
= addr
>> TARGET_PAGE_BITS
;
569 unsigned long *bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
571 ret
= test_and_clear_bit(nr
, bitmap
);
574 migration_dirty_pages
--;
579 static void migration_bitmap_sync_range(ram_addr_t start
, ram_addr_t length
)
581 unsigned long *bitmap
;
582 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
583 migration_dirty_pages
+=
584 cpu_physical_memory_sync_dirty_bitmap(bitmap
, start
, length
);
587 /* Fix me: there are too many global variables used in migration process. */
588 static int64_t start_time
;
589 static int64_t bytes_xfer_prev
;
590 static int64_t num_dirty_pages_period
;
591 static uint64_t xbzrle_cache_miss_prev
;
592 static uint64_t iterations_prev
;
594 static void migration_bitmap_sync_init(void)
598 num_dirty_pages_period
= 0;
599 xbzrle_cache_miss_prev
= 0;
603 static void migration_bitmap_sync(void)
606 uint64_t num_dirty_pages_init
= migration_dirty_pages
;
607 MigrationState
*s
= migrate_get_current();
609 int64_t bytes_xfer_now
;
613 if (!bytes_xfer_prev
) {
614 bytes_xfer_prev
= ram_bytes_transferred();
618 start_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
621 trace_migration_bitmap_sync_start();
622 memory_global_dirty_log_sync();
624 qemu_mutex_lock(&migration_bitmap_mutex
);
626 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
627 migration_bitmap_sync_range(block
->offset
, block
->used_length
);
630 qemu_mutex_unlock(&migration_bitmap_mutex
);
632 trace_migration_bitmap_sync_end(migration_dirty_pages
633 - num_dirty_pages_init
);
634 num_dirty_pages_period
+= migration_dirty_pages
- num_dirty_pages_init
;
635 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
637 /* more than 1 second = 1000 millisecons */
638 if (end_time
> start_time
+ 1000) {
639 if (migrate_auto_converge()) {
640 /* The following detection logic can be refined later. For now:
641 Check to see if the dirtied bytes is 50% more than the approx.
642 amount of bytes that just got transferred since the last time we
643 were in this routine. If that happens twice, start or increase
645 bytes_xfer_now
= ram_bytes_transferred();
647 if (s
->dirty_pages_rate
&&
648 (num_dirty_pages_period
* TARGET_PAGE_SIZE
>
649 (bytes_xfer_now
- bytes_xfer_prev
)/2) &&
650 (dirty_rate_high_cnt
++ >= 2)) {
651 trace_migration_throttle();
652 dirty_rate_high_cnt
= 0;
653 mig_throttle_guest_down();
655 bytes_xfer_prev
= bytes_xfer_now
;
658 if (migrate_use_xbzrle()) {
659 if (iterations_prev
!= acct_info
.iterations
) {
660 acct_info
.xbzrle_cache_miss_rate
=
661 (double)(acct_info
.xbzrle_cache_miss
-
662 xbzrle_cache_miss_prev
) /
663 (acct_info
.iterations
- iterations_prev
);
665 iterations_prev
= acct_info
.iterations
;
666 xbzrle_cache_miss_prev
= acct_info
.xbzrle_cache_miss
;
668 s
->dirty_pages_rate
= num_dirty_pages_period
* 1000
669 / (end_time
- start_time
);
670 s
->dirty_bytes_rate
= s
->dirty_pages_rate
* TARGET_PAGE_SIZE
;
671 start_time
= end_time
;
672 num_dirty_pages_period
= 0;
674 s
->dirty_sync_count
= bitmap_sync_count
;
675 if (migrate_use_events()) {
676 qapi_event_send_migration_pass(bitmap_sync_count
, NULL
);
681 * save_zero_page: Send the zero page to the stream
683 * Returns: Number of pages written.
685 * @f: QEMUFile where to send the data
686 * @block: block that contains the page we want to send
687 * @offset: offset inside the block for the page
688 * @p: pointer to the page
689 * @bytes_transferred: increase it with the number of transferred bytes
691 static int save_zero_page(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
,
692 uint8_t *p
, uint64_t *bytes_transferred
)
696 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
697 acct_info
.dup_pages
++;
698 *bytes_transferred
+= save_page_header(f
, block
,
699 offset
| RAM_SAVE_FLAG_COMPRESS
);
701 *bytes_transferred
+= 1;
709 * ram_save_page: Send the given page to the stream
711 * Returns: Number of pages written.
713 * >=0 - Number of pages written - this might legally be 0
714 * if xbzrle noticed the page was the same.
716 * @ms: The current migration state.
717 * @f: QEMUFile where to send the data
718 * @block: block that contains the page we want to send
719 * @offset: offset inside the block for the page
720 * @last_stage: if we are at the completion stage
721 * @bytes_transferred: increase it with the number of transferred bytes
723 static int ram_save_page(MigrationState
*ms
, QEMUFile
*f
, PageSearchStatus
*pss
,
724 bool last_stage
, uint64_t *bytes_transferred
)
728 ram_addr_t current_addr
;
731 bool send_async
= true;
732 RAMBlock
*block
= pss
->block
;
733 ram_addr_t offset
= pss
->offset
;
735 p
= block
->host
+ offset
;
737 /* In doubt sent page as normal */
739 ret
= ram_control_save_page(f
, block
->offset
,
740 offset
, TARGET_PAGE_SIZE
, &bytes_xmit
);
742 *bytes_transferred
+= bytes_xmit
;
748 current_addr
= block
->offset
+ offset
;
750 if (block
== last_sent_block
) {
751 offset
|= RAM_SAVE_FLAG_CONTINUE
;
753 if (ret
!= RAM_SAVE_CONTROL_NOT_SUPP
) {
754 if (ret
!= RAM_SAVE_CONTROL_DELAYED
) {
755 if (bytes_xmit
> 0) {
756 acct_info
.norm_pages
++;
757 } else if (bytes_xmit
== 0) {
758 acct_info
.dup_pages
++;
762 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
764 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
765 * page would be stale
767 xbzrle_cache_zero_page(current_addr
);
768 } else if (!ram_bulk_stage
&&
769 !migration_in_postcopy(ms
) && migrate_use_xbzrle()) {
770 pages
= save_xbzrle_page(f
, &p
, current_addr
, block
,
771 offset
, last_stage
, bytes_transferred
);
773 /* Can't send this cached data async, since the cache page
774 * might get updated before it gets to the wire
781 /* XBZRLE overflow or normal page */
783 *bytes_transferred
+= save_page_header(f
, block
,
784 offset
| RAM_SAVE_FLAG_PAGE
);
786 qemu_put_buffer_async(f
, p
, TARGET_PAGE_SIZE
);
788 qemu_put_buffer(f
, p
, TARGET_PAGE_SIZE
);
790 *bytes_transferred
+= TARGET_PAGE_SIZE
;
792 acct_info
.norm_pages
++;
795 XBZRLE_cache_unlock();
800 static int do_compress_ram_page(QEMUFile
*f
, RAMBlock
*block
,
803 int bytes_sent
, blen
;
804 uint8_t *p
= block
->host
+ (offset
& TARGET_PAGE_MASK
);
806 bytes_sent
= save_page_header(f
, block
, offset
|
807 RAM_SAVE_FLAG_COMPRESS_PAGE
);
808 blen
= qemu_put_compression_data(f
, p
, TARGET_PAGE_SIZE
,
809 migrate_compress_level());
812 qemu_file_set_error(migrate_get_current()->to_dst_file
, blen
);
813 error_report("compressed data failed!");
821 static uint64_t bytes_transferred
;
823 static void flush_compressed_data(QEMUFile
*f
)
825 int idx
, len
, thread_count
;
827 if (!migrate_use_compression()) {
830 thread_count
= migrate_compress_threads();
832 qemu_mutex_lock(&comp_done_lock
);
833 for (idx
= 0; idx
< thread_count
; idx
++) {
834 while (!comp_param
[idx
].done
) {
835 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
838 qemu_mutex_unlock(&comp_done_lock
);
840 for (idx
= 0; idx
< thread_count
; idx
++) {
841 qemu_mutex_lock(&comp_param
[idx
].mutex
);
842 if (!comp_param
[idx
].quit
) {
843 len
= qemu_put_qemu_file(f
, comp_param
[idx
].file
);
844 bytes_transferred
+= len
;
846 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
850 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
853 param
->block
= block
;
854 param
->offset
= offset
;
857 static int compress_page_with_multi_thread(QEMUFile
*f
, RAMBlock
*block
,
859 uint64_t *bytes_transferred
)
861 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
863 thread_count
= migrate_compress_threads();
864 qemu_mutex_lock(&comp_done_lock
);
866 for (idx
= 0; idx
< thread_count
; idx
++) {
867 if (comp_param
[idx
].done
) {
868 comp_param
[idx
].done
= false;
869 bytes_xmit
= qemu_put_qemu_file(f
, comp_param
[idx
].file
);
870 qemu_mutex_lock(&comp_param
[idx
].mutex
);
871 set_compress_params(&comp_param
[idx
], block
, offset
);
872 qemu_cond_signal(&comp_param
[idx
].cond
);
873 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
875 acct_info
.norm_pages
++;
876 *bytes_transferred
+= bytes_xmit
;
883 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
886 qemu_mutex_unlock(&comp_done_lock
);
892 * ram_save_compressed_page: compress the given page and send it to the stream
894 * Returns: Number of pages written.
896 * @ms: The current migration state.
897 * @f: QEMUFile where to send the data
898 * @block: block that contains the page we want to send
899 * @offset: offset inside the block for the page
900 * @last_stage: if we are at the completion stage
901 * @bytes_transferred: increase it with the number of transferred bytes
903 static int ram_save_compressed_page(MigrationState
*ms
, QEMUFile
*f
,
904 PageSearchStatus
*pss
, bool last_stage
,
905 uint64_t *bytes_transferred
)
908 uint64_t bytes_xmit
= 0;
911 RAMBlock
*block
= pss
->block
;
912 ram_addr_t offset
= pss
->offset
;
914 p
= block
->host
+ offset
;
916 ret
= ram_control_save_page(f
, block
->offset
,
917 offset
, TARGET_PAGE_SIZE
, &bytes_xmit
);
919 *bytes_transferred
+= bytes_xmit
;
922 if (ret
!= RAM_SAVE_CONTROL_NOT_SUPP
) {
923 if (ret
!= RAM_SAVE_CONTROL_DELAYED
) {
924 if (bytes_xmit
> 0) {
925 acct_info
.norm_pages
++;
926 } else if (bytes_xmit
== 0) {
927 acct_info
.dup_pages
++;
931 /* When starting the process of a new block, the first page of
932 * the block should be sent out before other pages in the same
933 * block, and all the pages in last block should have been sent
934 * out, keeping this order is important, because the 'cont' flag
935 * is used to avoid resending the block name.
937 if (block
!= last_sent_block
) {
938 flush_compressed_data(f
);
939 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
941 /* Make sure the first page is sent out before other pages */
942 bytes_xmit
= save_page_header(f
, block
, offset
|
943 RAM_SAVE_FLAG_COMPRESS_PAGE
);
944 blen
= qemu_put_compression_data(f
, p
, TARGET_PAGE_SIZE
,
945 migrate_compress_level());
947 *bytes_transferred
+= bytes_xmit
+ blen
;
948 acct_info
.norm_pages
++;
951 qemu_file_set_error(f
, blen
);
952 error_report("compressed data failed!");
956 offset
|= RAM_SAVE_FLAG_CONTINUE
;
957 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
959 pages
= compress_page_with_multi_thread(f
, block
, offset
,
969 * Find the next dirty page and update any state associated with
970 * the search process.
972 * Returns: True if a page is found
974 * @f: Current migration stream.
975 * @pss: Data about the state of the current dirty page scan.
976 * @*again: Set to false if the search has scanned the whole of RAM
977 * *ram_addr_abs: Pointer into which to store the address of the dirty page
978 * within the global ram_addr space
980 static bool find_dirty_block(QEMUFile
*f
, PageSearchStatus
*pss
,
981 bool *again
, ram_addr_t
*ram_addr_abs
)
983 pss
->offset
= migration_bitmap_find_dirty(pss
->block
, pss
->offset
,
985 if (pss
->complete_round
&& pss
->block
== last_seen_block
&&
986 pss
->offset
>= last_offset
) {
988 * We've been once around the RAM and haven't found anything.
994 if (pss
->offset
>= pss
->block
->used_length
) {
995 /* Didn't find anything in this RAM Block */
997 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
999 /* Hit the end of the list */
1000 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1001 /* Flag that we've looped */
1002 pss
->complete_round
= true;
1003 ram_bulk_stage
= false;
1004 if (migrate_use_xbzrle()) {
1005 /* If xbzrle is on, stop using the data compression at this
1006 * point. In theory, xbzrle can do better than compression.
1008 flush_compressed_data(f
);
1009 compression_switch
= false;
1012 /* Didn't find anything this time, but try again on the new block */
1016 /* Can go around again, but... */
1018 /* We've found something so probably don't need to */
1024 * Helper for 'get_queued_page' - gets a page off the queue
1025 * ms: MigrationState in
1026 * *offset: Used to return the offset within the RAMBlock
1027 * ram_addr_abs: global offset in the dirty/sent bitmaps
1029 * Returns: block (or NULL if none available)
1031 static RAMBlock
*unqueue_page(MigrationState
*ms
, ram_addr_t
*offset
,
1032 ram_addr_t
*ram_addr_abs
)
1034 RAMBlock
*block
= NULL
;
1036 qemu_mutex_lock(&ms
->src_page_req_mutex
);
1037 if (!QSIMPLEQ_EMPTY(&ms
->src_page_requests
)) {
1038 struct MigrationSrcPageRequest
*entry
=
1039 QSIMPLEQ_FIRST(&ms
->src_page_requests
);
1041 *offset
= entry
->offset
;
1042 *ram_addr_abs
= (entry
->offset
+ entry
->rb
->offset
) &
1045 if (entry
->len
> TARGET_PAGE_SIZE
) {
1046 entry
->len
-= TARGET_PAGE_SIZE
;
1047 entry
->offset
+= TARGET_PAGE_SIZE
;
1049 memory_region_unref(block
->mr
);
1050 QSIMPLEQ_REMOVE_HEAD(&ms
->src_page_requests
, next_req
);
1054 qemu_mutex_unlock(&ms
->src_page_req_mutex
);
1060 * Unqueue a page from the queue fed by postcopy page requests; skips pages
1061 * that are already sent (!dirty)
1063 * ms: MigrationState in
1064 * pss: PageSearchStatus structure updated with found block/offset
1065 * ram_addr_abs: global offset in the dirty/sent bitmaps
1067 * Returns: true if a queued page is found
1069 static bool get_queued_page(MigrationState
*ms
, PageSearchStatus
*pss
,
1070 ram_addr_t
*ram_addr_abs
)
1077 block
= unqueue_page(ms
, &offset
, ram_addr_abs
);
1079 * We're sending this page, and since it's postcopy nothing else
1080 * will dirty it, and we must make sure it doesn't get sent again
1081 * even if this queue request was received after the background
1082 * search already sent it.
1085 unsigned long *bitmap
;
1086 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1087 dirty
= test_bit(*ram_addr_abs
>> TARGET_PAGE_BITS
, bitmap
);
1089 trace_get_queued_page_not_dirty(
1090 block
->idstr
, (uint64_t)offset
,
1091 (uint64_t)*ram_addr_abs
,
1092 test_bit(*ram_addr_abs
>> TARGET_PAGE_BITS
,
1093 atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
));
1095 trace_get_queued_page(block
->idstr
,
1097 (uint64_t)*ram_addr_abs
);
1101 } while (block
&& !dirty
);
1105 * As soon as we start servicing pages out of order, then we have
1106 * to kill the bulk stage, since the bulk stage assumes
1107 * in (migration_bitmap_find_and_reset_dirty) that every page is
1108 * dirty, that's no longer true.
1110 ram_bulk_stage
= false;
1113 * We want the background search to continue from the queued page
1114 * since the guest is likely to want other pages near to the page
1115 * it just requested.
1118 pss
->offset
= offset
;
1125 * flush_page_queue: Flush any remaining pages in the ram request queue
1126 * it should be empty at the end anyway, but in error cases there may be
1129 * ms: MigrationState
1131 void flush_page_queue(MigrationState
*ms
)
1133 struct MigrationSrcPageRequest
*mspr
, *next_mspr
;
1134 /* This queue generally should be empty - but in the case of a failed
1135 * migration might have some droppings in.
1138 QSIMPLEQ_FOREACH_SAFE(mspr
, &ms
->src_page_requests
, next_req
, next_mspr
) {
1139 memory_region_unref(mspr
->rb
->mr
);
1140 QSIMPLEQ_REMOVE_HEAD(&ms
->src_page_requests
, next_req
);
1147 * Queue the pages for transmission, e.g. a request from postcopy destination
1148 * ms: MigrationStatus in which the queue is held
1149 * rbname: The RAMBlock the request is for - may be NULL (to mean reuse last)
1150 * start: Offset from the start of the RAMBlock
1151 * len: Length (in bytes) to send
1152 * Return: 0 on success
1154 int ram_save_queue_pages(MigrationState
*ms
, const char *rbname
,
1155 ram_addr_t start
, ram_addr_t len
)
1159 ms
->postcopy_requests
++;
1162 /* Reuse last RAMBlock */
1163 ramblock
= ms
->last_req_rb
;
1167 * Shouldn't happen, we can't reuse the last RAMBlock if
1168 * it's the 1st request.
1170 error_report("ram_save_queue_pages no previous block");
1174 ramblock
= qemu_ram_block_by_name(rbname
);
1177 /* We shouldn't be asked for a non-existent RAMBlock */
1178 error_report("ram_save_queue_pages no block '%s'", rbname
);
1181 ms
->last_req_rb
= ramblock
;
1183 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
1184 if (start
+len
> ramblock
->used_length
) {
1185 error_report("%s request overrun start=" RAM_ADDR_FMT
" len="
1186 RAM_ADDR_FMT
" blocklen=" RAM_ADDR_FMT
,
1187 __func__
, start
, len
, ramblock
->used_length
);
1191 struct MigrationSrcPageRequest
*new_entry
=
1192 g_malloc0(sizeof(struct MigrationSrcPageRequest
));
1193 new_entry
->rb
= ramblock
;
1194 new_entry
->offset
= start
;
1195 new_entry
->len
= len
;
1197 memory_region_ref(ramblock
->mr
);
1198 qemu_mutex_lock(&ms
->src_page_req_mutex
);
1199 QSIMPLEQ_INSERT_TAIL(&ms
->src_page_requests
, new_entry
, next_req
);
1200 qemu_mutex_unlock(&ms
->src_page_req_mutex
);
1211 * ram_save_target_page: Save one target page
1214 * @f: QEMUFile where to send the data
1215 * @block: pointer to block that contains the page we want to send
1216 * @offset: offset inside the block for the page;
1217 * @last_stage: if we are at the completion stage
1218 * @bytes_transferred: increase it with the number of transferred bytes
1219 * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
1221 * Returns: Number of pages written.
1223 static int ram_save_target_page(MigrationState
*ms
, QEMUFile
*f
,
1224 PageSearchStatus
*pss
,
1226 uint64_t *bytes_transferred
,
1227 ram_addr_t dirty_ram_abs
)
1231 /* Check the pages is dirty and if it is send it */
1232 if (migration_bitmap_clear_dirty(dirty_ram_abs
)) {
1233 unsigned long *unsentmap
;
1234 if (compression_switch
&& migrate_use_compression()) {
1235 res
= ram_save_compressed_page(ms
, f
, pss
,
1239 res
= ram_save_page(ms
, f
, pss
, last_stage
,
1246 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1248 clear_bit(dirty_ram_abs
>> TARGET_PAGE_BITS
, unsentmap
);
1250 /* Only update last_sent_block if a block was actually sent; xbzrle
1251 * might have decided the page was identical so didn't bother writing
1255 last_sent_block
= pss
->block
;
1263 * ram_save_host_page: Starting at *offset send pages up to the end
1264 * of the current host page. It's valid for the initial
1265 * offset to point into the middle of a host page
1266 * in which case the remainder of the hostpage is sent.
1267 * Only dirty target pages are sent.
1269 * Returns: Number of pages written.
1271 * @f: QEMUFile where to send the data
1272 * @block: pointer to block that contains the page we want to send
1273 * @offset: offset inside the block for the page; updated to last target page
1275 * @last_stage: if we are at the completion stage
1276 * @bytes_transferred: increase it with the number of transferred bytes
1277 * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
1279 static int ram_save_host_page(MigrationState
*ms
, QEMUFile
*f
,
1280 PageSearchStatus
*pss
,
1282 uint64_t *bytes_transferred
,
1283 ram_addr_t dirty_ram_abs
)
1285 int tmppages
, pages
= 0;
1287 tmppages
= ram_save_target_page(ms
, f
, pss
, last_stage
,
1288 bytes_transferred
, dirty_ram_abs
);
1294 pss
->offset
+= TARGET_PAGE_SIZE
;
1295 dirty_ram_abs
+= TARGET_PAGE_SIZE
;
1296 } while (pss
->offset
& (qemu_host_page_size
- 1));
1298 /* The offset we leave with is the last one we looked at */
1299 pss
->offset
-= TARGET_PAGE_SIZE
;
1304 * ram_find_and_save_block: Finds a dirty page and sends it to f
1306 * Called within an RCU critical section.
1308 * Returns: The number of pages written
1309 * 0 means no dirty pages
1311 * @f: QEMUFile where to send the data
1312 * @last_stage: if we are at the completion stage
1313 * @bytes_transferred: increase it with the number of transferred bytes
1315 * On systems where host-page-size > target-page-size it will send all the
1316 * pages in a host page that are dirty.
1319 static int ram_find_and_save_block(QEMUFile
*f
, bool last_stage
,
1320 uint64_t *bytes_transferred
)
1322 PageSearchStatus pss
;
1323 MigrationState
*ms
= migrate_get_current();
1326 ram_addr_t dirty_ram_abs
; /* Address of the start of the dirty page in
1329 pss
.block
= last_seen_block
;
1330 pss
.offset
= last_offset
;
1331 pss
.complete_round
= false;
1334 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1339 found
= get_queued_page(ms
, &pss
, &dirty_ram_abs
);
1342 /* priority queue empty, so just search for something dirty */
1343 found
= find_dirty_block(f
, &pss
, &again
, &dirty_ram_abs
);
1347 pages
= ram_save_host_page(ms
, f
, &pss
,
1348 last_stage
, bytes_transferred
,
1351 } while (!pages
&& again
);
1353 last_seen_block
= pss
.block
;
1354 last_offset
= pss
.offset
;
1359 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
1361 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
1363 acct_info
.dup_pages
+= pages
;
1365 acct_info
.norm_pages
+= pages
;
1366 bytes_transferred
+= size
;
1367 qemu_update_position(f
, size
);
1371 static ram_addr_t
ram_save_remaining(void)
1373 return migration_dirty_pages
;
1376 uint64_t ram_bytes_remaining(void)
1378 return ram_save_remaining() * TARGET_PAGE_SIZE
;
1381 uint64_t ram_bytes_transferred(void)
1383 return bytes_transferred
;
1386 uint64_t ram_bytes_total(void)
1392 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
)
1393 total
+= block
->used_length
;
1398 void free_xbzrle_decoded_buf(void)
1400 g_free(xbzrle_decoded_buf
);
1401 xbzrle_decoded_buf
= NULL
;
1404 static void migration_bitmap_free(struct BitmapRcu
*bmap
)
1407 g_free(bmap
->unsentmap
);
1411 static void ram_migration_cleanup(void *opaque
)
1413 /* caller have hold iothread lock or is in a bh, so there is
1414 * no writing race against this migration_bitmap
1416 struct BitmapRcu
*bitmap
= migration_bitmap_rcu
;
1417 atomic_rcu_set(&migration_bitmap_rcu
, NULL
);
1419 memory_global_dirty_log_stop();
1420 call_rcu(bitmap
, migration_bitmap_free
, rcu
);
1423 XBZRLE_cache_lock();
1425 cache_fini(XBZRLE
.cache
);
1426 g_free(XBZRLE
.encoded_buf
);
1427 g_free(XBZRLE
.current_buf
);
1428 g_free(ZERO_TARGET_PAGE
);
1429 XBZRLE
.cache
= NULL
;
1430 XBZRLE
.encoded_buf
= NULL
;
1431 XBZRLE
.current_buf
= NULL
;
1433 XBZRLE_cache_unlock();
1436 static void reset_ram_globals(void)
1438 last_seen_block
= NULL
;
1439 last_sent_block
= NULL
;
1441 last_version
= ram_list
.version
;
1442 ram_bulk_stage
= true;
1445 #define MAX_WAIT 50 /* ms, half buffered_file limit */
1447 void migration_bitmap_extend(ram_addr_t old
, ram_addr_t
new)
1449 /* called in qemu main thread, so there is
1450 * no writing race against this migration_bitmap
1452 if (migration_bitmap_rcu
) {
1453 struct BitmapRcu
*old_bitmap
= migration_bitmap_rcu
, *bitmap
;
1454 bitmap
= g_new(struct BitmapRcu
, 1);
1455 bitmap
->bmap
= bitmap_new(new);
1457 /* prevent migration_bitmap content from being set bit
1458 * by migration_bitmap_sync_range() at the same time.
1459 * it is safe to migration if migration_bitmap is cleared bit
1462 qemu_mutex_lock(&migration_bitmap_mutex
);
1463 bitmap_copy(bitmap
->bmap
, old_bitmap
->bmap
, old
);
1464 bitmap_set(bitmap
->bmap
, old
, new - old
);
1466 /* We don't have a way to safely extend the sentmap
1467 * with RCU; so mark it as missing, entry to postcopy
1470 bitmap
->unsentmap
= NULL
;
1472 atomic_rcu_set(&migration_bitmap_rcu
, bitmap
);
1473 qemu_mutex_unlock(&migration_bitmap_mutex
);
1474 migration_dirty_pages
+= new - old
;
1475 call_rcu(old_bitmap
, migration_bitmap_free
, rcu
);
1480 * 'expected' is the value you expect the bitmap mostly to be full
1481 * of; it won't bother printing lines that are all this value.
1482 * If 'todump' is null the migration bitmap is dumped.
1484 void ram_debug_dump_bitmap(unsigned long *todump
, bool expected
)
1486 int64_t ram_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
1489 int64_t linelen
= 128;
1493 todump
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1496 for (cur
= 0; cur
< ram_pages
; cur
+= linelen
) {
1500 * Last line; catch the case where the line length
1501 * is longer than remaining ram
1503 if (cur
+ linelen
> ram_pages
) {
1504 linelen
= ram_pages
- cur
;
1506 for (curb
= 0; curb
< linelen
; curb
++) {
1507 bool thisbit
= test_bit(cur
+ curb
, todump
);
1508 linebuf
[curb
] = thisbit
? '1' : '.';
1509 found
= found
|| (thisbit
!= expected
);
1512 linebuf
[curb
] = '\0';
1513 fprintf(stderr
, "0x%08" PRIx64
" : %s\n", cur
, linebuf
);
1518 /* **** functions for postcopy ***** */
1521 * Callback from postcopy_each_ram_send_discard for each RAMBlock
1522 * Note: At this point the 'unsentmap' is the processed bitmap combined
1523 * with the dirtymap; so a '1' means it's either dirty or unsent.
1524 * start,length: Indexes into the bitmap for the first bit
1525 * representing the named block and length in target-pages
1527 static int postcopy_send_discard_bm_ram(MigrationState
*ms
,
1528 PostcopyDiscardState
*pds
,
1529 unsigned long start
,
1530 unsigned long length
)
1532 unsigned long end
= start
+ length
; /* one after the end */
1533 unsigned long current
;
1534 unsigned long *unsentmap
;
1536 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1537 for (current
= start
; current
< end
; ) {
1538 unsigned long one
= find_next_bit(unsentmap
, end
, current
);
1541 unsigned long zero
= find_next_zero_bit(unsentmap
, end
, one
+ 1);
1542 unsigned long discard_length
;
1545 discard_length
= end
- one
;
1547 discard_length
= zero
- one
;
1549 if (discard_length
) {
1550 postcopy_discard_send_range(ms
, pds
, one
, discard_length
);
1552 current
= one
+ discard_length
;
1562 * Utility for the outgoing postcopy code.
1563 * Calls postcopy_send_discard_bm_ram for each RAMBlock
1564 * passing it bitmap indexes and name.
1565 * Returns: 0 on success
1566 * (qemu_ram_foreach_block ends up passing unscaled lengths
1567 * which would mean postcopy code would have to deal with target page)
1569 static int postcopy_each_ram_send_discard(MigrationState
*ms
)
1571 struct RAMBlock
*block
;
1574 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1575 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1576 PostcopyDiscardState
*pds
= postcopy_discard_send_init(ms
,
1581 * Postcopy sends chunks of bitmap over the wire, but it
1582 * just needs indexes at this point, avoids it having
1583 * target page specific code.
1585 ret
= postcopy_send_discard_bm_ram(ms
, pds
, first
,
1586 block
->used_length
>> TARGET_PAGE_BITS
);
1587 postcopy_discard_send_finish(ms
, pds
);
1597 * Helper for postcopy_chunk_hostpages; it's called twice to cleanup
1598 * the two bitmaps, that are similar, but one is inverted.
1600 * We search for runs of target-pages that don't start or end on a
1601 * host page boundary;
1602 * unsent_pass=true: Cleans up partially unsent host pages by searching
1604 * unsent_pass=false: Cleans up partially dirty host pages by searching
1605 * the main migration bitmap
1608 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, bool unsent_pass
,
1610 PostcopyDiscardState
*pds
)
1612 unsigned long *bitmap
;
1613 unsigned long *unsentmap
;
1614 unsigned int host_ratio
= qemu_host_page_size
/ TARGET_PAGE_SIZE
;
1615 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1616 unsigned long len
= block
->used_length
>> TARGET_PAGE_BITS
;
1617 unsigned long last
= first
+ (len
- 1);
1618 unsigned long run_start
;
1620 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1621 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1624 /* Find a sent page */
1625 run_start
= find_next_zero_bit(unsentmap
, last
+ 1, first
);
1627 /* Find a dirty page */
1628 run_start
= find_next_bit(bitmap
, last
+ 1, first
);
1631 while (run_start
<= last
) {
1632 bool do_fixup
= false;
1633 unsigned long fixup_start_addr
;
1634 unsigned long host_offset
;
1637 * If the start of this run of pages is in the middle of a host
1638 * page, then we need to fixup this host page.
1640 host_offset
= run_start
% host_ratio
;
1643 run_start
-= host_offset
;
1644 fixup_start_addr
= run_start
;
1645 /* For the next pass */
1646 run_start
= run_start
+ host_ratio
;
1648 /* Find the end of this run */
1649 unsigned long run_end
;
1651 run_end
= find_next_bit(unsentmap
, last
+ 1, run_start
+ 1);
1653 run_end
= find_next_zero_bit(bitmap
, last
+ 1, run_start
+ 1);
1656 * If the end isn't at the start of a host page, then the
1657 * run doesn't finish at the end of a host page
1658 * and we need to discard.
1660 host_offset
= run_end
% host_ratio
;
1663 fixup_start_addr
= run_end
- host_offset
;
1665 * This host page has gone, the next loop iteration starts
1666 * from after the fixup
1668 run_start
= fixup_start_addr
+ host_ratio
;
1671 * No discards on this iteration, next loop starts from
1672 * next sent/dirty page
1674 run_start
= run_end
+ 1;
1681 /* Tell the destination to discard this page */
1682 if (unsent_pass
|| !test_bit(fixup_start_addr
, unsentmap
)) {
1683 /* For the unsent_pass we:
1684 * discard partially sent pages
1685 * For the !unsent_pass (dirty) we:
1686 * discard partially dirty pages that were sent
1687 * (any partially sent pages were already discarded
1688 * by the previous unsent_pass)
1690 postcopy_discard_send_range(ms
, pds
, fixup_start_addr
,
1694 /* Clean up the bitmap */
1695 for (page
= fixup_start_addr
;
1696 page
< fixup_start_addr
+ host_ratio
; page
++) {
1697 /* All pages in this host page are now not sent */
1698 set_bit(page
, unsentmap
);
1701 * Remark them as dirty, updating the count for any pages
1702 * that weren't previously dirty.
1704 migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
1709 /* Find the next sent page for the next iteration */
1710 run_start
= find_next_zero_bit(unsentmap
, last
+ 1,
1713 /* Find the next dirty page for the next iteration */
1714 run_start
= find_next_bit(bitmap
, last
+ 1, run_start
);
1720 * Utility for the outgoing postcopy code.
1722 * Discard any partially sent host-page size chunks, mark any partially
1723 * dirty host-page size chunks as all dirty.
1725 * Returns: 0 on success
1727 static int postcopy_chunk_hostpages(MigrationState
*ms
)
1729 struct RAMBlock
*block
;
1731 if (qemu_host_page_size
== TARGET_PAGE_SIZE
) {
1732 /* Easy case - TPS==HPS - nothing to be done */
1736 /* Easiest way to make sure we don't resume in the middle of a host-page */
1737 last_seen_block
= NULL
;
1738 last_sent_block
= NULL
;
1741 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1742 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1744 PostcopyDiscardState
*pds
=
1745 postcopy_discard_send_init(ms
, first
, block
->idstr
);
1747 /* First pass: Discard all partially sent host pages */
1748 postcopy_chunk_hostpages_pass(ms
, true, block
, pds
);
1750 * Second pass: Ensure that all partially dirty host pages are made
1753 postcopy_chunk_hostpages_pass(ms
, false, block
, pds
);
1755 postcopy_discard_send_finish(ms
, pds
);
1756 } /* ram_list loop */
1762 * Transmit the set of pages to be discarded after precopy to the target
1763 * these are pages that:
1764 * a) Have been previously transmitted but are now dirty again
1765 * b) Pages that have never been transmitted, this ensures that
1766 * any pages on the destination that have been mapped by background
1767 * tasks get discarded (transparent huge pages is the specific concern)
1768 * Hopefully this is pretty sparse
1770 int ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
1773 unsigned long *bitmap
, *unsentmap
;
1777 /* This should be our last sync, the src is now paused */
1778 migration_bitmap_sync();
1780 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1782 /* We don't have a safe way to resize the sentmap, so
1783 * if the bitmap was resized it will be NULL at this
1786 error_report("migration ram resized during precopy phase");
1791 /* Deal with TPS != HPS */
1792 ret
= postcopy_chunk_hostpages(ms
);
1799 * Update the unsentmap to be unsentmap = unsentmap | dirty
1801 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1802 bitmap_or(unsentmap
, unsentmap
, bitmap
,
1803 last_ram_offset() >> TARGET_PAGE_BITS
);
1806 trace_ram_postcopy_send_discard_bitmap();
1807 #ifdef DEBUG_POSTCOPY
1808 ram_debug_dump_bitmap(unsentmap
, true);
1811 ret
= postcopy_each_ram_send_discard(ms
);
1818 * At the start of the postcopy phase of migration, any now-dirty
1819 * precopied pages are discarded.
1821 * start, length describe a byte address range within the RAMBlock
1823 * Returns 0 on success.
1825 int ram_discard_range(MigrationIncomingState
*mis
,
1826 const char *block_name
,
1827 uint64_t start
, size_t length
)
1832 RAMBlock
*rb
= qemu_ram_block_by_name(block_name
);
1835 error_report("ram_discard_range: Failed to find block '%s'",
1840 uint8_t *host_startaddr
= rb
->host
+ start
;
1842 if ((uintptr_t)host_startaddr
& (qemu_host_page_size
- 1)) {
1843 error_report("ram_discard_range: Unaligned start address: %p",
1848 if ((start
+ length
) <= rb
->used_length
) {
1849 uint8_t *host_endaddr
= host_startaddr
+ length
;
1850 if ((uintptr_t)host_endaddr
& (qemu_host_page_size
- 1)) {
1851 error_report("ram_discard_range: Unaligned end address: %p",
1855 ret
= postcopy_ram_discard_range(mis
, host_startaddr
, length
);
1857 error_report("ram_discard_range: Overrun block '%s' (%" PRIu64
1858 "/%zx/" RAM_ADDR_FMT
")",
1859 block_name
, start
, length
, rb
->used_length
);
1868 static int ram_save_init_globals(void)
1870 int64_t ram_bitmap_pages
; /* Size of bitmap in pages, including gaps */
1872 dirty_rate_high_cnt
= 0;
1873 bitmap_sync_count
= 0;
1874 migration_bitmap_sync_init();
1875 qemu_mutex_init(&migration_bitmap_mutex
);
1877 if (migrate_use_xbzrle()) {
1878 XBZRLE_cache_lock();
1879 ZERO_TARGET_PAGE
= g_malloc0(TARGET_PAGE_SIZE
);
1880 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size() /
1883 if (!XBZRLE
.cache
) {
1884 XBZRLE_cache_unlock();
1885 error_report("Error creating cache");
1888 XBZRLE_cache_unlock();
1890 /* We prefer not to abort if there is no memory */
1891 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
1892 if (!XBZRLE
.encoded_buf
) {
1893 error_report("Error allocating encoded_buf");
1897 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
1898 if (!XBZRLE
.current_buf
) {
1899 error_report("Error allocating current_buf");
1900 g_free(XBZRLE
.encoded_buf
);
1901 XBZRLE
.encoded_buf
= NULL
;
1908 /* For memory_global_dirty_log_start below. */
1909 qemu_mutex_lock_iothread();
1911 qemu_mutex_lock_ramlist();
1913 bytes_transferred
= 0;
1914 reset_ram_globals();
1916 ram_bitmap_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
1917 migration_bitmap_rcu
= g_new0(struct BitmapRcu
, 1);
1918 migration_bitmap_rcu
->bmap
= bitmap_new(ram_bitmap_pages
);
1919 bitmap_set(migration_bitmap_rcu
->bmap
, 0, ram_bitmap_pages
);
1921 if (migrate_postcopy_ram()) {
1922 migration_bitmap_rcu
->unsentmap
= bitmap_new(ram_bitmap_pages
);
1923 bitmap_set(migration_bitmap_rcu
->unsentmap
, 0, ram_bitmap_pages
);
1927 * Count the total number of pages used by ram blocks not including any
1928 * gaps due to alignment or unplugs.
1930 migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
1932 memory_global_dirty_log_start();
1933 migration_bitmap_sync();
1934 qemu_mutex_unlock_ramlist();
1935 qemu_mutex_unlock_iothread();
1941 /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
1942 * long-running RCU critical section. When rcu-reclaims in the code
1943 * start to become numerous it will be necessary to reduce the
1944 * granularity of these critical sections.
1947 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
1951 /* migration has already setup the bitmap, reuse it. */
1952 if (!migration_in_colo_state()) {
1953 if (ram_save_init_globals() < 0) {
1960 qemu_put_be64(f
, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE
);
1962 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1963 qemu_put_byte(f
, strlen(block
->idstr
));
1964 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
1965 qemu_put_be64(f
, block
->used_length
);
1970 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
1971 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
1973 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
1978 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
1986 if (ram_list
.version
!= last_version
) {
1987 reset_ram_globals();
1990 /* Read version before ram_list.blocks */
1993 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
1995 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
1997 while ((ret
= qemu_file_rate_limit(f
)) == 0) {
2000 pages
= ram_find_and_save_block(f
, false, &bytes_transferred
);
2001 /* no more pages to sent */
2006 acct_info
.iterations
++;
2008 /* we want to check in the 1st loop, just in case it was the 1st time
2009 and we had to sync the dirty bitmap.
2010 qemu_get_clock_ns() is a bit expensive, so we only check each some
2013 if ((i
& 63) == 0) {
2014 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) / 1000000;
2015 if (t1
> MAX_WAIT
) {
2016 trace_ram_save_iterate_big_wait(t1
, i
);
2022 flush_compressed_data(f
);
2026 * Must occur before EOS (or any QEMUFile operation)
2027 * because of RDMA protocol.
2029 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
2031 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2032 bytes_transferred
+= 8;
2034 ret
= qemu_file_get_error(f
);
2042 /* Called with iothread lock */
2043 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
2047 if (!migration_in_postcopy(migrate_get_current())) {
2048 migration_bitmap_sync();
2051 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
2053 /* try transferring iterative blocks of memory */
2055 /* flush all remaining blocks regardless of rate limiting */
2059 pages
= ram_find_and_save_block(f
, !migration_in_colo_state(),
2060 &bytes_transferred
);
2061 /* no more blocks to sent */
2067 flush_compressed_data(f
);
2068 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
2072 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2077 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
2078 uint64_t *non_postcopiable_pending
,
2079 uint64_t *postcopiable_pending
)
2081 uint64_t remaining_size
;
2083 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
2085 if (!migration_in_postcopy(migrate_get_current()) &&
2086 remaining_size
< max_size
) {
2087 qemu_mutex_lock_iothread();
2089 migration_bitmap_sync();
2091 qemu_mutex_unlock_iothread();
2092 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
2095 /* We can do postcopy, and all the data is postcopiable */
2096 *postcopiable_pending
+= remaining_size
;
2099 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
2101 unsigned int xh_len
;
2103 uint8_t *loaded_data
;
2105 if (!xbzrle_decoded_buf
) {
2106 xbzrle_decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
2108 loaded_data
= xbzrle_decoded_buf
;
2110 /* extract RLE header */
2111 xh_flags
= qemu_get_byte(f
);
2112 xh_len
= qemu_get_be16(f
);
2114 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
2115 error_report("Failed to load XBZRLE page - wrong compression!");
2119 if (xh_len
> TARGET_PAGE_SIZE
) {
2120 error_report("Failed to load XBZRLE page - len overflow!");
2123 /* load data and decode */
2124 qemu_get_buffer_in_place(f
, &loaded_data
, xh_len
);
2127 if (xbzrle_decode_buffer(loaded_data
, xh_len
, host
,
2128 TARGET_PAGE_SIZE
) == -1) {
2129 error_report("Failed to load XBZRLE page - decode error!");
2136 /* Must be called from within a rcu critical section.
2137 * Returns a pointer from within the RCU-protected ram_list.
2140 * Read a RAMBlock ID from the stream f.
2142 * f: Stream to read from
2143 * flags: Page flags (mostly to see if it's a continuation of previous block)
2145 static inline RAMBlock
*ram_block_from_stream(QEMUFile
*f
,
2148 static RAMBlock
*block
= NULL
;
2152 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
2154 error_report("Ack, bad migration stream!");
2160 len
= qemu_get_byte(f
);
2161 qemu_get_buffer(f
, (uint8_t *)id
, len
);
2164 block
= qemu_ram_block_by_name(id
);
2166 error_report("Can't find block %s", id
);
2173 static inline void *host_from_ram_block_offset(RAMBlock
*block
,
2176 if (!offset_in_ramblock(block
, offset
)) {
2180 return block
->host
+ offset
;
2184 * If a page (or a whole RDMA chunk) has been
2185 * determined to be zero, then zap it.
2187 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
2189 if (ch
!= 0 || !is_zero_range(host
, size
)) {
2190 memset(host
, ch
, size
);
2194 static void *do_data_decompress(void *opaque
)
2196 DecompressParam
*param
= opaque
;
2197 unsigned long pagesize
;
2201 qemu_mutex_lock(¶m
->mutex
);
2202 while (!param
->quit
) {
2207 qemu_mutex_unlock(¶m
->mutex
);
2209 pagesize
= TARGET_PAGE_SIZE
;
2210 /* uncompress() will return failed in some case, especially
2211 * when the page is dirted when doing the compression, it's
2212 * not a problem because the dirty page will be retransferred
2213 * and uncompress() won't break the data in other pages.
2215 uncompress((Bytef
*)des
, &pagesize
,
2216 (const Bytef
*)param
->compbuf
, len
);
2218 qemu_mutex_lock(&decomp_done_lock
);
2220 qemu_cond_signal(&decomp_done_cond
);
2221 qemu_mutex_unlock(&decomp_done_lock
);
2223 qemu_mutex_lock(¶m
->mutex
);
2225 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
2228 qemu_mutex_unlock(¶m
->mutex
);
2233 static void wait_for_decompress_done(void)
2235 int idx
, thread_count
;
2237 if (!migrate_use_compression()) {
2241 thread_count
= migrate_decompress_threads();
2242 qemu_mutex_lock(&decomp_done_lock
);
2243 for (idx
= 0; idx
< thread_count
; idx
++) {
2244 while (!decomp_param
[idx
].done
) {
2245 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
2248 qemu_mutex_unlock(&decomp_done_lock
);
2251 void migrate_decompress_threads_create(void)
2253 int i
, thread_count
;
2255 thread_count
= migrate_decompress_threads();
2256 decompress_threads
= g_new0(QemuThread
, thread_count
);
2257 decomp_param
= g_new0(DecompressParam
, thread_count
);
2258 qemu_mutex_init(&decomp_done_lock
);
2259 qemu_cond_init(&decomp_done_cond
);
2260 for (i
= 0; i
< thread_count
; i
++) {
2261 qemu_mutex_init(&decomp_param
[i
].mutex
);
2262 qemu_cond_init(&decomp_param
[i
].cond
);
2263 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
2264 decomp_param
[i
].done
= true;
2265 decomp_param
[i
].quit
= false;
2266 qemu_thread_create(decompress_threads
+ i
, "decompress",
2267 do_data_decompress
, decomp_param
+ i
,
2268 QEMU_THREAD_JOINABLE
);
2272 void migrate_decompress_threads_join(void)
2274 int i
, thread_count
;
2276 thread_count
= migrate_decompress_threads();
2277 for (i
= 0; i
< thread_count
; i
++) {
2278 qemu_mutex_lock(&decomp_param
[i
].mutex
);
2279 decomp_param
[i
].quit
= true;
2280 qemu_cond_signal(&decomp_param
[i
].cond
);
2281 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
2283 for (i
= 0; i
< thread_count
; i
++) {
2284 qemu_thread_join(decompress_threads
+ i
);
2285 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
2286 qemu_cond_destroy(&decomp_param
[i
].cond
);
2287 g_free(decomp_param
[i
].compbuf
);
2289 g_free(decompress_threads
);
2290 g_free(decomp_param
);
2291 decompress_threads
= NULL
;
2292 decomp_param
= NULL
;
2295 static void decompress_data_with_multi_threads(QEMUFile
*f
,
2296 void *host
, int len
)
2298 int idx
, thread_count
;
2300 thread_count
= migrate_decompress_threads();
2301 qemu_mutex_lock(&decomp_done_lock
);
2303 for (idx
= 0; idx
< thread_count
; idx
++) {
2304 if (decomp_param
[idx
].done
) {
2305 decomp_param
[idx
].done
= false;
2306 qemu_mutex_lock(&decomp_param
[idx
].mutex
);
2307 qemu_get_buffer(f
, decomp_param
[idx
].compbuf
, len
);
2308 decomp_param
[idx
].des
= host
;
2309 decomp_param
[idx
].len
= len
;
2310 qemu_cond_signal(&decomp_param
[idx
].cond
);
2311 qemu_mutex_unlock(&decomp_param
[idx
].mutex
);
2315 if (idx
< thread_count
) {
2318 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
2321 qemu_mutex_unlock(&decomp_done_lock
);
2325 * Allocate data structures etc needed by incoming migration with postcopy-ram
2326 * postcopy-ram's similarly names postcopy_ram_incoming_init does the work
2328 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
2330 size_t ram_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
2332 return postcopy_ram_incoming_init(mis
, ram_pages
);
2336 * Called in postcopy mode by ram_load().
2337 * rcu_read_lock is taken prior to this being called.
2339 static int ram_load_postcopy(QEMUFile
*f
)
2341 int flags
= 0, ret
= 0;
2342 bool place_needed
= false;
2343 bool matching_page_sizes
= qemu_host_page_size
== TARGET_PAGE_SIZE
;
2344 MigrationIncomingState
*mis
= migration_incoming_get_current();
2345 /* Temporary page that is later 'placed' */
2346 void *postcopy_host_page
= postcopy_get_tmp_page(mis
);
2347 void *last_host
= NULL
;
2348 bool all_zero
= false;
2350 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
2353 void *page_buffer
= NULL
;
2354 void *place_source
= NULL
;
2357 addr
= qemu_get_be64(f
);
2358 flags
= addr
& ~TARGET_PAGE_MASK
;
2359 addr
&= TARGET_PAGE_MASK
;
2361 trace_ram_load_postcopy_loop((uint64_t)addr
, flags
);
2362 place_needed
= false;
2363 if (flags
& (RAM_SAVE_FLAG_COMPRESS
| RAM_SAVE_FLAG_PAGE
)) {
2364 RAMBlock
*block
= ram_block_from_stream(f
, flags
);
2366 host
= host_from_ram_block_offset(block
, addr
);
2368 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
2373 * Postcopy requires that we place whole host pages atomically.
2374 * To make it atomic, the data is read into a temporary page
2375 * that's moved into place later.
2376 * The migration protocol uses, possibly smaller, target-pages
2377 * however the source ensures it always sends all the components
2378 * of a host page in order.
2380 page_buffer
= postcopy_host_page
+
2381 ((uintptr_t)host
& ~qemu_host_page_mask
);
2382 /* If all TP are zero then we can optimise the place */
2383 if (!((uintptr_t)host
& ~qemu_host_page_mask
)) {
2386 /* not the 1st TP within the HP */
2387 if (host
!= (last_host
+ TARGET_PAGE_SIZE
)) {
2388 error_report("Non-sequential target page %p/%p",
2397 * If it's the last part of a host page then we place the host
2400 place_needed
= (((uintptr_t)host
+ TARGET_PAGE_SIZE
) &
2401 ~qemu_host_page_mask
) == 0;
2402 place_source
= postcopy_host_page
;
2406 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
2407 case RAM_SAVE_FLAG_COMPRESS
:
2408 ch
= qemu_get_byte(f
);
2409 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
2415 case RAM_SAVE_FLAG_PAGE
:
2417 if (!place_needed
|| !matching_page_sizes
) {
2418 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
2420 /* Avoids the qemu_file copy during postcopy, which is
2421 * going to do a copy later; can only do it when we
2422 * do this read in one go (matching page sizes)
2424 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
2428 case RAM_SAVE_FLAG_EOS
:
2432 error_report("Unknown combination of migration flags: %#x"
2433 " (postcopy mode)", flags
);
2438 /* This gets called at the last target page in the host page */
2440 ret
= postcopy_place_page_zero(mis
,
2441 host
+ TARGET_PAGE_SIZE
-
2442 qemu_host_page_size
);
2444 ret
= postcopy_place_page(mis
, host
+ TARGET_PAGE_SIZE
-
2445 qemu_host_page_size
,
2450 ret
= qemu_file_get_error(f
);
2457 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
2459 int flags
= 0, ret
= 0;
2460 static uint64_t seq_iter
;
2463 * If system is running in postcopy mode, page inserts to host memory must
2466 bool postcopy_running
= postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING
;
2470 if (version_id
!= 4) {
2474 /* This RCU critical section can be very long running.
2475 * When RCU reclaims in the code start to become numerous,
2476 * it will be necessary to reduce the granularity of this
2481 if (postcopy_running
) {
2482 ret
= ram_load_postcopy(f
);
2485 while (!postcopy_running
&& !ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
2486 ram_addr_t addr
, total_ram_bytes
;
2490 addr
= qemu_get_be64(f
);
2491 flags
= addr
& ~TARGET_PAGE_MASK
;
2492 addr
&= TARGET_PAGE_MASK
;
2494 if (flags
& (RAM_SAVE_FLAG_COMPRESS
| RAM_SAVE_FLAG_PAGE
|
2495 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
2496 RAMBlock
*block
= ram_block_from_stream(f
, flags
);
2498 host
= host_from_ram_block_offset(block
, addr
);
2500 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
2506 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
2507 case RAM_SAVE_FLAG_MEM_SIZE
:
2508 /* Synchronize RAM block list */
2509 total_ram_bytes
= addr
;
2510 while (!ret
&& total_ram_bytes
) {
2515 len
= qemu_get_byte(f
);
2516 qemu_get_buffer(f
, (uint8_t *)id
, len
);
2518 length
= qemu_get_be64(f
);
2520 block
= qemu_ram_block_by_name(id
);
2522 if (length
!= block
->used_length
) {
2523 Error
*local_err
= NULL
;
2525 ret
= qemu_ram_resize(block
, length
,
2528 error_report_err(local_err
);
2531 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
2534 error_report("Unknown ramblock \"%s\", cannot "
2535 "accept migration", id
);
2539 total_ram_bytes
-= length
;
2543 case RAM_SAVE_FLAG_COMPRESS
:
2544 ch
= qemu_get_byte(f
);
2545 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
2548 case RAM_SAVE_FLAG_PAGE
:
2549 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
2552 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
2553 len
= qemu_get_be32(f
);
2554 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
2555 error_report("Invalid compressed data length: %d", len
);
2559 decompress_data_with_multi_threads(f
, host
, len
);
2562 case RAM_SAVE_FLAG_XBZRLE
:
2563 if (load_xbzrle(f
, addr
, host
) < 0) {
2564 error_report("Failed to decompress XBZRLE page at "
2565 RAM_ADDR_FMT
, addr
);
2570 case RAM_SAVE_FLAG_EOS
:
2574 if (flags
& RAM_SAVE_FLAG_HOOK
) {
2575 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
2577 error_report("Unknown combination of migration flags: %#x",
2583 ret
= qemu_file_get_error(f
);
2587 wait_for_decompress_done();
2589 trace_ram_load_complete(ret
, seq_iter
);
2593 static SaveVMHandlers savevm_ram_handlers
= {
2594 .save_live_setup
= ram_save_setup
,
2595 .save_live_iterate
= ram_save_iterate
,
2596 .save_live_complete_postcopy
= ram_save_complete
,
2597 .save_live_complete_precopy
= ram_save_complete
,
2598 .save_live_pending
= ram_save_pending
,
2599 .load_state
= ram_load
,
2600 .cleanup
= ram_migration_cleanup
,
2603 void ram_mig_init(void)
2605 qemu_mutex_init(&XBZRLE
.lock
);
2606 register_savevm_live(NULL
, "ram", 0, 4, &savevm_ram_handlers
, NULL
);