4 * Copyright IBM, Corp. 2008
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
16 #include "qemu-common.h"
17 #include "migration/migration.h"
18 #include "monitor/monitor.h"
19 #include "migration/qemu-file.h"
20 #include "sysemu/sysemu.h"
21 #include "block/block.h"
22 #include "qemu/sockets.h"
23 #include "migration/block.h"
24 #include "qemu/thread.h"
25 #include "qmp-commands.h"
28 //#define DEBUG_MIGRATION
30 #ifdef DEBUG_MIGRATION
31 #define DPRINTF(fmt, ...) \
32 do { printf("migration: " fmt, ## __VA_ARGS__); } while (0)
34 #define DPRINTF(fmt, ...) \
47 #define MAX_THROTTLE (32 << 20) /* Migration speed throttling */
49 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
51 #define BUFFER_DELAY 100
52 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
54 /* Migration XBZRLE default cache size */
55 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
57 static NotifierList migration_state_notifiers
=
58 NOTIFIER_LIST_INITIALIZER(migration_state_notifiers
);
60 /* When we add fault tolerance, we could have several
61 migrations at once. For now we don't need to add
62 dynamic creation of migration */
64 MigrationState
*migrate_get_current(void)
66 static MigrationState current_migration
= {
67 .state
= MIG_STATE_NONE
,
68 .bandwidth_limit
= MAX_THROTTLE
,
69 .xbzrle_cache_size
= DEFAULT_MIGRATE_CACHE_SIZE
,
73 return ¤t_migration
;
76 void qemu_start_incoming_migration(const char *uri
, Error
**errp
)
80 if (strstart(uri
, "tcp:", &p
))
81 tcp_start_incoming_migration(p
, errp
);
83 else if (strstart(uri
, "x-rdma:", &p
))
84 rdma_start_incoming_migration(p
, errp
);
87 else if (strstart(uri
, "exec:", &p
))
88 exec_start_incoming_migration(p
, errp
);
89 else if (strstart(uri
, "unix:", &p
))
90 unix_start_incoming_migration(p
, errp
);
91 else if (strstart(uri
, "fd:", &p
))
92 fd_start_incoming_migration(p
, errp
);
95 error_setg(errp
, "unknown migration protocol: %s", uri
);
99 static void process_incoming_migration_co(void *opaque
)
101 QEMUFile
*f
= opaque
;
104 ret
= qemu_loadvm_state(f
);
107 fprintf(stderr
, "load of migration failed\n");
110 qemu_announce_self();
111 DPRINTF("successfully loaded vm state\n");
113 bdrv_clear_incoming_migration_all();
114 /* Make sure all file formats flush their mutable metadata */
115 bdrv_invalidate_cache_all();
120 runstate_set(RUN_STATE_PAUSED
);
124 void process_incoming_migration(QEMUFile
*f
)
126 Coroutine
*co
= qemu_coroutine_create(process_incoming_migration_co
);
127 int fd
= qemu_get_fd(f
);
130 qemu_set_nonblock(fd
);
131 qemu_coroutine_enter(co
, f
);
134 /* amount of nanoseconds we are willing to wait for migration to be down.
135 * the choice of nanoseconds is because it is the maximum resolution that
136 * get_clock() can achieve. It is an internal measure. All user-visible
137 * units must be in seconds */
138 static uint64_t max_downtime
= 30000000;
140 uint64_t migrate_max_downtime(void)
145 MigrationCapabilityStatusList
*qmp_query_migrate_capabilities(Error
**errp
)
147 MigrationCapabilityStatusList
*head
= NULL
;
148 MigrationCapabilityStatusList
*caps
;
149 MigrationState
*s
= migrate_get_current();
152 for (i
= 0; i
< MIGRATION_CAPABILITY_MAX
; i
++) {
154 head
= g_malloc0(sizeof(*caps
));
157 caps
->next
= g_malloc0(sizeof(*caps
));
161 g_malloc(sizeof(*caps
->value
));
162 caps
->value
->capability
= i
;
163 caps
->value
->state
= s
->enabled_capabilities
[i
];
169 static void get_xbzrle_cache_stats(MigrationInfo
*info
)
171 if (migrate_use_xbzrle()) {
172 info
->has_xbzrle_cache
= true;
173 info
->xbzrle_cache
= g_malloc0(sizeof(*info
->xbzrle_cache
));
174 info
->xbzrle_cache
->cache_size
= migrate_xbzrle_cache_size();
175 info
->xbzrle_cache
->bytes
= xbzrle_mig_bytes_transferred();
176 info
->xbzrle_cache
->pages
= xbzrle_mig_pages_transferred();
177 info
->xbzrle_cache
->cache_miss
= xbzrle_mig_pages_cache_miss();
178 info
->xbzrle_cache
->overflow
= xbzrle_mig_pages_overflow();
182 MigrationInfo
*qmp_query_migrate(Error
**errp
)
184 MigrationInfo
*info
= g_malloc0(sizeof(*info
));
185 MigrationState
*s
= migrate_get_current();
189 /* no migration has happened ever */
191 case MIG_STATE_SETUP
:
192 info
->has_status
= true;
193 info
->status
= g_strdup("setup");
195 case MIG_STATE_ACTIVE
:
196 info
->has_status
= true;
197 info
->status
= g_strdup("active");
198 info
->has_total_time
= true;
199 info
->total_time
= qemu_get_clock_ms(rt_clock
)
201 info
->has_expected_downtime
= true;
202 info
->expected_downtime
= s
->expected_downtime
;
204 info
->has_ram
= true;
205 info
->ram
= g_malloc0(sizeof(*info
->ram
));
206 info
->ram
->transferred
= ram_bytes_transferred();
207 info
->ram
->remaining
= ram_bytes_remaining();
208 info
->ram
->total
= ram_bytes_total();
209 info
->ram
->duplicate
= dup_mig_pages_transferred();
210 info
->ram
->skipped
= skipped_mig_pages_transferred();
211 info
->ram
->normal
= norm_mig_pages_transferred();
212 info
->ram
->normal_bytes
= norm_mig_bytes_transferred();
213 info
->ram
->dirty_pages_rate
= s
->dirty_pages_rate
;
214 info
->ram
->mbps
= s
->mbps
;
216 if (blk_mig_active()) {
217 info
->has_disk
= true;
218 info
->disk
= g_malloc0(sizeof(*info
->disk
));
219 info
->disk
->transferred
= blk_mig_bytes_transferred();
220 info
->disk
->remaining
= blk_mig_bytes_remaining();
221 info
->disk
->total
= blk_mig_bytes_total();
224 get_xbzrle_cache_stats(info
);
226 case MIG_STATE_COMPLETED
:
227 get_xbzrle_cache_stats(info
);
229 info
->has_status
= true;
230 info
->status
= g_strdup("completed");
231 info
->total_time
= s
->total_time
;
232 info
->has_downtime
= true;
233 info
->downtime
= s
->downtime
;
235 info
->has_ram
= true;
236 info
->ram
= g_malloc0(sizeof(*info
->ram
));
237 info
->ram
->transferred
= ram_bytes_transferred();
238 info
->ram
->remaining
= 0;
239 info
->ram
->total
= ram_bytes_total();
240 info
->ram
->duplicate
= dup_mig_pages_transferred();
241 info
->ram
->skipped
= skipped_mig_pages_transferred();
242 info
->ram
->normal
= norm_mig_pages_transferred();
243 info
->ram
->normal_bytes
= norm_mig_bytes_transferred();
244 info
->ram
->mbps
= s
->mbps
;
246 case MIG_STATE_ERROR
:
247 info
->has_status
= true;
248 info
->status
= g_strdup("failed");
250 case MIG_STATE_CANCELLED
:
251 info
->has_status
= true;
252 info
->status
= g_strdup("cancelled");
259 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList
*params
,
262 MigrationState
*s
= migrate_get_current();
263 MigrationCapabilityStatusList
*cap
;
265 if (s
->state
== MIG_STATE_ACTIVE
|| s
->state
== MIG_STATE_SETUP
) {
266 error_set(errp
, QERR_MIGRATION_ACTIVE
);
270 for (cap
= params
; cap
; cap
= cap
->next
) {
271 s
->enabled_capabilities
[cap
->value
->capability
] = cap
->value
->state
;
275 /* shared migration helpers */
277 static void migrate_fd_cleanup(void *opaque
)
279 MigrationState
*s
= opaque
;
281 qemu_bh_delete(s
->cleanup_bh
);
282 s
->cleanup_bh
= NULL
;
285 DPRINTF("closing file\n");
286 qemu_mutex_unlock_iothread();
287 qemu_thread_join(&s
->thread
);
288 qemu_mutex_lock_iothread();
290 qemu_fclose(s
->file
);
294 assert(s
->state
!= MIG_STATE_ACTIVE
);
296 if (s
->state
!= MIG_STATE_COMPLETED
) {
297 qemu_savevm_state_cancel();
300 notifier_list_notify(&migration_state_notifiers
, s
);
303 static void migrate_set_state(MigrationState
*s
, int old_state
, int new_state
)
305 if (atomic_cmpxchg(&s
->state
, old_state
, new_state
) == new_state
) {
306 trace_migrate_set_state(new_state
);
310 void migrate_fd_error(MigrationState
*s
)
312 DPRINTF("setting error state\n");
313 assert(s
->file
== NULL
);
314 s
->state
= MIG_STATE_ERROR
;
315 trace_migrate_set_state(MIG_STATE_ERROR
);
316 notifier_list_notify(&migration_state_notifiers
, s
);
319 static void migrate_fd_cancel(MigrationState
*s
)
321 DPRINTF("cancelling migration\n");
323 migrate_set_state(s
, s
->state
, MIG_STATE_CANCELLED
);
326 void add_migration_state_change_notifier(Notifier
*notify
)
328 notifier_list_add(&migration_state_notifiers
, notify
);
331 void remove_migration_state_change_notifier(Notifier
*notify
)
333 notifier_remove(notify
);
336 bool migration_is_active(MigrationState
*s
)
338 return s
->state
== MIG_STATE_ACTIVE
;
341 bool migration_has_finished(MigrationState
*s
)
343 return s
->state
== MIG_STATE_COMPLETED
;
346 bool migration_has_failed(MigrationState
*s
)
348 return (s
->state
== MIG_STATE_CANCELLED
||
349 s
->state
== MIG_STATE_ERROR
);
352 static MigrationState
*migrate_init(const MigrationParams
*params
)
354 MigrationState
*s
= migrate_get_current();
355 int64_t bandwidth_limit
= s
->bandwidth_limit
;
356 bool enabled_capabilities
[MIGRATION_CAPABILITY_MAX
];
357 int64_t xbzrle_cache_size
= s
->xbzrle_cache_size
;
359 memcpy(enabled_capabilities
, s
->enabled_capabilities
,
360 sizeof(enabled_capabilities
));
362 memset(s
, 0, sizeof(*s
));
364 memcpy(s
->enabled_capabilities
, enabled_capabilities
,
365 sizeof(enabled_capabilities
));
366 s
->xbzrle_cache_size
= xbzrle_cache_size
;
368 s
->bandwidth_limit
= bandwidth_limit
;
369 s
->state
= MIG_STATE_SETUP
;
370 trace_migrate_set_state(MIG_STATE_SETUP
);
372 s
->total_time
= qemu_get_clock_ms(rt_clock
);
376 static GSList
*migration_blockers
;
378 void migrate_add_blocker(Error
*reason
)
380 migration_blockers
= g_slist_prepend(migration_blockers
, reason
);
383 void migrate_del_blocker(Error
*reason
)
385 migration_blockers
= g_slist_remove(migration_blockers
, reason
);
388 void qmp_migrate(const char *uri
, bool has_blk
, bool blk
,
389 bool has_inc
, bool inc
, bool has_detach
, bool detach
,
392 Error
*local_err
= NULL
;
393 MigrationState
*s
= migrate_get_current();
394 MigrationParams params
;
400 if (s
->state
== MIG_STATE_ACTIVE
|| s
->state
== MIG_STATE_SETUP
) {
401 error_set(errp
, QERR_MIGRATION_ACTIVE
);
405 if (qemu_savevm_state_blocked(errp
)) {
409 if (migration_blockers
) {
410 *errp
= error_copy(migration_blockers
->data
);
414 s
= migrate_init(¶ms
);
416 if (strstart(uri
, "tcp:", &p
)) {
417 tcp_start_outgoing_migration(s
, p
, &local_err
);
419 } else if (strstart(uri
, "x-rdma:", &p
)) {
420 rdma_start_outgoing_migration(s
, p
, &local_err
);
423 } else if (strstart(uri
, "exec:", &p
)) {
424 exec_start_outgoing_migration(s
, p
, &local_err
);
425 } else if (strstart(uri
, "unix:", &p
)) {
426 unix_start_outgoing_migration(s
, p
, &local_err
);
427 } else if (strstart(uri
, "fd:", &p
)) {
428 fd_start_outgoing_migration(s
, p
, &local_err
);
431 error_set(errp
, QERR_INVALID_PARAMETER_VALUE
, "uri", "a valid migration protocol");
437 error_propagate(errp
, local_err
);
442 void qmp_migrate_cancel(Error
**errp
)
444 migrate_fd_cancel(migrate_get_current());
447 void qmp_migrate_set_cache_size(int64_t value
, Error
**errp
)
449 MigrationState
*s
= migrate_get_current();
451 /* Check for truncation */
452 if (value
!= (size_t)value
) {
453 error_set(errp
, QERR_INVALID_PARAMETER_VALUE
, "cache size",
454 "exceeding address space");
458 s
->xbzrle_cache_size
= xbzrle_cache_resize(value
);
461 int64_t qmp_query_migrate_cache_size(Error
**errp
)
463 return migrate_xbzrle_cache_size();
466 void qmp_migrate_set_speed(int64_t value
, Error
**errp
)
473 if (value
> SIZE_MAX
) {
477 s
= migrate_get_current();
478 s
->bandwidth_limit
= value
;
480 qemu_file_set_rate_limit(s
->file
, s
->bandwidth_limit
/ XFER_LIMIT_RATIO
);
484 void qmp_migrate_set_downtime(double value
, Error
**errp
)
487 value
= MAX(0, MIN(UINT64_MAX
, value
));
488 max_downtime
= (uint64_t)value
;
491 bool migrate_rdma_pin_all(void)
495 s
= migrate_get_current();
497 return s
->enabled_capabilities
[MIGRATION_CAPABILITY_X_RDMA_PIN_ALL
];
500 bool migrate_auto_converge(void)
504 s
= migrate_get_current();
506 return s
->enabled_capabilities
[MIGRATION_CAPABILITY_AUTO_CONVERGE
];
509 bool migrate_zero_blocks(void)
513 s
= migrate_get_current();
515 return s
->enabled_capabilities
[MIGRATION_CAPABILITY_ZERO_BLOCKS
];
518 int migrate_use_xbzrle(void)
522 s
= migrate_get_current();
524 return s
->enabled_capabilities
[MIGRATION_CAPABILITY_XBZRLE
];
527 int64_t migrate_xbzrle_cache_size(void)
531 s
= migrate_get_current();
533 return s
->xbzrle_cache_size
;
536 /* migration thread support */
538 static void *migration_thread(void *opaque
)
540 MigrationState
*s
= opaque
;
541 int64_t initial_time
= qemu_get_clock_ms(rt_clock
);
542 int64_t initial_bytes
= 0;
543 int64_t max_size
= 0;
544 int64_t start_time
= initial_time
;
545 bool old_vm_running
= false;
547 DPRINTF("beginning savevm\n");
548 qemu_savevm_state_begin(s
->file
, &s
->params
);
550 migrate_set_state(s
, MIG_STATE_SETUP
, MIG_STATE_ACTIVE
);
552 while (s
->state
== MIG_STATE_ACTIVE
) {
553 int64_t current_time
;
554 uint64_t pending_size
;
556 if (!qemu_file_rate_limit(s
->file
)) {
557 DPRINTF("iterate\n");
558 pending_size
= qemu_savevm_state_pending(s
->file
, max_size
);
559 DPRINTF("pending size %lu max %lu\n", pending_size
, max_size
);
560 if (pending_size
&& pending_size
>= max_size
) {
561 qemu_savevm_state_iterate(s
->file
);
565 DPRINTF("done iterating\n");
566 qemu_mutex_lock_iothread();
567 start_time
= qemu_get_clock_ms(rt_clock
);
568 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER
);
569 old_vm_running
= runstate_is_running();
571 ret
= vm_stop_force_state(RUN_STATE_FINISH_MIGRATE
);
573 qemu_file_set_rate_limit(s
->file
, INT_MAX
);
574 qemu_savevm_state_complete(s
->file
);
576 qemu_mutex_unlock_iothread();
579 migrate_set_state(s
, MIG_STATE_ACTIVE
, MIG_STATE_ERROR
);
583 if (!qemu_file_get_error(s
->file
)) {
584 migrate_set_state(s
, MIG_STATE_ACTIVE
, MIG_STATE_COMPLETED
);
590 if (qemu_file_get_error(s
->file
)) {
591 migrate_set_state(s
, MIG_STATE_ACTIVE
, MIG_STATE_ERROR
);
594 current_time
= qemu_get_clock_ms(rt_clock
);
595 if (current_time
>= initial_time
+ BUFFER_DELAY
) {
596 uint64_t transferred_bytes
= qemu_ftell(s
->file
) - initial_bytes
;
597 uint64_t time_spent
= current_time
- initial_time
;
598 double bandwidth
= transferred_bytes
/ time_spent
;
599 max_size
= bandwidth
* migrate_max_downtime() / 1000000;
601 s
->mbps
= time_spent
? (((double) transferred_bytes
* 8.0) /
602 ((double) time_spent
/ 1000.0)) / 1000.0 / 1000.0 : -1;
604 DPRINTF("transferred %" PRIu64
" time_spent %" PRIu64
605 " bandwidth %g max_size %" PRId64
"\n",
606 transferred_bytes
, time_spent
, bandwidth
, max_size
);
607 /* if we haven't sent anything, we don't want to recalculate
608 10000 is a small enough number for our purposes */
609 if (s
->dirty_bytes_rate
&& transferred_bytes
> 10000) {
610 s
->expected_downtime
= s
->dirty_bytes_rate
/ bandwidth
;
613 qemu_file_reset_rate_limit(s
->file
);
614 initial_time
= current_time
;
615 initial_bytes
= qemu_ftell(s
->file
);
617 if (qemu_file_rate_limit(s
->file
)) {
618 /* usleep expects microseconds */
619 g_usleep((initial_time
+ BUFFER_DELAY
- current_time
)*1000);
623 qemu_mutex_lock_iothread();
624 if (s
->state
== MIG_STATE_COMPLETED
) {
625 int64_t end_time
= qemu_get_clock_ms(rt_clock
);
626 s
->total_time
= end_time
- s
->total_time
;
627 s
->downtime
= end_time
- start_time
;
628 runstate_set(RUN_STATE_POSTMIGRATE
);
630 if (old_vm_running
) {
634 qemu_bh_schedule(s
->cleanup_bh
);
635 qemu_mutex_unlock_iothread();
640 void migrate_fd_connect(MigrationState
*s
)
642 s
->state
= MIG_STATE_SETUP
;
643 trace_migrate_set_state(MIG_STATE_SETUP
);
645 /* This is a best 1st approximation. ns to ms */
646 s
->expected_downtime
= max_downtime
/1000000;
647 s
->cleanup_bh
= qemu_bh_new(migrate_fd_cleanup
, s
);
649 qemu_file_set_rate_limit(s
->file
,
650 s
->bandwidth_limit
/ XFER_LIMIT_RATIO
);
652 qemu_thread_create(&s
->thread
, migration_thread
, s
,
653 QEMU_THREAD_JOINABLE
);
654 notifier_list_notify(&migration_state_notifiers
, s
);