4 * Copyright IBM, Corp. 2008
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
16 #include "qemu-common.h"
17 #include "migration/migration.h"
18 #include "monitor/monitor.h"
19 #include "buffered_file.h"
20 #include "sysemu/sysemu.h"
21 #include "block/block.h"
22 #include "qemu/sockets.h"
23 #include "migration/block.h"
24 #include "qemu/thread.h"
25 #include "qmp-commands.h"
27 //#define DEBUG_MIGRATION
29 #ifdef DEBUG_MIGRATION
30 #define DPRINTF(fmt, ...) \
31 do { printf("migration: " fmt, ## __VA_ARGS__); } while (0)
33 #define DPRINTF(fmt, ...) \
45 #define MAX_THROTTLE (32 << 20) /* Migration speed throttling */
47 /* Migration XBZRLE default cache size */
48 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
50 static NotifierList migration_state_notifiers
=
51 NOTIFIER_LIST_INITIALIZER(migration_state_notifiers
);
53 /* When we add fault tolerance, we could have several
54 migrations at once. For now we don't need to add
55 dynamic creation of migration */
57 MigrationState
*migrate_get_current(void)
59 static MigrationState current_migration
= {
60 .state
= MIG_STATE_SETUP
,
61 .bandwidth_limit
= MAX_THROTTLE
,
62 .xbzrle_cache_size
= DEFAULT_MIGRATE_CACHE_SIZE
,
65 return ¤t_migration
;
68 void qemu_start_incoming_migration(const char *uri
, Error
**errp
)
72 if (strstart(uri
, "tcp:", &p
))
73 tcp_start_incoming_migration(p
, errp
);
75 else if (strstart(uri
, "exec:", &p
))
76 exec_start_incoming_migration(p
, errp
);
77 else if (strstart(uri
, "unix:", &p
))
78 unix_start_incoming_migration(p
, errp
);
79 else if (strstart(uri
, "fd:", &p
))
80 fd_start_incoming_migration(p
, errp
);
83 error_setg(errp
, "unknown migration protocol: %s\n", uri
);
87 static void process_incoming_migration_co(void *opaque
)
92 ret
= qemu_loadvm_state(f
);
93 qemu_set_fd_handler(qemu_get_fd(f
), NULL
, NULL
, NULL
);
96 fprintf(stderr
, "load of migration failed\n");
100 DPRINTF("successfully loaded vm state\n");
102 bdrv_clear_incoming_migration_all();
103 /* Make sure all file formats flush their mutable metadata */
104 bdrv_invalidate_cache_all();
109 runstate_set(RUN_STATE_PAUSED
);
113 static void enter_migration_coroutine(void *opaque
)
115 Coroutine
*co
= opaque
;
116 qemu_coroutine_enter(co
, NULL
);
119 void process_incoming_migration(QEMUFile
*f
)
121 Coroutine
*co
= qemu_coroutine_create(process_incoming_migration_co
);
122 int fd
= qemu_get_fd(f
);
125 socket_set_nonblock(fd
);
126 qemu_set_fd_handler(fd
, enter_migration_coroutine
, NULL
, co
);
127 qemu_coroutine_enter(co
, f
);
130 /* amount of nanoseconds we are willing to wait for migration to be down.
131 * the choice of nanoseconds is because it is the maximum resolution that
132 * get_clock() can achieve. It is an internal measure. All user-visible
133 * units must be in seconds */
134 static uint64_t max_downtime
= 30000000;
136 uint64_t migrate_max_downtime(void)
141 MigrationCapabilityStatusList
*qmp_query_migrate_capabilities(Error
**errp
)
143 MigrationCapabilityStatusList
*head
= NULL
;
144 MigrationCapabilityStatusList
*caps
;
145 MigrationState
*s
= migrate_get_current();
148 for (i
= 0; i
< MIGRATION_CAPABILITY_MAX
; i
++) {
150 head
= g_malloc0(sizeof(*caps
));
153 caps
->next
= g_malloc0(sizeof(*caps
));
157 g_malloc(sizeof(*caps
->value
));
158 caps
->value
->capability
= i
;
159 caps
->value
->state
= s
->enabled_capabilities
[i
];
165 static void get_xbzrle_cache_stats(MigrationInfo
*info
)
167 if (migrate_use_xbzrle()) {
168 info
->has_xbzrle_cache
= true;
169 info
->xbzrle_cache
= g_malloc0(sizeof(*info
->xbzrle_cache
));
170 info
->xbzrle_cache
->cache_size
= migrate_xbzrle_cache_size();
171 info
->xbzrle_cache
->bytes
= xbzrle_mig_bytes_transferred();
172 info
->xbzrle_cache
->pages
= xbzrle_mig_pages_transferred();
173 info
->xbzrle_cache
->cache_miss
= xbzrle_mig_pages_cache_miss();
174 info
->xbzrle_cache
->overflow
= xbzrle_mig_pages_overflow();
178 MigrationInfo
*qmp_query_migrate(Error
**errp
)
180 MigrationInfo
*info
= g_malloc0(sizeof(*info
));
181 MigrationState
*s
= migrate_get_current();
184 case MIG_STATE_SETUP
:
185 /* no migration has happened ever */
187 case MIG_STATE_ACTIVE
:
188 info
->has_status
= true;
189 info
->status
= g_strdup("active");
190 info
->has_total_time
= true;
191 info
->total_time
= qemu_get_clock_ms(rt_clock
)
193 info
->has_expected_downtime
= true;
194 info
->expected_downtime
= s
->expected_downtime
;
196 info
->has_ram
= true;
197 info
->ram
= g_malloc0(sizeof(*info
->ram
));
198 info
->ram
->transferred
= ram_bytes_transferred();
199 info
->ram
->remaining
= ram_bytes_remaining();
200 info
->ram
->total
= ram_bytes_total();
201 info
->ram
->duplicate
= dup_mig_pages_transferred();
202 info
->ram
->normal
= norm_mig_pages_transferred();
203 info
->ram
->normal_bytes
= norm_mig_bytes_transferred();
204 info
->ram
->dirty_pages_rate
= s
->dirty_pages_rate
;
207 if (blk_mig_active()) {
208 info
->has_disk
= true;
209 info
->disk
= g_malloc0(sizeof(*info
->disk
));
210 info
->disk
->transferred
= blk_mig_bytes_transferred();
211 info
->disk
->remaining
= blk_mig_bytes_remaining();
212 info
->disk
->total
= blk_mig_bytes_total();
215 get_xbzrle_cache_stats(info
);
217 case MIG_STATE_COMPLETED
:
218 get_xbzrle_cache_stats(info
);
220 info
->has_status
= true;
221 info
->status
= g_strdup("completed");
222 info
->total_time
= s
->total_time
;
223 info
->has_downtime
= true;
224 info
->downtime
= s
->downtime
;
226 info
->has_ram
= true;
227 info
->ram
= g_malloc0(sizeof(*info
->ram
));
228 info
->ram
->transferred
= ram_bytes_transferred();
229 info
->ram
->remaining
= 0;
230 info
->ram
->total
= ram_bytes_total();
231 info
->ram
->duplicate
= dup_mig_pages_transferred();
232 info
->ram
->normal
= norm_mig_pages_transferred();
233 info
->ram
->normal_bytes
= norm_mig_bytes_transferred();
235 case MIG_STATE_ERROR
:
236 info
->has_status
= true;
237 info
->status
= g_strdup("failed");
239 case MIG_STATE_CANCELLED
:
240 info
->has_status
= true;
241 info
->status
= g_strdup("cancelled");
248 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList
*params
,
251 MigrationState
*s
= migrate_get_current();
252 MigrationCapabilityStatusList
*cap
;
254 if (s
->state
== MIG_STATE_ACTIVE
) {
255 error_set(errp
, QERR_MIGRATION_ACTIVE
);
259 for (cap
= params
; cap
; cap
= cap
->next
) {
260 s
->enabled_capabilities
[cap
->value
->capability
] = cap
->value
->state
;
264 /* shared migration helpers */
266 static int migrate_fd_cleanup(MigrationState
*s
)
271 DPRINTF("closing file\n");
272 ret
= qemu_fclose(s
->file
);
280 void migrate_fd_error(MigrationState
*s
)
282 DPRINTF("setting error state\n");
283 s
->state
= MIG_STATE_ERROR
;
284 notifier_list_notify(&migration_state_notifiers
, s
);
285 migrate_fd_cleanup(s
);
288 static void migrate_fd_completed(MigrationState
*s
)
290 DPRINTF("setting completed state\n");
291 if (migrate_fd_cleanup(s
) < 0) {
292 s
->state
= MIG_STATE_ERROR
;
294 s
->state
= MIG_STATE_COMPLETED
;
295 runstate_set(RUN_STATE_POSTMIGRATE
);
297 notifier_list_notify(&migration_state_notifiers
, s
);
300 ssize_t
migrate_fd_put_buffer(MigrationState
*s
, const void *data
,
305 if (s
->state
!= MIG_STATE_ACTIVE
) {
310 ret
= s
->write(s
, data
, size
);
311 } while (ret
== -1 && ((s
->get_error(s
)) == EINTR
));
314 ret
= -(s
->get_error(s
));
319 void migrate_fd_put_ready(MigrationState
*s
)
323 if (s
->state
!= MIG_STATE_ACTIVE
) {
324 DPRINTF("put_ready returning because of non-active state\n");
328 s
->first_time
= false;
329 DPRINTF("beginning savevm\n");
330 ret
= qemu_savevm_state_begin(s
->file
, &s
->params
);
332 DPRINTF("failed, %d\n", ret
);
338 DPRINTF("iterate\n");
339 ret
= qemu_savevm_state_iterate(s
->file
);
342 } else if (ret
== 1) {
343 int old_vm_running
= runstate_is_running();
344 int64_t start_time
, end_time
;
346 DPRINTF("done iterating\n");
347 start_time
= qemu_get_clock_ms(rt_clock
);
348 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER
);
349 if (old_vm_running
) {
350 vm_stop(RUN_STATE_FINISH_MIGRATE
);
352 vm_stop_force_state(RUN_STATE_FINISH_MIGRATE
);
355 if (qemu_savevm_state_complete(s
->file
) < 0) {
358 migrate_fd_completed(s
);
360 end_time
= qemu_get_clock_ms(rt_clock
);
361 s
->total_time
= end_time
- s
->total_time
;
362 s
->downtime
= end_time
- start_time
;
363 if (s
->state
!= MIG_STATE_COMPLETED
) {
364 if (old_vm_running
) {
371 static void migrate_fd_cancel(MigrationState
*s
)
373 if (s
->state
!= MIG_STATE_ACTIVE
)
376 DPRINTF("cancelling migration\n");
378 s
->state
= MIG_STATE_CANCELLED
;
379 notifier_list_notify(&migration_state_notifiers
, s
);
380 qemu_savevm_state_cancel(s
->file
);
382 migrate_fd_cleanup(s
);
385 int migrate_fd_wait_for_unfreeze(MigrationState
*s
)
389 DPRINTF("wait for unfreeze\n");
390 if (s
->state
!= MIG_STATE_ACTIVE
)
397 FD_SET(s
->fd
, &wfds
);
399 ret
= select(s
->fd
+ 1, NULL
, &wfds
, NULL
, NULL
);
400 } while (ret
== -1 && (s
->get_error(s
)) == EINTR
);
403 return -s
->get_error(s
);
408 int migrate_fd_close(MigrationState
*s
)
418 void add_migration_state_change_notifier(Notifier
*notify
)
420 notifier_list_add(&migration_state_notifiers
, notify
);
423 void remove_migration_state_change_notifier(Notifier
*notify
)
425 notifier_remove(notify
);
428 bool migration_is_active(MigrationState
*s
)
430 return s
->state
== MIG_STATE_ACTIVE
;
433 bool migration_has_finished(MigrationState
*s
)
435 return s
->state
== MIG_STATE_COMPLETED
;
438 bool migration_has_failed(MigrationState
*s
)
440 return (s
->state
== MIG_STATE_CANCELLED
||
441 s
->state
== MIG_STATE_ERROR
);
444 void migrate_fd_connect(MigrationState
*s
)
446 s
->state
= MIG_STATE_ACTIVE
;
447 s
->first_time
= true;
448 qemu_fopen_ops_buffered(s
);
451 static MigrationState
*migrate_init(const MigrationParams
*params
)
453 MigrationState
*s
= migrate_get_current();
454 int64_t bandwidth_limit
= s
->bandwidth_limit
;
455 bool enabled_capabilities
[MIGRATION_CAPABILITY_MAX
];
456 int64_t xbzrle_cache_size
= s
->xbzrle_cache_size
;
458 memcpy(enabled_capabilities
, s
->enabled_capabilities
,
459 sizeof(enabled_capabilities
));
461 memset(s
, 0, sizeof(*s
));
462 s
->bandwidth_limit
= bandwidth_limit
;
464 memcpy(s
->enabled_capabilities
, enabled_capabilities
,
465 sizeof(enabled_capabilities
));
466 s
->xbzrle_cache_size
= xbzrle_cache_size
;
468 s
->bandwidth_limit
= bandwidth_limit
;
469 s
->state
= MIG_STATE_SETUP
;
470 s
->total_time
= qemu_get_clock_ms(rt_clock
);
475 static GSList
*migration_blockers
;
477 void migrate_add_blocker(Error
*reason
)
479 migration_blockers
= g_slist_prepend(migration_blockers
, reason
);
482 void migrate_del_blocker(Error
*reason
)
484 migration_blockers
= g_slist_remove(migration_blockers
, reason
);
487 void qmp_migrate(const char *uri
, bool has_blk
, bool blk
,
488 bool has_inc
, bool inc
, bool has_detach
, bool detach
,
491 Error
*local_err
= NULL
;
492 MigrationState
*s
= migrate_get_current();
493 MigrationParams params
;
499 if (s
->state
== MIG_STATE_ACTIVE
) {
500 error_set(errp
, QERR_MIGRATION_ACTIVE
);
504 if (qemu_savevm_state_blocked(errp
)) {
508 if (migration_blockers
) {
509 *errp
= error_copy(migration_blockers
->data
);
513 s
= migrate_init(¶ms
);
515 if (strstart(uri
, "tcp:", &p
)) {
516 tcp_start_outgoing_migration(s
, p
, &local_err
);
518 } else if (strstart(uri
, "exec:", &p
)) {
519 exec_start_outgoing_migration(s
, p
, &local_err
);
520 } else if (strstart(uri
, "unix:", &p
)) {
521 unix_start_outgoing_migration(s
, p
, &local_err
);
522 } else if (strstart(uri
, "fd:", &p
)) {
523 fd_start_outgoing_migration(s
, p
, &local_err
);
526 error_set(errp
, QERR_INVALID_PARAMETER_VALUE
, "uri", "a valid migration protocol");
532 error_propagate(errp
, local_err
);
536 notifier_list_notify(&migration_state_notifiers
, s
);
539 void qmp_migrate_cancel(Error
**errp
)
541 migrate_fd_cancel(migrate_get_current());
544 void qmp_migrate_set_cache_size(int64_t value
, Error
**errp
)
546 MigrationState
*s
= migrate_get_current();
548 /* Check for truncation */
549 if (value
!= (size_t)value
) {
550 error_set(errp
, QERR_INVALID_PARAMETER_VALUE
, "cache size",
551 "exceeding address space");
555 s
->xbzrle_cache_size
= xbzrle_cache_resize(value
);
558 int64_t qmp_query_migrate_cache_size(Error
**errp
)
560 return migrate_xbzrle_cache_size();
563 void qmp_migrate_set_speed(int64_t value
, Error
**errp
)
571 s
= migrate_get_current();
572 s
->bandwidth_limit
= value
;
573 qemu_file_set_rate_limit(s
->file
, s
->bandwidth_limit
);
576 void qmp_migrate_set_downtime(double value
, Error
**errp
)
579 value
= MAX(0, MIN(UINT64_MAX
, value
));
580 max_downtime
= (uint64_t)value
;
583 int migrate_use_xbzrle(void)
587 s
= migrate_get_current();
589 return s
->enabled_capabilities
[MIGRATION_CAPABILITY_XBZRLE
];
592 int64_t migrate_xbzrle_cache_size(void)
596 s
= migrate_get_current();
598 return s
->xbzrle_cache_size
;