debian/patches/pve/0017-PVE-add-savevm-async-for-background-state-snapshots.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Dietmar Maurer <dietmar@proxmox.com>
   3 Date: Mon, 6 Apr 2020 12:16:46 +0200
   4 Subject: [PATCH] PVE: add savevm-async for background state snapshots
   5
   6 Put qemu_savevm_state_{header,setup} into the main loop and the rest
   7 of the iteration into a coroutine. The former need to lock the
   8 iothread (and we can't unlock it in the coroutine), and the latter
   9 can't deal with being in a separate thread, so a coroutine it must
  10 be.
  11
  12 Truncate output file at 1024 boundary.
  13
  14 Do not block the VM and save the state on aborting a snapshot, as the
  15 snapshot will be invalid anyway.
  16
  17 Also, when aborting, wait for the target file to be closed, otherwise a
  18 client might run into race-conditions when trying to remove the file
  19 still opened by QEMU.
  20
  21 Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
  22 Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
  23 Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
  24 [improve aborting]
  25 Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
  26 ---
  27  hmp-commands-info.hx         |  13 +
  28  hmp-commands.hx              |  33 ++
  29  include/migration/snapshot.h |   1 +
  30  include/monitor/hmp.h        |   5 +
  31  migration/meson.build        |   1 +
  32  migration/savevm-async.c     | 591 +++++++++++++++++++++++++++++++++++
  33  monitor/hmp-cmds.c           |  57 ++++
  34  qapi/migration.json          |  34 ++
  35  qapi/misc.json               |  32 ++
  36  qemu-options.hx              |  12 +
  37  softmmu/vl.c                 |  10 +
  38  11 files changed, 789 insertions(+)
  39  create mode 100644 migration/savevm-async.c
  40
  41 diff --git a/hmp-commands-info.hx b/hmp-commands-info.hx
  42 index 117ba25f91..b3b797ca28 100644
  43 --- a/hmp-commands-info.hx
  44 +++ b/hmp-commands-info.hx
  45 @@ -580,6 +580,19 @@ SRST
  46      Show current migration xbzrle cache size.
  47  ERST
  48
  49 +    {
  50 +        .name       = "savevm",
  51 +        .args_type  = "",
  52 +        .params     = "",
  53 +        .help       = "show savevm status",
  54 +        .cmd = hmp_info_savevm,
  55 +    },
  56 +
  57 +SRST
  58 +  ``info savevm``
  59 +    Show savevm status.
  60 +ERST
  61 +
  62      {
  63          .name       = "balloon",
  64          .args_type  = "",
  65 diff --git a/hmp-commands.hx b/hmp-commands.hx
  66 index ff2d7aa8f3..d294c234a5 100644
  67 --- a/hmp-commands.hx
  68 +++ b/hmp-commands.hx
  69 @@ -1866,3 +1866,36 @@ ERST
  70          .flags      = "p",
  71      },
  72
  73 +
  74 +    {
  75 +        .name       = "savevm-start",
  76 +        .args_type  = "statefile:s?",
  77 +        .params     = "[statefile]",
  78 +        .help       = "Prepare for snapshot and halt VM. Save VM state to statefile.",
  79 +        .cmd = hmp_savevm_start,
  80 +    },
  81 +
  82 +    {
  83 +        .name       = "snapshot-drive",
  84 +        .args_type  = "device:s,name:s",
  85 +        .params     = "device name",
  86 +        .help       = "Create internal snapshot.",
  87 +        .cmd = hmp_snapshot_drive,
  88 +    },
  89 +
  90 +    {
  91 +        .name       = "delete-drive-snapshot",
  92 +        .args_type  = "device:s,name:s",
  93 +        .params     = "device name",
  94 +        .help       = "Delete internal snapshot.",
  95 +        .cmd = hmp_delete_drive_snapshot,
  96 +    },
  97 +
  98 +    {
  99 +        .name       = "savevm-end",
 100 +        .args_type  = "",
 101 +        .params     = "",
 102 +        .help       = "Resume VM after snaphot.",
 103 +        .cmd        = hmp_savevm_end,
 104 +        .coroutine  = true,
 105 +    },
 106 diff --git a/include/migration/snapshot.h b/include/migration/snapshot.h
 107 index c85b6ec75b..4411b7121d 100644
 108 --- a/include/migration/snapshot.h
 109 +++ b/include/migration/snapshot.h
 110 @@ -17,5 +17,6 @@
 111
 112  int save_snapshot(const char *name, Error **errp);
 113  int load_snapshot(const char *name, Error **errp);
 114 +int load_snapshot_from_blockdev(const char *filename, Error **errp);
 115
 116  #endif
 117 diff --git a/include/monitor/hmp.h b/include/monitor/hmp.h
 118 index ed2913fd18..4e06f89e8e 100644
 119 --- a/include/monitor/hmp.h
 120 +++ b/include/monitor/hmp.h
 121 @@ -25,6 +25,7 @@ void hmp_info_status(Monitor *mon, const QDict *qdict);
 122  void hmp_info_uuid(Monitor *mon, const QDict *qdict);
 123  void hmp_info_chardev(Monitor *mon, const QDict *qdict);
 124  void hmp_info_mice(Monitor *mon, const QDict *qdict);
 125 +void hmp_info_savevm(Monitor *mon, const QDict *qdict);
 126  void hmp_info_migrate(Monitor *mon, const QDict *qdict);
 127  void hmp_info_migrate_capabilities(Monitor *mon, const QDict *qdict);
 128  void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict);
 129 @@ -83,6 +84,10 @@ void hmp_netdev_add(Monitor *mon, const QDict *qdict);
 130  void hmp_netdev_del(Monitor *mon, const QDict *qdict);
 131  void hmp_getfd(Monitor *mon, const QDict *qdict);
 132  void hmp_closefd(Monitor *mon, const QDict *qdict);
 133 +void hmp_savevm_start(Monitor *mon, const QDict *qdict);
 134 +void hmp_snapshot_drive(Monitor *mon, const QDict *qdict);
 135 +void hmp_delete_drive_snapshot(Monitor *mon, const QDict *qdict);
 136 +void hmp_savevm_end(Monitor *mon, const QDict *qdict);
 137  void hmp_sendkey(Monitor *mon, const QDict *qdict);
 138  void hmp_screendump(Monitor *mon, const QDict *qdict);
 139  void hmp_chardev_add(Monitor *mon, const QDict *qdict);
 140 diff --git a/migration/meson.build b/migration/meson.build
 141 index 980e37865c..e62b79b60f 100644
 142 --- a/migration/meson.build
 143 +++ b/migration/meson.build
 144 @@ -23,6 +23,7 @@ softmmu_ss.add(files(
 145    'multifd-zlib.c',
 146    'postcopy-ram.c',
 147    'savevm.c',
 148 +  'savevm-async.c',
 149    'socket.c',
 150    'tls.c',
 151  ))
 152 diff --git a/migration/savevm-async.c b/migration/savevm-async.c
 153 new file mode 100644
 154 index 0000000000..4e345c1a7d
 155 --- /dev/null
 156 +++ b/migration/savevm-async.c
 157 @@ -0,0 +1,591 @@
 158 +#include "qemu/osdep.h"
 159 +#include "migration/migration.h"
 160 +#include "migration/savevm.h"
 161 +#include "migration/snapshot.h"
 162 +#include "migration/global_state.h"
 163 +#include "migration/ram.h"
 164 +#include "migration/qemu-file.h"
 165 +#include "sysemu/sysemu.h"
 166 +#include "sysemu/runstate.h"
 167 +#include "block/block.h"
 168 +#include "sysemu/block-backend.h"
 169 +#include "qapi/error.h"
 170 +#include "qapi/qmp/qerror.h"
 171 +#include "qapi/qmp/qdict.h"
 172 +#include "qapi/qapi-commands-migration.h"
 173 +#include "qapi/qapi-commands-misc.h"
 174 +#include "qapi/qapi-commands-block.h"
 175 +#include "qemu/cutils.h"
 176 +#include "qemu/timer.h"
 177 +#include "qemu/main-loop.h"
 178 +#include "qemu/rcu.h"
 179 +
 180 +/* #define DEBUG_SAVEVM_STATE */
 181 +
 182 +/* used while emulated sync operation in progress */
 183 +#define NOT_DONE -EINPROGRESS
 184 +
 185 +#ifdef DEBUG_SAVEVM_STATE
 186 +#define DPRINTF(fmt, ...) \
 187 +    do { printf("savevm-async: " fmt, ## __VA_ARGS__); } while (0)
 188 +#else
 189 +#define DPRINTF(fmt, ...) \
 190 +    do { } while (0)
 191 +#endif
 192 +
 193 +enum {
 194 +    SAVE_STATE_DONE,
 195 +    SAVE_STATE_ERROR,
 196 +    SAVE_STATE_ACTIVE,
 197 +    SAVE_STATE_COMPLETED,
 198 +    SAVE_STATE_CANCELLED
 199 +};
 200 +
 201 +
 202 +static struct SnapshotState {
 203 +    BlockBackend *target;
 204 +    size_t bs_pos;
 205 +    int state;
 206 +    Error *error;
 207 +    Error *blocker;
 208 +    int saved_vm_running;
 209 +    QEMUFile *file;
 210 +    int64_t total_time;
 211 +    QEMUBH *finalize_bh;
 212 +    Coroutine *co;
 213 +    QemuCoSleepState *target_close_wait;
 214 +} snap_state;
 215 +
 216 +static bool savevm_aborted(void)
 217 +{
 218 +    return snap_state.state == SAVE_STATE_CANCELLED ||
 219 +        snap_state.state == SAVE_STATE_ERROR;
 220 +}
 221 +
 222 +SaveVMInfo *qmp_query_savevm(Error **errp)
 223 +{
 224 +    SaveVMInfo *info = g_malloc0(sizeof(*info));
 225 +    struct SnapshotState *s = &snap_state;
 226 +
 227 +    if (s->state != SAVE_STATE_DONE) {
 228 +        info->has_bytes = true;
 229 +        info->bytes = s->bs_pos;
 230 +        switch (s->state) {
 231 +        case SAVE_STATE_ERROR:
 232 +            info->has_status = true;
 233 +            info->status = g_strdup("failed");
 234 +            info->has_total_time = true;
 235 +            info->total_time = s->total_time;
 236 +            if (s->error) {
 237 +                info->has_error = true;
 238 +                info->error = g_strdup(error_get_pretty(s->error));
 239 +            }
 240 +            break;
 241 +        case SAVE_STATE_ACTIVE:
 242 +            info->has_status = true;
 243 +            info->status = g_strdup("active");
 244 +            info->has_total_time = true;
 245 +            info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME)
 246 +                - s->total_time;
 247 +            break;
 248 +        case SAVE_STATE_COMPLETED:
 249 +            info->has_status = true;
 250 +            info->status = g_strdup("completed");
 251 +            info->has_total_time = true;
 252 +            info->total_time = s->total_time;
 253 +            break;
 254 +        }
 255 +    }
 256 +
 257 +    return info;
 258 +}
 259 +
 260 +static int save_snapshot_cleanup(void)
 261 +{
 262 +    int ret = 0;
 263 +
 264 +    DPRINTF("save_snapshot_cleanup\n");
 265 +
 266 +    snap_state.total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 267 +        snap_state.total_time;
 268 +
 269 +    if (snap_state.file) {
 270 +        ret = qemu_fclose(snap_state.file);
 271 +    }
 272 +
 273 +    if (snap_state.target) {
 274 +        if (!savevm_aborted()) {
 275 +            /* try to truncate, but ignore errors (will fail on block devices).
 276 +            * note1: bdrv_read() need whole blocks, so we need to round up
 277 +            * note2: PVE requires 1024 (BDRV_SECTOR_SIZE*2) alignment
 278 +            */
 279 +            size_t size = QEMU_ALIGN_UP(snap_state.bs_pos, BDRV_SECTOR_SIZE*2);
 280 +            blk_truncate(snap_state.target, size, false, PREALLOC_MODE_OFF, 0, NULL);
 281 +        }
 282 +        blk_op_unblock_all(snap_state.target, snap_state.blocker);
 283 +        error_free(snap_state.blocker);
 284 +        snap_state.blocker = NULL;
 285 +        blk_unref(snap_state.target);
 286 +        snap_state.target = NULL;
 287 +
 288 +        if (snap_state.target_close_wait) {
 289 +            qemu_co_sleep_wake(snap_state.target_close_wait);
 290 +        }
 291 +    }
 292 +
 293 +    return ret;
 294 +}
 295 +
 296 +static void save_snapshot_error(const char *fmt, ...)
 297 +{
 298 +    va_list ap;
 299 +    char *msg;
 300 +
 301 +    va_start(ap, fmt);
 302 +    msg = g_strdup_vprintf(fmt, ap);
 303 +    va_end(ap);
 304 +
 305 +    DPRINTF("save_snapshot_error: %s\n", msg);
 306 +
 307 +    if (!snap_state.error) {
 308 +        error_set(&snap_state.error, ERROR_CLASS_GENERIC_ERROR, "%s", msg);
 309 +    }
 310 +
 311 +    g_free (msg);
 312 +
 313 +    snap_state.state = SAVE_STATE_ERROR;
 314 +}
 315 +
 316 +static int block_state_close(void *opaque, Error **errp)
 317 +{
 318 +    snap_state.file = NULL;
 319 +    return blk_flush(snap_state.target);
 320 +}
 321 +
 322 +typedef struct BlkRwCo {
 323 +    int64_t offset;
 324 +    QEMUIOVector *qiov;
 325 +    ssize_t ret;
 326 +} BlkRwCo;
 327 +
 328 +static void coroutine_fn block_state_write_entry(void *opaque) {
 329 +    BlkRwCo *rwco = opaque;
 330 +    rwco->ret = blk_co_pwritev(snap_state.target, rwco->offset, rwco->qiov->size,
 331 +                               rwco->qiov, 0);
 332 +    aio_wait_kick();
 333 +}
 334 +
 335 +static ssize_t block_state_writev_buffer(void *opaque, struct iovec *iov,
 336 +                                         int iovcnt, int64_t pos, Error **errp)
 337 +{
 338 +    QEMUIOVector qiov;
 339 +    BlkRwCo rwco;
 340 +
 341 +    assert(pos == snap_state.bs_pos);
 342 +    rwco = (BlkRwCo) {
 343 +        .offset = pos,
 344 +        .qiov = &qiov,
 345 +        .ret = NOT_DONE,
 346 +    };
 347 +
 348 +    qemu_iovec_init_external(&qiov, iov, iovcnt);
 349 +
 350 +    if (qemu_in_coroutine()) {
 351 +        block_state_write_entry(&rwco);
 352 +    } else {
 353 +        Coroutine *co = qemu_coroutine_create(&block_state_write_entry, &rwco);
 354 +        bdrv_coroutine_enter(blk_bs(snap_state.target), co);
 355 +        BDRV_POLL_WHILE(blk_bs(snap_state.target), rwco.ret == NOT_DONE);
 356 +    }
 357 +    if (rwco.ret < 0) {
 358 +        return rwco.ret;
 359 +    }
 360 +
 361 +    snap_state.bs_pos += qiov.size;
 362 +    return qiov.size;
 363 +}
 364 +
 365 +static const QEMUFileOps block_file_ops = {
 366 +    .writev_buffer =  block_state_writev_buffer,
 367 +    .close =          block_state_close,
 368 +};
 369 +
 370 +static void process_savevm_finalize(void *opaque)
 371 +{
 372 +    int ret;
 373 +    AioContext *iohandler_ctx = iohandler_get_aio_context();
 374 +    MigrationState *ms = migrate_get_current();
 375 +
 376 +    bool aborted = savevm_aborted();
 377 +
 378 +#ifdef DEBUG_SAVEVM_STATE
 379 +    int64_t start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 380 +#endif
 381 +
 382 +    qemu_bh_delete(snap_state.finalize_bh);
 383 +    snap_state.finalize_bh = NULL;
 384 +    snap_state.co = NULL;
 385 +
 386 +    /* We need to own the target bdrv's context for the following functions,
 387 +     * so move it back. It can stay in the main context and live out its live
 388 +     * there, since we're done with it after this method ends anyway.
 389 +     */
 390 +    aio_context_acquire(iohandler_ctx);
 391 +    blk_set_aio_context(snap_state.target, qemu_get_aio_context(), NULL);
 392 +    aio_context_release(iohandler_ctx);
 393 +
 394 +    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
 395 +    if (ret < 0) {
 396 +        save_snapshot_error("vm_stop_force_state error %d", ret);
 397 +    }
 398 +
 399 +    if (!aborted) {
 400 +        /* skip state saving if we aborted, snapshot will be invalid anyway */
 401 +        (void)qemu_savevm_state_complete_precopy(snap_state.file, false, false);
 402 +        ret = qemu_file_get_error(snap_state.file);
 403 +        if (ret < 0) {
 404 +                save_snapshot_error("qemu_savevm_state_iterate error %d", ret);
 405 +        }
 406 +    }
 407 +
 408 +    DPRINTF("state saving complete\n");
 409 +    DPRINTF("timing: process_savevm_finalize (state saving) took %ld ms\n",
 410 +        qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - start_time);
 411 +
 412 +    /* clear migration state */
 413 +    migrate_set_state(&ms->state, MIGRATION_STATUS_SETUP,
 414 +        ret || aborted ? MIGRATION_STATUS_FAILED : MIGRATION_STATUS_COMPLETED);
 415 +    ms->to_dst_file = NULL;
 416 +
 417 +    qemu_savevm_state_cleanup();
 418 +
 419 +    ret = save_snapshot_cleanup();
 420 +    if (ret < 0) {
 421 +        save_snapshot_error("save_snapshot_cleanup error %d", ret);
 422 +    } else if (snap_state.state == SAVE_STATE_ACTIVE) {
 423 +        snap_state.state = SAVE_STATE_COMPLETED;
 424 +    } else if (aborted) {
 425 +        save_snapshot_error("process_savevm_cleanup: found aborted state: %d",
 426 +                            snap_state.state);
 427 +    } else {
 428 +        save_snapshot_error("process_savevm_cleanup: invalid state: %d",
 429 +                            snap_state.state);
 430 +    }
 431 +    if (snap_state.saved_vm_running) {
 432 +        vm_start();
 433 +        snap_state.saved_vm_running = false;
 434 +    }
 435 +
 436 +    DPRINTF("timing: process_savevm_finalize (full) took %ld ms\n",
 437 +        qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - start_time);
 438 +}
 439 +
 440 +static void coroutine_fn process_savevm_co(void *opaque)
 441 +{
 442 +    int ret;
 443 +    int64_t maxlen;
 444 +    BdrvNextIterator it;
 445 +    BlockDriverState *bs = NULL;
 446 +
 447 +#ifdef DEBUG_SAVEVM_STATE
 448 +    int64_t start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 449 +#endif
 450 +
 451 +    ret = qemu_file_get_error(snap_state.file);
 452 +    if (ret < 0) {
 453 +        save_snapshot_error("qemu_savevm_state_setup failed");
 454 +        return;
 455 +    }
 456 +
 457 +    while (snap_state.state == SAVE_STATE_ACTIVE) {
 458 +        uint64_t pending_size, pend_precopy, pend_compatible, pend_postcopy;
 459 +
 460 +        qemu_savevm_state_pending(snap_state.file, 0, &pend_precopy, &pend_compatible, &pend_postcopy);
 461 +        pending_size = pend_precopy + pend_compatible + pend_postcopy;
 462 +
 463 +        maxlen = blk_getlength(snap_state.target) - 30*1024*1024;
 464 +
 465 +        if (pending_size > 400000 && snap_state.bs_pos + pending_size < maxlen) {
 466 +            ret = qemu_savevm_state_iterate(snap_state.file, false);
 467 +            if (ret < 0) {
 468 +                save_snapshot_error("qemu_savevm_state_iterate error %d", ret);
 469 +                break;
 470 +            }
 471 +            DPRINTF("savevm iterate pending size %lu ret %d\n", pending_size, ret);
 472 +        } else {
 473 +            qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
 474 +            ret = global_state_store();
 475 +            if (ret) {
 476 +                save_snapshot_error("global_state_store error %d", ret);
 477 +                break;
 478 +            }
 479 +
 480 +            DPRINTF("savevm iterate complete\n");
 481 +            break;
 482 +        }
 483 +    }
 484 +
 485 +    DPRINTF("timing: process_savevm_co took %ld ms\n",
 486 +        qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - start_time);
 487 +
 488 +#ifdef DEBUG_SAVEVM_STATE
 489 +    int64_t start_time_flush = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 490 +#endif
 491 +    /* If a drive runs in an IOThread we can flush it async, and only
 492 +     * need to sync-flush whatever IO happens between now and
 493 +     * vm_stop_force_state. bdrv_next can only be called from main AioContext,
 494 +     * so move there now and after every flush.
 495 +     */
 496 +    aio_co_reschedule_self(qemu_get_aio_context());
 497 +    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
 498 +        /* target has BDRV_O_NO_FLUSH, no sense calling bdrv_flush on it */
 499 +        if (bs == blk_bs(snap_state.target)) {
 500 +            continue;
 501 +        }
 502 +
 503 +        AioContext *bs_ctx = bdrv_get_aio_context(bs);
 504 +        if (bs_ctx != qemu_get_aio_context()) {
 505 +            DPRINTF("savevm: async flushing drive %s\n", bs->filename);
 506 +            aio_co_reschedule_self(bs_ctx);
 507 +            bdrv_flush(bs);
 508 +            aio_co_reschedule_self(qemu_get_aio_context());
 509 +        }
 510 +    }
 511 +
 512 +    DPRINTF("timing: async flushing took %ld ms\n",
 513 +        qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - start_time_flush);
 514 +
 515 +    qemu_bh_schedule(snap_state.finalize_bh);
 516 +}
 517 +
 518 +void qmp_savevm_start(bool has_statefile, const char *statefile, Error **errp)
 519 +{
 520 +    Error *local_err = NULL;
 521 +    MigrationState *ms = migrate_get_current();
 522 +    AioContext *iohandler_ctx = iohandler_get_aio_context();
 523 +
 524 +    int bdrv_oflags = BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_NO_FLUSH;
 525 +
 526 +    if (snap_state.state != SAVE_STATE_DONE) {
 527 +        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
 528 +                  "VM snapshot already started\n");
 529 +        return;
 530 +    }
 531 +
 532 +    if (migration_is_running(ms->state)) {
 533 +        error_set(errp, ERROR_CLASS_GENERIC_ERROR, QERR_MIGRATION_ACTIVE);
 534 +        return;
 535 +    }
 536 +
 537 +    if (migrate_use_block()) {
 538 +        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
 539 +                  "Block migration and snapshots are incompatible");
 540 +        return;
 541 +    }
 542 +
 543 +    /* initialize snapshot info */
 544 +    snap_state.saved_vm_running = runstate_is_running();
 545 +    snap_state.bs_pos = 0;
 546 +    snap_state.total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 547 +    snap_state.blocker = NULL;
 548 +
 549 +    if (snap_state.error) {
 550 +        error_free(snap_state.error);
 551 +        snap_state.error = NULL;
 552 +    }
 553 +
 554 +    if (!has_statefile) {
 555 +        vm_stop(RUN_STATE_SAVE_VM);
 556 +        snap_state.state = SAVE_STATE_COMPLETED;
 557 +        return;
 558 +    }
 559 +
 560 +    if (qemu_savevm_state_blocked(errp)) {
 561 +        return;
 562 +    }
 563 +
 564 +    /* Open the image */
 565 +    QDict *options = NULL;
 566 +    options = qdict_new();
 567 +    qdict_put_str(options, "driver", "raw");
 568 +    snap_state.target = blk_new_open(statefile, NULL, options, bdrv_oflags, &local_err);
 569 +    if (!snap_state.target) {
 570 +        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "failed to open '%s'", statefile);
 571 +        goto restart;
 572 +    }
 573 +
 574 +    snap_state.file = qemu_fopen_ops(&snap_state, &block_file_ops);
 575 +
 576 +    if (!snap_state.file) {
 577 +        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "failed to open '%s'", statefile);
 578 +        goto restart;
 579 +    }
 580 +
 581 +    /*
 582 +     * qemu_savevm_* paths use migration code and expect a migration state.
 583 +     * State is cleared in process_savevm_co, but has to be initialized
 584 +     * here (blocking main thread, from QMP) to avoid race conditions.
 585 +     */
 586 +    migrate_init(ms);
 587 +    memset(&ram_counters, 0, sizeof(ram_counters));
 588 +    ms->to_dst_file = snap_state.file;
 589 +
 590 +    error_setg(&snap_state.blocker, "block device is in use by savevm");
 591 +    blk_op_block_all(snap_state.target, snap_state.blocker);
 592 +
 593 +    snap_state.state = SAVE_STATE_ACTIVE;
 594 +    snap_state.finalize_bh = qemu_bh_new(process_savevm_finalize, &snap_state);
 595 +    snap_state.co = qemu_coroutine_create(&process_savevm_co, NULL);
 596 +    qemu_mutex_unlock_iothread();
 597 +    qemu_savevm_state_header(snap_state.file);
 598 +    qemu_savevm_state_setup(snap_state.file);
 599 +    qemu_mutex_lock_iothread();
 600 +
 601 +    /* Async processing from here on out happens in iohandler context, so let
 602 +     * the target bdrv have its home there.
 603 +     */
 604 +    blk_set_aio_context(snap_state.target, iohandler_ctx, &local_err);
 605 +
 606 +    aio_co_schedule(iohandler_ctx, snap_state.co);
 607 +
 608 +    return;
 609 +
 610 +restart:
 611 +
 612 +    save_snapshot_error("setup failed");
 613 +
 614 +    if (snap_state.saved_vm_running) {
 615 +        vm_start();
 616 +        snap_state.saved_vm_running = false;
 617 +    }
 618 +}
 619 +
 620 +void coroutine_fn qmp_savevm_end(Error **errp)
 621 +{
 622 +    int64_t timeout;
 623 +
 624 +    if (snap_state.state == SAVE_STATE_DONE) {
 625 +        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
 626 +                  "VM snapshot not started\n");
 627 +        return;
 628 +    }
 629 +
 630 +    if (snap_state.state == SAVE_STATE_ACTIVE) {
 631 +        snap_state.state = SAVE_STATE_CANCELLED;
 632 +        goto wait_for_close;
 633 +    }
 634 +
 635 +    if (snap_state.saved_vm_running) {
 636 +        vm_start();
 637 +        snap_state.saved_vm_running = false;
 638 +    }
 639 +
 640 +    snap_state.state = SAVE_STATE_DONE;
 641 +
 642 +wait_for_close:
 643 +    if (!snap_state.target) {
 644 +        DPRINTF("savevm-end: no target file open\n");
 645 +        return;
 646 +    }
 647 +
 648 +    /* wait until cleanup is done before returning, this ensures that after this
 649 +     * call exits the statefile will be closed and can be removed immediately */
 650 +    DPRINTF("savevm-end: waiting for cleanup\n");
 651 +    timeout = 30L * 1000 * 1000 * 1000;
 652 +    qemu_co_sleep_ns_wakeable(QEMU_CLOCK_REALTIME, timeout,
 653 +                              &snap_state.target_close_wait);
 654 +    snap_state.target_close_wait = NULL;
 655 +    if (snap_state.target) {
 656 +        save_snapshot_error("timeout waiting for target file close in "
 657 +                            "qmp_savevm_end");
 658 +        /* we cannot assume the snapshot finished in this case, so leave the
 659 +         * state alone - caller has to figure something out */
 660 +        return;
 661 +    }
 662 +
 663 +    DPRINTF("savevm-end: cleanup done\n");
 664 +}
 665 +
 666 +// FIXME: Deprecated
 667 +void qmp_snapshot_drive(const char *device, const char *name, Error **errp)
 668 +{
 669 +    // Compatibility to older qemu-server.
 670 +    qmp_blockdev_snapshot_internal_sync(device, name, errp);
 671 +}
 672 +
 673 +// FIXME: Deprecated
 674 +void qmp_delete_drive_snapshot(const char *device, const char *name,
 675 +                               Error **errp)
 676 +{
 677 +    // Compatibility to older qemu-server.
 678 +    (void)qmp_blockdev_snapshot_delete_internal_sync(device, false, NULL,
 679 +                                                     true, name, errp);
 680 +}
 681 +
 682 +static ssize_t loadstate_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
 683 +                                    size_t size, Error **errp)
 684 +{
 685 +    BlockBackend *be = opaque;
 686 +    int64_t maxlen = blk_getlength(be);
 687 +    if (pos > maxlen) {
 688 +        return -EIO;
 689 +    }
 690 +    if ((pos + size) > maxlen) {
 691 +        size = maxlen - pos - 1;
 692 +    }
 693 +    if (size == 0) {
 694 +        return 0;
 695 +    }
 696 +    return blk_pread(be, pos, buf, size);
 697 +}
 698 +
 699 +static const QEMUFileOps loadstate_file_ops = {
 700 +    .get_buffer = loadstate_get_buffer,
 701 +};
 702 +
 703 +int load_snapshot_from_blockdev(const char *filename, Error **errp)
 704 +{
 705 +    BlockBackend *be;
 706 +    Error *local_err = NULL;
 707 +    Error *blocker = NULL;
 708 +
 709 +    QEMUFile *f;
 710 +    int ret = -EINVAL;
 711 +
 712 +    be = blk_new_open(filename, NULL, NULL, 0, &local_err);
 713 +
 714 +    if (!be) {
 715 +        error_setg(errp, "Could not open VM state file");
 716 +        goto the_end;
 717 +    }
 718 +
 719 +    error_setg(&blocker, "block device is in use by load state");
 720 +    blk_op_block_all(be, blocker);
 721 +
 722 +    /* restore the VM state */
 723 +    f = qemu_fopen_ops(be, &loadstate_file_ops);
 724 +    if (!f) {
 725 +        error_setg(errp, "Could not open VM state file");
 726 +        goto the_end;
 727 +    }
 728 +
 729 +    qemu_system_reset(SHUTDOWN_CAUSE_NONE);
 730 +    ret = qemu_loadvm_state(f);
 731 +
 732 +    qemu_fclose(f);
 733 +    migration_incoming_state_destroy();
 734 +    if (ret < 0) {
 735 +        error_setg_errno(errp, -ret, "Error while loading VM state");
 736 +        goto the_end;
 737 +    }
 738 +
 739 +    ret = 0;
 740 +
 741 + the_end:
 742 +    if (be) {
 743 +        blk_op_unblock_all(be, blocker);
 744 +        error_free(blocker);
 745 +        blk_unref(be);
 746 +    }
 747 +    return ret;
 748 +}
 749 diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
 750 index 705f08a8f1..77ab152aab 100644
 751 --- a/monitor/hmp-cmds.c
 752 +++ b/monitor/hmp-cmds.c
 753 @@ -1949,6 +1949,63 @@ void hmp_info_memory_devices(Monitor *mon, const QDict *qdict)
 754      hmp_handle_error(mon, err);
 755  }
 756
 757 +void hmp_savevm_start(Monitor *mon, const QDict *qdict)
 758 +{
 759 +    Error *errp = NULL;
 760 +    const char *statefile = qdict_get_try_str(qdict, "statefile");
 761 +
 762 +    qmp_savevm_start(statefile != NULL, statefile, &errp);
 763 +    hmp_handle_error(mon, errp);
 764 +}
 765 +
 766 +void hmp_snapshot_drive(Monitor *mon, const QDict *qdict)
 767 +{
 768 +    Error *errp = NULL;
 769 +    const char *name = qdict_get_str(qdict, "name");
 770 +    const char *device = qdict_get_str(qdict, "device");
 771 +
 772 +    qmp_snapshot_drive(device, name, &errp);
 773 +    hmp_handle_error(mon, errp);
 774 +}
 775 +
 776 +void hmp_delete_drive_snapshot(Monitor *mon, const QDict *qdict)
 777 +{
 778 +    Error *errp = NULL;
 779 +    const char *name = qdict_get_str(qdict, "name");
 780 +    const char *device = qdict_get_str(qdict, "device");
 781 +
 782 +    qmp_delete_drive_snapshot(device, name, &errp);
 783 +    hmp_handle_error(mon, errp);
 784 +}
 785 +
 786 +void coroutine_fn hmp_savevm_end(Monitor *mon, const QDict *qdict)
 787 +{
 788 +    Error *errp = NULL;
 789 +
 790 +    qmp_savevm_end(&errp);
 791 +    hmp_handle_error(mon, errp);
 792 +}
 793 +
 794 +void hmp_info_savevm(Monitor *mon, const QDict *qdict)
 795 +{
 796 +    SaveVMInfo *info;
 797 +    info = qmp_query_savevm(NULL);
 798 +
 799 +    if (info->has_status) {
 800 +        monitor_printf(mon, "savevm status: %s\n", info->status);
 801 +        monitor_printf(mon, "total time: %" PRIu64 " milliseconds\n",
 802 +                       info->total_time);
 803 +    } else {
 804 +        monitor_printf(mon, "savevm status: not running\n");
 805 +    }
 806 +    if (info->has_bytes) {
 807 +        monitor_printf(mon, "Bytes saved: %"PRIu64"\n", info->bytes);
 808 +    }
 809 +    if (info->has_error) {
 810 +        monitor_printf(mon, "Error: %s\n", info->error);
 811 +    }
 812 +}
 813 +
 814  void hmp_info_iothreads(Monitor *mon, const QDict *qdict)
 815  {
 816      IOThreadInfoList *info_list = qmp_query_iothreads(NULL);
 817 diff --git a/qapi/migration.json b/qapi/migration.json
 818 index 3c75820527..cb3627884c 100644
 819 --- a/qapi/migration.json
 820 +++ b/qapi/migration.json
 821 @@ -242,6 +242,40 @@
 822             '*compression': 'CompressionStats',
 823             '*socket-address': ['SocketAddress'] } }
 824
 825 +##
 826 +# @SaveVMInfo:
 827 +#
 828 +# Information about current migration process.
 829 +#
 830 +# @status: string describing the current savevm status.
 831 +#          This can be 'active', 'completed', 'failed'.
 832 +#          If this field is not returned, no savevm process
 833 +#          has been initiated
 834 +#
 835 +# @error: string containing error message is status is failed.
 836 +#
 837 +# @total-time: total amount of milliseconds since savevm started.
 838 +#              If savevm has ended, it returns the total save time
 839 +#
 840 +# @bytes: total amount of data transfered
 841 +#
 842 +# Since: 1.3
 843 +##
 844 +{ 'struct': 'SaveVMInfo',
 845 +  'data': {'*status': 'str', '*error': 'str',
 846 +           '*total-time': 'int', '*bytes': 'int'} }
 847 +
 848 +##
 849 +# @query-savevm:
 850 +#
 851 +# Returns information about current savevm process.
 852 +#
 853 +# Returns: @SaveVMInfo
 854 +#
 855 +# Since: 1.3
 856 +##
 857 +{ 'command': 'query-savevm', 'returns': 'SaveVMInfo' }
 858 +
 859  ##
 860  # @query-migrate:
 861  #
 862 diff --git a/qapi/misc.json b/qapi/misc.json
 863 index 40df513856..4f5333d960 100644
 864 --- a/qapi/misc.json
 865 +++ b/qapi/misc.json
 866 @@ -476,6 +476,38 @@
 867  ##
 868  { 'command': 'query-fdsets', 'returns': ['FdsetInfo'] }
 869
 870 +##
 871 +# @savevm-start:
 872 +#
 873 +# Prepare for snapshot and halt VM. Save VM state to statefile.
 874 +#
 875 +##
 876 +{ 'command': 'savevm-start', 'data': { '*statefile': 'str' } }
 877 +
 878 +##
 879 +# @snapshot-drive:
 880 +#
 881 +# Create an internal drive snapshot.
 882 +#
 883 +##
 884 +{ 'command': 'snapshot-drive', 'data': { 'device': 'str', 'name': 'str' } }
 885 +
 886 +##
 887 +# @delete-drive-snapshot:
 888 +#
 889 +# Delete a drive snapshot.
 890 +#
 891 +##
 892 +{ 'command': 'delete-drive-snapshot', 'data': { 'device': 'str', 'name': 'str' } }
 893 +
 894 +##
 895 +# @savevm-end:
 896 +#
 897 +# Resume VM after a snapshot.
 898 +#
 899 +##
 900 +{ 'command': 'savevm-end', 'coroutine': true }
 901 +
 902  ##
 903  # @CommandLineParameterType:
 904  #
 905 diff --git a/qemu-options.hx b/qemu-options.hx
 906 index 104632ea34..c1352312c2 100644
 907 --- a/qemu-options.hx
 908 +++ b/qemu-options.hx
 909 @@ -3903,6 +3903,18 @@ SRST
 910      Start right away with a saved state (``loadvm`` in monitor)
 911  ERST
 912
 913 +DEF("loadstate", HAS_ARG, QEMU_OPTION_loadstate, \
 914 +    "-loadstate file\n" \
 915 +    "                start right away with a saved state\n",
 916 +    QEMU_ARCH_ALL)
 917 +SRST
 918 +``-loadstate file``
 919 +  Start right away with a saved state. This option does not rollback
 920 +  disk state like @code{loadvm}, so user must make sure that disk
 921 +  have correct state. @var{file} can be any valid device URL. See the section
 922 +  for "Device URL Syntax" for more information.
 923 +ERST
 924 +
 925  #ifndef _WIN32
 926  DEF("daemonize", 0, QEMU_OPTION_daemonize, \
 927      "-daemonize      daemonize QEMU after initializing\n", QEMU_ARCH_ALL)
 928 diff --git a/softmmu/vl.c b/softmmu/vl.c
 929 index e6e0ad5a92..03152c816c 100644
 930 --- a/softmmu/vl.c
 931 +++ b/softmmu/vl.c
 932 @@ -2878,6 +2878,7 @@ void qemu_init(int argc, char **argv, char **envp)
 933      int optind;
 934      const char *optarg;
 935      const char *loadvm = NULL;
 936 +    const char *loadstate = NULL;
 937      MachineClass *machine_class;
 938      const char *cpu_option;
 939      const char *vga_model = NULL;
 940 @@ -3439,6 +3440,9 @@ void qemu_init(int argc, char **argv, char **envp)
 941              case QEMU_OPTION_loadvm:
 942                  loadvm = optarg;
 943                  break;
 944 +            case QEMU_OPTION_loadstate:
 945 +                loadstate = optarg;
 946 +                break;
 947              case QEMU_OPTION_full_screen:
 948                  dpy.has_full_screen = true;
 949                  dpy.full_screen = true;
 950 @@ -4478,6 +4482,12 @@ void qemu_init(int argc, char **argv, char **envp)
 951              autostart = 0;
 952              exit(1);
 953          }
 954 +    } else if (loadstate) {
 955 +        Error *local_err = NULL;
 956 +        if (load_snapshot_from_blockdev(loadstate, &local_err) < 0) {
 957 +            error_report_err(local_err);
 958 +            autostart = 0;
 959 +        }
 960      }
 961      if (replay_mode != REPLAY_MODE_NONE) {
 962          replay_vmstate_init();