migration/block.c

   1 /*
   2  * QEMU live block migration
   3  *
   4  * Copyright IBM, Corp. 2009
   5  *
   6  * Authors:
   7  *  Liran Schour   <lirans@il.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qapi/error.h"
  18 #include "qemu-common.h"
  19 #include "block/block.h"
  20 #include "qemu/error-report.h"
  21 #include "qemu/main-loop.h"
  22 #include "hw/hw.h"
  23 #include "qemu/queue.h"
  24 #include "qemu/timer.h"
  25 #include "migration/block.h"
  26 #include "migration/migration.h"
  27 #include "sysemu/blockdev.h"
  28 #include "sysemu/block-backend.h"
  29
  30 #define BLOCK_SIZE                       (1 << 20)
  31 #define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
  32
  33 #define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
  34 #define BLK_MIG_FLAG_EOS                0x02
  35 #define BLK_MIG_FLAG_PROGRESS           0x04
  36 #define BLK_MIG_FLAG_ZERO_BLOCK         0x08
  37
  38 #define MAX_IS_ALLOCATED_SEARCH 65536
  39
  40 #define MAX_INFLIGHT_IO 512
  41
  42 //#define DEBUG_BLK_MIGRATION
  43
  44 #ifdef DEBUG_BLK_MIGRATION
  45 #define DPRINTF(fmt, ...) \
  46     do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0)
  47 #else
  48 #define DPRINTF(fmt, ...) \
  49     do { } while (0)
  50 #endif
  51
  52 typedef struct BlkMigDevState {
  53     /* Written during setup phase.  Can be read without a lock.  */
  54     BlockDriverState *bs;
  55     int shared_base;
  56     int64_t total_sectors;
  57     QSIMPLEQ_ENTRY(BlkMigDevState) entry;
  58     Error *blocker;
  59
  60     /* Only used by migration thread.  Does not need a lock.  */
  61     int bulk_completed;
  62     int64_t cur_sector;
  63     int64_t cur_dirty;
  64
  65     /* Data in the aio_bitmap is protected by block migration lock.
  66      * Allocation and free happen during setup and cleanup respectively.
  67      */
  68     unsigned long *aio_bitmap;
  69
  70     /* Protected by block migration lock.  */
  71     int64_t completed_sectors;
  72
  73     /* During migration this is protected by iothread lock / AioContext.
  74      * Allocation and free happen during setup and cleanup respectively.
  75      */
  76     BdrvDirtyBitmap *dirty_bitmap;
  77 } BlkMigDevState;
  78
  79 typedef struct BlkMigBlock {
  80     /* Only used by migration thread.  */
  81     uint8_t *buf;
  82     BlkMigDevState *bmds;
  83     int64_t sector;
  84     int nr_sectors;
  85     struct iovec iov;
  86     QEMUIOVector qiov;
  87     BlockAIOCB *aiocb;
  88
  89     /* Protected by block migration lock.  */
  90     int ret;
  91     QSIMPLEQ_ENTRY(BlkMigBlock) entry;
  92 } BlkMigBlock;
  93
  94 typedef struct BlkMigState {
  95     /* Written during setup phase.  Can be read without a lock.  */
  96     int blk_enable;
  97     int shared_base;
  98     QSIMPLEQ_HEAD(bmds_list, BlkMigDevState) bmds_list;
  99     int64_t total_sector_sum;
 100     bool zero_blocks;
 101
 102     /* Protected by lock.  */
 103     QSIMPLEQ_HEAD(blk_list, BlkMigBlock) blk_list;
 104     int submitted;
 105     int read_done;
 106
 107     /* Only used by migration thread.  Does not need a lock.  */
 108     int transferred;
 109     int prev_progress;
 110     int bulk_completed;
 111
 112     /* Lock must be taken _inside_ the iothread lock and any AioContexts.  */
 113     QemuMutex lock;
 114 } BlkMigState;
 115
 116 static BlkMigState block_mig_state;
 117
 118 static void blk_mig_lock(void)
 119 {
 120     qemu_mutex_lock(&block_mig_state.lock);
 121 }
 122
 123 static void blk_mig_unlock(void)
 124 {
 125     qemu_mutex_unlock(&block_mig_state.lock);
 126 }
 127
 128 /* Must run outside of the iothread lock during the bulk phase,
 129  * or the VM will stall.
 130  */
 131
 132 static void blk_send(QEMUFile *f, BlkMigBlock * blk)
 133 {
 134     int len;
 135     uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;
 136
 137     if (block_mig_state.zero_blocks &&
 138         buffer_is_zero(blk->buf, BLOCK_SIZE)) {
 139         flags |= BLK_MIG_FLAG_ZERO_BLOCK;
 140     }
 141
 142     /* sector number and flags */
 143     qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS)
 144                      | flags);
 145
 146     /* device name */
 147     len = strlen(bdrv_get_device_name(blk->bmds->bs));
 148     qemu_put_byte(f, len);
 149     qemu_put_buffer(f, (uint8_t *)bdrv_get_device_name(blk->bmds->bs), len);
 150
 151     /* if a block is zero we need to flush here since the network
 152      * bandwidth is now a lot higher than the storage device bandwidth.
 153      * thus if we queue zero blocks we slow down the migration */
 154     if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
 155         qemu_fflush(f);
 156         return;
 157     }
 158
 159     qemu_put_buffer(f, blk->buf, BLOCK_SIZE);
 160 }
 161
 162 int blk_mig_active(void)
 163 {
 164     return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list);
 165 }
 166
 167 uint64_t blk_mig_bytes_transferred(void)
 168 {
 169     BlkMigDevState *bmds;
 170     uint64_t sum = 0;
 171
 172     blk_mig_lock();
 173     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 174         sum += bmds->completed_sectors;
 175     }
 176     blk_mig_unlock();
 177     return sum << BDRV_SECTOR_BITS;
 178 }
 179
 180 uint64_t blk_mig_bytes_remaining(void)
 181 {
 182     return blk_mig_bytes_total() - blk_mig_bytes_transferred();
 183 }
 184
 185 uint64_t blk_mig_bytes_total(void)
 186 {
 187     BlkMigDevState *bmds;
 188     uint64_t sum = 0;
 189
 190     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 191         sum += bmds->total_sectors;
 192     }
 193     return sum << BDRV_SECTOR_BITS;
 194 }
 195
 196
 197 /* Called with migration lock held.  */
 198
 199 static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
 200 {
 201     int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
 202
 203     if (sector < bdrv_nb_sectors(bmds->bs)) {
 204         return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
 205             (1UL << (chunk % (sizeof(unsigned long) * 8))));
 206     } else {
 207         return 0;
 208     }
 209 }
 210
 211 /* Called with migration lock held.  */
 212
 213 static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num,
 214                              int nb_sectors, int set)
 215 {
 216     int64_t start, end;
 217     unsigned long val, idx, bit;
 218
 219     start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
 220     end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
 221
 222     for (; start <= end; start++) {
 223         idx = start / (sizeof(unsigned long) * 8);
 224         bit = start % (sizeof(unsigned long) * 8);
 225         val = bmds->aio_bitmap[idx];
 226         if (set) {
 227             val |= 1UL << bit;
 228         } else {
 229             val &= ~(1UL << bit);
 230         }
 231         bmds->aio_bitmap[idx] = val;
 232     }
 233 }
 234
 235 static void alloc_aio_bitmap(BlkMigDevState *bmds)
 236 {
 237     BlockDriverState *bs = bmds->bs;
 238     int64_t bitmap_size;
 239
 240     bitmap_size = bdrv_nb_sectors(bs) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
 241     bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;
 242
 243     bmds->aio_bitmap = g_malloc0(bitmap_size);
 244 }
 245
 246 /* Never hold migration lock when yielding to the main loop!  */
 247
 248 static void blk_mig_read_cb(void *opaque, int ret)
 249 {
 250     BlkMigBlock *blk = opaque;
 251
 252     blk_mig_lock();
 253     blk->ret = ret;
 254
 255     QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
 256     bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0);
 257
 258     block_mig_state.submitted--;
 259     block_mig_state.read_done++;
 260     assert(block_mig_state.submitted >= 0);
 261     blk_mig_unlock();
 262 }
 263
 264 /* Called with no lock taken.  */
 265
 266 static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
 267 {
 268     int64_t total_sectors = bmds->total_sectors;
 269     int64_t cur_sector = bmds->cur_sector;
 270     BlockDriverState *bs = bmds->bs;
 271     BlkMigBlock *blk;
 272     int nr_sectors;
 273
 274     if (bmds->shared_base) {
 275         qemu_mutex_lock_iothread();
 276         aio_context_acquire(bdrv_get_aio_context(bs));
 277         while (cur_sector < total_sectors &&
 278                !bdrv_is_allocated(bs, cur_sector, MAX_IS_ALLOCATED_SEARCH,
 279                                   &nr_sectors)) {
 280             cur_sector += nr_sectors;
 281         }
 282         aio_context_release(bdrv_get_aio_context(bs));
 283         qemu_mutex_unlock_iothread();
 284     }
 285
 286     if (cur_sector >= total_sectors) {
 287         bmds->cur_sector = bmds->completed_sectors = total_sectors;
 288         return 1;
 289     }
 290
 291     bmds->completed_sectors = cur_sector;
 292
 293     cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);
 294
 295     /* we are going to transfer a full block even if it is not allocated */
 296     nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 297
 298     if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 299         nr_sectors = total_sectors - cur_sector;
 300     }
 301
 302     blk = g_new(BlkMigBlock, 1);
 303     blk->buf = g_malloc(BLOCK_SIZE);
 304     blk->bmds = bmds;
 305     blk->sector = cur_sector;
 306     blk->nr_sectors = nr_sectors;
 307
 308     blk->iov.iov_base = blk->buf;
 309     blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
 310     qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
 311
 312     blk_mig_lock();
 313     block_mig_state.submitted++;
 314     blk_mig_unlock();
 315
 316     /* We do not know if bs is under the main thread (and thus does
 317      * not acquire the AioContext when doing AIO) or rather under
 318      * dataplane.  Thus acquire both the iothread mutex and the
 319      * AioContext.
 320      *
 321      * This is ugly and will disappear when we make bdrv_* thread-safe,
 322      * without the need to acquire the AioContext.
 323      */
 324     qemu_mutex_lock_iothread();
 325     aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 326     blk->aiocb = bdrv_aio_readv(bs, cur_sector, &blk->qiov,
 327                                 nr_sectors, blk_mig_read_cb, blk);
 328
 329     bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, cur_sector, nr_sectors);
 330     aio_context_release(bdrv_get_aio_context(bmds->bs));
 331     qemu_mutex_unlock_iothread();
 332
 333     bmds->cur_sector = cur_sector + nr_sectors;
 334     return (bmds->cur_sector >= total_sectors);
 335 }
 336
 337 /* Called with iothread lock taken.  */
 338
 339 static int set_dirty_tracking(void)
 340 {
 341     BlkMigDevState *bmds;
 342     int ret;
 343
 344     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 345         aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 346         bmds->dirty_bitmap = bdrv_create_dirty_bitmap(bmds->bs, BLOCK_SIZE,
 347                                                       NULL, NULL);
 348         aio_context_release(bdrv_get_aio_context(bmds->bs));
 349         if (!bmds->dirty_bitmap) {
 350             ret = -errno;
 351             goto fail;
 352         }
 353     }
 354     return 0;
 355
 356 fail:
 357     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 358         if (bmds->dirty_bitmap) {
 359             aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 360             bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
 361             aio_context_release(bdrv_get_aio_context(bmds->bs));
 362         }
 363     }
 364     return ret;
 365 }
 366
 367 /* Called with iothread lock taken.  */
 368
 369 static void unset_dirty_tracking(void)
 370 {
 371     BlkMigDevState *bmds;
 372
 373     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 374         aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 375         bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
 376         aio_context_release(bdrv_get_aio_context(bmds->bs));
 377     }
 378 }
 379
 380 static void init_blk_migration(QEMUFile *f)
 381 {
 382     BlockDriverState *bs;
 383     BlkMigDevState *bmds;
 384     int64_t sectors;
 385
 386     block_mig_state.submitted = 0;
 387     block_mig_state.read_done = 0;
 388     block_mig_state.transferred = 0;
 389     block_mig_state.total_sector_sum = 0;
 390     block_mig_state.prev_progress = -1;
 391     block_mig_state.bulk_completed = 0;
 392     block_mig_state.zero_blocks = migrate_zero_blocks();
 393
 394     for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
 395         if (bdrv_is_read_only(bs)) {
 396             continue;
 397         }
 398
 399         sectors = bdrv_nb_sectors(bs);
 400         if (sectors <= 0) {
 401             return;
 402         }
 403
 404         bmds = g_new0(BlkMigDevState, 1);
 405         bmds->bs = bs;
 406         bmds->bulk_completed = 0;
 407         bmds->total_sectors = sectors;
 408         bmds->completed_sectors = 0;
 409         bmds->shared_base = block_mig_state.shared_base;
 410         alloc_aio_bitmap(bmds);
 411         error_setg(&bmds->blocker, "block device is in use by migration");
 412         bdrv_op_block_all(bs, bmds->blocker);
 413         bdrv_ref(bs);
 414
 415         block_mig_state.total_sector_sum += sectors;
 416
 417         if (bmds->shared_base) {
 418             DPRINTF("Start migration for %s with shared base image\n",
 419                     bdrv_get_device_name(bs));
 420         } else {
 421             DPRINTF("Start full migration for %s\n", bdrv_get_device_name(bs));
 422         }
 423
 424         QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
 425     }
 426 }
 427
 428 /* Called with no lock taken.  */
 429
 430 static int blk_mig_save_bulked_block(QEMUFile *f)
 431 {
 432     int64_t completed_sector_sum = 0;
 433     BlkMigDevState *bmds;
 434     int progress;
 435     int ret = 0;
 436
 437     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 438         if (bmds->bulk_completed == 0) {
 439             if (mig_save_device_bulk(f, bmds) == 1) {
 440                 /* completed bulk section for this device */
 441                 bmds->bulk_completed = 1;
 442             }
 443             completed_sector_sum += bmds->completed_sectors;
 444             ret = 1;
 445             break;
 446         } else {
 447             completed_sector_sum += bmds->completed_sectors;
 448         }
 449     }
 450
 451     if (block_mig_state.total_sector_sum != 0) {
 452         progress = completed_sector_sum * 100 /
 453                    block_mig_state.total_sector_sum;
 454     } else {
 455         progress = 100;
 456     }
 457     if (progress != block_mig_state.prev_progress) {
 458         block_mig_state.prev_progress = progress;
 459         qemu_put_be64(f, (progress << BDRV_SECTOR_BITS)
 460                          | BLK_MIG_FLAG_PROGRESS);
 461         DPRINTF("Completed %d %%\r", progress);
 462     }
 463
 464     return ret;
 465 }
 466
 467 static void blk_mig_reset_dirty_cursor(void)
 468 {
 469     BlkMigDevState *bmds;
 470
 471     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 472         bmds->cur_dirty = 0;
 473     }
 474 }
 475
 476 /* Called with iothread lock and AioContext taken.  */
 477
 478 static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
 479                                  int is_async)
 480 {
 481     BlkMigBlock *blk;
 482     int64_t total_sectors = bmds->total_sectors;
 483     int64_t sector;
 484     int nr_sectors;
 485     int ret = -EIO;
 486
 487     for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) {
 488         blk_mig_lock();
 489         if (bmds_aio_inflight(bmds, sector)) {
 490             blk_mig_unlock();
 491             bdrv_drain(bmds->bs);
 492         } else {
 493             blk_mig_unlock();
 494         }
 495         if (bdrv_get_dirty(bmds->bs, bmds->dirty_bitmap, sector)) {
 496
 497             if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 498                 nr_sectors = total_sectors - sector;
 499             } else {
 500                 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 501             }
 502             blk = g_new(BlkMigBlock, 1);
 503             blk->buf = g_malloc(BLOCK_SIZE);
 504             blk->bmds = bmds;
 505             blk->sector = sector;
 506             blk->nr_sectors = nr_sectors;
 507
 508             if (is_async) {
 509                 blk->iov.iov_base = blk->buf;
 510                 blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
 511                 qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
 512
 513                 blk->aiocb = bdrv_aio_readv(bmds->bs, sector, &blk->qiov,
 514                                             nr_sectors, blk_mig_read_cb, blk);
 515
 516                 blk_mig_lock();
 517                 block_mig_state.submitted++;
 518                 bmds_set_aio_inflight(bmds, sector, nr_sectors, 1);
 519                 blk_mig_unlock();
 520             } else {
 521                 ret = bdrv_read(bmds->bs, sector, blk->buf, nr_sectors);
 522                 if (ret < 0) {
 523                     goto error;
 524                 }
 525                 blk_send(f, blk);
 526
 527                 g_free(blk->buf);
 528                 g_free(blk);
 529             }
 530
 531             bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, sector, nr_sectors);
 532             break;
 533         }
 534         sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
 535         bmds->cur_dirty = sector;
 536     }
 537
 538     return (bmds->cur_dirty >= bmds->total_sectors);
 539
 540 error:
 541     DPRINTF("Error reading sector %" PRId64 "\n", sector);
 542     g_free(blk->buf);
 543     g_free(blk);
 544     return ret;
 545 }
 546
 547 /* Called with iothread lock taken.
 548  *
 549  * return value:
 550  * 0: too much data for max_downtime
 551  * 1: few enough data for max_downtime
 552 */
 553 static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
 554 {
 555     BlkMigDevState *bmds;
 556     int ret = 1;
 557
 558     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 559         aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 560         ret = mig_save_device_dirty(f, bmds, is_async);
 561         aio_context_release(bdrv_get_aio_context(bmds->bs));
 562         if (ret <= 0) {
 563             break;
 564         }
 565     }
 566
 567     return ret;
 568 }
 569
 570 /* Called with no locks taken.  */
 571
 572 static int flush_blks(QEMUFile *f)
 573 {
 574     BlkMigBlock *blk;
 575     int ret = 0;
 576
 577     DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
 578             __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
 579             block_mig_state.transferred);
 580
 581     blk_mig_lock();
 582     while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
 583         if (qemu_file_rate_limit(f)) {
 584             break;
 585         }
 586         if (blk->ret < 0) {
 587             ret = blk->ret;
 588             break;
 589         }
 590
 591         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
 592         blk_mig_unlock();
 593         blk_send(f, blk);
 594         blk_mig_lock();
 595
 596         g_free(blk->buf);
 597         g_free(blk);
 598
 599         block_mig_state.read_done--;
 600         block_mig_state.transferred++;
 601         assert(block_mig_state.read_done >= 0);
 602     }
 603     blk_mig_unlock();
 604
 605     DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
 606             block_mig_state.submitted, block_mig_state.read_done,
 607             block_mig_state.transferred);
 608     return ret;
 609 }
 610
 611 /* Called with iothread lock taken.  */
 612
 613 static int64_t get_remaining_dirty(void)
 614 {
 615     BlkMigDevState *bmds;
 616     int64_t dirty = 0;
 617
 618     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 619         aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 620         dirty += bdrv_get_dirty_count(bmds->dirty_bitmap);
 621         aio_context_release(bdrv_get_aio_context(bmds->bs));
 622     }
 623
 624     return dirty << BDRV_SECTOR_BITS;
 625 }
 626
 627 /* Called with iothread lock taken.  */
 628
 629 static void block_migration_cleanup(void *opaque)
 630 {
 631     BlkMigDevState *bmds;
 632     BlkMigBlock *blk;
 633     AioContext *ctx;
 634
 635     bdrv_drain_all();
 636
 637     unset_dirty_tracking();
 638
 639     while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
 640         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
 641         bdrv_op_unblock_all(bmds->bs, bmds->blocker);
 642         error_free(bmds->blocker);
 643
 644         /* Save ctx, because bmds->bs can disappear during bdrv_unref.  */
 645         ctx = bdrv_get_aio_context(bmds->bs);
 646         aio_context_acquire(ctx);
 647         bdrv_unref(bmds->bs);
 648         aio_context_release(ctx);
 649
 650         g_free(bmds->aio_bitmap);
 651         g_free(bmds);
 652     }
 653
 654     blk_mig_lock();
 655     while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
 656         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
 657         g_free(blk->buf);
 658         g_free(blk);
 659     }
 660     blk_mig_unlock();
 661 }
 662
 663 static int block_save_setup(QEMUFile *f, void *opaque)
 664 {
 665     int ret;
 666
 667     DPRINTF("Enter save live setup submitted %d transferred %d\n",
 668             block_mig_state.submitted, block_mig_state.transferred);
 669
 670     qemu_mutex_lock_iothread();
 671     init_blk_migration(f);
 672
 673     /* start track dirty blocks */
 674     ret = set_dirty_tracking();
 675
 676     qemu_mutex_unlock_iothread();
 677
 678     if (ret) {
 679         return ret;
 680     }
 681
 682     ret = flush_blks(f);
 683     blk_mig_reset_dirty_cursor();
 684     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 685
 686     return ret;
 687 }
 688
 689 static int block_save_iterate(QEMUFile *f, void *opaque)
 690 {
 691     int ret;
 692     int64_t last_ftell = qemu_ftell(f);
 693     int64_t delta_ftell;
 694
 695     DPRINTF("Enter save live iterate submitted %d transferred %d\n",
 696             block_mig_state.submitted, block_mig_state.transferred);
 697
 698     ret = flush_blks(f);
 699     if (ret) {
 700         return ret;
 701     }
 702
 703     blk_mig_reset_dirty_cursor();
 704
 705     /* control the rate of transfer */
 706     blk_mig_lock();
 707     while ((block_mig_state.submitted +
 708             block_mig_state.read_done) * BLOCK_SIZE <
 709            qemu_file_get_rate_limit(f) &&
 710            (block_mig_state.submitted +
 711             block_mig_state.read_done) <
 712            MAX_INFLIGHT_IO) {
 713         blk_mig_unlock();
 714         if (block_mig_state.bulk_completed == 0) {
 715             /* first finish the bulk phase */
 716             if (blk_mig_save_bulked_block(f) == 0) {
 717                 /* finished saving bulk on all devices */
 718                 block_mig_state.bulk_completed = 1;
 719             }
 720             ret = 0;
 721         } else {
 722             /* Always called with iothread lock taken for
 723              * simplicity, block_save_complete also calls it.
 724              */
 725             qemu_mutex_lock_iothread();
 726             ret = blk_mig_save_dirty_block(f, 1);
 727             qemu_mutex_unlock_iothread();
 728         }
 729         if (ret < 0) {
 730             return ret;
 731         }
 732         blk_mig_lock();
 733         if (ret != 0) {
 734             /* no more dirty blocks */
 735             break;
 736         }
 737     }
 738     blk_mig_unlock();
 739
 740     ret = flush_blks(f);
 741     if (ret) {
 742         return ret;
 743     }
 744
 745     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 746     delta_ftell = qemu_ftell(f) - last_ftell;
 747     if (delta_ftell > 0) {
 748         return 1;
 749     } else if (delta_ftell < 0) {
 750         return -1;
 751     } else {
 752         return 0;
 753     }
 754 }
 755
 756 /* Called with iothread lock taken.  */
 757
 758 static int block_save_complete(QEMUFile *f, void *opaque)
 759 {
 760     int ret;
 761
 762     DPRINTF("Enter save live complete submitted %d transferred %d\n",
 763             block_mig_state.submitted, block_mig_state.transferred);
 764
 765     ret = flush_blks(f);
 766     if (ret) {
 767         return ret;
 768     }
 769
 770     blk_mig_reset_dirty_cursor();
 771
 772     /* we know for sure that save bulk is completed and
 773        all async read completed */
 774     blk_mig_lock();
 775     assert(block_mig_state.submitted == 0);
 776     blk_mig_unlock();
 777
 778     do {
 779         ret = blk_mig_save_dirty_block(f, 0);
 780         if (ret < 0) {
 781             return ret;
 782         }
 783     } while (ret == 0);
 784
 785     /* report completion */
 786     qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
 787
 788     DPRINTF("Block migration completed\n");
 789
 790     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 791
 792     return 0;
 793 }
 794
 795 static void block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
 796                                uint64_t *non_postcopiable_pending,
 797                                uint64_t *postcopiable_pending)
 798 {
 799     /* Estimate pending number of bytes to send */
 800     uint64_t pending;
 801
 802     qemu_mutex_lock_iothread();
 803     pending = get_remaining_dirty();
 804     qemu_mutex_unlock_iothread();
 805
 806     blk_mig_lock();
 807     pending += block_mig_state.submitted * BLOCK_SIZE +
 808                block_mig_state.read_done * BLOCK_SIZE;
 809     blk_mig_unlock();
 810
 811     /* Report at least one block pending during bulk phase */
 812     if (pending <= max_size && !block_mig_state.bulk_completed) {
 813         pending = max_size + BLOCK_SIZE;
 814     }
 815
 816     DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
 817     /* We don't do postcopy */
 818     *non_postcopiable_pending += pending;
 819 }
 820
 821 static int block_load(QEMUFile *f, void *opaque, int version_id)
 822 {
 823     static int banner_printed;
 824     int len, flags;
 825     char device_name[256];
 826     int64_t addr;
 827     BlockDriverState *bs, *bs_prev = NULL;
 828     BlockBackend *blk;
 829     Error *local_err = NULL;
 830     uint8_t *buf;
 831     int64_t total_sectors = 0;
 832     int nr_sectors;
 833     int ret;
 834
 835     do {
 836         addr = qemu_get_be64(f);
 837
 838         flags = addr & ~BDRV_SECTOR_MASK;
 839         addr >>= BDRV_SECTOR_BITS;
 840
 841         if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) {
 842             /* get device name */
 843             len = qemu_get_byte(f);
 844             qemu_get_buffer(f, (uint8_t *)device_name, len);
 845             device_name[len] = '\0';
 846
 847             blk = blk_by_name(device_name);
 848             if (!blk) {
 849                 fprintf(stderr, "Error unknown block device %s\n",
 850                         device_name);
 851                 return -EINVAL;
 852             }
 853             bs = blk_bs(blk);
 854             if (!bs) {
 855                 fprintf(stderr, "Block device %s has no medium\n",
 856                         device_name);
 857                 return -EINVAL;
 858             }
 859
 860             if (bs != bs_prev) {
 861                 bs_prev = bs;
 862                 total_sectors = bdrv_nb_sectors(bs);
 863                 if (total_sectors <= 0) {
 864                     error_report("Error getting length of block device %s",
 865                                  device_name);
 866                     return -EINVAL;
 867                 }
 868
 869                 bdrv_invalidate_cache(bs, &local_err);
 870                 if (local_err) {
 871                     error_report_err(local_err);
 872                     return -EINVAL;
 873                 }
 874             }
 875
 876             if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 877                 nr_sectors = total_sectors - addr;
 878             } else {
 879                 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 880             }
 881
 882             if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
 883                 ret = bdrv_write_zeroes(bs, addr, nr_sectors,
 884                                         BDRV_REQ_MAY_UNMAP);
 885             } else {
 886                 buf = g_malloc(BLOCK_SIZE);
 887                 qemu_get_buffer(f, buf, BLOCK_SIZE);
 888                 ret = bdrv_write(bs, addr, buf, nr_sectors);
 889                 g_free(buf);
 890             }
 891
 892             if (ret < 0) {
 893                 return ret;
 894             }
 895         } else if (flags & BLK_MIG_FLAG_PROGRESS) {
 896             if (!banner_printed) {
 897                 printf("Receiving block device images\n");
 898                 banner_printed = 1;
 899             }
 900             printf("Completed %d %%%c", (int)addr,
 901                    (addr == 100) ? '\n' : '\r');
 902             fflush(stdout);
 903         } else if (!(flags & BLK_MIG_FLAG_EOS)) {
 904             fprintf(stderr, "Unknown block migration flags: %#x\n", flags);
 905             return -EINVAL;
 906         }
 907         ret = qemu_file_get_error(f);
 908         if (ret != 0) {
 909             return ret;
 910         }
 911     } while (!(flags & BLK_MIG_FLAG_EOS));
 912
 913     return 0;
 914 }
 915
 916 static void block_set_params(const MigrationParams *params, void *opaque)
 917 {
 918     block_mig_state.blk_enable = params->blk;
 919     block_mig_state.shared_base = params->shared;
 920
 921     /* shared base means that blk_enable = 1 */
 922     block_mig_state.blk_enable |= params->shared;
 923 }
 924
 925 static bool block_is_active(void *opaque)
 926 {
 927     return block_mig_state.blk_enable == 1;
 928 }
 929
 930 static SaveVMHandlers savevm_block_handlers = {
 931     .set_params = block_set_params,
 932     .save_live_setup = block_save_setup,
 933     .save_live_iterate = block_save_iterate,
 934     .save_live_complete_precopy = block_save_complete,
 935     .save_live_pending = block_save_pending,
 936     .load_state = block_load,
 937     .cleanup = block_migration_cleanup,
 938     .is_active = block_is_active,
 939 };
 940
 941 void blk_mig_init(void)
 942 {
 943     QSIMPLEQ_INIT(&block_mig_state.bmds_list);
 944     QSIMPLEQ_INIT(&block_mig_state.blk_list);
 945     qemu_mutex_init(&block_mig_state.lock);
 946
 947     register_savevm_live(NULL, "block", 0, 1, &savevm_block_handlers,
 948                          &block_mig_state);
 949 }