return ret;
}
+static bool ramblock_is_ignored(RAMBlock *block)
+{
+ return !qemu_ram_is_migratable(block) ||
+ (migrate_ignore_shared() && qemu_ram_is_shared(block));
+}
+
/* Should be holding either ram_list.mutex, or the RCU lock. */
+#define RAMBLOCK_FOREACH_NOT_IGNORED(block) \
+ INTERNAL_RAMBLOCK_FOREACH(block) \
+ if (ramblock_is_ignored(block)) {} else
+
#define RAMBLOCK_FOREACH_MIGRATABLE(block) \
INTERNAL_RAMBLOCK_FOREACH(block) \
if (!qemu_ram_is_migratable(block)) {} else
#undef RAMBLOCK_FOREACH
+int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
+{
+ RAMBlock *block;
+ int ret = 0;
+
+ rcu_read_lock();
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ ret = func(block, opaque);
+ if (ret) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ return ret;
+}
+
static void ramblock_recv_map_init(void)
{
RAMBlock *rb;
- RAMBLOCK_FOREACH_MIGRATABLE(rb) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
assert(!rb->receivedmap);
rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
}
uint32_t last_version;
/* We are in the first round */
bool ram_bulk_stage;
+ /* The free page optimization is enabled */
+ bool fpo_enabled;
/* How many times we have dirty too many pages */
int dirty_rate_high_cnt;
/* these variables are used for bitmap sync */
uint64_t num_dirty_pages_period;
/* xbzrle misses since the beginning of the period */
uint64_t xbzrle_cache_miss_prev;
- /* number of iterations at the beginning of period */
- uint64_t iterations_prev;
- /* Iterations since start */
- uint64_t iterations;
+
+ /* compression statistics since the beginning of the period */
+ /* amount of count that no free thread to compress data */
+ uint64_t compress_thread_busy_prev;
+ /* amount bytes after compression */
+ uint64_t compressed_size_prev;
+ /* amount of compressed pages */
+ uint64_t compress_pages_prev;
+
+ /* total handled target pages at the beginning of period */
+ uint64_t target_page_count_prev;
+ /* total handled target pages since start */
+ uint64_t target_page_count;
/* number of dirty bits in the bitmap */
uint64_t migration_dirty_pages;
- /* protects modification of the bitmap */
+ /* Protects modification of the bitmap and migration dirty pages */
QemuMutex bitmap_mutex;
/* The RAMBlock used in the last src_page_requests */
RAMBlock *last_req_rb;
/* Queue of outstanding page requests from the destination */
QemuMutex src_page_req_mutex;
- QSIMPLEQ_HEAD(src_page_requests, RAMSrcPageRequest) src_page_requests;
+ QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
};
typedef struct RAMState RAMState;
static RAMState *ram_state;
+static NotifierWithReturnList precopy_notifier_list;
+
+void precopy_infrastructure_init(void)
+{
+ notifier_with_return_list_init(&precopy_notifier_list);
+}
+
+void precopy_add_notifier(NotifierWithReturn *n)
+{
+ notifier_with_return_list_add(&precopy_notifier_list, n);
+}
+
+void precopy_remove_notifier(NotifierWithReturn *n)
+{
+ notifier_with_return_remove(n);
+}
+
+int precopy_notify(PrecopyNotifyReason reason, Error **errp)
+{
+ PrecopyNotifyData pnd;
+ pnd.reason = reason;
+ pnd.errp = errp;
+
+ return notifier_with_return_list_notify(&precopy_notifier_list, &pnd);
+}
+
+void precopy_enable_free_page_optimization(void)
+{
+ if (!ram_state) {
+ return;
+ }
+
+ ram_state->fpo_enabled = true;
+}
+
uint64_t ram_bytes_remaining(void)
{
return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
};
typedef struct PageSearchStatus PageSearchStatus;
+CompressionStats compression_counters;
+
struct CompressParam {
bool done;
bool quit;
+ bool zero_page;
QEMUFile *file;
QemuMutex mutex;
QemuCond cond;
static QemuMutex decomp_done_lock;
static QemuCond decomp_done_cond;
-static int do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
- ram_addr_t offset, uint8_t *source_buf);
+static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
+ ram_addr_t offset, uint8_t *source_buf);
static void *do_data_compress(void *opaque)
{
CompressParam *param = opaque;
RAMBlock *block;
ram_addr_t offset;
+ bool zero_page;
qemu_mutex_lock(¶m->mutex);
while (!param->quit) {
param->block = NULL;
qemu_mutex_unlock(¶m->mutex);
- do_compress_ram_page(param->file, ¶m->stream, block, offset,
- param->originbuf);
+ zero_page = do_compress_ram_page(param->file, ¶m->stream,
+ block, offset, param->originbuf);
qemu_mutex_lock(&comp_done_lock);
param->done = true;
+ param->zero_page = zero_page;
qemu_cond_signal(&comp_done_cond);
qemu_mutex_unlock(&comp_done_lock);
return NULL;
}
-static inline void terminate_compression_threads(void)
-{
- int idx, thread_count;
-
- thread_count = migrate_compress_threads();
-
- for (idx = 0; idx < thread_count; idx++) {
- qemu_mutex_lock(&comp_param[idx].mutex);
- comp_param[idx].quit = true;
- qemu_cond_signal(&comp_param[idx].cond);
- qemu_mutex_unlock(&comp_param[idx].mutex);
- }
-}
-
static void compress_threads_save_cleanup(void)
{
int i, thread_count;
- if (!migrate_use_compression()) {
+ if (!migrate_use_compression() || !comp_param) {
return;
}
- terminate_compression_threads();
+
thread_count = migrate_compress_threads();
for (i = 0; i < thread_count; i++) {
/*
if (!comp_param[i].file) {
break;
}
+
+ qemu_mutex_lock(&comp_param[i].mutex);
+ comp_param[i].quit = true;
+ qemu_cond_signal(&comp_param[i].cond);
+ qemu_mutex_unlock(&comp_param[i].mutex);
+
qemu_thread_join(compress_threads + i);
qemu_mutex_destroy(&comp_param[i].mutex);
qemu_cond_destroy(&comp_param[i].cond);
#define MULTIFD_FLAG_SYNC (1 << 0)
+/* This value needs to be a multiple of qemu_target_page_size() */
+#define MULTIFD_PACKET_SIZE (512 * 1024)
+
typedef struct {
uint32_t magic;
uint32_t version;
unsigned char uuid[16]; /* QemuUUID */
uint8_t id;
+ uint8_t unused1[7]; /* Reserved for future use */
+ uint64_t unused2[4]; /* Reserved for future use */
} __attribute__((packed)) MultiFDInit_t;
typedef struct {
uint32_t magic;
uint32_t version;
uint32_t flags;
- uint32_t size;
- uint32_t used;
+ /* maximum number of allocated pages */
+ uint32_t pages_alloc;
+ uint32_t pages_used;
+ /* size of the next packet that contains pages */
+ uint32_t next_packet_size;
uint64_t packet_num;
+ uint64_t unused[4]; /* Reserved for future use */
char ramblock[256];
uint64_t offset[];
} __attribute__((packed)) MultiFDPacket_t;
MultiFDPacket_t *packet;
/* multifd flags for each packet */
uint32_t flags;
+ /* size of the next packet that contains pages */
+ uint32_t next_packet_size;
/* global number of generated multifd packets */
uint64_t packet_num;
/* thread local variables */
/* global number of generated multifd packets */
uint64_t packet_num;
/* thread local variables */
+ /* size of the next packet that contains pages */
+ uint32_t next_packet_size;
/* packets sent through this channel */
uint64_t num_packets;
/* pages sent through this channel */
return -1;
}
- be32_to_cpus(&msg.magic);
- be32_to_cpus(&msg.version);
+ msg.magic = be32_to_cpu(msg.magic);
+ msg.version = be32_to_cpu(msg.version);
if (msg.magic != MULTIFD_MAGIC) {
error_setg(errp, "multifd: received packet magic %x "
static void multifd_send_fill_packet(MultiFDSendParams *p)
{
MultiFDPacket_t *packet = p->packet;
+ uint32_t page_max = MULTIFD_PACKET_SIZE / qemu_target_page_size();
int i;
packet->magic = cpu_to_be32(MULTIFD_MAGIC);
packet->version = cpu_to_be32(MULTIFD_VERSION);
packet->flags = cpu_to_be32(p->flags);
- packet->size = cpu_to_be32(migrate_multifd_page_count());
- packet->used = cpu_to_be32(p->pages->used);
+ packet->pages_alloc = cpu_to_be32(page_max);
+ packet->pages_used = cpu_to_be32(p->pages->used);
+ packet->next_packet_size = cpu_to_be32(p->next_packet_size);
packet->packet_num = cpu_to_be64(p->packet_num);
if (p->pages->block) {
static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp)
{
MultiFDPacket_t *packet = p->packet;
+ uint32_t pages_max = MULTIFD_PACKET_SIZE / qemu_target_page_size();
RAMBlock *block;
int i;
- be32_to_cpus(&packet->magic);
+ packet->magic = be32_to_cpu(packet->magic);
if (packet->magic != MULTIFD_MAGIC) {
error_setg(errp, "multifd: received packet "
"magic %x and expected magic %x",
return -1;
}
- be32_to_cpus(&packet->version);
+ packet->version = be32_to_cpu(packet->version);
if (packet->version != MULTIFD_VERSION) {
error_setg(errp, "multifd: received packet "
"version %d and expected version %d",
p->flags = be32_to_cpu(packet->flags);
- be32_to_cpus(&packet->size);
- if (packet->size > migrate_multifd_page_count()) {
+ packet->pages_alloc = be32_to_cpu(packet->pages_alloc);
+ /*
+ * If we recevied a packet that is 100 times bigger than expected
+ * just stop migration. It is a magic number.
+ */
+ if (packet->pages_alloc > pages_max * 100) {
error_setg(errp, "multifd: received packet "
- "with size %d and expected maximum size %d",
- packet->size, migrate_multifd_page_count()) ;
+ "with size %d and expected a maximum size of %d",
+ packet->pages_alloc, pages_max * 100) ;
return -1;
}
+ /*
+ * We received a packet that is bigger than expected but inside
+ * reasonable limits (see previous comment). Just reallocate.
+ */
+ if (packet->pages_alloc > p->pages->allocated) {
+ multifd_pages_clear(p->pages);
+ multifd_pages_init(packet->pages_alloc);
+ }
- p->pages->used = be32_to_cpu(packet->used);
- if (p->pages->used > packet->size) {
+ p->pages->used = be32_to_cpu(packet->pages_used);
+ if (p->pages->used > packet->pages_alloc) {
error_setg(errp, "multifd: received packet "
- "with size %d and expected maximum size %d",
- p->pages->used, packet->size) ;
+ "with %d pages and expected maximum pages are %d",
+ p->pages->used, packet->pages_alloc) ;
return -1;
}
+ p->next_packet_size = be32_to_cpu(packet->next_packet_size);
p->packet_num = be64_to_cpu(packet->packet_num);
if (p->pages->used) {
}
}
-int multifd_save_cleanup(Error **errp)
+void multifd_save_cleanup(void)
{
int i;
- int ret = 0;
if (!migrate_use_multifd()) {
- return 0;
+ return;
}
multifd_send_terminate_threads(NULL);
for (i = 0; i < migrate_multifd_channels(); i++) {
multifd_send_state->pages = NULL;
g_free(multifd_send_state);
multifd_send_state = NULL;
- return ret;
}
static void multifd_send_sync_main(void)
uint64_t packet_num = p->packet_num;
uint32_t flags = p->flags;
+ p->next_packet_size = used * qemu_target_page_size();
multifd_send_fill_packet(p);
p->flags = 0;
p->num_packets++;
p->pages->used = 0;
qemu_mutex_unlock(&p->mutex);
- trace_multifd_send(p->id, packet_num, used, flags);
+ trace_multifd_send(p->id, packet_num, used, flags,
+ p->next_packet_size);
ret = qio_channel_write_all(p->c, (void *)p->packet,
p->packet_len, &local_err);
break;
}
- ret = qio_channel_writev_all(p->c, p->pages->iov, used, &local_err);
- if (ret != 0) {
- break;
+ if (used) {
+ ret = qio_channel_writev_all(p->c, p->pages->iov,
+ used, &local_err);
+ if (ret != 0) {
+ break;
+ }
}
qemu_mutex_lock(&p->mutex);
Error *local_err = NULL;
if (qio_task_propagate_error(task, &local_err)) {
- if (multifd_save_cleanup(&local_err) != 0) {
- migrate_set_error(migrate_get_current(), local_err);
- }
+ migrate_set_error(migrate_get_current(), local_err);
+ multifd_save_cleanup();
} else {
p->c = QIO_CHANNEL(sioc);
qio_channel_set_delay(p->c, false);
int multifd_save_setup(void)
{
int thread_count;
- uint32_t page_count = migrate_multifd_page_count();
+ uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size();
uint8_t i;
if (!migrate_use_multifd()) {
used = p->pages->used;
flags = p->flags;
- trace_multifd_recv(p->id, p->packet_num, used, flags);
+ trace_multifd_recv(p->id, p->packet_num, used, flags,
+ p->next_packet_size);
p->num_packets++;
p->num_pages += used;
qemu_mutex_unlock(&p->mutex);
- ret = qio_channel_readv_all(p->c, p->pages->iov, used, &local_err);
- if (ret != 0) {
- break;
+ if (used) {
+ ret = qio_channel_readv_all(p->c, p->pages->iov,
+ used, &local_err);
+ if (ret != 0) {
+ break;
+ }
}
if (flags & MULTIFD_FLAG_SYNC) {
int multifd_load_setup(void)
{
int thread_count;
- uint32_t page_count = migrate_multifd_page_count();
+ uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size();
uint8_t i;
if (!migrate_use_multifd()) {
return thread_count == atomic_read(&multifd_recv_state->count);
}
-/* Return true if multifd is ready for the migration, otherwise false */
-bool multifd_recv_new_channel(QIOChannel *ioc)
+/*
+ * Try to receive all multifd channels to get ready for the migration.
+ * - Return true and do not set @errp when correctly receving all channels;
+ * - Return false and do not set @errp when correctly receiving the current one;
+ * - Return false and set @errp when failing to receive the current channel.
+ */
+bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp)
{
MultiFDRecvParams *p;
Error *local_err = NULL;
id = multifd_recv_initial_packet(ioc, &local_err);
if (id < 0) {
multifd_recv_terminate_threads(local_err);
+ error_propagate_prepend(errp, local_err,
+ "failed to receive packet"
+ " via multifd channel %d: ",
+ atomic_read(&multifd_recv_state->count));
return false;
}
error_setg(&local_err, "multifd: received id '%d' already setup'",
id);
multifd_recv_terminate_threads(local_err);
+ error_propagate(errp, local_err);
return false;
}
p->c = ioc;
qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p,
QEMU_THREAD_JOINABLE);
atomic_inc(&multifd_recv_state->count);
- return multifd_recv_state->count == migrate_multifd_channels();
+ return atomic_read(&multifd_recv_state->count) ==
+ migrate_multifd_channels();
}
/**
unsigned long *bitmap = rb->bmap;
unsigned long next;
- if (!qemu_ram_is_migratable(rb)) {
+ if (ramblock_is_ignored(rb)) {
return size;
}
- if (rs->ram_bulk_stage && start > 0) {
+ /*
+ * When the free page optimization is enabled, we need to check the bitmap
+ * to send the non-free pages rather than all the pages in the bulk stage.
+ */
+ if (!rs->fpo_enabled && rs->ram_bulk_stage && start > 0) {
next = start + 1;
} else {
next = find_next_bit(bitmap, size, start);
{
bool ret;
+ qemu_mutex_lock(&rs->bitmap_mutex);
ret = test_and_clear_bit(page, rb->bmap);
if (ret) {
rs->migration_dirty_pages--;
}
+ qemu_mutex_unlock(&rs->bitmap_mutex);
+
return ret;
}
RAMBlock *block;
uint64_t summary = 0;
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
summary |= block->page_size;
}
return summary;
}
+uint64_t ram_get_total_transferred_pages(void)
+{
+ return ram_counters.normal + ram_counters.duplicate +
+ compression_counters.pages + xbzrle_counters.pages;
+}
+
static void migration_update_rates(RAMState *rs, int64_t end_time)
{
- uint64_t iter_count = rs->iterations - rs->iterations_prev;
+ uint64_t page_count = rs->target_page_count - rs->target_page_count_prev;
+ double compressed_size;
/* calculate period counters */
ram_counters.dirty_pages_rate = rs->num_dirty_pages_period * 1000
/ (end_time - rs->time_last_bitmap_sync);
- if (!iter_count) {
+ if (!page_count) {
return;
}
if (migrate_use_xbzrle()) {
xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
- rs->xbzrle_cache_miss_prev) / iter_count;
+ rs->xbzrle_cache_miss_prev) / page_count;
rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
}
+
+ if (migrate_use_compression()) {
+ compression_counters.busy_rate = (double)(compression_counters.busy -
+ rs->compress_thread_busy_prev) / page_count;
+ rs->compress_thread_busy_prev = compression_counters.busy;
+
+ compressed_size = compression_counters.compressed_size -
+ rs->compressed_size_prev;
+ if (compressed_size) {
+ double uncompressed_size = (compression_counters.pages -
+ rs->compress_pages_prev) * TARGET_PAGE_SIZE;
+
+ /* Compression-Ratio = Uncompressed-size / Compressed-size */
+ compression_counters.compression_rate =
+ uncompressed_size / compressed_size;
+
+ rs->compress_pages_prev = compression_counters.pages;
+ rs->compressed_size_prev = compression_counters.compressed_size;
+ }
+ }
}
static void migration_bitmap_sync(RAMState *rs)
qemu_mutex_lock(&rs->bitmap_mutex);
rcu_read_lock();
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
migration_bitmap_sync_range(rs, block, 0, block->used_length);
}
ram_counters.remaining = ram_bytes_remaining();
migration_update_rates(rs, end_time);
- rs->iterations_prev = rs->iterations;
+ rs->target_page_count_prev = rs->target_page_count;
/* reset period counters */
rs->time_last_bitmap_sync = end_time;
rs->bytes_xfer_prev = bytes_xfer_now;
}
if (migrate_use_events()) {
- qapi_event_send_migration_pass(ram_counters.dirty_sync_count, NULL);
+ qapi_event_send_migration_pass(ram_counters.dirty_sync_count);
+ }
+}
+
+static void migration_bitmap_sync_precopy(RAMState *rs)
+{
+ Error *local_err = NULL;
+
+ /*
+ * The current notifier usage is just an optimization to migration, so we
+ * don't stop the normal migration process in the error case.
+ */
+ if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) {
+ error_report_err(local_err);
+ }
+
+ migration_bitmap_sync(rs);
+
+ if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) {
+ error_report_err(local_err);
}
}
+/**
+ * save_zero_page_to_file: send the zero page to the file
+ *
+ * Returns the size of data written to the file, 0 means the page is not
+ * a zero page
+ *
+ * @rs: current RAM state
+ * @file: the file where the data is saved
+ * @block: block that contains the page we want to send
+ * @offset: offset inside the block for the page
+ */
+static int save_zero_page_to_file(RAMState *rs, QEMUFile *file,
+ RAMBlock *block, ram_addr_t offset)
+{
+ uint8_t *p = block->host + offset;
+ int len = 0;
+
+ if (is_zero_range(p, TARGET_PAGE_SIZE)) {
+ len += save_page_header(rs, file, block, offset | RAM_SAVE_FLAG_ZERO);
+ qemu_put_byte(file, 0);
+ len += 1;
+ }
+ return len;
+}
+
/**
* save_zero_page: send the zero page to the stream
*
*/
static int save_zero_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
{
- uint8_t *p = block->host + offset;
- int pages = -1;
+ int len = save_zero_page_to_file(rs, rs->f, block, offset);
- if (is_zero_range(p, TARGET_PAGE_SIZE)) {
+ if (len) {
ram_counters.duplicate++;
- ram_counters.transferred +=
- save_page_header(rs, rs->f, block, offset | RAM_SAVE_FLAG_ZERO);
- qemu_put_byte(rs->f, 0);
- ram_counters.transferred += 1;
- pages = 1;
+ ram_counters.transferred += len;
+ return 1;
}
-
- return pages;
+ return -1;
}
static void ram_release_pages(const char *rbname, uint64_t offset, int pages)
return 1;
}
-static int do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
- ram_addr_t offset, uint8_t *source_buf)
+static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
+ ram_addr_t offset, uint8_t *source_buf)
{
RAMState *rs = ram_state;
- int bytes_sent, blen;
uint8_t *p = block->host + (offset & TARGET_PAGE_MASK);
+ bool zero_page = false;
+ int ret;
+
+ if (save_zero_page_to_file(rs, f, block, offset)) {
+ zero_page = true;
+ goto exit;
+ }
- bytes_sent = save_page_header(rs, f, block, offset |
- RAM_SAVE_FLAG_COMPRESS_PAGE);
+ save_page_header(rs, f, block, offset | RAM_SAVE_FLAG_COMPRESS_PAGE);
/*
* copy it to a internal buffer to avoid it being modified by VM
* decompression
*/
memcpy(source_buf, p, TARGET_PAGE_SIZE);
- blen = qemu_put_compression_data(f, stream, source_buf, TARGET_PAGE_SIZE);
- if (blen < 0) {
- bytes_sent = 0;
- qemu_file_set_error(migrate_get_current()->to_dst_file, blen);
+ ret = qemu_put_compression_data(f, stream, source_buf, TARGET_PAGE_SIZE);
+ if (ret < 0) {
+ qemu_file_set_error(migrate_get_current()->to_dst_file, ret);
error_report("compressed data failed!");
- } else {
- bytes_sent += blen;
- ram_release_pages(block->idstr, offset & TARGET_PAGE_MASK, 1);
+ return false;
+ }
+
+exit:
+ ram_release_pages(block->idstr, offset & TARGET_PAGE_MASK, 1);
+ return zero_page;
+}
+
+static void
+update_compress_thread_counts(const CompressParam *param, int bytes_xmit)
+{
+ ram_counters.transferred += bytes_xmit;
+
+ if (param->zero_page) {
+ ram_counters.duplicate++;
+ return;
}
- return bytes_sent;
+ /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
+ compression_counters.compressed_size += bytes_xmit - 8;
+ compression_counters.pages++;
}
+static bool save_page_use_compression(RAMState *rs);
+
static void flush_compressed_data(RAMState *rs)
{
int idx, len, thread_count;
- if (!migrate_use_compression()) {
+ if (!save_page_use_compression(rs)) {
return;
}
thread_count = migrate_compress_threads();
qemu_mutex_lock(&comp_param[idx].mutex);
if (!comp_param[idx].quit) {
len = qemu_put_qemu_file(rs->f, comp_param[idx].file);
- ram_counters.transferred += len;
+ /*
+ * it's safe to fetch zero_page without holding comp_done_lock
+ * as there is no further request submitted to the thread,
+ * i.e, the thread should be waiting for a request at this point.
+ */
+ update_compress_thread_counts(&comp_param[idx], len);
}
qemu_mutex_unlock(&comp_param[idx].mutex);
}
ram_addr_t offset)
{
int idx, thread_count, bytes_xmit = -1, pages = -1;
+ bool wait = migrate_compress_wait_thread();
thread_count = migrate_compress_threads();
qemu_mutex_lock(&comp_done_lock);
- while (true) {
- for (idx = 0; idx < thread_count; idx++) {
- if (comp_param[idx].done) {
- comp_param[idx].done = false;
- bytes_xmit = qemu_put_qemu_file(rs->f, comp_param[idx].file);
- qemu_mutex_lock(&comp_param[idx].mutex);
- set_compress_params(&comp_param[idx], block, offset);
- qemu_cond_signal(&comp_param[idx].cond);
- qemu_mutex_unlock(&comp_param[idx].mutex);
- pages = 1;
- ram_counters.normal++;
- ram_counters.transferred += bytes_xmit;
- break;
- }
- }
- if (pages > 0) {
+retry:
+ for (idx = 0; idx < thread_count; idx++) {
+ if (comp_param[idx].done) {
+ comp_param[idx].done = false;
+ bytes_xmit = qemu_put_qemu_file(rs->f, comp_param[idx].file);
+ qemu_mutex_lock(&comp_param[idx].mutex);
+ set_compress_params(&comp_param[idx], block, offset);
+ qemu_cond_signal(&comp_param[idx].cond);
+ qemu_mutex_unlock(&comp_param[idx].mutex);
+ pages = 1;
+ update_compress_thread_counts(&comp_param[idx], bytes_xmit);
break;
- } else {
- qemu_cond_wait(&comp_done_cond, &comp_done_lock);
}
}
+
+ /*
+ * wait for the free thread if the user specifies 'compress-wait-thread',
+ * otherwise we will post the page out in the main thread as normal page.
+ */
+ if (pages < 0 && wait) {
+ qemu_cond_wait(&comp_done_cond, &comp_done_lock);
+ goto retry;
+ }
qemu_mutex_unlock(&comp_done_lock);
return pages;
pss->page = 0;
pss->block = QLIST_NEXT_RCU(pss->block, next);
if (!pss->block) {
+ /*
+ * If memory migration starts over, we will meet a dirtied page
+ * which may still exists in compression threads's ring, so we
+ * should flush the compressed data to make sure the new page
+ * is not overwritten by the old one in the destination.
+ *
+ * Also If xbzrle is on, stop using the data compression at this
+ * point. In theory, xbzrle can do better than compression.
+ */
+ flush_compressed_data(rs);
+
/* Hit the end of the list */
pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
/* Flag that we've looped */
pss->complete_round = true;
rs->ram_bulk_stage = false;
- if (migrate_use_xbzrle()) {
- /* If xbzrle is on, stop using the data compression at this
- * point. In theory, xbzrle can do better than compression.
- */
- flush_compressed_data(rs);
- }
}
/* Didn't find anything this time, but try again on the new block */
*again = true;
{
RAMBlock *block = NULL;
+ if (QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests)) {
+ return NULL;
+ }
+
qemu_mutex_lock(&rs->src_page_req_mutex);
if (!QSIMPLEQ_EMPTY(&rs->src_page_requests)) {
struct RAMSrcPageRequest *entry =
return false;
}
+/*
+ * try to compress the page before posting it out, return true if the page
+ * has been properly handled by compression, otherwise needs other
+ * paths to handle it
+ */
+static bool save_compress_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
+{
+ if (!save_page_use_compression(rs)) {
+ return false;
+ }
+
+ /*
+ * When starting the process of a new block, the first page of
+ * the block should be sent out before other pages in the same
+ * block, and all the pages in last block should have been sent
+ * out, keeping this order is important, because the 'cont' flag
+ * is used to avoid resending the block name.
+ *
+ * We post the fist page as normal page as compression will take
+ * much CPU resource.
+ */
+ if (block != rs->last_sent_block) {
+ flush_compressed_data(rs);
+ return false;
+ }
+
+ if (compress_page_with_multi_thread(rs, block, offset) > 0) {
+ return true;
+ }
+
+ compression_counters.busy++;
+ return false;
+}
+
/**
* ram_save_target_page: save one target page
*
return res;
}
- /*
- * When starting the process of a new block, the first page of
- * the block should be sent out before other pages in the same
- * block, and all the pages in last block should have been sent
- * out, keeping this order is important, because the 'cont' flag
- * is used to avoid resending the block name.
- */
- if (block != rs->last_sent_block && save_page_use_compression(rs)) {
- flush_compressed_data(rs);
+ if (save_compress_page(rs, block, offset)) {
+ return 1;
}
res = save_zero_page(rs, block, offset);
}
/*
- * Make sure the first page is sent out before other pages.
- *
- * we post it as normal page as compression will take much
- * CPU resource.
+ * do not use multifd for compression as the first page in the new
+ * block should be posted out before sending the compressed page
*/
- if (block == rs->last_sent_block && save_page_use_compression(rs)) {
- return compress_page_with_multi_thread(rs, block, offset);
- } else if (migrate_use_multifd()) {
+ if (!save_page_use_compression(rs) && migrate_use_multifd()) {
return ram_save_multifd_page(rs, block, offset);
}
size_t pagesize_bits =
qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
- if (!qemu_ram_is_migratable(pss->block)) {
+ if (ramblock_is_ignored(pss->block)) {
error_report("block %s should not be migrated !", pss->block->idstr);
return 0;
}
*
* Called within an RCU critical section.
*
- * Returns the number of pages written where zero means no dirty pages
+ * Returns the number of pages written where zero means no dirty pages,
+ * or negative on error
*
* @rs: current RAM state
* @last_stage: if we are at the completion stage
}
}
-uint64_t ram_bytes_total(void)
+static uint64_t ram_bytes_total_common(bool count_ignored)
{
RAMBlock *block;
uint64_t total = 0;
rcu_read_lock();
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
- total += block->used_length;
+ if (count_ignored) {
+ RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ total += block->used_length;
+ }
+ } else {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ total += block->used_length;
+ }
}
rcu_read_unlock();
return total;
}
+uint64_t ram_bytes_total(void)
+{
+ return ram_bytes_total_common(false);
+}
+
static void xbzrle_load_setup(void)
{
XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
*/
memory_global_dirty_log_stop();
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
g_free(block->bmap);
block->bmap = NULL;
g_free(block->unsentmap);
rs->last_page = 0;
rs->last_version = ram_list.version;
rs->ram_bulk_stage = true;
+ rs->fpo_enabled = false;
}
#define MAX_WAIT 50 /* ms, half buffered_file limit */
{
struct RAMBlock *block;
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
unsigned long *bitmap = block->bmap;
unsigned long range = block->used_length >> TARGET_PAGE_BITS;
unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
struct RAMBlock *block;
int ret;
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
PostcopyDiscardState *pds =
postcopy_discard_send_init(ms, block->idstr);
rs->last_sent_block = NULL;
rs->last_page = 0;
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
unsigned long *bitmap = block->bmap;
unsigned long *unsentmap = block->unsentmap;
/* Skip setting bitmap if there is no RAM */
if (ram_bytes_total()) {
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
pages = block->max_length >> TARGET_PAGE_BITS;
block->bmap = bitmap_new(pages);
bitmap_set(block->bmap, 0, pages);
ram_list_init_bitmaps();
memory_global_dirty_log_start();
- migration_bitmap_sync(rs);
+ migration_bitmap_sync_precopy(rs);
rcu_read_unlock();
qemu_mutex_unlock_ramlist();
* about dirty page logging as well.
*/
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
pages += bitmap_count_one(block->bmap,
block->used_length >> TARGET_PAGE_BITS);
}
trace_ram_state_resume_prepare(pages);
}
+/*
+ * This function clears bits of the free pages reported by the caller from the
+ * migration dirty bitmap. @addr is the host address corresponding to the
+ * start of the continuous guest free pages, and @len is the total bytes of
+ * those pages.
+ */
+void qemu_guest_free_page_hint(void *addr, size_t len)
+{
+ RAMBlock *block;
+ ram_addr_t offset;
+ size_t used_len, start, npages;
+ MigrationState *s = migrate_get_current();
+
+ /* This function is currently expected to be used during live migration */
+ if (!migration_is_setup_or_active(s->state)) {
+ return;
+ }
+
+ for (; len > 0; len -= used_len, addr += used_len) {
+ block = qemu_ram_block_from_host(addr, false, &offset);
+ if (unlikely(!block || offset >= block->used_length)) {
+ /*
+ * The implementation might not support RAMBlock resize during
+ * live migration, but it could happen in theory with future
+ * updates. So we add a check here to capture that case.
+ */
+ error_report_once("%s unexpected error", __func__);
+ return;
+ }
+
+ if (len <= block->used_length - offset) {
+ used_len = len;
+ } else {
+ used_len = block->used_length - offset;
+ }
+
+ start = offset >> TARGET_PAGE_BITS;
+ npages = used_len >> TARGET_PAGE_BITS;
+
+ qemu_mutex_lock(&ram_state->bitmap_mutex);
+ ram_state->migration_dirty_pages -=
+ bitmap_count_one_with_offset(block->bmap, start, npages);
+ bitmap_clear(block->bmap, start, npages);
+ qemu_mutex_unlock(&ram_state->bitmap_mutex);
+ }
+}
+
/*
* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
* long-running RCU critical section. When rcu-reclaims in the code
rcu_read_lock();
- qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
+ qemu_put_be64(f, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE);
RAMBLOCK_FOREACH_MIGRATABLE(block) {
qemu_put_byte(f, strlen(block->idstr));
if (migrate_postcopy_ram() && block->page_size != qemu_host_page_size) {
qemu_put_be64(f, block->page_size);
}
+ if (migrate_ignore_shared()) {
+ qemu_put_be64(f, block->mr->addr);
+ qemu_put_byte(f, ramblock_is_ignored(block) ? 1 : 0);
+ }
}
rcu_read_unlock();
done = 1;
break;
}
- rs->iterations++;
+
+ if (pages < 0) {
+ qemu_file_set_error(f, pages);
+ break;
+ }
+
+ rs->target_page_count += pages;
/* we want to check in the 1st loop, just in case it was the 1st time
and we had to sync the dirty bitmap.
}
i++;
}
- flush_compressed_data(rs);
rcu_read_unlock();
/*
/**
* ram_save_complete: function called to send the remaining amount of ram
*
- * Returns zero to indicate success
+ * Returns zero to indicate success or negative on error
*
* Called with iothread lock
*
{
RAMState **temp = opaque;
RAMState *rs = *temp;
+ int ret = 0;
rcu_read_lock();
if (!migration_in_postcopy()) {
- migration_bitmap_sync(rs);
+ migration_bitmap_sync_precopy(rs);
}
ram_control_before_iterate(f, RAM_CONTROL_FINISH);
if (pages == 0) {
break;
}
+ if (pages < 0) {
+ ret = pages;
+ break;
+ }
}
flush_compressed_data(rs);
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
qemu_fflush(f);
- return 0;
+ return ret;
}
static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
remaining_size < max_size) {
qemu_mutex_lock_iothread();
rcu_read_lock();
- migration_bitmap_sync(rs);
+ migration_bitmap_sync_precopy(rs);
rcu_read_unlock();
qemu_mutex_unlock_iothread();
remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
return NULL;
}
- if (!qemu_ram_is_migratable(block)) {
+ if (ramblock_is_ignored(block)) {
error_report("block %s should not be migrated !", id);
return NULL;
}
return block->host + offset;
}
+static inline void *colo_cache_from_block_offset(RAMBlock *block,
+ ram_addr_t offset)
+{
+ if (!offset_in_ramblock(block, offset)) {
+ return NULL;
+ }
+ if (!block->colo_cache) {
+ error_report("%s: colo_cache is NULL in block :%s",
+ __func__, block->idstr);
+ return NULL;
+ }
+
+ /*
+ * During colo checkpoint, we need bitmap of these migrated pages.
+ * It help us to decide which pages in ram cache should be flushed
+ * into VM's RAM later.
+ */
+ if (!test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) {
+ ram_state->migration_dirty_pages++;
+ }
+ return block->colo_cache + offset;
+}
+
/**
* ram_handle_compressed: handle the zero page case
*
qemu_mutex_unlock(&decomp_done_lock);
}
+/*
+ * colo cache: this is for secondary VM, we cache the whole
+ * memory of the secondary VM, it is need to hold the global lock
+ * to call this helper.
+ */
+int colo_init_ram_cache(void)
+{
+ RAMBlock *block;
+
+ rcu_read_lock();
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ block->colo_cache = qemu_anon_ram_alloc(block->used_length,
+ NULL,
+ false);
+ if (!block->colo_cache) {
+ error_report("%s: Can't alloc memory for COLO cache of block %s,"
+ "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
+ block->used_length);
+ goto out_locked;
+ }
+ memcpy(block->colo_cache, block->host, block->used_length);
+ }
+ rcu_read_unlock();
+ /*
+ * Record the dirty pages that sent by PVM, we use this dirty bitmap together
+ * with to decide which page in cache should be flushed into SVM's RAM. Here
+ * we use the same name 'ram_bitmap' as for migration.
+ */
+ if (ram_bytes_total()) {
+ RAMBlock *block;
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
+
+ block->bmap = bitmap_new(pages);
+ bitmap_set(block->bmap, 0, pages);
+ }
+ }
+ ram_state = g_new0(RAMState, 1);
+ ram_state->migration_dirty_pages = 0;
+ memory_global_dirty_log_start();
+
+ return 0;
+
+out_locked:
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ if (block->colo_cache) {
+ qemu_anon_ram_free(block->colo_cache, block->used_length);
+ block->colo_cache = NULL;
+ }
+ }
+
+ rcu_read_unlock();
+ return -errno;
+}
+
+/* It is need to hold the global lock to call this helper */
+void colo_release_ram_cache(void)
+{
+ RAMBlock *block;
+
+ memory_global_dirty_log_stop();
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ g_free(block->bmap);
+ block->bmap = NULL;
+ }
+
+ rcu_read_lock();
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ if (block->colo_cache) {
+ qemu_anon_ram_free(block->colo_cache, block->used_length);
+ block->colo_cache = NULL;
+ }
+ }
+
+ rcu_read_unlock();
+ g_free(ram_state);
+ ram_state = NULL;
+}
+
/**
* ram_load_setup: Setup RAM for migration incoming side
*
xbzrle_load_setup();
ramblock_recv_map_init();
+
return 0;
}
{
RAMBlock *rb;
- RAMBLOCK_FOREACH_MIGRATABLE(rb) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
if (ramblock_is_pmem(rb)) {
pmem_persist(rb->host, rb->used_length);
}
xbzrle_load_cleanup();
compress_threads_load_cleanup();
- RAMBLOCK_FOREACH_MIGRATABLE(rb) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
g_free(rb->receivedmap);
rb->receivedmap = NULL;
}
+
return 0;
}
return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
}
+/*
+ * Flush content of RAM cache into SVM's memory.
+ * Only flush the pages that be dirtied by PVM or SVM or both.
+ */
+static void colo_flush_ram_cache(void)
+{
+ RAMBlock *block = NULL;
+ void *dst_host;
+ void *src_host;
+ unsigned long offset = 0;
+
+ memory_global_dirty_log_sync();
+ rcu_read_lock();
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ migration_bitmap_sync_range(ram_state, block, 0, block->used_length);
+ }
+ rcu_read_unlock();
+
+ trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
+ rcu_read_lock();
+ block = QLIST_FIRST_RCU(&ram_list.blocks);
+
+ while (block) {
+ offset = migration_bitmap_find_dirty(ram_state, block, offset);
+
+ if (offset << TARGET_PAGE_BITS >= block->used_length) {
+ offset = 0;
+ block = QLIST_NEXT_RCU(block, next);
+ } else {
+ migration_bitmap_clear_dirty(ram_state, block, offset);
+ dst_host = block->host + (offset << TARGET_PAGE_BITS);
+ src_host = block->colo_cache + (offset << TARGET_PAGE_BITS);
+ memcpy(dst_host, src_host, TARGET_PAGE_SIZE);
+ }
+ }
+
+ rcu_read_unlock();
+ trace_colo_flush_ram_cache_end();
+}
+
static int ram_load(QEMUFile *f, void *opaque, int version_id)
{
int flags = 0, ret = 0, invalid_flags = 0;
RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
RAMBlock *block = ram_block_from_stream(f, flags);
- host = host_from_ram_block_offset(block, addr);
+ /*
+ * After going into COLO, we should load the Page into colo_cache.
+ */
+ if (migration_incoming_in_colo_state()) {
+ host = colo_cache_from_block_offset(block, addr);
+ } else {
+ host = host_from_ram_block_offset(block, addr);
+ }
if (!host) {
error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
ret = -EINVAL;
break;
}
- ramblock_recv_bitmap_set(block, host);
+
+ if (!migration_incoming_in_colo_state()) {
+ ramblock_recv_bitmap_set(block, host);
+ }
+
trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
}
ret = -EINVAL;
}
}
+ if (migrate_ignore_shared()) {
+ hwaddr addr = qemu_get_be64(f);
+ bool ignored = qemu_get_byte(f);
+ if (ignored != ramblock_is_ignored(block)) {
+ error_report("RAM block %s should %s be migrated",
+ id, ignored ? "" : "not");
+ ret = -EINVAL;
+ }
+ if (ramblock_is_ignored(block) &&
+ block->mr->addr != addr) {
+ error_report("Mismatched GPAs for block %s "
+ "%" PRId64 "!= %" PRId64,
+ id, (uint64_t)addr,
+ (uint64_t)block->mr->addr);
+ ret = -EINVAL;
+ }
+ }
ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
block->idstr);
} else {
ret |= wait_for_decompress_done();
rcu_read_unlock();
trace_ram_load_complete(ret, seq_iter);
+
+ if (!ret && migration_incoming_in_colo_state()) {
+ colo_flush_ram_cache();
+ }
return ret;
}
static bool ram_has_postcopy(void *opaque)
{
RAMBlock *rb;
- RAMBLOCK_FOREACH_MIGRATABLE(rb) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
if (ramblock_is_pmem(rb)) {
info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
"is not supported now!", rb->idstr, rb->host);
trace_ram_dirty_bitmap_sync_start();
- RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
qemu_savevm_send_recv_bitmap(file, block->idstr);
trace_ram_dirty_bitmap_request(block->idstr);
ramblock_count++;
projects / mirror_qemu.git / blobdiff