typedef struct RDMAContext {
char *host;
int port;
- char *host_port;
RDMAWorkRequestData wr_data[RDMA_WRID_MAX];
local->block = g_new0(RDMALocalBlock, local->nb_blocks + 1);
if (local->nb_blocks) {
- int x;
-
if (rdma->blockmap) {
- for (x = 0; x < local->nb_blocks; x++) {
+ for (int x = 0; x < local->nb_blocks; x++) {
g_hash_table_remove(rdma->blockmap,
(void *)(uintptr_t)old[x].offset);
g_hash_table_insert(rdma->blockmap,
{
RDMALocalBlocks *local = &rdma->local_ram_blocks;
RDMALocalBlock *old = local->block;
- int x;
if (rdma->blockmap) {
g_hash_table_remove(rdma->blockmap, (void *)(uintptr_t)block->offset);
}
if (block->pmr) {
- int j;
-
- for (j = 0; j < block->nb_chunks; j++) {
+ for (int j = 0; j < block->nb_chunks; j++) {
if (!block->pmr[j]) {
continue;
}
block->block_name = NULL;
if (rdma->blockmap) {
- for (x = 0; x < local->nb_blocks; x++) {
+ for (int x = 0; x < local->nb_blocks; x++) {
g_hash_table_remove(rdma->blockmap,
(void *)(uintptr_t)old[x].offset);
}
memcpy(local->block + block->index, old + (block->index + 1),
sizeof(RDMALocalBlock) *
(local->nb_blocks - (block->index + 1)));
- for (x = block->index; x < local->nb_blocks - 1; x++) {
+ for (int x = block->index; x < local->nb_blocks - 1; x++) {
local->block[x].index--;
}
}
local->nb_blocks--;
if (local->nb_blocks && rdma->blockmap) {
- for (x = 0; x < local->nb_blocks; x++) {
+ for (int x = 0; x < local->nb_blocks; x++) {
g_hash_table_insert(rdma->blockmap,
(void *)(uintptr_t)local->block[x].offset,
&local->block[x]);
}
/*
- * Put in the log file which RDMA device was opened and the details
- * associated with that device.
+ * Trace RDMA device open, with device details.
*/
static void qemu_rdma_dump_id(const char *who, struct ibv_context *verbs)
{
struct ibv_port_attr port;
if (ibv_query_port(verbs, 1, &port)) {
- error_report("Failed to query port information");
+ trace_qemu_rdma_dump_id_failed(who);
return;
}
- printf("%s RDMA Device opened: kernel name %s "
- "uverbs device name %s, "
- "infiniband_verbs class device path %s, "
- "infiniband class device path %s, "
- "transport: (%d) %s\n",
- who,
+ trace_qemu_rdma_dump_id(who,
verbs->device->name,
verbs->device->dev_name,
verbs->device->dev_path,
verbs->device->ibdev_path,
port.link_layer,
- (port.link_layer == IBV_LINK_LAYER_INFINIBAND) ? "Infiniband" :
- ((port.link_layer == IBV_LINK_LAYER_ETHERNET)
- ? "Ethernet" : "Unknown"));
+ port.link_layer == IBV_LINK_LAYER_INFINIBAND ? "Infiniband"
+ : port.link_layer == IBV_LINK_LAYER_ETHERNET ? "Ethernet"
+ : "Unknown");
}
/*
- * Put in the log file the RDMA gid addressing information,
- * useful for folks who have trouble understanding the
- * RDMA device hierarchy in the kernel.
+ * Trace RDMA gid addressing information.
+ * Useful for understanding the RDMA device hierarchy in the kernel.
*/
static void qemu_rdma_dump_gid(const char *who, struct rdma_cm_id *id)
{
* Otherwise, there are no guarantees until the bug is fixed in linux.
*/
if (!verbs) {
- int num_devices, x;
+ int num_devices;
struct ibv_device **dev_list = ibv_get_device_list(&num_devices);
bool roce_found = false;
bool ib_found = false;
- for (x = 0; x < num_devices; x++) {
+ for (int x = 0; x < num_devices; x++) {
verbs = ibv_open_device(dev_list[x]);
/*
* ibv_open_device() is not documented to set errno. If
if (roce_found) {
if (ib_found) {
- fprintf(stderr, "WARN: migrations may fail:"
- " IPv6 over RoCE / iWARP in linux"
- " is broken. But since you appear to have a"
- " mixed RoCE / IB environment, be sure to only"
- " migrate over the IB fabric until the kernel "
- " fixes the bug.\n");
+ warn_report("migrations may fail:"
+ " IPv6 over RoCE / iWARP in linux"
+ " is broken. But since you appear to have a"
+ " mixed RoCE / IB environment, be sure to only"
+ " migrate over the IB fabric until the kernel "
+ " fixes the bug.");
} else {
error_setg(errp, "RDMA ERROR: "
"You only have RoCE / iWARP devices in your systems"
char port_str[16];
struct rdma_cm_event *cm_event;
char ip[40] = "unknown";
- struct rdma_addrinfo *e;
if (rdma->host == NULL || !strcmp(rdma->host, "")) {
error_setg(errp, "RDMA ERROR: RDMA hostname has not been set");
}
/* Try all addresses, saving the first error in @err */
- for (e = res; e != NULL; e = e->ai_next) {
+ for (struct rdma_addrinfo *e = res; e != NULL; e = e->ai_next) {
Error **local_errp = err ? NULL : &err;
inet_ntop(e->ai_family,
error_setg(errp,
"RDMA ERROR: result not equal to event_addr_resolved %s",
rdma_event_str(cm_event->event));
- error_report("rdma_resolve_addr");
rdma_ack_cm_event(cm_event);
goto err_resolve_get_addr;
}
/*
* Create protection domain and completion queues
*/
-static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma)
+static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma, Error **errp)
{
/* allocate pd */
rdma->pd = ibv_alloc_pd(rdma->verbs);
if (!rdma->pd) {
- error_report("failed to allocate protection domain");
+ error_setg(errp, "failed to allocate protection domain");
return -1;
}
/* create receive completion channel */
rdma->recv_comp_channel = ibv_create_comp_channel(rdma->verbs);
if (!rdma->recv_comp_channel) {
- error_report("failed to allocate receive completion channel");
+ error_setg(errp, "failed to allocate receive completion channel");
goto err_alloc_pd_cq;
}
rdma->recv_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3),
NULL, rdma->recv_comp_channel, 0);
if (!rdma->recv_cq) {
- error_report("failed to allocate receive completion queue");
+ error_setg(errp, "failed to allocate receive completion queue");
goto err_alloc_pd_cq;
}
/* create send completion channel */
rdma->send_comp_channel = ibv_create_comp_channel(rdma->verbs);
if (!rdma->send_comp_channel) {
- error_report("failed to allocate send completion channel");
+ error_setg(errp, "failed to allocate send completion channel");
goto err_alloc_pd_cq;
}
rdma->send_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3),
NULL, rdma->send_comp_channel, 0);
if (!rdma->send_cq) {
- error_report("failed to allocate send completion queue");
+ error_setg(errp, "failed to allocate send completion queue");
goto err_alloc_pd_cq;
}
static int qemu_rdma_alloc_qp(RDMAContext *rdma)
{
struct ibv_qp_init_attr attr = { 0 };
- int ret;
attr.cap.max_send_wr = RDMA_SIGNALED_SEND_MAX;
attr.cap.max_recv_wr = 3;
attr.recv_cq = rdma->recv_cq;
attr.qp_type = IBV_QPT_RC;
- ret = rdma_create_qp(rdma->cm_id, rdma->pd, &attr);
- if (ret < 0) {
+ if (rdma_create_qp(rdma->cm_id, rdma->pd, &attr) < 0) {
return -1;
}
static bool rdma_support_odp(struct ibv_context *dev)
{
struct ibv_device_attr_ex attr = {0};
- int ret = ibv_query_device_ex(dev, NULL, &attr);
- if (ret) {
+
+ if (ibv_query_device_ex(dev, NULL, &attr)) {
return false;
}
}
}
if (!block->pmr[chunk]) {
- perror("Failed to register chunk!");
- fprintf(stderr, "Chunk details: block: %d chunk index %d"
- " start %" PRIuPTR " end %" PRIuPTR
- " host %" PRIuPTR
- " local %" PRIuPTR " registrations: %d\n",
- block->index, chunk, (uintptr_t)chunk_start,
- (uintptr_t)chunk_end, host_addr,
- (uintptr_t)block->local_host_addr,
- rdma->total_registrations);
return -1;
}
rdma->total_registrations++;
rdma->total_registrations++;
return 0;
}
- error_report("qemu_rdma_reg_control failed");
return -1;
}
block->remote_keys[chunk] = 0;
if (ret != 0) {
- /*
- * FIXME perror() is problematic, bcause ibv_dereg_mr() is
- * not documented to set errno. Will go away later in
- * this series.
- */
- perror("unregistration chunk failed");
+ error_report("unregistration chunk failed: %s",
+ strerror(ret));
return -1;
}
rdma->total_registrations--;
}
if (ret < 0) {
- error_report("ibv_poll_cq failed");
return -1;
}
wr_id = wc.wr_id & RDMA_WRID_TYPE_MASK;
if (wc.status != IBV_WC_SUCCESS) {
- fprintf(stderr, "ibv_poll_cq wc.status=%d %s!\n",
- wc.status, ibv_wc_status_str(wc.status));
- fprintf(stderr, "ibv_poll_cq wrid=%" PRIu64 "!\n", wr_id);
-
return -1;
}
struct ibv_comp_channel *comp_channel)
{
struct rdma_cm_event *cm_event;
- int ret;
/*
* Coroutine doesn't start until migration_fd_process_incoming()
}
if (pfds[1].revents) {
- ret = rdma_get_cm_event(rdma->channel, &cm_event);
- if (ret < 0) {
- error_report("failed to get cm event while wait "
- "completion channel");
+ if (rdma_get_cm_event(rdma->channel, &cm_event) < 0) {
return -1;
}
- error_report("receive cm event while wait comp channel,"
- "cm event is %d", cm_event->event);
if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED ||
cm_event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) {
rdma_ack_cm_event(cm_event);
default: /* Error of some type -
* I don't trust errno from qemu_poll_ns
*/
- error_report("%s: poll failed", __func__);
return -1;
}
ret = ibv_get_cq_event(ch, &cq, &cq_ctx);
if (ret < 0) {
- /*
- * FIXME perror() is problematic, because ibv_reg_mr() is
- * not documented to set errno. Will go away later in
- * this series.
- */
- perror("ibv_get_cq_event");
goto err_block_for_wrid;
}
* containing some data and block until the post completes.
*/
static int qemu_rdma_post_send_control(RDMAContext *rdma, uint8_t *buf,
- RDMAControlHeader *head)
+ RDMAControlHeader *head,
+ Error **errp)
{
int ret;
RDMAWorkRequestData *wr = &rdma->wr_data[RDMA_WRID_CONTROL];
ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr);
if (ret > 0) {
- error_report("Failed to use post IB SEND for control");
+ error_setg(errp, "Failed to use post IB SEND for control");
return -1;
}
ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_SEND_CONTROL, NULL);
if (ret < 0) {
- error_report("rdma migration: send polling control error");
+ error_setg(errp, "rdma migration: send polling control error");
return -1;
}
* Post a RECV work request in anticipation of some future receipt
* of data on the control channel.
*/
-static int qemu_rdma_post_recv_control(RDMAContext *rdma, int idx)
+static int qemu_rdma_post_recv_control(RDMAContext *rdma, int idx,
+ Error **errp)
{
struct ibv_recv_wr *bad_wr;
struct ibv_sge sge = {
if (ibv_post_recv(rdma->qp, &recv_wr, &bad_wr)) {
+ error_setg(errp, "error posting control recv");
return -1;
}
* If the user is expecting a response, post a WR in anticipation of it.
*/
if (resp) {
- ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA);
+ ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA, errp);
if (ret < 0) {
- error_setg(errp, "rdma migration: error posting"
- " extra control recv for anticipated result!");
return -1;
}
}
/*
* Post a WR to replace the one we just consumed for the READY message.
*/
- ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+ ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, errp);
if (ret < 0) {
- error_setg(errp, "rdma migration: error posting first control recv!");
return -1;
}
/*
* Deliver the control message that was requested.
*/
- ret = qemu_rdma_post_send_control(rdma, data, head);
+ ret = qemu_rdma_post_send_control(rdma, data, head, errp);
if (ret < 0) {
- error_setg(errp, "Failed to send control buffer!");
return -1;
}
/*
* Inform the source that we're ready to receive a message.
*/
- ret = qemu_rdma_post_send_control(rdma, NULL, &ready);
+ ret = qemu_rdma_post_send_control(rdma, NULL, &ready, errp);
if (ret < 0) {
- error_setg(errp, "Failed to send control buffer!");
return -1;
}
/*
* Post a new RECV work request to replace the one we just consumed.
*/
- ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+ ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, errp);
if (ret < 0) {
- error_setg(errp, "rdma migration: error posting second control recv!");
return -1;
}
stat64_add(&mig_stats.normal_pages, sge.length / qemu_target_page_size());
/*
* We are adding to transferred the amount of data written, but no
- * overhead at all. I will asume that RDMA is magicaly and don't
+ * overhead at all. I will assume that RDMA is magicaly and don't
* need to transfer (at least) the addresses where it wants to
* write the pages. Here it looks like it should be something
* like:
*/
static int qemu_rdma_write(RDMAContext *rdma,
uint64_t block_offset, uint64_t offset,
- uint64_t len)
+ uint64_t len, Error **errp)
{
- Error *err = NULL;
uint64_t current_addr = block_offset + offset;
uint64_t index = rdma->current_index;
uint64_t chunk = rdma->current_chunk;
- int ret;
/* If we cannot merge it, we flush the current buffer first. */
if (!qemu_rdma_buffer_mergeable(rdma, current_addr, len)) {
- ret = qemu_rdma_write_flush(rdma, &err);
- if (ret < 0) {
- error_report_err(err);
+ if (qemu_rdma_write_flush(rdma, errp) < 0) {
return -1;
}
rdma->current_length = 0;
/* flush it if buffer is too large */
if (rdma->current_length >= RDMA_MERGE_MAX) {
- if (qemu_rdma_write_flush(rdma, &err) < 0) {
- error_report_err(err);
- return -1;
- }
+ return qemu_rdma_write_flush(rdma, errp);
}
return 0;
static void qemu_rdma_cleanup(RDMAContext *rdma)
{
- int idx;
+ Error *err = NULL;
if (rdma->cm_id && rdma->connected) {
if ((rdma->errored ||
.type = RDMA_CONTROL_ERROR,
.repeat = 1,
};
- error_report("Early error. Sending error.");
- qemu_rdma_post_send_control(rdma, NULL, &head);
+ warn_report("Early error. Sending error.");
+ if (qemu_rdma_post_send_control(rdma, NULL, &head, &err) < 0) {
+ warn_report_err(err);
+ }
}
rdma_disconnect(rdma->cm_id);
g_free(rdma->dest_blocks);
rdma->dest_blocks = NULL;
- for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
- if (rdma->wr_data[idx].control_mr) {
+ for (int i = 0; i < RDMA_WRID_MAX; i++) {
+ if (rdma->wr_data[i].control_mr) {
rdma->total_registrations--;
- ibv_dereg_mr(rdma->wr_data[idx].control_mr);
+ ibv_dereg_mr(rdma->wr_data[i].control_mr);
}
- rdma->wr_data[idx].control_mr = NULL;
+ rdma->wr_data[i].control_mr = NULL;
}
if (rdma->local_ram_blocks.block) {
rdma->channel = NULL;
}
g_free(rdma->host);
- g_free(rdma->host_port);
rdma->host = NULL;
- rdma->host_port = NULL;
}
static int qemu_rdma_source_init(RDMAContext *rdma, bool pin_all, Error **errp)
{
- int ret, idx;
+ int ret;
/*
* Will be validated against destination's actual capabilities
goto err_rdma_source_init;
}
- ret = qemu_rdma_alloc_pd_cq(rdma);
+ ret = qemu_rdma_alloc_pd_cq(rdma, errp);
if (ret < 0) {
- error_setg(errp, "RDMA ERROR: "
- "rdma migration: error allocating pd and cq! Your mlock()"
- " limits may be too low. Please check $ ulimit -a # and "
- "search for 'ulimit -l' in the output");
goto err_rdma_source_init;
}
/* Build the hash that maps from offset to RAMBlock */
rdma->blockmap = g_hash_table_new(g_direct_hash, g_direct_equal);
- for (idx = 0; idx < rdma->local_ram_blocks.nb_blocks; idx++) {
+ for (int i = 0; i < rdma->local_ram_blocks.nb_blocks; i++) {
g_hash_table_insert(rdma->blockmap,
- (void *)(uintptr_t)rdma->local_ram_blocks.block[idx].offset,
- &rdma->local_ram_blocks.block[idx]);
+ (void *)(uintptr_t)rdma->local_ram_blocks.block[i].offset,
+ &rdma->local_ram_blocks.block[i]);
}
- for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
- ret = qemu_rdma_reg_control(rdma, idx);
+ for (int i = 0; i < RDMA_WRID_MAX; i++) {
+ ret = qemu_rdma_reg_control(rdma, i);
if (ret < 0) {
- error_setg(errp,
- "RDMA ERROR: rdma migration: error registering %d control!",
- idx);
+ error_setg(errp, "RDMA ERROR: rdma migration: error "
+ "registering %d control!", i);
goto err_rdma_source_init;
}
}
caps_to_network(&cap);
- ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+ ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, errp);
if (ret < 0) {
- error_setg(errp, "RDMA ERROR: posting second control recv");
goto err_rdma_source_connect;
}
ret = rdma_connect(rdma->cm_id, &conn_param);
if (ret < 0) {
- perror("rdma_connect");
- error_setg(errp, "RDMA ERROR: connecting to destination!");
+ error_setg_errno(errp, errno,
+ "RDMA ERROR: connecting to destination!");
goto err_rdma_source_connect;
}
} else {
ret = rdma_get_cm_event(rdma->channel, &cm_event);
if (ret < 0) {
- error_setg(errp, "RDMA ERROR: failed to get cm event");
+ error_setg_errno(errp, errno,
+ "RDMA ERROR: failed to get cm event");
}
}
if (ret < 0) {
- /*
- * FIXME perror() is wrong, because
- * qemu_get_cm_event_timeout() can fail without setting errno.
- * Will go away later in this series.
- */
- perror("rdma_get_cm_event after rdma_connect");
goto err_rdma_source_connect;
}
if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) {
- error_report("rdma_get_cm_event != EVENT_ESTABLISHED after rdma_connect");
error_setg(errp, "RDMA ERROR: connecting to destination!");
rdma_ack_cm_event(cm_event);
goto err_rdma_source_connect;
static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
{
Error *err = NULL;
- int ret, idx;
+ int ret;
struct rdma_cm_id *listen_id;
char ip[40] = "unknown";
struct rdma_addrinfo *res, *e;
char port_str[16];
int reuse = 1;
- for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
- rdma->wr_data[idx].control_len = 0;
- rdma->wr_data[idx].control_curr = NULL;
+ for (int i = 0; i < RDMA_WRID_MAX; i++) {
+ rdma->wr_data[i].control_len = 0;
+ rdma->wr_data[i].control_curr = NULL;
}
if (!rdma->host || !rdma->host[0]) {
static void qemu_rdma_return_path_dest_init(RDMAContext *rdma_return_path,
RDMAContext *rdma)
{
- int idx;
-
- for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
- rdma_return_path->wr_data[idx].control_len = 0;
- rdma_return_path->wr_data[idx].control_curr = NULL;
+ for (int i = 0; i < RDMA_WRID_MAX; i++) {
+ rdma_return_path->wr_data[i].control_len = 0;
+ rdma_return_path->wr_data[i].control_curr = NULL;
}
/*the CM channel and CM id is shared*/
rdma_return_path->is_return_path = true;
}
-static RDMAContext *qemu_rdma_data_init(const char *host_port, Error **errp)
+static RDMAContext *qemu_rdma_data_init(InetSocketAddress *saddr, Error **errp)
{
RDMAContext *rdma = NULL;
- InetSocketAddress *addr;
rdma = g_new0(RDMAContext, 1);
rdma->current_index = -1;
rdma->current_chunk = -1;
- addr = g_new(InetSocketAddress, 1);
- if (!inet_parse(addr, host_port, NULL)) {
- rdma->port = atoi(addr->port);
- rdma->host = g_strdup(addr->host);
- rdma->host_port = g_strdup(host_port);
- } else {
- error_setg(errp, "RDMA ERROR: bad RDMA migration address '%s'",
- host_port);
- g_free(rdma);
- rdma = NULL;
- }
-
- qapi_free_InetSocketAddress(addr);
+ rdma->host = g_strdup(saddr->host);
+ rdma->port = atoi(saddr->port);
return rdma;
}
RDMAContext *rdma;
int ret;
ssize_t done = 0;
- size_t i, len;
+ size_t len;
RCU_READ_LOCK_GUARD();
rdma = qatomic_rcu_read(&rioc->rdmaout);
return -1;
}
- for (i = 0; i < niov; i++) {
+ for (int i = 0; i < niov; i++) {
size_t remaining = iov[i].iov_len;
uint8_t * data = (void *)iov[i].iov_base;
while (remaining) {
RDMAControlHeader head;
int ret;
ssize_t done = 0;
- size_t i, len;
+ size_t len;
RCU_READ_LOCK_GUARD();
rdma = qatomic_rcu_read(&rioc->rdmain);
return -1;
}
- for (i = 0; i < niov; i++) {
+ for (int i = 0; i < niov; i++) {
size_t want = iov[i].iov_len;
uint8_t *data = (void *)iov[i].iov_base;
static int qemu_rdma_drain_cq(RDMAContext *rdma)
{
Error *err = NULL;
- int ret;
if (qemu_rdma_write_flush(rdma, &err) < 0) {
error_report_err(err);
}
while (rdma->nb_sent) {
- ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
- if (ret < 0) {
+ if (qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL) < 0) {
error_report("rdma migration: complete polling error!");
return -1;
}
ram_addr_t offset, size_t size)
{
QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
+ Error *err = NULL;
RDMAContext *rdma;
int ret;
- if (migration_in_postcopy()) {
- return RAM_SAVE_CONTROL_NOT_SUPP;
- }
-
RCU_READ_LOCK_GUARD();
rdma = qatomic_rcu_read(&rioc->rdmaout);
* is full, or the page doesn't belong to the current chunk,
* an actual RDMA write will occur and a new chunk will be formed.
*/
- ret = qemu_rdma_write(rdma, block_offset, offset, size);
+ ret = qemu_rdma_write(rdma, block_offset, offset, size, &err);
if (ret < 0) {
- error_report("rdma migration: write error");
+ error_report_err(err);
goto err;
}
return -1;
}
+int rdma_control_save_page(QEMUFile *f, ram_addr_t block_offset,
+ ram_addr_t offset, size_t size)
+{
+ if (!migrate_rdma() || migration_in_postcopy()) {
+ return RAM_SAVE_CONTROL_NOT_SUPP;
+ }
+
+ int ret = qemu_rdma_save_page(f, block_offset, offset, size);
+
+ if (ret != RAM_SAVE_CONTROL_DELAYED &&
+ ret != RAM_SAVE_CONTROL_NOT_SUPP) {
+ if (ret < 0) {
+ qemu_file_set_error(f, ret);
+ }
+ }
+ return ret;
+}
+
static void rdma_accept_incoming_migration(void *opaque);
static void rdma_cm_poll_handler(void *opaque)
{
RDMAContext *rdma = opaque;
- int ret;
struct rdma_cm_event *cm_event;
MigrationIncomingState *mis = migration_incoming_get_current();
- ret = rdma_get_cm_event(rdma->channel, &cm_event);
- if (ret < 0) {
+ if (rdma_get_cm_event(rdma->channel, &cm_event) < 0) {
error_report("get_cm_event failed %d", errno);
return;
}
static int qemu_rdma_accept(RDMAContext *rdma)
{
+ Error *err = NULL;
RDMACapabilities cap;
struct rdma_conn_param conn_param = {
.responder_resources = 2,
.private_data_len = sizeof(cap),
};
RDMAContext *rdma_return_path = NULL;
+ g_autoptr(InetSocketAddress) isock = g_new0(InetSocketAddress, 1);
struct rdma_cm_event *cm_event;
struct ibv_context *verbs;
int ret;
- int idx;
ret = rdma_get_cm_event(rdma->channel, &cm_event);
if (ret < 0) {
goto err_rdma_dest_wait;
}
+ isock->host = rdma->host;
+ isock->port = g_strdup_printf("%d", rdma->port);
+
/*
* initialize the RDMAContext for return path for postcopy after first
* connection request reached.
*/
if ((migrate_postcopy() || migrate_return_path())
&& !rdma->is_return_path) {
- rdma_return_path = qemu_rdma_data_init(rdma->host_port, NULL);
+ rdma_return_path = qemu_rdma_data_init(isock, NULL);
if (rdma_return_path == NULL) {
rdma_ack_cm_event(cm_event);
goto err_rdma_dest_wait;
qemu_rdma_dump_id("dest_init", verbs);
- ret = qemu_rdma_alloc_pd_cq(rdma);
+ ret = qemu_rdma_alloc_pd_cq(rdma, &err);
if (ret < 0) {
- error_report("rdma migration: error allocating pd and cq!");
+ error_report_err(err);
goto err_rdma_dest_wait;
}
qemu_rdma_init_ram_blocks(rdma);
- for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
- ret = qemu_rdma_reg_control(rdma, idx);
+ for (int i = 0; i < RDMA_WRID_MAX; i++) {
+ ret = qemu_rdma_reg_control(rdma, i);
if (ret < 0) {
- error_report("rdma: error registering %d control", idx);
+ error_report("rdma: error registering %d control", i);
goto err_rdma_dest_wait;
}
}
rdma_ack_cm_event(cm_event);
rdma->connected = true;
- ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+ ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, &err);
if (ret < 0) {
- error_report("rdma migration: error posting second control recv");
+ error_report_err(err);
goto err_rdma_dest_wait;
}
*
* Keep doing this until the source tells us to stop.
*/
-static int qemu_rdma_registration_handle(QEMUFile *f)
+int rdma_registration_handle(QEMUFile *f)
{
RDMAControlHeader reg_resp = { .len = sizeof(RDMARegisterResult),
.type = RDMA_CONTROL_REGISTER_RESULT,
};
RDMAControlHeader blocks = { .type = RDMA_CONTROL_RAM_BLOCKS_RESULT,
.repeat = 1 };
- QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
+ QIOChannelRDMA *rioc;
Error *err = NULL;
RDMAContext *rdma;
RDMALocalBlocks *local;
void *host_addr;
int ret;
int idx = 0;
- int count = 0;
- int i = 0;
+
+ if (!migrate_rdma()) {
+ return 0;
+ }
RCU_READ_LOCK_GUARD();
+ rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
rdma = qatomic_rcu_read(&rioc->rdmain);
if (!rdma) {
local = &rdma->local_ram_blocks;
do {
- trace_qemu_rdma_registration_handle_wait();
+ trace_rdma_registration_handle_wait();
ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_NONE, &err);
comp = (RDMACompress *) rdma->wr_data[idx].control_curr;
network_to_compress(comp);
- trace_qemu_rdma_registration_handle_compress(comp->length,
- comp->block_idx,
- comp->offset);
+ trace_rdma_registration_handle_compress(comp->length,
+ comp->block_idx,
+ comp->offset);
if (comp->block_idx >= rdma->local_ram_blocks.nb_blocks) {
error_report("rdma: 'compress' bad block index %u (vs %d)",
(unsigned int)comp->block_idx,
host_addr = block->local_host_addr +
(comp->offset - block->offset);
-
- ram_handle_compressed(host_addr, comp->value, comp->length);
+ if (comp->value) {
+ error_report("rdma: Zero page with non-zero (%d) value",
+ comp->value);
+ goto err;
+ }
+ ram_handle_zero(host_addr, comp->length);
break;
case RDMA_CONTROL_REGISTER_FINISHED:
- trace_qemu_rdma_registration_handle_finished();
+ trace_rdma_registration_handle_finished();
return 0;
case RDMA_CONTROL_RAM_BLOCKS_REQUEST:
- trace_qemu_rdma_registration_handle_ram_blocks();
+ trace_rdma_registration_handle_ram_blocks();
/* Sort our local RAM Block list so it's the same as the source,
* we can do this since we've filled in a src_index in the list
qsort(rdma->local_ram_blocks.block,
rdma->local_ram_blocks.nb_blocks,
sizeof(RDMALocalBlock), dest_ram_sort_func);
- for (i = 0; i < local->nb_blocks; i++) {
+ for (int i = 0; i < local->nb_blocks; i++) {
local->block[i].index = i;
}
* Both sides use the "remote" structure to communicate and update
* their "local" descriptions with what was sent.
*/
- for (i = 0; i < local->nb_blocks; i++) {
+ for (int i = 0; i < local->nb_blocks; i++) {
rdma->dest_blocks[i].remote_host_addr =
(uintptr_t)(local->block[i].local_host_addr);
rdma->dest_blocks[i].length = local->block[i].length;
dest_block_to_network(&rdma->dest_blocks[i]);
- trace_qemu_rdma_registration_handle_ram_blocks_loop(
+ trace_rdma_registration_handle_ram_blocks_loop(
local->block[i].block_name,
local->block[i].offset,
local->block[i].length,
ret = qemu_rdma_post_send_control(rdma,
- (uint8_t *) rdma->dest_blocks, &blocks);
+ (uint8_t *) rdma->dest_blocks, &blocks,
+ &err);
if (ret < 0) {
- error_report("rdma migration: error sending remote info");
+ error_report_err(err);
goto err;
}
break;
case RDMA_CONTROL_REGISTER_REQUEST:
- trace_qemu_rdma_registration_handle_register(head.repeat);
+ trace_rdma_registration_handle_register(head.repeat);
reg_resp.repeat = head.repeat;
registers = (RDMARegister *) rdma->wr_data[idx].control_curr;
- for (count = 0; count < head.repeat; count++) {
+ for (int count = 0; count < head.repeat; count++) {
uint64_t chunk;
uint8_t *chunk_start, *chunk_end;
reg_result = &results[count];
- trace_qemu_rdma_registration_handle_register_loop(count,
+ trace_rdma_registration_handle_register_loop(count,
reg->current_index, reg->key.current_addr, reg->chunks);
if (reg->current_index >= rdma->local_ram_blocks.nb_blocks) {
reg_result->host_addr = (uintptr_t)block->local_host_addr;
- trace_qemu_rdma_registration_handle_register_rkey(
- reg_result->rkey);
+ trace_rdma_registration_handle_register_rkey(reg_result->rkey);
result_to_network(reg_result);
}
ret = qemu_rdma_post_send_control(rdma,
- (uint8_t *) results, ®_resp);
+ (uint8_t *) results, ®_resp, &err);
if (ret < 0) {
- error_report("Failed to send control buffer");
+ error_report_err(err);
goto err;
}
break;
case RDMA_CONTROL_UNREGISTER_REQUEST:
- trace_qemu_rdma_registration_handle_unregister(head.repeat);
+ trace_rdma_registration_handle_unregister(head.repeat);
unreg_resp.repeat = head.repeat;
registers = (RDMARegister *) rdma->wr_data[idx].control_curr;
- for (count = 0; count < head.repeat; count++) {
+ for (int count = 0; count < head.repeat; count++) {
reg = ®isters[count];
network_to_register(reg);
- trace_qemu_rdma_registration_handle_unregister_loop(count,
+ trace_rdma_registration_handle_unregister_loop(count,
reg->current_index, reg->key.chunk);
block = &(rdma->local_ram_blocks.block[reg->current_index]);
block->pmr[reg->key.chunk] = NULL;
if (ret != 0) {
- perror("rdma unregistration chunk failed");
+ error_report("rdma unregistration chunk failed: %s",
+ strerror(errno));
goto err;
}
rdma->total_registrations--;
- trace_qemu_rdma_registration_handle_unregister_success(
- reg->key.chunk);
+ trace_rdma_registration_handle_unregister_success(reg->key.chunk);
}
- ret = qemu_rdma_post_send_control(rdma, NULL, &unreg_resp);
+ ret = qemu_rdma_post_send_control(rdma, NULL, &unreg_resp, &err);
if (ret < 0) {
- error_report("Failed to send control buffer");
+ error_report_err(err);
goto err;
}
break;
}
/* Destination:
- * Called via a ram_control_load_hook during the initial RAM load section which
- * lists the RAMBlocks by name. This lets us know the order of the RAMBlocks
- * on the source.
- * We've already built our local RAMBlock list, but not yet sent the list to
- * the source.
+ * Called during the initial RAM load section which lists the
+ * RAMBlocks by name. This lets us know the order of the RAMBlocks on
+ * the source. We've already built our local RAMBlock list, but not
+ * yet sent the list to the source.
*/
-static int
-rdma_block_notification_handle(QEMUFile *f, const char *name)
+int rdma_block_notification_handle(QEMUFile *f, const char *name)
{
- RDMAContext *rdma;
- QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
int curr;
int found = -1;
+ if (!migrate_rdma()) {
+ return 0;
+ }
+
RCU_READ_LOCK_GUARD();
- rdma = qatomic_rcu_read(&rioc->rdmain);
+ QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
+ RDMAContext *rdma = qatomic_rcu_read(&rioc->rdmain);
if (!rdma) {
return -1;
return 0;
}
-static int rdma_load_hook(QEMUFile *f, uint64_t flags, void *data)
-{
- switch (flags) {
- case RAM_CONTROL_BLOCK_REG:
- return rdma_block_notification_handle(f, data);
-
- case RAM_CONTROL_HOOK:
- return qemu_rdma_registration_handle(f);
-
- default:
- /* Shouldn't be called with any other values */
- abort();
- }
-}
-
-static int qemu_rdma_registration_start(QEMUFile *f,
- uint64_t flags, void *data)
+int rdma_registration_start(QEMUFile *f, uint64_t flags)
{
- QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
- RDMAContext *rdma;
-
- if (migration_in_postcopy()) {
+ if (!migrate_rdma() || migration_in_postcopy()) {
return 0;
}
+ QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
RCU_READ_LOCK_GUARD();
- rdma = qatomic_rcu_read(&rioc->rdmaout);
+ RDMAContext *rdma = qatomic_rcu_read(&rioc->rdmaout);
if (!rdma) {
return -1;
}
return -1;
}
- trace_qemu_rdma_registration_start(flags);
+ trace_rdma_registration_start(flags);
qemu_put_be64(f, RAM_SAVE_FLAG_HOOK);
- qemu_fflush(f);
-
- return 0;
+ return qemu_fflush(f);
}
/*
* Inform dest that dynamic registrations are done for now.
* First, flush writes, if any.
*/
-static int qemu_rdma_registration_stop(QEMUFile *f,
- uint64_t flags, void *data)
+int rdma_registration_stop(QEMUFile *f, uint64_t flags)
{
- QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
+ QIOChannelRDMA *rioc;
Error *err = NULL;
RDMAContext *rdma;
RDMAControlHeader head = { .len = 0, .repeat = 1 };
int ret;
- if (migration_in_postcopy()) {
+ if (!migrate_rdma() || migration_in_postcopy()) {
return 0;
}
RCU_READ_LOCK_GUARD();
+ rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
rdma = qatomic_rcu_read(&rioc->rdmaout);
if (!rdma) {
return -1;
if (flags == RAM_CONTROL_SETUP) {
RDMAControlHeader resp = {.type = RDMA_CONTROL_RAM_BLOCKS_RESULT };
RDMALocalBlocks *local = &rdma->local_ram_blocks;
- int reg_result_idx, i, nb_dest_blocks;
+ int reg_result_idx, nb_dest_blocks;
head.type = RDMA_CONTROL_RAM_BLOCKS_REQUEST;
- trace_qemu_rdma_registration_stop_ram();
+ trace_rdma_registration_stop_ram();
/*
* Make sure that we parallelize the pinning on both sides.
*/
if (local->nb_blocks != nb_dest_blocks) {
- fprintf(stderr, "ram blocks mismatch (Number of blocks %d vs %d) "
- "Your QEMU command line parameters are probably "
- "not identical on both the source and destination.",
- local->nb_blocks, nb_dest_blocks);
+ error_report("ram blocks mismatch (Number of blocks %d vs %d)",
+ local->nb_blocks, nb_dest_blocks);
+ error_printf("Your QEMU command line parameters are probably "
+ "not identical on both the source and destination.");
rdma->errored = true;
return -1;
}
qemu_rdma_move_header(rdma, reg_result_idx, &resp);
memcpy(rdma->dest_blocks,
rdma->wr_data[reg_result_idx].control_curr, resp.len);
- for (i = 0; i < nb_dest_blocks; i++) {
+ for (int i = 0; i < nb_dest_blocks; i++) {
network_to_dest_block(&rdma->dest_blocks[i]);
/* We require that the blocks are in the same order */
if (rdma->dest_blocks[i].length != local->block[i].length) {
- fprintf(stderr, "Block %s/%d has a different length %" PRIu64
- "vs %" PRIu64, local->block[i].block_name, i,
- local->block[i].length,
- rdma->dest_blocks[i].length);
+ error_report("Block %s/%d has a different length %" PRIu64
+ "vs %" PRIu64,
+ local->block[i].block_name, i,
+ local->block[i].length,
+ rdma->dest_blocks[i].length);
rdma->errored = true;
return -1;
}
}
}
- trace_qemu_rdma_registration_stop(flags);
+ trace_rdma_registration_stop(flags);
head.type = RDMA_CONTROL_REGISTER_FINISHED;
ret = qemu_rdma_exchange_send(rdma, &head, NULL, NULL, NULL, NULL, &err);
return -1;
}
-static const QEMUFileHooks rdma_read_hooks = {
- .hook_ram_load = rdma_load_hook,
-};
-
-static const QEMUFileHooks rdma_write_hooks = {
- .before_ram_iterate = qemu_rdma_registration_start,
- .after_ram_iterate = qemu_rdma_registration_stop,
- .save_page = qemu_rdma_save_page,
-};
-
-
static void qio_channel_rdma_finalize(Object *obj)
{
QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(obj);
rioc->file = qemu_file_new_input(QIO_CHANNEL(rioc));
rioc->rdmain = rdma;
rioc->rdmaout = rdma->return_path;
- qemu_file_set_hooks(rioc->file, &rdma_read_hooks);
return rioc->file;
}
rioc->file = qemu_file_new_output(QIO_CHANNEL(rioc));
rioc->rdmaout = rdma;
rioc->rdmain = rdma->return_path;
- qemu_file_set_hooks(rioc->file, &rdma_write_hooks);
return rioc->file;
}
static void rdma_accept_incoming_migration(void *opaque)
{
RDMAContext *rdma = opaque;
- int ret;
QEMUFile *f;
Error *local_err = NULL;
trace_qemu_rdma_accept_incoming_migration();
- ret = qemu_rdma_accept(rdma);
-
- if (ret < 0) {
- fprintf(stderr, "RDMA ERROR: Migration initialization failed\n");
+ if (qemu_rdma_accept(rdma) < 0) {
+ error_report("RDMA ERROR: Migration initialization failed");
return;
}
f = rdma_new_input(rdma);
if (f == NULL) {
- fprintf(stderr, "RDMA ERROR: could not open RDMA for input\n");
+ error_report("RDMA ERROR: could not open RDMA for input");
qemu_rdma_cleanup(rdma);
return;
}
}
}
-void rdma_start_incoming_migration(const char *host_port, Error **errp)
+void rdma_start_incoming_migration(InetSocketAddress *host_port,
+ Error **errp)
{
+ MigrationState *s = migrate_get_current();
int ret;
RDMAContext *rdma;
}
trace_rdma_start_incoming_migration_after_rdma_listen();
-
+ s->rdma_migration = true;
qemu_set_fd_handler(rdma->channel->fd, rdma_accept_incoming_migration,
NULL, (void *)(intptr_t)rdma);
return;
err:
if (rdma) {
g_free(rdma->host);
- g_free(rdma->host_port);
}
g_free(rdma);
}
void rdma_start_outgoing_migration(void *opaque,
- const char *host_port, Error **errp)
+ InetSocketAddress *host_port, Error **errp)
{
MigrationState *s = opaque;
RDMAContext *rdma_return_path = NULL;
trace_rdma_start_outgoing_migration_after_rdma_connect();
s->to_dst_file = rdma_new_output(rdma);
+ s->rdma_migration = true;
migrate_fd_connect(s, NULL);
return;
return_path_err:
projects / mirror_qemu.git / blobdiff