[mirror_qemu.git] / hw / rdma / vmw / pvrdma_qp_ops.c

/*
 * QEMU paravirtual RDMA - QP implementation
 *
 * Copyright (C) 2018 Oracle
 * Copyright (C) 2018 Red Hat Inc
 *
 * Authors:
 *     Yuval Shaia <yuval.shaia@oracle.com>
 *     Marcel Apfelbaum <marcel@redhat.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 *
 */

#include "qemu/osdep.h"

#include "../rdma_utils.h"
#include "../rdma_rm.h"
#include "../rdma_backend.h"

#include "pvrdma.h"
#include "standard-headers/rdma/vmw_pvrdma-abi.h"
#include "pvrdma_qp_ops.h"

typedef struct CompHandlerCtx {
    PVRDMADev *dev;
    uint32_t cq_handle;
    struct pvrdma_cqe cqe;
} CompHandlerCtx;

/* Send Queue WQE */
typedef struct PvrdmaSqWqe {
    struct pvrdma_sq_wqe_hdr hdr;
    struct pvrdma_sge sge[0];
} PvrdmaSqWqe;

/* Recv Queue WQE */
typedef struct PvrdmaRqWqe {
    struct pvrdma_rq_wqe_hdr hdr;
    struct pvrdma_sge sge[0];
} PvrdmaRqWqe;

/*
 * 1. Put CQE on send CQ ring
 * 2. Put CQ number on dsr completion ring
 * 3. Interrupt host
 */
static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq_handle,
                           struct pvrdma_cqe *cqe, struct ibv_wc *wc)
{
    struct pvrdma_cqe *cqe1;
    struct pvrdma_cqne *cqne;
    PvrdmaRing *ring;
    RdmaRmCQ *cq = rdma_rm_get_cq(&dev->rdma_dev_res, cq_handle);

    if (unlikely(!cq)) {
        pr_dbg("Invalid cqn %d\n", cq_handle);
        return -EINVAL;
    }

    ring = (PvrdmaRing *)cq->opaque;
    pr_dbg("ring=%p\n", ring);

    /* Step #1: Put CQE on CQ ring */
    pr_dbg("Writing CQE\n");
    cqe1 = pvrdma_ring_next_elem_write(ring);
    if (unlikely(!cqe1)) {
        pr_dbg("No CQEs in ring\n");
        return -EINVAL;
    }

    memset(cqe1, 0, sizeof(*cqe1));
    cqe1->wr_id = cqe->wr_id;
    cqe1->qp = cqe->qp;
    cqe1->opcode = cqe->opcode;
    cqe1->status = wc->status;
    cqe1->byte_len = wc->byte_len;
    cqe1->src_qp = wc->src_qp;
    cqe1->wc_flags = wc->wc_flags;
    cqe1->vendor_err = wc->vendor_err;

    pr_dbg("wr_id=%" PRIx64 "\n", cqe1->wr_id);
    pr_dbg("qp=0x%lx\n", cqe1->qp);
    pr_dbg("opcode=%d\n", cqe1->opcode);
    pr_dbg("status=%d\n", cqe1->status);
    pr_dbg("byte_len=%d\n", cqe1->byte_len);
    pr_dbg("src_qp=%d\n", cqe1->src_qp);
    pr_dbg("wc_flags=%d\n", cqe1->wc_flags);
    pr_dbg("vendor_err=%d\n", cqe1->vendor_err);

    pvrdma_ring_write_inc(ring);

    /* Step #2: Put CQ number on dsr completion ring */
    pr_dbg("Writing CQNE\n");
    cqne = pvrdma_ring_next_elem_write(&dev->dsr_info.cq);
    if (unlikely(!cqne)) {
        return -EINVAL;
    }

    cqne->info = cq_handle;
    pvrdma_ring_write_inc(&dev->dsr_info.cq);

    pr_dbg("cq->notify=%d\n", cq->notify);
    if (cq->notify != CNT_CLEAR) {
        if (cq->notify == CNT_ARM) {
            cq->notify = CNT_CLEAR;
        }
        post_interrupt(dev, INTR_VEC_CMD_COMPLETION_Q);
    }

    return 0;
}

static void pvrdma_qp_ops_comp_handler(void *ctx, struct ibv_wc *wc)
{
    CompHandlerCtx *comp_ctx = (CompHandlerCtx *)ctx;

    pvrdma_post_cqe(comp_ctx->dev, comp_ctx->cq_handle, &comp_ctx->cqe, wc);

    g_free(ctx);
}

static void complete_with_error(uint32_t vendor_err, void *ctx)
{
    struct ibv_wc wc = {0};

    wc.status = IBV_WC_GENERAL_ERR;
    wc.vendor_err = vendor_err;

    pvrdma_qp_ops_comp_handler(ctx, &wc);
}

void pvrdma_qp_ops_fini(void)
{
    rdma_backend_unregister_comp_handler();
}

int pvrdma_qp_ops_init(void)
{
    rdma_backend_register_comp_handler(pvrdma_qp_ops_comp_handler);

    return 0;
}

int pvrdma_qp_send(PVRDMADev *dev, uint32_t qp_handle)
{
    RdmaRmQP *qp;
    PvrdmaSqWqe *wqe;
    PvrdmaRing *ring;
    int sgid_idx;
    union ibv_gid *sgid;

    pr_dbg("qp_handle=0x%x\n", qp_handle);

    qp = rdma_rm_get_qp(&dev->rdma_dev_res, qp_handle);
    if (unlikely(!qp)) {
        return -EINVAL;
    }

    ring = (PvrdmaRing *)qp->opaque;
    pr_dbg("sring=%p\n", ring);

    wqe = (struct PvrdmaSqWqe *)pvrdma_ring_next_elem_read(ring);
    while (wqe) {
        CompHandlerCtx *comp_ctx;

        pr_dbg("wr_id=%" PRIx64 "\n", wqe->hdr.wr_id);

        /* Prepare CQE */
        comp_ctx = g_malloc(sizeof(CompHandlerCtx));
        comp_ctx->dev = dev;
        comp_ctx->cq_handle = qp->send_cq_handle;
        comp_ctx->cqe.wr_id = wqe->hdr.wr_id;
        comp_ctx->cqe.qp = qp_handle;
        comp_ctx->cqe.opcode = IBV_WC_SEND;

        sgid = rdma_rm_get_gid(&dev->rdma_dev_res, wqe->hdr.wr.ud.av.gid_index);
        if (!sgid) {
            pr_dbg("Fail to get gid for idx %d\n", wqe->hdr.wr.ud.av.gid_index);
            complete_with_error(VENDOR_ERR_INV_GID_IDX, comp_ctx);
            continue;
        }
        pr_dbg("sgid_id=%d, sgid=0x%llx\n", wqe->hdr.wr.ud.av.gid_index,
               sgid->global.interface_id);

        sgid_idx = rdma_rm_get_backend_gid_index(&dev->rdma_dev_res,
                                                 &dev->backend_dev,
                                                 wqe->hdr.wr.ud.av.gid_index);
        if (sgid_idx <= 0) {
            pr_dbg("Fail to get bk sgid_idx for sgid_idx %d\n",
                   wqe->hdr.wr.ud.av.gid_index);
            complete_with_error(VENDOR_ERR_INV_GID_IDX, comp_ctx);
            continue;
        }

        if (wqe->hdr.num_sge > dev->dev_attr.max_sge) {
            pr_dbg("Invalid num_sge=%d (max %d)\n", wqe->hdr.num_sge,
                   dev->dev_attr.max_sge);
            complete_with_error(VENDOR_ERR_INV_NUM_SGE, comp_ctx);
            continue;
        }

        rdma_backend_post_send(&dev->backend_dev, &qp->backend_qp, qp->qp_type,
                               (struct ibv_sge *)&wqe->sge[0], wqe->hdr.num_sge,
                               sgid_idx, sgid,
                               (union ibv_gid *)wqe->hdr.wr.ud.av.dgid,
                               wqe->hdr.wr.ud.remote_qpn,
                               wqe->hdr.wr.ud.remote_qkey, comp_ctx);

        pvrdma_ring_read_inc(ring);

        wqe = pvrdma_ring_next_elem_read(ring);
    }

    return 0;
}

int pvrdma_qp_recv(PVRDMADev *dev, uint32_t qp_handle)
{
    RdmaRmQP *qp;
    PvrdmaRqWqe *wqe;
    PvrdmaRing *ring;

    pr_dbg("qp_handle=0x%x\n", qp_handle);

    qp = rdma_rm_get_qp(&dev->rdma_dev_res, qp_handle);
    if (unlikely(!qp)) {
        return -EINVAL;
    }

    ring = &((PvrdmaRing *)qp->opaque)[1];
    pr_dbg("rring=%p\n", ring);

    wqe = (struct PvrdmaRqWqe *)pvrdma_ring_next_elem_read(ring);
    while (wqe) {
        CompHandlerCtx *comp_ctx;

        pr_dbg("wr_id=%" PRIx64 "\n", wqe->hdr.wr_id);

        /* Prepare CQE */
        comp_ctx = g_malloc(sizeof(CompHandlerCtx));
        comp_ctx->dev = dev;
        comp_ctx->cq_handle = qp->recv_cq_handle;
        comp_ctx->cqe.wr_id = wqe->hdr.wr_id;
        comp_ctx->cqe.qp = qp_handle;
        comp_ctx->cqe.opcode = IBV_WC_RECV;

        if (wqe->hdr.num_sge > dev->dev_attr.max_sge) {
            pr_dbg("Invalid num_sge=%d (max %d)\n", wqe->hdr.num_sge,
                   dev->dev_attr.max_sge);
            complete_with_error(VENDOR_ERR_INV_NUM_SGE, comp_ctx);
            continue;
        }

        rdma_backend_post_recv(&dev->backend_dev, &dev->rdma_dev_res,
                               &qp->backend_qp, qp->qp_type,
                               (struct ibv_sge *)&wqe->sge[0], wqe->hdr.num_sge,
                               comp_ctx);

        pvrdma_ring_read_inc(ring);

        wqe = pvrdma_ring_next_elem_read(ring);
    }

    return 0;
}

void pvrdma_cq_poll(RdmaDeviceResources *dev_res, uint32_t cq_handle)
{
    RdmaRmCQ *cq;

    cq = rdma_rm_get_cq(dev_res, cq_handle);
    if (!cq) {
        pr_dbg("Invalid CQ# %d\n", cq_handle);
        return;
    }

    rdma_backend_poll_cq(dev_res, &cq->backend_cq);
}
Commit	Line	Data
98d176f8 YS	1	/*
	2	* QEMU paravirtual RDMA - QP implementation
	3	*
	4	* Copyright (C) 2018 Oracle
	5	* Copyright (C) 2018 Red Hat Inc
	6	*
	7	* Authors:
	8	* Yuval Shaia <yuval.shaia@oracle.com>
	9	* Marcel Apfelbaum <marcel@redhat.com>
	10	*
	11	* This work is licensed under the terms of the GNU GPL, version 2 or later.
	12	* See the COPYING file in the top-level directory.
	13	*
	14	*/
	15
0efc9511	16	#include "qemu/osdep.h"
98d176f8 YS	17
	18	#include "../rdma_utils.h"
	19	#include "../rdma_rm.h"
	20	#include "../rdma_backend.h"
	21
	22	#include "pvrdma.h"
0efc9511	23	#include "standard-headers/rdma/vmw_pvrdma-abi.h"
98d176f8 YS	24	#include "pvrdma_qp_ops.h"
	25
	26	typedef struct CompHandlerCtx {
	27	PVRDMADev *dev;
	28	uint32_t cq_handle;
	29	struct pvrdma_cqe cqe;
	30	} CompHandlerCtx;
	31
	32	/* Send Queue WQE */
	33	typedef struct PvrdmaSqWqe {
	34	struct pvrdma_sq_wqe_hdr hdr;
	35	struct pvrdma_sge sge[0];
	36	} PvrdmaSqWqe;
	37
	38	/* Recv Queue WQE */
	39	typedef struct PvrdmaRqWqe {
	40	struct pvrdma_rq_wqe_hdr hdr;
	41	struct pvrdma_sge sge[0];
	42	} PvrdmaRqWqe;
	43
	44	/*
	45	* 1. Put CQE on send CQ ring
	46	* 2. Put CQ number on dsr completion ring
	47	* 3. Interrupt host
	48	*/
	49	static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq_handle,
eaac0100	50	struct pvrdma_cqe cqe, struct ibv_wc wc)
98d176f8 YS	51	{
	52	struct pvrdma_cqe *cqe1;
	53	struct pvrdma_cqne *cqne;
	54	PvrdmaRing *ring;
	55	RdmaRmCQ *cq = rdma_rm_get_cq(&dev->rdma_dev_res, cq_handle);
	56
	57	if (unlikely(!cq)) {
	58	pr_dbg("Invalid cqn %d\n", cq_handle);
	59	return -EINVAL;
	60	}
	61
	62	ring = (PvrdmaRing *)cq->opaque;
	63	pr_dbg("ring=%p\n", ring);
	64
	65	/* Step #1: Put CQE on CQ ring */
	66	pr_dbg("Writing CQE\n");
	67	cqe1 = pvrdma_ring_next_elem_write(ring);
	68	if (unlikely(!cqe1)) {
eaac0100	69	pr_dbg("No CQEs in ring\n");
98d176f8 YS	70	return -EINVAL;
	71	}
	72
eca0f2a6	73	memset(cqe1, 0, sizeof(*cqe1));
98d176f8 YS	74	cqe1->wr_id = cqe->wr_id;
	75	cqe1->qp = cqe->qp;
	76	cqe1->opcode = cqe->opcode;
eaac0100 YS	77	cqe1->status = wc->status;
	78	cqe1->byte_len = wc->byte_len;
	79	cqe1->src_qp = wc->src_qp;
	80	cqe1->wc_flags = wc->wc_flags;
	81	cqe1->vendor_err = wc->vendor_err;
	82
	83	pr_dbg("wr_id=%" PRIx64 "\n", cqe1->wr_id);
	84	pr_dbg("qp=0x%lx\n", cqe1->qp);
	85	pr_dbg("opcode=%d\n", cqe1->opcode);
	86	pr_dbg("status=%d\n", cqe1->status);
	87	pr_dbg("byte_len=%d\n", cqe1->byte_len);
	88	pr_dbg("src_qp=%d\n", cqe1->src_qp);
	89	pr_dbg("wc_flags=%d\n", cqe1->wc_flags);
	90	pr_dbg("vendor_err=%d\n", cqe1->vendor_err);
98d176f8 YS	91
	92	pvrdma_ring_write_inc(ring);
	93
	94	/* Step #2: Put CQ number on dsr completion ring */
	95	pr_dbg("Writing CQNE\n");
	96	cqne = pvrdma_ring_next_elem_write(&dev->dsr_info.cq);
	97	if (unlikely(!cqne)) {
	98	return -EINVAL;
	99	}
	100
	101	cqne->info = cq_handle;
	102	pvrdma_ring_write_inc(&dev->dsr_info.cq);
	103
	104	pr_dbg("cq->notify=%d\n", cq->notify);
4082e533 YS	105	if (cq->notify != CNT_CLEAR) {
	106	if (cq->notify == CNT_ARM) {
	107	cq->notify = CNT_CLEAR;
	108	}
98d176f8 YS	109	post_interrupt(dev, INTR_VEC_CMD_COMPLETION_Q);
	110	}
	111
	112	return 0;
	113	}
	114
eaac0100	115	static void pvrdma_qp_ops_comp_handler(void ctx, struct ibv_wc wc)
98d176f8 YS	116	{
	117	CompHandlerCtx comp_ctx = (CompHandlerCtx )ctx;
	118
eaac0100 YS	119	pvrdma_post_cqe(comp_ctx->dev, comp_ctx->cq_handle, &comp_ctx->cqe, wc);
eaac0100 YS	120
98d176f8 YS	121	g_free(ctx);
	122	}
	123
ffef4775 YS	124	static void complete_with_error(uint32_t vendor_err, void *ctx)
	125	{
	126	struct ibv_wc wc = {0};
	127
	128	wc.status = IBV_WC_GENERAL_ERR;
	129	wc.vendor_err = vendor_err;
	130
	131	pvrdma_qp_ops_comp_handler(ctx, &wc);
	132	}
	133
98d176f8 YS	134	void pvrdma_qp_ops_fini(void)
	135	{
	136	rdma_backend_unregister_comp_handler();
	137	}
	138
	139	int pvrdma_qp_ops_init(void)
	140	{
	141	rdma_backend_register_comp_handler(pvrdma_qp_ops_comp_handler);
	142
	143	return 0;
	144	}
	145
	146	int pvrdma_qp_send(PVRDMADev *dev, uint32_t qp_handle)
	147	{
	148	RdmaRmQP *qp;
	149	PvrdmaSqWqe *wqe;
	150	PvrdmaRing *ring;
2b05705d YS	151	int sgid_idx;
2b05705d YS	152	union ibv_gid *sgid;
98d176f8	153
abc665aa	154	pr_dbg("qp_handle=0x%x\n", qp_handle);
98d176f8 YS	155
	156	qp = rdma_rm_get_qp(&dev->rdma_dev_res, qp_handle);
	157	if (unlikely(!qp)) {
	158	return -EINVAL;
	159	}
	160
	161	ring = (PvrdmaRing *)qp->opaque;
	162	pr_dbg("sring=%p\n", ring);
	163
	164	wqe = (struct PvrdmaSqWqe *)pvrdma_ring_next_elem_read(ring);
	165	while (wqe) {
	166	CompHandlerCtx *comp_ctx;
	167
6f559013	168	pr_dbg("wr_id=%" PRIx64 "\n", wqe->hdr.wr_id);
98d176f8 YS	169
	170	/* Prepare CQE */
	171	comp_ctx = g_malloc(sizeof(CompHandlerCtx));
	172	comp_ctx->dev = dev;
	173	comp_ctx->cq_handle = qp->send_cq_handle;
	174	comp_ctx->cqe.wr_id = wqe->hdr.wr_id;
	175	comp_ctx->cqe.qp = qp_handle;
1625bb13	176	comp_ctx->cqe.opcode = IBV_WC_SEND;
98d176f8	177
2b05705d YS	178	sgid = rdma_rm_get_gid(&dev->rdma_dev_res, wqe->hdr.wr.ud.av.gid_index);
	179	if (!sgid) {
	180	pr_dbg("Fail to get gid for idx %d\n", wqe->hdr.wr.ud.av.gid_index);
26fd8695 YS	181	complete_with_error(VENDOR_ERR_INV_GID_IDX, comp_ctx);
26fd8695 YS	182	continue;
2b05705d YS	183	}
	184	pr_dbg("sgid_id=%d, sgid=0x%llx\n", wqe->hdr.wr.ud.av.gid_index,
	185	sgid->global.interface_id);
	186
	187	sgid_idx = rdma_rm_get_backend_gid_index(&dev->rdma_dev_res,
	188	&dev->backend_dev,
	189	wqe->hdr.wr.ud.av.gid_index);
	190	if (sgid_idx <= 0) {
	191	pr_dbg("Fail to get bk sgid_idx for sgid_idx %d\n",
	192	wqe->hdr.wr.ud.av.gid_index);
26fd8695 YS	193	complete_with_error(VENDOR_ERR_INV_GID_IDX, comp_ctx);
26fd8695 YS	194	continue;
2b05705d YS	195	}
2b05705d YS	196
ffef4775 YS	197	if (wqe->hdr.num_sge > dev->dev_attr.max_sge) {
	198	pr_dbg("Invalid num_sge=%d (max %d)\n", wqe->hdr.num_sge,
	199	dev->dev_attr.max_sge);
	200	complete_with_error(VENDOR_ERR_INV_NUM_SGE, comp_ctx);
	201	continue;
	202	}
	203
98d176f8 YS	204	rdma_backend_post_send(&dev->backend_dev, &qp->backend_qp, qp->qp_type,
98d176f8 YS	205	(struct ibv_sge *)&wqe->sge[0], wqe->hdr.num_sge,
2b05705d	206	sgid_idx, sgid,
98d176f8 YS	207	(union ibv_gid *)wqe->hdr.wr.ud.av.dgid,
	208	wqe->hdr.wr.ud.remote_qpn,
	209	wqe->hdr.wr.ud.remote_qkey, comp_ctx);
	210
	211	pvrdma_ring_read_inc(ring);
	212
	213	wqe = pvrdma_ring_next_elem_read(ring);
	214	}
	215
	216	return 0;
	217	}
	218
	219	int pvrdma_qp_recv(PVRDMADev *dev, uint32_t qp_handle)
	220	{
	221	RdmaRmQP *qp;
	222	PvrdmaRqWqe *wqe;
	223	PvrdmaRing *ring;
	224
abc665aa	225	pr_dbg("qp_handle=0x%x\n", qp_handle);
98d176f8 YS	226
	227	qp = rdma_rm_get_qp(&dev->rdma_dev_res, qp_handle);
	228	if (unlikely(!qp)) {
	229	return -EINVAL;
	230	}
	231
	232	ring = &((PvrdmaRing *)qp->opaque)[1];
	233	pr_dbg("rring=%p\n", ring);
	234
	235	wqe = (struct PvrdmaRqWqe *)pvrdma_ring_next_elem_read(ring);
	236	while (wqe) {
	237	CompHandlerCtx *comp_ctx;
	238
6f559013	239	pr_dbg("wr_id=%" PRIx64 "\n", wqe->hdr.wr_id);
98d176f8 YS	240
	241	/* Prepare CQE */
	242	comp_ctx = g_malloc(sizeof(CompHandlerCtx));
	243	comp_ctx->dev = dev;
	244	comp_ctx->cq_handle = qp->recv_cq_handle;
98d176f8	245	comp_ctx->cqe.wr_id = wqe->hdr.wr_id;
2bff59e6 YS	246	comp_ctx->cqe.qp = qp_handle;
2bff59e6 YS	247	comp_ctx->cqe.opcode = IBV_WC_RECV;
98d176f8	248
ffef4775 YS	249	if (wqe->hdr.num_sge > dev->dev_attr.max_sge) {
	250	pr_dbg("Invalid num_sge=%d (max %d)\n", wqe->hdr.num_sge,
	251	dev->dev_attr.max_sge);
	252	complete_with_error(VENDOR_ERR_INV_NUM_SGE, comp_ctx);
	253	continue;
	254	}
	255
98d176f8 YS	256	rdma_backend_post_recv(&dev->backend_dev, &dev->rdma_dev_res,
	257	&qp->backend_qp, qp->qp_type,
	258	(struct ibv_sge *)&wqe->sge[0], wqe->hdr.num_sge,
	259	comp_ctx);
	260
	261	pvrdma_ring_read_inc(ring);
	262
	263	wqe = pvrdma_ring_next_elem_read(ring);
	264	}
	265
	266	return 0;
	267	}
	268
	269	void pvrdma_cq_poll(RdmaDeviceResources *dev_res, uint32_t cq_handle)
	270	{
	271	RdmaRmCQ *cq;
	272
	273	cq = rdma_rm_get_cq(dev_res, cq_handle);
	274	if (!cq) {
	275	pr_dbg("Invalid CQ# %d\n", cq_handle);
b0197cf8	276	return;
98d176f8 YS	277	}
	278
	279	rdma_backend_poll_cq(dev_res, &cq->backend_cq);
	280	}