/**
* enum queue_type - type of queue
* @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and
- * read by client. Used by rxe driver only.
+ * read by client which may be a user space
+ * application or a kernel ulp.
+ * Used by rxe internals only.
* @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and
- * read by rxe driver. Used by rxe driver only.
- * @QUEUE_TYPE_TO_DRIVER: Queue is written by client and
- * read by rxe driver. Used by kernel client only.
- * @QUEUE_TYPE_FROM_DRIVER: Queue is written by rxe driver and
- * read by client. Used by kernel client only.
+ * read by rxe driver.
+ * Used by rxe internals only.
+ * @QUEUE_TYPE_FROM_ULP: Queue is written by kernel ulp and
+ * read by rxe driver.
+ * Used by kernel verbs APIs only on
+ * behalf of ulps.
+ * @QUEUE_TYPE_TO_ULP: Queue is written by rxe driver and
+ * read by kernel ulp.
+ * Used by kernel verbs APIs only on
+ * behalf of ulps.
*/
enum queue_type {
QUEUE_TYPE_TO_CLIENT,
QUEUE_TYPE_FROM_CLIENT,
- QUEUE_TYPE_TO_DRIVER,
- QUEUE_TYPE_FROM_DRIVER,
+ QUEUE_TYPE_FROM_ULP,
+ QUEUE_TYPE_TO_ULP,
};
struct rxe_queue_buf;
u32 index_mask;
enum queue_type type;
/* private copy of index for shared queues between
- * kernel space and user space. Kernel reads and writes
+ * driver and clients. Driver reads and writes
* this copy and then replicates to rxe_queue_buf
- * for read access by user space.
+ * for read access by clients.
*/
u32 index;
};
switch (type) {
case QUEUE_TYPE_FROM_CLIENT:
- /* protect user index */
+ /* used by rxe, client owns the index */
prod = smp_load_acquire(&q->buf->producer_index);
break;
case QUEUE_TYPE_TO_CLIENT:
+ /* used by rxe which owns the index */
prod = q->index;
break;
- case QUEUE_TYPE_FROM_DRIVER:
- /* protect driver index */
- prod = smp_load_acquire(&q->buf->producer_index);
- break;
- case QUEUE_TYPE_TO_DRIVER:
+ case QUEUE_TYPE_FROM_ULP:
+ /* used by ulp which owns the index */
prod = q->buf->producer_index;
break;
+ case QUEUE_TYPE_TO_ULP:
+ /* used by ulp, rxe owns the index */
+ prod = smp_load_acquire(&q->buf->producer_index);
+ break;
}
return prod;
switch (type) {
case QUEUE_TYPE_FROM_CLIENT:
+ /* used by rxe which owns the index */
cons = q->index;
break;
case QUEUE_TYPE_TO_CLIENT:
- /* protect user index */
+ /* used by rxe, client owns the index */
cons = smp_load_acquire(&q->buf->consumer_index);
break;
- case QUEUE_TYPE_FROM_DRIVER:
- cons = q->buf->consumer_index;
- break;
- case QUEUE_TYPE_TO_DRIVER:
- /* protect driver index */
+ case QUEUE_TYPE_FROM_ULP:
+ /* used by ulp, rxe owns the index */
cons = smp_load_acquire(&q->buf->consumer_index);
break;
+ case QUEUE_TYPE_TO_ULP:
+ /* used by ulp which owns the index */
+ cons = q->buf->consumer_index;
+ break;
}
return cons;
switch (type) {
case QUEUE_TYPE_FROM_CLIENT:
- pr_warn("%s: attempt to advance client index\n",
- __func__);
+ /* used by rxe, client owns the index */
+ if (WARN_ON(1))
+ pr_warn("%s: attempt to advance client index\n",
+ __func__);
break;
case QUEUE_TYPE_TO_CLIENT:
+ /* used by rxe which owns the index */
prod = q->index;
prod = (prod + 1) & q->index_mask;
q->index = prod;
- /* protect user index */
+ /* release so client can read it safely */
smp_store_release(&q->buf->producer_index, prod);
break;
- case QUEUE_TYPE_FROM_DRIVER:
- pr_warn("%s: attempt to advance driver index\n",
- __func__);
- break;
- case QUEUE_TYPE_TO_DRIVER:
+ case QUEUE_TYPE_FROM_ULP:
+ /* used by ulp which owns the index */
prod = q->buf->producer_index;
prod = (prod + 1) & q->index_mask;
- q->buf->producer_index = prod;
+ /* release so rxe can read it safely */
+ smp_store_release(&q->buf->producer_index, prod);
+ break;
+ case QUEUE_TYPE_TO_ULP:
+ /* used by ulp, rxe owns the index */
+ if (WARN_ON(1))
+ pr_warn("%s: attempt to advance driver index\n",
+ __func__);
break;
}
}
switch (type) {
case QUEUE_TYPE_FROM_CLIENT:
- cons = q->index;
- cons = (cons + 1) & q->index_mask;
+ /* used by rxe which owns the index */
+ cons = (q->index + 1) & q->index_mask;
q->index = cons;
- /* protect user index */
+ /* release so client can read it safely */
smp_store_release(&q->buf->consumer_index, cons);
break;
case QUEUE_TYPE_TO_CLIENT:
- pr_warn("%s: attempt to advance client index\n",
- __func__);
+ /* used by rxe, client owns the index */
+ if (WARN_ON(1))
+ pr_warn("%s: attempt to advance client index\n",
+ __func__);
+ break;
+ case QUEUE_TYPE_FROM_ULP:
+ /* used by ulp, rxe owns the index */
+ if (WARN_ON(1))
+ pr_warn("%s: attempt to advance driver index\n",
+ __func__);
break;
- case QUEUE_TYPE_FROM_DRIVER:
+ case QUEUE_TYPE_TO_ULP:
+ /* used by ulp which owns the index */
cons = q->buf->consumer_index;
cons = (cons + 1) & q->index_mask;
- q->buf->consumer_index = cons;
- break;
- case QUEUE_TYPE_TO_DRIVER:
- pr_warn("%s: attempt to advance driver index\n",
- __func__);
+ /* release so rxe can read it safely */
+ smp_store_release(&q->buf->consumer_index, cons);
break;
}
}
int num_sge = ibwr->num_sge;
int full;
- full = queue_full(rq->queue, QUEUE_TYPE_TO_DRIVER);
+ full = queue_full(rq->queue, QUEUE_TYPE_FROM_ULP);
if (unlikely(full))
return -ENOMEM;
for (i = 0; i < num_sge; i++)
length += ibwr->sg_list[i].length;
- recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_TO_DRIVER);
+ recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_FROM_ULP);
recv_wqe->wr_id = ibwr->wr_id;
memcpy(recv_wqe->dma.sge, ibwr->sg_list,
recv_wqe->dma.cur_sge = 0;
recv_wqe->dma.sge_offset = 0;
- queue_advance_producer(rq->queue, QUEUE_TYPE_TO_DRIVER);
+ queue_advance_producer(rq->queue, QUEUE_TYPE_FROM_ULP);
return 0;
}
spin_lock_irqsave(&qp->sq.sq_lock, flags);
- full = queue_full(sq->queue, QUEUE_TYPE_TO_DRIVER);
+ full = queue_full(sq->queue, QUEUE_TYPE_FROM_ULP);
if (unlikely(full)) {
spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
return -ENOMEM;
}
- send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_TO_DRIVER);
+ send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_FROM_ULP);
init_send_wqe(qp, ibwr, mask, length, send_wqe);
- queue_advance_producer(sq->queue, QUEUE_TYPE_TO_DRIVER);
+ queue_advance_producer(sq->queue, QUEUE_TYPE_FROM_ULP);
spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
spin_lock_irqsave(&cq->cq_lock, flags);
for (i = 0; i < num_entries; i++) {
- cqe = queue_head(cq->queue, QUEUE_TYPE_FROM_DRIVER);
+ cqe = queue_head(cq->queue, QUEUE_TYPE_TO_ULP);
if (!cqe)
break;
memcpy(wc++, &cqe->ibwc, sizeof(*wc));
- queue_advance_consumer(cq->queue, QUEUE_TYPE_FROM_DRIVER);
+ queue_advance_consumer(cq->queue, QUEUE_TYPE_TO_ULP);
}
spin_unlock_irqrestore(&cq->cq_lock, flags);
struct rxe_cq *cq = to_rcq(ibcq);
int count;
- count = queue_count(cq->queue, QUEUE_TYPE_FROM_DRIVER);
+ count = queue_count(cq->queue, QUEUE_TYPE_TO_ULP);
return (count > wc_cnt) ? wc_cnt : count;
}
if (cq->notify != IB_CQ_NEXT_COMP)
cq->notify = flags & IB_CQ_SOLICITED_MASK;
- empty = queue_empty(cq->queue, QUEUE_TYPE_FROM_DRIVER);
+ empty = queue_empty(cq->queue, QUEUE_TYPE_TO_ULP);
if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !empty)
ret = 1;