2 * Copyright (c) 2012-2016 VMware, Inc. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of EITHER the GNU General Public License
6 * version 2 as published by the Free Software Foundation or the BSD
7 * 2-Clause License. This program is distributed in the hope that it
8 * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
9 * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
10 * See the GNU General Public License version 2 for more details at
11 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
13 * You should have received a copy of the GNU General Public License
14 * along with this program available in the file COPYING in the main
15 * directory of this source tree.
17 * The BSD 2-Clause License
19 * Redistribution and use in source and binary forms, with or
20 * without modification, are permitted provided that the following
23 * - Redistributions of source code must retain the above
24 * copyright notice, this list of conditions and the following
27 * - Redistributions in binary form must reproduce the above
28 * copyright notice, this list of conditions and the following
29 * disclaimer in the documentation and/or other materials
30 * provided with the distribution.
32 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
33 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
34 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
35 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
37 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
38 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
39 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
41 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
43 * OF THE POSSIBILITY OF SUCH DAMAGE.
48 #include <linux/wait.h>
49 #include <rdma/ib_addr.h>
50 #include <rdma/ib_smi.h>
51 #include <rdma/ib_user_verbs.h>
55 static inline void get_cqs(struct pvrdma_qp
*qp
, struct pvrdma_cq
**send_cq
,
56 struct pvrdma_cq
**recv_cq
)
58 *send_cq
= to_vcq(qp
->ibqp
.send_cq
);
59 *recv_cq
= to_vcq(qp
->ibqp
.recv_cq
);
62 static void pvrdma_lock_cqs(struct pvrdma_cq
*scq
, struct pvrdma_cq
*rcq
,
63 unsigned long *scq_flags
,
64 unsigned long *rcq_flags
)
65 __acquires(scq
->cq_lock
) __acquires(rcq
->cq_lock
)
68 spin_lock_irqsave(&scq
->cq_lock
, *scq_flags
);
69 __acquire(rcq
->cq_lock
);
70 } else if (scq
->cq_handle
< rcq
->cq_handle
) {
71 spin_lock_irqsave(&scq
->cq_lock
, *scq_flags
);
72 spin_lock_irqsave_nested(&rcq
->cq_lock
, *rcq_flags
,
73 SINGLE_DEPTH_NESTING
);
75 spin_lock_irqsave(&rcq
->cq_lock
, *rcq_flags
);
76 spin_lock_irqsave_nested(&scq
->cq_lock
, *scq_flags
,
77 SINGLE_DEPTH_NESTING
);
81 static void pvrdma_unlock_cqs(struct pvrdma_cq
*scq
, struct pvrdma_cq
*rcq
,
82 unsigned long *scq_flags
,
83 unsigned long *rcq_flags
)
84 __releases(scq
->cq_lock
) __releases(rcq
->cq_lock
)
87 __release(rcq
->cq_lock
);
88 spin_unlock_irqrestore(&scq
->cq_lock
, *scq_flags
);
89 } else if (scq
->cq_handle
< rcq
->cq_handle
) {
90 spin_unlock_irqrestore(&rcq
->cq_lock
, *rcq_flags
);
91 spin_unlock_irqrestore(&scq
->cq_lock
, *scq_flags
);
93 spin_unlock_irqrestore(&scq
->cq_lock
, *scq_flags
);
94 spin_unlock_irqrestore(&rcq
->cq_lock
, *rcq_flags
);
98 static void pvrdma_reset_qp(struct pvrdma_qp
*qp
)
100 struct pvrdma_cq
*scq
, *rcq
;
101 unsigned long scq_flags
, rcq_flags
;
104 get_cqs(qp
, &scq
, &rcq
);
105 pvrdma_lock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
107 _pvrdma_flush_cqe(qp
, scq
);
109 _pvrdma_flush_cqe(qp
, rcq
);
111 pvrdma_unlock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
114 * Reset queuepair. The checks are because usermode queuepairs won't
115 * have kernel ringstates.
118 atomic_set(&qp
->rq
.ring
->cons_head
, 0);
119 atomic_set(&qp
->rq
.ring
->prod_tail
, 0);
122 atomic_set(&qp
->sq
.ring
->cons_head
, 0);
123 atomic_set(&qp
->sq
.ring
->prod_tail
, 0);
127 static int pvrdma_set_rq_size(struct pvrdma_dev
*dev
,
128 struct ib_qp_cap
*req_cap
,
129 struct pvrdma_qp
*qp
)
131 if (req_cap
->max_recv_wr
> dev
->dsr
->caps
.max_qp_wr
||
132 req_cap
->max_recv_sge
> dev
->dsr
->caps
.max_sge
) {
133 dev_warn(&dev
->pdev
->dev
, "recv queue size invalid\n");
137 qp
->rq
.wqe_cnt
= roundup_pow_of_two(max(1U, req_cap
->max_recv_wr
));
138 qp
->rq
.max_sg
= roundup_pow_of_two(max(1U, req_cap
->max_recv_sge
));
141 req_cap
->max_recv_wr
= qp
->rq
.wqe_cnt
;
142 req_cap
->max_recv_sge
= qp
->rq
.max_sg
;
144 qp
->rq
.wqe_size
= roundup_pow_of_two(sizeof(struct pvrdma_rq_wqe_hdr
) +
145 sizeof(struct pvrdma_sge
) *
147 qp
->npages_recv
= (qp
->rq
.wqe_cnt
* qp
->rq
.wqe_size
+ PAGE_SIZE
- 1) /
153 static int pvrdma_set_sq_size(struct pvrdma_dev
*dev
, struct ib_qp_cap
*req_cap
,
154 struct pvrdma_qp
*qp
)
156 if (req_cap
->max_send_wr
> dev
->dsr
->caps
.max_qp_wr
||
157 req_cap
->max_send_sge
> dev
->dsr
->caps
.max_sge
) {
158 dev_warn(&dev
->pdev
->dev
, "send queue size invalid\n");
162 qp
->sq
.wqe_cnt
= roundup_pow_of_two(max(1U, req_cap
->max_send_wr
));
163 qp
->sq
.max_sg
= roundup_pow_of_two(max(1U, req_cap
->max_send_sge
));
166 req_cap
->max_send_wr
= qp
->sq
.wqe_cnt
;
167 req_cap
->max_send_sge
= qp
->sq
.max_sg
;
169 qp
->sq
.wqe_size
= roundup_pow_of_two(sizeof(struct pvrdma_sq_wqe_hdr
) +
170 sizeof(struct pvrdma_sge
) *
172 /* Note: one extra page for the header. */
173 qp
->npages_send
= PVRDMA_QP_NUM_HEADER_PAGES
+
174 (qp
->sq
.wqe_cnt
* qp
->sq
.wqe_size
+ PAGE_SIZE
- 1) /
181 * pvrdma_create_qp - create queue pair
182 * @pd: protection domain
183 * @init_attr: queue pair attributes
186 * @return: the ib_qp pointer on success, otherwise returns an errno.
188 struct ib_qp
*pvrdma_create_qp(struct ib_pd
*pd
,
189 struct ib_qp_init_attr
*init_attr
,
190 struct ib_udata
*udata
)
192 struct pvrdma_qp
*qp
= NULL
;
193 struct pvrdma_dev
*dev
= to_vdev(pd
->device
);
194 union pvrdma_cmd_req req
;
195 union pvrdma_cmd_resp rsp
;
196 struct pvrdma_cmd_create_qp
*cmd
= &req
.create_qp
;
197 struct pvrdma_cmd_create_qp_resp
*resp
= &rsp
.create_qp_resp
;
198 struct pvrdma_create_qp ucmd
;
201 bool is_srq
= !!init_attr
->srq
;
203 if (init_attr
->create_flags
) {
204 dev_warn(&dev
->pdev
->dev
,
205 "invalid create queuepair flags %#x\n",
206 init_attr
->create_flags
);
207 return ERR_PTR(-EINVAL
);
210 if (init_attr
->qp_type
!= IB_QPT_RC
&&
211 init_attr
->qp_type
!= IB_QPT_UD
&&
212 init_attr
->qp_type
!= IB_QPT_GSI
) {
213 dev_warn(&dev
->pdev
->dev
, "queuepair type %d not supported\n",
215 return ERR_PTR(-EINVAL
);
218 if (is_srq
&& !dev
->dsr
->caps
.max_srq
) {
219 dev_warn(&dev
->pdev
->dev
,
220 "SRQs not supported by device\n");
221 return ERR_PTR(-EINVAL
);
224 if (!atomic_add_unless(&dev
->num_qps
, 1, dev
->dsr
->caps
.max_qp
))
225 return ERR_PTR(-ENOMEM
);
227 switch (init_attr
->qp_type
) {
229 if (init_attr
->port_num
== 0 ||
230 init_attr
->port_num
> pd
->device
->phys_port_cnt
||
232 dev_warn(&dev
->pdev
->dev
, "invalid queuepair attrs\n");
239 qp
= kzalloc(sizeof(*qp
), GFP_KERNEL
);
245 spin_lock_init(&qp
->sq
.lock
);
246 spin_lock_init(&qp
->rq
.lock
);
247 mutex_init(&qp
->mutex
);
248 refcount_set(&qp
->refcnt
, 1);
249 init_completion(&qp
->free
);
251 qp
->state
= IB_QPS_RESET
;
252 qp
->is_kernel
= !udata
;
254 if (!qp
->is_kernel
) {
255 dev_dbg(&dev
->pdev
->dev
,
256 "create queuepair from user space\n");
258 if (ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
))) {
264 /* set qp->sq.wqe_cnt, shift, buf_size.. */
265 qp
->rumem
= ib_umem_get(pd
->uobject
->context
,
267 ucmd
.rbuf_size
, 0, 0);
268 if (IS_ERR(qp
->rumem
)) {
269 ret
= PTR_ERR(qp
->rumem
);
275 qp
->srq
= to_vsrq(init_attr
->srq
);
278 qp
->sumem
= ib_umem_get(pd
->uobject
->context
,
280 ucmd
.sbuf_size
, 0, 0);
281 if (IS_ERR(qp
->sumem
)) {
283 ib_umem_release(qp
->rumem
);
284 ret
= PTR_ERR(qp
->sumem
);
288 qp
->npages_send
= ib_umem_page_count(qp
->sumem
);
290 qp
->npages_recv
= ib_umem_page_count(qp
->rumem
);
293 qp
->npages
= qp
->npages_send
+ qp
->npages_recv
;
295 ret
= pvrdma_set_sq_size(to_vdev(pd
->device
),
296 &init_attr
->cap
, qp
);
300 ret
= pvrdma_set_rq_size(to_vdev(pd
->device
),
301 &init_attr
->cap
, qp
);
305 qp
->npages
= qp
->npages_send
+ qp
->npages_recv
;
307 /* Skip header page. */
308 qp
->sq
.offset
= PVRDMA_QP_NUM_HEADER_PAGES
* PAGE_SIZE
;
310 /* Recv queue pages are after send pages. */
311 qp
->rq
.offset
= qp
->npages_send
* PAGE_SIZE
;
314 if (qp
->npages
< 0 || qp
->npages
> PVRDMA_PAGE_DIR_MAX_PAGES
) {
315 dev_warn(&dev
->pdev
->dev
,
316 "overflow pages in queuepair\n");
321 ret
= pvrdma_page_dir_init(dev
, &qp
->pdir
, qp
->npages
,
324 dev_warn(&dev
->pdev
->dev
,
325 "could not allocate page directory\n");
329 if (!qp
->is_kernel
) {
330 pvrdma_page_dir_insert_umem(&qp
->pdir
, qp
->sumem
, 0);
332 pvrdma_page_dir_insert_umem(&qp
->pdir
,
336 /* Ring state is always the first page. */
337 qp
->sq
.ring
= qp
->pdir
.pages
[0];
338 qp
->rq
.ring
= is_srq
? NULL
: &qp
->sq
.ring
[1];
347 init_attr
->cap
.max_inline_data
= 0;
349 memset(cmd
, 0, sizeof(*cmd
));
350 cmd
->hdr
.cmd
= PVRDMA_CMD_CREATE_QP
;
351 cmd
->pd_handle
= to_vpd(pd
)->pd_handle
;
352 cmd
->send_cq_handle
= to_vcq(init_attr
->send_cq
)->cq_handle
;
353 cmd
->recv_cq_handle
= to_vcq(init_attr
->recv_cq
)->cq_handle
;
355 cmd
->srq_handle
= to_vsrq(init_attr
->srq
)->srq_handle
;
358 cmd
->max_send_wr
= init_attr
->cap
.max_send_wr
;
359 cmd
->max_recv_wr
= init_attr
->cap
.max_recv_wr
;
360 cmd
->max_send_sge
= init_attr
->cap
.max_send_sge
;
361 cmd
->max_recv_sge
= init_attr
->cap
.max_recv_sge
;
362 cmd
->max_inline_data
= init_attr
->cap
.max_inline_data
;
363 cmd
->sq_sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
) ? 1 : 0;
364 cmd
->qp_type
= ib_qp_type_to_pvrdma(init_attr
->qp_type
);
365 cmd
->is_srq
= is_srq
;
367 cmd
->access_flags
= IB_ACCESS_LOCAL_WRITE
;
368 cmd
->total_chunks
= qp
->npages
;
369 cmd
->send_chunks
= qp
->npages_send
- PVRDMA_QP_NUM_HEADER_PAGES
;
370 cmd
->pdir_dma
= qp
->pdir
.dir_dma
;
372 dev_dbg(&dev
->pdev
->dev
, "create queuepair with %d, %d, %d, %d\n",
373 cmd
->max_send_wr
, cmd
->max_recv_wr
, cmd
->max_send_sge
,
376 ret
= pvrdma_cmd_post(dev
, &req
, &rsp
, PVRDMA_CMD_CREATE_QP_RESP
);
378 dev_warn(&dev
->pdev
->dev
,
379 "could not create queuepair, error: %d\n", ret
);
383 /* max_send_wr/_recv_wr/_send_sge/_recv_sge/_inline_data */
384 qp
->qp_handle
= resp
->qpn
;
385 qp
->port
= init_attr
->port_num
;
386 qp
->ibqp
.qp_num
= resp
->qpn
;
387 spin_lock_irqsave(&dev
->qp_tbl_lock
, flags
);
388 dev
->qp_tbl
[qp
->qp_handle
% dev
->dsr
->caps
.max_qp
] = qp
;
389 spin_unlock_irqrestore(&dev
->qp_tbl_lock
, flags
);
394 pvrdma_page_dir_cleanup(dev
, &qp
->pdir
);
396 if (!qp
->is_kernel
) {
398 ib_umem_release(qp
->rumem
);
400 ib_umem_release(qp
->sumem
);
404 atomic_dec(&dev
->num_qps
);
409 static void pvrdma_free_qp(struct pvrdma_qp
*qp
)
411 struct pvrdma_dev
*dev
= to_vdev(qp
->ibqp
.device
);
412 struct pvrdma_cq
*scq
;
413 struct pvrdma_cq
*rcq
;
414 unsigned long flags
, scq_flags
, rcq_flags
;
416 /* In case cq is polling */
417 get_cqs(qp
, &scq
, &rcq
);
418 pvrdma_lock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
420 _pvrdma_flush_cqe(qp
, scq
);
422 _pvrdma_flush_cqe(qp
, rcq
);
424 spin_lock_irqsave(&dev
->qp_tbl_lock
, flags
);
425 dev
->qp_tbl
[qp
->qp_handle
] = NULL
;
426 spin_unlock_irqrestore(&dev
->qp_tbl_lock
, flags
);
428 pvrdma_unlock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
430 if (refcount_dec_and_test(&qp
->refcnt
))
432 wait_for_completion(&qp
->free
);
434 if (!qp
->is_kernel
) {
436 ib_umem_release(qp
->rumem
);
438 ib_umem_release(qp
->sumem
);
441 pvrdma_page_dir_cleanup(dev
, &qp
->pdir
);
445 atomic_dec(&dev
->num_qps
);
449 * pvrdma_destroy_qp - destroy a queue pair
450 * @qp: the queue pair to destroy
452 * @return: 0 on success.
454 int pvrdma_destroy_qp(struct ib_qp
*qp
)
456 struct pvrdma_qp
*vqp
= to_vqp(qp
);
457 union pvrdma_cmd_req req
;
458 struct pvrdma_cmd_destroy_qp
*cmd
= &req
.destroy_qp
;
461 memset(cmd
, 0, sizeof(*cmd
));
462 cmd
->hdr
.cmd
= PVRDMA_CMD_DESTROY_QP
;
463 cmd
->qp_handle
= vqp
->qp_handle
;
465 ret
= pvrdma_cmd_post(to_vdev(qp
->device
), &req
, NULL
, 0);
467 dev_warn(&to_vdev(qp
->device
)->pdev
->dev
,
468 "destroy queuepair failed, error: %d\n", ret
);
476 * pvrdma_modify_qp - modify queue pair attributes
477 * @ibqp: the queue pair
478 * @attr: the new queue pair's attributes
479 * @attr_mask: attributes mask
482 * @returns 0 on success, otherwise returns an errno.
484 int pvrdma_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
485 int attr_mask
, struct ib_udata
*udata
)
487 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
488 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
489 union pvrdma_cmd_req req
;
490 union pvrdma_cmd_resp rsp
;
491 struct pvrdma_cmd_modify_qp
*cmd
= &req
.modify_qp
;
492 enum ib_qp_state cur_state
, next_state
;
495 /* Sanity checking. Should need lock here */
496 mutex_lock(&qp
->mutex
);
497 cur_state
= (attr_mask
& IB_QP_CUR_STATE
) ? attr
->cur_qp_state
:
499 next_state
= (attr_mask
& IB_QP_STATE
) ? attr
->qp_state
: cur_state
;
501 if (!ib_modify_qp_is_ok(cur_state
, next_state
, ibqp
->qp_type
,
507 if (attr_mask
& IB_QP_PORT
) {
508 if (attr
->port_num
== 0 ||
509 attr
->port_num
> ibqp
->device
->phys_port_cnt
) {
515 if (attr_mask
& IB_QP_MIN_RNR_TIMER
) {
516 if (attr
->min_rnr_timer
> 31) {
522 if (attr_mask
& IB_QP_PKEY_INDEX
) {
523 if (attr
->pkey_index
>= dev
->dsr
->caps
.max_pkeys
) {
529 if (attr_mask
& IB_QP_QKEY
)
530 qp
->qkey
= attr
->qkey
;
532 if (cur_state
== next_state
&& cur_state
== IB_QPS_RESET
) {
537 qp
->state
= next_state
;
538 memset(cmd
, 0, sizeof(*cmd
));
539 cmd
->hdr
.cmd
= PVRDMA_CMD_MODIFY_QP
;
540 cmd
->qp_handle
= qp
->qp_handle
;
541 cmd
->attr_mask
= ib_qp_attr_mask_to_pvrdma(attr_mask
);
542 cmd
->attrs
.qp_state
= ib_qp_state_to_pvrdma(attr
->qp_state
);
543 cmd
->attrs
.cur_qp_state
=
544 ib_qp_state_to_pvrdma(attr
->cur_qp_state
);
545 cmd
->attrs
.path_mtu
= ib_mtu_to_pvrdma(attr
->path_mtu
);
546 cmd
->attrs
.path_mig_state
=
547 ib_mig_state_to_pvrdma(attr
->path_mig_state
);
548 cmd
->attrs
.qkey
= attr
->qkey
;
549 cmd
->attrs
.rq_psn
= attr
->rq_psn
;
550 cmd
->attrs
.sq_psn
= attr
->sq_psn
;
551 cmd
->attrs
.dest_qp_num
= attr
->dest_qp_num
;
552 cmd
->attrs
.qp_access_flags
=
553 ib_access_flags_to_pvrdma(attr
->qp_access_flags
);
554 cmd
->attrs
.pkey_index
= attr
->pkey_index
;
555 cmd
->attrs
.alt_pkey_index
= attr
->alt_pkey_index
;
556 cmd
->attrs
.en_sqd_async_notify
= attr
->en_sqd_async_notify
;
557 cmd
->attrs
.sq_draining
= attr
->sq_draining
;
558 cmd
->attrs
.max_rd_atomic
= attr
->max_rd_atomic
;
559 cmd
->attrs
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
560 cmd
->attrs
.min_rnr_timer
= attr
->min_rnr_timer
;
561 cmd
->attrs
.port_num
= attr
->port_num
;
562 cmd
->attrs
.timeout
= attr
->timeout
;
563 cmd
->attrs
.retry_cnt
= attr
->retry_cnt
;
564 cmd
->attrs
.rnr_retry
= attr
->rnr_retry
;
565 cmd
->attrs
.alt_port_num
= attr
->alt_port_num
;
566 cmd
->attrs
.alt_timeout
= attr
->alt_timeout
;
567 ib_qp_cap_to_pvrdma(&cmd
->attrs
.cap
, &attr
->cap
);
568 rdma_ah_attr_to_pvrdma(&cmd
->attrs
.ah_attr
, &attr
->ah_attr
);
569 rdma_ah_attr_to_pvrdma(&cmd
->attrs
.alt_ah_attr
, &attr
->alt_ah_attr
);
571 ret
= pvrdma_cmd_post(dev
, &req
, &rsp
, PVRDMA_CMD_MODIFY_QP_RESP
);
573 dev_warn(&dev
->pdev
->dev
,
574 "could not modify queuepair, error: %d\n", ret
);
575 } else if (rsp
.hdr
.err
> 0) {
576 dev_warn(&dev
->pdev
->dev
,
577 "cannot modify queuepair, error: %d\n", rsp
.hdr
.err
);
581 if (ret
== 0 && next_state
== IB_QPS_RESET
)
585 mutex_unlock(&qp
->mutex
);
590 static inline void *get_sq_wqe(struct pvrdma_qp
*qp
, unsigned int n
)
592 return pvrdma_page_dir_get_ptr(&qp
->pdir
,
593 qp
->sq
.offset
+ n
* qp
->sq
.wqe_size
);
596 static inline void *get_rq_wqe(struct pvrdma_qp
*qp
, unsigned int n
)
598 return pvrdma_page_dir_get_ptr(&qp
->pdir
,
599 qp
->rq
.offset
+ n
* qp
->rq
.wqe_size
);
602 static int set_reg_seg(struct pvrdma_sq_wqe_hdr
*wqe_hdr
,
603 const struct ib_reg_wr
*wr
)
605 struct pvrdma_user_mr
*mr
= to_vmr(wr
->mr
);
607 wqe_hdr
->wr
.fast_reg
.iova_start
= mr
->ibmr
.iova
;
608 wqe_hdr
->wr
.fast_reg
.pl_pdir_dma
= mr
->pdir
.dir_dma
;
609 wqe_hdr
->wr
.fast_reg
.page_shift
= mr
->page_shift
;
610 wqe_hdr
->wr
.fast_reg
.page_list_len
= mr
->npages
;
611 wqe_hdr
->wr
.fast_reg
.length
= mr
->ibmr
.length
;
612 wqe_hdr
->wr
.fast_reg
.access_flags
= wr
->access
;
613 wqe_hdr
->wr
.fast_reg
.rkey
= wr
->key
;
615 return pvrdma_page_dir_insert_page_list(&mr
->pdir
, mr
->pages
,
620 * pvrdma_post_send - post send work request entries on a QP
622 * @wr: work request list to post
623 * @bad_wr: the first bad WR returned
625 * @return: 0 on success, otherwise errno returned.
627 int pvrdma_post_send(struct ib_qp
*ibqp
, const struct ib_send_wr
*wr
,
628 const struct ib_send_wr
**bad_wr
)
630 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
631 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
633 struct pvrdma_sq_wqe_hdr
*wqe_hdr
;
634 struct pvrdma_sge
*sge
;
638 * In states lower than RTS, we can fail immediately. In other states,
639 * just post and let the device figure it out.
641 if (qp
->state
< IB_QPS_RTS
) {
646 spin_lock_irqsave(&qp
->sq
.lock
, flags
);
649 unsigned int tail
= 0;
651 if (unlikely(!pvrdma_idx_ring_has_space(
652 qp
->sq
.ring
, qp
->sq
.wqe_cnt
, &tail
))) {
653 dev_warn_ratelimited(&dev
->pdev
->dev
,
654 "send queue is full\n");
660 if (unlikely(wr
->num_sge
> qp
->sq
.max_sg
|| wr
->num_sge
< 0)) {
661 dev_warn_ratelimited(&dev
->pdev
->dev
,
662 "send SGE overflow\n");
668 if (unlikely(wr
->opcode
< 0)) {
669 dev_warn_ratelimited(&dev
->pdev
->dev
,
670 "invalid send opcode\n");
677 * Only support UD, RC.
678 * Need to check opcode table for thorough checking.
681 * _SEND_WITH_IMM x x x
683 * _RDMA_WRITE_WITH_IMM x x
687 * _ATOMIC_CMP_AND_SWP x
688 * _ATOMIC_FETCH_AND_ADD x
689 * _MASK_ATOMIC_CMP_AND_SWP x
690 * _MASK_ATOMIC_FETCH_AND_ADD x
694 if (qp
->ibqp
.qp_type
!= IB_QPT_UD
&&
695 qp
->ibqp
.qp_type
!= IB_QPT_RC
&&
696 wr
->opcode
!= IB_WR_SEND
) {
697 dev_warn_ratelimited(&dev
->pdev
->dev
,
698 "unsupported queuepair type\n");
702 } else if (qp
->ibqp
.qp_type
== IB_QPT_UD
||
703 qp
->ibqp
.qp_type
== IB_QPT_GSI
) {
704 if (wr
->opcode
!= IB_WR_SEND
&&
705 wr
->opcode
!= IB_WR_SEND_WITH_IMM
) {
706 dev_warn_ratelimited(&dev
->pdev
->dev
,
707 "invalid send opcode\n");
714 wqe_hdr
= (struct pvrdma_sq_wqe_hdr
*)get_sq_wqe(qp
, tail
);
715 memset(wqe_hdr
, 0, sizeof(*wqe_hdr
));
716 wqe_hdr
->wr_id
= wr
->wr_id
;
717 wqe_hdr
->num_sge
= wr
->num_sge
;
718 wqe_hdr
->opcode
= ib_wr_opcode_to_pvrdma(wr
->opcode
);
719 wqe_hdr
->send_flags
= ib_send_flags_to_pvrdma(wr
->send_flags
);
720 if (wr
->opcode
== IB_WR_SEND_WITH_IMM
||
721 wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
)
722 wqe_hdr
->ex
.imm_data
= wr
->ex
.imm_data
;
724 if (unlikely(wqe_hdr
->opcode
== PVRDMA_WR_ERROR
)) {
730 switch (qp
->ibqp
.qp_type
) {
733 if (unlikely(!ud_wr(wr
)->ah
)) {
734 dev_warn_ratelimited(&dev
->pdev
->dev
,
735 "invalid address handle\n");
742 * Use qkey from qp context if high order bit set,
743 * otherwise from work request.
745 wqe_hdr
->wr
.ud
.remote_qpn
= ud_wr(wr
)->remote_qpn
;
746 wqe_hdr
->wr
.ud
.remote_qkey
=
747 ud_wr(wr
)->remote_qkey
& 0x80000000 ?
748 qp
->qkey
: ud_wr(wr
)->remote_qkey
;
749 wqe_hdr
->wr
.ud
.av
= to_vah(ud_wr(wr
)->ah
)->av
;
753 switch (wr
->opcode
) {
754 case IB_WR_RDMA_READ
:
755 case IB_WR_RDMA_WRITE
:
756 case IB_WR_RDMA_WRITE_WITH_IMM
:
757 wqe_hdr
->wr
.rdma
.remote_addr
=
758 rdma_wr(wr
)->remote_addr
;
759 wqe_hdr
->wr
.rdma
.rkey
= rdma_wr(wr
)->rkey
;
761 case IB_WR_LOCAL_INV
:
762 case IB_WR_SEND_WITH_INV
:
763 wqe_hdr
->ex
.invalidate_rkey
=
764 wr
->ex
.invalidate_rkey
;
766 case IB_WR_ATOMIC_CMP_AND_SWP
:
767 case IB_WR_ATOMIC_FETCH_AND_ADD
:
768 wqe_hdr
->wr
.atomic
.remote_addr
=
769 atomic_wr(wr
)->remote_addr
;
770 wqe_hdr
->wr
.atomic
.rkey
= atomic_wr(wr
)->rkey
;
771 wqe_hdr
->wr
.atomic
.compare_add
=
772 atomic_wr(wr
)->compare_add
;
773 if (wr
->opcode
== IB_WR_ATOMIC_CMP_AND_SWP
)
774 wqe_hdr
->wr
.atomic
.swap
=
778 ret
= set_reg_seg(wqe_hdr
, reg_wr(wr
));
780 dev_warn_ratelimited(&dev
->pdev
->dev
,
781 "Failed to set fast register work request\n");
792 dev_warn_ratelimited(&dev
->pdev
->dev
,
793 "invalid queuepair type\n");
799 sge
= (struct pvrdma_sge
*)(wqe_hdr
+ 1);
800 for (i
= 0; i
< wr
->num_sge
; i
++) {
801 /* Need to check wqe_size 0 or max size */
802 sge
->addr
= wr
->sg_list
[i
].addr
;
803 sge
->length
= wr
->sg_list
[i
].length
;
804 sge
->lkey
= wr
->sg_list
[i
].lkey
;
808 /* Make sure wqe is written before index update */
811 /* Update shared sq ring */
812 pvrdma_idx_ring_inc(&qp
->sq
.ring
->prod_tail
,
821 spin_unlock_irqrestore(&qp
->sq
.lock
, flags
);
824 pvrdma_write_uar_qp(dev
, PVRDMA_UAR_QP_SEND
| qp
->qp_handle
);
830 * pvrdma_post_receive - post receive work request entries on a QP
832 * @wr: the work request list to post
833 * @bad_wr: the first bad WR returned
835 * @return: 0 on success, otherwise errno returned.
837 int pvrdma_post_recv(struct ib_qp
*ibqp
, const struct ib_recv_wr
*wr
,
838 const struct ib_recv_wr
**bad_wr
)
840 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
842 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
843 struct pvrdma_rq_wqe_hdr
*wqe_hdr
;
844 struct pvrdma_sge
*sge
;
849 * In the RESET state, we can fail immediately. For other states,
850 * just post and let the device figure it out.
852 if (qp
->state
== IB_QPS_RESET
) {
858 dev_warn(&dev
->pdev
->dev
, "QP associated with SRQ\n");
863 spin_lock_irqsave(&qp
->rq
.lock
, flags
);
866 unsigned int tail
= 0;
868 if (unlikely(wr
->num_sge
> qp
->rq
.max_sg
||
872 dev_warn_ratelimited(&dev
->pdev
->dev
,
873 "recv SGE overflow\n");
877 if (unlikely(!pvrdma_idx_ring_has_space(
878 qp
->rq
.ring
, qp
->rq
.wqe_cnt
, &tail
))) {
881 dev_warn_ratelimited(&dev
->pdev
->dev
,
882 "recv queue full\n");
886 wqe_hdr
= (struct pvrdma_rq_wqe_hdr
*)get_rq_wqe(qp
, tail
);
887 wqe_hdr
->wr_id
= wr
->wr_id
;
888 wqe_hdr
->num_sge
= wr
->num_sge
;
889 wqe_hdr
->total_len
= 0;
891 sge
= (struct pvrdma_sge
*)(wqe_hdr
+ 1);
892 for (i
= 0; i
< wr
->num_sge
; i
++) {
893 sge
->addr
= wr
->sg_list
[i
].addr
;
894 sge
->length
= wr
->sg_list
[i
].length
;
895 sge
->lkey
= wr
->sg_list
[i
].lkey
;
899 /* Make sure wqe is written before index update */
902 /* Update shared rq ring */
903 pvrdma_idx_ring_inc(&qp
->rq
.ring
->prod_tail
,
909 spin_unlock_irqrestore(&qp
->rq
.lock
, flags
);
911 pvrdma_write_uar_qp(dev
, PVRDMA_UAR_QP_RECV
| qp
->qp_handle
);
916 spin_unlock_irqrestore(&qp
->rq
.lock
, flags
);
922 * pvrdma_query_qp - query a queue pair's attributes
923 * @ibqp: the queue pair to query
924 * @attr: the queue pair's attributes
925 * @attr_mask: attributes mask
926 * @init_attr: initial queue pair attributes
928 * @returns 0 on success, otherwise returns an errno.
930 int pvrdma_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
931 int attr_mask
, struct ib_qp_init_attr
*init_attr
)
933 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
934 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
935 union pvrdma_cmd_req req
;
936 union pvrdma_cmd_resp rsp
;
937 struct pvrdma_cmd_query_qp
*cmd
= &req
.query_qp
;
938 struct pvrdma_cmd_query_qp_resp
*resp
= &rsp
.query_qp_resp
;
941 mutex_lock(&qp
->mutex
);
943 if (qp
->state
== IB_QPS_RESET
) {
944 attr
->qp_state
= IB_QPS_RESET
;
948 memset(cmd
, 0, sizeof(*cmd
));
949 cmd
->hdr
.cmd
= PVRDMA_CMD_QUERY_QP
;
950 cmd
->qp_handle
= qp
->qp_handle
;
951 cmd
->attr_mask
= ib_qp_attr_mask_to_pvrdma(attr_mask
);
953 ret
= pvrdma_cmd_post(dev
, &req
, &rsp
, PVRDMA_CMD_QUERY_QP_RESP
);
955 dev_warn(&dev
->pdev
->dev
,
956 "could not query queuepair, error: %d\n", ret
);
960 attr
->qp_state
= pvrdma_qp_state_to_ib(resp
->attrs
.qp_state
);
962 pvrdma_qp_state_to_ib(resp
->attrs
.cur_qp_state
);
963 attr
->path_mtu
= pvrdma_mtu_to_ib(resp
->attrs
.path_mtu
);
964 attr
->path_mig_state
=
965 pvrdma_mig_state_to_ib(resp
->attrs
.path_mig_state
);
966 attr
->qkey
= resp
->attrs
.qkey
;
967 attr
->rq_psn
= resp
->attrs
.rq_psn
;
968 attr
->sq_psn
= resp
->attrs
.sq_psn
;
969 attr
->dest_qp_num
= resp
->attrs
.dest_qp_num
;
970 attr
->qp_access_flags
=
971 pvrdma_access_flags_to_ib(resp
->attrs
.qp_access_flags
);
972 attr
->pkey_index
= resp
->attrs
.pkey_index
;
973 attr
->alt_pkey_index
= resp
->attrs
.alt_pkey_index
;
974 attr
->en_sqd_async_notify
= resp
->attrs
.en_sqd_async_notify
;
975 attr
->sq_draining
= resp
->attrs
.sq_draining
;
976 attr
->max_rd_atomic
= resp
->attrs
.max_rd_atomic
;
977 attr
->max_dest_rd_atomic
= resp
->attrs
.max_dest_rd_atomic
;
978 attr
->min_rnr_timer
= resp
->attrs
.min_rnr_timer
;
979 attr
->port_num
= resp
->attrs
.port_num
;
980 attr
->timeout
= resp
->attrs
.timeout
;
981 attr
->retry_cnt
= resp
->attrs
.retry_cnt
;
982 attr
->rnr_retry
= resp
->attrs
.rnr_retry
;
983 attr
->alt_port_num
= resp
->attrs
.alt_port_num
;
984 attr
->alt_timeout
= resp
->attrs
.alt_timeout
;
985 pvrdma_qp_cap_to_ib(&attr
->cap
, &resp
->attrs
.cap
);
986 pvrdma_ah_attr_to_rdma(&attr
->ah_attr
, &resp
->attrs
.ah_attr
);
987 pvrdma_ah_attr_to_rdma(&attr
->alt_ah_attr
, &resp
->attrs
.alt_ah_attr
);
989 qp
->state
= attr
->qp_state
;
994 attr
->cur_qp_state
= attr
->qp_state
;
996 init_attr
->event_handler
= qp
->ibqp
.event_handler
;
997 init_attr
->qp_context
= qp
->ibqp
.qp_context
;
998 init_attr
->send_cq
= qp
->ibqp
.send_cq
;
999 init_attr
->recv_cq
= qp
->ibqp
.recv_cq
;
1000 init_attr
->srq
= qp
->ibqp
.srq
;
1001 init_attr
->xrcd
= NULL
;
1002 init_attr
->cap
= attr
->cap
;
1003 init_attr
->sq_sig_type
= 0;
1004 init_attr
->qp_type
= qp
->ibqp
.qp_type
;
1005 init_attr
->create_flags
= 0;
1006 init_attr
->port_num
= qp
->port
;
1008 mutex_unlock(&qp
->mutex
);