2 * Copyright (c) 2012-2016 VMware, Inc. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of EITHER the GNU General Public License
6 * version 2 as published by the Free Software Foundation or the BSD
7 * 2-Clause License. This program is distributed in the hope that it
8 * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
9 * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
10 * See the GNU General Public License version 2 for more details at
11 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
13 * You should have received a copy of the GNU General Public License
14 * along with this program available in the file COPYING in the main
15 * directory of this source tree.
17 * The BSD 2-Clause License
19 * Redistribution and use in source and binary forms, with or
20 * without modification, are permitted provided that the following
23 * - Redistributions of source code must retain the above
24 * copyright notice, this list of conditions and the following
27 * - Redistributions in binary form must reproduce the above
28 * copyright notice, this list of conditions and the following
29 * disclaimer in the documentation and/or other materials
30 * provided with the distribution.
32 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
33 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
34 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
35 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
37 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
38 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
39 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
41 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
43 * OF THE POSSIBILITY OF SUCH DAMAGE.
48 #include <linux/wait.h>
49 #include <rdma/ib_addr.h>
50 #include <rdma/ib_smi.h>
51 #include <rdma/ib_user_verbs.h>
55 static inline void get_cqs(struct pvrdma_qp
*qp
, struct pvrdma_cq
**send_cq
,
56 struct pvrdma_cq
**recv_cq
)
58 *send_cq
= to_vcq(qp
->ibqp
.send_cq
);
59 *recv_cq
= to_vcq(qp
->ibqp
.recv_cq
);
62 static void pvrdma_lock_cqs(struct pvrdma_cq
*scq
, struct pvrdma_cq
*rcq
,
63 unsigned long *scq_flags
,
64 unsigned long *rcq_flags
)
65 __acquires(scq
->cq_lock
) __acquires(rcq
->cq_lock
)
68 spin_lock_irqsave(&scq
->cq_lock
, *scq_flags
);
69 __acquire(rcq
->cq_lock
);
70 } else if (scq
->cq_handle
< rcq
->cq_handle
) {
71 spin_lock_irqsave(&scq
->cq_lock
, *scq_flags
);
72 spin_lock_irqsave_nested(&rcq
->cq_lock
, *rcq_flags
,
73 SINGLE_DEPTH_NESTING
);
75 spin_lock_irqsave(&rcq
->cq_lock
, *rcq_flags
);
76 spin_lock_irqsave_nested(&scq
->cq_lock
, *scq_flags
,
77 SINGLE_DEPTH_NESTING
);
81 static void pvrdma_unlock_cqs(struct pvrdma_cq
*scq
, struct pvrdma_cq
*rcq
,
82 unsigned long *scq_flags
,
83 unsigned long *rcq_flags
)
84 __releases(scq
->cq_lock
) __releases(rcq
->cq_lock
)
87 __release(rcq
->cq_lock
);
88 spin_unlock_irqrestore(&scq
->cq_lock
, *scq_flags
);
89 } else if (scq
->cq_handle
< rcq
->cq_handle
) {
90 spin_unlock_irqrestore(&rcq
->cq_lock
, *rcq_flags
);
91 spin_unlock_irqrestore(&scq
->cq_lock
, *scq_flags
);
93 spin_unlock_irqrestore(&scq
->cq_lock
, *scq_flags
);
94 spin_unlock_irqrestore(&rcq
->cq_lock
, *rcq_flags
);
98 static void pvrdma_reset_qp(struct pvrdma_qp
*qp
)
100 struct pvrdma_cq
*scq
, *rcq
;
101 unsigned long scq_flags
, rcq_flags
;
104 get_cqs(qp
, &scq
, &rcq
);
105 pvrdma_lock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
107 _pvrdma_flush_cqe(qp
, scq
);
109 _pvrdma_flush_cqe(qp
, rcq
);
111 pvrdma_unlock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
114 * Reset queuepair. The checks are because usermode queuepairs won't
115 * have kernel ringstates.
118 atomic_set(&qp
->rq
.ring
->cons_head
, 0);
119 atomic_set(&qp
->rq
.ring
->prod_tail
, 0);
122 atomic_set(&qp
->sq
.ring
->cons_head
, 0);
123 atomic_set(&qp
->sq
.ring
->prod_tail
, 0);
127 static int pvrdma_set_rq_size(struct pvrdma_dev
*dev
,
128 struct ib_qp_cap
*req_cap
,
129 struct pvrdma_qp
*qp
)
131 if (req_cap
->max_recv_wr
> dev
->dsr
->caps
.max_qp_wr
||
132 req_cap
->max_recv_sge
> dev
->dsr
->caps
.max_sge
) {
133 dev_warn(&dev
->pdev
->dev
, "recv queue size invalid\n");
137 qp
->rq
.wqe_cnt
= roundup_pow_of_two(max(1U, req_cap
->max_recv_wr
));
138 qp
->rq
.max_sg
= roundup_pow_of_two(max(1U, req_cap
->max_recv_sge
));
141 req_cap
->max_recv_wr
= qp
->rq
.wqe_cnt
;
142 req_cap
->max_recv_sge
= qp
->rq
.max_sg
;
144 qp
->rq
.wqe_size
= roundup_pow_of_two(sizeof(struct pvrdma_rq_wqe_hdr
) +
145 sizeof(struct pvrdma_sge
) *
147 qp
->npages_recv
= (qp
->rq
.wqe_cnt
* qp
->rq
.wqe_size
+ PAGE_SIZE
- 1) /
153 static int pvrdma_set_sq_size(struct pvrdma_dev
*dev
, struct ib_qp_cap
*req_cap
,
154 struct pvrdma_qp
*qp
)
156 if (req_cap
->max_send_wr
> dev
->dsr
->caps
.max_qp_wr
||
157 req_cap
->max_send_sge
> dev
->dsr
->caps
.max_sge
) {
158 dev_warn(&dev
->pdev
->dev
, "send queue size invalid\n");
162 qp
->sq
.wqe_cnt
= roundup_pow_of_two(max(1U, req_cap
->max_send_wr
));
163 qp
->sq
.max_sg
= roundup_pow_of_two(max(1U, req_cap
->max_send_sge
));
166 req_cap
->max_send_wr
= qp
->sq
.wqe_cnt
;
167 req_cap
->max_send_sge
= qp
->sq
.max_sg
;
169 qp
->sq
.wqe_size
= roundup_pow_of_two(sizeof(struct pvrdma_sq_wqe_hdr
) +
170 sizeof(struct pvrdma_sge
) *
172 /* Note: one extra page for the header. */
173 qp
->npages_send
= PVRDMA_QP_NUM_HEADER_PAGES
+
174 (qp
->sq
.wqe_cnt
* qp
->sq
.wqe_size
+ PAGE_SIZE
- 1) /
181 * pvrdma_create_qp - create queue pair
182 * @pd: protection domain
183 * @init_attr: queue pair attributes
186 * @return: the ib_qp pointer on success, otherwise returns an errno.
188 struct ib_qp
*pvrdma_create_qp(struct ib_pd
*pd
,
189 struct ib_qp_init_attr
*init_attr
,
190 struct ib_udata
*udata
)
192 struct pvrdma_qp
*qp
= NULL
;
193 struct pvrdma_dev
*dev
= to_vdev(pd
->device
);
194 union pvrdma_cmd_req req
;
195 union pvrdma_cmd_resp rsp
;
196 struct pvrdma_cmd_create_qp
*cmd
= &req
.create_qp
;
197 struct pvrdma_cmd_create_qp_resp
*resp
= &rsp
.create_qp_resp
;
198 struct pvrdma_create_qp ucmd
;
201 bool is_srq
= !!init_attr
->srq
;
203 if (init_attr
->create_flags
) {
204 dev_warn(&dev
->pdev
->dev
,
205 "invalid create queuepair flags %#x\n",
206 init_attr
->create_flags
);
207 return ERR_PTR(-EINVAL
);
210 if (init_attr
->qp_type
!= IB_QPT_RC
&&
211 init_attr
->qp_type
!= IB_QPT_UD
&&
212 init_attr
->qp_type
!= IB_QPT_GSI
) {
213 dev_warn(&dev
->pdev
->dev
, "queuepair type %d not supported\n",
215 return ERR_PTR(-EINVAL
);
218 if (is_srq
&& !dev
->dsr
->caps
.max_srq
) {
219 dev_warn(&dev
->pdev
->dev
,
220 "SRQs not supported by device\n");
221 return ERR_PTR(-EINVAL
);
224 if (!atomic_add_unless(&dev
->num_qps
, 1, dev
->dsr
->caps
.max_qp
))
225 return ERR_PTR(-ENOMEM
);
227 switch (init_attr
->qp_type
) {
229 if (init_attr
->port_num
== 0 ||
230 init_attr
->port_num
> pd
->device
->phys_port_cnt
||
232 dev_warn(&dev
->pdev
->dev
, "invalid queuepair attrs\n");
239 qp
= kzalloc(sizeof(*qp
), GFP_KERNEL
);
245 spin_lock_init(&qp
->sq
.lock
);
246 spin_lock_init(&qp
->rq
.lock
);
247 mutex_init(&qp
->mutex
);
248 refcount_set(&qp
->refcnt
, 1);
249 init_completion(&qp
->free
);
251 qp
->state
= IB_QPS_RESET
;
252 qp
->is_kernel
= !udata
;
254 if (!qp
->is_kernel
) {
255 dev_dbg(&dev
->pdev
->dev
,
256 "create queuepair from user space\n");
258 if (ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
))) {
264 /* set qp->sq.wqe_cnt, shift, buf_size.. */
266 ib_umem_get(pd
->device
, ucmd
.rbuf_addr
,
268 if (IS_ERR(qp
->rumem
)) {
269 ret
= PTR_ERR(qp
->rumem
);
275 qp
->srq
= to_vsrq(init_attr
->srq
);
278 qp
->sumem
= ib_umem_get(pd
->device
, ucmd
.sbuf_addr
,
280 if (IS_ERR(qp
->sumem
)) {
282 ib_umem_release(qp
->rumem
);
283 ret
= PTR_ERR(qp
->sumem
);
287 qp
->npages_send
= ib_umem_page_count(qp
->sumem
);
289 qp
->npages_recv
= ib_umem_page_count(qp
->rumem
);
292 qp
->npages
= qp
->npages_send
+ qp
->npages_recv
;
294 ret
= pvrdma_set_sq_size(to_vdev(pd
->device
),
295 &init_attr
->cap
, qp
);
299 ret
= pvrdma_set_rq_size(to_vdev(pd
->device
),
300 &init_attr
->cap
, qp
);
304 qp
->npages
= qp
->npages_send
+ qp
->npages_recv
;
306 /* Skip header page. */
307 qp
->sq
.offset
= PVRDMA_QP_NUM_HEADER_PAGES
* PAGE_SIZE
;
309 /* Recv queue pages are after send pages. */
310 qp
->rq
.offset
= qp
->npages_send
* PAGE_SIZE
;
313 if (qp
->npages
< 0 || qp
->npages
> PVRDMA_PAGE_DIR_MAX_PAGES
) {
314 dev_warn(&dev
->pdev
->dev
,
315 "overflow pages in queuepair\n");
320 ret
= pvrdma_page_dir_init(dev
, &qp
->pdir
, qp
->npages
,
323 dev_warn(&dev
->pdev
->dev
,
324 "could not allocate page directory\n");
328 if (!qp
->is_kernel
) {
329 pvrdma_page_dir_insert_umem(&qp
->pdir
, qp
->sumem
, 0);
331 pvrdma_page_dir_insert_umem(&qp
->pdir
,
335 /* Ring state is always the first page. */
336 qp
->sq
.ring
= qp
->pdir
.pages
[0];
337 qp
->rq
.ring
= is_srq
? NULL
: &qp
->sq
.ring
[1];
346 init_attr
->cap
.max_inline_data
= 0;
348 memset(cmd
, 0, sizeof(*cmd
));
349 cmd
->hdr
.cmd
= PVRDMA_CMD_CREATE_QP
;
350 cmd
->pd_handle
= to_vpd(pd
)->pd_handle
;
351 cmd
->send_cq_handle
= to_vcq(init_attr
->send_cq
)->cq_handle
;
352 cmd
->recv_cq_handle
= to_vcq(init_attr
->recv_cq
)->cq_handle
;
354 cmd
->srq_handle
= to_vsrq(init_attr
->srq
)->srq_handle
;
357 cmd
->max_send_wr
= init_attr
->cap
.max_send_wr
;
358 cmd
->max_recv_wr
= init_attr
->cap
.max_recv_wr
;
359 cmd
->max_send_sge
= init_attr
->cap
.max_send_sge
;
360 cmd
->max_recv_sge
= init_attr
->cap
.max_recv_sge
;
361 cmd
->max_inline_data
= init_attr
->cap
.max_inline_data
;
362 cmd
->sq_sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
) ? 1 : 0;
363 cmd
->qp_type
= ib_qp_type_to_pvrdma(init_attr
->qp_type
);
364 cmd
->is_srq
= is_srq
;
366 cmd
->access_flags
= IB_ACCESS_LOCAL_WRITE
;
367 cmd
->total_chunks
= qp
->npages
;
368 cmd
->send_chunks
= qp
->npages_send
- PVRDMA_QP_NUM_HEADER_PAGES
;
369 cmd
->pdir_dma
= qp
->pdir
.dir_dma
;
371 dev_dbg(&dev
->pdev
->dev
, "create queuepair with %d, %d, %d, %d\n",
372 cmd
->max_send_wr
, cmd
->max_recv_wr
, cmd
->max_send_sge
,
375 ret
= pvrdma_cmd_post(dev
, &req
, &rsp
, PVRDMA_CMD_CREATE_QP_RESP
);
377 dev_warn(&dev
->pdev
->dev
,
378 "could not create queuepair, error: %d\n", ret
);
382 /* max_send_wr/_recv_wr/_send_sge/_recv_sge/_inline_data */
383 qp
->qp_handle
= resp
->qpn
;
384 qp
->port
= init_attr
->port_num
;
385 qp
->ibqp
.qp_num
= resp
->qpn
;
386 spin_lock_irqsave(&dev
->qp_tbl_lock
, flags
);
387 dev
->qp_tbl
[qp
->qp_handle
% dev
->dsr
->caps
.max_qp
] = qp
;
388 spin_unlock_irqrestore(&dev
->qp_tbl_lock
, flags
);
393 pvrdma_page_dir_cleanup(dev
, &qp
->pdir
);
395 ib_umem_release(qp
->rumem
);
396 ib_umem_release(qp
->sumem
);
399 atomic_dec(&dev
->num_qps
);
404 static void pvrdma_free_qp(struct pvrdma_qp
*qp
)
406 struct pvrdma_dev
*dev
= to_vdev(qp
->ibqp
.device
);
407 struct pvrdma_cq
*scq
;
408 struct pvrdma_cq
*rcq
;
409 unsigned long flags
, scq_flags
, rcq_flags
;
411 /* In case cq is polling */
412 get_cqs(qp
, &scq
, &rcq
);
413 pvrdma_lock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
415 _pvrdma_flush_cqe(qp
, scq
);
417 _pvrdma_flush_cqe(qp
, rcq
);
419 spin_lock_irqsave(&dev
->qp_tbl_lock
, flags
);
420 dev
->qp_tbl
[qp
->qp_handle
] = NULL
;
421 spin_unlock_irqrestore(&dev
->qp_tbl_lock
, flags
);
423 pvrdma_unlock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
425 if (refcount_dec_and_test(&qp
->refcnt
))
427 wait_for_completion(&qp
->free
);
429 ib_umem_release(qp
->rumem
);
430 ib_umem_release(qp
->sumem
);
432 pvrdma_page_dir_cleanup(dev
, &qp
->pdir
);
436 atomic_dec(&dev
->num_qps
);
440 * pvrdma_destroy_qp - destroy a queue pair
441 * @qp: the queue pair to destroy
442 * @udata: user data or null for kernel object
444 * @return: 0 on success.
446 int pvrdma_destroy_qp(struct ib_qp
*qp
, struct ib_udata
*udata
)
448 struct pvrdma_qp
*vqp
= to_vqp(qp
);
449 union pvrdma_cmd_req req
;
450 struct pvrdma_cmd_destroy_qp
*cmd
= &req
.destroy_qp
;
453 memset(cmd
, 0, sizeof(*cmd
));
454 cmd
->hdr
.cmd
= PVRDMA_CMD_DESTROY_QP
;
455 cmd
->qp_handle
= vqp
->qp_handle
;
457 ret
= pvrdma_cmd_post(to_vdev(qp
->device
), &req
, NULL
, 0);
459 dev_warn(&to_vdev(qp
->device
)->pdev
->dev
,
460 "destroy queuepair failed, error: %d\n", ret
);
468 * pvrdma_modify_qp - modify queue pair attributes
469 * @ibqp: the queue pair
470 * @attr: the new queue pair's attributes
471 * @attr_mask: attributes mask
474 * @returns 0 on success, otherwise returns an errno.
476 int pvrdma_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
477 int attr_mask
, struct ib_udata
*udata
)
479 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
480 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
481 union pvrdma_cmd_req req
;
482 union pvrdma_cmd_resp rsp
;
483 struct pvrdma_cmd_modify_qp
*cmd
= &req
.modify_qp
;
484 enum ib_qp_state cur_state
, next_state
;
487 /* Sanity checking. Should need lock here */
488 mutex_lock(&qp
->mutex
);
489 cur_state
= (attr_mask
& IB_QP_CUR_STATE
) ? attr
->cur_qp_state
:
491 next_state
= (attr_mask
& IB_QP_STATE
) ? attr
->qp_state
: cur_state
;
493 if (!ib_modify_qp_is_ok(cur_state
, next_state
, ibqp
->qp_type
,
499 if (attr_mask
& IB_QP_PORT
) {
500 if (attr
->port_num
== 0 ||
501 attr
->port_num
> ibqp
->device
->phys_port_cnt
) {
507 if (attr_mask
& IB_QP_MIN_RNR_TIMER
) {
508 if (attr
->min_rnr_timer
> 31) {
514 if (attr_mask
& IB_QP_PKEY_INDEX
) {
515 if (attr
->pkey_index
>= dev
->dsr
->caps
.max_pkeys
) {
521 if (attr_mask
& IB_QP_QKEY
)
522 qp
->qkey
= attr
->qkey
;
524 if (cur_state
== next_state
&& cur_state
== IB_QPS_RESET
) {
529 qp
->state
= next_state
;
530 memset(cmd
, 0, sizeof(*cmd
));
531 cmd
->hdr
.cmd
= PVRDMA_CMD_MODIFY_QP
;
532 cmd
->qp_handle
= qp
->qp_handle
;
533 cmd
->attr_mask
= ib_qp_attr_mask_to_pvrdma(attr_mask
);
534 cmd
->attrs
.qp_state
= ib_qp_state_to_pvrdma(attr
->qp_state
);
535 cmd
->attrs
.cur_qp_state
=
536 ib_qp_state_to_pvrdma(attr
->cur_qp_state
);
537 cmd
->attrs
.path_mtu
= ib_mtu_to_pvrdma(attr
->path_mtu
);
538 cmd
->attrs
.path_mig_state
=
539 ib_mig_state_to_pvrdma(attr
->path_mig_state
);
540 cmd
->attrs
.qkey
= attr
->qkey
;
541 cmd
->attrs
.rq_psn
= attr
->rq_psn
;
542 cmd
->attrs
.sq_psn
= attr
->sq_psn
;
543 cmd
->attrs
.dest_qp_num
= attr
->dest_qp_num
;
544 cmd
->attrs
.qp_access_flags
=
545 ib_access_flags_to_pvrdma(attr
->qp_access_flags
);
546 cmd
->attrs
.pkey_index
= attr
->pkey_index
;
547 cmd
->attrs
.alt_pkey_index
= attr
->alt_pkey_index
;
548 cmd
->attrs
.en_sqd_async_notify
= attr
->en_sqd_async_notify
;
549 cmd
->attrs
.sq_draining
= attr
->sq_draining
;
550 cmd
->attrs
.max_rd_atomic
= attr
->max_rd_atomic
;
551 cmd
->attrs
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
552 cmd
->attrs
.min_rnr_timer
= attr
->min_rnr_timer
;
553 cmd
->attrs
.port_num
= attr
->port_num
;
554 cmd
->attrs
.timeout
= attr
->timeout
;
555 cmd
->attrs
.retry_cnt
= attr
->retry_cnt
;
556 cmd
->attrs
.rnr_retry
= attr
->rnr_retry
;
557 cmd
->attrs
.alt_port_num
= attr
->alt_port_num
;
558 cmd
->attrs
.alt_timeout
= attr
->alt_timeout
;
559 ib_qp_cap_to_pvrdma(&cmd
->attrs
.cap
, &attr
->cap
);
560 rdma_ah_attr_to_pvrdma(&cmd
->attrs
.ah_attr
, &attr
->ah_attr
);
561 rdma_ah_attr_to_pvrdma(&cmd
->attrs
.alt_ah_attr
, &attr
->alt_ah_attr
);
563 ret
= pvrdma_cmd_post(dev
, &req
, &rsp
, PVRDMA_CMD_MODIFY_QP_RESP
);
565 dev_warn(&dev
->pdev
->dev
,
566 "could not modify queuepair, error: %d\n", ret
);
567 } else if (rsp
.hdr
.err
> 0) {
568 dev_warn(&dev
->pdev
->dev
,
569 "cannot modify queuepair, error: %d\n", rsp
.hdr
.err
);
573 if (ret
== 0 && next_state
== IB_QPS_RESET
)
577 mutex_unlock(&qp
->mutex
);
582 static inline void *get_sq_wqe(struct pvrdma_qp
*qp
, unsigned int n
)
584 return pvrdma_page_dir_get_ptr(&qp
->pdir
,
585 qp
->sq
.offset
+ n
* qp
->sq
.wqe_size
);
588 static inline void *get_rq_wqe(struct pvrdma_qp
*qp
, unsigned int n
)
590 return pvrdma_page_dir_get_ptr(&qp
->pdir
,
591 qp
->rq
.offset
+ n
* qp
->rq
.wqe_size
);
594 static int set_reg_seg(struct pvrdma_sq_wqe_hdr
*wqe_hdr
,
595 const struct ib_reg_wr
*wr
)
597 struct pvrdma_user_mr
*mr
= to_vmr(wr
->mr
);
599 wqe_hdr
->wr
.fast_reg
.iova_start
= mr
->ibmr
.iova
;
600 wqe_hdr
->wr
.fast_reg
.pl_pdir_dma
= mr
->pdir
.dir_dma
;
601 wqe_hdr
->wr
.fast_reg
.page_shift
= mr
->page_shift
;
602 wqe_hdr
->wr
.fast_reg
.page_list_len
= mr
->npages
;
603 wqe_hdr
->wr
.fast_reg
.length
= mr
->ibmr
.length
;
604 wqe_hdr
->wr
.fast_reg
.access_flags
= wr
->access
;
605 wqe_hdr
->wr
.fast_reg
.rkey
= wr
->key
;
607 return pvrdma_page_dir_insert_page_list(&mr
->pdir
, mr
->pages
,
612 * pvrdma_post_send - post send work request entries on a QP
614 * @wr: work request list to post
615 * @bad_wr: the first bad WR returned
617 * @return: 0 on success, otherwise errno returned.
619 int pvrdma_post_send(struct ib_qp
*ibqp
, const struct ib_send_wr
*wr
,
620 const struct ib_send_wr
**bad_wr
)
622 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
623 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
625 struct pvrdma_sq_wqe_hdr
*wqe_hdr
;
626 struct pvrdma_sge
*sge
;
630 * In states lower than RTS, we can fail immediately. In other states,
631 * just post and let the device figure it out.
633 if (qp
->state
< IB_QPS_RTS
) {
638 spin_lock_irqsave(&qp
->sq
.lock
, flags
);
641 unsigned int tail
= 0;
643 if (unlikely(!pvrdma_idx_ring_has_space(
644 qp
->sq
.ring
, qp
->sq
.wqe_cnt
, &tail
))) {
645 dev_warn_ratelimited(&dev
->pdev
->dev
,
646 "send queue is full\n");
652 if (unlikely(wr
->num_sge
> qp
->sq
.max_sg
|| wr
->num_sge
< 0)) {
653 dev_warn_ratelimited(&dev
->pdev
->dev
,
654 "send SGE overflow\n");
660 if (unlikely(wr
->opcode
< 0)) {
661 dev_warn_ratelimited(&dev
->pdev
->dev
,
662 "invalid send opcode\n");
669 * Only support UD, RC.
670 * Need to check opcode table for thorough checking.
673 * _SEND_WITH_IMM x x x
675 * _RDMA_WRITE_WITH_IMM x x
679 * _ATOMIC_CMP_AND_SWP x
680 * _ATOMIC_FETCH_AND_ADD x
681 * _MASK_ATOMIC_CMP_AND_SWP x
682 * _MASK_ATOMIC_FETCH_AND_ADD x
686 if (qp
->ibqp
.qp_type
!= IB_QPT_UD
&&
687 qp
->ibqp
.qp_type
!= IB_QPT_RC
&&
688 wr
->opcode
!= IB_WR_SEND
) {
689 dev_warn_ratelimited(&dev
->pdev
->dev
,
690 "unsupported queuepair type\n");
694 } else if (qp
->ibqp
.qp_type
== IB_QPT_UD
||
695 qp
->ibqp
.qp_type
== IB_QPT_GSI
) {
696 if (wr
->opcode
!= IB_WR_SEND
&&
697 wr
->opcode
!= IB_WR_SEND_WITH_IMM
) {
698 dev_warn_ratelimited(&dev
->pdev
->dev
,
699 "invalid send opcode\n");
706 wqe_hdr
= (struct pvrdma_sq_wqe_hdr
*)get_sq_wqe(qp
, tail
);
707 memset(wqe_hdr
, 0, sizeof(*wqe_hdr
));
708 wqe_hdr
->wr_id
= wr
->wr_id
;
709 wqe_hdr
->num_sge
= wr
->num_sge
;
710 wqe_hdr
->opcode
= ib_wr_opcode_to_pvrdma(wr
->opcode
);
711 wqe_hdr
->send_flags
= ib_send_flags_to_pvrdma(wr
->send_flags
);
712 if (wr
->opcode
== IB_WR_SEND_WITH_IMM
||
713 wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
)
714 wqe_hdr
->ex
.imm_data
= wr
->ex
.imm_data
;
716 if (unlikely(wqe_hdr
->opcode
== PVRDMA_WR_ERROR
)) {
722 switch (qp
->ibqp
.qp_type
) {
725 if (unlikely(!ud_wr(wr
)->ah
)) {
726 dev_warn_ratelimited(&dev
->pdev
->dev
,
727 "invalid address handle\n");
734 * Use qkey from qp context if high order bit set,
735 * otherwise from work request.
737 wqe_hdr
->wr
.ud
.remote_qpn
= ud_wr(wr
)->remote_qpn
;
738 wqe_hdr
->wr
.ud
.remote_qkey
=
739 ud_wr(wr
)->remote_qkey
& 0x80000000 ?
740 qp
->qkey
: ud_wr(wr
)->remote_qkey
;
741 wqe_hdr
->wr
.ud
.av
= to_vah(ud_wr(wr
)->ah
)->av
;
745 switch (wr
->opcode
) {
746 case IB_WR_RDMA_READ
:
747 case IB_WR_RDMA_WRITE
:
748 case IB_WR_RDMA_WRITE_WITH_IMM
:
749 wqe_hdr
->wr
.rdma
.remote_addr
=
750 rdma_wr(wr
)->remote_addr
;
751 wqe_hdr
->wr
.rdma
.rkey
= rdma_wr(wr
)->rkey
;
753 case IB_WR_LOCAL_INV
:
754 case IB_WR_SEND_WITH_INV
:
755 wqe_hdr
->ex
.invalidate_rkey
=
756 wr
->ex
.invalidate_rkey
;
758 case IB_WR_ATOMIC_CMP_AND_SWP
:
759 case IB_WR_ATOMIC_FETCH_AND_ADD
:
760 wqe_hdr
->wr
.atomic
.remote_addr
=
761 atomic_wr(wr
)->remote_addr
;
762 wqe_hdr
->wr
.atomic
.rkey
= atomic_wr(wr
)->rkey
;
763 wqe_hdr
->wr
.atomic
.compare_add
=
764 atomic_wr(wr
)->compare_add
;
765 if (wr
->opcode
== IB_WR_ATOMIC_CMP_AND_SWP
)
766 wqe_hdr
->wr
.atomic
.swap
=
770 ret
= set_reg_seg(wqe_hdr
, reg_wr(wr
));
772 dev_warn_ratelimited(&dev
->pdev
->dev
,
773 "Failed to set fast register work request\n");
784 dev_warn_ratelimited(&dev
->pdev
->dev
,
785 "invalid queuepair type\n");
791 sge
= (struct pvrdma_sge
*)(wqe_hdr
+ 1);
792 for (i
= 0; i
< wr
->num_sge
; i
++) {
793 /* Need to check wqe_size 0 or max size */
794 sge
->addr
= wr
->sg_list
[i
].addr
;
795 sge
->length
= wr
->sg_list
[i
].length
;
796 sge
->lkey
= wr
->sg_list
[i
].lkey
;
800 /* Make sure wqe is written before index update */
803 /* Update shared sq ring */
804 pvrdma_idx_ring_inc(&qp
->sq
.ring
->prod_tail
,
813 spin_unlock_irqrestore(&qp
->sq
.lock
, flags
);
816 pvrdma_write_uar_qp(dev
, PVRDMA_UAR_QP_SEND
| qp
->qp_handle
);
822 * pvrdma_post_receive - post receive work request entries on a QP
824 * @wr: the work request list to post
825 * @bad_wr: the first bad WR returned
827 * @return: 0 on success, otherwise errno returned.
829 int pvrdma_post_recv(struct ib_qp
*ibqp
, const struct ib_recv_wr
*wr
,
830 const struct ib_recv_wr
**bad_wr
)
832 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
834 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
835 struct pvrdma_rq_wqe_hdr
*wqe_hdr
;
836 struct pvrdma_sge
*sge
;
841 * In the RESET state, we can fail immediately. For other states,
842 * just post and let the device figure it out.
844 if (qp
->state
== IB_QPS_RESET
) {
850 dev_warn(&dev
->pdev
->dev
, "QP associated with SRQ\n");
855 spin_lock_irqsave(&qp
->rq
.lock
, flags
);
858 unsigned int tail
= 0;
860 if (unlikely(wr
->num_sge
> qp
->rq
.max_sg
||
864 dev_warn_ratelimited(&dev
->pdev
->dev
,
865 "recv SGE overflow\n");
869 if (unlikely(!pvrdma_idx_ring_has_space(
870 qp
->rq
.ring
, qp
->rq
.wqe_cnt
, &tail
))) {
873 dev_warn_ratelimited(&dev
->pdev
->dev
,
874 "recv queue full\n");
878 wqe_hdr
= (struct pvrdma_rq_wqe_hdr
*)get_rq_wqe(qp
, tail
);
879 wqe_hdr
->wr_id
= wr
->wr_id
;
880 wqe_hdr
->num_sge
= wr
->num_sge
;
881 wqe_hdr
->total_len
= 0;
883 sge
= (struct pvrdma_sge
*)(wqe_hdr
+ 1);
884 for (i
= 0; i
< wr
->num_sge
; i
++) {
885 sge
->addr
= wr
->sg_list
[i
].addr
;
886 sge
->length
= wr
->sg_list
[i
].length
;
887 sge
->lkey
= wr
->sg_list
[i
].lkey
;
891 /* Make sure wqe is written before index update */
894 /* Update shared rq ring */
895 pvrdma_idx_ring_inc(&qp
->rq
.ring
->prod_tail
,
901 spin_unlock_irqrestore(&qp
->rq
.lock
, flags
);
903 pvrdma_write_uar_qp(dev
, PVRDMA_UAR_QP_RECV
| qp
->qp_handle
);
908 spin_unlock_irqrestore(&qp
->rq
.lock
, flags
);
914 * pvrdma_query_qp - query a queue pair's attributes
915 * @ibqp: the queue pair to query
916 * @attr: the queue pair's attributes
917 * @attr_mask: attributes mask
918 * @init_attr: initial queue pair attributes
920 * @returns 0 on success, otherwise returns an errno.
922 int pvrdma_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
923 int attr_mask
, struct ib_qp_init_attr
*init_attr
)
925 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
926 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
927 union pvrdma_cmd_req req
;
928 union pvrdma_cmd_resp rsp
;
929 struct pvrdma_cmd_query_qp
*cmd
= &req
.query_qp
;
930 struct pvrdma_cmd_query_qp_resp
*resp
= &rsp
.query_qp_resp
;
933 mutex_lock(&qp
->mutex
);
935 if (qp
->state
== IB_QPS_RESET
) {
936 attr
->qp_state
= IB_QPS_RESET
;
940 memset(cmd
, 0, sizeof(*cmd
));
941 cmd
->hdr
.cmd
= PVRDMA_CMD_QUERY_QP
;
942 cmd
->qp_handle
= qp
->qp_handle
;
943 cmd
->attr_mask
= ib_qp_attr_mask_to_pvrdma(attr_mask
);
945 ret
= pvrdma_cmd_post(dev
, &req
, &rsp
, PVRDMA_CMD_QUERY_QP_RESP
);
947 dev_warn(&dev
->pdev
->dev
,
948 "could not query queuepair, error: %d\n", ret
);
952 attr
->qp_state
= pvrdma_qp_state_to_ib(resp
->attrs
.qp_state
);
954 pvrdma_qp_state_to_ib(resp
->attrs
.cur_qp_state
);
955 attr
->path_mtu
= pvrdma_mtu_to_ib(resp
->attrs
.path_mtu
);
956 attr
->path_mig_state
=
957 pvrdma_mig_state_to_ib(resp
->attrs
.path_mig_state
);
958 attr
->qkey
= resp
->attrs
.qkey
;
959 attr
->rq_psn
= resp
->attrs
.rq_psn
;
960 attr
->sq_psn
= resp
->attrs
.sq_psn
;
961 attr
->dest_qp_num
= resp
->attrs
.dest_qp_num
;
962 attr
->qp_access_flags
=
963 pvrdma_access_flags_to_ib(resp
->attrs
.qp_access_flags
);
964 attr
->pkey_index
= resp
->attrs
.pkey_index
;
965 attr
->alt_pkey_index
= resp
->attrs
.alt_pkey_index
;
966 attr
->en_sqd_async_notify
= resp
->attrs
.en_sqd_async_notify
;
967 attr
->sq_draining
= resp
->attrs
.sq_draining
;
968 attr
->max_rd_atomic
= resp
->attrs
.max_rd_atomic
;
969 attr
->max_dest_rd_atomic
= resp
->attrs
.max_dest_rd_atomic
;
970 attr
->min_rnr_timer
= resp
->attrs
.min_rnr_timer
;
971 attr
->port_num
= resp
->attrs
.port_num
;
972 attr
->timeout
= resp
->attrs
.timeout
;
973 attr
->retry_cnt
= resp
->attrs
.retry_cnt
;
974 attr
->rnr_retry
= resp
->attrs
.rnr_retry
;
975 attr
->alt_port_num
= resp
->attrs
.alt_port_num
;
976 attr
->alt_timeout
= resp
->attrs
.alt_timeout
;
977 pvrdma_qp_cap_to_ib(&attr
->cap
, &resp
->attrs
.cap
);
978 pvrdma_ah_attr_to_rdma(&attr
->ah_attr
, &resp
->attrs
.ah_attr
);
979 pvrdma_ah_attr_to_rdma(&attr
->alt_ah_attr
, &resp
->attrs
.alt_ah_attr
);
981 qp
->state
= attr
->qp_state
;
986 attr
->cur_qp_state
= attr
->qp_state
;
988 init_attr
->event_handler
= qp
->ibqp
.event_handler
;
989 init_attr
->qp_context
= qp
->ibqp
.qp_context
;
990 init_attr
->send_cq
= qp
->ibqp
.send_cq
;
991 init_attr
->recv_cq
= qp
->ibqp
.recv_cq
;
992 init_attr
->srq
= qp
->ibqp
.srq
;
993 init_attr
->xrcd
= NULL
;
994 init_attr
->cap
= attr
->cap
;
995 init_attr
->sq_sig_type
= 0;
996 init_attr
->qp_type
= qp
->ibqp
.qp_type
;
997 init_attr
->create_flags
= 0;
998 init_attr
->port_num
= qp
->port
;
1000 mutex_unlock(&qp
->mutex
);