1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
7 #include <linux/dma-mapping.h>
8 #include <net/addrconf.h>
9 #include <rdma/uverbs_ioctl.h>
12 #include "rxe_queue.h"
13 #include "rxe_hw_counters.h"
15 static int rxe_query_device(struct ib_device
*dev
,
16 struct ib_device_attr
*attr
,
19 struct rxe_dev
*rxe
= to_rdev(dev
);
21 if (uhw
->inlen
|| uhw
->outlen
)
28 static int rxe_query_port(struct ib_device
*dev
,
29 u8 port_num
, struct ib_port_attr
*attr
)
31 struct rxe_dev
*rxe
= to_rdev(dev
);
32 struct rxe_port
*port
;
37 /* *attr being zeroed by the caller, avoid zeroing it here */
40 mutex_lock(&rxe
->usdev_lock
);
41 rc
= ib_get_eth_speed(dev
, port_num
, &attr
->active_speed
,
44 if (attr
->state
== IB_PORT_ACTIVE
)
45 attr
->phys_state
= IB_PORT_PHYS_STATE_LINK_UP
;
46 else if (dev_get_flags(rxe
->ndev
) & IFF_UP
)
47 attr
->phys_state
= IB_PORT_PHYS_STATE_POLLING
;
49 attr
->phys_state
= IB_PORT_PHYS_STATE_DISABLED
;
51 mutex_unlock(&rxe
->usdev_lock
);
56 static int rxe_query_pkey(struct ib_device
*device
,
57 u8 port_num
, u16 index
, u16
*pkey
)
62 *pkey
= IB_DEFAULT_PKEY_FULL
;
66 static int rxe_modify_device(struct ib_device
*dev
,
67 int mask
, struct ib_device_modify
*attr
)
69 struct rxe_dev
*rxe
= to_rdev(dev
);
71 if (mask
& ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID
|
72 IB_DEVICE_MODIFY_NODE_DESC
))
75 if (mask
& IB_DEVICE_MODIFY_SYS_IMAGE_GUID
)
76 rxe
->attr
.sys_image_guid
= cpu_to_be64(attr
->sys_image_guid
);
78 if (mask
& IB_DEVICE_MODIFY_NODE_DESC
) {
79 memcpy(rxe
->ib_dev
.node_desc
,
80 attr
->node_desc
, sizeof(rxe
->ib_dev
.node_desc
));
86 static int rxe_modify_port(struct ib_device
*dev
,
87 u8 port_num
, int mask
, struct ib_port_modify
*attr
)
89 struct rxe_dev
*rxe
= to_rdev(dev
);
90 struct rxe_port
*port
;
94 port
->attr
.port_cap_flags
|= attr
->set_port_cap_mask
;
95 port
->attr
.port_cap_flags
&= ~attr
->clr_port_cap_mask
;
97 if (mask
& IB_PORT_RESET_QKEY_CNTR
)
98 port
->attr
.qkey_viol_cntr
= 0;
103 static enum rdma_link_layer
rxe_get_link_layer(struct ib_device
*dev
,
106 return IB_LINK_LAYER_ETHERNET
;
109 static int rxe_alloc_ucontext(struct ib_ucontext
*uctx
, struct ib_udata
*udata
)
111 struct rxe_dev
*rxe
= to_rdev(uctx
->device
);
112 struct rxe_ucontext
*uc
= to_ruc(uctx
);
114 return rxe_add_to_pool(&rxe
->uc_pool
, &uc
->pelem
);
117 static void rxe_dealloc_ucontext(struct ib_ucontext
*ibuc
)
119 struct rxe_ucontext
*uc
= to_ruc(ibuc
);
124 static int rxe_port_immutable(struct ib_device
*dev
, u8 port_num
,
125 struct ib_port_immutable
*immutable
)
128 struct ib_port_attr attr
;
130 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP
;
132 err
= ib_query_port(dev
, port_num
, &attr
);
136 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
137 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
138 immutable
->max_mad_size
= IB_MGMT_MAD_SIZE
;
143 static int rxe_alloc_pd(struct ib_pd
*ibpd
, struct ib_udata
*udata
)
145 struct rxe_dev
*rxe
= to_rdev(ibpd
->device
);
146 struct rxe_pd
*pd
= to_rpd(ibpd
);
148 return rxe_add_to_pool(&rxe
->pd_pool
, &pd
->pelem
);
151 static int rxe_dealloc_pd(struct ib_pd
*ibpd
, struct ib_udata
*udata
)
153 struct rxe_pd
*pd
= to_rpd(ibpd
);
159 static int rxe_create_ah(struct ib_ah
*ibah
,
160 struct rdma_ah_init_attr
*init_attr
,
161 struct ib_udata
*udata
)
165 struct rxe_dev
*rxe
= to_rdev(ibah
->device
);
166 struct rxe_ah
*ah
= to_rah(ibah
);
168 err
= rxe_av_chk_attr(rxe
, init_attr
->ah_attr
);
172 err
= rxe_add_to_pool(&rxe
->ah_pool
, &ah
->pelem
);
176 rxe_init_av(init_attr
->ah_attr
, &ah
->av
);
180 static int rxe_modify_ah(struct ib_ah
*ibah
, struct rdma_ah_attr
*attr
)
183 struct rxe_dev
*rxe
= to_rdev(ibah
->device
);
184 struct rxe_ah
*ah
= to_rah(ibah
);
186 err
= rxe_av_chk_attr(rxe
, attr
);
190 rxe_init_av(attr
, &ah
->av
);
194 static int rxe_query_ah(struct ib_ah
*ibah
, struct rdma_ah_attr
*attr
)
196 struct rxe_ah
*ah
= to_rah(ibah
);
198 memset(attr
, 0, sizeof(*attr
));
199 attr
->type
= ibah
->type
;
200 rxe_av_to_attr(&ah
->av
, attr
);
204 static int rxe_destroy_ah(struct ib_ah
*ibah
, u32 flags
)
206 struct rxe_ah
*ah
= to_rah(ibah
);
212 static int post_one_recv(struct rxe_rq
*rq
, const struct ib_recv_wr
*ibwr
)
217 struct rxe_recv_wqe
*recv_wqe
;
218 int num_sge
= ibwr
->num_sge
;
220 if (unlikely(queue_full(rq
->queue
))) {
225 if (unlikely(num_sge
> rq
->max_sge
)) {
231 for (i
= 0; i
< num_sge
; i
++)
232 length
+= ibwr
->sg_list
[i
].length
;
234 recv_wqe
= producer_addr(rq
->queue
);
235 recv_wqe
->wr_id
= ibwr
->wr_id
;
236 recv_wqe
->num_sge
= num_sge
;
238 memcpy(recv_wqe
->dma
.sge
, ibwr
->sg_list
,
239 num_sge
* sizeof(struct ib_sge
));
241 recv_wqe
->dma
.length
= length
;
242 recv_wqe
->dma
.resid
= length
;
243 recv_wqe
->dma
.num_sge
= num_sge
;
244 recv_wqe
->dma
.cur_sge
= 0;
245 recv_wqe
->dma
.sge_offset
= 0;
247 /* make sure all changes to the work queue are written before we
248 * update the producer pointer
252 advance_producer(rq
->queue
);
259 static int rxe_create_srq(struct ib_srq
*ibsrq
, struct ib_srq_init_attr
*init
,
260 struct ib_udata
*udata
)
263 struct rxe_dev
*rxe
= to_rdev(ibsrq
->device
);
264 struct rxe_pd
*pd
= to_rpd(ibsrq
->pd
);
265 struct rxe_srq
*srq
= to_rsrq(ibsrq
);
266 struct rxe_create_srq_resp __user
*uresp
= NULL
;
269 if (udata
->outlen
< sizeof(*uresp
))
271 uresp
= udata
->outbuf
;
274 err
= rxe_srq_chk_attr(rxe
, NULL
, &init
->attr
, IB_SRQ_INIT_MASK
);
278 err
= rxe_add_to_pool(&rxe
->srq_pool
, &srq
->pelem
);
285 err
= rxe_srq_from_init(rxe
, srq
, init
, udata
, uresp
);
298 static int rxe_modify_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
,
299 enum ib_srq_attr_mask mask
,
300 struct ib_udata
*udata
)
303 struct rxe_srq
*srq
= to_rsrq(ibsrq
);
304 struct rxe_dev
*rxe
= to_rdev(ibsrq
->device
);
305 struct rxe_modify_srq_cmd ucmd
= {};
308 if (udata
->inlen
< sizeof(ucmd
))
311 err
= ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
));
316 err
= rxe_srq_chk_attr(rxe
, srq
, attr
, mask
);
320 err
= rxe_srq_from_attr(rxe
, srq
, attr
, mask
, &ucmd
, udata
);
330 static int rxe_query_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
)
332 struct rxe_srq
*srq
= to_rsrq(ibsrq
);
337 attr
->max_wr
= srq
->rq
.queue
->buf
->index_mask
;
338 attr
->max_sge
= srq
->rq
.max_sge
;
339 attr
->srq_limit
= srq
->limit
;
343 static int rxe_destroy_srq(struct ib_srq
*ibsrq
, struct ib_udata
*udata
)
345 struct rxe_srq
*srq
= to_rsrq(ibsrq
);
348 rxe_queue_cleanup(srq
->rq
.queue
);
350 rxe_drop_ref(srq
->pd
);
355 static int rxe_post_srq_recv(struct ib_srq
*ibsrq
, const struct ib_recv_wr
*wr
,
356 const struct ib_recv_wr
**bad_wr
)
360 struct rxe_srq
*srq
= to_rsrq(ibsrq
);
362 spin_lock_irqsave(&srq
->rq
.producer_lock
, flags
);
365 err
= post_one_recv(&srq
->rq
, wr
);
371 spin_unlock_irqrestore(&srq
->rq
.producer_lock
, flags
);
379 static struct ib_qp
*rxe_create_qp(struct ib_pd
*ibpd
,
380 struct ib_qp_init_attr
*init
,
381 struct ib_udata
*udata
)
384 struct rxe_dev
*rxe
= to_rdev(ibpd
->device
);
385 struct rxe_pd
*pd
= to_rpd(ibpd
);
387 struct rxe_create_qp_resp __user
*uresp
= NULL
;
390 if (udata
->outlen
< sizeof(*uresp
))
391 return ERR_PTR(-EINVAL
);
392 uresp
= udata
->outbuf
;
395 err
= rxe_qp_chk_init(rxe
, init
);
399 qp
= rxe_alloc(&rxe
->qp_pool
);
415 err
= rxe_qp_from_init(rxe
, qp
, pd
, init
, uresp
, ibpd
, udata
);
429 static int rxe_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
430 int mask
, struct ib_udata
*udata
)
433 struct rxe_dev
*rxe
= to_rdev(ibqp
->device
);
434 struct rxe_qp
*qp
= to_rqp(ibqp
);
436 err
= rxe_qp_chk_attr(rxe
, qp
, attr
, mask
);
440 err
= rxe_qp_from_attr(qp
, attr
, mask
, udata
);
450 static int rxe_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
451 int mask
, struct ib_qp_init_attr
*init
)
453 struct rxe_qp
*qp
= to_rqp(ibqp
);
455 rxe_qp_to_init(qp
, init
);
456 rxe_qp_to_attr(qp
, attr
, mask
);
461 static int rxe_destroy_qp(struct ib_qp
*ibqp
, struct ib_udata
*udata
)
463 struct rxe_qp
*qp
= to_rqp(ibqp
);
471 static int validate_send_wr(struct rxe_qp
*qp
, const struct ib_send_wr
*ibwr
,
472 unsigned int mask
, unsigned int length
)
474 int num_sge
= ibwr
->num_sge
;
475 struct rxe_sq
*sq
= &qp
->sq
;
477 if (unlikely(num_sge
> sq
->max_sge
))
480 if (unlikely(mask
& WR_ATOMIC_MASK
)) {
484 if (atomic_wr(ibwr
)->remote_addr
& 0x7)
488 if (unlikely((ibwr
->send_flags
& IB_SEND_INLINE
) &&
489 (length
> sq
->max_inline
)))
498 static void init_send_wr(struct rxe_qp
*qp
, struct rxe_send_wr
*wr
,
499 const struct ib_send_wr
*ibwr
)
501 wr
->wr_id
= ibwr
->wr_id
;
502 wr
->num_sge
= ibwr
->num_sge
;
503 wr
->opcode
= ibwr
->opcode
;
504 wr
->send_flags
= ibwr
->send_flags
;
506 if (qp_type(qp
) == IB_QPT_UD
||
507 qp_type(qp
) == IB_QPT_SMI
||
508 qp_type(qp
) == IB_QPT_GSI
) {
509 wr
->wr
.ud
.remote_qpn
= ud_wr(ibwr
)->remote_qpn
;
510 wr
->wr
.ud
.remote_qkey
= ud_wr(ibwr
)->remote_qkey
;
511 if (qp_type(qp
) == IB_QPT_GSI
)
512 wr
->wr
.ud
.pkey_index
= ud_wr(ibwr
)->pkey_index
;
513 if (wr
->opcode
== IB_WR_SEND_WITH_IMM
)
514 wr
->ex
.imm_data
= ibwr
->ex
.imm_data
;
516 switch (wr
->opcode
) {
517 case IB_WR_RDMA_WRITE_WITH_IMM
:
518 wr
->ex
.imm_data
= ibwr
->ex
.imm_data
;
520 case IB_WR_RDMA_READ
:
521 case IB_WR_RDMA_WRITE
:
522 wr
->wr
.rdma
.remote_addr
= rdma_wr(ibwr
)->remote_addr
;
523 wr
->wr
.rdma
.rkey
= rdma_wr(ibwr
)->rkey
;
525 case IB_WR_SEND_WITH_IMM
:
526 wr
->ex
.imm_data
= ibwr
->ex
.imm_data
;
528 case IB_WR_SEND_WITH_INV
:
529 wr
->ex
.invalidate_rkey
= ibwr
->ex
.invalidate_rkey
;
531 case IB_WR_ATOMIC_CMP_AND_SWP
:
532 case IB_WR_ATOMIC_FETCH_AND_ADD
:
533 wr
->wr
.atomic
.remote_addr
=
534 atomic_wr(ibwr
)->remote_addr
;
535 wr
->wr
.atomic
.compare_add
=
536 atomic_wr(ibwr
)->compare_add
;
537 wr
->wr
.atomic
.swap
= atomic_wr(ibwr
)->swap
;
538 wr
->wr
.atomic
.rkey
= atomic_wr(ibwr
)->rkey
;
540 case IB_WR_LOCAL_INV
:
541 wr
->ex
.invalidate_rkey
= ibwr
->ex
.invalidate_rkey
;
544 wr
->wr
.reg
.mr
= reg_wr(ibwr
)->mr
;
545 wr
->wr
.reg
.key
= reg_wr(ibwr
)->key
;
546 wr
->wr
.reg
.access
= reg_wr(ibwr
)->access
;
554 static int init_send_wqe(struct rxe_qp
*qp
, const struct ib_send_wr
*ibwr
,
555 unsigned int mask
, unsigned int length
,
556 struct rxe_send_wqe
*wqe
)
558 int num_sge
= ibwr
->num_sge
;
563 init_send_wr(qp
, &wqe
->wr
, ibwr
);
565 if (qp_type(qp
) == IB_QPT_UD
||
566 qp_type(qp
) == IB_QPT_SMI
||
567 qp_type(qp
) == IB_QPT_GSI
)
568 memcpy(&wqe
->av
, &to_rah(ud_wr(ibwr
)->ah
)->av
, sizeof(wqe
->av
));
570 if (unlikely(ibwr
->send_flags
& IB_SEND_INLINE
)) {
571 p
= wqe
->dma
.inline_data
;
574 for (i
= 0; i
< num_sge
; i
++, sge
++) {
575 memcpy(p
, (void *)(uintptr_t)sge
->addr
,
580 } else if (mask
& WR_REG_MASK
) {
582 wqe
->state
= wqe_state_posted
;
585 memcpy(wqe
->dma
.sge
, ibwr
->sg_list
,
586 num_sge
* sizeof(struct ib_sge
));
588 wqe
->iova
= mask
& WR_ATOMIC_MASK
? atomic_wr(ibwr
)->remote_addr
:
589 mask
& WR_READ_OR_WRITE_MASK
? rdma_wr(ibwr
)->remote_addr
: 0;
591 wqe
->dma
.length
= length
;
592 wqe
->dma
.resid
= length
;
593 wqe
->dma
.num_sge
= num_sge
;
594 wqe
->dma
.cur_sge
= 0;
595 wqe
->dma
.sge_offset
= 0;
596 wqe
->state
= wqe_state_posted
;
597 wqe
->ssn
= atomic_add_return(1, &qp
->ssn
);
602 static int post_one_send(struct rxe_qp
*qp
, const struct ib_send_wr
*ibwr
,
603 unsigned int mask
, u32 length
)
606 struct rxe_sq
*sq
= &qp
->sq
;
607 struct rxe_send_wqe
*send_wqe
;
610 err
= validate_send_wr(qp
, ibwr
, mask
, length
);
614 spin_lock_irqsave(&qp
->sq
.sq_lock
, flags
);
616 if (unlikely(queue_full(sq
->queue
))) {
621 send_wqe
= producer_addr(sq
->queue
);
623 err
= init_send_wqe(qp
, ibwr
, mask
, length
, send_wqe
);
628 * make sure all changes to the work queue are
629 * written before we update the producer pointer
633 advance_producer(sq
->queue
);
634 spin_unlock_irqrestore(&qp
->sq
.sq_lock
, flags
);
639 spin_unlock_irqrestore(&qp
->sq
.sq_lock
, flags
);
643 static int rxe_post_send_kernel(struct rxe_qp
*qp
, const struct ib_send_wr
*wr
,
644 const struct ib_send_wr
**bad_wr
)
648 unsigned int length
= 0;
650 struct ib_send_wr
*next
;
653 mask
= wr_opcode_mask(wr
->opcode
, qp
);
654 if (unlikely(!mask
)) {
660 if (unlikely((wr
->send_flags
& IB_SEND_INLINE
) &&
661 !(mask
& WR_INLINE_MASK
))) {
670 for (i
= 0; i
< wr
->num_sge
; i
++)
671 length
+= wr
->sg_list
[i
].length
;
673 err
= post_one_send(qp
, wr
, mask
, length
);
682 rxe_run_task(&qp
->req
.task
, 1);
683 if (unlikely(qp
->req
.state
== QP_STATE_ERROR
))
684 rxe_run_task(&qp
->comp
.task
, 1);
689 static int rxe_post_send(struct ib_qp
*ibqp
, const struct ib_send_wr
*wr
,
690 const struct ib_send_wr
**bad_wr
)
692 struct rxe_qp
*qp
= to_rqp(ibqp
);
694 if (unlikely(!qp
->valid
)) {
699 if (unlikely(qp
->req
.state
< QP_STATE_READY
)) {
705 /* Utilize process context to do protocol processing */
706 rxe_run_task(&qp
->req
.task
, 0);
709 return rxe_post_send_kernel(qp
, wr
, bad_wr
);
712 static int rxe_post_recv(struct ib_qp
*ibqp
, const struct ib_recv_wr
*wr
,
713 const struct ib_recv_wr
**bad_wr
)
716 struct rxe_qp
*qp
= to_rqp(ibqp
);
717 struct rxe_rq
*rq
= &qp
->rq
;
720 if (unlikely((qp_state(qp
) < IB_QPS_INIT
) || !qp
->valid
)) {
726 if (unlikely(qp
->srq
)) {
732 spin_lock_irqsave(&rq
->producer_lock
, flags
);
735 err
= post_one_recv(rq
, wr
);
743 spin_unlock_irqrestore(&rq
->producer_lock
, flags
);
745 if (qp
->resp
.state
== QP_STATE_ERROR
)
746 rxe_run_task(&qp
->resp
.task
, 1);
752 static int rxe_create_cq(struct ib_cq
*ibcq
, const struct ib_cq_init_attr
*attr
,
753 struct ib_udata
*udata
)
756 struct ib_device
*dev
= ibcq
->device
;
757 struct rxe_dev
*rxe
= to_rdev(dev
);
758 struct rxe_cq
*cq
= to_rcq(ibcq
);
759 struct rxe_create_cq_resp __user
*uresp
= NULL
;
762 if (udata
->outlen
< sizeof(*uresp
))
764 uresp
= udata
->outbuf
;
770 err
= rxe_cq_chk_attr(rxe
, NULL
, attr
->cqe
, attr
->comp_vector
);
774 err
= rxe_cq_from_init(rxe
, cq
, attr
->cqe
, attr
->comp_vector
, udata
,
779 return rxe_add_to_pool(&rxe
->cq_pool
, &cq
->pelem
);
782 static int rxe_destroy_cq(struct ib_cq
*ibcq
, struct ib_udata
*udata
)
784 struct rxe_cq
*cq
= to_rcq(ibcq
);
792 static int rxe_resize_cq(struct ib_cq
*ibcq
, int cqe
, struct ib_udata
*udata
)
795 struct rxe_cq
*cq
= to_rcq(ibcq
);
796 struct rxe_dev
*rxe
= to_rdev(ibcq
->device
);
797 struct rxe_resize_cq_resp __user
*uresp
= NULL
;
800 if (udata
->outlen
< sizeof(*uresp
))
802 uresp
= udata
->outbuf
;
805 err
= rxe_cq_chk_attr(rxe
, cq
, cqe
, 0);
809 err
= rxe_cq_resize_queue(cq
, cqe
, uresp
, udata
);
819 static int rxe_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
)
822 struct rxe_cq
*cq
= to_rcq(ibcq
);
826 spin_lock_irqsave(&cq
->cq_lock
, flags
);
827 for (i
= 0; i
< num_entries
; i
++) {
828 cqe
= queue_head(cq
->queue
);
832 memcpy(wc
++, &cqe
->ibwc
, sizeof(*wc
));
833 advance_consumer(cq
->queue
);
835 spin_unlock_irqrestore(&cq
->cq_lock
, flags
);
840 static int rxe_peek_cq(struct ib_cq
*ibcq
, int wc_cnt
)
842 struct rxe_cq
*cq
= to_rcq(ibcq
);
843 int count
= queue_count(cq
->queue
);
845 return (count
> wc_cnt
) ? wc_cnt
: count
;
848 static int rxe_req_notify_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
850 struct rxe_cq
*cq
= to_rcq(ibcq
);
851 unsigned long irq_flags
;
854 spin_lock_irqsave(&cq
->cq_lock
, irq_flags
);
855 if (cq
->notify
!= IB_CQ_NEXT_COMP
)
856 cq
->notify
= flags
& IB_CQ_SOLICITED_MASK
;
858 if ((flags
& IB_CQ_REPORT_MISSED_EVENTS
) && !queue_empty(cq
->queue
))
861 spin_unlock_irqrestore(&cq
->cq_lock
, irq_flags
);
866 static struct ib_mr
*rxe_get_dma_mr(struct ib_pd
*ibpd
, int access
)
868 struct rxe_dev
*rxe
= to_rdev(ibpd
->device
);
869 struct rxe_pd
*pd
= to_rpd(ibpd
);
872 mr
= rxe_alloc(&rxe
->mr_pool
);
874 return ERR_PTR(-ENOMEM
);
878 rxe_mem_init_dma(pd
, access
, mr
);
883 static struct ib_mr
*rxe_reg_user_mr(struct ib_pd
*ibpd
,
887 int access
, struct ib_udata
*udata
)
890 struct rxe_dev
*rxe
= to_rdev(ibpd
->device
);
891 struct rxe_pd
*pd
= to_rpd(ibpd
);
894 mr
= rxe_alloc(&rxe
->mr_pool
);
904 err
= rxe_mem_init_user(pd
, start
, length
, iova
,
919 static int rxe_dereg_mr(struct ib_mr
*ibmr
, struct ib_udata
*udata
)
921 struct rxe_mem
*mr
= to_rmr(ibmr
);
923 mr
->state
= RXE_MEM_STATE_ZOMBIE
;
924 rxe_drop_ref(mr_pd(mr
));
930 static struct ib_mr
*rxe_alloc_mr(struct ib_pd
*ibpd
, enum ib_mr_type mr_type
,
933 struct rxe_dev
*rxe
= to_rdev(ibpd
->device
);
934 struct rxe_pd
*pd
= to_rpd(ibpd
);
938 if (mr_type
!= IB_MR_TYPE_MEM_REG
)
939 return ERR_PTR(-EINVAL
);
941 mr
= rxe_alloc(&rxe
->mr_pool
);
951 err
= rxe_mem_init_fast(pd
, max_num_sg
, mr
);
965 static int rxe_set_page(struct ib_mr
*ibmr
, u64 addr
)
967 struct rxe_mem
*mr
= to_rmr(ibmr
);
969 struct rxe_phys_buf
*buf
;
971 if (unlikely(mr
->nbuf
== mr
->num_buf
))
974 map
= mr
->map
[mr
->nbuf
/ RXE_BUF_PER_MAP
];
975 buf
= &map
->buf
[mr
->nbuf
% RXE_BUF_PER_MAP
];
978 buf
->size
= ibmr
->page_size
;
984 static int rxe_map_mr_sg(struct ib_mr
*ibmr
, struct scatterlist
*sg
,
985 int sg_nents
, unsigned int *sg_offset
)
987 struct rxe_mem
*mr
= to_rmr(ibmr
);
992 n
= ib_sg_to_pages(ibmr
, sg
, sg_nents
, sg_offset
, rxe_set_page
);
995 mr
->iova
= ibmr
->iova
;
996 mr
->length
= ibmr
->length
;
997 mr
->page_shift
= ilog2(ibmr
->page_size
);
998 mr
->page_mask
= ibmr
->page_size
- 1;
999 mr
->offset
= mr
->iova
& mr
->page_mask
;
1004 static int rxe_attach_mcast(struct ib_qp
*ibqp
, union ib_gid
*mgid
, u16 mlid
)
1007 struct rxe_dev
*rxe
= to_rdev(ibqp
->device
);
1008 struct rxe_qp
*qp
= to_rqp(ibqp
);
1009 struct rxe_mc_grp
*grp
;
1011 /* takes a ref on grp if successful */
1012 err
= rxe_mcast_get_grp(rxe
, mgid
, &grp
);
1016 err
= rxe_mcast_add_grp_elem(rxe
, qp
, grp
);
1022 static int rxe_detach_mcast(struct ib_qp
*ibqp
, union ib_gid
*mgid
, u16 mlid
)
1024 struct rxe_dev
*rxe
= to_rdev(ibqp
->device
);
1025 struct rxe_qp
*qp
= to_rqp(ibqp
);
1027 return rxe_mcast_drop_grp_elem(rxe
, qp
, mgid
);
1030 static ssize_t
parent_show(struct device
*device
,
1031 struct device_attribute
*attr
, char *buf
)
1033 struct rxe_dev
*rxe
=
1034 rdma_device_to_drv_device(device
, struct rxe_dev
, ib_dev
);
1036 return scnprintf(buf
, PAGE_SIZE
, "%s\n", rxe_parent_name(rxe
, 1));
1039 static DEVICE_ATTR_RO(parent
);
1041 static struct attribute
*rxe_dev_attributes
[] = {
1042 &dev_attr_parent
.attr
,
1046 static const struct attribute_group rxe_attr_group
= {
1047 .attrs
= rxe_dev_attributes
,
1050 static int rxe_enable_driver(struct ib_device
*ib_dev
)
1052 struct rxe_dev
*rxe
= container_of(ib_dev
, struct rxe_dev
, ib_dev
);
1054 rxe_set_port_state(rxe
);
1055 dev_info(&rxe
->ib_dev
.dev
, "added %s\n", netdev_name(rxe
->ndev
));
1059 static const struct ib_device_ops rxe_dev_ops
= {
1060 .owner
= THIS_MODULE
,
1061 .driver_id
= RDMA_DRIVER_RXE
,
1062 .uverbs_abi_ver
= RXE_UVERBS_ABI_VERSION
,
1064 .alloc_hw_stats
= rxe_ib_alloc_hw_stats
,
1065 .alloc_mr
= rxe_alloc_mr
,
1066 .alloc_pd
= rxe_alloc_pd
,
1067 .alloc_ucontext
= rxe_alloc_ucontext
,
1068 .attach_mcast
= rxe_attach_mcast
,
1069 .create_ah
= rxe_create_ah
,
1070 .create_cq
= rxe_create_cq
,
1071 .create_qp
= rxe_create_qp
,
1072 .create_srq
= rxe_create_srq
,
1073 .dealloc_driver
= rxe_dealloc
,
1074 .dealloc_pd
= rxe_dealloc_pd
,
1075 .dealloc_ucontext
= rxe_dealloc_ucontext
,
1076 .dereg_mr
= rxe_dereg_mr
,
1077 .destroy_ah
= rxe_destroy_ah
,
1078 .destroy_cq
= rxe_destroy_cq
,
1079 .destroy_qp
= rxe_destroy_qp
,
1080 .destroy_srq
= rxe_destroy_srq
,
1081 .detach_mcast
= rxe_detach_mcast
,
1082 .enable_driver
= rxe_enable_driver
,
1083 .get_dma_mr
= rxe_get_dma_mr
,
1084 .get_hw_stats
= rxe_ib_get_hw_stats
,
1085 .get_link_layer
= rxe_get_link_layer
,
1086 .get_port_immutable
= rxe_port_immutable
,
1087 .map_mr_sg
= rxe_map_mr_sg
,
1089 .modify_ah
= rxe_modify_ah
,
1090 .modify_device
= rxe_modify_device
,
1091 .modify_port
= rxe_modify_port
,
1092 .modify_qp
= rxe_modify_qp
,
1093 .modify_srq
= rxe_modify_srq
,
1094 .peek_cq
= rxe_peek_cq
,
1095 .poll_cq
= rxe_poll_cq
,
1096 .post_recv
= rxe_post_recv
,
1097 .post_send
= rxe_post_send
,
1098 .post_srq_recv
= rxe_post_srq_recv
,
1099 .query_ah
= rxe_query_ah
,
1100 .query_device
= rxe_query_device
,
1101 .query_pkey
= rxe_query_pkey
,
1102 .query_port
= rxe_query_port
,
1103 .query_qp
= rxe_query_qp
,
1104 .query_srq
= rxe_query_srq
,
1105 .reg_user_mr
= rxe_reg_user_mr
,
1106 .req_notify_cq
= rxe_req_notify_cq
,
1107 .resize_cq
= rxe_resize_cq
,
1109 INIT_RDMA_OBJ_SIZE(ib_ah
, rxe_ah
, ibah
),
1110 INIT_RDMA_OBJ_SIZE(ib_cq
, rxe_cq
, ibcq
),
1111 INIT_RDMA_OBJ_SIZE(ib_pd
, rxe_pd
, ibpd
),
1112 INIT_RDMA_OBJ_SIZE(ib_srq
, rxe_srq
, ibsrq
),
1113 INIT_RDMA_OBJ_SIZE(ib_ucontext
, rxe_ucontext
, ibuc
),
1116 int rxe_register_device(struct rxe_dev
*rxe
, const char *ibdev_name
)
1119 struct ib_device
*dev
= &rxe
->ib_dev
;
1120 struct crypto_shash
*tfm
;
1123 strlcpy(dev
->node_desc
, "rxe", sizeof(dev
->node_desc
));
1125 dev
->node_type
= RDMA_NODE_IB_CA
;
1126 dev
->phys_port_cnt
= 1;
1127 dev
->num_comp_vectors
= num_possible_cpus();
1128 dev
->dev
.parent
= rxe_dma_device(rxe
);
1129 dev
->local_dma_lkey
= 0;
1130 addrconf_addr_eui48((unsigned char *)&dev
->node_guid
,
1131 rxe
->ndev
->dev_addr
);
1132 dev
->dev
.dma_parms
= &rxe
->dma_parms
;
1133 dma_set_max_seg_size(&dev
->dev
, UINT_MAX
);
1134 dma_mask
= IS_ENABLED(CONFIG_64BIT
) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32);
1135 err
= dma_coerce_mask_and_coherent(&dev
->dev
, dma_mask
);
1139 dev
->uverbs_cmd_mask
= BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT
)
1140 | BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
)
1141 | BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE
)
1142 | BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT
)
1143 | BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD
)
1144 | BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD
)
1145 | BIT_ULL(IB_USER_VERBS_CMD_CREATE_SRQ
)
1146 | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_SRQ
)
1147 | BIT_ULL(IB_USER_VERBS_CMD_QUERY_SRQ
)
1148 | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_SRQ
)
1149 | BIT_ULL(IB_USER_VERBS_CMD_POST_SRQ_RECV
)
1150 | BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP
)
1151 | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP
)
1152 | BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP
)
1153 | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP
)
1154 | BIT_ULL(IB_USER_VERBS_CMD_POST_SEND
)
1155 | BIT_ULL(IB_USER_VERBS_CMD_POST_RECV
)
1156 | BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ
)
1157 | BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ
)
1158 | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ
)
1159 | BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ
)
1160 | BIT_ULL(IB_USER_VERBS_CMD_PEEK_CQ
)
1161 | BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
)
1162 | BIT_ULL(IB_USER_VERBS_CMD_REG_MR
)
1163 | BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR
)
1164 | BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH
)
1165 | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_AH
)
1166 | BIT_ULL(IB_USER_VERBS_CMD_QUERY_AH
)
1167 | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH
)
1168 | BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST
)
1169 | BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST
)
1172 ib_set_device_ops(dev
, &rxe_dev_ops
);
1173 err
= ib_device_set_netdev(&rxe
->ib_dev
, rxe
->ndev
, 1);
1177 tfm
= crypto_alloc_shash("crc32", 0, 0);
1179 pr_err("failed to allocate crc algorithm err:%ld\n",
1181 return PTR_ERR(tfm
);
1185 rdma_set_device_sysfs_group(dev
, &rxe_attr_group
);
1186 err
= ib_register_device(dev
, ibdev_name
, NULL
);
1188 pr_warn("%s failed with error %d\n", __func__
, err
);
1191 * Note that rxe may be invalid at this point if another thread