1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
6 * irdma_query_device - get device attributes
7 * @ibdev: device pointer from stack
8 * @props: returning device attributes
11 static int irdma_query_device(struct ib_device
*ibdev
,
12 struct ib_device_attr
*props
,
13 struct ib_udata
*udata
)
15 struct irdma_device
*iwdev
= to_iwdev(ibdev
);
16 struct irdma_pci_f
*rf
= iwdev
->rf
;
17 struct pci_dev
*pcidev
= iwdev
->rf
->pcidev
;
18 struct irdma_hw_attrs
*hw_attrs
= &rf
->sc_dev
.hw_attrs
;
20 if (udata
->inlen
|| udata
->outlen
)
23 memset(props
, 0, sizeof(*props
));
24 ether_addr_copy((u8
*)&props
->sys_image_guid
, iwdev
->netdev
->dev_addr
);
25 props
->fw_ver
= (u64
)irdma_fw_major_ver(&rf
->sc_dev
) << 32 |
26 irdma_fw_minor_ver(&rf
->sc_dev
);
27 props
->device_cap_flags
= iwdev
->device_cap_flags
;
28 props
->vendor_id
= pcidev
->vendor
;
29 props
->vendor_part_id
= pcidev
->device
;
31 props
->hw_ver
= rf
->pcidev
->revision
;
32 props
->page_size_cap
= SZ_4K
| SZ_2M
| SZ_1G
;
33 props
->max_mr_size
= hw_attrs
->max_mr_size
;
34 props
->max_qp
= rf
->max_qp
- rf
->used_qps
;
35 props
->max_qp_wr
= hw_attrs
->max_qp_wr
;
36 props
->max_send_sge
= hw_attrs
->uk_attrs
.max_hw_wq_frags
;
37 props
->max_recv_sge
= hw_attrs
->uk_attrs
.max_hw_wq_frags
;
38 props
->max_cq
= rf
->max_cq
- rf
->used_cqs
;
39 props
->max_cqe
= rf
->max_cqe
;
40 props
->max_mr
= rf
->max_mr
- rf
->used_mrs
;
41 props
->max_mw
= props
->max_mr
;
42 props
->max_pd
= rf
->max_pd
- rf
->used_pds
;
43 props
->max_sge_rd
= hw_attrs
->uk_attrs
.max_hw_read_sges
;
44 props
->max_qp_rd_atom
= hw_attrs
->max_hw_ird
;
45 props
->max_qp_init_rd_atom
= hw_attrs
->max_hw_ord
;
46 if (rdma_protocol_roce(ibdev
, 1))
47 props
->max_pkeys
= IRDMA_PKEY_TBL_SZ
;
48 props
->max_ah
= rf
->max_ah
;
49 props
->max_mcast_grp
= rf
->max_mcg
;
50 props
->max_mcast_qp_attach
= IRDMA_MAX_MGS_PER_CTX
;
51 props
->max_total_mcast_qp_attach
= rf
->max_qp
* IRDMA_MAX_MGS_PER_CTX
;
52 props
->max_fast_reg_page_list_len
= IRDMA_MAX_PAGES_PER_FMR
;
53 #define HCA_CLOCK_TIMESTAMP_MASK 0x1ffff
54 if (hw_attrs
->uk_attrs
.hw_rev
>= IRDMA_GEN_2
)
55 props
->timestamp_mask
= HCA_CLOCK_TIMESTAMP_MASK
;
61 * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed
62 * @link_speed: netdev phy link speed
63 * @active_speed: IB port speed
64 * @active_width: IB port width
66 static void irdma_get_eth_speed_and_width(u32 link_speed
, u16
*active_speed
,
69 if (link_speed
<= SPEED_1000
) {
70 *active_width
= IB_WIDTH_1X
;
71 *active_speed
= IB_SPEED_SDR
;
72 } else if (link_speed
<= SPEED_10000
) {
73 *active_width
= IB_WIDTH_1X
;
74 *active_speed
= IB_SPEED_FDR10
;
75 } else if (link_speed
<= SPEED_20000
) {
76 *active_width
= IB_WIDTH_4X
;
77 *active_speed
= IB_SPEED_DDR
;
78 } else if (link_speed
<= SPEED_25000
) {
79 *active_width
= IB_WIDTH_1X
;
80 *active_speed
= IB_SPEED_EDR
;
81 } else if (link_speed
<= SPEED_40000
) {
82 *active_width
= IB_WIDTH_4X
;
83 *active_speed
= IB_SPEED_FDR10
;
85 *active_width
= IB_WIDTH_4X
;
86 *active_speed
= IB_SPEED_EDR
;
91 * irdma_query_port - get port attributes
92 * @ibdev: device pointer from stack
93 * @port: port number for query
94 * @props: returning device attributes
96 static int irdma_query_port(struct ib_device
*ibdev
, u32 port
,
97 struct ib_port_attr
*props
)
99 struct irdma_device
*iwdev
= to_iwdev(ibdev
);
100 struct net_device
*netdev
= iwdev
->netdev
;
102 /* no need to zero out pros here. done by caller */
104 props
->max_mtu
= IB_MTU_4096
;
105 props
->active_mtu
= ib_mtu_int_to_enum(netdev
->mtu
);
110 if (netif_carrier_ok(netdev
) && netif_running(netdev
)) {
111 props
->state
= IB_PORT_ACTIVE
;
112 props
->phys_state
= IB_PORT_PHYS_STATE_LINK_UP
;
114 props
->state
= IB_PORT_DOWN
;
115 props
->phys_state
= IB_PORT_PHYS_STATE_DISABLED
;
117 irdma_get_eth_speed_and_width(SPEED_100000
, &props
->active_speed
,
118 &props
->active_width
);
120 if (rdma_protocol_roce(ibdev
, 1)) {
121 props
->gid_tbl_len
= 32;
122 props
->ip_gids
= true;
123 props
->pkey_tbl_len
= IRDMA_PKEY_TBL_SZ
;
125 props
->gid_tbl_len
= 1;
127 props
->qkey_viol_cntr
= 0;
128 props
->port_cap_flags
|= IB_PORT_CM_SUP
| IB_PORT_REINIT_SUP
;
129 props
->max_msg_sz
= iwdev
->rf
->sc_dev
.hw_attrs
.max_hw_outbound_msg_size
;
135 * irdma_disassociate_ucontext - Disassociate user context
136 * @context: ib user context
138 static void irdma_disassociate_ucontext(struct ib_ucontext
*context
)
142 static int irdma_mmap_legacy(struct irdma_ucontext
*ucontext
,
143 struct vm_area_struct
*vma
)
147 if (vma
->vm_pgoff
|| vma
->vm_end
- vma
->vm_start
!= PAGE_SIZE
)
150 vma
->vm_private_data
= ucontext
;
151 pfn
= ((uintptr_t)ucontext
->iwdev
->rf
->sc_dev
.hw_regs
[IRDMA_DB_ADDR_OFFSET
] +
152 pci_resource_start(ucontext
->iwdev
->rf
->pcidev
, 0)) >> PAGE_SHIFT
;
154 return rdma_user_mmap_io(&ucontext
->ibucontext
, vma
, pfn
, PAGE_SIZE
,
155 pgprot_noncached(vma
->vm_page_prot
), NULL
);
158 static void irdma_mmap_free(struct rdma_user_mmap_entry
*rdma_entry
)
160 struct irdma_user_mmap_entry
*entry
= to_irdma_mmap_entry(rdma_entry
);
165 static struct rdma_user_mmap_entry
*
166 irdma_user_mmap_entry_insert(struct irdma_ucontext
*ucontext
, u64 bar_offset
,
167 enum irdma_mmap_flag mmap_flag
, u64
*mmap_offset
)
169 struct irdma_user_mmap_entry
*entry
= kzalloc(sizeof(*entry
), GFP_KERNEL
);
175 entry
->bar_offset
= bar_offset
;
176 entry
->mmap_flag
= mmap_flag
;
178 ret
= rdma_user_mmap_entry_insert(&ucontext
->ibucontext
,
179 &entry
->rdma_entry
, PAGE_SIZE
);
184 *mmap_offset
= rdma_user_mmap_get_offset(&entry
->rdma_entry
);
186 return &entry
->rdma_entry
;
190 * irdma_mmap - user memory map
191 * @context: context created during alloc
192 * @vma: kernel info for user memory map
194 static int irdma_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
196 struct rdma_user_mmap_entry
*rdma_entry
;
197 struct irdma_user_mmap_entry
*entry
;
198 struct irdma_ucontext
*ucontext
;
202 ucontext
= to_ucontext(context
);
204 /* Legacy support for libi40iw with hard-coded mmap key */
205 if (ucontext
->legacy_mode
)
206 return irdma_mmap_legacy(ucontext
, vma
);
208 rdma_entry
= rdma_user_mmap_entry_get(&ucontext
->ibucontext
, vma
);
210 ibdev_dbg(&ucontext
->iwdev
->ibdev
,
211 "VERBS: pgoff[0x%lx] does not have valid entry\n",
216 entry
= to_irdma_mmap_entry(rdma_entry
);
217 ibdev_dbg(&ucontext
->iwdev
->ibdev
,
218 "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n",
219 entry
->bar_offset
, entry
->mmap_flag
);
221 pfn
= (entry
->bar_offset
+
222 pci_resource_start(ucontext
->iwdev
->rf
->pcidev
, 0)) >> PAGE_SHIFT
;
224 switch (entry
->mmap_flag
) {
225 case IRDMA_MMAP_IO_NC
:
226 ret
= rdma_user_mmap_io(context
, vma
, pfn
, PAGE_SIZE
,
227 pgprot_noncached(vma
->vm_page_prot
),
230 case IRDMA_MMAP_IO_WC
:
231 ret
= rdma_user_mmap_io(context
, vma
, pfn
, PAGE_SIZE
,
232 pgprot_writecombine(vma
->vm_page_prot
),
240 ibdev_dbg(&ucontext
->iwdev
->ibdev
,
241 "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n",
242 entry
->bar_offset
, entry
->mmap_flag
, ret
);
243 rdma_user_mmap_entry_put(rdma_entry
);
249 * irdma_alloc_push_page - allocate a push page for qp
252 static void irdma_alloc_push_page(struct irdma_qp
*iwqp
)
254 struct irdma_cqp_request
*cqp_request
;
255 struct cqp_cmds_info
*cqp_info
;
256 struct irdma_device
*iwdev
= iwqp
->iwdev
;
257 struct irdma_sc_qp
*qp
= &iwqp
->sc_qp
;
258 enum irdma_status_code status
;
260 cqp_request
= irdma_alloc_and_get_cqp_request(&iwdev
->rf
->cqp
, true);
264 cqp_info
= &cqp_request
->info
;
265 cqp_info
->cqp_cmd
= IRDMA_OP_MANAGE_PUSH_PAGE
;
266 cqp_info
->post_sq
= 1;
267 cqp_info
->in
.u
.manage_push_page
.info
.push_idx
= 0;
268 cqp_info
->in
.u
.manage_push_page
.info
.qs_handle
=
269 qp
->vsi
->qos
[qp
->user_pri
].qs_handle
;
270 cqp_info
->in
.u
.manage_push_page
.info
.free_page
= 0;
271 cqp_info
->in
.u
.manage_push_page
.info
.push_page_type
= 0;
272 cqp_info
->in
.u
.manage_push_page
.cqp
= &iwdev
->rf
->cqp
.sc_cqp
;
273 cqp_info
->in
.u
.manage_push_page
.scratch
= (uintptr_t)cqp_request
;
275 status
= irdma_handle_cqp_op(iwdev
->rf
, cqp_request
);
276 if (!status
&& cqp_request
->compl_info
.op_ret_val
<
277 iwdev
->rf
->sc_dev
.hw_attrs
.max_hw_device_pages
) {
278 qp
->push_idx
= cqp_request
->compl_info
.op_ret_val
;
282 irdma_put_cqp_request(&iwdev
->rf
->cqp
, cqp_request
);
286 * irdma_alloc_ucontext - Allocate the user context data structure
287 * @uctx: uverbs context pointer
290 * This keeps track of all objects associated with a particular
293 static int irdma_alloc_ucontext(struct ib_ucontext
*uctx
,
294 struct ib_udata
*udata
)
296 struct ib_device
*ibdev
= uctx
->device
;
297 struct irdma_device
*iwdev
= to_iwdev(ibdev
);
298 struct irdma_alloc_ucontext_req req
;
299 struct irdma_alloc_ucontext_resp uresp
= {};
300 struct irdma_ucontext
*ucontext
= to_ucontext(uctx
);
301 struct irdma_uk_attrs
*uk_attrs
;
303 if (ib_copy_from_udata(&req
, udata
, min(sizeof(req
), udata
->inlen
)))
306 if (req
.userspace_ver
< 4 || req
.userspace_ver
> IRDMA_ABI_VER
)
309 ucontext
->iwdev
= iwdev
;
310 ucontext
->abi_ver
= req
.userspace_ver
;
312 uk_attrs
= &iwdev
->rf
->sc_dev
.hw_attrs
.uk_attrs
;
313 /* GEN_1 legacy support with libi40iw */
314 if (udata
->outlen
< sizeof(uresp
)) {
315 if (uk_attrs
->hw_rev
!= IRDMA_GEN_1
)
318 ucontext
->legacy_mode
= true;
319 uresp
.max_qps
= iwdev
->rf
->max_qp
;
320 uresp
.max_pds
= iwdev
->rf
->sc_dev
.hw_attrs
.max_hw_pds
;
321 uresp
.wq_size
= iwdev
->rf
->sc_dev
.hw_attrs
.max_qp_wr
* 2;
322 uresp
.kernel_ver
= req
.userspace_ver
;
323 if (ib_copy_to_udata(udata
, &uresp
,
324 min(sizeof(uresp
), udata
->outlen
)))
327 u64 bar_off
= (uintptr_t)iwdev
->rf
->sc_dev
.hw_regs
[IRDMA_DB_ADDR_OFFSET
];
329 ucontext
->db_mmap_entry
=
330 irdma_user_mmap_entry_insert(ucontext
, bar_off
,
333 if (!ucontext
->db_mmap_entry
)
336 uresp
.kernel_ver
= IRDMA_ABI_VER
;
337 uresp
.feature_flags
= uk_attrs
->feature_flags
;
338 uresp
.max_hw_wq_frags
= uk_attrs
->max_hw_wq_frags
;
339 uresp
.max_hw_read_sges
= uk_attrs
->max_hw_read_sges
;
340 uresp
.max_hw_inline
= uk_attrs
->max_hw_inline
;
341 uresp
.max_hw_rq_quanta
= uk_attrs
->max_hw_rq_quanta
;
342 uresp
.max_hw_wq_quanta
= uk_attrs
->max_hw_wq_quanta
;
343 uresp
.max_hw_sq_chunk
= uk_attrs
->max_hw_sq_chunk
;
344 uresp
.max_hw_cq_size
= uk_attrs
->max_hw_cq_size
;
345 uresp
.min_hw_cq_size
= uk_attrs
->min_hw_cq_size
;
346 uresp
.hw_rev
= uk_attrs
->hw_rev
;
347 if (ib_copy_to_udata(udata
, &uresp
,
348 min(sizeof(uresp
), udata
->outlen
))) {
349 rdma_user_mmap_entry_remove(ucontext
->db_mmap_entry
);
354 INIT_LIST_HEAD(&ucontext
->cq_reg_mem_list
);
355 spin_lock_init(&ucontext
->cq_reg_mem_list_lock
);
356 INIT_LIST_HEAD(&ucontext
->qp_reg_mem_list
);
357 spin_lock_init(&ucontext
->qp_reg_mem_list_lock
);
362 ibdev_err(&iwdev
->ibdev
,
363 "Invalid userspace driver version detected. Detected version %d, should be %d\n",
364 req
.userspace_ver
, IRDMA_ABI_VER
);
369 * irdma_dealloc_ucontext - deallocate the user context data structure
370 * @context: user context created during alloc
372 static void irdma_dealloc_ucontext(struct ib_ucontext
*context
)
374 struct irdma_ucontext
*ucontext
= to_ucontext(context
);
376 rdma_user_mmap_entry_remove(ucontext
->db_mmap_entry
);
380 * irdma_alloc_pd - allocate protection domain
384 static int irdma_alloc_pd(struct ib_pd
*pd
, struct ib_udata
*udata
)
386 struct irdma_pd
*iwpd
= to_iwpd(pd
);
387 struct irdma_device
*iwdev
= to_iwdev(pd
->device
);
388 struct irdma_sc_dev
*dev
= &iwdev
->rf
->sc_dev
;
389 struct irdma_pci_f
*rf
= iwdev
->rf
;
390 struct irdma_alloc_pd_resp uresp
= {};
391 struct irdma_sc_pd
*sc_pd
;
395 err
= irdma_alloc_rsrc(rf
, rf
->allocated_pds
, rf
->max_pd
, &pd_id
,
400 sc_pd
= &iwpd
->sc_pd
;
402 struct irdma_ucontext
*ucontext
=
403 rdma_udata_to_drv_context(udata
, struct irdma_ucontext
,
405 irdma_sc_pd_init(dev
, sc_pd
, pd_id
, ucontext
->abi_ver
);
407 if (ib_copy_to_udata(udata
, &uresp
,
408 min(sizeof(uresp
), udata
->outlen
))) {
413 irdma_sc_pd_init(dev
, sc_pd
, pd_id
, IRDMA_ABI_VER
);
418 irdma_free_rsrc(rf
, rf
->allocated_pds
, pd_id
);
424 * irdma_dealloc_pd - deallocate pd
425 * @ibpd: ptr of pd to be deallocated
428 static int irdma_dealloc_pd(struct ib_pd
*ibpd
, struct ib_udata
*udata
)
430 struct irdma_pd
*iwpd
= to_iwpd(ibpd
);
431 struct irdma_device
*iwdev
= to_iwdev(ibpd
->device
);
433 irdma_free_rsrc(iwdev
->rf
, iwdev
->rf
->allocated_pds
, iwpd
->sc_pd
.pd_id
);
439 * irdma_get_pbl - Retrieve pbl from a list given a virtual
441 * @va: user virtual address
442 * @pbl_list: pbl list to search in (QP's or CQ's)
444 static struct irdma_pbl
*irdma_get_pbl(unsigned long va
,
445 struct list_head
*pbl_list
)
447 struct irdma_pbl
*iwpbl
;
449 list_for_each_entry (iwpbl
, pbl_list
, list
) {
450 if (iwpbl
->user_base
== va
) {
451 list_del(&iwpbl
->list
);
452 iwpbl
->on_list
= false;
461 * irdma_clean_cqes - clean cq entries for qp
462 * @iwqp: qp ptr (user or kernel)
465 static void irdma_clean_cqes(struct irdma_qp
*iwqp
, struct irdma_cq
*iwcq
)
467 struct irdma_cq_uk
*ukcq
= &iwcq
->sc_cq
.cq_uk
;
470 spin_lock_irqsave(&iwcq
->lock
, flags
);
471 irdma_uk_clean_cq(&iwqp
->sc_qp
.qp_uk
, ukcq
);
472 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
475 static void irdma_remove_push_mmap_entries(struct irdma_qp
*iwqp
)
477 if (iwqp
->push_db_mmap_entry
) {
478 rdma_user_mmap_entry_remove(iwqp
->push_db_mmap_entry
);
479 iwqp
->push_db_mmap_entry
= NULL
;
481 if (iwqp
->push_wqe_mmap_entry
) {
482 rdma_user_mmap_entry_remove(iwqp
->push_wqe_mmap_entry
);
483 iwqp
->push_wqe_mmap_entry
= NULL
;
487 static int irdma_setup_push_mmap_entries(struct irdma_ucontext
*ucontext
,
488 struct irdma_qp
*iwqp
,
489 u64
*push_wqe_mmap_key
,
490 u64
*push_db_mmap_key
)
492 struct irdma_device
*iwdev
= ucontext
->iwdev
;
495 rsvd
= IRDMA_PF_BAR_RSVD
;
496 bar_off
= (uintptr_t)iwdev
->rf
->sc_dev
.hw_regs
[IRDMA_DB_ADDR_OFFSET
];
497 /* skip over db page */
498 bar_off
+= IRDMA_HW_PAGE_SIZE
;
500 bar_off
+= rsvd
+ iwqp
->sc_qp
.push_idx
* IRDMA_HW_PAGE_SIZE
;
501 iwqp
->push_wqe_mmap_entry
= irdma_user_mmap_entry_insert(ucontext
,
502 bar_off
, IRDMA_MMAP_IO_WC
,
504 if (!iwqp
->push_wqe_mmap_entry
)
507 /* push doorbell page */
508 bar_off
+= IRDMA_HW_PAGE_SIZE
;
509 iwqp
->push_db_mmap_entry
= irdma_user_mmap_entry_insert(ucontext
,
510 bar_off
, IRDMA_MMAP_IO_NC
,
512 if (!iwqp
->push_db_mmap_entry
) {
513 rdma_user_mmap_entry_remove(iwqp
->push_wqe_mmap_entry
);
521 * irdma_destroy_qp - destroy qp
522 * @ibqp: qp's ib pointer also to get to device's qp address
525 static int irdma_destroy_qp(struct ib_qp
*ibqp
, struct ib_udata
*udata
)
527 struct irdma_qp
*iwqp
= to_iwqp(ibqp
);
528 struct irdma_device
*iwdev
= iwqp
->iwdev
;
530 iwqp
->sc_qp
.qp_uk
.destroy_pending
= true;
532 if (iwqp
->iwarp_state
== IRDMA_QP_STATE_RTS
)
533 irdma_modify_qp_to_err(&iwqp
->sc_qp
);
535 irdma_qp_rem_ref(&iwqp
->ibqp
);
536 wait_for_completion(&iwqp
->free_qp
);
537 irdma_free_lsmm_rsrc(iwqp
);
539 irdma_cqp_qp_destroy_cmd(&iwdev
->rf
->sc_dev
, &iwqp
->sc_qp
);
541 if (!iwqp
->user_mode
) {
543 irdma_clean_cqes(iwqp
, iwqp
->iwscq
);
544 if (iwqp
->iwrcq
!= iwqp
->iwscq
)
545 irdma_clean_cqes(iwqp
, iwqp
->iwrcq
);
548 irdma_remove_push_mmap_entries(iwqp
);
549 irdma_free_qp_rsrc(iwqp
);
555 * irdma_setup_virt_qp - setup for allocation of virtual qp
556 * @iwdev: irdma device
558 * @init_info: initialize info to return
560 static void irdma_setup_virt_qp(struct irdma_device
*iwdev
,
561 struct irdma_qp
*iwqp
,
562 struct irdma_qp_init_info
*init_info
)
564 struct irdma_pbl
*iwpbl
= iwqp
->iwpbl
;
565 struct irdma_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
567 iwqp
->page
= qpmr
->sq_page
;
568 init_info
->shadow_area_pa
= qpmr
->shadow
;
569 if (iwpbl
->pbl_allocated
) {
570 init_info
->virtual_map
= true;
571 init_info
->sq_pa
= qpmr
->sq_pbl
.idx
;
572 init_info
->rq_pa
= qpmr
->rq_pbl
.idx
;
574 init_info
->sq_pa
= qpmr
->sq_pbl
.addr
;
575 init_info
->rq_pa
= qpmr
->rq_pbl
.addr
;
580 * irdma_setup_kmode_qp - setup initialization for kernel mode qp
581 * @iwdev: iwarp device
582 * @iwqp: qp ptr (user or kernel)
583 * @info: initialize info to return
584 * @init_attr: Initial QP create attributes
586 static int irdma_setup_kmode_qp(struct irdma_device
*iwdev
,
587 struct irdma_qp
*iwqp
,
588 struct irdma_qp_init_info
*info
,
589 struct ib_qp_init_attr
*init_attr
)
591 struct irdma_dma_mem
*mem
= &iwqp
->kqp
.dma_mem
;
592 u32 sqdepth
, rqdepth
;
595 enum irdma_status_code status
;
596 struct irdma_qp_uk_init_info
*ukinfo
= &info
->qp_uk_init_info
;
597 struct irdma_uk_attrs
*uk_attrs
= &iwdev
->rf
->sc_dev
.hw_attrs
.uk_attrs
;
599 irdma_get_wqe_shift(uk_attrs
,
600 uk_attrs
->hw_rev
>= IRDMA_GEN_2
? ukinfo
->max_sq_frag_cnt
+ 1 :
601 ukinfo
->max_sq_frag_cnt
,
602 ukinfo
->max_inline_data
, &sqshift
);
603 status
= irdma_get_sqdepth(uk_attrs
, ukinfo
->sq_size
, sqshift
,
608 if (uk_attrs
->hw_rev
== IRDMA_GEN_1
)
609 rqshift
= IRDMA_MAX_RQ_WQE_SHIFT_GEN1
;
611 irdma_get_wqe_shift(uk_attrs
, ukinfo
->max_rq_frag_cnt
, 0,
614 status
= irdma_get_rqdepth(uk_attrs
, ukinfo
->rq_size
, rqshift
,
619 iwqp
->kqp
.sq_wrid_mem
=
620 kcalloc(sqdepth
, sizeof(*iwqp
->kqp
.sq_wrid_mem
), GFP_KERNEL
);
621 if (!iwqp
->kqp
.sq_wrid_mem
)
624 iwqp
->kqp
.rq_wrid_mem
=
625 kcalloc(rqdepth
, sizeof(*iwqp
->kqp
.rq_wrid_mem
), GFP_KERNEL
);
626 if (!iwqp
->kqp
.rq_wrid_mem
) {
627 kfree(iwqp
->kqp
.sq_wrid_mem
);
628 iwqp
->kqp
.sq_wrid_mem
= NULL
;
632 ukinfo
->sq_wrtrk_array
= iwqp
->kqp
.sq_wrid_mem
;
633 ukinfo
->rq_wrid_array
= iwqp
->kqp
.rq_wrid_mem
;
635 size
= (sqdepth
+ rqdepth
) * IRDMA_QP_WQE_MIN_SIZE
;
636 size
+= (IRDMA_SHADOW_AREA_SIZE
<< 3);
638 mem
->size
= ALIGN(size
, 256);
639 mem
->va
= dma_alloc_coherent(iwdev
->rf
->hw
.device
, mem
->size
,
640 &mem
->pa
, GFP_KERNEL
);
642 kfree(iwqp
->kqp
.sq_wrid_mem
);
643 iwqp
->kqp
.sq_wrid_mem
= NULL
;
644 kfree(iwqp
->kqp
.rq_wrid_mem
);
645 iwqp
->kqp
.rq_wrid_mem
= NULL
;
649 ukinfo
->sq
= mem
->va
;
650 info
->sq_pa
= mem
->pa
;
651 ukinfo
->rq
= &ukinfo
->sq
[sqdepth
];
652 info
->rq_pa
= info
->sq_pa
+ (sqdepth
* IRDMA_QP_WQE_MIN_SIZE
);
653 ukinfo
->shadow_area
= ukinfo
->rq
[rqdepth
].elem
;
654 info
->shadow_area_pa
= info
->rq_pa
+ (rqdepth
* IRDMA_QP_WQE_MIN_SIZE
);
655 ukinfo
->sq_size
= sqdepth
>> sqshift
;
656 ukinfo
->rq_size
= rqdepth
>> rqshift
;
657 ukinfo
->qp_id
= iwqp
->ibqp
.qp_num
;
659 init_attr
->cap
.max_send_wr
= (sqdepth
- IRDMA_SQ_RSVD
) >> sqshift
;
660 init_attr
->cap
.max_recv_wr
= (rqdepth
- IRDMA_RQ_RSVD
) >> rqshift
;
665 static int irdma_cqp_create_qp_cmd(struct irdma_qp
*iwqp
)
667 struct irdma_pci_f
*rf
= iwqp
->iwdev
->rf
;
668 struct irdma_cqp_request
*cqp_request
;
669 struct cqp_cmds_info
*cqp_info
;
670 struct irdma_create_qp_info
*qp_info
;
671 enum irdma_status_code status
;
673 cqp_request
= irdma_alloc_and_get_cqp_request(&rf
->cqp
, true);
677 cqp_info
= &cqp_request
->info
;
678 qp_info
= &cqp_request
->info
.in
.u
.qp_create
.info
;
679 memset(qp_info
, 0, sizeof(*qp_info
));
680 qp_info
->mac_valid
= true;
681 qp_info
->cq_num_valid
= true;
682 qp_info
->next_iwarp_state
= IRDMA_QP_STATE_IDLE
;
684 cqp_info
->cqp_cmd
= IRDMA_OP_QP_CREATE
;
685 cqp_info
->post_sq
= 1;
686 cqp_info
->in
.u
.qp_create
.qp
= &iwqp
->sc_qp
;
687 cqp_info
->in
.u
.qp_create
.scratch
= (uintptr_t)cqp_request
;
688 status
= irdma_handle_cqp_op(rf
, cqp_request
);
689 irdma_put_cqp_request(&rf
->cqp
, cqp_request
);
691 return status
? -ENOMEM
: 0;
694 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp
*iwqp
,
695 struct irdma_qp_host_ctx_info
*ctx_info
)
697 struct irdma_device
*iwdev
= iwqp
->iwdev
;
698 struct irdma_sc_dev
*dev
= &iwdev
->rf
->sc_dev
;
699 struct irdma_roce_offload_info
*roce_info
;
700 struct irdma_udp_offload_info
*udp_info
;
702 udp_info
= &iwqp
->udp_info
;
703 udp_info
->snd_mss
= ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev
->vsi
.mtu
));
704 udp_info
->cwnd
= iwdev
->roce_cwnd
;
705 udp_info
->rexmit_thresh
= 2;
706 udp_info
->rnr_nak_thresh
= 2;
707 udp_info
->src_port
= 0xc000;
708 udp_info
->dst_port
= ROCE_V2_UDP_DPORT
;
709 roce_info
= &iwqp
->roce_info
;
710 ether_addr_copy(roce_info
->mac_addr
, iwdev
->netdev
->dev_addr
);
712 roce_info
->rd_en
= true;
713 roce_info
->wr_rdresp_en
= true;
714 roce_info
->bind_en
= true;
715 roce_info
->dcqcn_en
= false;
716 roce_info
->rtomin
= 5;
718 roce_info
->ack_credits
= iwdev
->roce_ackcreds
;
719 roce_info
->ird_size
= dev
->hw_attrs
.max_hw_ird
;
720 roce_info
->ord_size
= dev
->hw_attrs
.max_hw_ord
;
722 if (!iwqp
->user_mode
) {
723 roce_info
->priv_mode_en
= true;
724 roce_info
->fast_reg_en
= true;
725 roce_info
->udprivcq_en
= true;
727 roce_info
->roce_tver
= 0;
729 ctx_info
->roce_info
= &iwqp
->roce_info
;
730 ctx_info
->udp_info
= &iwqp
->udp_info
;
731 irdma_sc_qp_setctx_roce(&iwqp
->sc_qp
, iwqp
->host_ctx
.va
, ctx_info
);
734 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp
*iwqp
,
735 struct irdma_qp_host_ctx_info
*ctx_info
)
737 struct irdma_device
*iwdev
= iwqp
->iwdev
;
738 struct irdma_sc_dev
*dev
= &iwdev
->rf
->sc_dev
;
739 struct irdma_iwarp_offload_info
*iwarp_info
;
741 iwarp_info
= &iwqp
->iwarp_info
;
742 ether_addr_copy(iwarp_info
->mac_addr
, iwdev
->netdev
->dev_addr
);
743 iwarp_info
->rd_en
= true;
744 iwarp_info
->wr_rdresp_en
= true;
745 iwarp_info
->bind_en
= true;
746 iwarp_info
->ecn_en
= true;
747 iwarp_info
->rtomin
= 5;
749 if (dev
->hw_attrs
.uk_attrs
.hw_rev
>= IRDMA_GEN_2
)
750 iwarp_info
->ib_rd_en
= true;
751 if (!iwqp
->user_mode
) {
752 iwarp_info
->priv_mode_en
= true;
753 iwarp_info
->fast_reg_en
= true;
755 iwarp_info
->ddp_ver
= 1;
756 iwarp_info
->rdmap_ver
= 1;
758 ctx_info
->iwarp_info
= &iwqp
->iwarp_info
;
759 ctx_info
->iwarp_info_valid
= true;
760 irdma_sc_qp_setctx(&iwqp
->sc_qp
, iwqp
->host_ctx
.va
, ctx_info
);
761 ctx_info
->iwarp_info_valid
= false;
764 static int irdma_validate_qp_attrs(struct ib_qp_init_attr
*init_attr
,
765 struct irdma_device
*iwdev
)
767 struct irdma_sc_dev
*dev
= &iwdev
->rf
->sc_dev
;
768 struct irdma_uk_attrs
*uk_attrs
= &dev
->hw_attrs
.uk_attrs
;
770 if (init_attr
->create_flags
)
773 if (init_attr
->cap
.max_inline_data
> uk_attrs
->max_hw_inline
||
774 init_attr
->cap
.max_send_sge
> uk_attrs
->max_hw_wq_frags
||
775 init_attr
->cap
.max_recv_sge
> uk_attrs
->max_hw_wq_frags
)
778 if (rdma_protocol_roce(&iwdev
->ibdev
, 1)) {
779 if (init_attr
->qp_type
!= IB_QPT_RC
&&
780 init_attr
->qp_type
!= IB_QPT_UD
&&
781 init_attr
->qp_type
!= IB_QPT_GSI
)
784 if (init_attr
->qp_type
!= IB_QPT_RC
)
792 * irdma_create_qp - create qp
794 * @init_attr: attributes for qp
795 * @udata: user data for create qp
797 static int irdma_create_qp(struct ib_qp
*ibqp
,
798 struct ib_qp_init_attr
*init_attr
,
799 struct ib_udata
*udata
)
801 struct ib_pd
*ibpd
= ibqp
->pd
;
802 struct irdma_pd
*iwpd
= to_iwpd(ibpd
);
803 struct irdma_device
*iwdev
= to_iwdev(ibpd
->device
);
804 struct irdma_pci_f
*rf
= iwdev
->rf
;
805 struct irdma_qp
*iwqp
= to_iwqp(ibqp
);
806 struct irdma_create_qp_req req
;
807 struct irdma_create_qp_resp uresp
= {};
809 enum irdma_status_code ret
;
813 struct irdma_sc_qp
*qp
;
814 struct irdma_sc_dev
*dev
= &rf
->sc_dev
;
815 struct irdma_uk_attrs
*uk_attrs
= &dev
->hw_attrs
.uk_attrs
;
816 struct irdma_qp_init_info init_info
= {};
817 struct irdma_qp_host_ctx_info
*ctx_info
;
820 err_code
= irdma_validate_qp_attrs(init_attr
, iwdev
);
824 sq_size
= init_attr
->cap
.max_send_wr
;
825 rq_size
= init_attr
->cap
.max_recv_wr
;
827 init_info
.vsi
= &iwdev
->vsi
;
828 init_info
.qp_uk_init_info
.uk_attrs
= uk_attrs
;
829 init_info
.qp_uk_init_info
.sq_size
= sq_size
;
830 init_info
.qp_uk_init_info
.rq_size
= rq_size
;
831 init_info
.qp_uk_init_info
.max_sq_frag_cnt
= init_attr
->cap
.max_send_sge
;
832 init_info
.qp_uk_init_info
.max_rq_frag_cnt
= init_attr
->cap
.max_recv_sge
;
833 init_info
.qp_uk_init_info
.max_inline_data
= init_attr
->cap
.max_inline_data
;
836 qp
->qp_uk
.back_qp
= iwqp
;
837 qp
->qp_uk
.lock
= &iwqp
->lock
;
838 qp
->push_idx
= IRDMA_INVALID_PUSH_PAGE_INDEX
;
841 iwqp
->q2_ctx_mem
.size
= ALIGN(IRDMA_Q2_BUF_SIZE
+ IRDMA_QP_CTX_SIZE
,
843 iwqp
->q2_ctx_mem
.va
= dma_alloc_coherent(dev
->hw
->device
,
844 iwqp
->q2_ctx_mem
.size
,
845 &iwqp
->q2_ctx_mem
.pa
,
847 if (!iwqp
->q2_ctx_mem
.va
)
850 init_info
.q2
= iwqp
->q2_ctx_mem
.va
;
851 init_info
.q2_pa
= iwqp
->q2_ctx_mem
.pa
;
852 init_info
.host_ctx
= (__le64
*)(init_info
.q2
+ IRDMA_Q2_BUF_SIZE
);
853 init_info
.host_ctx_pa
= init_info
.q2_pa
+ IRDMA_Q2_BUF_SIZE
;
855 if (init_attr
->qp_type
== IB_QPT_GSI
)
858 err_code
= irdma_alloc_rsrc(rf
, rf
->allocated_qps
, rf
->max_qp
,
859 &qp_num
, &rf
->next_qp
);
864 iwqp
->ibqp
.qp_num
= qp_num
;
866 iwqp
->iwscq
= to_iwcq(init_attr
->send_cq
);
867 iwqp
->iwrcq
= to_iwcq(init_attr
->recv_cq
);
868 iwqp
->host_ctx
.va
= init_info
.host_ctx
;
869 iwqp
->host_ctx
.pa
= init_info
.host_ctx_pa
;
870 iwqp
->host_ctx
.size
= IRDMA_QP_CTX_SIZE
;
872 init_info
.pd
= &iwpd
->sc_pd
;
873 init_info
.qp_uk_init_info
.qp_id
= iwqp
->ibqp
.qp_num
;
874 if (!rdma_protocol_roce(&iwdev
->ibdev
, 1))
875 init_info
.qp_uk_init_info
.first_sq_wq
= 1;
876 iwqp
->ctx_info
.qp_compl_ctx
= (uintptr_t)qp
;
877 init_waitqueue_head(&iwqp
->waitq
);
878 init_waitqueue_head(&iwqp
->mod_qp_waitq
);
881 err_code
= ib_copy_from_udata(&req
, udata
,
882 min(sizeof(req
), udata
->inlen
));
884 ibdev_dbg(&iwdev
->ibdev
,
885 "VERBS: ib_copy_from_data fail\n");
889 iwqp
->ctx_info
.qp_compl_ctx
= req
.user_compl_ctx
;
891 if (req
.user_wqe_bufs
) {
892 struct irdma_ucontext
*ucontext
=
893 rdma_udata_to_drv_context(udata
,
894 struct irdma_ucontext
,
897 init_info
.qp_uk_init_info
.legacy_mode
= ucontext
->legacy_mode
;
898 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
899 iwqp
->iwpbl
= irdma_get_pbl((unsigned long)req
.user_wqe_bufs
,
900 &ucontext
->qp_reg_mem_list
);
901 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
905 ibdev_dbg(&iwdev
->ibdev
, "VERBS: no pbl info\n");
909 init_info
.qp_uk_init_info
.abi_ver
= iwpd
->sc_pd
.abi_ver
;
910 irdma_setup_virt_qp(iwdev
, iwqp
, &init_info
);
912 init_info
.qp_uk_init_info
.abi_ver
= IRDMA_ABI_VER
;
913 err_code
= irdma_setup_kmode_qp(iwdev
, iwqp
, &init_info
, init_attr
);
917 ibdev_dbg(&iwdev
->ibdev
, "VERBS: setup qp failed\n");
921 if (rdma_protocol_roce(&iwdev
->ibdev
, 1)) {
922 if (init_attr
->qp_type
== IB_QPT_RC
) {
923 init_info
.qp_uk_init_info
.type
= IRDMA_QP_TYPE_ROCE_RC
;
924 init_info
.qp_uk_init_info
.qp_caps
= IRDMA_SEND_WITH_IMM
|
925 IRDMA_WRITE_WITH_IMM
|
928 init_info
.qp_uk_init_info
.type
= IRDMA_QP_TYPE_ROCE_UD
;
929 init_info
.qp_uk_init_info
.qp_caps
= IRDMA_SEND_WITH_IMM
|
933 init_info
.qp_uk_init_info
.type
= IRDMA_QP_TYPE_IWARP
;
934 init_info
.qp_uk_init_info
.qp_caps
= IRDMA_WRITE_WITH_IMM
;
937 if (dev
->hw_attrs
.uk_attrs
.hw_rev
> IRDMA_GEN_1
)
938 init_info
.qp_uk_init_info
.qp_caps
|= IRDMA_PUSH_MODE
;
940 ret
= irdma_sc_qp_init(qp
, &init_info
);
943 ibdev_dbg(&iwdev
->ibdev
, "VERBS: qp_init fail\n");
947 ctx_info
= &iwqp
->ctx_info
;
948 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
949 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
951 if (rdma_protocol_roce(&iwdev
->ibdev
, 1))
952 irdma_roce_fill_and_set_qpctx_info(iwqp
, ctx_info
);
954 irdma_iw_fill_and_set_qpctx_info(iwqp
, ctx_info
);
956 err_code
= irdma_cqp_create_qp_cmd(iwqp
);
960 refcount_set(&iwqp
->refcnt
, 1);
961 spin_lock_init(&iwqp
->lock
);
962 spin_lock_init(&iwqp
->sc_qp
.pfpdu
.lock
);
963 iwqp
->sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
) ? 1 : 0;
964 rf
->qp_table
[qp_num
] = iwqp
;
965 iwqp
->max_send_wr
= sq_size
;
966 iwqp
->max_recv_wr
= rq_size
;
968 if (rdma_protocol_roce(&iwdev
->ibdev
, 1)) {
969 if (dev
->ws_add(&iwdev
->vsi
, 0)) {
970 irdma_cqp_qp_destroy_cmd(&rf
->sc_dev
, &iwqp
->sc_qp
);
975 irdma_qp_add_qos(&iwqp
->sc_qp
);
979 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */
980 if (udata
->outlen
< sizeof(uresp
)) {
982 uresp
.push_idx
= IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1
;
984 if (rdma_protocol_iwarp(&iwdev
->ibdev
, 1))
987 uresp
.actual_sq_size
= sq_size
;
988 uresp
.actual_rq_size
= rq_size
;
989 uresp
.qp_id
= qp_num
;
990 uresp
.qp_caps
= qp
->qp_uk
.qp_caps
;
992 err_code
= ib_copy_to_udata(udata
, &uresp
,
993 min(sizeof(uresp
), udata
->outlen
));
995 ibdev_dbg(&iwdev
->ibdev
, "VERBS: copy_to_udata failed\n");
996 irdma_destroy_qp(&iwqp
->ibqp
, udata
);
1001 init_completion(&iwqp
->free_qp
);
1005 irdma_free_qp_rsrc(iwqp
);
1009 static int irdma_get_ib_acc_flags(struct irdma_qp
*iwqp
)
1013 if (rdma_protocol_roce(iwqp
->ibqp
.device
, 1)) {
1014 if (iwqp
->roce_info
.wr_rdresp_en
) {
1015 acc_flags
|= IB_ACCESS_LOCAL_WRITE
;
1016 acc_flags
|= IB_ACCESS_REMOTE_WRITE
;
1018 if (iwqp
->roce_info
.rd_en
)
1019 acc_flags
|= IB_ACCESS_REMOTE_READ
;
1020 if (iwqp
->roce_info
.bind_en
)
1021 acc_flags
|= IB_ACCESS_MW_BIND
;
1023 if (iwqp
->iwarp_info
.wr_rdresp_en
) {
1024 acc_flags
|= IB_ACCESS_LOCAL_WRITE
;
1025 acc_flags
|= IB_ACCESS_REMOTE_WRITE
;
1027 if (iwqp
->iwarp_info
.rd_en
)
1028 acc_flags
|= IB_ACCESS_REMOTE_READ
;
1029 if (iwqp
->iwarp_info
.bind_en
)
1030 acc_flags
|= IB_ACCESS_MW_BIND
;
1036 * irdma_query_qp - query qp attributes
1038 * @attr: attributes pointer
1039 * @attr_mask: Not used
1040 * @init_attr: qp attributes to return
1042 static int irdma_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
1043 int attr_mask
, struct ib_qp_init_attr
*init_attr
)
1045 struct irdma_qp
*iwqp
= to_iwqp(ibqp
);
1046 struct irdma_sc_qp
*qp
= &iwqp
->sc_qp
;
1048 memset(attr
, 0, sizeof(*attr
));
1049 memset(init_attr
, 0, sizeof(*init_attr
));
1051 attr
->qp_state
= iwqp
->ibqp_state
;
1052 attr
->cur_qp_state
= iwqp
->ibqp_state
;
1053 attr
->cap
.max_send_wr
= iwqp
->max_send_wr
;
1054 attr
->cap
.max_recv_wr
= iwqp
->max_recv_wr
;
1055 attr
->cap
.max_inline_data
= qp
->qp_uk
.max_inline_data
;
1056 attr
->cap
.max_send_sge
= qp
->qp_uk
.max_sq_frag_cnt
;
1057 attr
->cap
.max_recv_sge
= qp
->qp_uk
.max_rq_frag_cnt
;
1058 attr
->qp_access_flags
= irdma_get_ib_acc_flags(iwqp
);
1060 if (rdma_protocol_roce(ibqp
->device
, 1)) {
1061 attr
->path_mtu
= ib_mtu_int_to_enum(iwqp
->udp_info
.snd_mss
);
1062 attr
->qkey
= iwqp
->roce_info
.qkey
;
1063 attr
->rq_psn
= iwqp
->udp_info
.epsn
;
1064 attr
->sq_psn
= iwqp
->udp_info
.psn_nxt
;
1065 attr
->dest_qp_num
= iwqp
->roce_info
.dest_qp
;
1066 attr
->pkey_index
= iwqp
->roce_info
.p_key
;
1067 attr
->retry_cnt
= iwqp
->udp_info
.rexmit_thresh
;
1068 attr
->rnr_retry
= iwqp
->udp_info
.rnr_nak_thresh
;
1069 attr
->max_rd_atomic
= iwqp
->roce_info
.ord_size
;
1070 attr
->max_dest_rd_atomic
= iwqp
->roce_info
.ird_size
;
1073 init_attr
->event_handler
= iwqp
->ibqp
.event_handler
;
1074 init_attr
->qp_context
= iwqp
->ibqp
.qp_context
;
1075 init_attr
->send_cq
= iwqp
->ibqp
.send_cq
;
1076 init_attr
->recv_cq
= iwqp
->ibqp
.recv_cq
;
1077 init_attr
->cap
= attr
->cap
;
1083 * irdma_query_pkey - Query partition key
1084 * @ibdev: device pointer from stack
1085 * @port: port number
1086 * @index: index of pkey
1087 * @pkey: pointer to store the pkey
1089 static int irdma_query_pkey(struct ib_device
*ibdev
, u32 port
, u16 index
,
1092 if (index
>= IRDMA_PKEY_TBL_SZ
)
1095 *pkey
= IRDMA_DEFAULT_PKEY
;
1100 * irdma_modify_qp_roce - modify qp request
1101 * @ibqp: qp's pointer for modify
1102 * @attr: access attributes
1103 * @attr_mask: state mask
1106 int irdma_modify_qp_roce(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
1107 int attr_mask
, struct ib_udata
*udata
)
1109 struct irdma_pd
*iwpd
= to_iwpd(ibqp
->pd
);
1110 struct irdma_qp
*iwqp
= to_iwqp(ibqp
);
1111 struct irdma_device
*iwdev
= iwqp
->iwdev
;
1112 struct irdma_sc_dev
*dev
= &iwdev
->rf
->sc_dev
;
1113 struct irdma_qp_host_ctx_info
*ctx_info
;
1114 struct irdma_roce_offload_info
*roce_info
;
1115 struct irdma_udp_offload_info
*udp_info
;
1116 struct irdma_modify_qp_info info
= {};
1117 struct irdma_modify_qp_resp uresp
= {};
1118 struct irdma_modify_qp_req ureq
= {};
1119 unsigned long flags
;
1120 u8 issue_modify_qp
= 0;
1123 ctx_info
= &iwqp
->ctx_info
;
1124 roce_info
= &iwqp
->roce_info
;
1125 udp_info
= &iwqp
->udp_info
;
1127 if (attr_mask
& ~IB_QP_ATTR_STANDARD_BITS
)
1130 if (attr_mask
& IB_QP_DEST_QPN
)
1131 roce_info
->dest_qp
= attr
->dest_qp_num
;
1133 if (attr_mask
& IB_QP_PKEY_INDEX
) {
1134 ret
= irdma_query_pkey(ibqp
->device
, 0, attr
->pkey_index
,
1140 if (attr_mask
& IB_QP_QKEY
)
1141 roce_info
->qkey
= attr
->qkey
;
1143 if (attr_mask
& IB_QP_PATH_MTU
)
1144 udp_info
->snd_mss
= ib_mtu_enum_to_int(attr
->path_mtu
);
1146 if (attr_mask
& IB_QP_SQ_PSN
) {
1147 udp_info
->psn_nxt
= attr
->sq_psn
;
1148 udp_info
->lsn
= 0xffff;
1149 udp_info
->psn_una
= attr
->sq_psn
;
1150 udp_info
->psn_max
= attr
->sq_psn
;
1153 if (attr_mask
& IB_QP_RQ_PSN
)
1154 udp_info
->epsn
= attr
->rq_psn
;
1156 if (attr_mask
& IB_QP_RNR_RETRY
)
1157 udp_info
->rnr_nak_thresh
= attr
->rnr_retry
;
1159 if (attr_mask
& IB_QP_RETRY_CNT
)
1160 udp_info
->rexmit_thresh
= attr
->retry_cnt
;
1162 ctx_info
->roce_info
->pd_id
= iwpd
->sc_pd
.pd_id
;
1164 if (attr_mask
& IB_QP_AV
) {
1165 struct irdma_av
*av
= &iwqp
->roce_ah
.av
;
1166 const struct ib_gid_attr
*sgid_attr
;
1167 u16 vlan_id
= VLAN_N_VID
;
1170 memset(&iwqp
->roce_ah
, 0, sizeof(iwqp
->roce_ah
));
1171 if (attr
->ah_attr
.ah_flags
& IB_AH_GRH
) {
1172 udp_info
->ttl
= attr
->ah_attr
.grh
.hop_limit
;
1173 udp_info
->flow_label
= attr
->ah_attr
.grh
.flow_label
;
1174 udp_info
->tos
= attr
->ah_attr
.grh
.traffic_class
;
1175 irdma_qp_rem_qos(&iwqp
->sc_qp
);
1176 dev
->ws_remove(iwqp
->sc_qp
.vsi
, ctx_info
->user_pri
);
1177 ctx_info
->user_pri
= rt_tos2priority(udp_info
->tos
);
1178 iwqp
->sc_qp
.user_pri
= ctx_info
->user_pri
;
1179 if (dev
->ws_add(iwqp
->sc_qp
.vsi
, ctx_info
->user_pri
))
1181 irdma_qp_add_qos(&iwqp
->sc_qp
);
1183 sgid_attr
= attr
->ah_attr
.grh
.sgid_attr
;
1184 ret
= rdma_read_gid_l2_fields(sgid_attr
, &vlan_id
,
1185 ctx_info
->roce_info
->mac_addr
);
1189 if (vlan_id
>= VLAN_N_VID
&& iwdev
->dcb
)
1191 if (vlan_id
< VLAN_N_VID
) {
1192 udp_info
->insert_vlan_tag
= true;
1193 udp_info
->vlan_tag
= vlan_id
|
1194 ctx_info
->user_pri
<< VLAN_PRIO_SHIFT
;
1196 udp_info
->insert_vlan_tag
= false;
1199 av
->attrs
= attr
->ah_attr
;
1200 rdma_gid2ip((struct sockaddr
*)&av
->sgid_addr
, &sgid_attr
->gid
);
1201 rdma_gid2ip((struct sockaddr
*)&av
->dgid_addr
, &attr
->ah_attr
.grh
.dgid
);
1202 roce_info
->local_qp
= ibqp
->qp_num
;
1203 if (av
->sgid_addr
.saddr
.sa_family
== AF_INET6
) {
1205 av
->dgid_addr
.saddr_in6
.sin6_addr
.in6_u
.u6_addr32
;
1207 av
->sgid_addr
.saddr_in6
.sin6_addr
.in6_u
.u6_addr32
;
1209 irdma_copy_ip_ntohl(&udp_info
->dest_ip_addr
[0], daddr
);
1210 irdma_copy_ip_ntohl(&udp_info
->local_ipaddr
[0], saddr
);
1212 udp_info
->ipv4
= false;
1213 irdma_copy_ip_ntohl(local_ip
, daddr
);
1215 udp_info
->arp_idx
= irdma_arp_table(iwdev
->rf
,
1220 __be32 saddr
= av
->sgid_addr
.saddr_in
.sin_addr
.s_addr
;
1221 __be32 daddr
= av
->dgid_addr
.saddr_in
.sin_addr
.s_addr
;
1223 local_ip
[0] = ntohl(daddr
);
1225 udp_info
->ipv4
= true;
1226 udp_info
->dest_ip_addr
[0] = 0;
1227 udp_info
->dest_ip_addr
[1] = 0;
1228 udp_info
->dest_ip_addr
[2] = 0;
1229 udp_info
->dest_ip_addr
[3] = local_ip
[0];
1231 udp_info
->local_ipaddr
[0] = 0;
1232 udp_info
->local_ipaddr
[1] = 0;
1233 udp_info
->local_ipaddr
[2] = 0;
1234 udp_info
->local_ipaddr
[3] = ntohl(saddr
);
1237 irdma_add_arp(iwdev
->rf
, local_ip
, udp_info
->ipv4
,
1238 attr
->ah_attr
.roce
.dmac
);
1241 if (attr_mask
& IB_QP_MAX_QP_RD_ATOMIC
) {
1242 if (attr
->max_rd_atomic
> dev
->hw_attrs
.max_hw_ord
) {
1243 ibdev_err(&iwdev
->ibdev
,
1244 "rd_atomic = %d, above max_hw_ord=%d\n",
1245 attr
->max_rd_atomic
,
1246 dev
->hw_attrs
.max_hw_ord
);
1249 if (attr
->max_rd_atomic
)
1250 roce_info
->ord_size
= attr
->max_rd_atomic
;
1251 info
.ord_valid
= true;
1254 if (attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
) {
1255 if (attr
->max_dest_rd_atomic
> dev
->hw_attrs
.max_hw_ird
) {
1256 ibdev_err(&iwdev
->ibdev
,
1257 "rd_atomic = %d, above max_hw_ird=%d\n",
1258 attr
->max_rd_atomic
,
1259 dev
->hw_attrs
.max_hw_ird
);
1262 if (attr
->max_dest_rd_atomic
)
1263 roce_info
->ird_size
= attr
->max_dest_rd_atomic
;
1266 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
1267 if (attr
->qp_access_flags
& IB_ACCESS_LOCAL_WRITE
)
1268 roce_info
->wr_rdresp_en
= true;
1269 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
)
1270 roce_info
->wr_rdresp_en
= true;
1271 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
)
1272 roce_info
->rd_en
= true;
1275 wait_event(iwqp
->mod_qp_waitq
, !atomic_read(&iwqp
->hw_mod_qp_pend
));
1277 ibdev_dbg(&iwdev
->ibdev
,
1278 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n",
1279 __builtin_return_address(0), ibqp
->qp_num
, attr
->qp_state
,
1280 iwqp
->ibqp_state
, iwqp
->iwarp_state
, attr_mask
);
1282 spin_lock_irqsave(&iwqp
->lock
, flags
);
1283 if (attr_mask
& IB_QP_STATE
) {
1284 if (!ib_modify_qp_is_ok(iwqp
->ibqp_state
, attr
->qp_state
,
1285 iwqp
->ibqp
.qp_type
, attr_mask
)) {
1286 ibdev_warn(&iwdev
->ibdev
, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n",
1287 iwqp
->ibqp
.qp_num
, iwqp
->ibqp_state
,
1292 info
.curr_iwarp_state
= iwqp
->iwarp_state
;
1294 switch (attr
->qp_state
) {
1296 if (iwqp
->iwarp_state
> IRDMA_QP_STATE_IDLE
) {
1301 if (iwqp
->iwarp_state
== IRDMA_QP_STATE_INVALID
) {
1302 info
.next_iwarp_state
= IRDMA_QP_STATE_IDLE
;
1303 issue_modify_qp
= 1;
1307 if (iwqp
->iwarp_state
> IRDMA_QP_STATE_IDLE
) {
1311 info
.arp_cache_idx_valid
= true;
1312 info
.cq_num_valid
= true;
1313 info
.next_iwarp_state
= IRDMA_QP_STATE_RTR
;
1314 issue_modify_qp
= 1;
1317 if (iwqp
->ibqp_state
< IB_QPS_RTR
||
1318 iwqp
->ibqp_state
== IB_QPS_ERR
) {
1323 info
.arp_cache_idx_valid
= true;
1324 info
.cq_num_valid
= true;
1325 info
.ord_valid
= true;
1326 info
.next_iwarp_state
= IRDMA_QP_STATE_RTS
;
1327 issue_modify_qp
= 1;
1328 if (iwdev
->push_mode
&& udata
&&
1329 iwqp
->sc_qp
.push_idx
== IRDMA_INVALID_PUSH_PAGE_INDEX
&&
1330 dev
->hw_attrs
.uk_attrs
.hw_rev
>= IRDMA_GEN_2
) {
1331 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1332 irdma_alloc_push_page(iwqp
);
1333 spin_lock_irqsave(&iwqp
->lock
, flags
);
1337 if (iwqp
->iwarp_state
== IRDMA_QP_STATE_SQD
)
1340 if (iwqp
->iwarp_state
!= IRDMA_QP_STATE_RTS
) {
1345 info
.next_iwarp_state
= IRDMA_QP_STATE_SQD
;
1346 issue_modify_qp
= 1;
1351 if (iwqp
->iwarp_state
== IRDMA_QP_STATE_RTS
) {
1352 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1353 info
.next_iwarp_state
= IRDMA_QP_STATE_SQD
;
1354 irdma_hw_modify_qp(iwdev
, iwqp
, &info
, true);
1355 spin_lock_irqsave(&iwqp
->lock
, flags
);
1358 if (iwqp
->iwarp_state
== IRDMA_QP_STATE_ERROR
) {
1359 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1361 if (ib_copy_from_udata(&ureq
, udata
,
1362 min(sizeof(ureq
), udata
->inlen
)))
1365 irdma_flush_wqes(iwqp
,
1366 (ureq
.sq_flush
? IRDMA_FLUSH_SQ
: 0) |
1367 (ureq
.rq_flush
? IRDMA_FLUSH_RQ
: 0) |
1373 info
.next_iwarp_state
= IRDMA_QP_STATE_ERROR
;
1374 issue_modify_qp
= 1;
1381 iwqp
->ibqp_state
= attr
->qp_state
;
1384 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
1385 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
1386 irdma_sc_qp_setctx_roce(&iwqp
->sc_qp
, iwqp
->host_ctx
.va
, ctx_info
);
1387 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1389 if (attr_mask
& IB_QP_STATE
) {
1390 if (issue_modify_qp
) {
1391 ctx_info
->rem_endpoint_idx
= udp_info
->arp_idx
;
1392 if (irdma_hw_modify_qp(iwdev
, iwqp
, &info
, true))
1394 spin_lock_irqsave(&iwqp
->lock
, flags
);
1395 if (iwqp
->iwarp_state
== info
.curr_iwarp_state
) {
1396 iwqp
->iwarp_state
= info
.next_iwarp_state
;
1397 iwqp
->ibqp_state
= attr
->qp_state
;
1399 if (iwqp
->ibqp_state
> IB_QPS_RTS
&&
1400 !iwqp
->flush_issued
) {
1401 iwqp
->flush_issued
= 1;
1402 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1403 irdma_flush_wqes(iwqp
, IRDMA_FLUSH_SQ
|
1407 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1410 iwqp
->ibqp_state
= attr
->qp_state
;
1412 if (udata
&& dev
->hw_attrs
.uk_attrs
.hw_rev
>= IRDMA_GEN_2
) {
1413 struct irdma_ucontext
*ucontext
;
1415 ucontext
= rdma_udata_to_drv_context(udata
,
1416 struct irdma_ucontext
, ibucontext
);
1417 if (iwqp
->sc_qp
.push_idx
!= IRDMA_INVALID_PUSH_PAGE_INDEX
&&
1418 !iwqp
->push_wqe_mmap_entry
&&
1419 !irdma_setup_push_mmap_entries(ucontext
, iwqp
,
1420 &uresp
.push_wqe_mmap_key
, &uresp
.push_db_mmap_key
)) {
1421 uresp
.push_valid
= 1;
1422 uresp
.push_offset
= iwqp
->sc_qp
.push_offset
;
1424 ret
= ib_copy_to_udata(udata
, &uresp
, min(sizeof(uresp
),
1427 irdma_remove_push_mmap_entries(iwqp
);
1428 ibdev_dbg(&iwdev
->ibdev
,
1429 "VERBS: copy_to_udata failed\n");
1437 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1443 * irdma_modify_qp - modify qp request
1444 * @ibqp: qp's pointer for modify
1445 * @attr: access attributes
1446 * @attr_mask: state mask
1449 int irdma_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
, int attr_mask
,
1450 struct ib_udata
*udata
)
1452 struct irdma_qp
*iwqp
= to_iwqp(ibqp
);
1453 struct irdma_device
*iwdev
= iwqp
->iwdev
;
1454 struct irdma_sc_dev
*dev
= &iwdev
->rf
->sc_dev
;
1455 struct irdma_qp_host_ctx_info
*ctx_info
;
1456 struct irdma_tcp_offload_info
*tcp_info
;
1457 struct irdma_iwarp_offload_info
*offload_info
;
1458 struct irdma_modify_qp_info info
= {};
1459 struct irdma_modify_qp_resp uresp
= {};
1460 struct irdma_modify_qp_req ureq
= {};
1461 u8 issue_modify_qp
= 0;
1464 unsigned long flags
;
1466 if (attr_mask
& ~IB_QP_ATTR_STANDARD_BITS
)
1469 ctx_info
= &iwqp
->ctx_info
;
1470 offload_info
= &iwqp
->iwarp_info
;
1471 tcp_info
= &iwqp
->tcp_info
;
1472 wait_event(iwqp
->mod_qp_waitq
, !atomic_read(&iwqp
->hw_mod_qp_pend
));
1473 ibdev_dbg(&iwdev
->ibdev
,
1474 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n",
1475 __builtin_return_address(0), ibqp
->qp_num
, attr
->qp_state
,
1476 iwqp
->ibqp_state
, iwqp
->iwarp_state
, iwqp
->last_aeq
,
1477 iwqp
->hw_tcp_state
, iwqp
->hw_iwarp_state
, attr_mask
);
1479 spin_lock_irqsave(&iwqp
->lock
, flags
);
1480 if (attr_mask
& IB_QP_STATE
) {
1481 info
.curr_iwarp_state
= iwqp
->iwarp_state
;
1482 switch (attr
->qp_state
) {
1485 if (iwqp
->iwarp_state
> IRDMA_QP_STATE_IDLE
) {
1490 if (iwqp
->iwarp_state
== IRDMA_QP_STATE_INVALID
) {
1491 info
.next_iwarp_state
= IRDMA_QP_STATE_IDLE
;
1492 issue_modify_qp
= 1;
1494 if (iwdev
->push_mode
&& udata
&&
1495 iwqp
->sc_qp
.push_idx
== IRDMA_INVALID_PUSH_PAGE_INDEX
&&
1496 dev
->hw_attrs
.uk_attrs
.hw_rev
>= IRDMA_GEN_2
) {
1497 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1498 irdma_alloc_push_page(iwqp
);
1499 spin_lock_irqsave(&iwqp
->lock
, flags
);
1503 if (iwqp
->iwarp_state
> IRDMA_QP_STATE_RTS
||
1509 issue_modify_qp
= 1;
1510 iwqp
->hw_tcp_state
= IRDMA_TCP_STATE_ESTABLISHED
;
1511 iwqp
->hte_added
= 1;
1512 info
.next_iwarp_state
= IRDMA_QP_STATE_RTS
;
1513 info
.tcp_ctx_valid
= true;
1514 info
.ord_valid
= true;
1515 info
.arp_cache_idx_valid
= true;
1516 info
.cq_num_valid
= true;
1519 if (iwqp
->hw_iwarp_state
> IRDMA_QP_STATE_RTS
) {
1524 if (iwqp
->iwarp_state
== IRDMA_QP_STATE_CLOSING
||
1525 iwqp
->iwarp_state
< IRDMA_QP_STATE_RTS
) {
1530 if (iwqp
->iwarp_state
> IRDMA_QP_STATE_CLOSING
) {
1535 info
.next_iwarp_state
= IRDMA_QP_STATE_CLOSING
;
1536 issue_modify_qp
= 1;
1539 if (iwqp
->iwarp_state
>= IRDMA_QP_STATE_TERMINATE
) {
1544 info
.next_iwarp_state
= IRDMA_QP_STATE_TERMINATE
;
1545 issue_modify_qp
= 1;
1549 if (iwqp
->iwarp_state
== IRDMA_QP_STATE_ERROR
) {
1550 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1552 if (ib_copy_from_udata(&ureq
, udata
,
1553 min(sizeof(ureq
), udata
->inlen
)))
1556 irdma_flush_wqes(iwqp
,
1557 (ureq
.sq_flush
? IRDMA_FLUSH_SQ
: 0) |
1558 (ureq
.rq_flush
? IRDMA_FLUSH_RQ
: 0) |
1564 if (iwqp
->sc_qp
.term_flags
) {
1565 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1566 irdma_terminate_del_timer(&iwqp
->sc_qp
);
1567 spin_lock_irqsave(&iwqp
->lock
, flags
);
1569 info
.next_iwarp_state
= IRDMA_QP_STATE_ERROR
;
1570 if (iwqp
->hw_tcp_state
> IRDMA_TCP_STATE_CLOSED
&&
1572 iwqp
->hw_tcp_state
!= IRDMA_TCP_STATE_TIME_WAIT
)
1573 info
.reset_tcp_conn
= true;
1577 issue_modify_qp
= 1;
1578 info
.next_iwarp_state
= IRDMA_QP_STATE_ERROR
;
1585 iwqp
->ibqp_state
= attr
->qp_state
;
1587 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
1588 ctx_info
->iwarp_info_valid
= true;
1589 if (attr
->qp_access_flags
& IB_ACCESS_LOCAL_WRITE
)
1590 offload_info
->wr_rdresp_en
= true;
1591 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
)
1592 offload_info
->wr_rdresp_en
= true;
1593 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
)
1594 offload_info
->rd_en
= true;
1597 if (ctx_info
->iwarp_info_valid
) {
1598 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
1599 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
1600 irdma_sc_qp_setctx(&iwqp
->sc_qp
, iwqp
->host_ctx
.va
, ctx_info
);
1602 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1604 if (attr_mask
& IB_QP_STATE
) {
1605 if (issue_modify_qp
) {
1606 ctx_info
->rem_endpoint_idx
= tcp_info
->arp_idx
;
1607 if (irdma_hw_modify_qp(iwdev
, iwqp
, &info
, true))
1611 spin_lock_irqsave(&iwqp
->lock
, flags
);
1612 if (iwqp
->iwarp_state
== info
.curr_iwarp_state
) {
1613 iwqp
->iwarp_state
= info
.next_iwarp_state
;
1614 iwqp
->ibqp_state
= attr
->qp_state
;
1616 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1619 if (issue_modify_qp
&& iwqp
->ibqp_state
> IB_QPS_RTS
) {
1621 if (iwqp
->cm_id
&& iwqp
->hw_tcp_state
) {
1622 spin_lock_irqsave(&iwqp
->lock
, flags
);
1623 iwqp
->hw_tcp_state
= IRDMA_TCP_STATE_CLOSED
;
1624 iwqp
->last_aeq
= IRDMA_AE_RESET_SENT
;
1625 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1626 irdma_cm_disconn(iwqp
);
1629 int close_timer_started
;
1631 spin_lock_irqsave(&iwdev
->cm_core
.ht_lock
, flags
);
1633 if (iwqp
->cm_node
) {
1634 refcount_inc(&iwqp
->cm_node
->refcnt
);
1635 spin_unlock_irqrestore(&iwdev
->cm_core
.ht_lock
, flags
);
1636 close_timer_started
= atomic_inc_return(&iwqp
->close_timer_started
);
1637 if (iwqp
->cm_id
&& close_timer_started
== 1)
1638 irdma_schedule_cm_timer(iwqp
->cm_node
,
1639 (struct irdma_puda_buf
*)iwqp
,
1640 IRDMA_TIMER_TYPE_CLOSE
, 1, 0);
1642 irdma_rem_ref_cm_node(iwqp
->cm_node
);
1644 spin_unlock_irqrestore(&iwdev
->cm_core
.ht_lock
, flags
);
1648 if (attr_mask
& IB_QP_STATE
&& udata
&&
1649 dev
->hw_attrs
.uk_attrs
.hw_rev
>= IRDMA_GEN_2
) {
1650 struct irdma_ucontext
*ucontext
;
1652 ucontext
= rdma_udata_to_drv_context(udata
,
1653 struct irdma_ucontext
, ibucontext
);
1654 if (iwqp
->sc_qp
.push_idx
!= IRDMA_INVALID_PUSH_PAGE_INDEX
&&
1655 !iwqp
->push_wqe_mmap_entry
&&
1656 !irdma_setup_push_mmap_entries(ucontext
, iwqp
,
1657 &uresp
.push_wqe_mmap_key
, &uresp
.push_db_mmap_key
)) {
1658 uresp
.push_valid
= 1;
1659 uresp
.push_offset
= iwqp
->sc_qp
.push_offset
;
1662 err
= ib_copy_to_udata(udata
, &uresp
, min(sizeof(uresp
),
1665 irdma_remove_push_mmap_entries(iwqp
);
1666 ibdev_dbg(&iwdev
->ibdev
,
1667 "VERBS: copy_to_udata failed\n");
1674 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1680 * irdma_cq_free_rsrc - free up resources for cq
1681 * @rf: RDMA PCI function
1684 static void irdma_cq_free_rsrc(struct irdma_pci_f
*rf
, struct irdma_cq
*iwcq
)
1686 struct irdma_sc_cq
*cq
= &iwcq
->sc_cq
;
1688 if (!iwcq
->user_mode
) {
1689 dma_free_coherent(rf
->sc_dev
.hw
->device
, iwcq
->kmem
.size
,
1690 iwcq
->kmem
.va
, iwcq
->kmem
.pa
);
1691 iwcq
->kmem
.va
= NULL
;
1692 dma_free_coherent(rf
->sc_dev
.hw
->device
,
1693 iwcq
->kmem_shadow
.size
,
1694 iwcq
->kmem_shadow
.va
, iwcq
->kmem_shadow
.pa
);
1695 iwcq
->kmem_shadow
.va
= NULL
;
1698 irdma_free_rsrc(rf
, rf
->allocated_cqs
, cq
->cq_uk
.cq_id
);
1702 * irdma_free_cqbuf - worker to free a cq buffer
1703 * @work: provides access to the cq buffer to free
1705 static void irdma_free_cqbuf(struct work_struct
*work
)
1707 struct irdma_cq_buf
*cq_buf
= container_of(work
, struct irdma_cq_buf
, work
);
1709 dma_free_coherent(cq_buf
->hw
->device
, cq_buf
->kmem_buf
.size
,
1710 cq_buf
->kmem_buf
.va
, cq_buf
->kmem_buf
.pa
);
1711 cq_buf
->kmem_buf
.va
= NULL
;
1716 * irdma_process_resize_list - remove resized cq buffers from the resize_list
1717 * @iwcq: cq which owns the resize_list
1718 * @iwdev: irdma device
1719 * @lcqe_buf: the buffer where the last cqe is received
1721 static int irdma_process_resize_list(struct irdma_cq
*iwcq
,
1722 struct irdma_device
*iwdev
,
1723 struct irdma_cq_buf
*lcqe_buf
)
1725 struct list_head
*tmp_node
, *list_node
;
1726 struct irdma_cq_buf
*cq_buf
;
1729 list_for_each_safe(list_node
, tmp_node
, &iwcq
->resize_list
) {
1730 cq_buf
= list_entry(list_node
, struct irdma_cq_buf
, list
);
1731 if (cq_buf
== lcqe_buf
)
1734 list_del(&cq_buf
->list
);
1735 queue_work(iwdev
->cleanup_wq
, &cq_buf
->work
);
1743 * irdma_destroy_cq - destroy cq
1744 * @ib_cq: cq pointer
1747 static int irdma_destroy_cq(struct ib_cq
*ib_cq
, struct ib_udata
*udata
)
1749 struct irdma_device
*iwdev
= to_iwdev(ib_cq
->device
);
1750 struct irdma_cq
*iwcq
= to_iwcq(ib_cq
);
1751 struct irdma_sc_cq
*cq
= &iwcq
->sc_cq
;
1752 struct irdma_sc_dev
*dev
= cq
->dev
;
1753 struct irdma_sc_ceq
*ceq
= dev
->ceq
[cq
->ceq_id
];
1754 struct irdma_ceq
*iwceq
= container_of(ceq
, struct irdma_ceq
, sc_ceq
);
1755 unsigned long flags
;
1757 spin_lock_irqsave(&iwcq
->lock
, flags
);
1758 if (!list_empty(&iwcq
->resize_list
))
1759 irdma_process_resize_list(iwcq
, iwdev
, NULL
);
1760 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
1762 irdma_cq_wq_destroy(iwdev
->rf
, cq
);
1763 irdma_cq_free_rsrc(iwdev
->rf
, iwcq
);
1765 spin_lock_irqsave(&iwceq
->ce_lock
, flags
);
1766 irdma_sc_cleanup_ceqes(cq
, ceq
);
1767 spin_unlock_irqrestore(&iwceq
->ce_lock
, flags
);
1773 * irdma_resize_cq - resize cq
1774 * @ibcq: cq to be resized
1775 * @entries: desired cq size
1778 static int irdma_resize_cq(struct ib_cq
*ibcq
, int entries
,
1779 struct ib_udata
*udata
)
1781 struct irdma_cq
*iwcq
= to_iwcq(ibcq
);
1782 struct irdma_sc_dev
*dev
= iwcq
->sc_cq
.dev
;
1783 struct irdma_cqp_request
*cqp_request
;
1784 struct cqp_cmds_info
*cqp_info
;
1785 struct irdma_modify_cq_info
*m_info
;
1786 struct irdma_modify_cq_info info
= {};
1787 struct irdma_dma_mem kmem_buf
;
1788 struct irdma_cq_mr
*cqmr_buf
;
1789 struct irdma_pbl
*iwpbl_buf
;
1790 struct irdma_device
*iwdev
;
1791 struct irdma_pci_f
*rf
;
1792 struct irdma_cq_buf
*cq_buf
= NULL
;
1793 enum irdma_status_code status
= 0;
1794 unsigned long flags
;
1797 iwdev
= to_iwdev(ibcq
->device
);
1800 if (!(rf
->sc_dev
.hw_attrs
.uk_attrs
.feature_flags
&
1801 IRDMA_FEATURE_CQ_RESIZE
))
1804 if (entries
> rf
->max_cqe
)
1807 if (!iwcq
->user_mode
) {
1809 if (rf
->sc_dev
.hw_attrs
.uk_attrs
.hw_rev
>= IRDMA_GEN_2
)
1813 info
.cq_size
= max(entries
, 4);
1815 if (info
.cq_size
== iwcq
->sc_cq
.cq_uk
.cq_size
- 1)
1819 struct irdma_resize_cq_req req
= {};
1820 struct irdma_ucontext
*ucontext
=
1821 rdma_udata_to_drv_context(udata
, struct irdma_ucontext
,
1824 /* CQ resize not supported with legacy GEN_1 libi40iw */
1825 if (ucontext
->legacy_mode
)
1828 if (ib_copy_from_udata(&req
, udata
,
1829 min(sizeof(req
), udata
->inlen
)))
1832 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1833 iwpbl_buf
= irdma_get_pbl((unsigned long)req
.user_cq_buffer
,
1834 &ucontext
->cq_reg_mem_list
);
1835 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1840 cqmr_buf
= &iwpbl_buf
->cq_mr
;
1841 if (iwpbl_buf
->pbl_allocated
) {
1842 info
.virtual_map
= true;
1843 info
.pbl_chunk_size
= 1;
1844 info
.first_pm_pbl_idx
= cqmr_buf
->cq_pbl
.idx
;
1846 info
.cq_pa
= cqmr_buf
->cq_pbl
.addr
;
1849 /* Kmode CQ resize */
1852 rsize
= info
.cq_size
* sizeof(struct irdma_cqe
);
1853 kmem_buf
.size
= ALIGN(round_up(rsize
, 256), 256);
1854 kmem_buf
.va
= dma_alloc_coherent(dev
->hw
->device
,
1855 kmem_buf
.size
, &kmem_buf
.pa
,
1860 info
.cq_base
= kmem_buf
.va
;
1861 info
.cq_pa
= kmem_buf
.pa
;
1862 cq_buf
= kzalloc(sizeof(*cq_buf
), GFP_KERNEL
);
1869 cqp_request
= irdma_alloc_and_get_cqp_request(&rf
->cqp
, true);
1875 info
.shadow_read_threshold
= iwcq
->sc_cq
.shadow_read_threshold
;
1876 info
.cq_resize
= true;
1878 cqp_info
= &cqp_request
->info
;
1879 m_info
= &cqp_info
->in
.u
.cq_modify
.info
;
1880 memcpy(m_info
, &info
, sizeof(*m_info
));
1882 cqp_info
->cqp_cmd
= IRDMA_OP_CQ_MODIFY
;
1883 cqp_info
->in
.u
.cq_modify
.cq
= &iwcq
->sc_cq
;
1884 cqp_info
->in
.u
.cq_modify
.scratch
= (uintptr_t)cqp_request
;
1885 cqp_info
->post_sq
= 1;
1886 status
= irdma_handle_cqp_op(rf
, cqp_request
);
1887 irdma_put_cqp_request(&rf
->cqp
, cqp_request
);
1893 spin_lock_irqsave(&iwcq
->lock
, flags
);
1895 cq_buf
->kmem_buf
= iwcq
->kmem
;
1896 cq_buf
->hw
= dev
->hw
;
1897 memcpy(&cq_buf
->cq_uk
, &iwcq
->sc_cq
.cq_uk
, sizeof(cq_buf
->cq_uk
));
1898 INIT_WORK(&cq_buf
->work
, irdma_free_cqbuf
);
1899 list_add_tail(&cq_buf
->list
, &iwcq
->resize_list
);
1900 iwcq
->kmem
= kmem_buf
;
1903 irdma_sc_cq_resize(&iwcq
->sc_cq
, &info
);
1904 ibcq
->cqe
= info
.cq_size
- 1;
1905 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
1910 dma_free_coherent(dev
->hw
->device
, kmem_buf
.size
, kmem_buf
.va
,
1919 static inline int cq_validate_flags(u32 flags
, u8 hw_rev
)
1921 /* GEN1 does not support CQ create flags */
1922 if (hw_rev
== IRDMA_GEN_1
)
1923 return flags
? -EOPNOTSUPP
: 0;
1925 return flags
& ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION
? -EOPNOTSUPP
: 0;
1929 * irdma_create_cq - create cq
1930 * @ibcq: CQ allocated
1931 * @attr: attributes for cq
1934 static int irdma_create_cq(struct ib_cq
*ibcq
,
1935 const struct ib_cq_init_attr
*attr
,
1936 struct ib_udata
*udata
)
1938 struct ib_device
*ibdev
= ibcq
->device
;
1939 struct irdma_device
*iwdev
= to_iwdev(ibdev
);
1940 struct irdma_pci_f
*rf
= iwdev
->rf
;
1941 struct irdma_cq
*iwcq
= to_iwcq(ibcq
);
1943 struct irdma_sc_cq
*cq
;
1944 struct irdma_sc_dev
*dev
= &rf
->sc_dev
;
1945 struct irdma_cq_init_info info
= {};
1946 enum irdma_status_code status
;
1947 struct irdma_cqp_request
*cqp_request
;
1948 struct cqp_cmds_info
*cqp_info
;
1949 struct irdma_cq_uk_init_info
*ukinfo
= &info
.cq_uk_init_info
;
1950 unsigned long flags
;
1952 int entries
= attr
->cqe
;
1954 err_code
= cq_validate_flags(attr
->flags
, dev
->hw_attrs
.uk_attrs
.hw_rev
);
1957 err_code
= irdma_alloc_rsrc(rf
, rf
->allocated_cqs
, rf
->max_cq
, &cq_num
,
1964 spin_lock_init(&iwcq
->lock
);
1965 INIT_LIST_HEAD(&iwcq
->resize_list
);
1967 ukinfo
->cq_size
= max(entries
, 4);
1968 ukinfo
->cq_id
= cq_num
;
1969 iwcq
->ibcq
.cqe
= info
.cq_uk_init_info
.cq_size
;
1970 if (attr
->comp_vector
< rf
->ceqs_count
)
1971 info
.ceq_id
= attr
->comp_vector
;
1972 info
.ceq_id_valid
= true;
1974 info
.type
= IRDMA_CQ_TYPE_IWARP
;
1975 info
.vsi
= &iwdev
->vsi
;
1978 struct irdma_ucontext
*ucontext
;
1979 struct irdma_create_cq_req req
= {};
1980 struct irdma_cq_mr
*cqmr
;
1981 struct irdma_pbl
*iwpbl
;
1982 struct irdma_pbl
*iwpbl_shadow
;
1983 struct irdma_cq_mr
*cqmr_shadow
;
1985 iwcq
->user_mode
= true;
1987 rdma_udata_to_drv_context(udata
, struct irdma_ucontext
,
1989 if (ib_copy_from_udata(&req
, udata
,
1990 min(sizeof(req
), udata
->inlen
))) {
1995 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1996 iwpbl
= irdma_get_pbl((unsigned long)req
.user_cq_buf
,
1997 &ucontext
->cq_reg_mem_list
);
1998 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
2004 iwcq
->iwpbl
= iwpbl
;
2005 iwcq
->cq_mem_size
= 0;
2006 cqmr
= &iwpbl
->cq_mr
;
2008 if (rf
->sc_dev
.hw_attrs
.uk_attrs
.feature_flags
&
2009 IRDMA_FEATURE_CQ_RESIZE
&& !ucontext
->legacy_mode
) {
2010 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
2011 iwpbl_shadow
= irdma_get_pbl(
2012 (unsigned long)req
.user_shadow_area
,
2013 &ucontext
->cq_reg_mem_list
);
2014 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
2016 if (!iwpbl_shadow
) {
2020 iwcq
->iwpbl_shadow
= iwpbl_shadow
;
2021 cqmr_shadow
= &iwpbl_shadow
->cq_mr
;
2022 info
.shadow_area_pa
= cqmr_shadow
->cq_pbl
.addr
;
2025 info
.shadow_area_pa
= cqmr
->shadow
;
2027 if (iwpbl
->pbl_allocated
) {
2028 info
.virtual_map
= true;
2029 info
.pbl_chunk_size
= 1;
2030 info
.first_pm_pbl_idx
= cqmr
->cq_pbl
.idx
;
2032 info
.cq_base_pa
= cqmr
->cq_pbl
.addr
;
2035 /* Kmode allocations */
2038 if (entries
> rf
->max_cqe
) {
2044 if (dev
->hw_attrs
.uk_attrs
.hw_rev
>= IRDMA_GEN_2
)
2046 ukinfo
->cq_size
= entries
;
2048 rsize
= info
.cq_uk_init_info
.cq_size
* sizeof(struct irdma_cqe
);
2049 iwcq
->kmem
.size
= ALIGN(round_up(rsize
, 256), 256);
2050 iwcq
->kmem
.va
= dma_alloc_coherent(dev
->hw
->device
,
2052 &iwcq
->kmem
.pa
, GFP_KERNEL
);
2053 if (!iwcq
->kmem
.va
) {
2058 iwcq
->kmem_shadow
.size
= ALIGN(IRDMA_SHADOW_AREA_SIZE
<< 3,
2060 iwcq
->kmem_shadow
.va
= dma_alloc_coherent(dev
->hw
->device
,
2061 iwcq
->kmem_shadow
.size
,
2062 &iwcq
->kmem_shadow
.pa
,
2064 if (!iwcq
->kmem_shadow
.va
) {
2068 info
.shadow_area_pa
= iwcq
->kmem_shadow
.pa
;
2069 ukinfo
->shadow_area
= iwcq
->kmem_shadow
.va
;
2070 ukinfo
->cq_base
= iwcq
->kmem
.va
;
2071 info
.cq_base_pa
= iwcq
->kmem
.pa
;
2074 if (dev
->hw_attrs
.uk_attrs
.hw_rev
>= IRDMA_GEN_2
)
2075 info
.shadow_read_threshold
= min(info
.cq_uk_init_info
.cq_size
/ 2,
2076 (u32
)IRDMA_MAX_CQ_READ_THRESH
);
2078 if (irdma_sc_cq_init(cq
, &info
)) {
2079 ibdev_dbg(&iwdev
->ibdev
, "VERBS: init cq fail\n");
2084 cqp_request
= irdma_alloc_and_get_cqp_request(&rf
->cqp
, true);
2090 cqp_info
= &cqp_request
->info
;
2091 cqp_info
->cqp_cmd
= IRDMA_OP_CQ_CREATE
;
2092 cqp_info
->post_sq
= 1;
2093 cqp_info
->in
.u
.cq_create
.cq
= cq
;
2094 cqp_info
->in
.u
.cq_create
.check_overflow
= true;
2095 cqp_info
->in
.u
.cq_create
.scratch
= (uintptr_t)cqp_request
;
2096 status
= irdma_handle_cqp_op(rf
, cqp_request
);
2097 irdma_put_cqp_request(&rf
->cqp
, cqp_request
);
2104 struct irdma_create_cq_resp resp
= {};
2106 resp
.cq_id
= info
.cq_uk_init_info
.cq_id
;
2107 resp
.cq_size
= info
.cq_uk_init_info
.cq_size
;
2108 if (ib_copy_to_udata(udata
, &resp
,
2109 min(sizeof(resp
), udata
->outlen
))) {
2110 ibdev_dbg(&iwdev
->ibdev
,
2111 "VERBS: copy to user data\n");
2118 irdma_cq_wq_destroy(rf
, cq
);
2120 irdma_cq_free_rsrc(rf
, iwcq
);
2126 * irdma_get_mr_access - get hw MR access permissions from IB access flags
2127 * @access: IB access flags
2129 static inline u16
irdma_get_mr_access(int access
)
2133 hw_access
|= (access
& IB_ACCESS_LOCAL_WRITE
) ?
2134 IRDMA_ACCESS_FLAGS_LOCALWRITE
: 0;
2135 hw_access
|= (access
& IB_ACCESS_REMOTE_WRITE
) ?
2136 IRDMA_ACCESS_FLAGS_REMOTEWRITE
: 0;
2137 hw_access
|= (access
& IB_ACCESS_REMOTE_READ
) ?
2138 IRDMA_ACCESS_FLAGS_REMOTEREAD
: 0;
2139 hw_access
|= (access
& IB_ACCESS_MW_BIND
) ?
2140 IRDMA_ACCESS_FLAGS_BIND_WINDOW
: 0;
2141 hw_access
|= (access
& IB_ZERO_BASED
) ?
2142 IRDMA_ACCESS_FLAGS_ZERO_BASED
: 0;
2143 hw_access
|= IRDMA_ACCESS_FLAGS_LOCALREAD
;
2149 * irdma_free_stag - free stag resource
2150 * @iwdev: irdma device
2151 * @stag: stag to free
2153 static void irdma_free_stag(struct irdma_device
*iwdev
, u32 stag
)
2157 stag_idx
= (stag
& iwdev
->rf
->mr_stagmask
) >> IRDMA_CQPSQ_STAG_IDX_S
;
2158 irdma_free_rsrc(iwdev
->rf
, iwdev
->rf
->allocated_mrs
, stag_idx
);
2162 * irdma_create_stag - create random stag
2163 * @iwdev: irdma device
2165 static u32
irdma_create_stag(struct irdma_device
*iwdev
)
2169 u32 next_stag_index
;
2175 get_random_bytes(&random
, sizeof(random
));
2176 consumer_key
= (u8
)random
;
2178 driver_key
= random
& ~iwdev
->rf
->mr_stagmask
;
2179 next_stag_index
= (random
& iwdev
->rf
->mr_stagmask
) >> 8;
2180 next_stag_index
%= iwdev
->rf
->max_mr
;
2182 ret
= irdma_alloc_rsrc(iwdev
->rf
, iwdev
->rf
->allocated_mrs
,
2183 iwdev
->rf
->max_mr
, &stag_index
,
2187 stag
= stag_index
<< IRDMA_CQPSQ_STAG_IDX_S
;
2189 stag
+= (u32
)consumer_key
;
2195 * irdma_next_pbl_addr - Get next pbl address
2196 * @pbl: pointer to a pble
2197 * @pinfo: info pointer
2200 static inline u64
*irdma_next_pbl_addr(u64
*pbl
, struct irdma_pble_info
**pinfo
,
2204 if (!(*pinfo
) || *idx
!= (*pinfo
)->cnt
)
2209 return (*pinfo
)->addr
;
2213 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally
2214 * @iwmr: iwmr for IB's user page addresses
2215 * @pbl: ple pointer to save 1 level or 0 level pble
2216 * @level: indicated level 0, 1 or 2
2218 static void irdma_copy_user_pgaddrs(struct irdma_mr
*iwmr
, u64
*pbl
,
2219 enum irdma_pble_level level
)
2221 struct ib_umem
*region
= iwmr
->region
;
2222 struct irdma_pbl
*iwpbl
= &iwmr
->iwpbl
;
2223 struct irdma_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
2224 struct irdma_pble_info
*pinfo
;
2225 struct ib_block_iter biter
;
2229 pinfo
= (level
== PBLE_LEVEL_1
) ? NULL
: palloc
->level2
.leaf
;
2231 if (iwmr
->type
== IRDMA_MEMREG_TYPE_QP
)
2232 iwpbl
->qp_mr
.sq_page
= sg_page(region
->sgt_append
.sgt
.sgl
);
2234 rdma_umem_for_each_dma_block(region
, &biter
, iwmr
->page_size
) {
2235 *pbl
= rdma_block_iter_dma_address(&biter
);
2236 if (++pbl_cnt
== palloc
->total_cnt
)
2238 pbl
= irdma_next_pbl_addr(pbl
, &pinfo
, &idx
);
2243 * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous
2244 * @arr: lvl1 pbl array
2245 * @npages: page count
2246 * @pg_size: page size
2249 static bool irdma_check_mem_contiguous(u64
*arr
, u32 npages
, u32 pg_size
)
2253 for (pg_idx
= 0; pg_idx
< npages
; pg_idx
++) {
2254 if ((*arr
+ (pg_size
* pg_idx
)) != arr
[pg_idx
])
2262 * irdma_check_mr_contiguous - check if MR is physically contiguous
2263 * @palloc: pbl allocation struct
2264 * @pg_size: page size
2266 static bool irdma_check_mr_contiguous(struct irdma_pble_alloc
*palloc
,
2269 struct irdma_pble_level2
*lvl2
= &palloc
->level2
;
2270 struct irdma_pble_info
*leaf
= lvl2
->leaf
;
2272 u64
*start_addr
= NULL
;
2276 if (palloc
->level
== PBLE_LEVEL_1
) {
2277 arr
= palloc
->level1
.addr
;
2278 ret
= irdma_check_mem_contiguous(arr
, palloc
->total_cnt
,
2283 start_addr
= leaf
->addr
;
2285 for (i
= 0; i
< lvl2
->leaf_cnt
; i
++, leaf
++) {
2287 if ((*start_addr
+ (i
* pg_size
* PBLE_PER_PAGE
)) != *arr
)
2289 ret
= irdma_check_mem_contiguous(arr
, leaf
->cnt
, pg_size
);
2298 * irdma_setup_pbles - copy user pg address to pble's
2299 * @rf: RDMA PCI function
2300 * @iwmr: mr pointer for this memory registration
2301 * @use_pbles: flag if to use pble's
2303 static int irdma_setup_pbles(struct irdma_pci_f
*rf
, struct irdma_mr
*iwmr
,
2306 struct irdma_pbl
*iwpbl
= &iwmr
->iwpbl
;
2307 struct irdma_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
2308 struct irdma_pble_info
*pinfo
;
2310 enum irdma_status_code status
;
2311 enum irdma_pble_level level
= PBLE_LEVEL_1
;
2314 status
= irdma_get_pble(rf
->pble_rsrc
, palloc
, iwmr
->page_cnt
,
2319 iwpbl
->pbl_allocated
= true;
2320 level
= palloc
->level
;
2321 pinfo
= (level
== PBLE_LEVEL_1
) ? &palloc
->level1
:
2322 palloc
->level2
.leaf
;
2325 pbl
= iwmr
->pgaddrmem
;
2328 irdma_copy_user_pgaddrs(iwmr
, pbl
, level
);
2331 iwmr
->pgaddrmem
[0] = *pbl
;
2337 * irdma_handle_q_mem - handle memory for qp and cq
2338 * @iwdev: irdma device
2339 * @req: information for q memory management
2340 * @iwpbl: pble struct
2341 * @use_pbles: flag to use pble
2343 static int irdma_handle_q_mem(struct irdma_device
*iwdev
,
2344 struct irdma_mem_reg_req
*req
,
2345 struct irdma_pbl
*iwpbl
, bool use_pbles
)
2347 struct irdma_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
2348 struct irdma_mr
*iwmr
= iwpbl
->iwmr
;
2349 struct irdma_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
2350 struct irdma_cq_mr
*cqmr
= &iwpbl
->cq_mr
;
2351 struct irdma_hmc_pble
*hmc_p
;
2352 u64
*arr
= iwmr
->pgaddrmem
;
2357 pg_size
= iwmr
->page_size
;
2358 err
= irdma_setup_pbles(iwdev
->rf
, iwmr
, use_pbles
);
2362 if (use_pbles
&& palloc
->level
!= PBLE_LEVEL_1
) {
2363 irdma_free_pble(iwdev
->rf
->pble_rsrc
, palloc
);
2364 iwpbl
->pbl_allocated
= false;
2369 arr
= palloc
->level1
.addr
;
2371 switch (iwmr
->type
) {
2372 case IRDMA_MEMREG_TYPE_QP
:
2373 total
= req
->sq_pages
+ req
->rq_pages
;
2374 hmc_p
= &qpmr
->sq_pbl
;
2375 qpmr
->shadow
= (dma_addr_t
)arr
[total
];
2378 ret
= irdma_check_mem_contiguous(arr
, req
->sq_pages
,
2381 ret
= irdma_check_mem_contiguous(&arr
[req
->sq_pages
],
2387 hmc_p
->idx
= palloc
->level1
.idx
;
2388 hmc_p
= &qpmr
->rq_pbl
;
2389 hmc_p
->idx
= palloc
->level1
.idx
+ req
->sq_pages
;
2391 hmc_p
->addr
= arr
[0];
2392 hmc_p
= &qpmr
->rq_pbl
;
2393 hmc_p
->addr
= arr
[req
->sq_pages
];
2396 case IRDMA_MEMREG_TYPE_CQ
:
2397 hmc_p
= &cqmr
->cq_pbl
;
2400 cqmr
->shadow
= (dma_addr_t
)arr
[req
->cq_pages
];
2403 ret
= irdma_check_mem_contiguous(arr
, req
->cq_pages
,
2407 hmc_p
->idx
= palloc
->level1
.idx
;
2409 hmc_p
->addr
= arr
[0];
2412 ibdev_dbg(&iwdev
->ibdev
, "VERBS: MR type error\n");
2416 if (use_pbles
&& ret
) {
2417 irdma_free_pble(iwdev
->rf
->pble_rsrc
, palloc
);
2418 iwpbl
->pbl_allocated
= false;
2425 * irdma_hw_alloc_mw - create the hw memory window
2426 * @iwdev: irdma device
2427 * @iwmr: pointer to memory window info
2429 static int irdma_hw_alloc_mw(struct irdma_device
*iwdev
, struct irdma_mr
*iwmr
)
2431 struct irdma_mw_alloc_info
*info
;
2432 struct irdma_pd
*iwpd
= to_iwpd(iwmr
->ibmr
.pd
);
2433 struct irdma_cqp_request
*cqp_request
;
2434 struct cqp_cmds_info
*cqp_info
;
2435 enum irdma_status_code status
;
2437 cqp_request
= irdma_alloc_and_get_cqp_request(&iwdev
->rf
->cqp
, true);
2441 cqp_info
= &cqp_request
->info
;
2442 info
= &cqp_info
->in
.u
.mw_alloc
.info
;
2443 memset(info
, 0, sizeof(*info
));
2444 if (iwmr
->ibmw
.type
== IB_MW_TYPE_1
)
2445 info
->mw_wide
= true;
2447 info
->page_size
= PAGE_SIZE
;
2448 info
->mw_stag_index
= iwmr
->stag
>> IRDMA_CQPSQ_STAG_IDX_S
;
2449 info
->pd_id
= iwpd
->sc_pd
.pd_id
;
2450 info
->remote_access
= true;
2451 cqp_info
->cqp_cmd
= IRDMA_OP_MW_ALLOC
;
2452 cqp_info
->post_sq
= 1;
2453 cqp_info
->in
.u
.mw_alloc
.dev
= &iwdev
->rf
->sc_dev
;
2454 cqp_info
->in
.u
.mw_alloc
.scratch
= (uintptr_t)cqp_request
;
2455 status
= irdma_handle_cqp_op(iwdev
->rf
, cqp_request
);
2456 irdma_put_cqp_request(&iwdev
->rf
->cqp
, cqp_request
);
2458 return status
? -ENOMEM
: 0;
2462 * irdma_alloc_mw - Allocate memory window
2463 * @ibmw: Memory Window
2464 * @udata: user data pointer
2466 static int irdma_alloc_mw(struct ib_mw
*ibmw
, struct ib_udata
*udata
)
2468 struct irdma_device
*iwdev
= to_iwdev(ibmw
->device
);
2469 struct irdma_mr
*iwmr
= to_iwmw(ibmw
);
2473 stag
= irdma_create_stag(iwdev
);
2480 err_code
= irdma_hw_alloc_mw(iwdev
, iwmr
);
2482 irdma_free_stag(iwdev
, stag
);
2490 * irdma_dealloc_mw - Dealloc memory window
2491 * @ibmw: memory window structure.
2493 static int irdma_dealloc_mw(struct ib_mw
*ibmw
)
2495 struct ib_pd
*ibpd
= ibmw
->pd
;
2496 struct irdma_pd
*iwpd
= to_iwpd(ibpd
);
2497 struct irdma_mr
*iwmr
= to_iwmr((struct ib_mr
*)ibmw
);
2498 struct irdma_device
*iwdev
= to_iwdev(ibmw
->device
);
2499 struct irdma_cqp_request
*cqp_request
;
2500 struct cqp_cmds_info
*cqp_info
;
2501 struct irdma_dealloc_stag_info
*info
;
2503 cqp_request
= irdma_alloc_and_get_cqp_request(&iwdev
->rf
->cqp
, true);
2507 cqp_info
= &cqp_request
->info
;
2508 info
= &cqp_info
->in
.u
.dealloc_stag
.info
;
2509 memset(info
, 0, sizeof(*info
));
2510 info
->pd_id
= iwpd
->sc_pd
.pd_id
& 0x00007fff;
2511 info
->stag_idx
= ibmw
->rkey
>> IRDMA_CQPSQ_STAG_IDX_S
;
2513 cqp_info
->cqp_cmd
= IRDMA_OP_DEALLOC_STAG
;
2514 cqp_info
->post_sq
= 1;
2515 cqp_info
->in
.u
.dealloc_stag
.dev
= &iwdev
->rf
->sc_dev
;
2516 cqp_info
->in
.u
.dealloc_stag
.scratch
= (uintptr_t)cqp_request
;
2517 irdma_handle_cqp_op(iwdev
->rf
, cqp_request
);
2518 irdma_put_cqp_request(&iwdev
->rf
->cqp
, cqp_request
);
2519 irdma_free_stag(iwdev
, iwmr
->stag
);
2525 * irdma_hw_alloc_stag - cqp command to allocate stag
2526 * @iwdev: irdma device
2527 * @iwmr: irdma mr pointer
2529 static int irdma_hw_alloc_stag(struct irdma_device
*iwdev
,
2530 struct irdma_mr
*iwmr
)
2532 struct irdma_allocate_stag_info
*info
;
2533 struct irdma_pd
*iwpd
= to_iwpd(iwmr
->ibmr
.pd
);
2534 enum irdma_status_code status
;
2536 struct irdma_cqp_request
*cqp_request
;
2537 struct cqp_cmds_info
*cqp_info
;
2539 cqp_request
= irdma_alloc_and_get_cqp_request(&iwdev
->rf
->cqp
, true);
2543 cqp_info
= &cqp_request
->info
;
2544 info
= &cqp_info
->in
.u
.alloc_stag
.info
;
2545 memset(info
, 0, sizeof(*info
));
2546 info
->page_size
= PAGE_SIZE
;
2547 info
->stag_idx
= iwmr
->stag
>> IRDMA_CQPSQ_STAG_IDX_S
;
2548 info
->pd_id
= iwpd
->sc_pd
.pd_id
;
2549 info
->total_len
= iwmr
->len
;
2550 info
->remote_access
= true;
2551 cqp_info
->cqp_cmd
= IRDMA_OP_ALLOC_STAG
;
2552 cqp_info
->post_sq
= 1;
2553 cqp_info
->in
.u
.alloc_stag
.dev
= &iwdev
->rf
->sc_dev
;
2554 cqp_info
->in
.u
.alloc_stag
.scratch
= (uintptr_t)cqp_request
;
2555 status
= irdma_handle_cqp_op(iwdev
->rf
, cqp_request
);
2556 irdma_put_cqp_request(&iwdev
->rf
->cqp
, cqp_request
);
2564 * irdma_alloc_mr - register stag for fast memory registration
2566 * @mr_type: memory for stag registrion
2567 * @max_num_sg: man number of pages
2569 static struct ib_mr
*irdma_alloc_mr(struct ib_pd
*pd
, enum ib_mr_type mr_type
,
2572 struct irdma_device
*iwdev
= to_iwdev(pd
->device
);
2573 struct irdma_pble_alloc
*palloc
;
2574 struct irdma_pbl
*iwpbl
;
2575 struct irdma_mr
*iwmr
;
2576 enum irdma_status_code status
;
2578 int err_code
= -ENOMEM
;
2580 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
2582 return ERR_PTR(-ENOMEM
);
2584 stag
= irdma_create_stag(iwdev
);
2591 iwmr
->ibmr
.rkey
= stag
;
2592 iwmr
->ibmr
.lkey
= stag
;
2594 iwmr
->ibmr
.device
= pd
->device
;
2595 iwpbl
= &iwmr
->iwpbl
;
2597 iwmr
->type
= IRDMA_MEMREG_TYPE_MEM
;
2598 palloc
= &iwpbl
->pble_alloc
;
2599 iwmr
->page_cnt
= max_num_sg
;
2600 status
= irdma_get_pble(iwdev
->rf
->pble_rsrc
, palloc
, iwmr
->page_cnt
,
2605 err_code
= irdma_hw_alloc_stag(iwdev
, iwmr
);
2607 goto err_alloc_stag
;
2609 iwpbl
->pbl_allocated
= true;
2613 irdma_free_pble(iwdev
->rf
->pble_rsrc
, palloc
);
2615 irdma_free_stag(iwdev
, stag
);
2619 return ERR_PTR(err_code
);
2623 * irdma_set_page - populate pbl list for fmr
2624 * @ibmr: ib mem to access iwarp mr pointer
2625 * @addr: page dma address fro pbl list
2627 static int irdma_set_page(struct ib_mr
*ibmr
, u64 addr
)
2629 struct irdma_mr
*iwmr
= to_iwmr(ibmr
);
2630 struct irdma_pbl
*iwpbl
= &iwmr
->iwpbl
;
2631 struct irdma_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
2634 if (unlikely(iwmr
->npages
== iwmr
->page_cnt
))
2637 pbl
= palloc
->level1
.addr
;
2638 pbl
[iwmr
->npages
++] = addr
;
2644 * irdma_map_mr_sg - map of sg list for fmr
2645 * @ibmr: ib mem to access iwarp mr pointer
2646 * @sg: scatter gather list
2647 * @sg_nents: number of sg pages
2648 * @sg_offset: scatter gather list for fmr
2650 static int irdma_map_mr_sg(struct ib_mr
*ibmr
, struct scatterlist
*sg
,
2651 int sg_nents
, unsigned int *sg_offset
)
2653 struct irdma_mr
*iwmr
= to_iwmr(ibmr
);
2657 return ib_sg_to_pages(ibmr
, sg
, sg_nents
, sg_offset
, irdma_set_page
);
2661 * irdma_hwreg_mr - send cqp command for memory registration
2662 * @iwdev: irdma device
2663 * @iwmr: irdma mr pointer
2664 * @access: access for MR
2666 static int irdma_hwreg_mr(struct irdma_device
*iwdev
, struct irdma_mr
*iwmr
,
2669 struct irdma_pbl
*iwpbl
= &iwmr
->iwpbl
;
2670 struct irdma_reg_ns_stag_info
*stag_info
;
2671 struct irdma_pd
*iwpd
= to_iwpd(iwmr
->ibmr
.pd
);
2672 struct irdma_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
2673 enum irdma_status_code status
;
2675 struct irdma_cqp_request
*cqp_request
;
2676 struct cqp_cmds_info
*cqp_info
;
2678 cqp_request
= irdma_alloc_and_get_cqp_request(&iwdev
->rf
->cqp
, true);
2682 cqp_info
= &cqp_request
->info
;
2683 stag_info
= &cqp_info
->in
.u
.mr_reg_non_shared
.info
;
2684 memset(stag_info
, 0, sizeof(*stag_info
));
2685 stag_info
->va
= iwpbl
->user_base
;
2686 stag_info
->stag_idx
= iwmr
->stag
>> IRDMA_CQPSQ_STAG_IDX_S
;
2687 stag_info
->stag_key
= (u8
)iwmr
->stag
;
2688 stag_info
->total_len
= iwmr
->len
;
2689 stag_info
->access_rights
= irdma_get_mr_access(access
);
2690 stag_info
->pd_id
= iwpd
->sc_pd
.pd_id
;
2691 if (stag_info
->access_rights
& IRDMA_ACCESS_FLAGS_ZERO_BASED
)
2692 stag_info
->addr_type
= IRDMA_ADDR_TYPE_ZERO_BASED
;
2694 stag_info
->addr_type
= IRDMA_ADDR_TYPE_VA_BASED
;
2695 stag_info
->page_size
= iwmr
->page_size
;
2697 if (iwpbl
->pbl_allocated
) {
2698 if (palloc
->level
== PBLE_LEVEL_1
) {
2699 stag_info
->first_pm_pbl_index
= palloc
->level1
.idx
;
2700 stag_info
->chunk_size
= 1;
2702 stag_info
->first_pm_pbl_index
= palloc
->level2
.root
.idx
;
2703 stag_info
->chunk_size
= 3;
2706 stag_info
->reg_addr_pa
= iwmr
->pgaddrmem
[0];
2709 cqp_info
->cqp_cmd
= IRDMA_OP_MR_REG_NON_SHARED
;
2710 cqp_info
->post_sq
= 1;
2711 cqp_info
->in
.u
.mr_reg_non_shared
.dev
= &iwdev
->rf
->sc_dev
;
2712 cqp_info
->in
.u
.mr_reg_non_shared
.scratch
= (uintptr_t)cqp_request
;
2713 status
= irdma_handle_cqp_op(iwdev
->rf
, cqp_request
);
2714 irdma_put_cqp_request(&iwdev
->rf
->cqp
, cqp_request
);
2722 * irdma_reg_user_mr - Register a user memory region
2724 * @start: virtual start address
2725 * @len: length of mr
2726 * @virt: virtual address
2727 * @access: access of mr
2730 static struct ib_mr
*irdma_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 len
,
2731 u64 virt
, int access
,
2732 struct ib_udata
*udata
)
2734 struct irdma_device
*iwdev
= to_iwdev(pd
->device
);
2735 struct irdma_ucontext
*ucontext
;
2736 struct irdma_pble_alloc
*palloc
;
2737 struct irdma_pbl
*iwpbl
;
2738 struct irdma_mr
*iwmr
;
2739 struct ib_umem
*region
;
2740 struct irdma_mem_reg_req req
;
2741 u32 total
, stag
= 0;
2742 u8 shadow_pgcnt
= 1;
2743 bool use_pbles
= false;
2744 unsigned long flags
;
2748 if (len
> iwdev
->rf
->sc_dev
.hw_attrs
.max_mr_size
)
2749 return ERR_PTR(-EINVAL
);
2751 region
= ib_umem_get(pd
->device
, start
, len
, access
);
2753 if (IS_ERR(region
)) {
2754 ibdev_dbg(&iwdev
->ibdev
,
2755 "VERBS: Failed to create ib_umem region\n");
2756 return (struct ib_mr
*)region
;
2759 if (ib_copy_from_udata(&req
, udata
, min(sizeof(req
), udata
->inlen
))) {
2760 ib_umem_release(region
);
2761 return ERR_PTR(-EFAULT
);
2764 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
2766 ib_umem_release(region
);
2767 return ERR_PTR(-ENOMEM
);
2770 iwpbl
= &iwmr
->iwpbl
;
2772 iwmr
->region
= region
;
2774 iwmr
->ibmr
.device
= pd
->device
;
2775 iwmr
->ibmr
.iova
= virt
;
2776 iwmr
->page_size
= PAGE_SIZE
;
2778 if (req
.reg_type
== IRDMA_MEMREG_TYPE_MEM
) {
2779 iwmr
->page_size
= ib_umem_find_best_pgsz(region
,
2780 SZ_4K
| SZ_2M
| SZ_1G
,
2782 if (unlikely(!iwmr
->page_size
)) {
2784 ib_umem_release(region
);
2785 return ERR_PTR(-EOPNOTSUPP
);
2788 iwmr
->len
= region
->length
;
2789 iwpbl
->user_base
= virt
;
2790 palloc
= &iwpbl
->pble_alloc
;
2791 iwmr
->type
= req
.reg_type
;
2792 iwmr
->page_cnt
= ib_umem_num_dma_blocks(region
, iwmr
->page_size
);
2794 switch (req
.reg_type
) {
2795 case IRDMA_MEMREG_TYPE_QP
:
2796 total
= req
.sq_pages
+ req
.rq_pages
+ shadow_pgcnt
;
2797 if (total
> iwmr
->page_cnt
) {
2801 total
= req
.sq_pages
+ req
.rq_pages
;
2802 use_pbles
= (total
> 2);
2803 err
= irdma_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
2807 ucontext
= rdma_udata_to_drv_context(udata
, struct irdma_ucontext
,
2809 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
2810 list_add_tail(&iwpbl
->list
, &ucontext
->qp_reg_mem_list
);
2811 iwpbl
->on_list
= true;
2812 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
2814 case IRDMA_MEMREG_TYPE_CQ
:
2815 if (iwdev
->rf
->sc_dev
.hw_attrs
.uk_attrs
.feature_flags
& IRDMA_FEATURE_CQ_RESIZE
)
2817 total
= req
.cq_pages
+ shadow_pgcnt
;
2818 if (total
> iwmr
->page_cnt
) {
2823 use_pbles
= (req
.cq_pages
> 1);
2824 err
= irdma_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
2828 ucontext
= rdma_udata_to_drv_context(udata
, struct irdma_ucontext
,
2830 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
2831 list_add_tail(&iwpbl
->list
, &ucontext
->cq_reg_mem_list
);
2832 iwpbl
->on_list
= true;
2833 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
2835 case IRDMA_MEMREG_TYPE_MEM
:
2836 use_pbles
= (iwmr
->page_cnt
!= 1);
2838 err
= irdma_setup_pbles(iwdev
->rf
, iwmr
, use_pbles
);
2843 ret
= irdma_check_mr_contiguous(palloc
,
2846 irdma_free_pble(iwdev
->rf
->pble_rsrc
, palloc
);
2847 iwpbl
->pbl_allocated
= false;
2851 stag
= irdma_create_stag(iwdev
);
2858 iwmr
->ibmr
.rkey
= stag
;
2859 iwmr
->ibmr
.lkey
= stag
;
2860 err
= irdma_hwreg_mr(iwdev
, iwmr
, access
);
2862 irdma_free_stag(iwdev
, stag
);
2871 iwmr
->type
= req
.reg_type
;
2876 if (palloc
->level
!= PBLE_LEVEL_0
&& iwpbl
->pbl_allocated
)
2877 irdma_free_pble(iwdev
->rf
->pble_rsrc
, palloc
);
2878 ib_umem_release(region
);
2881 return ERR_PTR(err
);
2885 * irdma_reg_phys_mr - register kernel physical memory
2887 * @addr: physical address of memory to register
2888 * @size: size of memory to register
2889 * @access: Access rights
2890 * @iova_start: start of virtual address for physical buffers
2892 struct ib_mr
*irdma_reg_phys_mr(struct ib_pd
*pd
, u64 addr
, u64 size
, int access
,
2895 struct irdma_device
*iwdev
= to_iwdev(pd
->device
);
2896 struct irdma_pbl
*iwpbl
;
2897 struct irdma_mr
*iwmr
;
2898 enum irdma_status_code status
;
2902 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
2904 return ERR_PTR(-ENOMEM
);
2907 iwmr
->ibmr
.device
= pd
->device
;
2908 iwpbl
= &iwmr
->iwpbl
;
2910 iwmr
->type
= IRDMA_MEMREG_TYPE_MEM
;
2911 iwpbl
->user_base
= *iova_start
;
2912 stag
= irdma_create_stag(iwdev
);
2919 iwmr
->ibmr
.iova
= *iova_start
;
2920 iwmr
->ibmr
.rkey
= stag
;
2921 iwmr
->ibmr
.lkey
= stag
;
2923 iwmr
->pgaddrmem
[0] = addr
;
2925 iwmr
->page_size
= SZ_4K
;
2926 status
= irdma_hwreg_mr(iwdev
, iwmr
, access
);
2928 irdma_free_stag(iwdev
, stag
);
2938 return ERR_PTR(ret
);
2942 * irdma_get_dma_mr - register physical mem
2944 * @acc: access for memory
2946 static struct ib_mr
*irdma_get_dma_mr(struct ib_pd
*pd
, int acc
)
2950 return irdma_reg_phys_mr(pd
, 0, 0, acc
, &kva
);
2954 * irdma_del_memlist - Deleting pbl list entries for CQ/QP
2955 * @iwmr: iwmr for IB's user page addresses
2956 * @ucontext: ptr to user context
2958 static void irdma_del_memlist(struct irdma_mr
*iwmr
,
2959 struct irdma_ucontext
*ucontext
)
2961 struct irdma_pbl
*iwpbl
= &iwmr
->iwpbl
;
2962 unsigned long flags
;
2964 switch (iwmr
->type
) {
2965 case IRDMA_MEMREG_TYPE_CQ
:
2966 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
2967 if (iwpbl
->on_list
) {
2968 iwpbl
->on_list
= false;
2969 list_del(&iwpbl
->list
);
2971 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
2973 case IRDMA_MEMREG_TYPE_QP
:
2974 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
2975 if (iwpbl
->on_list
) {
2976 iwpbl
->on_list
= false;
2977 list_del(&iwpbl
->list
);
2979 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
2987 * irdma_dereg_mr - deregister mr
2988 * @ib_mr: mr ptr for dereg
2991 static int irdma_dereg_mr(struct ib_mr
*ib_mr
, struct ib_udata
*udata
)
2993 struct ib_pd
*ibpd
= ib_mr
->pd
;
2994 struct irdma_pd
*iwpd
= to_iwpd(ibpd
);
2995 struct irdma_mr
*iwmr
= to_iwmr(ib_mr
);
2996 struct irdma_device
*iwdev
= to_iwdev(ib_mr
->device
);
2997 struct irdma_dealloc_stag_info
*info
;
2998 struct irdma_pbl
*iwpbl
= &iwmr
->iwpbl
;
2999 struct irdma_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
3000 struct irdma_cqp_request
*cqp_request
;
3001 struct cqp_cmds_info
*cqp_info
;
3003 if (iwmr
->type
!= IRDMA_MEMREG_TYPE_MEM
) {
3005 struct irdma_ucontext
*ucontext
;
3007 ucontext
= rdma_udata_to_drv_context(udata
,
3008 struct irdma_ucontext
,
3010 irdma_del_memlist(iwmr
, ucontext
);
3015 cqp_request
= irdma_alloc_and_get_cqp_request(&iwdev
->rf
->cqp
, true);
3019 cqp_info
= &cqp_request
->info
;
3020 info
= &cqp_info
->in
.u
.dealloc_stag
.info
;
3021 memset(info
, 0, sizeof(*info
));
3022 info
->pd_id
= iwpd
->sc_pd
.pd_id
& 0x00007fff;
3023 info
->stag_idx
= ib_mr
->rkey
>> IRDMA_CQPSQ_STAG_IDX_S
;
3025 if (iwpbl
->pbl_allocated
)
3026 info
->dealloc_pbl
= true;
3028 cqp_info
->cqp_cmd
= IRDMA_OP_DEALLOC_STAG
;
3029 cqp_info
->post_sq
= 1;
3030 cqp_info
->in
.u
.dealloc_stag
.dev
= &iwdev
->rf
->sc_dev
;
3031 cqp_info
->in
.u
.dealloc_stag
.scratch
= (uintptr_t)cqp_request
;
3032 irdma_handle_cqp_op(iwdev
->rf
, cqp_request
);
3033 irdma_put_cqp_request(&iwdev
->rf
->cqp
, cqp_request
);
3034 irdma_free_stag(iwdev
, iwmr
->stag
);
3036 if (iwpbl
->pbl_allocated
)
3037 irdma_free_pble(iwdev
->rf
->pble_rsrc
, palloc
);
3038 ib_umem_release(iwmr
->region
);
3045 * irdma_copy_sg_list - copy sg list for qp
3046 * @sg_list: copied into sg_list
3047 * @sgl: copy from sgl
3048 * @num_sges: count of sg entries
3050 static void irdma_copy_sg_list(struct irdma_sge
*sg_list
, struct ib_sge
*sgl
,
3055 for (i
= 0; (i
< num_sges
) && (i
< IRDMA_MAX_WQ_FRAGMENT_COUNT
); i
++) {
3056 sg_list
[i
].tag_off
= sgl
[i
].addr
;
3057 sg_list
[i
].len
= sgl
[i
].length
;
3058 sg_list
[i
].stag
= sgl
[i
].lkey
;
3063 * irdma_post_send - kernel application wr
3064 * @ibqp: qp ptr for wr
3065 * @ib_wr: work request ptr
3066 * @bad_wr: return of bad wr if err
3068 static int irdma_post_send(struct ib_qp
*ibqp
,
3069 const struct ib_send_wr
*ib_wr
,
3070 const struct ib_send_wr
**bad_wr
)
3072 struct irdma_qp
*iwqp
;
3073 struct irdma_qp_uk
*ukqp
;
3074 struct irdma_sc_dev
*dev
;
3075 struct irdma_post_sq_info info
;
3076 enum irdma_status_code ret
;
3078 unsigned long flags
;
3080 struct irdma_ah
*ah
;
3081 bool reflush
= false;
3083 iwqp
= to_iwqp(ibqp
);
3084 ukqp
= &iwqp
->sc_qp
.qp_uk
;
3085 dev
= &iwqp
->iwdev
->rf
->sc_dev
;
3087 spin_lock_irqsave(&iwqp
->lock
, flags
);
3088 if (iwqp
->flush_issued
&& ukqp
->sq_flush_complete
)
3091 memset(&info
, 0, sizeof(info
));
3093 info
.wr_id
= (ib_wr
->wr_id
);
3094 if ((ib_wr
->send_flags
& IB_SEND_SIGNALED
) || iwqp
->sig_all
)
3095 info
.signaled
= true;
3096 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
3097 info
.read_fence
= true;
3098 switch (ib_wr
->opcode
) {
3099 case IB_WR_SEND_WITH_IMM
:
3100 if (ukqp
->qp_caps
& IRDMA_SEND_WITH_IMM
) {
3101 info
.imm_data_valid
= true;
3102 info
.imm_data
= ntohl(ib_wr
->ex
.imm_data
);
3109 case IB_WR_SEND_WITH_INV
:
3110 if (ib_wr
->opcode
== IB_WR_SEND
||
3111 ib_wr
->opcode
== IB_WR_SEND_WITH_IMM
) {
3112 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
3113 info
.op_type
= IRDMA_OP_TYPE_SEND_SOL
;
3115 info
.op_type
= IRDMA_OP_TYPE_SEND
;
3117 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
3118 info
.op_type
= IRDMA_OP_TYPE_SEND_SOL_INV
;
3120 info
.op_type
= IRDMA_OP_TYPE_SEND_INV
;
3121 info
.stag_to_inv
= ib_wr
->ex
.invalidate_rkey
;
3124 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
3125 info
.op
.inline_send
.data
= (void *)(unsigned long)
3126 ib_wr
->sg_list
[0].addr
;
3127 info
.op
.inline_send
.len
= ib_wr
->sg_list
[0].length
;
3128 if (iwqp
->ibqp
.qp_type
== IB_QPT_UD
||
3129 iwqp
->ibqp
.qp_type
== IB_QPT_GSI
) {
3130 ah
= to_iwah(ud_wr(ib_wr
)->ah
);
3131 info
.op
.inline_send
.ah_id
= ah
->sc_ah
.ah_info
.ah_idx
;
3132 info
.op
.inline_send
.qkey
= ud_wr(ib_wr
)->remote_qkey
;
3133 info
.op
.inline_send
.dest_qp
= ud_wr(ib_wr
)->remote_qpn
;
3135 ret
= irdma_uk_inline_send(ukqp
, &info
, false);
3137 info
.op
.send
.num_sges
= ib_wr
->num_sge
;
3138 info
.op
.send
.sg_list
= (struct irdma_sge
*)
3140 if (iwqp
->ibqp
.qp_type
== IB_QPT_UD
||
3141 iwqp
->ibqp
.qp_type
== IB_QPT_GSI
) {
3142 ah
= to_iwah(ud_wr(ib_wr
)->ah
);
3143 info
.op
.send
.ah_id
= ah
->sc_ah
.ah_info
.ah_idx
;
3144 info
.op
.send
.qkey
= ud_wr(ib_wr
)->remote_qkey
;
3145 info
.op
.send
.dest_qp
= ud_wr(ib_wr
)->remote_qpn
;
3147 ret
= irdma_uk_send(ukqp
, &info
, false);
3151 if (ret
== IRDMA_ERR_QP_TOOMANY_WRS_POSTED
)
3157 case IB_WR_RDMA_WRITE_WITH_IMM
:
3158 if (ukqp
->qp_caps
& IRDMA_WRITE_WITH_IMM
) {
3159 info
.imm_data_valid
= true;
3160 info
.imm_data
= ntohl(ib_wr
->ex
.imm_data
);
3166 case IB_WR_RDMA_WRITE
:
3167 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
3168 info
.op_type
= IRDMA_OP_TYPE_RDMA_WRITE_SOL
;
3170 info
.op_type
= IRDMA_OP_TYPE_RDMA_WRITE
;
3172 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
3173 info
.op
.inline_rdma_write
.data
= (void *)(uintptr_t)ib_wr
->sg_list
[0].addr
;
3174 info
.op
.inline_rdma_write
.len
= ib_wr
->sg_list
[0].length
;
3175 info
.op
.inline_rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
3176 info
.op
.inline_rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
3177 ret
= irdma_uk_inline_rdma_write(ukqp
, &info
, false);
3179 info
.op
.rdma_write
.lo_sg_list
= (void *)ib_wr
->sg_list
;
3180 info
.op
.rdma_write
.num_lo_sges
= ib_wr
->num_sge
;
3181 info
.op
.rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
3182 info
.op
.rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
3183 ret
= irdma_uk_rdma_write(ukqp
, &info
, false);
3187 if (ret
== IRDMA_ERR_QP_TOOMANY_WRS_POSTED
)
3193 case IB_WR_RDMA_READ_WITH_INV
:
3196 case IB_WR_RDMA_READ
:
3197 if (ib_wr
->num_sge
>
3198 dev
->hw_attrs
.uk_attrs
.max_hw_read_sges
) {
3202 info
.op_type
= IRDMA_OP_TYPE_RDMA_READ
;
3203 info
.op
.rdma_read
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
3204 info
.op
.rdma_read
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
3205 info
.op
.rdma_read
.lo_sg_list
= (void *)ib_wr
->sg_list
;
3206 info
.op
.rdma_read
.num_lo_sges
= ib_wr
->num_sge
;
3208 ret
= irdma_uk_rdma_read(ukqp
, &info
, inv_stag
, false);
3210 if (ret
== IRDMA_ERR_QP_TOOMANY_WRS_POSTED
)
3216 case IB_WR_LOCAL_INV
:
3217 info
.op_type
= IRDMA_OP_TYPE_INV_STAG
;
3218 info
.op
.inv_local_stag
.target_stag
= ib_wr
->ex
.invalidate_rkey
;
3219 ret
= irdma_uk_stag_local_invalidate(ukqp
, &info
, true);
3223 case IB_WR_REG_MR
: {
3224 struct irdma_mr
*iwmr
= to_iwmr(reg_wr(ib_wr
)->mr
);
3225 struct irdma_pble_alloc
*palloc
= &iwmr
->iwpbl
.pble_alloc
;
3226 struct irdma_fast_reg_stag_info stag_info
= {};
3228 stag_info
.signaled
= info
.signaled
;
3229 stag_info
.read_fence
= info
.read_fence
;
3230 stag_info
.access_rights
= irdma_get_mr_access(reg_wr(ib_wr
)->access
);
3231 stag_info
.stag_key
= reg_wr(ib_wr
)->key
& 0xff;
3232 stag_info
.stag_idx
= reg_wr(ib_wr
)->key
>> 8;
3233 stag_info
.page_size
= reg_wr(ib_wr
)->mr
->page_size
;
3234 stag_info
.wr_id
= ib_wr
->wr_id
;
3235 stag_info
.addr_type
= IRDMA_ADDR_TYPE_VA_BASED
;
3236 stag_info
.va
= (void *)(uintptr_t)iwmr
->ibmr
.iova
;
3237 stag_info
.total_len
= iwmr
->ibmr
.length
;
3238 stag_info
.reg_addr_pa
= *palloc
->level1
.addr
;
3239 stag_info
.first_pm_pbl_index
= palloc
->level1
.idx
;
3240 stag_info
.local_fence
= ib_wr
->send_flags
& IB_SEND_FENCE
;
3241 if (iwmr
->npages
> IRDMA_MIN_PAGES_PER_FMR
)
3242 stag_info
.chunk_size
= 1;
3243 ret
= irdma_sc_mr_fast_register(&iwqp
->sc_qp
, &stag_info
,
3251 ibdev_dbg(&iwqp
->iwdev
->ibdev
,
3252 "VERBS: upost_send bad opcode = 0x%x\n",
3259 ib_wr
= ib_wr
->next
;
3262 if (!iwqp
->flush_issued
&& iwqp
->hw_iwarp_state
<= IRDMA_QP_STATE_RTS
) {
3263 irdma_uk_qp_post_wr(ukqp
);
3264 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
3265 } else if (reflush
) {
3266 ukqp
->sq_flush_complete
= false;
3267 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
3268 irdma_flush_wqes(iwqp
, IRDMA_FLUSH_SQ
| IRDMA_REFLUSH
);
3270 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
3279 * irdma_post_recv - post receive wr for kernel application
3280 * @ibqp: ib qp pointer
3281 * @ib_wr: work request for receive
3282 * @bad_wr: bad wr caused an error
3284 static int irdma_post_recv(struct ib_qp
*ibqp
,
3285 const struct ib_recv_wr
*ib_wr
,
3286 const struct ib_recv_wr
**bad_wr
)
3288 struct irdma_qp
*iwqp
;
3289 struct irdma_qp_uk
*ukqp
;
3290 struct irdma_post_rq_info post_recv
= {};
3291 struct irdma_sge sg_list
[IRDMA_MAX_WQ_FRAGMENT_COUNT
];
3292 enum irdma_status_code ret
= 0;
3293 unsigned long flags
;
3295 bool reflush
= false;
3297 iwqp
= to_iwqp(ibqp
);
3298 ukqp
= &iwqp
->sc_qp
.qp_uk
;
3300 spin_lock_irqsave(&iwqp
->lock
, flags
);
3301 if (iwqp
->flush_issued
&& ukqp
->rq_flush_complete
)
3304 post_recv
.num_sges
= ib_wr
->num_sge
;
3305 post_recv
.wr_id
= ib_wr
->wr_id
;
3306 irdma_copy_sg_list(sg_list
, ib_wr
->sg_list
, ib_wr
->num_sge
);
3307 post_recv
.sg_list
= sg_list
;
3308 ret
= irdma_uk_post_receive(ukqp
, &post_recv
);
3310 ibdev_dbg(&iwqp
->iwdev
->ibdev
,
3311 "VERBS: post_recv err %d\n", ret
);
3312 if (ret
== IRDMA_ERR_QP_TOOMANY_WRS_POSTED
)
3319 ib_wr
= ib_wr
->next
;
3324 ukqp
->rq_flush_complete
= false;
3325 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
3326 irdma_flush_wqes(iwqp
, IRDMA_FLUSH_RQ
| IRDMA_REFLUSH
);
3328 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
3338 * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status
3339 * @opcode: iwarp flush code
3341 static enum ib_wc_status
irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode
)
3344 case FLUSH_PROT_ERR
:
3345 return IB_WC_LOC_PROT_ERR
;
3346 case FLUSH_REM_ACCESS_ERR
:
3347 return IB_WC_REM_ACCESS_ERR
;
3348 case FLUSH_LOC_QP_OP_ERR
:
3349 return IB_WC_LOC_QP_OP_ERR
;
3350 case FLUSH_REM_OP_ERR
:
3351 return IB_WC_REM_OP_ERR
;
3352 case FLUSH_LOC_LEN_ERR
:
3353 return IB_WC_LOC_LEN_ERR
;
3354 case FLUSH_GENERAL_ERR
:
3355 return IB_WC_WR_FLUSH_ERR
;
3356 case FLUSH_FATAL_ERR
:
3358 return IB_WC_FATAL_ERR
;
3363 * irdma_process_cqe - process cqe info
3364 * @entry: processed cqe
3365 * @cq_poll_info: cqe info
3367 static void irdma_process_cqe(struct ib_wc
*entry
,
3368 struct irdma_cq_poll_info
*cq_poll_info
)
3370 struct irdma_qp
*iwqp
;
3371 struct irdma_sc_qp
*qp
;
3373 entry
->wc_flags
= 0;
3374 entry
->pkey_index
= 0;
3375 entry
->wr_id
= cq_poll_info
->wr_id
;
3377 qp
= cq_poll_info
->qp_handle
;
3378 iwqp
= qp
->qp_uk
.back_qp
;
3379 entry
->qp
= qp
->qp_uk
.back_qp
;
3381 if (cq_poll_info
->error
) {
3382 entry
->status
= (cq_poll_info
->comp_status
== IRDMA_COMPL_STATUS_FLUSHED
) ?
3383 irdma_flush_err_to_ib_wc_status(cq_poll_info
->minor_err
) : IB_WC_GENERAL_ERR
;
3385 entry
->vendor_err
= cq_poll_info
->major_err
<< 16 |
3386 cq_poll_info
->minor_err
;
3388 entry
->status
= IB_WC_SUCCESS
;
3389 if (cq_poll_info
->imm_valid
) {
3390 entry
->ex
.imm_data
= htonl(cq_poll_info
->imm_data
);
3391 entry
->wc_flags
|= IB_WC_WITH_IMM
;
3393 if (cq_poll_info
->ud_smac_valid
) {
3394 ether_addr_copy(entry
->smac
, cq_poll_info
->ud_smac
);
3395 entry
->wc_flags
|= IB_WC_WITH_SMAC
;
3398 if (cq_poll_info
->ud_vlan_valid
) {
3399 entry
->vlan_id
= cq_poll_info
->ud_vlan
& VLAN_VID_MASK
;
3400 entry
->wc_flags
|= IB_WC_WITH_VLAN
;
3401 entry
->sl
= cq_poll_info
->ud_vlan
>> VLAN_PRIO_SHIFT
;
3407 switch (cq_poll_info
->op_type
) {
3408 case IRDMA_OP_TYPE_RDMA_WRITE
:
3409 case IRDMA_OP_TYPE_RDMA_WRITE_SOL
:
3410 entry
->opcode
= IB_WC_RDMA_WRITE
;
3412 case IRDMA_OP_TYPE_RDMA_READ_INV_STAG
:
3413 case IRDMA_OP_TYPE_RDMA_READ
:
3414 entry
->opcode
= IB_WC_RDMA_READ
;
3416 case IRDMA_OP_TYPE_SEND_INV
:
3417 case IRDMA_OP_TYPE_SEND_SOL
:
3418 case IRDMA_OP_TYPE_SEND_SOL_INV
:
3419 case IRDMA_OP_TYPE_SEND
:
3420 entry
->opcode
= IB_WC_SEND
;
3422 case IRDMA_OP_TYPE_FAST_REG_NSMR
:
3423 entry
->opcode
= IB_WC_REG_MR
;
3425 case IRDMA_OP_TYPE_INV_STAG
:
3426 entry
->opcode
= IB_WC_LOCAL_INV
;
3428 case IRDMA_OP_TYPE_REC_IMM
:
3429 case IRDMA_OP_TYPE_REC
:
3430 entry
->opcode
= cq_poll_info
->op_type
== IRDMA_OP_TYPE_REC_IMM
?
3431 IB_WC_RECV_RDMA_WITH_IMM
: IB_WC_RECV
;
3432 if (qp
->qp_uk
.qp_type
!= IRDMA_QP_TYPE_ROCE_UD
&&
3433 cq_poll_info
->stag_invalid_set
) {
3434 entry
->ex
.invalidate_rkey
= cq_poll_info
->inv_stag
;
3435 entry
->wc_flags
|= IB_WC_WITH_INVALIDATE
;
3439 ibdev_err(&iwqp
->iwdev
->ibdev
,
3440 "Invalid opcode = %d in CQE\n", cq_poll_info
->op_type
);
3441 entry
->status
= IB_WC_GENERAL_ERR
;
3445 if (qp
->qp_uk
.qp_type
== IRDMA_QP_TYPE_ROCE_UD
) {
3446 entry
->src_qp
= cq_poll_info
->ud_src_qpn
;
3449 (IB_WC_GRH
| IB_WC_WITH_NETWORK_HDR_TYPE
);
3450 entry
->network_hdr_type
= cq_poll_info
->ipv4
?
3454 entry
->src_qp
= cq_poll_info
->qp_id
;
3457 entry
->byte_len
= cq_poll_info
->bytes_xfered
;
3461 * irdma_poll_one - poll one entry of the CQ
3462 * @ukcq: ukcq to poll
3463 * @cur_cqe: current CQE info to be filled in
3464 * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ
3466 * Returns the internal irdma device error code or 0 on success
3468 static inline int irdma_poll_one(struct irdma_cq_uk
*ukcq
,
3469 struct irdma_cq_poll_info
*cur_cqe
,
3470 struct ib_wc
*entry
)
3472 int ret
= irdma_uk_cq_poll_cmpl(ukcq
, cur_cqe
);
3477 irdma_process_cqe(entry
, cur_cqe
);
3483 * __irdma_poll_cq - poll cq for completion (kernel apps)
3485 * @num_entries: number of entries to poll
3486 * @entry: wr of a completed entry
3488 static int __irdma_poll_cq(struct irdma_cq
*iwcq
, int num_entries
, struct ib_wc
*entry
)
3490 struct list_head
*tmp_node
, *list_node
;
3491 struct irdma_cq_buf
*last_buf
= NULL
;
3492 struct irdma_cq_poll_info
*cur_cqe
= &iwcq
->cur_cqe
;
3493 struct irdma_cq_buf
*cq_buf
;
3494 enum irdma_status_code ret
;
3495 struct irdma_device
*iwdev
;
3496 struct irdma_cq_uk
*ukcq
;
3497 bool cq_new_cqe
= false;
3498 int resized_bufs
= 0;
3501 iwdev
= to_iwdev(iwcq
->ibcq
.device
);
3502 ukcq
= &iwcq
->sc_cq
.cq_uk
;
3504 /* go through the list of previously resized CQ buffers */
3505 list_for_each_safe(list_node
, tmp_node
, &iwcq
->resize_list
) {
3506 cq_buf
= container_of(list_node
, struct irdma_cq_buf
, list
);
3507 while (npolled
< num_entries
) {
3508 ret
= irdma_poll_one(&cq_buf
->cq_uk
, cur_cqe
, entry
+ npolled
);
3514 if (ret
== IRDMA_ERR_Q_EMPTY
)
3516 /* QP using the CQ is destroyed. Skip reporting this CQE */
3517 if (ret
== IRDMA_ERR_Q_DESTROYED
) {
3524 /* save the resized CQ buffer which received the last cqe */
3530 /* check the current CQ for new cqes */
3531 while (npolled
< num_entries
) {
3532 ret
= irdma_poll_one(ukcq
, cur_cqe
, entry
+ npolled
);
3539 if (ret
== IRDMA_ERR_Q_EMPTY
)
3541 /* QP using the CQ is destroyed. Skip reporting this CQE */
3542 if (ret
== IRDMA_ERR_Q_DESTROYED
) {
3550 /* all previous CQ resizes are complete */
3551 resized_bufs
= irdma_process_resize_list(iwcq
, iwdev
, NULL
);
3553 /* only CQ resizes up to the last_buf are complete */
3554 resized_bufs
= irdma_process_resize_list(iwcq
, iwdev
, last_buf
);
3556 /* report to the HW the number of complete CQ resizes */
3557 irdma_uk_cq_set_resized_cnt(ukcq
, resized_bufs
);
3561 ibdev_dbg(&iwdev
->ibdev
, "%s: Error polling CQ, irdma_err: %d\n",
3568 * irdma_poll_cq - poll cq for completion (kernel apps)
3570 * @num_entries: number of entries to poll
3571 * @entry: wr of a completed entry
3573 static int irdma_poll_cq(struct ib_cq
*ibcq
, int num_entries
,
3574 struct ib_wc
*entry
)
3576 struct irdma_cq
*iwcq
;
3577 unsigned long flags
;
3580 iwcq
= to_iwcq(ibcq
);
3582 spin_lock_irqsave(&iwcq
->lock
, flags
);
3583 ret
= __irdma_poll_cq(iwcq
, num_entries
, entry
);
3584 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
3590 * irdma_req_notify_cq - arm cq kernel application
3592 * @notify_flags: notofication flags
3594 static int irdma_req_notify_cq(struct ib_cq
*ibcq
,
3595 enum ib_cq_notify_flags notify_flags
)
3597 struct irdma_cq
*iwcq
;
3598 struct irdma_cq_uk
*ukcq
;
3599 unsigned long flags
;
3600 enum irdma_cmpl_notify cq_notify
= IRDMA_CQ_COMPL_EVENT
;
3602 iwcq
= to_iwcq(ibcq
);
3603 ukcq
= &iwcq
->sc_cq
.cq_uk
;
3604 if (notify_flags
== IB_CQ_SOLICITED
)
3605 cq_notify
= IRDMA_CQ_COMPL_SOLICITED
;
3607 spin_lock_irqsave(&iwcq
->lock
, flags
);
3608 irdma_uk_cq_request_notification(ukcq
, cq_notify
);
3609 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
3614 static int irdma_roce_port_immutable(struct ib_device
*ibdev
, u32 port_num
,
3615 struct ib_port_immutable
*immutable
)
3617 struct ib_port_attr attr
;
3620 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP
;
3621 err
= ib_query_port(ibdev
, port_num
, &attr
);
3625 immutable
->max_mad_size
= IB_MGMT_MAD_SIZE
;
3626 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
3627 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
3632 static int irdma_iw_port_immutable(struct ib_device
*ibdev
, u32 port_num
,
3633 struct ib_port_immutable
*immutable
)
3635 struct ib_port_attr attr
;
3638 immutable
->core_cap_flags
= RDMA_CORE_PORT_IWARP
;
3639 err
= ib_query_port(ibdev
, port_num
, &attr
);
3642 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
3647 static const char *const irdma_hw_stat_names
[] = {
3649 [IRDMA_HW_STAT_INDEX_RXVLANERR
] = "rxVlanErrors",
3650 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD
] = "ip4InDiscards",
3651 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC
] = "ip4InTruncatedPkts",
3652 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE
] = "ip4OutNoRoutes",
3653 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD
] = "ip6InDiscards",
3654 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC
] = "ip6InTruncatedPkts",
3655 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE
] = "ip6OutNoRoutes",
3656 [IRDMA_HW_STAT_INDEX_TCPRTXSEG
] = "tcpRetransSegs",
3657 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR
] = "tcpInOptErrors",
3658 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR
] = "tcpInProtoErrors",
3659 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED
] = "cnpHandled",
3660 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED
] = "cnpIgnored",
3661 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT
] = "cnpSent",
3664 [IRDMA_HW_STAT_INDEX_IP4RXOCTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3666 [IRDMA_HW_STAT_INDEX_IP4RXPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3668 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3670 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3672 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3674 [IRDMA_HW_STAT_INDEX_IP4TXOCTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3676 [IRDMA_HW_STAT_INDEX_IP4TXPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3678 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3680 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3681 "ip4OutMcastOctets",
3682 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3684 [IRDMA_HW_STAT_INDEX_IP6RXOCTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3686 [IRDMA_HW_STAT_INDEX_IP6RXPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3688 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3690 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3692 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3694 [IRDMA_HW_STAT_INDEX_IP6TXOCTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3696 [IRDMA_HW_STAT_INDEX_IP6TXPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3698 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3700 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3701 "ip6OutMcastOctets",
3702 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3704 [IRDMA_HW_STAT_INDEX_TCPRXSEGS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3706 [IRDMA_HW_STAT_INDEX_TCPTXSEG
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3708 [IRDMA_HW_STAT_INDEX_RDMARXRDS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3710 [IRDMA_HW_STAT_INDEX_RDMARXSNDS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3712 [IRDMA_HW_STAT_INDEX_RDMARXWRS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3714 [IRDMA_HW_STAT_INDEX_RDMATXRDS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3716 [IRDMA_HW_STAT_INDEX_RDMATXSNDS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3718 [IRDMA_HW_STAT_INDEX_RDMATXWRS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3720 [IRDMA_HW_STAT_INDEX_RDMAVBND
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3722 [IRDMA_HW_STAT_INDEX_RDMAVINV
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3724 [IRDMA_HW_STAT_INDEX_UDPRXPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3726 [IRDMA_HW_STAT_INDEX_UDPTXPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3728 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS
+ IRDMA_HW_STAT_INDEX_MAX_32
] =
3732 static void irdma_get_dev_fw_str(struct ib_device
*dev
, char *str
)
3734 struct irdma_device
*iwdev
= to_iwdev(dev
);
3736 snprintf(str
, IB_FW_VERSION_NAME_MAX
, "%u.%u",
3737 irdma_fw_major_ver(&iwdev
->rf
->sc_dev
),
3738 irdma_fw_minor_ver(&iwdev
->rf
->sc_dev
));
3742 * irdma_alloc_hw_port_stats - Allocate a hw stats structure
3743 * @ibdev: device pointer from stack
3744 * @port_num: port number
3746 static struct rdma_hw_stats
*irdma_alloc_hw_port_stats(struct ib_device
*ibdev
,
3749 int num_counters
= IRDMA_HW_STAT_INDEX_MAX_32
+
3750 IRDMA_HW_STAT_INDEX_MAX_64
;
3751 unsigned long lifespan
= RDMA_HW_STATS_DEFAULT_LIFESPAN
;
3753 BUILD_BUG_ON(ARRAY_SIZE(irdma_hw_stat_names
) !=
3754 (IRDMA_HW_STAT_INDEX_MAX_32
+ IRDMA_HW_STAT_INDEX_MAX_64
));
3756 return rdma_alloc_hw_stats_struct(irdma_hw_stat_names
, num_counters
,
3761 * irdma_get_hw_stats - Populates the rdma_hw_stats structure
3762 * @ibdev: device pointer from stack
3763 * @stats: stats pointer from stack
3764 * @port_num: port number
3765 * @index: which hw counter the stack is requesting we update
3767 static int irdma_get_hw_stats(struct ib_device
*ibdev
,
3768 struct rdma_hw_stats
*stats
, u32 port_num
,
3771 struct irdma_device
*iwdev
= to_iwdev(ibdev
);
3772 struct irdma_dev_hw_stats
*hw_stats
= &iwdev
->vsi
.pestat
->hw_stats
;
3774 if (iwdev
->rf
->rdma_ver
>= IRDMA_GEN_2
)
3775 irdma_cqp_gather_stats_cmd(&iwdev
->rf
->sc_dev
, iwdev
->vsi
.pestat
, true);
3777 irdma_cqp_gather_stats_gen1(&iwdev
->rf
->sc_dev
, iwdev
->vsi
.pestat
);
3779 memcpy(&stats
->value
[0], hw_stats
, sizeof(*hw_stats
));
3781 return stats
->num_counters
;
3785 * irdma_query_gid - Query port GID
3786 * @ibdev: device pointer from stack
3787 * @port: port number
3788 * @index: Entry index
3791 static int irdma_query_gid(struct ib_device
*ibdev
, u32 port
, int index
,
3794 struct irdma_device
*iwdev
= to_iwdev(ibdev
);
3796 memset(gid
->raw
, 0, sizeof(gid
->raw
));
3797 ether_addr_copy(gid
->raw
, iwdev
->netdev
->dev_addr
);
3803 * mcast_list_add - Add a new mcast item to list
3804 * @rf: RDMA PCI function
3805 * @new_elem: pointer to element to add
3807 static void mcast_list_add(struct irdma_pci_f
*rf
,
3808 struct mc_table_list
*new_elem
)
3810 list_add(&new_elem
->list
, &rf
->mc_qht_list
.list
);
3814 * mcast_list_del - Remove an mcast item from list
3815 * @mc_qht_elem: pointer to mcast table list element
3817 static void mcast_list_del(struct mc_table_list
*mc_qht_elem
)
3820 list_del(&mc_qht_elem
->list
);
3824 * mcast_list_lookup_ip - Search mcast list for address
3825 * @rf: RDMA PCI function
3826 * @ip_mcast: pointer to mcast IP address
3828 static struct mc_table_list
*mcast_list_lookup_ip(struct irdma_pci_f
*rf
,
3831 struct mc_table_list
*mc_qht_el
;
3832 struct list_head
*pos
, *q
;
3834 list_for_each_safe (pos
, q
, &rf
->mc_qht_list
.list
) {
3835 mc_qht_el
= list_entry(pos
, struct mc_table_list
, list
);
3836 if (!memcmp(mc_qht_el
->mc_info
.dest_ip
, ip_mcast
,
3837 sizeof(mc_qht_el
->mc_info
.dest_ip
)))
3845 * irdma_mcast_cqp_op - perform a mcast cqp operation
3846 * @iwdev: irdma device
3847 * @mc_grp_ctx: mcast group info
3850 * returns error status
3852 static int irdma_mcast_cqp_op(struct irdma_device
*iwdev
,
3853 struct irdma_mcast_grp_info
*mc_grp_ctx
, u8 op
)
3855 struct cqp_cmds_info
*cqp_info
;
3856 struct irdma_cqp_request
*cqp_request
;
3857 enum irdma_status_code status
;
3859 cqp_request
= irdma_alloc_and_get_cqp_request(&iwdev
->rf
->cqp
, true);
3863 cqp_request
->info
.in
.u
.mc_create
.info
= *mc_grp_ctx
;
3864 cqp_info
= &cqp_request
->info
;
3865 cqp_info
->cqp_cmd
= op
;
3866 cqp_info
->post_sq
= 1;
3867 cqp_info
->in
.u
.mc_create
.scratch
= (uintptr_t)cqp_request
;
3868 cqp_info
->in
.u
.mc_create
.cqp
= &iwdev
->rf
->cqp
.sc_cqp
;
3869 status
= irdma_handle_cqp_op(iwdev
->rf
, cqp_request
);
3870 irdma_put_cqp_request(&iwdev
->rf
->cqp
, cqp_request
);
3878 * irdma_mcast_mac - Get the multicast MAC for an IP address
3879 * @ip_addr: IPv4 or IPv6 address
3880 * @mac: pointer to result MAC address
3881 * @ipv4: flag indicating IPv4 or IPv6
3884 void irdma_mcast_mac(u32
*ip_addr
, u8
*mac
, bool ipv4
)
3886 u8
*ip
= (u8
*)ip_addr
;
3889 unsigned char mac4
[ETH_ALEN
] = {0x01, 0x00, 0x5E, 0x00,
3892 mac4
[3] = ip
[2] & 0x7F;
3895 ether_addr_copy(mac
, mac4
);
3897 unsigned char mac6
[ETH_ALEN
] = {0x33, 0x33, 0x00, 0x00,
3904 ether_addr_copy(mac
, mac6
);
3909 * irdma_attach_mcast - attach a qp to a multicast group
3911 * @ibgid: pointer to global ID
3914 * returns error status
3916 static int irdma_attach_mcast(struct ib_qp
*ibqp
, union ib_gid
*ibgid
, u16 lid
)
3918 struct irdma_qp
*iwqp
= to_iwqp(ibqp
);
3919 struct irdma_device
*iwdev
= iwqp
->iwdev
;
3920 struct irdma_pci_f
*rf
= iwdev
->rf
;
3921 struct mc_table_list
*mc_qht_elem
;
3922 struct irdma_mcast_grp_ctx_entry_info mcg_info
= {};
3923 unsigned long flags
;
3924 u32 ip_addr
[4] = {};
3931 struct sockaddr saddr
;
3932 struct sockaddr_in saddr_in
;
3933 struct sockaddr_in6 saddr_in6
;
3935 unsigned char dmac
[ETH_ALEN
];
3937 rdma_gid2ip((struct sockaddr
*)&sgid_addr
, ibgid
);
3939 if (!ipv6_addr_v4mapped((struct in6_addr
*)ibgid
)) {
3940 irdma_copy_ip_ntohl(ip_addr
,
3941 sgid_addr
.saddr_in6
.sin6_addr
.in6_u
.u6_addr32
);
3942 irdma_netdev_vlan_ipv6(ip_addr
, &vlan_id
, NULL
);
3944 ibdev_dbg(&iwdev
->ibdev
,
3945 "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp
->qp_num
,
3947 irdma_mcast_mac(ip_addr
, dmac
, false);
3949 ip_addr
[0] = ntohl(sgid_addr
.saddr_in
.sin_addr
.s_addr
);
3951 vlan_id
= irdma_get_vlan_ipv4(ip_addr
);
3952 irdma_mcast_mac(ip_addr
, dmac
, true);
3953 ibdev_dbg(&iwdev
->ibdev
,
3954 "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n",
3955 ibqp
->qp_num
, ip_addr
, dmac
);
3958 spin_lock_irqsave(&rf
->qh_list_lock
, flags
);
3959 mc_qht_elem
= mcast_list_lookup_ip(rf
, ip_addr
);
3961 struct irdma_dma_mem
*dma_mem_mc
;
3963 spin_unlock_irqrestore(&rf
->qh_list_lock
, flags
);
3964 mc_qht_elem
= kzalloc(sizeof(*mc_qht_elem
), GFP_KERNEL
);
3968 mc_qht_elem
->mc_info
.ipv4_valid
= ipv4
;
3969 memcpy(mc_qht_elem
->mc_info
.dest_ip
, ip_addr
,
3970 sizeof(mc_qht_elem
->mc_info
.dest_ip
));
3971 ret
= irdma_alloc_rsrc(rf
, rf
->allocated_mcgs
, rf
->max_mcg
,
3972 &mgn
, &rf
->next_mcg
);
3978 mc_qht_elem
->mc_info
.mgn
= mgn
;
3979 dma_mem_mc
= &mc_qht_elem
->mc_grp_ctx
.dma_mem_mc
;
3980 dma_mem_mc
->size
= ALIGN(sizeof(u64
) * IRDMA_MAX_MGS_PER_CTX
,
3981 IRDMA_HW_PAGE_SIZE
);
3982 dma_mem_mc
->va
= dma_alloc_coherent(rf
->hw
.device
,
3986 if (!dma_mem_mc
->va
) {
3987 irdma_free_rsrc(rf
, rf
->allocated_mcgs
, mgn
);
3992 mc_qht_elem
->mc_grp_ctx
.mg_id
= (u16
)mgn
;
3993 memcpy(mc_qht_elem
->mc_grp_ctx
.dest_ip_addr
, ip_addr
,
3994 sizeof(mc_qht_elem
->mc_grp_ctx
.dest_ip_addr
));
3995 mc_qht_elem
->mc_grp_ctx
.ipv4_valid
= ipv4
;
3996 mc_qht_elem
->mc_grp_ctx
.vlan_id
= vlan_id
;
3997 if (vlan_id
< VLAN_N_VID
)
3998 mc_qht_elem
->mc_grp_ctx
.vlan_valid
= true;
3999 mc_qht_elem
->mc_grp_ctx
.hmc_fcn_id
= iwdev
->vsi
.fcn_id
;
4000 mc_qht_elem
->mc_grp_ctx
.qs_handle
=
4001 iwqp
->sc_qp
.vsi
->qos
[iwqp
->sc_qp
.user_pri
].qs_handle
;
4002 ether_addr_copy(mc_qht_elem
->mc_grp_ctx
.dest_mac_addr
, dmac
);
4004 spin_lock_irqsave(&rf
->qh_list_lock
, flags
);
4005 mcast_list_add(rf
, mc_qht_elem
);
4007 if (mc_qht_elem
->mc_grp_ctx
.no_of_mgs
==
4008 IRDMA_MAX_MGS_PER_CTX
) {
4009 spin_unlock_irqrestore(&rf
->qh_list_lock
, flags
);
4014 mcg_info
.qp_id
= iwqp
->ibqp
.qp_num
;
4015 no_mgs
= mc_qht_elem
->mc_grp_ctx
.no_of_mgs
;
4016 irdma_sc_add_mcast_grp(&mc_qht_elem
->mc_grp_ctx
, &mcg_info
);
4017 spin_unlock_irqrestore(&rf
->qh_list_lock
, flags
);
4019 /* Only if there is a change do we need to modify or create */
4021 ret
= irdma_mcast_cqp_op(iwdev
, &mc_qht_elem
->mc_grp_ctx
,
4022 IRDMA_OP_MC_CREATE
);
4023 } else if (no_mgs
!= mc_qht_elem
->mc_grp_ctx
.no_of_mgs
) {
4024 ret
= irdma_mcast_cqp_op(iwdev
, &mc_qht_elem
->mc_grp_ctx
,
4025 IRDMA_OP_MC_MODIFY
);
4036 irdma_sc_del_mcast_grp(&mc_qht_elem
->mc_grp_ctx
, &mcg_info
);
4037 if (!mc_qht_elem
->mc_grp_ctx
.no_of_mgs
) {
4038 mcast_list_del(mc_qht_elem
);
4039 dma_free_coherent(rf
->hw
.device
,
4040 mc_qht_elem
->mc_grp_ctx
.dma_mem_mc
.size
,
4041 mc_qht_elem
->mc_grp_ctx
.dma_mem_mc
.va
,
4042 mc_qht_elem
->mc_grp_ctx
.dma_mem_mc
.pa
);
4043 mc_qht_elem
->mc_grp_ctx
.dma_mem_mc
.va
= NULL
;
4044 irdma_free_rsrc(rf
, rf
->allocated_mcgs
,
4045 mc_qht_elem
->mc_grp_ctx
.mg_id
);
4053 * irdma_detach_mcast - detach a qp from a multicast group
4055 * @ibgid: pointer to global ID
4058 * returns error status
4060 static int irdma_detach_mcast(struct ib_qp
*ibqp
, union ib_gid
*ibgid
, u16 lid
)
4062 struct irdma_qp
*iwqp
= to_iwqp(ibqp
);
4063 struct irdma_device
*iwdev
= iwqp
->iwdev
;
4064 struct irdma_pci_f
*rf
= iwdev
->rf
;
4065 u32 ip_addr
[4] = {};
4066 struct mc_table_list
*mc_qht_elem
;
4067 struct irdma_mcast_grp_ctx_entry_info mcg_info
= {};
4069 unsigned long flags
;
4071 struct sockaddr saddr
;
4072 struct sockaddr_in saddr_in
;
4073 struct sockaddr_in6 saddr_in6
;
4076 rdma_gid2ip((struct sockaddr
*)&sgid_addr
, ibgid
);
4077 if (!ipv6_addr_v4mapped((struct in6_addr
*)ibgid
))
4078 irdma_copy_ip_ntohl(ip_addr
,
4079 sgid_addr
.saddr_in6
.sin6_addr
.in6_u
.u6_addr32
);
4081 ip_addr
[0] = ntohl(sgid_addr
.saddr_in
.sin_addr
.s_addr
);
4083 spin_lock_irqsave(&rf
->qh_list_lock
, flags
);
4084 mc_qht_elem
= mcast_list_lookup_ip(rf
, ip_addr
);
4086 spin_unlock_irqrestore(&rf
->qh_list_lock
, flags
);
4087 ibdev_dbg(&iwdev
->ibdev
,
4088 "VERBS: address not found MCG\n");
4092 mcg_info
.qp_id
= iwqp
->ibqp
.qp_num
;
4093 irdma_sc_del_mcast_grp(&mc_qht_elem
->mc_grp_ctx
, &mcg_info
);
4094 if (!mc_qht_elem
->mc_grp_ctx
.no_of_mgs
) {
4095 mcast_list_del(mc_qht_elem
);
4096 spin_unlock_irqrestore(&rf
->qh_list_lock
, flags
);
4097 ret
= irdma_mcast_cqp_op(iwdev
, &mc_qht_elem
->mc_grp_ctx
,
4098 IRDMA_OP_MC_DESTROY
);
4100 ibdev_dbg(&iwdev
->ibdev
,
4101 "VERBS: failed MC_DESTROY MCG\n");
4102 spin_lock_irqsave(&rf
->qh_list_lock
, flags
);
4103 mcast_list_add(rf
, mc_qht_elem
);
4104 spin_unlock_irqrestore(&rf
->qh_list_lock
, flags
);
4108 dma_free_coherent(rf
->hw
.device
,
4109 mc_qht_elem
->mc_grp_ctx
.dma_mem_mc
.size
,
4110 mc_qht_elem
->mc_grp_ctx
.dma_mem_mc
.va
,
4111 mc_qht_elem
->mc_grp_ctx
.dma_mem_mc
.pa
);
4112 mc_qht_elem
->mc_grp_ctx
.dma_mem_mc
.va
= NULL
;
4113 irdma_free_rsrc(rf
, rf
->allocated_mcgs
,
4114 mc_qht_elem
->mc_grp_ctx
.mg_id
);
4117 spin_unlock_irqrestore(&rf
->qh_list_lock
, flags
);
4118 ret
= irdma_mcast_cqp_op(iwdev
, &mc_qht_elem
->mc_grp_ctx
,
4119 IRDMA_OP_MC_MODIFY
);
4121 ibdev_dbg(&iwdev
->ibdev
,
4122 "VERBS: failed Modify MCG\n");
4131 * irdma_create_ah - create address handle
4132 * @ibah: address handle
4133 * @attr: address handle attributes
4136 * returns 0 on success, error otherwise
4138 static int irdma_create_ah(struct ib_ah
*ibah
,
4139 struct rdma_ah_init_attr
*attr
,
4140 struct ib_udata
*udata
)
4142 struct irdma_pd
*pd
= to_iwpd(ibah
->pd
);
4143 struct irdma_ah
*ah
= container_of(ibah
, struct irdma_ah
, ibah
);
4144 struct rdma_ah_attr
*ah_attr
= attr
->ah_attr
;
4145 const struct ib_gid_attr
*sgid_attr
;
4146 struct irdma_device
*iwdev
= to_iwdev(ibah
->pd
->device
);
4147 struct irdma_pci_f
*rf
= iwdev
->rf
;
4148 struct irdma_sc_ah
*sc_ah
;
4150 struct irdma_ah_info
*ah_info
;
4151 struct irdma_create_ah_resp uresp
;
4153 struct sockaddr saddr
;
4154 struct sockaddr_in saddr_in
;
4155 struct sockaddr_in6 saddr_in6
;
4156 } sgid_addr
, dgid_addr
;
4160 err
= irdma_alloc_rsrc(rf
, rf
->allocated_ahs
, rf
->max_ah
, &ah_id
,
4167 sc_ah
->ah_info
.ah_idx
= ah_id
;
4168 sc_ah
->ah_info
.vsi
= &iwdev
->vsi
;
4169 irdma_sc_init_ah(&rf
->sc_dev
, sc_ah
);
4170 ah
->sgid_index
= ah_attr
->grh
.sgid_index
;
4171 sgid_attr
= ah_attr
->grh
.sgid_attr
;
4172 memcpy(&ah
->dgid
, &ah_attr
->grh
.dgid
, sizeof(ah
->dgid
));
4173 rdma_gid2ip((struct sockaddr
*)&sgid_addr
, &sgid_attr
->gid
);
4174 rdma_gid2ip((struct sockaddr
*)&dgid_addr
, &ah_attr
->grh
.dgid
);
4175 ah
->av
.attrs
= *ah_attr
;
4176 ah
->av
.net_type
= rdma_gid_attr_network_type(sgid_attr
);
4177 ah
->av
.sgid_addr
.saddr
= sgid_addr
.saddr
;
4178 ah
->av
.dgid_addr
.saddr
= dgid_addr
.saddr
;
4179 ah_info
= &sc_ah
->ah_info
;
4180 ah_info
->ah_idx
= ah_id
;
4181 ah_info
->pd_idx
= pd
->sc_pd
.pd_id
;
4182 if (ah_attr
->ah_flags
& IB_AH_GRH
) {
4183 ah_info
->flow_label
= ah_attr
->grh
.flow_label
;
4184 ah_info
->hop_ttl
= ah_attr
->grh
.hop_limit
;
4185 ah_info
->tc_tos
= ah_attr
->grh
.traffic_class
;
4188 ether_addr_copy(dmac
, ah_attr
->roce
.dmac
);
4189 if (rdma_gid_attr_network_type(sgid_attr
) == RDMA_NETWORK_IPV4
) {
4190 ah_info
->ipv4_valid
= true;
4191 ah_info
->dest_ip_addr
[0] =
4192 ntohl(dgid_addr
.saddr_in
.sin_addr
.s_addr
);
4193 ah_info
->src_ip_addr
[0] =
4194 ntohl(sgid_addr
.saddr_in
.sin_addr
.s_addr
);
4195 ah_info
->do_lpbk
= irdma_ipv4_is_lpb(ah_info
->src_ip_addr
[0],
4196 ah_info
->dest_ip_addr
[0]);
4197 if (ipv4_is_multicast(dgid_addr
.saddr_in
.sin_addr
.s_addr
)) {
4198 ah_info
->do_lpbk
= true;
4199 irdma_mcast_mac(ah_info
->dest_ip_addr
, dmac
, true);
4202 irdma_copy_ip_ntohl(ah_info
->dest_ip_addr
,
4203 dgid_addr
.saddr_in6
.sin6_addr
.in6_u
.u6_addr32
);
4204 irdma_copy_ip_ntohl(ah_info
->src_ip_addr
,
4205 sgid_addr
.saddr_in6
.sin6_addr
.in6_u
.u6_addr32
);
4206 ah_info
->do_lpbk
= irdma_ipv6_is_lpb(ah_info
->src_ip_addr
,
4207 ah_info
->dest_ip_addr
);
4208 if (rdma_is_multicast_addr(&dgid_addr
.saddr_in6
.sin6_addr
)) {
4209 ah_info
->do_lpbk
= true;
4210 irdma_mcast_mac(ah_info
->dest_ip_addr
, dmac
, false);
4214 err
= rdma_read_gid_l2_fields(sgid_attr
, &ah_info
->vlan_tag
,
4219 ah_info
->dst_arpindex
= irdma_add_arp(iwdev
->rf
, ah_info
->dest_ip_addr
,
4220 ah_info
->ipv4_valid
, dmac
);
4222 if (ah_info
->dst_arpindex
== -1) {
4227 if (ah_info
->vlan_tag
>= VLAN_N_VID
&& iwdev
->dcb
)
4228 ah_info
->vlan_tag
= 0;
4230 if (ah_info
->vlan_tag
< VLAN_N_VID
) {
4231 ah_info
->insert_vlan_tag
= true;
4232 ah_info
->vlan_tag
|=
4233 rt_tos2priority(ah_info
->tc_tos
) << VLAN_PRIO_SHIFT
;
4236 err
= irdma_ah_cqp_op(iwdev
->rf
, sc_ah
, IRDMA_OP_AH_CREATE
,
4237 attr
->flags
& RDMA_CREATE_AH_SLEEPABLE
,
4238 irdma_gsi_ud_qp_ah_cb
, sc_ah
);
4241 ibdev_dbg(&iwdev
->ibdev
,
4242 "VERBS: CQP-OP Create AH fail");
4246 if (!(attr
->flags
& RDMA_CREATE_AH_SLEEPABLE
)) {
4247 int cnt
= CQP_COMPL_WAIT_TIME_MS
* CQP_TIMEOUT_THRESHOLD
;
4250 irdma_cqp_ce_handler(rf
, &rf
->ccq
.sc_cq
);
4252 } while (!sc_ah
->ah_info
.ah_valid
&& --cnt
);
4255 ibdev_dbg(&iwdev
->ibdev
,
4256 "VERBS: CQP create AH timed out");
4263 uresp
.ah_id
= ah
->sc_ah
.ah_info
.ah_idx
;
4264 err
= ib_copy_to_udata(udata
, &uresp
,
4265 min(sizeof(uresp
), udata
->outlen
));
4270 irdma_free_rsrc(iwdev
->rf
, iwdev
->rf
->allocated_ahs
, ah_id
);
4276 * irdma_destroy_ah - Destroy address handle
4277 * @ibah: pointer to address handle
4278 * @ah_flags: flags for sleepable
4280 static int irdma_destroy_ah(struct ib_ah
*ibah
, u32 ah_flags
)
4282 struct irdma_device
*iwdev
= to_iwdev(ibah
->device
);
4283 struct irdma_ah
*ah
= to_iwah(ibah
);
4285 irdma_ah_cqp_op(iwdev
->rf
, &ah
->sc_ah
, IRDMA_OP_AH_DESTROY
,
4288 irdma_free_rsrc(iwdev
->rf
, iwdev
->rf
->allocated_ahs
,
4289 ah
->sc_ah
.ah_info
.ah_idx
);
4295 * irdma_query_ah - Query address handle
4296 * @ibah: pointer to address handle
4297 * @ah_attr: address handle attributes
4299 static int irdma_query_ah(struct ib_ah
*ibah
, struct rdma_ah_attr
*ah_attr
)
4301 struct irdma_ah
*ah
= to_iwah(ibah
);
4303 memset(ah_attr
, 0, sizeof(*ah_attr
));
4304 if (ah
->av
.attrs
.ah_flags
& IB_AH_GRH
) {
4305 ah_attr
->ah_flags
= IB_AH_GRH
;
4306 ah_attr
->grh
.flow_label
= ah
->sc_ah
.ah_info
.flow_label
;
4307 ah_attr
->grh
.traffic_class
= ah
->sc_ah
.ah_info
.tc_tos
;
4308 ah_attr
->grh
.hop_limit
= ah
->sc_ah
.ah_info
.hop_ttl
;
4309 ah_attr
->grh
.sgid_index
= ah
->sgid_index
;
4310 ah_attr
->grh
.sgid_index
= ah
->sgid_index
;
4311 memcpy(&ah_attr
->grh
.dgid
, &ah
->dgid
,
4312 sizeof(ah_attr
->grh
.dgid
));
4318 static enum rdma_link_layer
irdma_get_link_layer(struct ib_device
*ibdev
,
4321 return IB_LINK_LAYER_ETHERNET
;
4324 static __be64
irdma_mac_to_guid(struct net_device
*ndev
)
4326 unsigned char *mac
= ndev
->dev_addr
;
4328 unsigned char *dst
= (unsigned char *)&guid
;
4330 dst
[0] = mac
[0] ^ 2;
4342 static const struct ib_device_ops irdma_roce_dev_ops
= {
4343 .attach_mcast
= irdma_attach_mcast
,
4344 .create_ah
= irdma_create_ah
,
4345 .create_user_ah
= irdma_create_ah
,
4346 .destroy_ah
= irdma_destroy_ah
,
4347 .detach_mcast
= irdma_detach_mcast
,
4348 .get_link_layer
= irdma_get_link_layer
,
4349 .get_port_immutable
= irdma_roce_port_immutable
,
4350 .modify_qp
= irdma_modify_qp_roce
,
4351 .query_ah
= irdma_query_ah
,
4352 .query_pkey
= irdma_query_pkey
,
4355 static const struct ib_device_ops irdma_iw_dev_ops
= {
4356 .modify_qp
= irdma_modify_qp
,
4357 .get_port_immutable
= irdma_iw_port_immutable
,
4358 .query_gid
= irdma_query_gid
,
4361 static const struct ib_device_ops irdma_dev_ops
= {
4362 .owner
= THIS_MODULE
,
4363 .driver_id
= RDMA_DRIVER_IRDMA
,
4364 .uverbs_abi_ver
= IRDMA_ABI_VER
,
4366 .alloc_hw_port_stats
= irdma_alloc_hw_port_stats
,
4367 .alloc_mr
= irdma_alloc_mr
,
4368 .alloc_mw
= irdma_alloc_mw
,
4369 .alloc_pd
= irdma_alloc_pd
,
4370 .alloc_ucontext
= irdma_alloc_ucontext
,
4371 .create_cq
= irdma_create_cq
,
4372 .create_qp
= irdma_create_qp
,
4373 .dealloc_driver
= irdma_ib_dealloc_device
,
4374 .dealloc_mw
= irdma_dealloc_mw
,
4375 .dealloc_pd
= irdma_dealloc_pd
,
4376 .dealloc_ucontext
= irdma_dealloc_ucontext
,
4377 .dereg_mr
= irdma_dereg_mr
,
4378 .destroy_cq
= irdma_destroy_cq
,
4379 .destroy_qp
= irdma_destroy_qp
,
4380 .disassociate_ucontext
= irdma_disassociate_ucontext
,
4381 .get_dev_fw_str
= irdma_get_dev_fw_str
,
4382 .get_dma_mr
= irdma_get_dma_mr
,
4383 .get_hw_stats
= irdma_get_hw_stats
,
4384 .map_mr_sg
= irdma_map_mr_sg
,
4386 .mmap_free
= irdma_mmap_free
,
4387 .poll_cq
= irdma_poll_cq
,
4388 .post_recv
= irdma_post_recv
,
4389 .post_send
= irdma_post_send
,
4390 .query_device
= irdma_query_device
,
4391 .query_port
= irdma_query_port
,
4392 .query_qp
= irdma_query_qp
,
4393 .reg_user_mr
= irdma_reg_user_mr
,
4394 .req_notify_cq
= irdma_req_notify_cq
,
4395 .resize_cq
= irdma_resize_cq
,
4396 INIT_RDMA_OBJ_SIZE(ib_pd
, irdma_pd
, ibpd
),
4397 INIT_RDMA_OBJ_SIZE(ib_ucontext
, irdma_ucontext
, ibucontext
),
4398 INIT_RDMA_OBJ_SIZE(ib_ah
, irdma_ah
, ibah
),
4399 INIT_RDMA_OBJ_SIZE(ib_cq
, irdma_cq
, ibcq
),
4400 INIT_RDMA_OBJ_SIZE(ib_mw
, irdma_mr
, ibmw
),
4401 INIT_RDMA_OBJ_SIZE(ib_qp
, irdma_qp
, ibqp
),
4405 * irdma_init_roce_device - initialization of roce rdma device
4406 * @iwdev: irdma device
4408 static void irdma_init_roce_device(struct irdma_device
*iwdev
)
4410 iwdev
->ibdev
.node_type
= RDMA_NODE_IB_CA
;
4411 iwdev
->ibdev
.node_guid
= irdma_mac_to_guid(iwdev
->netdev
);
4412 ib_set_device_ops(&iwdev
->ibdev
, &irdma_roce_dev_ops
);
4416 * irdma_init_iw_device - initialization of iwarp rdma device
4417 * @iwdev: irdma device
4419 static int irdma_init_iw_device(struct irdma_device
*iwdev
)
4421 struct net_device
*netdev
= iwdev
->netdev
;
4423 iwdev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
4424 ether_addr_copy((u8
*)&iwdev
->ibdev
.node_guid
, netdev
->dev_addr
);
4425 iwdev
->ibdev
.ops
.iw_add_ref
= irdma_qp_add_ref
;
4426 iwdev
->ibdev
.ops
.iw_rem_ref
= irdma_qp_rem_ref
;
4427 iwdev
->ibdev
.ops
.iw_get_qp
= irdma_get_qp
;
4428 iwdev
->ibdev
.ops
.iw_connect
= irdma_connect
;
4429 iwdev
->ibdev
.ops
.iw_accept
= irdma_accept
;
4430 iwdev
->ibdev
.ops
.iw_reject
= irdma_reject
;
4431 iwdev
->ibdev
.ops
.iw_create_listen
= irdma_create_listen
;
4432 iwdev
->ibdev
.ops
.iw_destroy_listen
= irdma_destroy_listen
;
4433 memcpy(iwdev
->ibdev
.iw_ifname
, netdev
->name
,
4434 sizeof(iwdev
->ibdev
.iw_ifname
));
4435 ib_set_device_ops(&iwdev
->ibdev
, &irdma_iw_dev_ops
);
4441 * irdma_init_rdma_device - initialization of rdma device
4442 * @iwdev: irdma device
4444 static int irdma_init_rdma_device(struct irdma_device
*iwdev
)
4446 struct pci_dev
*pcidev
= iwdev
->rf
->pcidev
;
4449 if (iwdev
->roce_mode
) {
4450 irdma_init_roce_device(iwdev
);
4452 ret
= irdma_init_iw_device(iwdev
);
4456 iwdev
->ibdev
.phys_port_cnt
= 1;
4457 iwdev
->ibdev
.num_comp_vectors
= iwdev
->rf
->ceqs_count
;
4458 iwdev
->ibdev
.dev
.parent
= &pcidev
->dev
;
4459 ib_set_device_ops(&iwdev
->ibdev
, &irdma_dev_ops
);
4465 * irdma_port_ibevent - indicate port event
4466 * @iwdev: irdma device
4468 void irdma_port_ibevent(struct irdma_device
*iwdev
)
4470 struct ib_event event
;
4472 event
.device
= &iwdev
->ibdev
;
4473 event
.element
.port_num
= 1;
4475 iwdev
->iw_status
? IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
4476 ib_dispatch_event(&event
);
4480 * irdma_ib_unregister_device - unregister rdma device from IB
4482 * @iwdev: irdma device
4484 void irdma_ib_unregister_device(struct irdma_device
*iwdev
)
4486 iwdev
->iw_status
= 0;
4487 irdma_port_ibevent(iwdev
);
4488 ib_unregister_device(&iwdev
->ibdev
);
4492 * irdma_ib_register_device - register irdma device to IB core
4493 * @iwdev: irdma device
4495 int irdma_ib_register_device(struct irdma_device
*iwdev
)
4499 ret
= irdma_init_rdma_device(iwdev
);
4503 ret
= ib_device_set_netdev(&iwdev
->ibdev
, iwdev
->netdev
, 1);
4506 dma_set_max_seg_size(iwdev
->rf
->hw
.device
, UINT_MAX
);
4507 ret
= ib_register_device(&iwdev
->ibdev
, "irdma%d", iwdev
->rf
->hw
.device
);
4511 iwdev
->iw_status
= 1;
4512 irdma_port_ibevent(iwdev
);
4518 ibdev_dbg(&iwdev
->ibdev
, "VERBS: Register RDMA device fail\n");
4524 * irdma_ib_dealloc_device
4527 * callback from ibdev dealloc_driver to deallocate resources
4528 * unber irdma device
4530 void irdma_ib_dealloc_device(struct ib_device
*ibdev
)
4532 struct irdma_device
*iwdev
= to_iwdev(ibdev
);
4534 irdma_rt_deinit_hw(iwdev
);
4535 irdma_ctrl_deinit_hw(iwdev
->rf
);