1 /*******************************************************************************
3 * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 *******************************************************************************/
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/random.h>
38 #include <linux/highmem.h>
39 #include <linux/time.h>
40 #include <linux/hugetlb.h>
41 #include <asm/byteorder.h>
43 #include <rdma/ib_verbs.h>
44 #include <rdma/iw_cm.h>
45 #include <rdma/ib_user_verbs.h>
46 #include <rdma/ib_umem.h>
50 * i40iw_query_device - get device attributes
51 * @ibdev: device pointer from stack
52 * @props: returning device attributes
55 static int i40iw_query_device(struct ib_device
*ibdev
,
56 struct ib_device_attr
*props
,
57 struct ib_udata
*udata
)
59 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
61 if (udata
->inlen
|| udata
->outlen
)
63 memset(props
, 0, sizeof(*props
));
64 ether_addr_copy((u8
*)&props
->sys_image_guid
, iwdev
->netdev
->dev_addr
);
65 props
->fw_ver
= I40IW_FW_VERSION
;
66 props
->device_cap_flags
= iwdev
->device_cap_flags
;
67 props
->vendor_id
= iwdev
->ldev
->pcidev
->vendor
;
68 props
->vendor_part_id
= iwdev
->ldev
->pcidev
->device
;
69 props
->hw_ver
= (u32
)iwdev
->sc_dev
.hw_rev
;
70 props
->max_mr_size
= I40IW_MAX_OUTBOUND_MESSAGE_SIZE
;
71 props
->max_qp
= iwdev
->max_qp
- iwdev
->used_qps
;
72 props
->max_qp_wr
= (I40IW_MAX_WQ_ENTRIES
>> 2) - 1;
73 props
->max_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
74 props
->max_cq
= iwdev
->max_cq
- iwdev
->used_cqs
;
75 props
->max_cqe
= iwdev
->max_cqe
;
76 props
->max_mr
= iwdev
->max_mr
- iwdev
->used_mrs
;
77 props
->max_pd
= iwdev
->max_pd
- iwdev
->used_pds
;
78 props
->max_sge_rd
= I40IW_MAX_SGE_RD
;
79 props
->max_qp_rd_atom
= I40IW_MAX_IRD_SIZE
;
80 props
->max_qp_init_rd_atom
= props
->max_qp_rd_atom
;
81 props
->atomic_cap
= IB_ATOMIC_NONE
;
82 props
->max_map_per_fmr
= 1;
83 props
->max_fast_reg_page_list_len
= I40IW_MAX_PAGES_PER_FMR
;
88 * i40iw_query_port - get port attrubutes
89 * @ibdev: device pointer from stack
90 * @port: port number for query
91 * @props: returning device attributes
93 static int i40iw_query_port(struct ib_device
*ibdev
,
95 struct ib_port_attr
*props
)
97 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
98 struct net_device
*netdev
= iwdev
->netdev
;
100 memset(props
, 0, sizeof(*props
));
102 props
->max_mtu
= IB_MTU_4096
;
103 props
->active_mtu
= ib_mtu_int_to_enum(netdev
->mtu
);
106 if (netif_carrier_ok(iwdev
->netdev
))
107 props
->state
= IB_PORT_ACTIVE
;
109 props
->state
= IB_PORT_DOWN
;
110 props
->port_cap_flags
= IB_PORT_CM_SUP
| IB_PORT_REINIT_SUP
|
111 IB_PORT_VENDOR_CLASS_SUP
| IB_PORT_BOOT_MGMT_SUP
;
112 props
->gid_tbl_len
= 1;
113 props
->pkey_tbl_len
= 1;
114 props
->active_width
= IB_WIDTH_4X
;
115 props
->active_speed
= 1;
116 props
->max_msg_sz
= I40IW_MAX_OUTBOUND_MESSAGE_SIZE
;
121 * i40iw_alloc_ucontext - Allocate the user context data structure
122 * @ibdev: device pointer from stack
125 * This keeps track of all objects associated with a particular
128 static struct ib_ucontext
*i40iw_alloc_ucontext(struct ib_device
*ibdev
,
129 struct ib_udata
*udata
)
131 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
132 struct i40iw_alloc_ucontext_req req
;
133 struct i40iw_alloc_ucontext_resp uresp
;
134 struct i40iw_ucontext
*ucontext
;
136 if (ib_copy_from_udata(&req
, udata
, sizeof(req
)))
137 return ERR_PTR(-EINVAL
);
139 if (req
.userspace_ver
< 4 || req
.userspace_ver
> I40IW_ABI_VER
) {
140 i40iw_pr_err("Unsupported provider library version %u.\n", req
.userspace_ver
);
141 return ERR_PTR(-EINVAL
);
144 memset(&uresp
, 0, sizeof(uresp
));
145 uresp
.max_qps
= iwdev
->max_qp
;
146 uresp
.max_pds
= iwdev
->max_pd
;
147 uresp
.wq_size
= iwdev
->max_qp_wr
* 2;
148 uresp
.kernel_ver
= req
.userspace_ver
;
150 ucontext
= kzalloc(sizeof(*ucontext
), GFP_KERNEL
);
152 return ERR_PTR(-ENOMEM
);
154 ucontext
->iwdev
= iwdev
;
155 ucontext
->abi_ver
= req
.userspace_ver
;
157 if (ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
))) {
159 return ERR_PTR(-EFAULT
);
162 INIT_LIST_HEAD(&ucontext
->cq_reg_mem_list
);
163 spin_lock_init(&ucontext
->cq_reg_mem_list_lock
);
164 INIT_LIST_HEAD(&ucontext
->qp_reg_mem_list
);
165 spin_lock_init(&ucontext
->qp_reg_mem_list_lock
);
167 return &ucontext
->ibucontext
;
171 * i40iw_dealloc_ucontext - deallocate the user context data structure
172 * @context: user context created during alloc
174 static int i40iw_dealloc_ucontext(struct ib_ucontext
*context
)
176 struct i40iw_ucontext
*ucontext
= to_ucontext(context
);
179 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
180 if (!list_empty(&ucontext
->cq_reg_mem_list
)) {
181 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
184 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
185 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
186 if (!list_empty(&ucontext
->qp_reg_mem_list
)) {
187 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
190 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
197 * i40iw_mmap - user memory map
198 * @context: context created during alloc
199 * @vma: kernel info for user memory map
201 static int i40iw_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
203 struct i40iw_ucontext
*ucontext
;
207 ucontext
= to_ucontext(context
);
208 if (ucontext
->iwdev
->sc_dev
.is_pf
) {
209 db_addr_offset
= I40IW_DB_ADDR_OFFSET
;
210 push_offset
= I40IW_PUSH_OFFSET
;
212 vma
->vm_pgoff
+= I40IW_PF_FIRST_PUSH_PAGE_INDEX
- 1;
214 db_addr_offset
= I40IW_VF_DB_ADDR_OFFSET
;
215 push_offset
= I40IW_VF_PUSH_OFFSET
;
217 vma
->vm_pgoff
+= I40IW_VF_FIRST_PUSH_PAGE_INDEX
- 1;
220 vma
->vm_pgoff
+= db_addr_offset
>> PAGE_SHIFT
;
222 if (vma
->vm_pgoff
== (db_addr_offset
>> PAGE_SHIFT
)) {
223 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
224 vma
->vm_private_data
= ucontext
;
226 if ((vma
->vm_pgoff
- (push_offset
>> PAGE_SHIFT
)) % 2)
227 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
229 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
232 if (io_remap_pfn_range(vma
, vma
->vm_start
,
233 vma
->vm_pgoff
+ (pci_resource_start(ucontext
->iwdev
->ldev
->pcidev
, 0) >> PAGE_SHIFT
),
234 PAGE_SIZE
, vma
->vm_page_prot
))
241 * i40iw_alloc_push_page - allocate a push page for qp
242 * @iwdev: iwarp device
243 * @qp: hardware control qp
245 static void i40iw_alloc_push_page(struct i40iw_device
*iwdev
, struct i40iw_sc_qp
*qp
)
247 struct i40iw_cqp_request
*cqp_request
;
248 struct cqp_commands_info
*cqp_info
;
249 enum i40iw_status_code status
;
251 if (qp
->push_idx
!= I40IW_INVALID_PUSH_PAGE_INDEX
)
254 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
258 atomic_inc(&cqp_request
->refcount
);
260 cqp_info
= &cqp_request
->info
;
261 cqp_info
->cqp_cmd
= OP_MANAGE_PUSH_PAGE
;
262 cqp_info
->post_sq
= 1;
264 cqp_info
->in
.u
.manage_push_page
.info
.qs_handle
= qp
->qs_handle
;
265 cqp_info
->in
.u
.manage_push_page
.info
.free_page
= 0;
266 cqp_info
->in
.u
.manage_push_page
.cqp
= &iwdev
->cqp
.sc_cqp
;
267 cqp_info
->in
.u
.manage_push_page
.scratch
= (uintptr_t)cqp_request
;
269 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
271 qp
->push_idx
= cqp_request
->compl_info
.op_ret_val
;
273 i40iw_pr_err("CQP-OP Push page fail");
274 i40iw_put_cqp_request(&iwdev
->cqp
, cqp_request
);
278 * i40iw_dealloc_push_page - free a push page for qp
279 * @iwdev: iwarp device
280 * @qp: hardware control qp
282 static void i40iw_dealloc_push_page(struct i40iw_device
*iwdev
, struct i40iw_sc_qp
*qp
)
284 struct i40iw_cqp_request
*cqp_request
;
285 struct cqp_commands_info
*cqp_info
;
286 enum i40iw_status_code status
;
288 if (qp
->push_idx
== I40IW_INVALID_PUSH_PAGE_INDEX
)
291 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, false);
295 cqp_info
= &cqp_request
->info
;
296 cqp_info
->cqp_cmd
= OP_MANAGE_PUSH_PAGE
;
297 cqp_info
->post_sq
= 1;
299 cqp_info
->in
.u
.manage_push_page
.info
.push_idx
= qp
->push_idx
;
300 cqp_info
->in
.u
.manage_push_page
.info
.qs_handle
= qp
->qs_handle
;
301 cqp_info
->in
.u
.manage_push_page
.info
.free_page
= 1;
302 cqp_info
->in
.u
.manage_push_page
.cqp
= &iwdev
->cqp
.sc_cqp
;
303 cqp_info
->in
.u
.manage_push_page
.scratch
= (uintptr_t)cqp_request
;
305 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
307 qp
->push_idx
= I40IW_INVALID_PUSH_PAGE_INDEX
;
309 i40iw_pr_err("CQP-OP Push page fail");
313 * i40iw_alloc_pd - allocate protection domain
314 * @ibdev: device pointer from stack
315 * @context: user context created during alloc
318 static struct ib_pd
*i40iw_alloc_pd(struct ib_device
*ibdev
,
319 struct ib_ucontext
*context
,
320 struct ib_udata
*udata
)
322 struct i40iw_pd
*iwpd
;
323 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
324 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
325 struct i40iw_alloc_pd_resp uresp
;
326 struct i40iw_sc_pd
*sc_pd
;
327 struct i40iw_ucontext
*ucontext
;
332 return ERR_PTR(-ENODEV
);
334 err
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_pds
,
335 iwdev
->max_pd
, &pd_id
, &iwdev
->next_pd
);
337 i40iw_pr_err("alloc resource failed\n");
341 iwpd
= kzalloc(sizeof(*iwpd
), GFP_KERNEL
);
347 sc_pd
= &iwpd
->sc_pd
;
350 ucontext
= to_ucontext(context
);
351 dev
->iw_pd_ops
->pd_init(dev
, sc_pd
, pd_id
, ucontext
->abi_ver
);
352 memset(&uresp
, 0, sizeof(uresp
));
354 if (ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
))) {
359 dev
->iw_pd_ops
->pd_init(dev
, sc_pd
, pd_id
, -1);
362 i40iw_add_pdusecount(iwpd
);
367 i40iw_free_resource(iwdev
, iwdev
->allocated_pds
, pd_id
);
372 * i40iw_dealloc_pd - deallocate pd
373 * @ibpd: ptr of pd to be deallocated
375 static int i40iw_dealloc_pd(struct ib_pd
*ibpd
)
377 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
378 struct i40iw_device
*iwdev
= to_iwdev(ibpd
->device
);
380 i40iw_rem_pdusecount(iwpd
, iwdev
);
385 * i40iw_qp_roundup - return round up qp ring size
386 * @wr_ring_size: ring size to round up
388 static int i40iw_qp_roundup(u32 wr_ring_size
)
392 if (wr_ring_size
< I40IWQP_SW_MIN_WQSIZE
)
393 wr_ring_size
= I40IWQP_SW_MIN_WQSIZE
;
395 for (wr_ring_size
--; scount
<= 16; scount
*= 2)
396 wr_ring_size
|= wr_ring_size
>> scount
;
397 return ++wr_ring_size
;
401 * i40iw_get_pbl - Retrieve pbl from a list given a virtual
403 * @va: user virtual address
404 * @pbl_list: pbl list to search in (QP's or CQ's)
406 static struct i40iw_pbl
*i40iw_get_pbl(unsigned long va
,
407 struct list_head
*pbl_list
)
409 struct i40iw_pbl
*iwpbl
;
411 list_for_each_entry(iwpbl
, pbl_list
, list
) {
412 if (iwpbl
->user_base
== va
) {
413 list_del(&iwpbl
->list
);
421 * i40iw_free_qp_resources - free up memory resources for qp
422 * @iwdev: iwarp device
423 * @iwqp: qp ptr (user or kernel)
424 * @qp_num: qp number assigned
426 void i40iw_free_qp_resources(struct i40iw_device
*iwdev
,
427 struct i40iw_qp
*iwqp
,
430 i40iw_dealloc_push_page(iwdev
, &iwqp
->sc_qp
);
432 i40iw_free_resource(iwdev
, iwdev
->allocated_qps
, qp_num
);
433 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwqp
->q2_ctx_mem
);
434 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwqp
->kqp
.dma_mem
);
435 kfree(iwqp
->kqp
.wrid_mem
);
436 iwqp
->kqp
.wrid_mem
= NULL
;
437 kfree(iwqp
->allocated_buffer
);
441 * i40iw_clean_cqes - clean cq entries for qp
442 * @iwqp: qp ptr (user or kernel)
445 static void i40iw_clean_cqes(struct i40iw_qp
*iwqp
, struct i40iw_cq
*iwcq
)
447 struct i40iw_cq_uk
*ukcq
= &iwcq
->sc_cq
.cq_uk
;
449 ukcq
->ops
.iw_cq_clean(&iwqp
->sc_qp
.qp_uk
, ukcq
);
453 * i40iw_destroy_qp - destroy qp
454 * @ibqp: qp's ib pointer also to get to device's qp address
456 static int i40iw_destroy_qp(struct ib_qp
*ibqp
)
458 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
462 if (iwqp
->ibqp_state
>= IB_QPS_INIT
&& iwqp
->ibqp_state
< IB_QPS_RTS
)
463 i40iw_next_iw_state(iwqp
, I40IW_QP_STATE_ERROR
, 0, 0, 0);
465 if (!iwqp
->user_mode
) {
467 i40iw_clean_cqes(iwqp
, iwqp
->iwscq
);
468 if (iwqp
->iwrcq
!= iwqp
->iwscq
)
469 i40iw_clean_cqes(iwqp
, iwqp
->iwrcq
);
473 i40iw_rem_ref(&iwqp
->ibqp
);
478 * i40iw_setup_virt_qp - setup for allocation of virtual qp
481 * @init_info: initialize info to return
483 static int i40iw_setup_virt_qp(struct i40iw_device
*iwdev
,
484 struct i40iw_qp
*iwqp
,
485 struct i40iw_qp_init_info
*init_info
)
487 struct i40iw_pbl
*iwpbl
= iwqp
->iwpbl
;
488 struct i40iw_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
490 iwqp
->page
= qpmr
->sq_page
;
491 init_info
->shadow_area_pa
= cpu_to_le64(qpmr
->shadow
);
492 if (iwpbl
->pbl_allocated
) {
493 init_info
->virtual_map
= true;
494 init_info
->sq_pa
= qpmr
->sq_pbl
.idx
;
495 init_info
->rq_pa
= qpmr
->rq_pbl
.idx
;
497 init_info
->sq_pa
= qpmr
->sq_pbl
.addr
;
498 init_info
->rq_pa
= qpmr
->rq_pbl
.addr
;
504 * i40iw_setup_kmode_qp - setup initialization for kernel mode qp
505 * @iwdev: iwarp device
506 * @iwqp: qp ptr (user or kernel)
507 * @info: initialize info to return
509 static int i40iw_setup_kmode_qp(struct i40iw_device
*iwdev
,
510 struct i40iw_qp
*iwqp
,
511 struct i40iw_qp_init_info
*info
)
513 struct i40iw_dma_mem
*mem
= &iwqp
->kqp
.dma_mem
;
514 u32 sqdepth
, rqdepth
;
515 u32 sq_size
, rq_size
;
518 enum i40iw_status_code status
;
519 struct i40iw_qp_uk_init_info
*ukinfo
= &info
->qp_uk_init_info
;
521 sq_size
= i40iw_qp_roundup(ukinfo
->sq_size
+ 1);
522 rq_size
= i40iw_qp_roundup(ukinfo
->rq_size
+ 1);
524 status
= i40iw_get_wqe_shift(sq_size
, ukinfo
->max_sq_frag_cnt
, ukinfo
->max_inline_data
, &sqshift
);
528 sqdepth
= sq_size
<< sqshift
;
529 rqdepth
= rq_size
<< I40IW_MAX_RQ_WQE_SHIFT
;
531 size
= sqdepth
* sizeof(struct i40iw_sq_uk_wr_trk_info
) + (rqdepth
<< 3);
532 iwqp
->kqp
.wrid_mem
= kzalloc(size
, GFP_KERNEL
);
534 ukinfo
->sq_wrtrk_array
= (struct i40iw_sq_uk_wr_trk_info
*)iwqp
->kqp
.wrid_mem
;
535 if (!ukinfo
->sq_wrtrk_array
)
538 ukinfo
->rq_wrid_array
= (u64
*)&ukinfo
->sq_wrtrk_array
[sqdepth
];
540 size
= (sqdepth
+ rqdepth
) * I40IW_QP_WQE_MIN_SIZE
;
541 size
+= (I40IW_SHADOW_AREA_SIZE
<< 3);
543 status
= i40iw_allocate_dma_mem(iwdev
->sc_dev
.hw
, mem
, size
, 256);
545 kfree(ukinfo
->sq_wrtrk_array
);
546 ukinfo
->sq_wrtrk_array
= NULL
;
550 ukinfo
->sq
= mem
->va
;
551 info
->sq_pa
= mem
->pa
;
553 ukinfo
->rq
= &ukinfo
->sq
[sqdepth
];
554 info
->rq_pa
= info
->sq_pa
+ (sqdepth
* I40IW_QP_WQE_MIN_SIZE
);
556 ukinfo
->shadow_area
= ukinfo
->rq
[rqdepth
].elem
;
557 info
->shadow_area_pa
= info
->rq_pa
+ (rqdepth
* I40IW_QP_WQE_MIN_SIZE
);
559 ukinfo
->sq_size
= sq_size
;
560 ukinfo
->rq_size
= rq_size
;
561 ukinfo
->qp_id
= iwqp
->ibqp
.qp_num
;
566 * i40iw_create_qp - create qp
568 * @init_attr: attributes for qp
569 * @udata: user data for create qp
571 static struct ib_qp
*i40iw_create_qp(struct ib_pd
*ibpd
,
572 struct ib_qp_init_attr
*init_attr
,
573 struct ib_udata
*udata
)
575 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
576 struct i40iw_device
*iwdev
= to_iwdev(ibpd
->device
);
577 struct i40iw_cqp
*iwcqp
= &iwdev
->cqp
;
578 struct i40iw_qp
*iwqp
;
579 struct i40iw_ucontext
*ucontext
;
580 struct i40iw_create_qp_req req
;
581 struct i40iw_create_qp_resp uresp
;
584 enum i40iw_status_code ret
;
588 struct i40iw_sc_qp
*qp
;
589 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
590 struct i40iw_qp_init_info init_info
;
591 struct i40iw_create_qp_info
*qp_info
;
592 struct i40iw_cqp_request
*cqp_request
;
593 struct cqp_commands_info
*cqp_info
;
595 struct i40iw_qp_host_ctx_info
*ctx_info
;
596 struct i40iwarp_offload_info
*iwarp_info
;
600 return ERR_PTR(-ENODEV
);
602 if (init_attr
->create_flags
)
603 return ERR_PTR(-EINVAL
);
604 if (init_attr
->cap
.max_inline_data
> I40IW_MAX_INLINE_DATA_SIZE
)
605 init_attr
->cap
.max_inline_data
= I40IW_MAX_INLINE_DATA_SIZE
;
607 if (init_attr
->cap
.max_send_sge
> I40IW_MAX_WQ_FRAGMENT_COUNT
)
608 init_attr
->cap
.max_send_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
610 if (init_attr
->cap
.max_recv_sge
> I40IW_MAX_WQ_FRAGMENT_COUNT
)
611 init_attr
->cap
.max_recv_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
613 memset(&init_info
, 0, sizeof(init_info
));
615 sq_size
= init_attr
->cap
.max_send_wr
;
616 rq_size
= init_attr
->cap
.max_recv_wr
;
618 init_info
.vsi
= &iwdev
->vsi
;
619 init_info
.qp_uk_init_info
.sq_size
= sq_size
;
620 init_info
.qp_uk_init_info
.rq_size
= rq_size
;
621 init_info
.qp_uk_init_info
.max_sq_frag_cnt
= init_attr
->cap
.max_send_sge
;
622 init_info
.qp_uk_init_info
.max_rq_frag_cnt
= init_attr
->cap
.max_recv_sge
;
623 init_info
.qp_uk_init_info
.max_inline_data
= init_attr
->cap
.max_inline_data
;
625 mem
= kzalloc(sizeof(*iwqp
), GFP_KERNEL
);
627 return ERR_PTR(-ENOMEM
);
629 iwqp
= (struct i40iw_qp
*)mem
;
631 qp
->back_qp
= (void *)iwqp
;
632 qp
->push_idx
= I40IW_INVALID_PUSH_PAGE_INDEX
;
634 iwqp
->ctx_info
.iwarp_info
= &iwqp
->iwarp_info
;
636 if (i40iw_allocate_dma_mem(dev
->hw
,
638 I40IW_Q2_BUFFER_SIZE
+ I40IW_QP_CTX_SIZE
,
640 i40iw_pr_err("dma_mem failed\n");
645 init_info
.q2
= iwqp
->q2_ctx_mem
.va
;
646 init_info
.q2_pa
= iwqp
->q2_ctx_mem
.pa
;
648 init_info
.host_ctx
= (void *)init_info
.q2
+ I40IW_Q2_BUFFER_SIZE
;
649 init_info
.host_ctx_pa
= init_info
.q2_pa
+ I40IW_Q2_BUFFER_SIZE
;
651 err_code
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_qps
, iwdev
->max_qp
,
652 &qp_num
, &iwdev
->next_qp
);
654 i40iw_pr_err("qp resource\n");
658 iwqp
->allocated_buffer
= mem
;
661 iwqp
->ibqp
.qp_num
= qp_num
;
663 iwqp
->iwscq
= to_iwcq(init_attr
->send_cq
);
664 iwqp
->iwrcq
= to_iwcq(init_attr
->recv_cq
);
666 iwqp
->host_ctx
.va
= init_info
.host_ctx
;
667 iwqp
->host_ctx
.pa
= init_info
.host_ctx_pa
;
668 iwqp
->host_ctx
.size
= I40IW_QP_CTX_SIZE
;
670 init_info
.pd
= &iwpd
->sc_pd
;
671 init_info
.qp_uk_init_info
.qp_id
= iwqp
->ibqp
.qp_num
;
672 iwqp
->ctx_info
.qp_compl_ctx
= (uintptr_t)qp
;
674 if (init_attr
->qp_type
!= IB_QPT_RC
) {
678 if (iwdev
->push_mode
)
679 i40iw_alloc_push_page(iwdev
, qp
);
681 err_code
= ib_copy_from_udata(&req
, udata
, sizeof(req
));
683 i40iw_pr_err("ib_copy_from_data\n");
686 iwqp
->ctx_info
.qp_compl_ctx
= req
.user_compl_ctx
;
687 if (ibpd
->uobject
&& ibpd
->uobject
->context
) {
689 ucontext
= to_ucontext(ibpd
->uobject
->context
);
691 if (req
.user_wqe_buffers
) {
693 &ucontext
->qp_reg_mem_list_lock
, flags
);
694 iwqp
->iwpbl
= i40iw_get_pbl(
695 (unsigned long)req
.user_wqe_buffers
,
696 &ucontext
->qp_reg_mem_list
);
697 spin_unlock_irqrestore(
698 &ucontext
->qp_reg_mem_list_lock
, flags
);
702 i40iw_pr_err("no pbl info\n");
707 err_code
= i40iw_setup_virt_qp(iwdev
, iwqp
, &init_info
);
709 err_code
= i40iw_setup_kmode_qp(iwdev
, iwqp
, &init_info
);
713 i40iw_pr_err("setup qp failed\n");
717 init_info
.type
= I40IW_QP_TYPE_IWARP
;
718 ret
= dev
->iw_priv_qp_ops
->qp_init(qp
, &init_info
);
721 i40iw_pr_err("qp_init fail\n");
724 ctx_info
= &iwqp
->ctx_info
;
725 iwarp_info
= &iwqp
->iwarp_info
;
726 iwarp_info
->rd_enable
= true;
727 iwarp_info
->wr_rdresp_en
= true;
728 if (!iwqp
->user_mode
) {
729 iwarp_info
->fast_reg_en
= true;
730 iwarp_info
->priv_mode_en
= true;
732 iwarp_info
->ddp_ver
= 1;
733 iwarp_info
->rdmap_ver
= 1;
735 ctx_info
->iwarp_info_valid
= true;
736 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
737 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
738 if (qp
->push_idx
== I40IW_INVALID_PUSH_PAGE_INDEX
) {
739 ctx_info
->push_mode_en
= false;
741 ctx_info
->push_mode_en
= true;
742 ctx_info
->push_idx
= qp
->push_idx
;
745 ret
= dev
->iw_priv_qp_ops
->qp_setctx(&iwqp
->sc_qp
,
746 (u64
*)iwqp
->host_ctx
.va
,
748 ctx_info
->iwarp_info_valid
= false;
749 cqp_request
= i40iw_get_cqp_request(iwcqp
, true);
754 cqp_info
= &cqp_request
->info
;
755 qp_info
= &cqp_request
->info
.in
.u
.qp_create
.info
;
757 memset(qp_info
, 0, sizeof(*qp_info
));
759 qp_info
->cq_num_valid
= true;
760 qp_info
->next_iwarp_state
= I40IW_QP_STATE_IDLE
;
762 cqp_info
->cqp_cmd
= OP_QP_CREATE
;
763 cqp_info
->post_sq
= 1;
764 cqp_info
->in
.u
.qp_create
.qp
= qp
;
765 cqp_info
->in
.u
.qp_create
.scratch
= (uintptr_t)cqp_request
;
766 ret
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
768 i40iw_pr_err("CQP-OP QP create fail");
773 i40iw_add_ref(&iwqp
->ibqp
);
774 spin_lock_init(&iwqp
->lock
);
775 iwqp
->sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
) ? 1 : 0;
776 iwdev
->qp_table
[qp_num
] = iwqp
;
777 i40iw_add_pdusecount(iwqp
->iwpd
);
778 i40iw_add_devusecount(iwdev
);
779 if (ibpd
->uobject
&& udata
) {
780 memset(&uresp
, 0, sizeof(uresp
));
781 uresp
.actual_sq_size
= sq_size
;
782 uresp
.actual_rq_size
= rq_size
;
783 uresp
.qp_id
= qp_num
;
784 uresp
.push_idx
= qp
->push_idx
;
785 err_code
= ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
));
787 i40iw_pr_err("copy_to_udata failed\n");
788 i40iw_destroy_qp(&iwqp
->ibqp
);
789 /* let the completion of the qp destroy free the qp */
790 return ERR_PTR(err_code
);
793 init_completion(&iwqp
->sq_drained
);
794 init_completion(&iwqp
->rq_drained
);
798 i40iw_free_qp_resources(iwdev
, iwqp
, qp_num
);
799 return ERR_PTR(err_code
);
803 * i40iw_query - query qp attributes
805 * @attr: attributes pointer
806 * @attr_mask: Not used
807 * @init_attr: qp attributes to return
809 static int i40iw_query_qp(struct ib_qp
*ibqp
,
810 struct ib_qp_attr
*attr
,
812 struct ib_qp_init_attr
*init_attr
)
814 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
815 struct i40iw_sc_qp
*qp
= &iwqp
->sc_qp
;
817 attr
->qp_access_flags
= 0;
818 attr
->cap
.max_send_wr
= qp
->qp_uk
.sq_size
;
819 attr
->cap
.max_recv_wr
= qp
->qp_uk
.rq_size
;
820 attr
->cap
.max_inline_data
= I40IW_MAX_INLINE_DATA_SIZE
;
821 attr
->cap
.max_send_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
822 attr
->cap
.max_recv_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
823 init_attr
->event_handler
= iwqp
->ibqp
.event_handler
;
824 init_attr
->qp_context
= iwqp
->ibqp
.qp_context
;
825 init_attr
->send_cq
= iwqp
->ibqp
.send_cq
;
826 init_attr
->recv_cq
= iwqp
->ibqp
.recv_cq
;
827 init_attr
->srq
= iwqp
->ibqp
.srq
;
828 init_attr
->cap
= attr
->cap
;
833 * i40iw_hw_modify_qp - setup cqp for modify qp
834 * @iwdev: iwarp device
835 * @iwqp: qp ptr (user or kernel)
836 * @info: info for modify qp
837 * @wait: flag to wait or not for modify qp completion
839 void i40iw_hw_modify_qp(struct i40iw_device
*iwdev
, struct i40iw_qp
*iwqp
,
840 struct i40iw_modify_qp_info
*info
, bool wait
)
842 enum i40iw_status_code status
;
843 struct i40iw_cqp_request
*cqp_request
;
844 struct cqp_commands_info
*cqp_info
;
845 struct i40iw_modify_qp_info
*m_info
;
847 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, wait
);
851 cqp_info
= &cqp_request
->info
;
852 m_info
= &cqp_info
->in
.u
.qp_modify
.info
;
853 memcpy(m_info
, info
, sizeof(*m_info
));
854 cqp_info
->cqp_cmd
= OP_QP_MODIFY
;
855 cqp_info
->post_sq
= 1;
856 cqp_info
->in
.u
.qp_modify
.qp
= &iwqp
->sc_qp
;
857 cqp_info
->in
.u
.qp_modify
.scratch
= (uintptr_t)cqp_request
;
858 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
860 i40iw_pr_err("CQP-OP Modify QP fail");
864 * i40iw_modify_qp - modify qp request
865 * @ibqp: qp's pointer for modify
866 * @attr: access attributes
867 * @attr_mask: state mask
870 int i40iw_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
871 int attr_mask
, struct ib_udata
*udata
)
873 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
874 struct i40iw_device
*iwdev
= iwqp
->iwdev
;
875 struct i40iw_qp_host_ctx_info
*ctx_info
;
876 struct i40iwarp_offload_info
*iwarp_info
;
877 struct i40iw_modify_qp_info info
;
878 u8 issue_modify_qp
= 0;
883 memset(&info
, 0, sizeof(info
));
884 ctx_info
= &iwqp
->ctx_info
;
885 iwarp_info
= &iwqp
->iwarp_info
;
887 spin_lock_irqsave(&iwqp
->lock
, flags
);
889 if (attr_mask
& IB_QP_STATE
) {
890 if (iwdev
->closing
&& attr
->qp_state
!= IB_QPS_ERR
) {
895 switch (attr
->qp_state
) {
898 if (iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_IDLE
) {
902 if (iwqp
->iwarp_state
== I40IW_QP_STATE_INVALID
) {
903 info
.next_iwarp_state
= I40IW_QP_STATE_IDLE
;
908 if ((iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_RTS
) ||
915 iwqp
->hw_tcp_state
= I40IW_TCP_STATE_ESTABLISHED
;
917 info
.next_iwarp_state
= I40IW_QP_STATE_RTS
;
918 info
.tcp_ctx_valid
= true;
919 info
.ord_valid
= true;
920 info
.arp_cache_idx_valid
= true;
921 info
.cq_num_valid
= true;
924 if (iwqp
->hw_iwarp_state
> (u32
)I40IW_QP_STATE_RTS
) {
928 if ((iwqp
->iwarp_state
== (u32
)I40IW_QP_STATE_CLOSING
) ||
929 (iwqp
->iwarp_state
< (u32
)I40IW_QP_STATE_RTS
)) {
933 if (iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_CLOSING
) {
937 info
.next_iwarp_state
= I40IW_QP_STATE_CLOSING
;
941 if (iwqp
->iwarp_state
>= (u32
)I40IW_QP_STATE_TERMINATE
) {
945 info
.next_iwarp_state
= I40IW_QP_STATE_TERMINATE
;
950 if (iwqp
->iwarp_state
== (u32
)I40IW_QP_STATE_ERROR
) {
954 if (iwqp
->sc_qp
.term_flags
)
955 i40iw_terminate_del_timer(&iwqp
->sc_qp
);
956 info
.next_iwarp_state
= I40IW_QP_STATE_ERROR
;
957 if ((iwqp
->hw_tcp_state
> I40IW_TCP_STATE_CLOSED
) &&
959 (iwqp
->hw_tcp_state
!= I40IW_TCP_STATE_TIME_WAIT
))
960 info
.reset_tcp_conn
= true;
964 info
.next_iwarp_state
= I40IW_QP_STATE_ERROR
;
971 iwqp
->ibqp_state
= attr
->qp_state
;
974 iwqp
->iwarp_state
= info
.next_iwarp_state
;
976 info
.next_iwarp_state
= iwqp
->iwarp_state
;
978 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
979 ctx_info
->iwarp_info_valid
= true;
980 if (attr
->qp_access_flags
& IB_ACCESS_LOCAL_WRITE
)
981 iwarp_info
->wr_rdresp_en
= true;
982 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
)
983 iwarp_info
->wr_rdresp_en
= true;
984 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
)
985 iwarp_info
->rd_enable
= true;
986 if (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
)
987 iwarp_info
->bind_en
= true;
989 if (iwqp
->user_mode
) {
990 iwarp_info
->rd_enable
= true;
991 iwarp_info
->wr_rdresp_en
= true;
992 iwarp_info
->priv_mode_en
= false;
996 if (ctx_info
->iwarp_info_valid
) {
997 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
1000 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
1001 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
1002 ret
= dev
->iw_priv_qp_ops
->qp_setctx(&iwqp
->sc_qp
,
1003 (u64
*)iwqp
->host_ctx
.va
,
1006 i40iw_pr_err("setting QP context\n");
1012 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1014 if (issue_modify_qp
)
1015 i40iw_hw_modify_qp(iwdev
, iwqp
, &info
, true);
1017 if (issue_modify_qp
&& (iwqp
->ibqp_state
> IB_QPS_RTS
)) {
1019 if (iwqp
->cm_id
&& iwqp
->hw_tcp_state
) {
1020 spin_lock_irqsave(&iwqp
->lock
, flags
);
1021 iwqp
->hw_tcp_state
= I40IW_TCP_STATE_CLOSED
;
1022 iwqp
->last_aeq
= I40IW_AE_RESET_SENT
;
1023 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1029 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1034 * cq_free_resources - free up recources for cq
1035 * @iwdev: iwarp device
1038 static void cq_free_resources(struct i40iw_device
*iwdev
, struct i40iw_cq
*iwcq
)
1040 struct i40iw_sc_cq
*cq
= &iwcq
->sc_cq
;
1042 if (!iwcq
->user_mode
)
1043 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwcq
->kmem
);
1044 i40iw_free_resource(iwdev
, iwdev
->allocated_cqs
, cq
->cq_uk
.cq_id
);
1048 * i40iw_cq_wq_destroy - send cq destroy cqp
1049 * @iwdev: iwarp device
1050 * @cq: hardware control cq
1052 void i40iw_cq_wq_destroy(struct i40iw_device
*iwdev
, struct i40iw_sc_cq
*cq
)
1054 enum i40iw_status_code status
;
1055 struct i40iw_cqp_request
*cqp_request
;
1056 struct cqp_commands_info
*cqp_info
;
1058 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1062 cqp_info
= &cqp_request
->info
;
1064 cqp_info
->cqp_cmd
= OP_CQ_DESTROY
;
1065 cqp_info
->post_sq
= 1;
1066 cqp_info
->in
.u
.cq_destroy
.cq
= cq
;
1067 cqp_info
->in
.u
.cq_destroy
.scratch
= (uintptr_t)cqp_request
;
1068 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1070 i40iw_pr_err("CQP-OP Destroy QP fail");
1074 * i40iw_destroy_cq - destroy cq
1075 * @ib_cq: cq pointer
1077 static int i40iw_destroy_cq(struct ib_cq
*ib_cq
)
1079 struct i40iw_cq
*iwcq
;
1080 struct i40iw_device
*iwdev
;
1081 struct i40iw_sc_cq
*cq
;
1084 i40iw_pr_err("ib_cq == NULL\n");
1088 iwcq
= to_iwcq(ib_cq
);
1089 iwdev
= to_iwdev(ib_cq
->device
);
1091 i40iw_cq_wq_destroy(iwdev
, cq
);
1092 cq_free_resources(iwdev
, iwcq
);
1094 i40iw_rem_devusecount(iwdev
);
1099 * i40iw_create_cq - create cq
1100 * @ibdev: device pointer from stack
1101 * @attr: attributes for cq
1102 * @context: user context created during alloc
1105 static struct ib_cq
*i40iw_create_cq(struct ib_device
*ibdev
,
1106 const struct ib_cq_init_attr
*attr
,
1107 struct ib_ucontext
*context
,
1108 struct ib_udata
*udata
)
1110 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
1111 struct i40iw_cq
*iwcq
;
1112 struct i40iw_pbl
*iwpbl
;
1114 struct i40iw_sc_cq
*cq
;
1115 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
1116 struct i40iw_cq_init_info info
;
1117 enum i40iw_status_code status
;
1118 struct i40iw_cqp_request
*cqp_request
;
1119 struct cqp_commands_info
*cqp_info
;
1120 struct i40iw_cq_uk_init_info
*ukinfo
= &info
.cq_uk_init_info
;
1121 unsigned long flags
;
1123 int entries
= attr
->cqe
;
1126 return ERR_PTR(-ENODEV
);
1128 if (entries
> iwdev
->max_cqe
)
1129 return ERR_PTR(-EINVAL
);
1131 iwcq
= kzalloc(sizeof(*iwcq
), GFP_KERNEL
);
1133 return ERR_PTR(-ENOMEM
);
1135 memset(&info
, 0, sizeof(info
));
1137 err_code
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_cqs
,
1138 iwdev
->max_cq
, &cq_num
,
1144 cq
->back_cq
= (void *)iwcq
;
1145 spin_lock_init(&iwcq
->lock
);
1148 ukinfo
->cq_size
= max(entries
, 4);
1149 ukinfo
->cq_id
= cq_num
;
1150 iwcq
->ibcq
.cqe
= info
.cq_uk_init_info
.cq_size
;
1152 if (attr
->comp_vector
< iwdev
->ceqs_count
)
1153 info
.ceq_id
= attr
->comp_vector
;
1154 info
.ceq_id_valid
= true;
1156 info
.type
= I40IW_CQ_TYPE_IWARP
;
1158 struct i40iw_ucontext
*ucontext
;
1159 struct i40iw_create_cq_req req
;
1160 struct i40iw_cq_mr
*cqmr
;
1162 memset(&req
, 0, sizeof(req
));
1163 iwcq
->user_mode
= true;
1164 ucontext
= to_ucontext(context
);
1165 if (ib_copy_from_udata(&req
, udata
, sizeof(struct i40iw_create_cq_req
)))
1166 goto cq_free_resources
;
1168 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1169 iwpbl
= i40iw_get_pbl((unsigned long)req
.user_cq_buffer
,
1170 &ucontext
->cq_reg_mem_list
);
1171 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1174 goto cq_free_resources
;
1177 iwcq
->iwpbl
= iwpbl
;
1178 iwcq
->cq_mem_size
= 0;
1179 cqmr
= &iwpbl
->cq_mr
;
1180 info
.shadow_area_pa
= cpu_to_le64(cqmr
->shadow
);
1181 if (iwpbl
->pbl_allocated
) {
1182 info
.virtual_map
= true;
1183 info
.pbl_chunk_size
= 1;
1184 info
.first_pm_pbl_idx
= cqmr
->cq_pbl
.idx
;
1186 info
.cq_base_pa
= cqmr
->cq_pbl
.addr
;
1189 /* Kmode allocations */
1193 rsize
= info
.cq_uk_init_info
.cq_size
* sizeof(struct i40iw_cqe
);
1194 rsize
= round_up(rsize
, 256);
1195 shadow
= I40IW_SHADOW_AREA_SIZE
<< 3;
1196 status
= i40iw_allocate_dma_mem(dev
->hw
, &iwcq
->kmem
,
1197 rsize
+ shadow
, 256);
1200 goto cq_free_resources
;
1202 ukinfo
->cq_base
= iwcq
->kmem
.va
;
1203 info
.cq_base_pa
= iwcq
->kmem
.pa
;
1204 info
.shadow_area_pa
= info
.cq_base_pa
+ rsize
;
1205 ukinfo
->shadow_area
= iwcq
->kmem
.va
+ rsize
;
1208 if (dev
->iw_priv_cq_ops
->cq_init(cq
, &info
)) {
1209 i40iw_pr_err("init cq fail\n");
1211 goto cq_free_resources
;
1214 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1217 goto cq_free_resources
;
1220 cqp_info
= &cqp_request
->info
;
1221 cqp_info
->cqp_cmd
= OP_CQ_CREATE
;
1222 cqp_info
->post_sq
= 1;
1223 cqp_info
->in
.u
.cq_create
.cq
= cq
;
1224 cqp_info
->in
.u
.cq_create
.scratch
= (uintptr_t)cqp_request
;
1225 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1227 i40iw_pr_err("CQP-OP Create QP fail");
1229 goto cq_free_resources
;
1233 struct i40iw_create_cq_resp resp
;
1235 memset(&resp
, 0, sizeof(resp
));
1236 resp
.cq_id
= info
.cq_uk_init_info
.cq_id
;
1237 resp
.cq_size
= info
.cq_uk_init_info
.cq_size
;
1238 if (ib_copy_to_udata(udata
, &resp
, sizeof(resp
))) {
1239 i40iw_pr_err("copy to user data\n");
1245 i40iw_add_devusecount(iwdev
);
1246 return (struct ib_cq
*)iwcq
;
1249 i40iw_cq_wq_destroy(iwdev
, cq
);
1251 cq_free_resources(iwdev
, iwcq
);
1254 return ERR_PTR(err_code
);
1258 * i40iw_get_user_access - get hw access from IB access
1259 * @acc: IB access to return hw access
1261 static inline u16
i40iw_get_user_access(int acc
)
1265 access
|= (acc
& IB_ACCESS_LOCAL_WRITE
) ? I40IW_ACCESS_FLAGS_LOCALWRITE
: 0;
1266 access
|= (acc
& IB_ACCESS_REMOTE_WRITE
) ? I40IW_ACCESS_FLAGS_REMOTEWRITE
: 0;
1267 access
|= (acc
& IB_ACCESS_REMOTE_READ
) ? I40IW_ACCESS_FLAGS_REMOTEREAD
: 0;
1268 access
|= (acc
& IB_ACCESS_MW_BIND
) ? I40IW_ACCESS_FLAGS_BIND_WINDOW
: 0;
1273 * i40iw_free_stag - free stag resource
1274 * @iwdev: iwarp device
1275 * @stag: stag to free
1277 static void i40iw_free_stag(struct i40iw_device
*iwdev
, u32 stag
)
1281 stag_idx
= (stag
& iwdev
->mr_stagmask
) >> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1282 i40iw_free_resource(iwdev
, iwdev
->allocated_mrs
, stag_idx
);
1283 i40iw_rem_devusecount(iwdev
);
1287 * i40iw_create_stag - create random stag
1288 * @iwdev: iwarp device
1290 static u32
i40iw_create_stag(struct i40iw_device
*iwdev
)
1294 u32 next_stag_index
;
1300 get_random_bytes(&random
, sizeof(random
));
1301 consumer_key
= (u8
)random
;
1303 driver_key
= random
& ~iwdev
->mr_stagmask
;
1304 next_stag_index
= (random
& iwdev
->mr_stagmask
) >> 8;
1305 next_stag_index
%= iwdev
->max_mr
;
1307 ret
= i40iw_alloc_resource(iwdev
,
1308 iwdev
->allocated_mrs
, iwdev
->max_mr
,
1309 &stag_index
, &next_stag_index
);
1311 stag
= stag_index
<< I40IW_CQPSQ_STAG_IDX_SHIFT
;
1313 stag
+= (u32
)consumer_key
;
1314 i40iw_add_devusecount(iwdev
);
1320 * i40iw_next_pbl_addr - Get next pbl address
1321 * @pbl: pointer to a pble
1322 * @pinfo: info pointer
1325 static inline u64
*i40iw_next_pbl_addr(u64
*pbl
,
1326 struct i40iw_pble_info
**pinfo
,
1330 if ((!(*pinfo
)) || (*idx
!= (*pinfo
)->cnt
))
1334 return (u64
*)(*pinfo
)->addr
;
1338 * i40iw_copy_user_pgaddrs - copy user page address to pble's os locally
1339 * @iwmr: iwmr for IB's user page addresses
1340 * @pbl: ple pointer to save 1 level or 0 level pble
1341 * @level: indicated level 0, 1 or 2
1343 static void i40iw_copy_user_pgaddrs(struct i40iw_mr
*iwmr
,
1345 enum i40iw_pble_level level
)
1347 struct ib_umem
*region
= iwmr
->region
;
1348 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1349 int chunk_pages
, entry
, pg_shift
, i
;
1350 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1351 struct i40iw_pble_info
*pinfo
;
1352 struct scatterlist
*sg
;
1356 pinfo
= (level
== I40IW_LEVEL_1
) ? NULL
: palloc
->level2
.leaf
;
1358 pg_shift
= ffs(region
->page_size
) - 1;
1359 for_each_sg(region
->sg_head
.sgl
, sg
, region
->nmap
, entry
) {
1360 chunk_pages
= sg_dma_len(sg
) >> pg_shift
;
1361 if ((iwmr
->type
== IW_MEMREG_TYPE_QP
) &&
1362 !iwpbl
->qp_mr
.sq_page
)
1363 iwpbl
->qp_mr
.sq_page
= sg_page(sg
);
1364 for (i
= 0; i
< chunk_pages
; i
++) {
1365 pg_addr
= sg_dma_address(sg
) + region
->page_size
* i
;
1367 if ((entry
+ i
) == 0)
1368 *pbl
= cpu_to_le64(pg_addr
& iwmr
->page_msk
);
1369 else if (!(pg_addr
& ~iwmr
->page_msk
))
1370 *pbl
= cpu_to_le64(pg_addr
);
1373 pbl
= i40iw_next_pbl_addr(pbl
, &pinfo
, &idx
);
1379 * i40iw_set_hugetlb_params - set MR pg size and mask to huge pg values.
1380 * @addr: virtual address
1381 * @iwmr: mr pointer for this memory registration
1383 static void i40iw_set_hugetlb_values(u64 addr
, struct i40iw_mr
*iwmr
)
1385 struct vm_area_struct
*vma
;
1388 vma
= find_vma(current
->mm
, addr
);
1389 if (vma
&& is_vm_hugetlb_page(vma
)) {
1390 h
= hstate_vma(vma
);
1391 if (huge_page_size(h
) == 0x200000) {
1392 iwmr
->page_size
= huge_page_size(h
);
1393 iwmr
->page_msk
= huge_page_mask(h
);
1399 * i40iw_check_mem_contiguous - check if pbls stored in arr are contiguous
1400 * @arr: lvl1 pbl array
1401 * @npages: page count
1402 * pg_size: page size
1405 static bool i40iw_check_mem_contiguous(u64
*arr
, u32 npages
, u32 pg_size
)
1409 for (pg_idx
= 0; pg_idx
< npages
; pg_idx
++) {
1410 if ((*arr
+ (pg_size
* pg_idx
)) != arr
[pg_idx
])
1417 * i40iw_check_mr_contiguous - check if MR is physically contiguous
1418 * @palloc: pbl allocation struct
1419 * pg_size: page size
1421 static bool i40iw_check_mr_contiguous(struct i40iw_pble_alloc
*palloc
, u32 pg_size
)
1423 struct i40iw_pble_level2
*lvl2
= &palloc
->level2
;
1424 struct i40iw_pble_info
*leaf
= lvl2
->leaf
;
1426 u64
*start_addr
= NULL
;
1430 if (palloc
->level
== I40IW_LEVEL_1
) {
1431 arr
= (u64
*)palloc
->level1
.addr
;
1432 ret
= i40iw_check_mem_contiguous(arr
, palloc
->total_cnt
, pg_size
);
1436 start_addr
= (u64
*)leaf
->addr
;
1438 for (i
= 0; i
< lvl2
->leaf_cnt
; i
++, leaf
++) {
1439 arr
= (u64
*)leaf
->addr
;
1440 if ((*start_addr
+ (i
* pg_size
* PBLE_PER_PAGE
)) != *arr
)
1442 ret
= i40iw_check_mem_contiguous(arr
, leaf
->cnt
, pg_size
);
1451 * i40iw_setup_pbles - copy user pg address to pble's
1452 * @iwdev: iwarp device
1453 * @iwmr: mr pointer for this memory registration
1454 * @use_pbles: flag if to use pble's
1456 static int i40iw_setup_pbles(struct i40iw_device
*iwdev
,
1457 struct i40iw_mr
*iwmr
,
1460 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1461 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1462 struct i40iw_pble_info
*pinfo
;
1464 enum i40iw_status_code status
;
1465 enum i40iw_pble_level level
= I40IW_LEVEL_1
;
1468 mutex_lock(&iwdev
->pbl_mutex
);
1469 status
= i40iw_get_pble(&iwdev
->sc_dev
, iwdev
->pble_rsrc
, palloc
, iwmr
->page_cnt
);
1470 mutex_unlock(&iwdev
->pbl_mutex
);
1474 iwpbl
->pbl_allocated
= true;
1475 level
= palloc
->level
;
1476 pinfo
= (level
== I40IW_LEVEL_1
) ? &palloc
->level1
: palloc
->level2
.leaf
;
1477 pbl
= (u64
*)pinfo
->addr
;
1479 pbl
= iwmr
->pgaddrmem
;
1482 i40iw_copy_user_pgaddrs(iwmr
, pbl
, level
);
1485 iwmr
->pgaddrmem
[0] = *pbl
;
1491 * i40iw_handle_q_mem - handle memory for qp and cq
1492 * @iwdev: iwarp device
1493 * @req: information for q memory management
1494 * @iwpbl: pble struct
1495 * @use_pbles: flag to use pble
1497 static int i40iw_handle_q_mem(struct i40iw_device
*iwdev
,
1498 struct i40iw_mem_reg_req
*req
,
1499 struct i40iw_pbl
*iwpbl
,
1502 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1503 struct i40iw_mr
*iwmr
= iwpbl
->iwmr
;
1504 struct i40iw_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
1505 struct i40iw_cq_mr
*cqmr
= &iwpbl
->cq_mr
;
1506 struct i40iw_hmc_pble
*hmc_p
;
1507 u64
*arr
= iwmr
->pgaddrmem
;
1513 total
= req
->sq_pages
+ req
->rq_pages
+ req
->cq_pages
;
1514 pg_size
= iwmr
->page_size
;
1516 err
= i40iw_setup_pbles(iwdev
, iwmr
, use_pbles
);
1520 if (use_pbles
&& (palloc
->level
!= I40IW_LEVEL_1
)) {
1521 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1522 iwpbl
->pbl_allocated
= false;
1527 arr
= (u64
*)palloc
->level1
.addr
;
1529 if (iwmr
->type
== IW_MEMREG_TYPE_QP
) {
1530 hmc_p
= &qpmr
->sq_pbl
;
1531 qpmr
->shadow
= (dma_addr_t
)arr
[total
];
1534 ret
= i40iw_check_mem_contiguous(arr
, req
->sq_pages
, pg_size
);
1536 ret
= i40iw_check_mem_contiguous(&arr
[req
->sq_pages
], req
->rq_pages
, pg_size
);
1540 hmc_p
->idx
= palloc
->level1
.idx
;
1541 hmc_p
= &qpmr
->rq_pbl
;
1542 hmc_p
->idx
= palloc
->level1
.idx
+ req
->sq_pages
;
1544 hmc_p
->addr
= arr
[0];
1545 hmc_p
= &qpmr
->rq_pbl
;
1546 hmc_p
->addr
= arr
[req
->sq_pages
];
1549 hmc_p
= &cqmr
->cq_pbl
;
1550 cqmr
->shadow
= (dma_addr_t
)arr
[total
];
1553 ret
= i40iw_check_mem_contiguous(arr
, req
->cq_pages
, pg_size
);
1556 hmc_p
->idx
= palloc
->level1
.idx
;
1558 hmc_p
->addr
= arr
[0];
1561 if (use_pbles
&& ret
) {
1562 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1563 iwpbl
->pbl_allocated
= false;
1570 * i40iw_hw_alloc_stag - cqp command to allocate stag
1571 * @iwdev: iwarp device
1572 * @iwmr: iwarp mr pointer
1574 static int i40iw_hw_alloc_stag(struct i40iw_device
*iwdev
, struct i40iw_mr
*iwmr
)
1576 struct i40iw_allocate_stag_info
*info
;
1577 struct i40iw_pd
*iwpd
= to_iwpd(iwmr
->ibmr
.pd
);
1578 enum i40iw_status_code status
;
1580 struct i40iw_cqp_request
*cqp_request
;
1581 struct cqp_commands_info
*cqp_info
;
1583 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1587 cqp_info
= &cqp_request
->info
;
1588 info
= &cqp_info
->in
.u
.alloc_stag
.info
;
1589 memset(info
, 0, sizeof(*info
));
1590 info
->page_size
= PAGE_SIZE
;
1591 info
->stag_idx
= iwmr
->stag
>> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1592 info
->pd_id
= iwpd
->sc_pd
.pd_id
;
1593 info
->total_len
= iwmr
->length
;
1594 info
->remote_access
= true;
1595 cqp_info
->cqp_cmd
= OP_ALLOC_STAG
;
1596 cqp_info
->post_sq
= 1;
1597 cqp_info
->in
.u
.alloc_stag
.dev
= &iwdev
->sc_dev
;
1598 cqp_info
->in
.u
.alloc_stag
.scratch
= (uintptr_t)cqp_request
;
1600 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1603 i40iw_pr_err("CQP-OP MR Reg fail");
1609 * i40iw_alloc_mr - register stag for fast memory registration
1611 * @mr_type: memory for stag registrion
1612 * @max_num_sg: man number of pages
1614 static struct ib_mr
*i40iw_alloc_mr(struct ib_pd
*pd
,
1615 enum ib_mr_type mr_type
,
1618 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1619 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1620 struct i40iw_pble_alloc
*palloc
;
1621 struct i40iw_pbl
*iwpbl
;
1622 struct i40iw_mr
*iwmr
;
1623 enum i40iw_status_code status
;
1625 int err_code
= -ENOMEM
;
1627 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1629 return ERR_PTR(-ENOMEM
);
1631 stag
= i40iw_create_stag(iwdev
);
1633 err_code
= -EOVERFLOW
;
1637 iwmr
->ibmr
.rkey
= stag
;
1638 iwmr
->ibmr
.lkey
= stag
;
1640 iwmr
->ibmr
.device
= pd
->device
;
1641 iwpbl
= &iwmr
->iwpbl
;
1643 iwmr
->type
= IW_MEMREG_TYPE_MEM
;
1644 palloc
= &iwpbl
->pble_alloc
;
1645 iwmr
->page_cnt
= max_num_sg
;
1646 mutex_lock(&iwdev
->pbl_mutex
);
1647 status
= i40iw_get_pble(&iwdev
->sc_dev
, iwdev
->pble_rsrc
, palloc
, iwmr
->page_cnt
);
1648 mutex_unlock(&iwdev
->pbl_mutex
);
1652 if (palloc
->level
!= I40IW_LEVEL_1
)
1654 err_code
= i40iw_hw_alloc_stag(iwdev
, iwmr
);
1657 iwpbl
->pbl_allocated
= true;
1658 i40iw_add_pdusecount(iwpd
);
1661 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1663 i40iw_free_stag(iwdev
, stag
);
1666 return ERR_PTR(err_code
);
1670 * i40iw_set_page - populate pbl list for fmr
1671 * @ibmr: ib mem to access iwarp mr pointer
1672 * @addr: page dma address fro pbl list
1674 static int i40iw_set_page(struct ib_mr
*ibmr
, u64 addr
)
1676 struct i40iw_mr
*iwmr
= to_iwmr(ibmr
);
1677 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1678 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1681 if (unlikely(iwmr
->npages
== iwmr
->page_cnt
))
1684 pbl
= (u64
*)palloc
->level1
.addr
;
1685 pbl
[iwmr
->npages
++] = cpu_to_le64(addr
);
1690 * i40iw_map_mr_sg - map of sg list for fmr
1691 * @ibmr: ib mem to access iwarp mr pointer
1692 * @sg: scatter gather list for fmr
1693 * @sg_nents: number of sg pages
1695 static int i40iw_map_mr_sg(struct ib_mr
*ibmr
, struct scatterlist
*sg
,
1696 int sg_nents
, unsigned int *sg_offset
)
1698 struct i40iw_mr
*iwmr
= to_iwmr(ibmr
);
1701 return ib_sg_to_pages(ibmr
, sg
, sg_nents
, sg_offset
, i40iw_set_page
);
1705 * i40iw_drain_sq - drain the send queue
1706 * @ibqp: ib qp pointer
1708 static void i40iw_drain_sq(struct ib_qp
*ibqp
)
1710 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
1711 struct i40iw_sc_qp
*qp
= &iwqp
->sc_qp
;
1713 if (I40IW_RING_MORE_WORK(qp
->qp_uk
.sq_ring
))
1714 wait_for_completion(&iwqp
->sq_drained
);
1718 * i40iw_drain_rq - drain the receive queue
1719 * @ibqp: ib qp pointer
1721 static void i40iw_drain_rq(struct ib_qp
*ibqp
)
1723 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
1724 struct i40iw_sc_qp
*qp
= &iwqp
->sc_qp
;
1726 if (I40IW_RING_MORE_WORK(qp
->qp_uk
.rq_ring
))
1727 wait_for_completion(&iwqp
->rq_drained
);
1731 * i40iw_hwreg_mr - send cqp command for memory registration
1732 * @iwdev: iwarp device
1733 * @iwmr: iwarp mr pointer
1734 * @access: access for MR
1736 static int i40iw_hwreg_mr(struct i40iw_device
*iwdev
,
1737 struct i40iw_mr
*iwmr
,
1740 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1741 struct i40iw_reg_ns_stag_info
*stag_info
;
1742 struct i40iw_pd
*iwpd
= to_iwpd(iwmr
->ibmr
.pd
);
1743 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1744 enum i40iw_status_code status
;
1746 struct i40iw_cqp_request
*cqp_request
;
1747 struct cqp_commands_info
*cqp_info
;
1749 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1753 cqp_info
= &cqp_request
->info
;
1754 stag_info
= &cqp_info
->in
.u
.mr_reg_non_shared
.info
;
1755 memset(stag_info
, 0, sizeof(*stag_info
));
1756 stag_info
->va
= (void *)(unsigned long)iwpbl
->user_base
;
1757 stag_info
->stag_idx
= iwmr
->stag
>> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1758 stag_info
->stag_key
= (u8
)iwmr
->stag
;
1759 stag_info
->total_len
= iwmr
->length
;
1760 stag_info
->access_rights
= access
;
1761 stag_info
->pd_id
= iwpd
->sc_pd
.pd_id
;
1762 stag_info
->addr_type
= I40IW_ADDR_TYPE_VA_BASED
;
1763 stag_info
->page_size
= iwmr
->page_size
;
1765 if (iwpbl
->pbl_allocated
) {
1766 if (palloc
->level
== I40IW_LEVEL_1
) {
1767 stag_info
->first_pm_pbl_index
= palloc
->level1
.idx
;
1768 stag_info
->chunk_size
= 1;
1770 stag_info
->first_pm_pbl_index
= palloc
->level2
.root
.idx
;
1771 stag_info
->chunk_size
= 3;
1774 stag_info
->reg_addr_pa
= iwmr
->pgaddrmem
[0];
1777 cqp_info
->cqp_cmd
= OP_MR_REG_NON_SHARED
;
1778 cqp_info
->post_sq
= 1;
1779 cqp_info
->in
.u
.mr_reg_non_shared
.dev
= &iwdev
->sc_dev
;
1780 cqp_info
->in
.u
.mr_reg_non_shared
.scratch
= (uintptr_t)cqp_request
;
1782 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1785 i40iw_pr_err("CQP-OP MR Reg fail");
1791 * i40iw_reg_user_mr - Register a user memory region
1793 * @start: virtual start address
1794 * @length: length of mr
1795 * @virt: virtual address
1796 * @acc: access of mr
1799 static struct ib_mr
*i40iw_reg_user_mr(struct ib_pd
*pd
,
1804 struct ib_udata
*udata
)
1806 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1807 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1808 struct i40iw_ucontext
*ucontext
;
1809 struct i40iw_pble_alloc
*palloc
;
1810 struct i40iw_pbl
*iwpbl
;
1811 struct i40iw_mr
*iwmr
;
1812 struct ib_umem
*region
;
1813 struct i40iw_mem_reg_req req
;
1818 bool use_pbles
= false;
1819 unsigned long flags
;
1825 return ERR_PTR(-ENODEV
);
1827 if (length
> I40IW_MAX_MR_SIZE
)
1828 return ERR_PTR(-EINVAL
);
1829 region
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
1831 return (struct ib_mr
*)region
;
1833 if (ib_copy_from_udata(&req
, udata
, sizeof(req
))) {
1834 ib_umem_release(region
);
1835 return ERR_PTR(-EFAULT
);
1838 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1840 ib_umem_release(region
);
1841 return ERR_PTR(-ENOMEM
);
1844 iwpbl
= &iwmr
->iwpbl
;
1846 iwmr
->region
= region
;
1848 iwmr
->ibmr
.device
= pd
->device
;
1849 ucontext
= to_ucontext(pd
->uobject
->context
);
1851 iwmr
->page_size
= region
->page_size
;
1852 iwmr
->page_msk
= PAGE_MASK
;
1854 if (region
->hugetlb
&& (req
.reg_type
== IW_MEMREG_TYPE_MEM
))
1855 i40iw_set_hugetlb_values(start
, iwmr
);
1857 region_length
= region
->length
+ (start
& (iwmr
->page_size
- 1));
1858 pg_shift
= ffs(iwmr
->page_size
) - 1;
1859 pbl_depth
= region_length
>> pg_shift
;
1860 pbl_depth
+= (region_length
& (iwmr
->page_size
- 1)) ? 1 : 0;
1861 iwmr
->length
= region
->length
;
1863 iwpbl
->user_base
= virt
;
1864 palloc
= &iwpbl
->pble_alloc
;
1866 iwmr
->type
= req
.reg_type
;
1867 iwmr
->page_cnt
= (u32
)pbl_depth
;
1869 switch (req
.reg_type
) {
1870 case IW_MEMREG_TYPE_QP
:
1871 use_pbles
= ((req
.sq_pages
+ req
.rq_pages
) > 2);
1872 err
= i40iw_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
1875 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
1876 list_add_tail(&iwpbl
->list
, &ucontext
->qp_reg_mem_list
);
1877 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
1879 case IW_MEMREG_TYPE_CQ
:
1880 use_pbles
= (req
.cq_pages
> 1);
1881 err
= i40iw_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
1885 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1886 list_add_tail(&iwpbl
->list
, &ucontext
->cq_reg_mem_list
);
1887 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1889 case IW_MEMREG_TYPE_MEM
:
1890 use_pbles
= (iwmr
->page_cnt
!= 1);
1891 access
= I40IW_ACCESS_FLAGS_LOCALREAD
;
1893 err
= i40iw_setup_pbles(iwdev
, iwmr
, use_pbles
);
1898 ret
= i40iw_check_mr_contiguous(palloc
, iwmr
->page_size
);
1900 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1901 iwpbl
->pbl_allocated
= false;
1905 access
|= i40iw_get_user_access(acc
);
1906 stag
= i40iw_create_stag(iwdev
);
1913 iwmr
->ibmr
.rkey
= stag
;
1914 iwmr
->ibmr
.lkey
= stag
;
1916 err
= i40iw_hwreg_mr(iwdev
, iwmr
, access
);
1918 i40iw_free_stag(iwdev
, stag
);
1927 iwmr
->type
= req
.reg_type
;
1928 if (req
.reg_type
== IW_MEMREG_TYPE_MEM
)
1929 i40iw_add_pdusecount(iwpd
);
1933 if (palloc
->level
!= I40IW_LEVEL_0
&& iwpbl
->pbl_allocated
)
1934 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1935 ib_umem_release(region
);
1937 return ERR_PTR(err
);
1941 * i40iw_reg_phys_mr - register kernel physical memory
1943 * @addr: physical address of memory to register
1944 * @size: size of memory to register
1945 * @acc: Access rights
1946 * @iova_start: start of virtual address for physical buffers
1948 struct ib_mr
*i40iw_reg_phys_mr(struct ib_pd
*pd
,
1954 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1955 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1956 struct i40iw_pbl
*iwpbl
;
1957 struct i40iw_mr
*iwmr
;
1958 enum i40iw_status_code status
;
1960 u16 access
= I40IW_ACCESS_FLAGS_LOCALREAD
;
1963 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1965 return ERR_PTR(-ENOMEM
);
1967 iwmr
->ibmr
.device
= pd
->device
;
1968 iwpbl
= &iwmr
->iwpbl
;
1970 iwmr
->type
= IW_MEMREG_TYPE_MEM
;
1971 iwpbl
->user_base
= *iova_start
;
1972 stag
= i40iw_create_stag(iwdev
);
1977 access
|= i40iw_get_user_access(acc
);
1979 iwmr
->ibmr
.rkey
= stag
;
1980 iwmr
->ibmr
.lkey
= stag
;
1982 iwmr
->pgaddrmem
[0] = addr
;
1983 iwmr
->length
= size
;
1984 status
= i40iw_hwreg_mr(iwdev
, iwmr
, access
);
1986 i40iw_free_stag(iwdev
, stag
);
1991 i40iw_add_pdusecount(iwpd
);
1995 return ERR_PTR(ret
);
1999 * i40iw_get_dma_mr - register physical mem
2001 * @acc: access for memory
2003 static struct ib_mr
*i40iw_get_dma_mr(struct ib_pd
*pd
, int acc
)
2007 return i40iw_reg_phys_mr(pd
, 0, 0, acc
, &kva
);
2011 * i40iw_del_mem_list - Deleting pbl list entries for CQ/QP
2012 * @iwmr: iwmr for IB's user page addresses
2013 * @ucontext: ptr to user context
2015 static void i40iw_del_memlist(struct i40iw_mr
*iwmr
,
2016 struct i40iw_ucontext
*ucontext
)
2018 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
2019 unsigned long flags
;
2021 switch (iwmr
->type
) {
2022 case IW_MEMREG_TYPE_CQ
:
2023 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
2024 if (!list_empty(&ucontext
->cq_reg_mem_list
))
2025 list_del(&iwpbl
->list
);
2026 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
2028 case IW_MEMREG_TYPE_QP
:
2029 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
2030 if (!list_empty(&ucontext
->qp_reg_mem_list
))
2031 list_del(&iwpbl
->list
);
2032 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
2040 * i40iw_dereg_mr - deregister mr
2041 * @ib_mr: mr ptr for dereg
2043 static int i40iw_dereg_mr(struct ib_mr
*ib_mr
)
2045 struct ib_pd
*ibpd
= ib_mr
->pd
;
2046 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
2047 struct i40iw_mr
*iwmr
= to_iwmr(ib_mr
);
2048 struct i40iw_device
*iwdev
= to_iwdev(ib_mr
->device
);
2049 enum i40iw_status_code status
;
2050 struct i40iw_dealloc_stag_info
*info
;
2051 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
2052 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
2053 struct i40iw_cqp_request
*cqp_request
;
2054 struct cqp_commands_info
*cqp_info
;
2058 ib_umem_release(iwmr
->region
);
2060 if (iwmr
->type
!= IW_MEMREG_TYPE_MEM
) {
2061 if (ibpd
->uobject
) {
2062 struct i40iw_ucontext
*ucontext
;
2064 ucontext
= to_ucontext(ibpd
->uobject
->context
);
2065 i40iw_del_memlist(iwmr
, ucontext
);
2067 if (iwpbl
->pbl_allocated
)
2068 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
2073 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
2077 cqp_info
= &cqp_request
->info
;
2078 info
= &cqp_info
->in
.u
.dealloc_stag
.info
;
2079 memset(info
, 0, sizeof(*info
));
2081 info
->pd_id
= cpu_to_le32(iwpd
->sc_pd
.pd_id
& 0x00007fff);
2082 info
->stag_idx
= RS_64_1(ib_mr
->rkey
, I40IW_CQPSQ_STAG_IDX_SHIFT
);
2083 stag_idx
= info
->stag_idx
;
2085 if (iwpbl
->pbl_allocated
)
2086 info
->dealloc_pbl
= true;
2088 cqp_info
->cqp_cmd
= OP_DEALLOC_STAG
;
2089 cqp_info
->post_sq
= 1;
2090 cqp_info
->in
.u
.dealloc_stag
.dev
= &iwdev
->sc_dev
;
2091 cqp_info
->in
.u
.dealloc_stag
.scratch
= (uintptr_t)cqp_request
;
2092 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
2094 i40iw_pr_err("CQP-OP dealloc failed for stag_idx = 0x%x\n", stag_idx
);
2095 i40iw_rem_pdusecount(iwpd
, iwdev
);
2096 i40iw_free_stag(iwdev
, iwmr
->stag
);
2097 if (iwpbl
->pbl_allocated
)
2098 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
2106 static ssize_t
i40iw_show_rev(struct device
*dev
,
2107 struct device_attribute
*attr
, char *buf
)
2109 struct i40iw_ib_device
*iwibdev
= container_of(dev
,
2110 struct i40iw_ib_device
,
2112 u32 hw_rev
= iwibdev
->iwdev
->sc_dev
.hw_rev
;
2114 return sprintf(buf
, "%x\n", hw_rev
);
2120 static ssize_t
i40iw_show_hca(struct device
*dev
,
2121 struct device_attribute
*attr
, char *buf
)
2123 return sprintf(buf
, "I40IW\n");
2129 static ssize_t
i40iw_show_board(struct device
*dev
,
2130 struct device_attribute
*attr
,
2133 return sprintf(buf
, "%.*s\n", 32, "I40IW Board ID");
2136 static DEVICE_ATTR(hw_rev
, S_IRUGO
, i40iw_show_rev
, NULL
);
2137 static DEVICE_ATTR(hca_type
, S_IRUGO
, i40iw_show_hca
, NULL
);
2138 static DEVICE_ATTR(board_id
, S_IRUGO
, i40iw_show_board
, NULL
);
2140 static struct device_attribute
*i40iw_dev_attributes
[] = {
2147 * i40iw_copy_sg_list - copy sg list for qp
2148 * @sg_list: copied into sg_list
2149 * @sgl: copy from sgl
2150 * @num_sges: count of sg entries
2152 static void i40iw_copy_sg_list(struct i40iw_sge
*sg_list
, struct ib_sge
*sgl
, int num_sges
)
2156 for (i
= 0; (i
< num_sges
) && (i
< I40IW_MAX_WQ_FRAGMENT_COUNT
); i
++) {
2157 sg_list
[i
].tag_off
= sgl
[i
].addr
;
2158 sg_list
[i
].len
= sgl
[i
].length
;
2159 sg_list
[i
].stag
= sgl
[i
].lkey
;
2164 * i40iw_post_send - kernel application wr
2165 * @ibqp: qp ptr for wr
2166 * @ib_wr: work request ptr
2167 * @bad_wr: return of bad wr if err
2169 static int i40iw_post_send(struct ib_qp
*ibqp
,
2170 struct ib_send_wr
*ib_wr
,
2171 struct ib_send_wr
**bad_wr
)
2173 struct i40iw_qp
*iwqp
;
2174 struct i40iw_qp_uk
*ukqp
;
2175 struct i40iw_post_sq_info info
;
2176 enum i40iw_status_code ret
;
2178 unsigned long flags
;
2181 iwqp
= (struct i40iw_qp
*)ibqp
;
2182 ukqp
= &iwqp
->sc_qp
.qp_uk
;
2184 spin_lock_irqsave(&iwqp
->lock
, flags
);
2187 memset(&info
, 0, sizeof(info
));
2188 info
.wr_id
= (u64
)(ib_wr
->wr_id
);
2189 if ((ib_wr
->send_flags
& IB_SEND_SIGNALED
) || iwqp
->sig_all
)
2190 info
.signaled
= true;
2191 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
2192 info
.read_fence
= true;
2194 switch (ib_wr
->opcode
) {
2197 case IB_WR_SEND_WITH_INV
:
2198 if (ib_wr
->opcode
== IB_WR_SEND
) {
2199 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
2200 info
.op_type
= I40IW_OP_TYPE_SEND_SOL
;
2202 info
.op_type
= I40IW_OP_TYPE_SEND
;
2204 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
2205 info
.op_type
= I40IW_OP_TYPE_SEND_SOL_INV
;
2207 info
.op_type
= I40IW_OP_TYPE_SEND_INV
;
2210 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
2211 info
.op
.inline_send
.data
= (void *)(unsigned long)ib_wr
->sg_list
[0].addr
;
2212 info
.op
.inline_send
.len
= ib_wr
->sg_list
[0].length
;
2213 ret
= ukqp
->ops
.iw_inline_send(ukqp
, &info
, ib_wr
->ex
.invalidate_rkey
, false);
2215 info
.op
.send
.num_sges
= ib_wr
->num_sge
;
2216 info
.op
.send
.sg_list
= (struct i40iw_sge
*)ib_wr
->sg_list
;
2217 ret
= ukqp
->ops
.iw_send(ukqp
, &info
, ib_wr
->ex
.invalidate_rkey
, false);
2221 if (ret
== I40IW_ERR_QP_TOOMANY_WRS_POSTED
)
2227 case IB_WR_RDMA_WRITE
:
2228 info
.op_type
= I40IW_OP_TYPE_RDMA_WRITE
;
2230 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
2231 info
.op
.inline_rdma_write
.data
= (void *)(unsigned long)ib_wr
->sg_list
[0].addr
;
2232 info
.op
.inline_rdma_write
.len
= ib_wr
->sg_list
[0].length
;
2233 info
.op
.inline_rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
2234 info
.op
.inline_rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
2235 info
.op
.inline_rdma_write
.rem_addr
.len
= ib_wr
->sg_list
->length
;
2236 ret
= ukqp
->ops
.iw_inline_rdma_write(ukqp
, &info
, false);
2238 info
.op
.rdma_write
.lo_sg_list
= (void *)ib_wr
->sg_list
;
2239 info
.op
.rdma_write
.num_lo_sges
= ib_wr
->num_sge
;
2240 info
.op
.rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
2241 info
.op
.rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
2242 info
.op
.rdma_write
.rem_addr
.len
= ib_wr
->sg_list
->length
;
2243 ret
= ukqp
->ops
.iw_rdma_write(ukqp
, &info
, false);
2247 if (ret
== I40IW_ERR_QP_TOOMANY_WRS_POSTED
)
2253 case IB_WR_RDMA_READ_WITH_INV
:
2256 case IB_WR_RDMA_READ
:
2257 if (ib_wr
->num_sge
> I40IW_MAX_SGE_RD
) {
2261 info
.op_type
= I40IW_OP_TYPE_RDMA_READ
;
2262 info
.op
.rdma_read
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
2263 info
.op
.rdma_read
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
2264 info
.op
.rdma_read
.rem_addr
.len
= ib_wr
->sg_list
->length
;
2265 info
.op
.rdma_read
.lo_addr
.tag_off
= ib_wr
->sg_list
->addr
;
2266 info
.op
.rdma_read
.lo_addr
.stag
= ib_wr
->sg_list
->lkey
;
2267 info
.op
.rdma_read
.lo_addr
.len
= ib_wr
->sg_list
->length
;
2268 ret
= ukqp
->ops
.iw_rdma_read(ukqp
, &info
, inv_stag
, false);
2270 if (ret
== I40IW_ERR_QP_TOOMANY_WRS_POSTED
)
2276 case IB_WR_LOCAL_INV
:
2277 info
.op_type
= I40IW_OP_TYPE_INV_STAG
;
2278 info
.op
.inv_local_stag
.target_stag
= ib_wr
->ex
.invalidate_rkey
;
2279 ret
= ukqp
->ops
.iw_stag_local_invalidate(ukqp
, &info
, true);
2285 struct i40iw_mr
*iwmr
= to_iwmr(reg_wr(ib_wr
)->mr
);
2286 int flags
= reg_wr(ib_wr
)->access
;
2287 struct i40iw_pble_alloc
*palloc
= &iwmr
->iwpbl
.pble_alloc
;
2288 struct i40iw_sc_dev
*dev
= &iwqp
->iwdev
->sc_dev
;
2289 struct i40iw_fast_reg_stag_info info
;
2291 memset(&info
, 0, sizeof(info
));
2292 info
.access_rights
= I40IW_ACCESS_FLAGS_LOCALREAD
;
2293 info
.access_rights
|= i40iw_get_user_access(flags
);
2294 info
.stag_key
= reg_wr(ib_wr
)->key
& 0xff;
2295 info
.stag_idx
= reg_wr(ib_wr
)->key
>> 8;
2296 info
.page_size
= reg_wr(ib_wr
)->mr
->page_size
;
2297 info
.wr_id
= ib_wr
->wr_id
;
2299 info
.addr_type
= I40IW_ADDR_TYPE_VA_BASED
;
2300 info
.va
= (void *)(uintptr_t)iwmr
->ibmr
.iova
;
2301 info
.total_len
= iwmr
->ibmr
.length
;
2302 info
.reg_addr_pa
= *(u64
*)palloc
->level1
.addr
;
2303 info
.first_pm_pbl_index
= palloc
->level1
.idx
;
2304 info
.local_fence
= ib_wr
->send_flags
& IB_SEND_FENCE
;
2305 info
.signaled
= ib_wr
->send_flags
& IB_SEND_SIGNALED
;
2307 if (iwmr
->npages
> I40IW_MIN_PAGES_PER_FMR
)
2308 info
.chunk_size
= 1;
2310 ret
= dev
->iw_priv_qp_ops
->iw_mr_fast_register(&iwqp
->sc_qp
, &info
, true);
2317 i40iw_pr_err(" upost_send bad opcode = 0x%x\n",
2324 ib_wr
= ib_wr
->next
;
2330 ukqp
->ops
.iw_qp_post_wr(ukqp
);
2331 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
2337 * i40iw_post_recv - post receive wr for kernel application
2338 * @ibqp: ib qp pointer
2339 * @ib_wr: work request for receive
2340 * @bad_wr: bad wr caused an error
2342 static int i40iw_post_recv(struct ib_qp
*ibqp
,
2343 struct ib_recv_wr
*ib_wr
,
2344 struct ib_recv_wr
**bad_wr
)
2346 struct i40iw_qp
*iwqp
;
2347 struct i40iw_qp_uk
*ukqp
;
2348 struct i40iw_post_rq_info post_recv
;
2349 struct i40iw_sge sg_list
[I40IW_MAX_WQ_FRAGMENT_COUNT
];
2350 enum i40iw_status_code ret
= 0;
2351 unsigned long flags
;
2354 iwqp
= (struct i40iw_qp
*)ibqp
;
2355 ukqp
= &iwqp
->sc_qp
.qp_uk
;
2357 memset(&post_recv
, 0, sizeof(post_recv
));
2358 spin_lock_irqsave(&iwqp
->lock
, flags
);
2360 post_recv
.num_sges
= ib_wr
->num_sge
;
2361 post_recv
.wr_id
= ib_wr
->wr_id
;
2362 i40iw_copy_sg_list(sg_list
, ib_wr
->sg_list
, ib_wr
->num_sge
);
2363 post_recv
.sg_list
= sg_list
;
2364 ret
= ukqp
->ops
.iw_post_receive(ukqp
, &post_recv
);
2366 i40iw_pr_err(" post_recv err %d\n", ret
);
2367 if (ret
== I40IW_ERR_QP_TOOMANY_WRS_POSTED
)
2374 ib_wr
= ib_wr
->next
;
2377 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
2382 * i40iw_poll_cq - poll cq for completion (kernel apps)
2384 * @num_entries: number of entries to poll
2385 * @entry: wr of entry completed
2387 static int i40iw_poll_cq(struct ib_cq
*ibcq
,
2389 struct ib_wc
*entry
)
2391 struct i40iw_cq
*iwcq
;
2393 struct i40iw_cq_poll_info cq_poll_info
;
2394 enum i40iw_status_code ret
;
2395 struct i40iw_cq_uk
*ukcq
;
2396 struct i40iw_sc_qp
*qp
;
2397 struct i40iw_qp
*iwqp
;
2398 unsigned long flags
;
2400 iwcq
= (struct i40iw_cq
*)ibcq
;
2401 ukcq
= &iwcq
->sc_cq
.cq_uk
;
2403 spin_lock_irqsave(&iwcq
->lock
, flags
);
2404 while (cqe_count
< num_entries
) {
2405 ret
= ukcq
->ops
.iw_cq_poll_completion(ukcq
, &cq_poll_info
);
2406 if (ret
== I40IW_ERR_QUEUE_EMPTY
) {
2408 } else if (ret
== I40IW_ERR_QUEUE_DESTROYED
) {
2415 entry
->wc_flags
= 0;
2416 entry
->wr_id
= cq_poll_info
.wr_id
;
2417 if (cq_poll_info
.error
) {
2418 entry
->status
= IB_WC_WR_FLUSH_ERR
;
2419 entry
->vendor_err
= cq_poll_info
.major_err
<< 16 | cq_poll_info
.minor_err
;
2421 entry
->status
= IB_WC_SUCCESS
;
2424 switch (cq_poll_info
.op_type
) {
2425 case I40IW_OP_TYPE_RDMA_WRITE
:
2426 entry
->opcode
= IB_WC_RDMA_WRITE
;
2428 case I40IW_OP_TYPE_RDMA_READ_INV_STAG
:
2429 case I40IW_OP_TYPE_RDMA_READ
:
2430 entry
->opcode
= IB_WC_RDMA_READ
;
2432 case I40IW_OP_TYPE_SEND_SOL
:
2433 case I40IW_OP_TYPE_SEND_SOL_INV
:
2434 case I40IW_OP_TYPE_SEND_INV
:
2435 case I40IW_OP_TYPE_SEND
:
2436 entry
->opcode
= IB_WC_SEND
;
2438 case I40IW_OP_TYPE_REC
:
2439 entry
->opcode
= IB_WC_RECV
;
2442 entry
->opcode
= IB_WC_RECV
;
2446 entry
->ex
.imm_data
= 0;
2447 qp
= (struct i40iw_sc_qp
*)cq_poll_info
.qp_handle
;
2448 entry
->qp
= (struct ib_qp
*)qp
->back_qp
;
2449 entry
->src_qp
= cq_poll_info
.qp_id
;
2450 iwqp
= (struct i40iw_qp
*)qp
->back_qp
;
2451 if (iwqp
->iwarp_state
> I40IW_QP_STATE_RTS
) {
2452 if (!I40IW_RING_MORE_WORK(qp
->qp_uk
.sq_ring
))
2453 complete(&iwqp
->sq_drained
);
2454 if (!I40IW_RING_MORE_WORK(qp
->qp_uk
.rq_ring
))
2455 complete(&iwqp
->rq_drained
);
2457 entry
->byte_len
= cq_poll_info
.bytes_xfered
;
2461 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
2466 * i40iw_req_notify_cq - arm cq kernel application
2468 * @notify_flags: notofication flags
2470 static int i40iw_req_notify_cq(struct ib_cq
*ibcq
,
2471 enum ib_cq_notify_flags notify_flags
)
2473 struct i40iw_cq
*iwcq
;
2474 struct i40iw_cq_uk
*ukcq
;
2475 unsigned long flags
;
2476 enum i40iw_completion_notify cq_notify
= IW_CQ_COMPL_EVENT
;
2478 iwcq
= (struct i40iw_cq
*)ibcq
;
2479 ukcq
= &iwcq
->sc_cq
.cq_uk
;
2480 if (notify_flags
== IB_CQ_SOLICITED
)
2481 cq_notify
= IW_CQ_COMPL_SOLICITED
;
2482 spin_lock_irqsave(&iwcq
->lock
, flags
);
2483 ukcq
->ops
.iw_cq_request_notification(ukcq
, cq_notify
);
2484 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
2489 * i40iw_port_immutable - return port's immutable data
2490 * @ibdev: ib dev struct
2491 * @port_num: port number
2492 * @immutable: immutable data for the port return
2494 static int i40iw_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
2495 struct ib_port_immutable
*immutable
)
2497 struct ib_port_attr attr
;
2500 err
= i40iw_query_port(ibdev
, port_num
, &attr
);
2505 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
2506 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
2507 immutable
->core_cap_flags
= RDMA_CORE_PORT_IWARP
;
2512 static const char * const i40iw_hw_stat_names
[] = {
2514 [I40IW_HW_STAT_INDEX_IP4RXDISCARD
] = "ip4InDiscards",
2515 [I40IW_HW_STAT_INDEX_IP4RXTRUNC
] = "ip4InTruncatedPkts",
2516 [I40IW_HW_STAT_INDEX_IP4TXNOROUTE
] = "ip4OutNoRoutes",
2517 [I40IW_HW_STAT_INDEX_IP6RXDISCARD
] = "ip6InDiscards",
2518 [I40IW_HW_STAT_INDEX_IP6RXTRUNC
] = "ip6InTruncatedPkts",
2519 [I40IW_HW_STAT_INDEX_IP6TXNOROUTE
] = "ip6OutNoRoutes",
2520 [I40IW_HW_STAT_INDEX_TCPRTXSEG
] = "tcpRetransSegs",
2521 [I40IW_HW_STAT_INDEX_TCPRXOPTERR
] = "tcpInOptErrors",
2522 [I40IW_HW_STAT_INDEX_TCPRXPROTOERR
] = "tcpInProtoErrors",
2524 [I40IW_HW_STAT_INDEX_IP4RXOCTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2526 [I40IW_HW_STAT_INDEX_IP4RXPKTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2528 [I40IW_HW_STAT_INDEX_IP4RXFRAGS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2530 [I40IW_HW_STAT_INDEX_IP4RXMCPKTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2532 [I40IW_HW_STAT_INDEX_IP4TXOCTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2534 [I40IW_HW_STAT_INDEX_IP4TXPKTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2536 [I40IW_HW_STAT_INDEX_IP4TXFRAGS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2538 [I40IW_HW_STAT_INDEX_IP4TXMCPKTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2540 [I40IW_HW_STAT_INDEX_IP6RXOCTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2542 [I40IW_HW_STAT_INDEX_IP6RXPKTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2544 [I40IW_HW_STAT_INDEX_IP6RXFRAGS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2546 [I40IW_HW_STAT_INDEX_IP6RXMCPKTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2548 [I40IW_HW_STAT_INDEX_IP6TXOCTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2550 [I40IW_HW_STAT_INDEX_IP6TXPKTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2552 [I40IW_HW_STAT_INDEX_IP6TXFRAGS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2554 [I40IW_HW_STAT_INDEX_IP6TXMCPKTS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2556 [I40IW_HW_STAT_INDEX_TCPRXSEGS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2558 [I40IW_HW_STAT_INDEX_TCPTXSEG
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2560 [I40IW_HW_STAT_INDEX_RDMARXRDS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2562 [I40IW_HW_STAT_INDEX_RDMARXSNDS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2564 [I40IW_HW_STAT_INDEX_RDMARXWRS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2566 [I40IW_HW_STAT_INDEX_RDMATXRDS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2568 [I40IW_HW_STAT_INDEX_RDMATXSNDS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2570 [I40IW_HW_STAT_INDEX_RDMATXWRS
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2572 [I40IW_HW_STAT_INDEX_RDMAVBND
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2574 [I40IW_HW_STAT_INDEX_RDMAVINV
+ I40IW_HW_STAT_INDEX_MAX_32
] =
2578 static void i40iw_get_dev_fw_str(struct ib_device
*dev
, char *str
,
2581 u32 firmware_version
= I40IW_FW_VERSION
;
2583 snprintf(str
, str_len
, "%u.%u", firmware_version
,
2584 (firmware_version
& 0x000000ff));
2588 * i40iw_alloc_hw_stats - Allocate a hw stats structure
2589 * @ibdev: device pointer from stack
2590 * @port_num: port number
2592 static struct rdma_hw_stats
*i40iw_alloc_hw_stats(struct ib_device
*ibdev
,
2595 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
2596 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
2597 int num_counters
= I40IW_HW_STAT_INDEX_MAX_32
+
2598 I40IW_HW_STAT_INDEX_MAX_64
;
2599 unsigned long lifespan
= RDMA_HW_STATS_DEFAULT_LIFESPAN
;
2601 BUILD_BUG_ON(ARRAY_SIZE(i40iw_hw_stat_names
) !=
2602 (I40IW_HW_STAT_INDEX_MAX_32
+
2603 I40IW_HW_STAT_INDEX_MAX_64
));
2606 * PFs get the default update lifespan, but VFs only update once
2611 return rdma_alloc_hw_stats_struct(i40iw_hw_stat_names
, num_counters
,
2616 * i40iw_get_hw_stats - Populates the rdma_hw_stats structure
2617 * @ibdev: device pointer from stack
2618 * @stats: stats pointer from stack
2619 * @port_num: port number
2620 * @index: which hw counter the stack is requesting we update
2622 static int i40iw_get_hw_stats(struct ib_device
*ibdev
,
2623 struct rdma_hw_stats
*stats
,
2624 u8 port_num
, int index
)
2626 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
2627 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
2628 struct i40iw_vsi_pestat
*devstat
= iwdev
->vsi
.pestat
;
2629 struct i40iw_dev_hw_stats
*hw_stats
= &devstat
->hw_stats
;
2632 i40iw_hw_stats_read_all(devstat
, &devstat
->hw_stats
);
2634 if (i40iw_vchnl_vf_get_pe_stats(dev
, &devstat
->hw_stats
))
2638 memcpy(&stats
->value
[0], hw_stats
, sizeof(*hw_stats
));
2640 return stats
->num_counters
;
2644 * i40iw_query_gid - Query port GID
2645 * @ibdev: device pointer from stack
2646 * @port: port number
2647 * @index: Entry index
2650 static int i40iw_query_gid(struct ib_device
*ibdev
,
2655 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
2657 memset(gid
->raw
, 0, sizeof(gid
->raw
));
2658 ether_addr_copy(gid
->raw
, iwdev
->netdev
->dev_addr
);
2663 * i40iw_modify_port Modify port properties
2664 * @ibdev: device pointer from stack
2665 * @port: port number
2666 * @port_modify_mask: mask for port modifications
2667 * @props: port properties
2669 static int i40iw_modify_port(struct ib_device
*ibdev
,
2671 int port_modify_mask
,
2672 struct ib_port_modify
*props
)
2678 * i40iw_query_pkey - Query partition key
2679 * @ibdev: device pointer from stack
2680 * @port: port number
2681 * @index: index of pkey
2682 * @pkey: pointer to store the pkey
2684 static int i40iw_query_pkey(struct ib_device
*ibdev
,
2694 * i40iw_create_ah - create address handle
2696 * @ah_attr: address handle attributes
2698 static struct ib_ah
*i40iw_create_ah(struct ib_pd
*ibpd
,
2699 struct ib_ah_attr
*attr
,
2700 struct ib_udata
*udata
)
2703 return ERR_PTR(-ENOSYS
);
2707 * i40iw_destroy_ah - Destroy address handle
2708 * @ah: pointer to address handle
2710 static int i40iw_destroy_ah(struct ib_ah
*ah
)
2716 * i40iw_init_rdma_device - initialization of iwarp device
2717 * @iwdev: iwarp device
2719 static struct i40iw_ib_device
*i40iw_init_rdma_device(struct i40iw_device
*iwdev
)
2721 struct i40iw_ib_device
*iwibdev
;
2722 struct net_device
*netdev
= iwdev
->netdev
;
2723 struct pci_dev
*pcidev
= (struct pci_dev
*)iwdev
->hw
.dev_context
;
2725 iwibdev
= (struct i40iw_ib_device
*)ib_alloc_device(sizeof(*iwibdev
));
2727 i40iw_pr_err("iwdev == NULL\n");
2730 strlcpy(iwibdev
->ibdev
.name
, "i40iw%d", IB_DEVICE_NAME_MAX
);
2731 iwibdev
->ibdev
.owner
= THIS_MODULE
;
2732 iwdev
->iwibdev
= iwibdev
;
2733 iwibdev
->iwdev
= iwdev
;
2735 iwibdev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
2736 ether_addr_copy((u8
*)&iwibdev
->ibdev
.node_guid
, netdev
->dev_addr
);
2738 iwibdev
->ibdev
.uverbs_cmd_mask
=
2739 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
2740 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
2741 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
2742 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
2743 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
2744 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
2745 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
2746 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
2747 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
2748 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
2749 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
2750 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
2751 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
2752 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
2753 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
2754 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
2755 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
2756 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
2757 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
2758 (1ull << IB_USER_VERBS_CMD_POST_SEND
);
2759 iwibdev
->ibdev
.phys_port_cnt
= 1;
2760 iwibdev
->ibdev
.num_comp_vectors
= iwdev
->ceqs_count
;
2761 iwibdev
->ibdev
.dma_device
= &pcidev
->dev
;
2762 iwibdev
->ibdev
.dev
.parent
= &pcidev
->dev
;
2763 iwibdev
->ibdev
.query_port
= i40iw_query_port
;
2764 iwibdev
->ibdev
.modify_port
= i40iw_modify_port
;
2765 iwibdev
->ibdev
.query_pkey
= i40iw_query_pkey
;
2766 iwibdev
->ibdev
.query_gid
= i40iw_query_gid
;
2767 iwibdev
->ibdev
.alloc_ucontext
= i40iw_alloc_ucontext
;
2768 iwibdev
->ibdev
.dealloc_ucontext
= i40iw_dealloc_ucontext
;
2769 iwibdev
->ibdev
.mmap
= i40iw_mmap
;
2770 iwibdev
->ibdev
.alloc_pd
= i40iw_alloc_pd
;
2771 iwibdev
->ibdev
.dealloc_pd
= i40iw_dealloc_pd
;
2772 iwibdev
->ibdev
.create_qp
= i40iw_create_qp
;
2773 iwibdev
->ibdev
.modify_qp
= i40iw_modify_qp
;
2774 iwibdev
->ibdev
.query_qp
= i40iw_query_qp
;
2775 iwibdev
->ibdev
.destroy_qp
= i40iw_destroy_qp
;
2776 iwibdev
->ibdev
.create_cq
= i40iw_create_cq
;
2777 iwibdev
->ibdev
.destroy_cq
= i40iw_destroy_cq
;
2778 iwibdev
->ibdev
.get_dma_mr
= i40iw_get_dma_mr
;
2779 iwibdev
->ibdev
.reg_user_mr
= i40iw_reg_user_mr
;
2780 iwibdev
->ibdev
.dereg_mr
= i40iw_dereg_mr
;
2781 iwibdev
->ibdev
.alloc_hw_stats
= i40iw_alloc_hw_stats
;
2782 iwibdev
->ibdev
.get_hw_stats
= i40iw_get_hw_stats
;
2783 iwibdev
->ibdev
.query_device
= i40iw_query_device
;
2784 iwibdev
->ibdev
.create_ah
= i40iw_create_ah
;
2785 iwibdev
->ibdev
.destroy_ah
= i40iw_destroy_ah
;
2786 iwibdev
->ibdev
.drain_sq
= i40iw_drain_sq
;
2787 iwibdev
->ibdev
.drain_rq
= i40iw_drain_rq
;
2788 iwibdev
->ibdev
.alloc_mr
= i40iw_alloc_mr
;
2789 iwibdev
->ibdev
.map_mr_sg
= i40iw_map_mr_sg
;
2790 iwibdev
->ibdev
.iwcm
= kzalloc(sizeof(*iwibdev
->ibdev
.iwcm
), GFP_KERNEL
);
2791 if (!iwibdev
->ibdev
.iwcm
) {
2792 ib_dealloc_device(&iwibdev
->ibdev
);
2796 iwibdev
->ibdev
.iwcm
->add_ref
= i40iw_add_ref
;
2797 iwibdev
->ibdev
.iwcm
->rem_ref
= i40iw_rem_ref
;
2798 iwibdev
->ibdev
.iwcm
->get_qp
= i40iw_get_qp
;
2799 iwibdev
->ibdev
.iwcm
->connect
= i40iw_connect
;
2800 iwibdev
->ibdev
.iwcm
->accept
= i40iw_accept
;
2801 iwibdev
->ibdev
.iwcm
->reject
= i40iw_reject
;
2802 iwibdev
->ibdev
.iwcm
->create_listen
= i40iw_create_listen
;
2803 iwibdev
->ibdev
.iwcm
->destroy_listen
= i40iw_destroy_listen
;
2804 memcpy(iwibdev
->ibdev
.iwcm
->ifname
, netdev
->name
,
2805 sizeof(iwibdev
->ibdev
.iwcm
->ifname
));
2806 iwibdev
->ibdev
.get_port_immutable
= i40iw_port_immutable
;
2807 iwibdev
->ibdev
.get_dev_fw_str
= i40iw_get_dev_fw_str
;
2808 iwibdev
->ibdev
.poll_cq
= i40iw_poll_cq
;
2809 iwibdev
->ibdev
.req_notify_cq
= i40iw_req_notify_cq
;
2810 iwibdev
->ibdev
.post_send
= i40iw_post_send
;
2811 iwibdev
->ibdev
.post_recv
= i40iw_post_recv
;
2817 * i40iw_port_ibevent - indicate port event
2818 * @iwdev: iwarp device
2820 void i40iw_port_ibevent(struct i40iw_device
*iwdev
)
2822 struct i40iw_ib_device
*iwibdev
= iwdev
->iwibdev
;
2823 struct ib_event event
;
2825 event
.device
= &iwibdev
->ibdev
;
2826 event
.element
.port_num
= 1;
2827 event
.event
= iwdev
->iw_status
? IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
2828 ib_dispatch_event(&event
);
2832 * i40iw_unregister_rdma_device - unregister of iwarp from IB
2833 * @iwibdev: rdma device ptr
2835 static void i40iw_unregister_rdma_device(struct i40iw_ib_device
*iwibdev
)
2839 for (i
= 0; i
< ARRAY_SIZE(i40iw_dev_attributes
); ++i
)
2840 device_remove_file(&iwibdev
->ibdev
.dev
,
2841 i40iw_dev_attributes
[i
]);
2842 ib_unregister_device(&iwibdev
->ibdev
);
2846 * i40iw_destroy_rdma_device - destroy rdma device and free resources
2847 * @iwibdev: IB device ptr
2849 void i40iw_destroy_rdma_device(struct i40iw_ib_device
*iwibdev
)
2854 i40iw_unregister_rdma_device(iwibdev
);
2855 kfree(iwibdev
->ibdev
.iwcm
);
2856 iwibdev
->ibdev
.iwcm
= NULL
;
2857 wait_event_timeout(iwibdev
->iwdev
->close_wq
,
2858 !atomic64_read(&iwibdev
->iwdev
->use_count
),
2859 I40IW_EVENT_TIMEOUT
);
2860 ib_dealloc_device(&iwibdev
->ibdev
);
2864 * i40iw_register_rdma_device - register iwarp device to IB
2865 * @iwdev: iwarp device
2867 int i40iw_register_rdma_device(struct i40iw_device
*iwdev
)
2870 struct i40iw_ib_device
*iwibdev
;
2872 iwdev
->iwibdev
= i40iw_init_rdma_device(iwdev
);
2873 if (!iwdev
->iwibdev
)
2875 iwibdev
= iwdev
->iwibdev
;
2877 ret
= ib_register_device(&iwibdev
->ibdev
, NULL
);
2881 for (i
= 0; i
< ARRAY_SIZE(i40iw_dev_attributes
); ++i
) {
2883 device_create_file(&iwibdev
->ibdev
.dev
,
2884 i40iw_dev_attributes
[i
]);
2888 device_remove_file(&iwibdev
->ibdev
.dev
, i40iw_dev_attributes
[i
]);
2890 ib_unregister_device(&iwibdev
->ibdev
);
2896 kfree(iwdev
->iwibdev
->ibdev
.iwcm
);
2897 iwdev
->iwibdev
->ibdev
.iwcm
= NULL
;
2898 ib_dealloc_device(&iwdev
->iwibdev
->ibdev
);