2 * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/module.h>
33 #include <linux/moduleparam.h>
34 #include <linux/device.h>
35 #include <linux/netdevice.h>
36 #include <linux/etherdevice.h>
37 #include <linux/delay.h>
38 #include <linux/errno.h>
39 #include <linux/list.h>
40 #include <linux/sched.h>
41 #include <linux/spinlock.h>
42 #include <linux/ethtool.h>
43 #include <linux/rtnetlink.h>
44 #include <linux/inetdevice.h>
45 #include <linux/slab.h>
49 #include <asm/byteorder.h>
51 #include <rdma/iw_cm.h>
52 #include <rdma/ib_verbs.h>
53 #include <rdma/ib_smi.h>
54 #include <rdma/ib_umem.h>
55 #include <rdma/ib_user_verbs.h>
59 #include "iwch_provider.h"
61 #include "iwch_user.h"
64 static int iwch_modify_port(struct ib_device
*ibdev
,
65 u8 port
, int port_modify_mask
,
66 struct ib_port_modify
*props
)
71 static struct ib_ah
*iwch_ah_create(struct ib_pd
*pd
,
72 struct ib_ah_attr
*ah_attr
)
74 return ERR_PTR(-ENOSYS
);
77 static int iwch_ah_destroy(struct ib_ah
*ah
)
82 static int iwch_multicast_attach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
87 static int iwch_multicast_detach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
92 static int iwch_process_mad(struct ib_device
*ibdev
,
96 struct ib_grh
*in_grh
,
97 struct ib_mad
*in_mad
, struct ib_mad
*out_mad
)
102 static int iwch_dealloc_ucontext(struct ib_ucontext
*context
)
104 struct iwch_dev
*rhp
= to_iwch_dev(context
->device
);
105 struct iwch_ucontext
*ucontext
= to_iwch_ucontext(context
);
106 struct iwch_mm_entry
*mm
, *tmp
;
108 PDBG("%s context %p\n", __func__
, context
);
109 list_for_each_entry_safe(mm
, tmp
, &ucontext
->mmaps
, entry
)
111 cxio_release_ucontext(&rhp
->rdev
, &ucontext
->uctx
);
116 static struct ib_ucontext
*iwch_alloc_ucontext(struct ib_device
*ibdev
,
117 struct ib_udata
*udata
)
119 struct iwch_ucontext
*context
;
120 struct iwch_dev
*rhp
= to_iwch_dev(ibdev
);
122 PDBG("%s ibdev %p\n", __func__
, ibdev
);
123 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
125 return ERR_PTR(-ENOMEM
);
126 cxio_init_ucontext(&rhp
->rdev
, &context
->uctx
);
127 INIT_LIST_HEAD(&context
->mmaps
);
128 spin_lock_init(&context
->mmap_lock
);
129 return &context
->ibucontext
;
132 static int iwch_destroy_cq(struct ib_cq
*ib_cq
)
136 PDBG("%s ib_cq %p\n", __func__
, ib_cq
);
137 chp
= to_iwch_cq(ib_cq
);
139 remove_handle(chp
->rhp
, &chp
->rhp
->cqidr
, chp
->cq
.cqid
);
140 atomic_dec(&chp
->refcnt
);
141 wait_event(chp
->wait
, !atomic_read(&chp
->refcnt
));
143 cxio_destroy_cq(&chp
->rhp
->rdev
, &chp
->cq
);
148 static struct ib_cq
*iwch_create_cq(struct ib_device
*ibdev
, int entries
, int vector
,
149 struct ib_ucontext
*ib_context
,
150 struct ib_udata
*udata
)
152 struct iwch_dev
*rhp
;
154 struct iwch_create_cq_resp uresp
;
155 struct iwch_create_cq_req ureq
;
156 struct iwch_ucontext
*ucontext
= NULL
;
158 PDBG("%s ib_dev %p entries %d\n", __func__
, ibdev
, entries
);
159 rhp
= to_iwch_dev(ibdev
);
160 chp
= kzalloc(sizeof(*chp
), GFP_KERNEL
);
162 return ERR_PTR(-ENOMEM
);
165 ucontext
= to_iwch_ucontext(ib_context
);
166 if (!t3a_device(rhp
)) {
167 if (ib_copy_from_udata(&ureq
, udata
, sizeof (ureq
))) {
169 return ERR_PTR(-EFAULT
);
171 chp
->user_rptr_addr
= (u32 __user
*)(unsigned long)ureq
.user_rptr_addr
;
175 if (t3a_device(rhp
)) {
178 * T3A: Add some fluff to handle extra CQEs inserted
179 * for various errors.
180 * Additional CQE possibilities:
182 * incoming RDMA WRITE Failures
183 * incoming RDMA READ REQUEST FAILUREs
184 * NOTE: We cannot ensure the CQ won't overflow.
188 entries
= roundup_pow_of_two(entries
);
189 chp
->cq
.size_log2
= ilog2(entries
);
191 if (cxio_create_cq(&rhp
->rdev
, &chp
->cq
, !ucontext
)) {
193 return ERR_PTR(-ENOMEM
);
196 chp
->ibcq
.cqe
= 1 << chp
->cq
.size_log2
;
197 spin_lock_init(&chp
->lock
);
198 atomic_set(&chp
->refcnt
, 1);
199 init_waitqueue_head(&chp
->wait
);
200 if (insert_handle(rhp
, &rhp
->cqidr
, chp
, chp
->cq
.cqid
)) {
201 cxio_destroy_cq(&chp
->rhp
->rdev
, &chp
->cq
);
203 return ERR_PTR(-ENOMEM
);
207 struct iwch_mm_entry
*mm
;
209 mm
= kmalloc(sizeof *mm
, GFP_KERNEL
);
211 iwch_destroy_cq(&chp
->ibcq
);
212 return ERR_PTR(-ENOMEM
);
214 uresp
.cqid
= chp
->cq
.cqid
;
215 uresp
.size_log2
= chp
->cq
.size_log2
;
216 spin_lock(&ucontext
->mmap_lock
);
217 uresp
.key
= ucontext
->key
;
218 ucontext
->key
+= PAGE_SIZE
;
219 spin_unlock(&ucontext
->mmap_lock
);
220 if (ib_copy_to_udata(udata
, &uresp
, sizeof (uresp
))) {
222 iwch_destroy_cq(&chp
->ibcq
);
223 return ERR_PTR(-EFAULT
);
226 mm
->addr
= virt_to_phys(chp
->cq
.queue
);
227 mm
->len
= PAGE_ALIGN((1UL << uresp
.size_log2
) *
228 sizeof (struct t3_cqe
));
229 insert_mmap(ucontext
, mm
);
231 PDBG("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",
232 chp
->cq
.cqid
, chp
, (1 << chp
->cq
.size_log2
),
233 (unsigned long long) chp
->cq
.dma_addr
);
237 static int iwch_resize_cq(struct ib_cq
*cq
, int cqe
, struct ib_udata
*udata
)
240 struct iwch_cq
*chp
= to_iwch_cq(cq
);
241 struct t3_cq oldcq
, newcq
;
244 PDBG("%s ib_cq %p cqe %d\n", __func__
, cq
, cqe
);
246 /* We don't downsize... */
250 /* create new t3_cq with new size */
251 cqe
= roundup_pow_of_two(cqe
+1);
252 newcq
.size_log2
= ilog2(cqe
);
254 /* Dont allow resize to less than the current wce count */
255 if (cqe
< Q_COUNT(chp
->cq
.rptr
, chp
->cq
.wptr
)) {
259 /* Quiesce all QPs using this CQ */
260 ret
= iwch_quiesce_qps(chp
);
265 ret
= cxio_create_cq(&chp
->rhp
->rdev
, &newcq
);
271 memcpy(newcq
.queue
, chp
->cq
.queue
, (1 << chp
->cq
.size_log2
) *
272 sizeof(struct t3_cqe
));
274 /* old iwch_qp gets new t3_cq but keeps old cqid */
277 chp
->cq
.cqid
= oldcq
.cqid
;
279 /* resize new t3_cq to update the HW context */
280 ret
= cxio_resize_cq(&chp
->rhp
->rdev
, &chp
->cq
);
285 chp
->ibcq
.cqe
= (1<<chp
->cq
.size_log2
) - 1;
287 /* destroy old t3_cq */
288 oldcq
.cqid
= newcq
.cqid
;
289 ret
= cxio_destroy_cq(&chp
->rhp
->rdev
, &oldcq
);
291 printk(KERN_ERR MOD
"%s - cxio_destroy_cq failed %d\n",
295 /* add user hooks here */
298 ret
= iwch_resume_qps(chp
);
305 static int iwch_arm_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
307 struct iwch_dev
*rhp
;
309 enum t3_cq_opcode cq_op
;
314 chp
= to_iwch_cq(ibcq
);
316 if ((flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
)
320 if (chp
->user_rptr_addr
) {
321 if (get_user(rptr
, chp
->user_rptr_addr
))
323 spin_lock_irqsave(&chp
->lock
, flag
);
326 spin_lock_irqsave(&chp
->lock
, flag
);
327 PDBG("%s rptr 0x%x\n", __func__
, chp
->cq
.rptr
);
328 err
= cxio_hal_cq_op(&rhp
->rdev
, &chp
->cq
, cq_op
, 0);
329 spin_unlock_irqrestore(&chp
->lock
, flag
);
331 printk(KERN_ERR MOD
"Error %d rearming CQID 0x%x\n", err
,
333 if (err
> 0 && !(flags
& IB_CQ_REPORT_MISSED_EVENTS
))
338 static int iwch_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
340 int len
= vma
->vm_end
- vma
->vm_start
;
341 u32 key
= vma
->vm_pgoff
<< PAGE_SHIFT
;
342 struct cxio_rdev
*rdev_p
;
344 struct iwch_mm_entry
*mm
;
345 struct iwch_ucontext
*ucontext
;
348 PDBG("%s pgoff 0x%lx key 0x%x len %d\n", __func__
, vma
->vm_pgoff
,
351 if (vma
->vm_start
& (PAGE_SIZE
-1)) {
355 rdev_p
= &(to_iwch_dev(context
->device
)->rdev
);
356 ucontext
= to_iwch_ucontext(context
);
358 mm
= remove_mmap(ucontext
, key
, len
);
364 if ((addr
>= rdev_p
->rnic_info
.udbell_physbase
) &&
365 (addr
< (rdev_p
->rnic_info
.udbell_physbase
+
366 rdev_p
->rnic_info
.udbell_len
))) {
369 * Map T3 DB register.
371 if (vma
->vm_flags
& VM_READ
) {
375 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
376 vma
->vm_flags
|= VM_DONTCOPY
| VM_DONTEXPAND
;
377 vma
->vm_flags
&= ~VM_MAYREAD
;
378 ret
= io_remap_pfn_range(vma
, vma
->vm_start
,
380 len
, vma
->vm_page_prot
);
384 * Map WQ or CQ contig dma memory...
386 ret
= remap_pfn_range(vma
, vma
->vm_start
,
388 len
, vma
->vm_page_prot
);
394 static int iwch_deallocate_pd(struct ib_pd
*pd
)
396 struct iwch_dev
*rhp
;
399 php
= to_iwch_pd(pd
);
401 PDBG("%s ibpd %p pdid 0x%x\n", __func__
, pd
, php
->pdid
);
402 cxio_hal_put_pdid(rhp
->rdev
.rscp
, php
->pdid
);
407 static struct ib_pd
*iwch_allocate_pd(struct ib_device
*ibdev
,
408 struct ib_ucontext
*context
,
409 struct ib_udata
*udata
)
413 struct iwch_dev
*rhp
;
415 PDBG("%s ibdev %p\n", __func__
, ibdev
);
416 rhp
= (struct iwch_dev
*) ibdev
;
417 pdid
= cxio_hal_get_pdid(rhp
->rdev
.rscp
);
419 return ERR_PTR(-EINVAL
);
420 php
= kzalloc(sizeof(*php
), GFP_KERNEL
);
422 cxio_hal_put_pdid(rhp
->rdev
.rscp
, pdid
);
423 return ERR_PTR(-ENOMEM
);
428 if (ib_copy_to_udata(udata
, &php
->pdid
, sizeof (__u32
))) {
429 iwch_deallocate_pd(&php
->ibpd
);
430 return ERR_PTR(-EFAULT
);
433 PDBG("%s pdid 0x%0x ptr 0x%p\n", __func__
, pdid
, php
);
437 static int iwch_dereg_mr(struct ib_mr
*ib_mr
)
439 struct iwch_dev
*rhp
;
443 PDBG("%s ib_mr %p\n", __func__
, ib_mr
);
444 /* There can be no memory windows */
445 if (atomic_read(&ib_mr
->usecnt
))
448 mhp
= to_iwch_mr(ib_mr
);
450 mmid
= mhp
->attr
.stag
>> 8;
451 cxio_dereg_mem(&rhp
->rdev
, mhp
->attr
.stag
, mhp
->attr
.pbl_size
,
454 remove_handle(rhp
, &rhp
->mmidr
, mmid
);
456 kfree((void *) (unsigned long) mhp
->kva
);
458 ib_umem_release(mhp
->umem
);
459 PDBG("%s mmid 0x%x ptr %p\n", __func__
, mmid
, mhp
);
464 static struct ib_mr
*iwch_register_phys_mem(struct ib_pd
*pd
,
465 struct ib_phys_buf
*buffer_list
,
474 struct iwch_dev
*rhp
;
479 PDBG("%s ib_pd %p\n", __func__
, pd
);
480 php
= to_iwch_pd(pd
);
483 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
485 return ERR_PTR(-ENOMEM
);
489 /* First check that we have enough alignment */
490 if ((*iova_start
& ~PAGE_MASK
) != (buffer_list
[0].addr
& ~PAGE_MASK
)) {
495 if (num_phys_buf
> 1 &&
496 ((buffer_list
[0].addr
+ buffer_list
[0].size
) & ~PAGE_MASK
)) {
501 ret
= build_phys_page_list(buffer_list
, num_phys_buf
, iova_start
,
502 &total_size
, &npages
, &shift
, &page_list
);
506 ret
= iwch_alloc_pbl(mhp
, npages
);
512 ret
= iwch_write_pbl(mhp
, page_list
, npages
, 0);
517 mhp
->attr
.pdid
= php
->pdid
;
520 mhp
->attr
.perms
= iwch_ib_to_tpt_access(acc
);
521 mhp
->attr
.va_fbo
= *iova_start
;
522 mhp
->attr
.page_size
= shift
- 12;
524 mhp
->attr
.len
= (u32
) total_size
;
525 mhp
->attr
.pbl_size
= npages
;
526 ret
= iwch_register_mem(rhp
, php
, mhp
, shift
);
541 static int iwch_reregister_phys_mem(struct ib_mr
*mr
,
544 struct ib_phys_buf
*buffer_list
,
546 int acc
, u64
* iova_start
)
549 struct iwch_mr mh
, *mhp
;
551 struct iwch_dev
*rhp
;
552 __be64
*page_list
= NULL
;
558 PDBG("%s ib_mr %p ib_pd %p\n", __func__
, mr
, pd
);
560 /* There can be no memory windows */
561 if (atomic_read(&mr
->usecnt
))
564 mhp
= to_iwch_mr(mr
);
566 php
= to_iwch_pd(mr
->pd
);
568 /* make sure we are on the same adapter */
572 memcpy(&mh
, mhp
, sizeof *mhp
);
574 if (mr_rereg_mask
& IB_MR_REREG_PD
)
575 php
= to_iwch_pd(pd
);
576 if (mr_rereg_mask
& IB_MR_REREG_ACCESS
)
577 mh
.attr
.perms
= iwch_ib_to_tpt_access(acc
);
578 if (mr_rereg_mask
& IB_MR_REREG_TRANS
) {
579 ret
= build_phys_page_list(buffer_list
, num_phys_buf
,
581 &total_size
, &npages
,
587 ret
= iwch_reregister_mem(rhp
, php
, &mh
, shift
, npages
);
592 if (mr_rereg_mask
& IB_MR_REREG_PD
)
593 mhp
->attr
.pdid
= php
->pdid
;
594 if (mr_rereg_mask
& IB_MR_REREG_ACCESS
)
595 mhp
->attr
.perms
= iwch_ib_to_tpt_access(acc
);
596 if (mr_rereg_mask
& IB_MR_REREG_TRANS
) {
598 mhp
->attr
.va_fbo
= *iova_start
;
599 mhp
->attr
.page_size
= shift
- 12;
600 mhp
->attr
.len
= (u32
) total_size
;
601 mhp
->attr
.pbl_size
= npages
;
608 static struct ib_mr
*iwch_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
609 u64 virt
, int acc
, struct ib_udata
*udata
)
615 struct ib_umem_chunk
*chunk
;
616 struct iwch_dev
*rhp
;
619 struct iwch_reg_user_mr_resp uresp
;
621 PDBG("%s ib_pd %p\n", __func__
, pd
);
623 php
= to_iwch_pd(pd
);
625 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
627 return ERR_PTR(-ENOMEM
);
631 mhp
->umem
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
632 if (IS_ERR(mhp
->umem
)) {
633 err
= PTR_ERR(mhp
->umem
);
638 shift
= ffs(mhp
->umem
->page_size
) - 1;
641 list_for_each_entry(chunk
, &mhp
->umem
->chunk_list
, list
)
644 err
= iwch_alloc_pbl(mhp
, n
);
648 pages
= (__be64
*) __get_free_page(GFP_KERNEL
);
656 list_for_each_entry(chunk
, &mhp
->umem
->chunk_list
, list
)
657 for (j
= 0; j
< chunk
->nmap
; ++j
) {
658 len
= sg_dma_len(&chunk
->page_list
[j
]) >> shift
;
659 for (k
= 0; k
< len
; ++k
) {
660 pages
[i
++] = cpu_to_be64(sg_dma_address(
661 &chunk
->page_list
[j
]) +
662 mhp
->umem
->page_size
* k
);
663 if (i
== PAGE_SIZE
/ sizeof *pages
) {
664 err
= iwch_write_pbl(mhp
, pages
, i
, n
);
674 err
= iwch_write_pbl(mhp
, pages
, i
, n
);
677 free_page((unsigned long) pages
);
681 mhp
->attr
.pdid
= php
->pdid
;
683 mhp
->attr
.perms
= iwch_ib_to_tpt_access(acc
);
684 mhp
->attr
.va_fbo
= virt
;
685 mhp
->attr
.page_size
= shift
- 12;
686 mhp
->attr
.len
= (u32
) length
;
688 err
= iwch_register_mem(rhp
, php
, mhp
, shift
);
692 if (udata
&& !t3a_device(rhp
)) {
693 uresp
.pbl_addr
= (mhp
->attr
.pbl_addr
-
694 rhp
->rdev
.rnic_info
.pbl_base
) >> 3;
695 PDBG("%s user resp pbl_addr 0x%x\n", __func__
,
698 if (ib_copy_to_udata(udata
, &uresp
, sizeof (uresp
))) {
699 iwch_dereg_mr(&mhp
->ibmr
);
711 ib_umem_release(mhp
->umem
);
716 static struct ib_mr
*iwch_get_dma_mr(struct ib_pd
*pd
, int acc
)
718 struct ib_phys_buf bl
;
722 PDBG("%s ib_pd %p\n", __func__
, pd
);
725 * T3 only supports 32 bits of size.
727 bl
.size
= 0xffffffff;
730 ibmr
= iwch_register_phys_mem(pd
, &bl
, 1, acc
, &kva
);
734 static struct ib_mw
*iwch_alloc_mw(struct ib_pd
*pd
)
736 struct iwch_dev
*rhp
;
743 php
= to_iwch_pd(pd
);
745 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
747 return ERR_PTR(-ENOMEM
);
748 ret
= cxio_allocate_window(&rhp
->rdev
, &stag
, php
->pdid
);
754 mhp
->attr
.pdid
= php
->pdid
;
755 mhp
->attr
.type
= TPT_MW
;
756 mhp
->attr
.stag
= stag
;
758 mhp
->ibmw
.rkey
= stag
;
759 if (insert_handle(rhp
, &rhp
->mmidr
, mhp
, mmid
)) {
760 cxio_deallocate_window(&rhp
->rdev
, mhp
->attr
.stag
);
762 return ERR_PTR(-ENOMEM
);
764 PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__
, mmid
, mhp
, stag
);
768 static int iwch_dealloc_mw(struct ib_mw
*mw
)
770 struct iwch_dev
*rhp
;
774 mhp
= to_iwch_mw(mw
);
776 mmid
= (mw
->rkey
) >> 8;
777 cxio_deallocate_window(&rhp
->rdev
, mhp
->attr
.stag
);
778 remove_handle(rhp
, &rhp
->mmidr
, mmid
);
780 PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__
, mw
, mmid
, mhp
);
784 static struct ib_mr
*iwch_alloc_fast_reg_mr(struct ib_pd
*pd
, int pbl_depth
)
786 struct iwch_dev
*rhp
;
793 php
= to_iwch_pd(pd
);
795 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
800 ret
= iwch_alloc_pbl(mhp
, pbl_depth
);
803 mhp
->attr
.pbl_size
= pbl_depth
;
804 ret
= cxio_allocate_stag(&rhp
->rdev
, &stag
, php
->pdid
,
805 mhp
->attr
.pbl_size
, mhp
->attr
.pbl_addr
);
808 mhp
->attr
.pdid
= php
->pdid
;
809 mhp
->attr
.type
= TPT_NON_SHARED_MR
;
810 mhp
->attr
.stag
= stag
;
813 mhp
->ibmr
.rkey
= mhp
->ibmr
.lkey
= stag
;
814 if (insert_handle(rhp
, &rhp
->mmidr
, mhp
, mmid
))
817 PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__
, mmid
, mhp
, stag
);
820 cxio_dereg_mem(&rhp
->rdev
, stag
, mhp
->attr
.pbl_size
,
830 static struct ib_fast_reg_page_list
*iwch_alloc_fastreg_pbl(
831 struct ib_device
*device
,
834 struct ib_fast_reg_page_list
*page_list
;
836 page_list
= kmalloc(sizeof *page_list
+ page_list_len
* sizeof(u64
),
839 return ERR_PTR(-ENOMEM
);
841 page_list
->page_list
= (u64
*)(page_list
+ 1);
842 page_list
->max_page_list_len
= page_list_len
;
847 static void iwch_free_fastreg_pbl(struct ib_fast_reg_page_list
*page_list
)
852 static int iwch_destroy_qp(struct ib_qp
*ib_qp
)
854 struct iwch_dev
*rhp
;
856 struct iwch_qp_attributes attrs
;
857 struct iwch_ucontext
*ucontext
;
859 qhp
= to_iwch_qp(ib_qp
);
862 attrs
.next_state
= IWCH_QP_STATE_ERROR
;
863 iwch_modify_qp(rhp
, qhp
, IWCH_QP_ATTR_NEXT_STATE
, &attrs
, 0);
864 wait_event(qhp
->wait
, !qhp
->ep
);
866 remove_handle(rhp
, &rhp
->qpidr
, qhp
->wq
.qpid
);
868 atomic_dec(&qhp
->refcnt
);
869 wait_event(qhp
->wait
, !atomic_read(&qhp
->refcnt
));
871 ucontext
= ib_qp
->uobject
? to_iwch_ucontext(ib_qp
->uobject
->context
)
873 cxio_destroy_qp(&rhp
->rdev
, &qhp
->wq
,
874 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
);
876 PDBG("%s ib_qp %p qpid 0x%0x qhp %p\n", __func__
,
877 ib_qp
, qhp
->wq
.qpid
, qhp
);
882 static struct ib_qp
*iwch_create_qp(struct ib_pd
*pd
,
883 struct ib_qp_init_attr
*attrs
,
884 struct ib_udata
*udata
)
886 struct iwch_dev
*rhp
;
889 struct iwch_cq
*schp
;
890 struct iwch_cq
*rchp
;
891 struct iwch_create_qp_resp uresp
;
892 int wqsize
, sqsize
, rqsize
;
893 struct iwch_ucontext
*ucontext
;
895 PDBG("%s ib_pd %p\n", __func__
, pd
);
896 if (attrs
->qp_type
!= IB_QPT_RC
)
897 return ERR_PTR(-EINVAL
);
898 php
= to_iwch_pd(pd
);
900 schp
= get_chp(rhp
, ((struct iwch_cq
*) attrs
->send_cq
)->cq
.cqid
);
901 rchp
= get_chp(rhp
, ((struct iwch_cq
*) attrs
->recv_cq
)->cq
.cqid
);
903 return ERR_PTR(-EINVAL
);
905 /* The RQT size must be # of entries + 1 rounded up to a power of two */
906 rqsize
= roundup_pow_of_two(attrs
->cap
.max_recv_wr
);
907 if (rqsize
== attrs
->cap
.max_recv_wr
)
908 rqsize
= roundup_pow_of_two(attrs
->cap
.max_recv_wr
+1);
910 /* T3 doesn't support RQT depth < 16 */
914 if (rqsize
> T3_MAX_RQ_SIZE
)
915 return ERR_PTR(-EINVAL
);
917 if (attrs
->cap
.max_inline_data
> T3_MAX_INLINE
)
918 return ERR_PTR(-EINVAL
);
921 * NOTE: The SQ and total WQ sizes don't need to be
922 * a power of two. However, all the code assumes
923 * they are. EG: Q_FREECNT() and friends.
925 sqsize
= roundup_pow_of_two(attrs
->cap
.max_send_wr
);
926 wqsize
= roundup_pow_of_two(rqsize
+ sqsize
);
929 * Kernel users need more wq space for fastreg WRs which can take
932 ucontext
= pd
->uobject
? to_iwch_ucontext(pd
->uobject
->context
) : NULL
;
933 if (!ucontext
&& wqsize
< (rqsize
+ (2 * sqsize
)))
934 wqsize
= roundup_pow_of_two(rqsize
+
935 roundup_pow_of_two(attrs
->cap
.max_send_wr
* 2));
936 PDBG("%s wqsize %d sqsize %d rqsize %d\n", __func__
,
937 wqsize
, sqsize
, rqsize
);
938 qhp
= kzalloc(sizeof(*qhp
), GFP_KERNEL
);
940 return ERR_PTR(-ENOMEM
);
941 qhp
->wq
.size_log2
= ilog2(wqsize
);
942 qhp
->wq
.rq_size_log2
= ilog2(rqsize
);
943 qhp
->wq
.sq_size_log2
= ilog2(sqsize
);
944 if (cxio_create_qp(&rhp
->rdev
, !udata
, &qhp
->wq
,
945 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
)) {
947 return ERR_PTR(-ENOMEM
);
950 attrs
->cap
.max_recv_wr
= rqsize
- 1;
951 attrs
->cap
.max_send_wr
= sqsize
;
952 attrs
->cap
.max_inline_data
= T3_MAX_INLINE
;
955 qhp
->attr
.pd
= php
->pdid
;
956 qhp
->attr
.scq
= ((struct iwch_cq
*) attrs
->send_cq
)->cq
.cqid
;
957 qhp
->attr
.rcq
= ((struct iwch_cq
*) attrs
->recv_cq
)->cq
.cqid
;
958 qhp
->attr
.sq_num_entries
= attrs
->cap
.max_send_wr
;
959 qhp
->attr
.rq_num_entries
= attrs
->cap
.max_recv_wr
;
960 qhp
->attr
.sq_max_sges
= attrs
->cap
.max_send_sge
;
961 qhp
->attr
.sq_max_sges_rdma_write
= attrs
->cap
.max_send_sge
;
962 qhp
->attr
.rq_max_sges
= attrs
->cap
.max_recv_sge
;
963 qhp
->attr
.state
= IWCH_QP_STATE_IDLE
;
964 qhp
->attr
.next_state
= IWCH_QP_STATE_IDLE
;
967 * XXX - These don't get passed in from the openib user
968 * at create time. The CM sets them via a QP modify.
969 * Need to fix... I think the CM should
971 qhp
->attr
.enable_rdma_read
= 1;
972 qhp
->attr
.enable_rdma_write
= 1;
973 qhp
->attr
.enable_bind
= 1;
974 qhp
->attr
.max_ord
= 1;
975 qhp
->attr
.max_ird
= 1;
977 spin_lock_init(&qhp
->lock
);
978 init_waitqueue_head(&qhp
->wait
);
979 atomic_set(&qhp
->refcnt
, 1);
981 if (insert_handle(rhp
, &rhp
->qpidr
, qhp
, qhp
->wq
.qpid
)) {
982 cxio_destroy_qp(&rhp
->rdev
, &qhp
->wq
,
983 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
);
985 return ERR_PTR(-ENOMEM
);
990 struct iwch_mm_entry
*mm1
, *mm2
;
992 mm1
= kmalloc(sizeof *mm1
, GFP_KERNEL
);
994 iwch_destroy_qp(&qhp
->ibqp
);
995 return ERR_PTR(-ENOMEM
);
998 mm2
= kmalloc(sizeof *mm2
, GFP_KERNEL
);
1001 iwch_destroy_qp(&qhp
->ibqp
);
1002 return ERR_PTR(-ENOMEM
);
1005 uresp
.qpid
= qhp
->wq
.qpid
;
1006 uresp
.size_log2
= qhp
->wq
.size_log2
;
1007 uresp
.sq_size_log2
= qhp
->wq
.sq_size_log2
;
1008 uresp
.rq_size_log2
= qhp
->wq
.rq_size_log2
;
1009 spin_lock(&ucontext
->mmap_lock
);
1010 uresp
.key
= ucontext
->key
;
1011 ucontext
->key
+= PAGE_SIZE
;
1012 uresp
.db_key
= ucontext
->key
;
1013 ucontext
->key
+= PAGE_SIZE
;
1014 spin_unlock(&ucontext
->mmap_lock
);
1015 if (ib_copy_to_udata(udata
, &uresp
, sizeof (uresp
))) {
1018 iwch_destroy_qp(&qhp
->ibqp
);
1019 return ERR_PTR(-EFAULT
);
1021 mm1
->key
= uresp
.key
;
1022 mm1
->addr
= virt_to_phys(qhp
->wq
.queue
);
1023 mm1
->len
= PAGE_ALIGN(wqsize
* sizeof (union t3_wr
));
1024 insert_mmap(ucontext
, mm1
);
1025 mm2
->key
= uresp
.db_key
;
1026 mm2
->addr
= qhp
->wq
.udb
& PAGE_MASK
;
1027 mm2
->len
= PAGE_SIZE
;
1028 insert_mmap(ucontext
, mm2
);
1030 qhp
->ibqp
.qp_num
= qhp
->wq
.qpid
;
1031 init_timer(&(qhp
->timer
));
1032 PDBG("%s sq_num_entries %d, rq_num_entries %d "
1033 "qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",
1034 __func__
, qhp
->attr
.sq_num_entries
, qhp
->attr
.rq_num_entries
,
1035 qhp
->wq
.qpid
, qhp
, (unsigned long long) qhp
->wq
.dma_addr
,
1036 1 << qhp
->wq
.size_log2
, qhp
->wq
.rq_addr
);
1040 static int iwch_ib_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
1041 int attr_mask
, struct ib_udata
*udata
)
1043 struct iwch_dev
*rhp
;
1044 struct iwch_qp
*qhp
;
1045 enum iwch_qp_attr_mask mask
= 0;
1046 struct iwch_qp_attributes attrs
;
1048 PDBG("%s ib_qp %p\n", __func__
, ibqp
);
1050 /* iwarp does not support the RTR state */
1051 if ((attr_mask
& IB_QP_STATE
) && (attr
->qp_state
== IB_QPS_RTR
))
1052 attr_mask
&= ~IB_QP_STATE
;
1054 /* Make sure we still have something left to do */
1058 memset(&attrs
, 0, sizeof attrs
);
1059 qhp
= to_iwch_qp(ibqp
);
1062 attrs
.next_state
= iwch_convert_state(attr
->qp_state
);
1063 attrs
.enable_rdma_read
= (attr
->qp_access_flags
&
1064 IB_ACCESS_REMOTE_READ
) ? 1 : 0;
1065 attrs
.enable_rdma_write
= (attr
->qp_access_flags
&
1066 IB_ACCESS_REMOTE_WRITE
) ? 1 : 0;
1067 attrs
.enable_bind
= (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
) ? 1 : 0;
1070 mask
|= (attr_mask
& IB_QP_STATE
) ? IWCH_QP_ATTR_NEXT_STATE
: 0;
1071 mask
|= (attr_mask
& IB_QP_ACCESS_FLAGS
) ?
1072 (IWCH_QP_ATTR_ENABLE_RDMA_READ
|
1073 IWCH_QP_ATTR_ENABLE_RDMA_WRITE
|
1074 IWCH_QP_ATTR_ENABLE_RDMA_BIND
) : 0;
1076 return iwch_modify_qp(rhp
, qhp
, mask
, &attrs
, 0);
1079 void iwch_qp_add_ref(struct ib_qp
*qp
)
1081 PDBG("%s ib_qp %p\n", __func__
, qp
);
1082 atomic_inc(&(to_iwch_qp(qp
)->refcnt
));
1085 void iwch_qp_rem_ref(struct ib_qp
*qp
)
1087 PDBG("%s ib_qp %p\n", __func__
, qp
);
1088 if (atomic_dec_and_test(&(to_iwch_qp(qp
)->refcnt
)))
1089 wake_up(&(to_iwch_qp(qp
)->wait
));
1092 static struct ib_qp
*iwch_get_qp(struct ib_device
*dev
, int qpn
)
1094 PDBG("%s ib_dev %p qpn 0x%x\n", __func__
, dev
, qpn
);
1095 return (struct ib_qp
*)get_qhp(to_iwch_dev(dev
), qpn
);
1099 static int iwch_query_pkey(struct ib_device
*ibdev
,
1100 u8 port
, u16 index
, u16
* pkey
)
1102 PDBG("%s ibdev %p\n", __func__
, ibdev
);
1107 static int iwch_query_gid(struct ib_device
*ibdev
, u8 port
,
1108 int index
, union ib_gid
*gid
)
1110 struct iwch_dev
*dev
;
1112 PDBG("%s ibdev %p, port %d, index %d, gid %p\n",
1113 __func__
, ibdev
, port
, index
, gid
);
1114 dev
= to_iwch_dev(ibdev
);
1115 BUG_ON(port
== 0 || port
> 2);
1116 memset(&(gid
->raw
[0]), 0, sizeof(gid
->raw
));
1117 memcpy(&(gid
->raw
[0]), dev
->rdev
.port_info
.lldevs
[port
-1]->dev_addr
, 6);
1121 static u64
fw_vers_string_to_u64(struct iwch_dev
*iwch_dev
)
1123 struct ethtool_drvinfo info
;
1124 struct net_device
*lldev
= iwch_dev
->rdev
.t3cdev_p
->lldev
;
1126 unsigned fw_maj
, fw_min
, fw_mic
;
1128 lldev
->ethtool_ops
->get_drvinfo(lldev
, &info
);
1130 next
= info
.fw_version
+ 1;
1131 cp
= strsep(&next
, ".");
1132 sscanf(cp
, "%i", &fw_maj
);
1133 cp
= strsep(&next
, ".");
1134 sscanf(cp
, "%i", &fw_min
);
1135 cp
= strsep(&next
, ".");
1136 sscanf(cp
, "%i", &fw_mic
);
1138 return (((u64
)fw_maj
& 0xffff) << 32) | ((fw_min
& 0xffff) << 16) |
1142 static int iwch_query_device(struct ib_device
*ibdev
,
1143 struct ib_device_attr
*props
)
1146 struct iwch_dev
*dev
;
1147 PDBG("%s ibdev %p\n", __func__
, ibdev
);
1149 dev
= to_iwch_dev(ibdev
);
1150 memset(props
, 0, sizeof *props
);
1151 memcpy(&props
->sys_image_guid
, dev
->rdev
.t3cdev_p
->lldev
->dev_addr
, 6);
1152 props
->hw_ver
= dev
->rdev
.t3cdev_p
->type
;
1153 props
->fw_ver
= fw_vers_string_to_u64(dev
);
1154 props
->device_cap_flags
= dev
->device_cap_flags
;
1155 props
->page_size_cap
= dev
->attr
.mem_pgsizes_bitmask
;
1156 props
->vendor_id
= (u32
)dev
->rdev
.rnic_info
.pdev
->vendor
;
1157 props
->vendor_part_id
= (u32
)dev
->rdev
.rnic_info
.pdev
->device
;
1158 props
->max_mr_size
= dev
->attr
.max_mr_size
;
1159 props
->max_qp
= dev
->attr
.max_qps
;
1160 props
->max_qp_wr
= dev
->attr
.max_wrs
;
1161 props
->max_sge
= dev
->attr
.max_sge_per_wr
;
1162 props
->max_sge_rd
= 1;
1163 props
->max_qp_rd_atom
= dev
->attr
.max_rdma_reads_per_qp
;
1164 props
->max_qp_init_rd_atom
= dev
->attr
.max_rdma_reads_per_qp
;
1165 props
->max_cq
= dev
->attr
.max_cqs
;
1166 props
->max_cqe
= dev
->attr
.max_cqes_per_cq
;
1167 props
->max_mr
= dev
->attr
.max_mem_regs
;
1168 props
->max_pd
= dev
->attr
.max_pds
;
1169 props
->local_ca_ack_delay
= 0;
1170 props
->max_fast_reg_page_list_len
= T3_MAX_FASTREG_DEPTH
;
1175 static int iwch_query_port(struct ib_device
*ibdev
,
1176 u8 port
, struct ib_port_attr
*props
)
1178 struct iwch_dev
*dev
;
1179 struct net_device
*netdev
;
1180 struct in_device
*inetdev
;
1182 PDBG("%s ibdev %p\n", __func__
, ibdev
);
1184 dev
= to_iwch_dev(ibdev
);
1185 netdev
= dev
->rdev
.port_info
.lldevs
[port
-1];
1187 memset(props
, 0, sizeof(struct ib_port_attr
));
1188 props
->max_mtu
= IB_MTU_4096
;
1189 if (netdev
->mtu
>= 4096)
1190 props
->active_mtu
= IB_MTU_4096
;
1191 else if (netdev
->mtu
>= 2048)
1192 props
->active_mtu
= IB_MTU_2048
;
1193 else if (netdev
->mtu
>= 1024)
1194 props
->active_mtu
= IB_MTU_1024
;
1195 else if (netdev
->mtu
>= 512)
1196 props
->active_mtu
= IB_MTU_512
;
1198 props
->active_mtu
= IB_MTU_256
;
1200 if (!netif_carrier_ok(netdev
))
1201 props
->state
= IB_PORT_DOWN
;
1203 inetdev
= in_dev_get(netdev
);
1205 if (inetdev
->ifa_list
)
1206 props
->state
= IB_PORT_ACTIVE
;
1208 props
->state
= IB_PORT_INIT
;
1209 in_dev_put(inetdev
);
1211 props
->state
= IB_PORT_INIT
;
1214 props
->port_cap_flags
=
1216 IB_PORT_SNMP_TUNNEL_SUP
|
1217 IB_PORT_REINIT_SUP
|
1218 IB_PORT_DEVICE_MGMT_SUP
|
1219 IB_PORT_VENDOR_CLASS_SUP
| IB_PORT_BOOT_MGMT_SUP
;
1220 props
->gid_tbl_len
= 1;
1221 props
->pkey_tbl_len
= 1;
1222 props
->active_width
= 2;
1223 props
->active_speed
= 2;
1224 props
->max_msg_sz
= -1;
1229 static ssize_t
show_rev(struct device
*dev
, struct device_attribute
*attr
,
1232 struct iwch_dev
*iwch_dev
= container_of(dev
, struct iwch_dev
,
1234 PDBG("%s dev 0x%p\n", __func__
, dev
);
1235 return sprintf(buf
, "%d\n", iwch_dev
->rdev
.t3cdev_p
->type
);
1238 static ssize_t
show_fw_ver(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
1240 struct iwch_dev
*iwch_dev
= container_of(dev
, struct iwch_dev
,
1242 struct ethtool_drvinfo info
;
1243 struct net_device
*lldev
= iwch_dev
->rdev
.t3cdev_p
->lldev
;
1245 PDBG("%s dev 0x%p\n", __func__
, dev
);
1246 lldev
->ethtool_ops
->get_drvinfo(lldev
, &info
);
1247 return sprintf(buf
, "%s\n", info
.fw_version
);
1250 static ssize_t
show_hca(struct device
*dev
, struct device_attribute
*attr
,
1253 struct iwch_dev
*iwch_dev
= container_of(dev
, struct iwch_dev
,
1255 struct ethtool_drvinfo info
;
1256 struct net_device
*lldev
= iwch_dev
->rdev
.t3cdev_p
->lldev
;
1258 PDBG("%s dev 0x%p\n", __func__
, dev
);
1259 lldev
->ethtool_ops
->get_drvinfo(lldev
, &info
);
1260 return sprintf(buf
, "%s\n", info
.driver
);
1263 static ssize_t
show_board(struct device
*dev
, struct device_attribute
*attr
,
1266 struct iwch_dev
*iwch_dev
= container_of(dev
, struct iwch_dev
,
1268 PDBG("%s dev 0x%p\n", __func__
, dev
);
1269 return sprintf(buf
, "%x.%x\n", iwch_dev
->rdev
.rnic_info
.pdev
->vendor
,
1270 iwch_dev
->rdev
.rnic_info
.pdev
->device
);
1273 static int iwch_get_mib(struct ib_device
*ibdev
,
1274 union rdma_protocol_stats
*stats
)
1276 struct iwch_dev
*dev
;
1277 struct tp_mib_stats m
;
1280 PDBG("%s ibdev %p\n", __func__
, ibdev
);
1281 dev
= to_iwch_dev(ibdev
);
1282 ret
= dev
->rdev
.t3cdev_p
->ctl(dev
->rdev
.t3cdev_p
, RDMA_GET_MIB
, &m
);
1286 memset(stats
, 0, sizeof *stats
);
1287 stats
->iw
.ipInReceives
= ((u64
) m
.ipInReceive_hi
<< 32) +
1289 stats
->iw
.ipInHdrErrors
= ((u64
) m
.ipInHdrErrors_hi
<< 32) +
1291 stats
->iw
.ipInAddrErrors
= ((u64
) m
.ipInAddrErrors_hi
<< 32) +
1292 m
.ipInAddrErrors_lo
;
1293 stats
->iw
.ipInUnknownProtos
= ((u64
) m
.ipInUnknownProtos_hi
<< 32) +
1294 m
.ipInUnknownProtos_lo
;
1295 stats
->iw
.ipInDiscards
= ((u64
) m
.ipInDiscards_hi
<< 32) +
1297 stats
->iw
.ipInDelivers
= ((u64
) m
.ipInDelivers_hi
<< 32) +
1299 stats
->iw
.ipOutRequests
= ((u64
) m
.ipOutRequests_hi
<< 32) +
1301 stats
->iw
.ipOutDiscards
= ((u64
) m
.ipOutDiscards_hi
<< 32) +
1303 stats
->iw
.ipOutNoRoutes
= ((u64
) m
.ipOutNoRoutes_hi
<< 32) +
1305 stats
->iw
.ipReasmTimeout
= (u64
) m
.ipReasmTimeout
;
1306 stats
->iw
.ipReasmReqds
= (u64
) m
.ipReasmReqds
;
1307 stats
->iw
.ipReasmOKs
= (u64
) m
.ipReasmOKs
;
1308 stats
->iw
.ipReasmFails
= (u64
) m
.ipReasmFails
;
1309 stats
->iw
.tcpActiveOpens
= (u64
) m
.tcpActiveOpens
;
1310 stats
->iw
.tcpPassiveOpens
= (u64
) m
.tcpPassiveOpens
;
1311 stats
->iw
.tcpAttemptFails
= (u64
) m
.tcpAttemptFails
;
1312 stats
->iw
.tcpEstabResets
= (u64
) m
.tcpEstabResets
;
1313 stats
->iw
.tcpOutRsts
= (u64
) m
.tcpOutRsts
;
1314 stats
->iw
.tcpCurrEstab
= (u64
) m
.tcpCurrEstab
;
1315 stats
->iw
.tcpInSegs
= ((u64
) m
.tcpInSegs_hi
<< 32) +
1317 stats
->iw
.tcpOutSegs
= ((u64
) m
.tcpOutSegs_hi
<< 32) +
1319 stats
->iw
.tcpRetransSegs
= ((u64
) m
.tcpRetransSeg_hi
<< 32) +
1321 stats
->iw
.tcpInErrs
= ((u64
) m
.tcpInErrs_hi
<< 32) +
1323 stats
->iw
.tcpRtoMin
= (u64
) m
.tcpRtoMin
;
1324 stats
->iw
.tcpRtoMax
= (u64
) m
.tcpRtoMax
;
1328 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
1329 static DEVICE_ATTR(fw_ver
, S_IRUGO
, show_fw_ver
, NULL
);
1330 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
1331 static DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
1333 static struct device_attribute
*iwch_class_attributes
[] = {
1340 int iwch_register_device(struct iwch_dev
*dev
)
1345 PDBG("%s iwch_dev %p\n", __func__
, dev
);
1346 strlcpy(dev
->ibdev
.name
, "cxgb3_%d", IB_DEVICE_NAME_MAX
);
1347 memset(&dev
->ibdev
.node_guid
, 0, sizeof(dev
->ibdev
.node_guid
));
1348 memcpy(&dev
->ibdev
.node_guid
, dev
->rdev
.t3cdev_p
->lldev
->dev_addr
, 6);
1349 dev
->ibdev
.owner
= THIS_MODULE
;
1350 dev
->device_cap_flags
= IB_DEVICE_LOCAL_DMA_LKEY
|
1351 IB_DEVICE_MEM_WINDOW
|
1352 IB_DEVICE_MEM_MGT_EXTENSIONS
;
1354 /* cxgb3 supports STag 0. */
1355 dev
->ibdev
.local_dma_lkey
= 0;
1357 dev
->ibdev
.uverbs_cmd_mask
=
1358 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
1359 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
1360 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
1361 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
1362 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
1363 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
1364 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
1365 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
1366 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
1367 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
1368 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
1369 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
1370 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
1371 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
1372 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
1373 (1ull << IB_USER_VERBS_CMD_POST_SEND
) |
1374 (1ull << IB_USER_VERBS_CMD_POST_RECV
);
1375 dev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
1376 memcpy(dev
->ibdev
.node_desc
, IWCH_NODE_DESC
, sizeof(IWCH_NODE_DESC
));
1377 dev
->ibdev
.phys_port_cnt
= dev
->rdev
.port_info
.nports
;
1378 dev
->ibdev
.num_comp_vectors
= 1;
1379 dev
->ibdev
.dma_device
= &(dev
->rdev
.rnic_info
.pdev
->dev
);
1380 dev
->ibdev
.query_device
= iwch_query_device
;
1381 dev
->ibdev
.query_port
= iwch_query_port
;
1382 dev
->ibdev
.modify_port
= iwch_modify_port
;
1383 dev
->ibdev
.query_pkey
= iwch_query_pkey
;
1384 dev
->ibdev
.query_gid
= iwch_query_gid
;
1385 dev
->ibdev
.alloc_ucontext
= iwch_alloc_ucontext
;
1386 dev
->ibdev
.dealloc_ucontext
= iwch_dealloc_ucontext
;
1387 dev
->ibdev
.mmap
= iwch_mmap
;
1388 dev
->ibdev
.alloc_pd
= iwch_allocate_pd
;
1389 dev
->ibdev
.dealloc_pd
= iwch_deallocate_pd
;
1390 dev
->ibdev
.create_ah
= iwch_ah_create
;
1391 dev
->ibdev
.destroy_ah
= iwch_ah_destroy
;
1392 dev
->ibdev
.create_qp
= iwch_create_qp
;
1393 dev
->ibdev
.modify_qp
= iwch_ib_modify_qp
;
1394 dev
->ibdev
.destroy_qp
= iwch_destroy_qp
;
1395 dev
->ibdev
.create_cq
= iwch_create_cq
;
1396 dev
->ibdev
.destroy_cq
= iwch_destroy_cq
;
1397 dev
->ibdev
.resize_cq
= iwch_resize_cq
;
1398 dev
->ibdev
.poll_cq
= iwch_poll_cq
;
1399 dev
->ibdev
.get_dma_mr
= iwch_get_dma_mr
;
1400 dev
->ibdev
.reg_phys_mr
= iwch_register_phys_mem
;
1401 dev
->ibdev
.rereg_phys_mr
= iwch_reregister_phys_mem
;
1402 dev
->ibdev
.reg_user_mr
= iwch_reg_user_mr
;
1403 dev
->ibdev
.dereg_mr
= iwch_dereg_mr
;
1404 dev
->ibdev
.alloc_mw
= iwch_alloc_mw
;
1405 dev
->ibdev
.bind_mw
= iwch_bind_mw
;
1406 dev
->ibdev
.dealloc_mw
= iwch_dealloc_mw
;
1407 dev
->ibdev
.alloc_fast_reg_mr
= iwch_alloc_fast_reg_mr
;
1408 dev
->ibdev
.alloc_fast_reg_page_list
= iwch_alloc_fastreg_pbl
;
1409 dev
->ibdev
.free_fast_reg_page_list
= iwch_free_fastreg_pbl
;
1410 dev
->ibdev
.attach_mcast
= iwch_multicast_attach
;
1411 dev
->ibdev
.detach_mcast
= iwch_multicast_detach
;
1412 dev
->ibdev
.process_mad
= iwch_process_mad
;
1413 dev
->ibdev
.req_notify_cq
= iwch_arm_cq
;
1414 dev
->ibdev
.post_send
= iwch_post_send
;
1415 dev
->ibdev
.post_recv
= iwch_post_receive
;
1416 dev
->ibdev
.get_protocol_stats
= iwch_get_mib
;
1418 dev
->ibdev
.iwcm
= kmalloc(sizeof(struct iw_cm_verbs
), GFP_KERNEL
);
1419 if (!dev
->ibdev
.iwcm
)
1422 dev
->ibdev
.iwcm
->connect
= iwch_connect
;
1423 dev
->ibdev
.iwcm
->accept
= iwch_accept_cr
;
1424 dev
->ibdev
.iwcm
->reject
= iwch_reject_cr
;
1425 dev
->ibdev
.iwcm
->create_listen
= iwch_create_listen
;
1426 dev
->ibdev
.iwcm
->destroy_listen
= iwch_destroy_listen
;
1427 dev
->ibdev
.iwcm
->add_ref
= iwch_qp_add_ref
;
1428 dev
->ibdev
.iwcm
->rem_ref
= iwch_qp_rem_ref
;
1429 dev
->ibdev
.iwcm
->get_qp
= iwch_get_qp
;
1431 ret
= ib_register_device(&dev
->ibdev
);
1435 for (i
= 0; i
< ARRAY_SIZE(iwch_class_attributes
); ++i
) {
1436 ret
= device_create_file(&dev
->ibdev
.dev
,
1437 iwch_class_attributes
[i
]);
1444 ib_unregister_device(&dev
->ibdev
);
1446 kfree(dev
->ibdev
.iwcm
);
1450 void iwch_unregister_device(struct iwch_dev
*dev
)
1454 PDBG("%s iwch_dev %p\n", __func__
, dev
);
1455 for (i
= 0; i
< ARRAY_SIZE(iwch_class_attributes
); ++i
)
1456 device_remove_file(&dev
->ibdev
.dev
,
1457 iwch_class_attributes
[i
]);
1458 ib_unregister_device(&dev
->ibdev
);
1459 kfree(dev
->ibdev
.iwcm
);