2 * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/module.h>
33 #include <linux/moduleparam.h>
34 #include <linux/device.h>
35 #include <linux/netdevice.h>
36 #include <linux/etherdevice.h>
37 #include <linux/delay.h>
38 #include <linux/errno.h>
39 #include <linux/list.h>
40 #include <linux/spinlock.h>
41 #include <linux/ethtool.h>
45 #include <asm/byteorder.h>
47 #include <rdma/iw_cm.h>
48 #include <rdma/ib_verbs.h>
49 #include <rdma/ib_smi.h>
50 #include <rdma/ib_user_verbs.h>
54 #include "iwch_provider.h"
56 #include "iwch_user.h"
58 static int iwch_modify_port(struct ib_device
*ibdev
,
59 u8 port
, int port_modify_mask
,
60 struct ib_port_modify
*props
)
65 static struct ib_ah
*iwch_ah_create(struct ib_pd
*pd
,
66 struct ib_ah_attr
*ah_attr
)
68 return ERR_PTR(-ENOSYS
);
71 static int iwch_ah_destroy(struct ib_ah
*ah
)
76 static int iwch_multicast_attach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
81 static int iwch_multicast_detach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
86 static int iwch_process_mad(struct ib_device
*ibdev
,
90 struct ib_grh
*in_grh
,
91 struct ib_mad
*in_mad
, struct ib_mad
*out_mad
)
96 static int iwch_dealloc_ucontext(struct ib_ucontext
*context
)
98 struct iwch_dev
*rhp
= to_iwch_dev(context
->device
);
99 struct iwch_ucontext
*ucontext
= to_iwch_ucontext(context
);
100 struct iwch_mm_entry
*mm
, *tmp
;
102 PDBG("%s context %p\n", __FUNCTION__
, context
);
103 list_for_each_entry_safe(mm
, tmp
, &ucontext
->mmaps
, entry
)
105 cxio_release_ucontext(&rhp
->rdev
, &ucontext
->uctx
);
110 static struct ib_ucontext
*iwch_alloc_ucontext(struct ib_device
*ibdev
,
111 struct ib_udata
*udata
)
113 struct iwch_ucontext
*context
;
114 struct iwch_dev
*rhp
= to_iwch_dev(ibdev
);
116 PDBG("%s ibdev %p\n", __FUNCTION__
, ibdev
);
117 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
119 return ERR_PTR(-ENOMEM
);
120 cxio_init_ucontext(&rhp
->rdev
, &context
->uctx
);
121 INIT_LIST_HEAD(&context
->mmaps
);
122 spin_lock_init(&context
->mmap_lock
);
123 return &context
->ibucontext
;
126 static int iwch_destroy_cq(struct ib_cq
*ib_cq
)
130 PDBG("%s ib_cq %p\n", __FUNCTION__
, ib_cq
);
131 chp
= to_iwch_cq(ib_cq
);
133 remove_handle(chp
->rhp
, &chp
->rhp
->cqidr
, chp
->cq
.cqid
);
134 atomic_dec(&chp
->refcnt
);
135 wait_event(chp
->wait
, !atomic_read(&chp
->refcnt
));
137 cxio_destroy_cq(&chp
->rhp
->rdev
, &chp
->cq
);
142 static struct ib_cq
*iwch_create_cq(struct ib_device
*ibdev
, int entries
, int vector
,
143 struct ib_ucontext
*ib_context
,
144 struct ib_udata
*udata
)
146 struct iwch_dev
*rhp
;
148 struct iwch_create_cq_resp uresp
;
149 struct iwch_create_cq_req ureq
;
150 struct iwch_ucontext
*ucontext
= NULL
;
152 PDBG("%s ib_dev %p entries %d\n", __FUNCTION__
, ibdev
, entries
);
153 rhp
= to_iwch_dev(ibdev
);
154 chp
= kzalloc(sizeof(*chp
), GFP_KERNEL
);
156 return ERR_PTR(-ENOMEM
);
159 ucontext
= to_iwch_ucontext(ib_context
);
160 if (!t3a_device(rhp
)) {
161 if (ib_copy_from_udata(&ureq
, udata
, sizeof (ureq
))) {
163 return ERR_PTR(-EFAULT
);
165 chp
->user_rptr_addr
= (u32 __user
*)(unsigned long)ureq
.user_rptr_addr
;
169 if (t3a_device(rhp
)) {
172 * T3A: Add some fluff to handle extra CQEs inserted
173 * for various errors.
174 * Additional CQE possibilities:
176 * incoming RDMA WRITE Failures
177 * incoming RDMA READ REQUEST FAILUREs
178 * NOTE: We cannot ensure the CQ won't overflow.
182 entries
= roundup_pow_of_two(entries
);
183 chp
->cq
.size_log2
= ilog2(entries
);
185 if (cxio_create_cq(&rhp
->rdev
, &chp
->cq
)) {
187 return ERR_PTR(-ENOMEM
);
190 chp
->ibcq
.cqe
= (1 << chp
->cq
.size_log2
) - 1;
191 spin_lock_init(&chp
->lock
);
192 atomic_set(&chp
->refcnt
, 1);
193 init_waitqueue_head(&chp
->wait
);
194 insert_handle(rhp
, &rhp
->cqidr
, chp
, chp
->cq
.cqid
);
197 struct iwch_mm_entry
*mm
;
199 mm
= kmalloc(sizeof *mm
, GFP_KERNEL
);
201 iwch_destroy_cq(&chp
->ibcq
);
202 return ERR_PTR(-ENOMEM
);
204 uresp
.cqid
= chp
->cq
.cqid
;
205 uresp
.size_log2
= chp
->cq
.size_log2
;
206 spin_lock(&ucontext
->mmap_lock
);
207 uresp
.key
= ucontext
->key
;
208 ucontext
->key
+= PAGE_SIZE
;
209 spin_unlock(&ucontext
->mmap_lock
);
210 if (ib_copy_to_udata(udata
, &uresp
, sizeof (uresp
))) {
212 iwch_destroy_cq(&chp
->ibcq
);
213 return ERR_PTR(-EFAULT
);
216 mm
->addr
= virt_to_phys(chp
->cq
.queue
);
217 mm
->len
= PAGE_ALIGN((1UL << uresp
.size_log2
) *
218 sizeof (struct t3_cqe
));
219 insert_mmap(ucontext
, mm
);
221 PDBG("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",
222 chp
->cq
.cqid
, chp
, (1 << chp
->cq
.size_log2
),
223 (unsigned long long) chp
->cq
.dma_addr
);
227 static int iwch_resize_cq(struct ib_cq
*cq
, int cqe
, struct ib_udata
*udata
)
230 struct iwch_cq
*chp
= to_iwch_cq(cq
);
231 struct t3_cq oldcq
, newcq
;
234 PDBG("%s ib_cq %p cqe %d\n", __FUNCTION__
, cq
, cqe
);
236 /* We don't downsize... */
240 /* create new t3_cq with new size */
241 cqe
= roundup_pow_of_two(cqe
+1);
242 newcq
.size_log2
= ilog2(cqe
);
244 /* Dont allow resize to less than the current wce count */
245 if (cqe
< Q_COUNT(chp
->cq
.rptr
, chp
->cq
.wptr
)) {
249 /* Quiesce all QPs using this CQ */
250 ret
= iwch_quiesce_qps(chp
);
255 ret
= cxio_create_cq(&chp
->rhp
->rdev
, &newcq
);
261 memcpy(newcq
.queue
, chp
->cq
.queue
, (1 << chp
->cq
.size_log2
) *
262 sizeof(struct t3_cqe
));
264 /* old iwch_qp gets new t3_cq but keeps old cqid */
267 chp
->cq
.cqid
= oldcq
.cqid
;
269 /* resize new t3_cq to update the HW context */
270 ret
= cxio_resize_cq(&chp
->rhp
->rdev
, &chp
->cq
);
275 chp
->ibcq
.cqe
= (1<<chp
->cq
.size_log2
) - 1;
277 /* destroy old t3_cq */
278 oldcq
.cqid
= newcq
.cqid
;
279 ret
= cxio_destroy_cq(&chp
->rhp
->rdev
, &oldcq
);
281 printk(KERN_ERR MOD
"%s - cxio_destroy_cq failed %d\n",
285 /* add user hooks here */
288 ret
= iwch_resume_qps(chp
);
295 static int iwch_arm_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
297 struct iwch_dev
*rhp
;
299 enum t3_cq_opcode cq_op
;
304 chp
= to_iwch_cq(ibcq
);
306 if ((flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
)
310 if (chp
->user_rptr_addr
) {
311 if (get_user(rptr
, chp
->user_rptr_addr
))
313 spin_lock_irqsave(&chp
->lock
, flag
);
316 spin_lock_irqsave(&chp
->lock
, flag
);
317 PDBG("%s rptr 0x%x\n", __FUNCTION__
, chp
->cq
.rptr
);
318 err
= cxio_hal_cq_op(&rhp
->rdev
, &chp
->cq
, cq_op
, 0);
319 spin_unlock_irqrestore(&chp
->lock
, flag
);
321 printk(KERN_ERR MOD
"Error %d rearming CQID 0x%x\n", err
,
323 if (err
> 0 && !(flags
& IB_CQ_REPORT_MISSED_EVENTS
))
328 static int iwch_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
330 int len
= vma
->vm_end
- vma
->vm_start
;
331 u32 key
= vma
->vm_pgoff
<< PAGE_SHIFT
;
332 struct cxio_rdev
*rdev_p
;
334 struct iwch_mm_entry
*mm
;
335 struct iwch_ucontext
*ucontext
;
338 PDBG("%s pgoff 0x%lx key 0x%x len %d\n", __FUNCTION__
, vma
->vm_pgoff
,
341 if (vma
->vm_start
& (PAGE_SIZE
-1)) {
345 rdev_p
= &(to_iwch_dev(context
->device
)->rdev
);
346 ucontext
= to_iwch_ucontext(context
);
348 mm
= remove_mmap(ucontext
, key
, len
);
354 if ((addr
>= rdev_p
->rnic_info
.udbell_physbase
) &&
355 (addr
< (rdev_p
->rnic_info
.udbell_physbase
+
356 rdev_p
->rnic_info
.udbell_len
))) {
359 * Map T3 DB register.
361 if (vma
->vm_flags
& VM_READ
) {
365 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
366 vma
->vm_flags
|= VM_DONTCOPY
| VM_DONTEXPAND
;
367 vma
->vm_flags
&= ~VM_MAYREAD
;
368 ret
= io_remap_pfn_range(vma
, vma
->vm_start
,
370 len
, vma
->vm_page_prot
);
374 * Map WQ or CQ contig dma memory...
376 ret
= remap_pfn_range(vma
, vma
->vm_start
,
378 len
, vma
->vm_page_prot
);
384 static int iwch_deallocate_pd(struct ib_pd
*pd
)
386 struct iwch_dev
*rhp
;
389 php
= to_iwch_pd(pd
);
391 PDBG("%s ibpd %p pdid 0x%x\n", __FUNCTION__
, pd
, php
->pdid
);
392 cxio_hal_put_pdid(rhp
->rdev
.rscp
, php
->pdid
);
397 static struct ib_pd
*iwch_allocate_pd(struct ib_device
*ibdev
,
398 struct ib_ucontext
*context
,
399 struct ib_udata
*udata
)
403 struct iwch_dev
*rhp
;
405 PDBG("%s ibdev %p\n", __FUNCTION__
, ibdev
);
406 rhp
= (struct iwch_dev
*) ibdev
;
407 pdid
= cxio_hal_get_pdid(rhp
->rdev
.rscp
);
409 return ERR_PTR(-EINVAL
);
410 php
= kzalloc(sizeof(*php
), GFP_KERNEL
);
412 cxio_hal_put_pdid(rhp
->rdev
.rscp
, pdid
);
413 return ERR_PTR(-ENOMEM
);
418 if (ib_copy_to_udata(udata
, &php
->pdid
, sizeof (__u32
))) {
419 iwch_deallocate_pd(&php
->ibpd
);
420 return ERR_PTR(-EFAULT
);
423 PDBG("%s pdid 0x%0x ptr 0x%p\n", __FUNCTION__
, pdid
, php
);
427 static int iwch_dereg_mr(struct ib_mr
*ib_mr
)
429 struct iwch_dev
*rhp
;
433 PDBG("%s ib_mr %p\n", __FUNCTION__
, ib_mr
);
434 /* There can be no memory windows */
435 if (atomic_read(&ib_mr
->usecnt
))
438 mhp
= to_iwch_mr(ib_mr
);
440 mmid
= mhp
->attr
.stag
>> 8;
441 cxio_dereg_mem(&rhp
->rdev
, mhp
->attr
.stag
, mhp
->attr
.pbl_size
,
443 remove_handle(rhp
, &rhp
->mmidr
, mmid
);
445 kfree((void *) (unsigned long) mhp
->kva
);
446 PDBG("%s mmid 0x%x ptr %p\n", __FUNCTION__
, mmid
, mhp
);
451 static struct ib_mr
*iwch_register_phys_mem(struct ib_pd
*pd
,
452 struct ib_phys_buf
*buffer_list
,
461 struct iwch_dev
*rhp
;
466 PDBG("%s ib_pd %p\n", __FUNCTION__
, pd
);
467 php
= to_iwch_pd(pd
);
470 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
472 return ERR_PTR(-ENOMEM
);
474 /* First check that we have enough alignment */
475 if ((*iova_start
& ~PAGE_MASK
) != (buffer_list
[0].addr
& ~PAGE_MASK
)) {
480 if (num_phys_buf
> 1 &&
481 ((buffer_list
[0].addr
+ buffer_list
[0].size
) & ~PAGE_MASK
)) {
486 ret
= build_phys_page_list(buffer_list
, num_phys_buf
, iova_start
,
487 &total_size
, &npages
, &shift
, &page_list
);
492 mhp
->attr
.pdid
= php
->pdid
;
495 mhp
->attr
.perms
= iwch_ib_to_tpt_access(acc
);
496 mhp
->attr
.va_fbo
= *iova_start
;
497 mhp
->attr
.page_size
= shift
- 12;
499 mhp
->attr
.len
= (u32
) total_size
;
500 mhp
->attr
.pbl_size
= npages
;
501 ret
= iwch_register_mem(rhp
, php
, mhp
, shift
, page_list
);
513 static int iwch_reregister_phys_mem(struct ib_mr
*mr
,
516 struct ib_phys_buf
*buffer_list
,
518 int acc
, u64
* iova_start
)
521 struct iwch_mr mh
, *mhp
;
523 struct iwch_dev
*rhp
;
524 __be64
*page_list
= NULL
;
530 PDBG("%s ib_mr %p ib_pd %p\n", __FUNCTION__
, mr
, pd
);
532 /* There can be no memory windows */
533 if (atomic_read(&mr
->usecnt
))
536 mhp
= to_iwch_mr(mr
);
538 php
= to_iwch_pd(mr
->pd
);
540 /* make sure we are on the same adapter */
544 memcpy(&mh
, mhp
, sizeof *mhp
);
546 if (mr_rereg_mask
& IB_MR_REREG_PD
)
547 php
= to_iwch_pd(pd
);
548 if (mr_rereg_mask
& IB_MR_REREG_ACCESS
)
549 mh
.attr
.perms
= iwch_ib_to_tpt_access(acc
);
550 if (mr_rereg_mask
& IB_MR_REREG_TRANS
) {
551 ret
= build_phys_page_list(buffer_list
, num_phys_buf
,
553 &total_size
, &npages
,
559 ret
= iwch_reregister_mem(rhp
, php
, &mh
, shift
, page_list
, npages
);
564 if (mr_rereg_mask
& IB_MR_REREG_PD
)
565 mhp
->attr
.pdid
= php
->pdid
;
566 if (mr_rereg_mask
& IB_MR_REREG_ACCESS
)
567 mhp
->attr
.perms
= iwch_ib_to_tpt_access(acc
);
568 if (mr_rereg_mask
& IB_MR_REREG_TRANS
) {
570 mhp
->attr
.va_fbo
= *iova_start
;
571 mhp
->attr
.page_size
= shift
- 12;
572 mhp
->attr
.len
= (u32
) total_size
;
573 mhp
->attr
.pbl_size
= npages
;
580 static struct ib_mr
*iwch_reg_user_mr(struct ib_pd
*pd
, struct ib_umem
*region
,
581 int acc
, struct ib_udata
*udata
)
587 struct ib_umem_chunk
*chunk
;
588 struct iwch_dev
*rhp
;
591 struct iwch_reg_user_mr_resp uresp
;
593 PDBG("%s ib_pd %p\n", __FUNCTION__
, pd
);
594 shift
= ffs(region
->page_size
) - 1;
596 php
= to_iwch_pd(pd
);
598 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
600 return ERR_PTR(-ENOMEM
);
603 list_for_each_entry(chunk
, ®ion
->chunk_list
, list
)
606 pages
= kmalloc(n
* sizeof(u64
), GFP_KERNEL
);
614 list_for_each_entry(chunk
, ®ion
->chunk_list
, list
)
615 for (j
= 0; j
< chunk
->nmap
; ++j
) {
616 len
= sg_dma_len(&chunk
->page_list
[j
]) >> shift
;
617 for (k
= 0; k
< len
; ++k
) {
618 pages
[i
++] = cpu_to_be64(sg_dma_address(
619 &chunk
->page_list
[j
]) +
620 region
->page_size
* k
);
625 mhp
->attr
.pdid
= php
->pdid
;
627 mhp
->attr
.perms
= iwch_ib_to_tpt_access(acc
);
628 mhp
->attr
.va_fbo
= region
->virt_base
;
629 mhp
->attr
.page_size
= shift
- 12;
630 mhp
->attr
.len
= (u32
) region
->length
;
631 mhp
->attr
.pbl_size
= i
;
632 err
= iwch_register_mem(rhp
, php
, mhp
, shift
, pages
);
637 if (udata
&& t3b_device(rhp
)) {
638 uresp
.pbl_addr
= (mhp
->attr
.pbl_addr
-
639 rhp
->rdev
.rnic_info
.pbl_base
) >> 3;
640 PDBG("%s user resp pbl_addr 0x%x\n", __FUNCTION__
,
643 if (ib_copy_to_udata(udata
, &uresp
, sizeof (uresp
))) {
644 iwch_dereg_mr(&mhp
->ibmr
);
657 static struct ib_mr
*iwch_get_dma_mr(struct ib_pd
*pd
, int acc
)
659 struct ib_phys_buf bl
;
663 PDBG("%s ib_pd %p\n", __FUNCTION__
, pd
);
666 * T3 only supports 32 bits of size.
668 bl
.size
= 0xffffffff;
671 ibmr
= iwch_register_phys_mem(pd
, &bl
, 1, acc
, &kva
);
675 static struct ib_mw
*iwch_alloc_mw(struct ib_pd
*pd
)
677 struct iwch_dev
*rhp
;
684 php
= to_iwch_pd(pd
);
686 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
688 return ERR_PTR(-ENOMEM
);
689 ret
= cxio_allocate_window(&rhp
->rdev
, &stag
, php
->pdid
);
695 mhp
->attr
.pdid
= php
->pdid
;
696 mhp
->attr
.type
= TPT_MW
;
697 mhp
->attr
.stag
= stag
;
699 insert_handle(rhp
, &rhp
->mmidr
, mhp
, mmid
);
700 PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __FUNCTION__
, mmid
, mhp
, stag
);
704 static int iwch_dealloc_mw(struct ib_mw
*mw
)
706 struct iwch_dev
*rhp
;
710 mhp
= to_iwch_mw(mw
);
712 mmid
= (mw
->rkey
) >> 8;
713 cxio_deallocate_window(&rhp
->rdev
, mhp
->attr
.stag
);
714 remove_handle(rhp
, &rhp
->mmidr
, mmid
);
716 PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __FUNCTION__
, mw
, mmid
, mhp
);
720 static int iwch_destroy_qp(struct ib_qp
*ib_qp
)
722 struct iwch_dev
*rhp
;
724 struct iwch_qp_attributes attrs
;
725 struct iwch_ucontext
*ucontext
;
727 qhp
= to_iwch_qp(ib_qp
);
730 attrs
.next_state
= IWCH_QP_STATE_ERROR
;
731 iwch_modify_qp(rhp
, qhp
, IWCH_QP_ATTR_NEXT_STATE
, &attrs
, 0);
732 wait_event(qhp
->wait
, !qhp
->ep
);
734 remove_handle(rhp
, &rhp
->qpidr
, qhp
->wq
.qpid
);
736 atomic_dec(&qhp
->refcnt
);
737 wait_event(qhp
->wait
, !atomic_read(&qhp
->refcnt
));
739 ucontext
= ib_qp
->uobject
? to_iwch_ucontext(ib_qp
->uobject
->context
)
741 cxio_destroy_qp(&rhp
->rdev
, &qhp
->wq
,
742 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
);
744 PDBG("%s ib_qp %p qpid 0x%0x qhp %p\n", __FUNCTION__
,
745 ib_qp
, qhp
->wq
.qpid
, qhp
);
750 static struct ib_qp
*iwch_create_qp(struct ib_pd
*pd
,
751 struct ib_qp_init_attr
*attrs
,
752 struct ib_udata
*udata
)
754 struct iwch_dev
*rhp
;
757 struct iwch_cq
*schp
;
758 struct iwch_cq
*rchp
;
759 struct iwch_create_qp_resp uresp
;
760 int wqsize
, sqsize
, rqsize
;
761 struct iwch_ucontext
*ucontext
;
763 PDBG("%s ib_pd %p\n", __FUNCTION__
, pd
);
764 if (attrs
->qp_type
!= IB_QPT_RC
)
765 return ERR_PTR(-EINVAL
);
766 php
= to_iwch_pd(pd
);
768 schp
= get_chp(rhp
, ((struct iwch_cq
*) attrs
->send_cq
)->cq
.cqid
);
769 rchp
= get_chp(rhp
, ((struct iwch_cq
*) attrs
->recv_cq
)->cq
.cqid
);
771 return ERR_PTR(-EINVAL
);
773 /* The RQT size must be # of entries + 1 rounded up to a power of two */
774 rqsize
= roundup_pow_of_two(attrs
->cap
.max_recv_wr
);
775 if (rqsize
== attrs
->cap
.max_recv_wr
)
776 rqsize
= roundup_pow_of_two(attrs
->cap
.max_recv_wr
+1);
778 /* T3 doesn't support RQT depth < 16 */
782 if (rqsize
> T3_MAX_RQ_SIZE
)
783 return ERR_PTR(-EINVAL
);
785 if (attrs
->cap
.max_inline_data
> T3_MAX_INLINE
)
786 return ERR_PTR(-EINVAL
);
789 * NOTE: The SQ and total WQ sizes don't need to be
790 * a power of two. However, all the code assumes
791 * they are. EG: Q_FREECNT() and friends.
793 sqsize
= roundup_pow_of_two(attrs
->cap
.max_send_wr
);
794 wqsize
= roundup_pow_of_two(rqsize
+ sqsize
);
795 PDBG("%s wqsize %d sqsize %d rqsize %d\n", __FUNCTION__
,
796 wqsize
, sqsize
, rqsize
);
797 qhp
= kzalloc(sizeof(*qhp
), GFP_KERNEL
);
799 return ERR_PTR(-ENOMEM
);
800 qhp
->wq
.size_log2
= ilog2(wqsize
);
801 qhp
->wq
.rq_size_log2
= ilog2(rqsize
);
802 qhp
->wq
.sq_size_log2
= ilog2(sqsize
);
803 ucontext
= pd
->uobject
? to_iwch_ucontext(pd
->uobject
->context
) : NULL
;
804 if (cxio_create_qp(&rhp
->rdev
, !udata
, &qhp
->wq
,
805 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
)) {
807 return ERR_PTR(-ENOMEM
);
809 attrs
->cap
.max_recv_wr
= rqsize
- 1;
810 attrs
->cap
.max_send_wr
= sqsize
;
812 qhp
->attr
.pd
= php
->pdid
;
813 qhp
->attr
.scq
= ((struct iwch_cq
*) attrs
->send_cq
)->cq
.cqid
;
814 qhp
->attr
.rcq
= ((struct iwch_cq
*) attrs
->recv_cq
)->cq
.cqid
;
815 qhp
->attr
.sq_num_entries
= attrs
->cap
.max_send_wr
;
816 qhp
->attr
.rq_num_entries
= attrs
->cap
.max_recv_wr
;
817 qhp
->attr
.sq_max_sges
= attrs
->cap
.max_send_sge
;
818 qhp
->attr
.sq_max_sges_rdma_write
= attrs
->cap
.max_send_sge
;
819 qhp
->attr
.rq_max_sges
= attrs
->cap
.max_recv_sge
;
820 qhp
->attr
.state
= IWCH_QP_STATE_IDLE
;
821 qhp
->attr
.next_state
= IWCH_QP_STATE_IDLE
;
824 * XXX - These don't get passed in from the openib user
825 * at create time. The CM sets them via a QP modify.
826 * Need to fix... I think the CM should
828 qhp
->attr
.enable_rdma_read
= 1;
829 qhp
->attr
.enable_rdma_write
= 1;
830 qhp
->attr
.enable_bind
= 1;
831 qhp
->attr
.max_ord
= 1;
832 qhp
->attr
.max_ird
= 1;
834 spin_lock_init(&qhp
->lock
);
835 init_waitqueue_head(&qhp
->wait
);
836 atomic_set(&qhp
->refcnt
, 1);
837 insert_handle(rhp
, &rhp
->qpidr
, qhp
, qhp
->wq
.qpid
);
841 struct iwch_mm_entry
*mm1
, *mm2
;
843 mm1
= kmalloc(sizeof *mm1
, GFP_KERNEL
);
845 iwch_destroy_qp(&qhp
->ibqp
);
846 return ERR_PTR(-ENOMEM
);
849 mm2
= kmalloc(sizeof *mm2
, GFP_KERNEL
);
852 iwch_destroy_qp(&qhp
->ibqp
);
853 return ERR_PTR(-ENOMEM
);
856 uresp
.qpid
= qhp
->wq
.qpid
;
857 uresp
.size_log2
= qhp
->wq
.size_log2
;
858 uresp
.sq_size_log2
= qhp
->wq
.sq_size_log2
;
859 uresp
.rq_size_log2
= qhp
->wq
.rq_size_log2
;
860 spin_lock(&ucontext
->mmap_lock
);
861 uresp
.key
= ucontext
->key
;
862 ucontext
->key
+= PAGE_SIZE
;
863 uresp
.db_key
= ucontext
->key
;
864 ucontext
->key
+= PAGE_SIZE
;
865 spin_unlock(&ucontext
->mmap_lock
);
866 if (ib_copy_to_udata(udata
, &uresp
, sizeof (uresp
))) {
869 iwch_destroy_qp(&qhp
->ibqp
);
870 return ERR_PTR(-EFAULT
);
872 mm1
->key
= uresp
.key
;
873 mm1
->addr
= virt_to_phys(qhp
->wq
.queue
);
874 mm1
->len
= PAGE_ALIGN(wqsize
* sizeof (union t3_wr
));
875 insert_mmap(ucontext
, mm1
);
876 mm2
->key
= uresp
.db_key
;
877 mm2
->addr
= qhp
->wq
.udb
& PAGE_MASK
;
878 mm2
->len
= PAGE_SIZE
;
879 insert_mmap(ucontext
, mm2
);
881 qhp
->ibqp
.qp_num
= qhp
->wq
.qpid
;
882 init_timer(&(qhp
->timer
));
883 PDBG("%s sq_num_entries %d, rq_num_entries %d "
884 "qpid 0x%0x qhp %p dma_addr 0x%llx size %d\n",
885 __FUNCTION__
, qhp
->attr
.sq_num_entries
, qhp
->attr
.rq_num_entries
,
886 qhp
->wq
.qpid
, qhp
, (unsigned long long) qhp
->wq
.dma_addr
,
887 1 << qhp
->wq
.size_log2
);
891 static int iwch_ib_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
892 int attr_mask
, struct ib_udata
*udata
)
894 struct iwch_dev
*rhp
;
896 enum iwch_qp_attr_mask mask
= 0;
897 struct iwch_qp_attributes attrs
;
899 PDBG("%s ib_qp %p\n", __FUNCTION__
, ibqp
);
901 /* iwarp does not support the RTR state */
902 if ((attr_mask
& IB_QP_STATE
) && (attr
->qp_state
== IB_QPS_RTR
))
903 attr_mask
&= ~IB_QP_STATE
;
905 /* Make sure we still have something left to do */
909 memset(&attrs
, 0, sizeof attrs
);
910 qhp
= to_iwch_qp(ibqp
);
913 attrs
.next_state
= iwch_convert_state(attr
->qp_state
);
914 attrs
.enable_rdma_read
= (attr
->qp_access_flags
&
915 IB_ACCESS_REMOTE_READ
) ? 1 : 0;
916 attrs
.enable_rdma_write
= (attr
->qp_access_flags
&
917 IB_ACCESS_REMOTE_WRITE
) ? 1 : 0;
918 attrs
.enable_bind
= (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
) ? 1 : 0;
921 mask
|= (attr_mask
& IB_QP_STATE
) ? IWCH_QP_ATTR_NEXT_STATE
: 0;
922 mask
|= (attr_mask
& IB_QP_ACCESS_FLAGS
) ?
923 (IWCH_QP_ATTR_ENABLE_RDMA_READ
|
924 IWCH_QP_ATTR_ENABLE_RDMA_WRITE
|
925 IWCH_QP_ATTR_ENABLE_RDMA_BIND
) : 0;
927 return iwch_modify_qp(rhp
, qhp
, mask
, &attrs
, 0);
930 void iwch_qp_add_ref(struct ib_qp
*qp
)
932 PDBG("%s ib_qp %p\n", __FUNCTION__
, qp
);
933 atomic_inc(&(to_iwch_qp(qp
)->refcnt
));
936 void iwch_qp_rem_ref(struct ib_qp
*qp
)
938 PDBG("%s ib_qp %p\n", __FUNCTION__
, qp
);
939 if (atomic_dec_and_test(&(to_iwch_qp(qp
)->refcnt
)))
940 wake_up(&(to_iwch_qp(qp
)->wait
));
943 static struct ib_qp
*iwch_get_qp(struct ib_device
*dev
, int qpn
)
945 PDBG("%s ib_dev %p qpn 0x%x\n", __FUNCTION__
, dev
, qpn
);
946 return (struct ib_qp
*)get_qhp(to_iwch_dev(dev
), qpn
);
950 static int iwch_query_pkey(struct ib_device
*ibdev
,
951 u8 port
, u16 index
, u16
* pkey
)
953 PDBG("%s ibdev %p\n", __FUNCTION__
, ibdev
);
958 static int iwch_query_gid(struct ib_device
*ibdev
, u8 port
,
959 int index
, union ib_gid
*gid
)
961 struct iwch_dev
*dev
;
963 PDBG("%s ibdev %p, port %d, index %d, gid %p\n",
964 __FUNCTION__
, ibdev
, port
, index
, gid
);
965 dev
= to_iwch_dev(ibdev
);
966 BUG_ON(port
== 0 || port
> 2);
967 memset(&(gid
->raw
[0]), 0, sizeof(gid
->raw
));
968 memcpy(&(gid
->raw
[0]), dev
->rdev
.port_info
.lldevs
[port
-1]->dev_addr
, 6);
972 static int iwch_query_device(struct ib_device
*ibdev
,
973 struct ib_device_attr
*props
)
976 struct iwch_dev
*dev
;
977 PDBG("%s ibdev %p\n", __FUNCTION__
, ibdev
);
979 dev
= to_iwch_dev(ibdev
);
980 memset(props
, 0, sizeof *props
);
981 memcpy(&props
->sys_image_guid
, dev
->rdev
.t3cdev_p
->lldev
->dev_addr
, 6);
982 props
->device_cap_flags
= dev
->device_cap_flags
;
983 props
->vendor_id
= (u32
)dev
->rdev
.rnic_info
.pdev
->vendor
;
984 props
->vendor_part_id
= (u32
)dev
->rdev
.rnic_info
.pdev
->device
;
985 props
->max_mr_size
= ~0ull;
986 props
->max_qp
= dev
->attr
.max_qps
;
987 props
->max_qp_wr
= dev
->attr
.max_wrs
;
988 props
->max_sge
= dev
->attr
.max_sge_per_wr
;
989 props
->max_sge_rd
= 1;
990 props
->max_qp_rd_atom
= dev
->attr
.max_rdma_reads_per_qp
;
991 props
->max_cq
= dev
->attr
.max_cqs
;
992 props
->max_cqe
= dev
->attr
.max_cqes_per_cq
;
993 props
->max_mr
= dev
->attr
.max_mem_regs
;
994 props
->max_pd
= dev
->attr
.max_pds
;
995 props
->local_ca_ack_delay
= 0;
1000 static int iwch_query_port(struct ib_device
*ibdev
,
1001 u8 port
, struct ib_port_attr
*props
)
1003 PDBG("%s ibdev %p\n", __FUNCTION__
, ibdev
);
1004 props
->max_mtu
= IB_MTU_4096
;
1009 props
->state
= IB_PORT_ACTIVE
;
1010 props
->phys_state
= 0;
1011 props
->port_cap_flags
=
1013 IB_PORT_SNMP_TUNNEL_SUP
|
1014 IB_PORT_REINIT_SUP
|
1015 IB_PORT_DEVICE_MGMT_SUP
|
1016 IB_PORT_VENDOR_CLASS_SUP
| IB_PORT_BOOT_MGMT_SUP
;
1017 props
->gid_tbl_len
= 1;
1018 props
->pkey_tbl_len
= 1;
1019 props
->qkey_viol_cntr
= 0;
1020 props
->active_width
= 2;
1021 props
->active_speed
= 2;
1022 props
->max_msg_sz
= -1;
1027 static ssize_t
show_rev(struct class_device
*cdev
, char *buf
)
1029 struct iwch_dev
*dev
= container_of(cdev
, struct iwch_dev
,
1031 PDBG("%s class dev 0x%p\n", __FUNCTION__
, cdev
);
1032 return sprintf(buf
, "%d\n", dev
->rdev
.t3cdev_p
->type
);
1035 static ssize_t
show_fw_ver(struct class_device
*cdev
, char *buf
)
1037 struct iwch_dev
*dev
= container_of(cdev
, struct iwch_dev
,
1039 struct ethtool_drvinfo info
;
1040 struct net_device
*lldev
= dev
->rdev
.t3cdev_p
->lldev
;
1042 PDBG("%s class dev 0x%p\n", __FUNCTION__
, cdev
);
1043 lldev
->ethtool_ops
->get_drvinfo(lldev
, &info
);
1044 return sprintf(buf
, "%s\n", info
.fw_version
);
1047 static ssize_t
show_hca(struct class_device
*cdev
, char *buf
)
1049 struct iwch_dev
*dev
= container_of(cdev
, struct iwch_dev
,
1051 struct ethtool_drvinfo info
;
1052 struct net_device
*lldev
= dev
->rdev
.t3cdev_p
->lldev
;
1054 PDBG("%s class dev 0x%p\n", __FUNCTION__
, cdev
);
1055 lldev
->ethtool_ops
->get_drvinfo(lldev
, &info
);
1056 return sprintf(buf
, "%s\n", info
.driver
);
1059 static ssize_t
show_board(struct class_device
*cdev
, char *buf
)
1061 struct iwch_dev
*dev
= container_of(cdev
, struct iwch_dev
,
1063 PDBG("%s class dev 0x%p\n", __FUNCTION__
, dev
);
1064 return sprintf(buf
, "%x.%x\n", dev
->rdev
.rnic_info
.pdev
->vendor
,
1065 dev
->rdev
.rnic_info
.pdev
->device
);
1068 static CLASS_DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
1069 static CLASS_DEVICE_ATTR(fw_ver
, S_IRUGO
, show_fw_ver
, NULL
);
1070 static CLASS_DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
1071 static CLASS_DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
1073 static struct class_device_attribute
*iwch_class_attributes
[] = {
1074 &class_device_attr_hw_rev
,
1075 &class_device_attr_fw_ver
,
1076 &class_device_attr_hca_type
,
1077 &class_device_attr_board_id
1080 int iwch_register_device(struct iwch_dev
*dev
)
1085 PDBG("%s iwch_dev %p\n", __FUNCTION__
, dev
);
1086 strlcpy(dev
->ibdev
.name
, "cxgb3_%d", IB_DEVICE_NAME_MAX
);
1087 memset(&dev
->ibdev
.node_guid
, 0, sizeof(dev
->ibdev
.node_guid
));
1088 memcpy(&dev
->ibdev
.node_guid
, dev
->rdev
.t3cdev_p
->lldev
->dev_addr
, 6);
1089 dev
->ibdev
.owner
= THIS_MODULE
;
1090 dev
->device_cap_flags
=
1091 (IB_DEVICE_ZERO_STAG
|
1092 IB_DEVICE_SEND_W_INV
| IB_DEVICE_MEM_WINDOW
);
1094 dev
->ibdev
.uverbs_cmd_mask
=
1095 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
1096 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
1097 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
1098 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
1099 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
1100 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
1101 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
1102 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
1103 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
1104 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
1105 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
1106 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
1107 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
1108 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
1109 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
1110 (1ull << IB_USER_VERBS_CMD_POST_SEND
) |
1111 (1ull << IB_USER_VERBS_CMD_POST_RECV
);
1112 dev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
1113 memcpy(dev
->ibdev
.node_desc
, IWCH_NODE_DESC
, sizeof(IWCH_NODE_DESC
));
1114 dev
->ibdev
.phys_port_cnt
= dev
->rdev
.port_info
.nports
;
1115 dev
->ibdev
.num_comp_vectors
= 1;
1116 dev
->ibdev
.dma_device
= &(dev
->rdev
.rnic_info
.pdev
->dev
);
1117 dev
->ibdev
.query_device
= iwch_query_device
;
1118 dev
->ibdev
.query_port
= iwch_query_port
;
1119 dev
->ibdev
.modify_port
= iwch_modify_port
;
1120 dev
->ibdev
.query_pkey
= iwch_query_pkey
;
1121 dev
->ibdev
.query_gid
= iwch_query_gid
;
1122 dev
->ibdev
.alloc_ucontext
= iwch_alloc_ucontext
;
1123 dev
->ibdev
.dealloc_ucontext
= iwch_dealloc_ucontext
;
1124 dev
->ibdev
.mmap
= iwch_mmap
;
1125 dev
->ibdev
.alloc_pd
= iwch_allocate_pd
;
1126 dev
->ibdev
.dealloc_pd
= iwch_deallocate_pd
;
1127 dev
->ibdev
.create_ah
= iwch_ah_create
;
1128 dev
->ibdev
.destroy_ah
= iwch_ah_destroy
;
1129 dev
->ibdev
.create_qp
= iwch_create_qp
;
1130 dev
->ibdev
.modify_qp
= iwch_ib_modify_qp
;
1131 dev
->ibdev
.destroy_qp
= iwch_destroy_qp
;
1132 dev
->ibdev
.create_cq
= iwch_create_cq
;
1133 dev
->ibdev
.destroy_cq
= iwch_destroy_cq
;
1134 dev
->ibdev
.resize_cq
= iwch_resize_cq
;
1135 dev
->ibdev
.poll_cq
= iwch_poll_cq
;
1136 dev
->ibdev
.get_dma_mr
= iwch_get_dma_mr
;
1137 dev
->ibdev
.reg_phys_mr
= iwch_register_phys_mem
;
1138 dev
->ibdev
.rereg_phys_mr
= iwch_reregister_phys_mem
;
1139 dev
->ibdev
.reg_user_mr
= iwch_reg_user_mr
;
1140 dev
->ibdev
.dereg_mr
= iwch_dereg_mr
;
1141 dev
->ibdev
.alloc_mw
= iwch_alloc_mw
;
1142 dev
->ibdev
.bind_mw
= iwch_bind_mw
;
1143 dev
->ibdev
.dealloc_mw
= iwch_dealloc_mw
;
1145 dev
->ibdev
.attach_mcast
= iwch_multicast_attach
;
1146 dev
->ibdev
.detach_mcast
= iwch_multicast_detach
;
1147 dev
->ibdev
.process_mad
= iwch_process_mad
;
1149 dev
->ibdev
.req_notify_cq
= iwch_arm_cq
;
1150 dev
->ibdev
.post_send
= iwch_post_send
;
1151 dev
->ibdev
.post_recv
= iwch_post_receive
;
1155 (struct iw_cm_verbs
*) kmalloc(sizeof(struct iw_cm_verbs
),
1157 dev
->ibdev
.iwcm
->connect
= iwch_connect
;
1158 dev
->ibdev
.iwcm
->accept
= iwch_accept_cr
;
1159 dev
->ibdev
.iwcm
->reject
= iwch_reject_cr
;
1160 dev
->ibdev
.iwcm
->create_listen
= iwch_create_listen
;
1161 dev
->ibdev
.iwcm
->destroy_listen
= iwch_destroy_listen
;
1162 dev
->ibdev
.iwcm
->add_ref
= iwch_qp_add_ref
;
1163 dev
->ibdev
.iwcm
->rem_ref
= iwch_qp_rem_ref
;
1164 dev
->ibdev
.iwcm
->get_qp
= iwch_get_qp
;
1166 ret
= ib_register_device(&dev
->ibdev
);
1170 for (i
= 0; i
< ARRAY_SIZE(iwch_class_attributes
); ++i
) {
1171 ret
= class_device_create_file(&dev
->ibdev
.class_dev
,
1172 iwch_class_attributes
[i
]);
1179 ib_unregister_device(&dev
->ibdev
);
1184 void iwch_unregister_device(struct iwch_dev
*dev
)
1188 PDBG("%s iwch_dev %p\n", __FUNCTION__
, dev
);
1189 for (i
= 0; i
< ARRAY_SIZE(iwch_class_attributes
); ++i
)
1190 class_device_remove_file(&dev
->ibdev
.class_dev
,
1191 iwch_class_attributes
[i
]);
1192 ib_unregister_device(&dev
->ibdev
);