2 * Copyright (c) 2012 - 2018 Intel Corporation. All rights reserved.
3 * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
4 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <rdma/ib_mad.h>
36 #include <rdma/ib_user_verbs.h>
38 #include <linux/module.h>
39 #include <linux/utsname.h>
40 #include <linux/rculist.h>
42 #include <linux/random.h>
43 #include <linux/vmalloc.h>
44 #include <rdma/rdma_vt.h>
47 #include "qib_common.h"
49 static unsigned int ib_qib_qp_table_size
= 256;
50 module_param_named(qp_table_size
, ib_qib_qp_table_size
, uint
, S_IRUGO
);
51 MODULE_PARM_DESC(qp_table_size
, "QP table size");
53 static unsigned int qib_lkey_table_size
= 16;
54 module_param_named(lkey_table_size
, qib_lkey_table_size
, uint
,
56 MODULE_PARM_DESC(lkey_table_size
,
57 "LKEY table size in bits (2^n, 1 <= n <= 23)");
59 static unsigned int ib_qib_max_pds
= 0xFFFF;
60 module_param_named(max_pds
, ib_qib_max_pds
, uint
, S_IRUGO
);
61 MODULE_PARM_DESC(max_pds
,
62 "Maximum number of protection domains to support");
64 static unsigned int ib_qib_max_ahs
= 0xFFFF;
65 module_param_named(max_ahs
, ib_qib_max_ahs
, uint
, S_IRUGO
);
66 MODULE_PARM_DESC(max_ahs
, "Maximum number of address handles to support");
68 unsigned int ib_qib_max_cqes
= 0x2FFFF;
69 module_param_named(max_cqes
, ib_qib_max_cqes
, uint
, S_IRUGO
);
70 MODULE_PARM_DESC(max_cqes
,
71 "Maximum number of completion queue entries to support");
73 unsigned int ib_qib_max_cqs
= 0x1FFFF;
74 module_param_named(max_cqs
, ib_qib_max_cqs
, uint
, S_IRUGO
);
75 MODULE_PARM_DESC(max_cqs
, "Maximum number of completion queues to support");
77 unsigned int ib_qib_max_qp_wrs
= 0x3FFF;
78 module_param_named(max_qp_wrs
, ib_qib_max_qp_wrs
, uint
, S_IRUGO
);
79 MODULE_PARM_DESC(max_qp_wrs
, "Maximum number of QP WRs to support");
81 unsigned int ib_qib_max_qps
= 16384;
82 module_param_named(max_qps
, ib_qib_max_qps
, uint
, S_IRUGO
);
83 MODULE_PARM_DESC(max_qps
, "Maximum number of QPs to support");
85 unsigned int ib_qib_max_sges
= 0x60;
86 module_param_named(max_sges
, ib_qib_max_sges
, uint
, S_IRUGO
);
87 MODULE_PARM_DESC(max_sges
, "Maximum number of SGEs to support");
89 unsigned int ib_qib_max_mcast_grps
= 16384;
90 module_param_named(max_mcast_grps
, ib_qib_max_mcast_grps
, uint
, S_IRUGO
);
91 MODULE_PARM_DESC(max_mcast_grps
,
92 "Maximum number of multicast groups to support");
94 unsigned int ib_qib_max_mcast_qp_attached
= 16;
95 module_param_named(max_mcast_qp_attached
, ib_qib_max_mcast_qp_attached
,
97 MODULE_PARM_DESC(max_mcast_qp_attached
,
98 "Maximum number of attached QPs to support");
100 unsigned int ib_qib_max_srqs
= 1024;
101 module_param_named(max_srqs
, ib_qib_max_srqs
, uint
, S_IRUGO
);
102 MODULE_PARM_DESC(max_srqs
, "Maximum number of SRQs to support");
104 unsigned int ib_qib_max_srq_sges
= 128;
105 module_param_named(max_srq_sges
, ib_qib_max_srq_sges
, uint
, S_IRUGO
);
106 MODULE_PARM_DESC(max_srq_sges
, "Maximum number of SRQ SGEs to support");
108 unsigned int ib_qib_max_srq_wrs
= 0x1FFFF;
109 module_param_named(max_srq_wrs
, ib_qib_max_srq_wrs
, uint
, S_IRUGO
);
110 MODULE_PARM_DESC(max_srq_wrs
, "Maximum number of SRQ WRs support");
112 static unsigned int ib_qib_disable_sma
;
113 module_param_named(disable_sma
, ib_qib_disable_sma
, uint
, S_IWUSR
| S_IRUGO
);
114 MODULE_PARM_DESC(disable_sma
, "Disable the SMA");
117 * Translate ib_wr_opcode into ib_wc_opcode.
119 const enum ib_wc_opcode ib_qib_wc_opcode
[] = {
120 [IB_WR_RDMA_WRITE
] = IB_WC_RDMA_WRITE
,
121 [IB_WR_RDMA_WRITE_WITH_IMM
] = IB_WC_RDMA_WRITE
,
122 [IB_WR_SEND
] = IB_WC_SEND
,
123 [IB_WR_SEND_WITH_IMM
] = IB_WC_SEND
,
124 [IB_WR_RDMA_READ
] = IB_WC_RDMA_READ
,
125 [IB_WR_ATOMIC_CMP_AND_SWP
] = IB_WC_COMP_SWAP
,
126 [IB_WR_ATOMIC_FETCH_AND_ADD
] = IB_WC_FETCH_ADD
132 __be64 ib_qib_sys_image_guid
;
135 * Count the number of DMA descriptors needed to send length bytes of data.
136 * Don't modify the qib_sge_state to get the count.
137 * Return zero if any of the segments is not aligned.
139 static u32
qib_count_sge(struct rvt_sge_state
*ss
, u32 length
)
141 struct rvt_sge
*sg_list
= ss
->sg_list
;
142 struct rvt_sge sge
= ss
->sge
;
143 u8 num_sge
= ss
->num_sge
;
144 u32 ndesc
= 1; /* count the header */
147 u32 len
= sge
.length
;
151 if (len
> sge
.sge_length
)
152 len
= sge
.sge_length
;
153 if (((long) sge
.vaddr
& (sizeof(u32
) - 1)) ||
154 (len
!= length
&& (len
& (sizeof(u32
) - 1)))) {
161 sge
.sge_length
-= len
;
162 if (sge
.sge_length
== 0) {
165 } else if (sge
.length
== 0 && sge
.mr
->lkey
) {
166 if (++sge
.n
>= RVT_SEGSZ
) {
167 if (++sge
.m
>= sge
.mr
->mapsz
)
172 sge
.mr
->map
[sge
.m
]->segs
[sge
.n
].vaddr
;
174 sge
.mr
->map
[sge
.m
]->segs
[sge
.n
].length
;
182 * Copy from the SGEs to the data buffer.
184 static void qib_copy_from_sge(void *data
, struct rvt_sge_state
*ss
, u32 length
)
186 struct rvt_sge
*sge
= &ss
->sge
;
189 u32 len
= sge
->length
;
193 if (len
> sge
->sge_length
)
194 len
= sge
->sge_length
;
195 memcpy(data
, sge
->vaddr
, len
);
198 sge
->sge_length
-= len
;
199 if (sge
->sge_length
== 0) {
201 *sge
= *ss
->sg_list
++;
202 } else if (sge
->length
== 0 && sge
->mr
->lkey
) {
203 if (++sge
->n
>= RVT_SEGSZ
) {
204 if (++sge
->m
>= sge
->mr
->mapsz
)
209 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
211 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
219 * qib_qp_rcv - processing an incoming packet on a QP
220 * @rcd: the context pointer
221 * @hdr: the packet header
222 * @has_grh: true if the packet has a GRH
223 * @data: the packet data
224 * @tlen: the packet length
225 * @qp: the QP the packet came on
227 * This is called from qib_ib_rcv() to process an incoming packet
229 * Called at interrupt level.
231 static void qib_qp_rcv(struct qib_ctxtdata
*rcd
, struct ib_header
*hdr
,
232 int has_grh
, void *data
, u32 tlen
, struct rvt_qp
*qp
)
234 struct qib_ibport
*ibp
= &rcd
->ppd
->ibport_data
;
236 spin_lock(&qp
->r_lock
);
238 /* Check for valid receive state. */
239 if (!(ib_rvt_state_ops
[qp
->state
] & RVT_PROCESS_RECV_OK
)) {
240 ibp
->rvp
.n_pkt_drops
++;
244 switch (qp
->ibqp
.qp_type
) {
247 if (ib_qib_disable_sma
)
251 qib_ud_rcv(ibp
, hdr
, has_grh
, data
, tlen
, qp
);
255 qib_rc_rcv(rcd
, hdr
, has_grh
, data
, tlen
, qp
);
259 qib_uc_rcv(ibp
, hdr
, has_grh
, data
, tlen
, qp
);
267 spin_unlock(&qp
->r_lock
);
271 * qib_ib_rcv - process an incoming packet
272 * @rcd: the context pointer
273 * @rhdr: the header of the packet
274 * @data: the packet payload
275 * @tlen: the packet length
277 * This is called from qib_kreceive() to process an incoming packet at
278 * interrupt level. Tlen is the length of the header + data + CRC in bytes.
280 void qib_ib_rcv(struct qib_ctxtdata
*rcd
, void *rhdr
, void *data
, u32 tlen
)
282 struct qib_pportdata
*ppd
= rcd
->ppd
;
283 struct qib_ibport
*ibp
= &ppd
->ibport_data
;
284 struct ib_header
*hdr
= rhdr
;
285 struct qib_devdata
*dd
= ppd
->dd
;
286 struct rvt_dev_info
*rdi
= &dd
->verbs_dev
.rdi
;
287 struct ib_other_headers
*ohdr
;
294 /* 24 == LRH+BTH+CRC */
295 if (unlikely(tlen
< 24))
298 /* Check for a valid destination LID (see ch. 7.11.1). */
299 lid
= be16_to_cpu(hdr
->lrh
[1]);
300 if (lid
< be16_to_cpu(IB_MULTICAST_LID_BASE
)) {
301 lid
&= ~((1 << ppd
->lmc
) - 1);
302 if (unlikely(lid
!= ppd
->lid
))
307 lnh
= be16_to_cpu(hdr
->lrh
[0]) & 3;
308 if (lnh
== QIB_LRH_BTH
)
310 else if (lnh
== QIB_LRH_GRH
) {
313 ohdr
= &hdr
->u
.l
.oth
;
314 if (hdr
->u
.l
.grh
.next_hdr
!= IB_GRH_NEXT_HDR
)
316 vtf
= be32_to_cpu(hdr
->u
.l
.grh
.version_tclass_flow
);
317 if ((vtf
>> IB_GRH_VERSION_SHIFT
) != IB_GRH_VERSION
)
322 opcode
= (be32_to_cpu(ohdr
->bth
[0]) >> 24) & 0x7f;
323 #ifdef CONFIG_DEBUG_FS
324 rcd
->opstats
->stats
[opcode
].n_bytes
+= tlen
;
325 rcd
->opstats
->stats
[opcode
].n_packets
++;
328 /* Get the destination QP number. */
329 qp_num
= be32_to_cpu(ohdr
->bth
[1]) & RVT_QPN_MASK
;
330 if (qp_num
== QIB_MULTICAST_QPN
) {
331 struct rvt_mcast
*mcast
;
332 struct rvt_mcast_qp
*p
;
334 if (lnh
!= QIB_LRH_GRH
)
336 mcast
= rvt_mcast_find(&ibp
->rvp
, &hdr
->u
.l
.grh
.dgid
, lid
);
339 this_cpu_inc(ibp
->pmastats
->n_multicast_rcv
);
340 list_for_each_entry_rcu(p
, &mcast
->qp_list
, list
)
341 qib_qp_rcv(rcd
, hdr
, 1, data
, tlen
, p
->qp
);
343 * Notify rvt_multicast_detach() if it is waiting for us
346 if (atomic_dec_return(&mcast
->refcount
) <= 1)
347 wake_up(&mcast
->wait
);
350 qp
= rvt_lookup_qpn(rdi
, &ibp
->rvp
, qp_num
);
355 this_cpu_inc(ibp
->pmastats
->n_unicast_rcv
);
356 qib_qp_rcv(rcd
, hdr
, lnh
== QIB_LRH_GRH
, data
, tlen
, qp
);
362 ibp
->rvp
.n_pkt_drops
++;
366 * This is called from a timer to check for QPs
367 * which need kernel memory in order to send a packet.
369 static void mem_timer(struct timer_list
*t
)
371 struct qib_ibdev
*dev
= from_timer(dev
, t
, mem_timer
);
372 struct list_head
*list
= &dev
->memwait
;
373 struct rvt_qp
*qp
= NULL
;
374 struct qib_qp_priv
*priv
= NULL
;
377 spin_lock_irqsave(&dev
->rdi
.pending_lock
, flags
);
378 if (!list_empty(list
)) {
379 priv
= list_entry(list
->next
, struct qib_qp_priv
, iowait
);
381 list_del_init(&priv
->iowait
);
383 if (!list_empty(list
))
384 mod_timer(&dev
->mem_timer
, jiffies
+ 1);
386 spin_unlock_irqrestore(&dev
->rdi
.pending_lock
, flags
);
389 spin_lock_irqsave(&qp
->s_lock
, flags
);
390 if (qp
->s_flags
& RVT_S_WAIT_KMEM
) {
391 qp
->s_flags
&= ~RVT_S_WAIT_KMEM
;
392 qib_schedule_send(qp
);
394 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
399 #ifdef __LITTLE_ENDIAN
400 static inline u32
get_upper_bits(u32 data
, u32 shift
)
402 return data
>> shift
;
405 static inline u32
set_upper_bits(u32 data
, u32 shift
)
407 return data
<< shift
;
410 static inline u32
clear_upper_bytes(u32 data
, u32 n
, u32 off
)
412 data
<<= ((sizeof(u32
) - n
) * BITS_PER_BYTE
);
413 data
>>= ((sizeof(u32
) - n
- off
) * BITS_PER_BYTE
);
417 static inline u32
get_upper_bits(u32 data
, u32 shift
)
419 return data
<< shift
;
422 static inline u32
set_upper_bits(u32 data
, u32 shift
)
424 return data
>> shift
;
427 static inline u32
clear_upper_bytes(u32 data
, u32 n
, u32 off
)
429 data
>>= ((sizeof(u32
) - n
) * BITS_PER_BYTE
);
430 data
<<= ((sizeof(u32
) - n
- off
) * BITS_PER_BYTE
);
435 static void copy_io(u32 __iomem
*piobuf
, struct rvt_sge_state
*ss
,
436 u32 length
, unsigned flush_wc
)
443 u32 len
= ss
->sge
.length
;
448 if (len
> ss
->sge
.sge_length
)
449 len
= ss
->sge
.sge_length
;
450 /* If the source address is not aligned, try to align it. */
451 off
= (unsigned long)ss
->sge
.vaddr
& (sizeof(u32
) - 1);
453 u32
*addr
= (u32
*)((unsigned long)ss
->sge
.vaddr
&
455 u32 v
= get_upper_bits(*addr
, off
* BITS_PER_BYTE
);
458 y
= sizeof(u32
) - off
;
461 if (len
+ extra
>= sizeof(u32
)) {
462 data
|= set_upper_bits(v
, extra
*
464 len
= sizeof(u32
) - extra
;
469 __raw_writel(data
, piobuf
);
474 /* Clear unused upper bytes */
475 data
|= clear_upper_bytes(v
, len
, extra
);
483 /* Source address is aligned. */
484 u32
*addr
= (u32
*) ss
->sge
.vaddr
;
485 int shift
= extra
* BITS_PER_BYTE
;
486 int ushift
= 32 - shift
;
489 while (l
>= sizeof(u32
)) {
492 data
|= set_upper_bits(v
, shift
);
493 __raw_writel(data
, piobuf
);
494 data
= get_upper_bits(v
, ushift
);
500 * We still have 'extra' number of bytes leftover.
505 if (l
+ extra
>= sizeof(u32
)) {
506 data
|= set_upper_bits(v
, shift
);
507 len
-= l
+ extra
- sizeof(u32
);
512 __raw_writel(data
, piobuf
);
517 /* Clear unused upper bytes */
518 data
|= clear_upper_bytes(v
, l
, extra
);
525 } else if (len
== length
) {
529 } else if (len
== length
) {
533 * Need to round up for the last dword in the
537 qib_pio_copy(piobuf
, ss
->sge
.vaddr
, w
- 1);
539 last
= ((u32
*) ss
->sge
.vaddr
)[w
- 1];
544 qib_pio_copy(piobuf
, ss
->sge
.vaddr
, w
);
547 extra
= len
& (sizeof(u32
) - 1);
549 u32 v
= ((u32
*) ss
->sge
.vaddr
)[w
];
551 /* Clear unused upper bytes */
552 data
= clear_upper_bytes(v
, extra
, 0);
555 rvt_update_sge(ss
, len
, false);
558 /* Update address before sending packet. */
559 rvt_update_sge(ss
, length
, false);
561 /* must flush early everything before trigger word */
563 __raw_writel(last
, piobuf
);
564 /* be sure trigger word is written */
567 __raw_writel(last
, piobuf
);
570 static noinline
struct qib_verbs_txreq
*__get_txreq(struct qib_ibdev
*dev
,
573 struct qib_qp_priv
*priv
= qp
->priv
;
574 struct qib_verbs_txreq
*tx
;
577 spin_lock_irqsave(&qp
->s_lock
, flags
);
578 spin_lock(&dev
->rdi
.pending_lock
);
580 if (!list_empty(&dev
->txreq_free
)) {
581 struct list_head
*l
= dev
->txreq_free
.next
;
584 spin_unlock(&dev
->rdi
.pending_lock
);
585 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
586 tx
= list_entry(l
, struct qib_verbs_txreq
, txreq
.list
);
588 if (ib_rvt_state_ops
[qp
->state
] & RVT_PROCESS_RECV_OK
&&
589 list_empty(&priv
->iowait
)) {
591 qp
->s_flags
|= RVT_S_WAIT_TX
;
592 list_add_tail(&priv
->iowait
, &dev
->txwait
);
594 qp
->s_flags
&= ~RVT_S_BUSY
;
595 spin_unlock(&dev
->rdi
.pending_lock
);
596 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
597 tx
= ERR_PTR(-EBUSY
);
602 static inline struct qib_verbs_txreq
*get_txreq(struct qib_ibdev
*dev
,
605 struct qib_verbs_txreq
*tx
;
608 spin_lock_irqsave(&dev
->rdi
.pending_lock
, flags
);
609 /* assume the list non empty */
610 if (likely(!list_empty(&dev
->txreq_free
))) {
611 struct list_head
*l
= dev
->txreq_free
.next
;
614 spin_unlock_irqrestore(&dev
->rdi
.pending_lock
, flags
);
615 tx
= list_entry(l
, struct qib_verbs_txreq
, txreq
.list
);
617 /* call slow path to get the extra lock */
618 spin_unlock_irqrestore(&dev
->rdi
.pending_lock
, flags
);
619 tx
= __get_txreq(dev
, qp
);
624 void qib_put_txreq(struct qib_verbs_txreq
*tx
)
626 struct qib_ibdev
*dev
;
628 struct qib_qp_priv
*priv
;
632 dev
= to_idev(qp
->ibqp
.device
);
638 if (tx
->txreq
.flags
& QIB_SDMA_TXREQ_F_FREEBUF
) {
639 tx
->txreq
.flags
&= ~QIB_SDMA_TXREQ_F_FREEBUF
;
640 dma_unmap_single(&dd_from_dev(dev
)->pcidev
->dev
,
641 tx
->txreq
.addr
, tx
->hdr_dwords
<< 2,
643 kfree(tx
->align_buf
);
646 spin_lock_irqsave(&dev
->rdi
.pending_lock
, flags
);
648 /* Put struct back on free list */
649 list_add(&tx
->txreq
.list
, &dev
->txreq_free
);
651 if (!list_empty(&dev
->txwait
)) {
652 /* Wake up first QP wanting a free struct */
653 priv
= list_entry(dev
->txwait
.next
, struct qib_qp_priv
,
656 list_del_init(&priv
->iowait
);
658 spin_unlock_irqrestore(&dev
->rdi
.pending_lock
, flags
);
660 spin_lock_irqsave(&qp
->s_lock
, flags
);
661 if (qp
->s_flags
& RVT_S_WAIT_TX
) {
662 qp
->s_flags
&= ~RVT_S_WAIT_TX
;
663 qib_schedule_send(qp
);
665 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
669 spin_unlock_irqrestore(&dev
->rdi
.pending_lock
, flags
);
673 * This is called when there are send DMA descriptors that might be
676 * This is called with ppd->sdma_lock held.
678 void qib_verbs_sdma_desc_avail(struct qib_pportdata
*ppd
, unsigned avail
)
681 struct qib_qp_priv
*qpp
, *nqpp
;
682 struct rvt_qp
*qps
[20];
683 struct qib_ibdev
*dev
;
687 dev
= &ppd
->dd
->verbs_dev
;
688 spin_lock(&dev
->rdi
.pending_lock
);
690 /* Search wait list for first QP wanting DMA descriptors. */
691 list_for_each_entry_safe(qpp
, nqpp
, &dev
->dmawait
, iowait
) {
693 if (qp
->port_num
!= ppd
->port
)
695 if (n
== ARRAY_SIZE(qps
))
697 if (qpp
->s_tx
->txreq
.sg_count
> avail
)
699 avail
-= qpp
->s_tx
->txreq
.sg_count
;
700 list_del_init(&qpp
->iowait
);
705 spin_unlock(&dev
->rdi
.pending_lock
);
707 for (i
= 0; i
< n
; i
++) {
709 spin_lock(&qp
->s_lock
);
710 if (qp
->s_flags
& RVT_S_WAIT_DMA_DESC
) {
711 qp
->s_flags
&= ~RVT_S_WAIT_DMA_DESC
;
712 qib_schedule_send(qp
);
714 spin_unlock(&qp
->s_lock
);
720 * This is called with ppd->sdma_lock held.
722 static void sdma_complete(struct qib_sdma_txreq
*cookie
, int status
)
724 struct qib_verbs_txreq
*tx
=
725 container_of(cookie
, struct qib_verbs_txreq
, txreq
);
726 struct rvt_qp
*qp
= tx
->qp
;
727 struct qib_qp_priv
*priv
= qp
->priv
;
729 spin_lock(&qp
->s_lock
);
731 rvt_send_complete(qp
, tx
->wqe
, IB_WC_SUCCESS
);
732 else if (qp
->ibqp
.qp_type
== IB_QPT_RC
) {
733 struct ib_header
*hdr
;
735 if (tx
->txreq
.flags
& QIB_SDMA_TXREQ_F_FREEBUF
)
736 hdr
= &tx
->align_buf
->hdr
;
738 struct qib_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
740 hdr
= &dev
->pio_hdrs
[tx
->hdr_inx
].hdr
;
742 qib_rc_send_complete(qp
, hdr
);
744 if (atomic_dec_and_test(&priv
->s_dma_busy
)) {
745 if (qp
->state
== IB_QPS_RESET
)
746 wake_up(&priv
->wait_dma
);
747 else if (qp
->s_flags
& RVT_S_WAIT_DMA
) {
748 qp
->s_flags
&= ~RVT_S_WAIT_DMA
;
749 qib_schedule_send(qp
);
752 spin_unlock(&qp
->s_lock
);
757 static int wait_kmem(struct qib_ibdev
*dev
, struct rvt_qp
*qp
)
759 struct qib_qp_priv
*priv
= qp
->priv
;
763 spin_lock_irqsave(&qp
->s_lock
, flags
);
764 if (ib_rvt_state_ops
[qp
->state
] & RVT_PROCESS_RECV_OK
) {
765 spin_lock(&dev
->rdi
.pending_lock
);
766 if (list_empty(&priv
->iowait
)) {
767 if (list_empty(&dev
->memwait
))
768 mod_timer(&dev
->mem_timer
, jiffies
+ 1);
769 qp
->s_flags
|= RVT_S_WAIT_KMEM
;
770 list_add_tail(&priv
->iowait
, &dev
->memwait
);
772 spin_unlock(&dev
->rdi
.pending_lock
);
773 qp
->s_flags
&= ~RVT_S_BUSY
;
776 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
781 static int qib_verbs_send_dma(struct rvt_qp
*qp
, struct ib_header
*hdr
,
782 u32 hdrwords
, struct rvt_sge_state
*ss
, u32 len
,
783 u32 plen
, u32 dwords
)
785 struct qib_qp_priv
*priv
= qp
->priv
;
786 struct qib_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
787 struct qib_devdata
*dd
= dd_from_dev(dev
);
788 struct qib_ibport
*ibp
= to_iport(qp
->ibqp
.device
, qp
->port_num
);
789 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
790 struct qib_verbs_txreq
*tx
;
791 struct qib_pio_header
*phdr
;
799 /* resend previously constructed packet */
800 ret
= qib_sdma_verbs_send(ppd
, tx
->ss
, tx
->dwords
, tx
);
804 tx
= get_txreq(dev
, qp
);
808 control
= dd
->f_setpbc_control(ppd
, plen
, qp
->s_srate
,
809 be16_to_cpu(hdr
->lrh
[0]) >> 12);
812 tx
->mr
= qp
->s_rdma_mr
;
814 qp
->s_rdma_mr
= NULL
;
815 tx
->txreq
.callback
= sdma_complete
;
816 if (dd
->flags
& QIB_HAS_SDMA_TIMEOUT
)
817 tx
->txreq
.flags
= QIB_SDMA_TXREQ_F_HEADTOHOST
;
819 tx
->txreq
.flags
= QIB_SDMA_TXREQ_F_INTREQ
;
820 if (plen
+ 1 > dd
->piosize2kmax_dwords
)
821 tx
->txreq
.flags
|= QIB_SDMA_TXREQ_F_USELARGEBUF
;
825 * Don't try to DMA if it takes more descriptors than
828 ndesc
= qib_count_sge(ss
, len
);
829 if (ndesc
>= ppd
->sdma_descq_cnt
)
834 phdr
= &dev
->pio_hdrs
[tx
->hdr_inx
];
835 phdr
->pbc
[0] = cpu_to_le32(plen
);
836 phdr
->pbc
[1] = cpu_to_le32(control
);
837 memcpy(&phdr
->hdr
, hdr
, hdrwords
<< 2);
838 tx
->txreq
.flags
|= QIB_SDMA_TXREQ_F_FREEDESC
;
839 tx
->txreq
.sg_count
= ndesc
;
840 tx
->txreq
.addr
= dev
->pio_hdrs_phys
+
841 tx
->hdr_inx
* sizeof(struct qib_pio_header
);
842 tx
->hdr_dwords
= hdrwords
+ 2; /* add PBC length */
843 ret
= qib_sdma_verbs_send(ppd
, ss
, dwords
, tx
);
847 /* Allocate a buffer and copy the header and payload to it. */
848 tx
->hdr_dwords
= plen
+ 1;
849 phdr
= kmalloc(tx
->hdr_dwords
<< 2, GFP_ATOMIC
);
852 phdr
->pbc
[0] = cpu_to_le32(plen
);
853 phdr
->pbc
[1] = cpu_to_le32(control
);
854 memcpy(&phdr
->hdr
, hdr
, hdrwords
<< 2);
855 qib_copy_from_sge((u32
*) &phdr
->hdr
+ hdrwords
, ss
, len
);
857 tx
->txreq
.addr
= dma_map_single(&dd
->pcidev
->dev
, phdr
,
858 tx
->hdr_dwords
<< 2, DMA_TO_DEVICE
);
859 if (dma_mapping_error(&dd
->pcidev
->dev
, tx
->txreq
.addr
))
861 tx
->align_buf
= phdr
;
862 tx
->txreq
.flags
|= QIB_SDMA_TXREQ_F_FREEBUF
;
863 tx
->txreq
.sg_count
= 1;
864 ret
= qib_sdma_verbs_send(ppd
, NULL
, 0, tx
);
871 ret
= wait_kmem(dev
, qp
);
873 ibp
->rvp
.n_unaligned
++;
882 * If we are now in the error state, return zero to flush the
885 static int no_bufs_available(struct rvt_qp
*qp
)
887 struct qib_qp_priv
*priv
= qp
->priv
;
888 struct qib_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
889 struct qib_devdata
*dd
;
894 * Note that as soon as want_buffer() is called and
895 * possibly before it returns, qib_ib_piobufavail()
896 * could be called. Therefore, put QP on the I/O wait list before
897 * enabling the PIO avail interrupt.
899 spin_lock_irqsave(&qp
->s_lock
, flags
);
900 if (ib_rvt_state_ops
[qp
->state
] & RVT_PROCESS_RECV_OK
) {
901 spin_lock(&dev
->rdi
.pending_lock
);
902 if (list_empty(&priv
->iowait
)) {
904 qp
->s_flags
|= RVT_S_WAIT_PIO
;
905 list_add_tail(&priv
->iowait
, &dev
->piowait
);
906 dd
= dd_from_dev(dev
);
907 dd
->f_wantpiobuf_intr(dd
, 1);
909 spin_unlock(&dev
->rdi
.pending_lock
);
910 qp
->s_flags
&= ~RVT_S_BUSY
;
913 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
917 static int qib_verbs_send_pio(struct rvt_qp
*qp
, struct ib_header
*ibhdr
,
918 u32 hdrwords
, struct rvt_sge_state
*ss
, u32 len
,
919 u32 plen
, u32 dwords
)
921 struct qib_devdata
*dd
= dd_from_ibdev(qp
->ibqp
.device
);
922 struct qib_pportdata
*ppd
= dd
->pport
+ qp
->port_num
- 1;
923 u32
*hdr
= (u32
*) ibhdr
;
924 u32 __iomem
*piobuf_orig
;
932 control
= dd
->f_setpbc_control(ppd
, plen
, qp
->s_srate
,
933 be16_to_cpu(ibhdr
->lrh
[0]) >> 12);
934 pbc
= ((u64
) control
<< 32) | plen
;
935 piobuf
= dd
->f_getsendbuf(ppd
, pbc
, &pbufn
);
936 if (unlikely(piobuf
== NULL
))
937 return no_bufs_available(qp
);
941 * We have to flush after the PBC for correctness on some cpus
942 * or WC buffer can be written out of order.
945 piobuf_orig
= piobuf
;
948 flush_wc
= dd
->flags
& QIB_PIO_FLUSH_WC
;
951 * If there is just the header portion, must flush before
952 * writing last word of header for correctness, and after
953 * the last header word (trigger word).
957 qib_pio_copy(piobuf
, hdr
, hdrwords
- 1);
959 __raw_writel(hdr
[hdrwords
- 1], piobuf
+ hdrwords
- 1);
962 qib_pio_copy(piobuf
, hdr
, hdrwords
);
968 qib_pio_copy(piobuf
, hdr
, hdrwords
);
971 /* The common case is aligned and contained in one segment. */
972 if (likely(ss
->num_sge
== 1 && len
<= ss
->sge
.length
&&
973 !((unsigned long)ss
->sge
.vaddr
& (sizeof(u32
) - 1)))) {
974 u32
*addr
= (u32
*) ss
->sge
.vaddr
;
976 /* Update address before sending packet. */
977 rvt_update_sge(ss
, len
, false);
979 qib_pio_copy(piobuf
, addr
, dwords
- 1);
980 /* must flush early everything before trigger word */
982 __raw_writel(addr
[dwords
- 1], piobuf
+ dwords
- 1);
983 /* be sure trigger word is written */
986 qib_pio_copy(piobuf
, addr
, dwords
);
989 copy_io(piobuf
, ss
, len
, flush_wc
);
991 if (dd
->flags
& QIB_USE_SPCL_TRIG
) {
992 u32 spcl_off
= (pbufn
>= dd
->piobcnt2k
) ? 2047 : 1023;
995 __raw_writel(0xaebecede, piobuf_orig
+ spcl_off
);
997 qib_sendbuf_done(dd
, pbufn
);
999 rvt_put_mr(qp
->s_rdma_mr
);
1000 qp
->s_rdma_mr
= NULL
;
1003 spin_lock_irqsave(&qp
->s_lock
, flags
);
1004 rvt_send_complete(qp
, qp
->s_wqe
, IB_WC_SUCCESS
);
1005 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1006 } else if (qp
->ibqp
.qp_type
== IB_QPT_RC
) {
1007 spin_lock_irqsave(&qp
->s_lock
, flags
);
1008 qib_rc_send_complete(qp
, ibhdr
);
1009 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1015 * qib_verbs_send - send a packet
1016 * @qp: the QP to send on
1017 * @hdr: the packet header
1018 * @hdrwords: the number of 32-bit words in the header
1019 * @ss: the SGE to send
1020 * @len: the length of the packet in bytes
1022 * Return zero if packet is sent or queued OK.
1023 * Return non-zero and clear qp->s_flags RVT_S_BUSY otherwise.
1025 int qib_verbs_send(struct rvt_qp
*qp
, struct ib_header
*hdr
,
1026 u32 hdrwords
, struct rvt_sge_state
*ss
, u32 len
)
1028 struct qib_devdata
*dd
= dd_from_ibdev(qp
->ibqp
.device
);
1031 u32 dwords
= (len
+ 3) >> 2;
1034 * Calculate the send buffer trigger address.
1035 * The +1 counts for the pbc control dword following the pbc length.
1037 plen
= hdrwords
+ dwords
+ 1;
1040 * VL15 packets (IB_QPT_SMI) will always use PIO, so we
1041 * can defer SDMA restart until link goes ACTIVE without
1042 * worrying about just how we got there.
1044 if (qp
->ibqp
.qp_type
== IB_QPT_SMI
||
1045 !(dd
->flags
& QIB_HAS_SEND_DMA
))
1046 ret
= qib_verbs_send_pio(qp
, hdr
, hdrwords
, ss
, len
,
1049 ret
= qib_verbs_send_dma(qp
, hdr
, hdrwords
, ss
, len
,
1055 int qib_snapshot_counters(struct qib_pportdata
*ppd
, u64
*swords
,
1056 u64
*rwords
, u64
*spkts
, u64
*rpkts
,
1060 struct qib_devdata
*dd
= ppd
->dd
;
1062 if (!(dd
->flags
& QIB_PRESENT
)) {
1063 /* no hardware, freeze, etc. */
1067 *swords
= dd
->f_portcntr(ppd
, QIBPORTCNTR_WORDSEND
);
1068 *rwords
= dd
->f_portcntr(ppd
, QIBPORTCNTR_WORDRCV
);
1069 *spkts
= dd
->f_portcntr(ppd
, QIBPORTCNTR_PKTSEND
);
1070 *rpkts
= dd
->f_portcntr(ppd
, QIBPORTCNTR_PKTRCV
);
1071 *xmit_wait
= dd
->f_portcntr(ppd
, QIBPORTCNTR_SENDSTALL
);
1080 * qib_get_counters - get various chip counters
1081 * @dd: the qlogic_ib device
1082 * @cntrs: counters are placed here
1084 * Return the counters needed by recv_pma_get_portcounters().
1086 int qib_get_counters(struct qib_pportdata
*ppd
,
1087 struct qib_verbs_counters
*cntrs
)
1091 if (!(ppd
->dd
->flags
& QIB_PRESENT
)) {
1092 /* no hardware, freeze, etc. */
1096 cntrs
->symbol_error_counter
=
1097 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_IBSYMBOLERR
);
1098 cntrs
->link_error_recovery_counter
=
1099 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_IBLINKERRRECOV
);
1101 * The link downed counter counts when the other side downs the
1102 * connection. We add in the number of times we downed the link
1103 * due to local link integrity errors to compensate.
1105 cntrs
->link_downed_counter
=
1106 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_IBLINKDOWN
);
1107 cntrs
->port_rcv_errors
=
1108 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RXDROPPKT
) +
1109 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RCVOVFL
) +
1110 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERR_RLEN
) +
1111 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_INVALIDRLEN
) +
1112 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERRLINK
) +
1113 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERRICRC
) +
1114 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERRVCRC
) +
1115 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERRLPCRC
) +
1116 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_BADFORMAT
);
1117 cntrs
->port_rcv_errors
+=
1118 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RXLOCALPHYERR
);
1119 cntrs
->port_rcv_errors
+=
1120 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RXVLERR
);
1121 cntrs
->port_rcv_remphys_errors
=
1122 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RCVEBP
);
1123 cntrs
->port_xmit_discards
=
1124 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_UNSUPVL
);
1125 cntrs
->port_xmit_data
= ppd
->dd
->f_portcntr(ppd
,
1126 QIBPORTCNTR_WORDSEND
);
1127 cntrs
->port_rcv_data
= ppd
->dd
->f_portcntr(ppd
,
1128 QIBPORTCNTR_WORDRCV
);
1129 cntrs
->port_xmit_packets
= ppd
->dd
->f_portcntr(ppd
,
1130 QIBPORTCNTR_PKTSEND
);
1131 cntrs
->port_rcv_packets
= ppd
->dd
->f_portcntr(ppd
,
1132 QIBPORTCNTR_PKTRCV
);
1133 cntrs
->local_link_integrity_errors
=
1134 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_LLI
);
1135 cntrs
->excessive_buffer_overrun_errors
=
1136 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_EXCESSBUFOVFL
);
1137 cntrs
->vl15_dropped
=
1138 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_VL15PKTDROP
);
1147 * qib_ib_piobufavail - callback when a PIO buffer is available
1148 * @dd: the device pointer
1150 * This is called from qib_intr() at interrupt level when a PIO buffer is
1151 * available after qib_verbs_send() returned an error that no buffers were
1152 * available. Disable the interrupt if there are no more QPs waiting.
1154 void qib_ib_piobufavail(struct qib_devdata
*dd
)
1156 struct qib_ibdev
*dev
= &dd
->verbs_dev
;
1157 struct list_head
*list
;
1158 struct rvt_qp
*qps
[5];
1160 unsigned long flags
;
1162 struct qib_qp_priv
*priv
;
1164 list
= &dev
->piowait
;
1168 * Note: checking that the piowait list is empty and clearing
1169 * the buffer available interrupt needs to be atomic or we
1170 * could end up with QPs on the wait list with the interrupt
1173 spin_lock_irqsave(&dev
->rdi
.pending_lock
, flags
);
1174 while (!list_empty(list
)) {
1175 if (n
== ARRAY_SIZE(qps
))
1177 priv
= list_entry(list
->next
, struct qib_qp_priv
, iowait
);
1179 list_del_init(&priv
->iowait
);
1183 dd
->f_wantpiobuf_intr(dd
, 0);
1185 spin_unlock_irqrestore(&dev
->rdi
.pending_lock
, flags
);
1187 for (i
= 0; i
< n
; i
++) {
1190 spin_lock_irqsave(&qp
->s_lock
, flags
);
1191 if (qp
->s_flags
& RVT_S_WAIT_PIO
) {
1192 qp
->s_flags
&= ~RVT_S_WAIT_PIO
;
1193 qib_schedule_send(qp
);
1195 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1197 /* Notify qib_destroy_qp() if it is waiting. */
1202 static int qib_query_port(struct rvt_dev_info
*rdi
, u8 port_num
,
1203 struct ib_port_attr
*props
)
1205 struct qib_ibdev
*ibdev
= container_of(rdi
, struct qib_ibdev
, rdi
);
1206 struct qib_devdata
*dd
= dd_from_dev(ibdev
);
1207 struct qib_pportdata
*ppd
= &dd
->pport
[port_num
- 1];
1211 /* props being zeroed by the caller, avoid zeroing it here */
1212 props
->lid
= lid
? lid
: be16_to_cpu(IB_LID_PERMISSIVE
);
1213 props
->lmc
= ppd
->lmc
;
1214 props
->state
= dd
->f_iblink_state(ppd
->lastibcstat
);
1215 props
->phys_state
= dd
->f_ibphys_portstate(ppd
->lastibcstat
);
1216 props
->gid_tbl_len
= QIB_GUIDS_PER_PORT
;
1217 props
->active_width
= ppd
->link_width_active
;
1218 /* See rate_show() */
1219 props
->active_speed
= ppd
->link_speed_active
;
1220 props
->max_vl_num
= qib_num_vls(ppd
->vls_supported
);
1222 props
->max_mtu
= qib_ibmtu
? qib_ibmtu
: IB_MTU_4096
;
1223 switch (ppd
->ibmtu
) {
1242 props
->active_mtu
= mtu
;
1247 static int qib_modify_device(struct ib_device
*device
,
1248 int device_modify_mask
,
1249 struct ib_device_modify
*device_modify
)
1251 struct qib_devdata
*dd
= dd_from_ibdev(device
);
1255 if (device_modify_mask
& ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID
|
1256 IB_DEVICE_MODIFY_NODE_DESC
)) {
1261 if (device_modify_mask
& IB_DEVICE_MODIFY_NODE_DESC
) {
1262 memcpy(device
->node_desc
, device_modify
->node_desc
,
1263 IB_DEVICE_NODE_DESC_MAX
);
1264 for (i
= 0; i
< dd
->num_pports
; i
++) {
1265 struct qib_ibport
*ibp
= &dd
->pport
[i
].ibport_data
;
1267 qib_node_desc_chg(ibp
);
1271 if (device_modify_mask
& IB_DEVICE_MODIFY_SYS_IMAGE_GUID
) {
1272 ib_qib_sys_image_guid
=
1273 cpu_to_be64(device_modify
->sys_image_guid
);
1274 for (i
= 0; i
< dd
->num_pports
; i
++) {
1275 struct qib_ibport
*ibp
= &dd
->pport
[i
].ibport_data
;
1277 qib_sys_guid_chg(ibp
);
1287 static int qib_shut_down_port(struct rvt_dev_info
*rdi
, u8 port_num
)
1289 struct qib_ibdev
*ibdev
= container_of(rdi
, struct qib_ibdev
, rdi
);
1290 struct qib_devdata
*dd
= dd_from_dev(ibdev
);
1291 struct qib_pportdata
*ppd
= &dd
->pport
[port_num
- 1];
1293 qib_set_linkstate(ppd
, QIB_IB_LINKDOWN
);
1298 static int qib_get_guid_be(struct rvt_dev_info
*rdi
, struct rvt_ibport
*rvp
,
1299 int guid_index
, __be64
*guid
)
1301 struct qib_ibport
*ibp
= container_of(rvp
, struct qib_ibport
, rvp
);
1302 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
1304 if (guid_index
== 0)
1306 else if (guid_index
< QIB_GUIDS_PER_PORT
)
1307 *guid
= ibp
->guids
[guid_index
- 1];
1314 int qib_check_ah(struct ib_device
*ibdev
, struct rdma_ah_attr
*ah_attr
)
1316 if (rdma_ah_get_sl(ah_attr
) > 15)
1319 if (rdma_ah_get_dlid(ah_attr
) == 0)
1321 if (rdma_ah_get_dlid(ah_attr
) >=
1322 be16_to_cpu(IB_MULTICAST_LID_BASE
) &&
1323 rdma_ah_get_dlid(ah_attr
) !=
1324 be16_to_cpu(IB_LID_PERMISSIVE
) &&
1325 !(rdma_ah_get_ah_flags(ah_attr
) & IB_AH_GRH
))
1331 static void qib_notify_new_ah(struct ib_device
*ibdev
,
1332 struct rdma_ah_attr
*ah_attr
,
1335 struct qib_ibport
*ibp
;
1336 struct qib_pportdata
*ppd
;
1339 * Do not trust reading anything from rvt_ah at this point as it is not
1340 * done being setup. We can however modify things which we need to set.
1343 ibp
= to_iport(ibdev
, rdma_ah_get_port_num(ah_attr
));
1344 ppd
= ppd_from_ibp(ibp
);
1345 ah
->vl
= ibp
->sl_to_vl
[rdma_ah_get_sl(&ah
->attr
)];
1346 ah
->log_pmtu
= ilog2(ppd
->ibmtu
);
1349 struct ib_ah
*qib_create_qp0_ah(struct qib_ibport
*ibp
, u16 dlid
)
1351 struct rdma_ah_attr attr
;
1352 struct ib_ah
*ah
= ERR_PTR(-EINVAL
);
1354 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
1355 struct qib_devdata
*dd
= dd_from_ppd(ppd
);
1356 u8 port_num
= ppd
->port
;
1358 memset(&attr
, 0, sizeof(attr
));
1359 attr
.type
= rdma_ah_find_type(&dd
->verbs_dev
.rdi
.ibdev
, port_num
);
1360 rdma_ah_set_dlid(&attr
, dlid
);
1361 rdma_ah_set_port_num(&attr
, port_num
);
1363 qp0
= rcu_dereference(ibp
->rvp
.qp
[0]);
1365 ah
= rdma_create_ah(qp0
->ibqp
.pd
, &attr
, 0);
1371 * qib_get_npkeys - return the size of the PKEY table for context 0
1372 * @dd: the qlogic_ib device
1374 unsigned qib_get_npkeys(struct qib_devdata
*dd
)
1376 return ARRAY_SIZE(dd
->rcd
[0]->pkeys
);
1380 * Return the indexed PKEY from the port PKEY table.
1381 * No need to validate rcd[ctxt]; the port is setup if we are here.
1383 unsigned qib_get_pkey(struct qib_ibport
*ibp
, unsigned index
)
1385 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
1386 struct qib_devdata
*dd
= ppd
->dd
;
1387 unsigned ctxt
= ppd
->hw_pidx
;
1390 /* dd->rcd null if mini_init or some init failures */
1391 if (!dd
->rcd
|| index
>= ARRAY_SIZE(dd
->rcd
[ctxt
]->pkeys
))
1394 ret
= dd
->rcd
[ctxt
]->pkeys
[index
];
1399 static void init_ibport(struct qib_pportdata
*ppd
)
1401 struct qib_verbs_counters cntrs
;
1402 struct qib_ibport
*ibp
= &ppd
->ibport_data
;
1404 spin_lock_init(&ibp
->rvp
.lock
);
1405 /* Set the prefix to the default value (see ch. 4.1.1) */
1406 ibp
->rvp
.gid_prefix
= IB_DEFAULT_GID_PREFIX
;
1407 ibp
->rvp
.sm_lid
= be16_to_cpu(IB_LID_PERMISSIVE
);
1408 ibp
->rvp
.port_cap_flags
= IB_PORT_SYS_IMAGE_GUID_SUP
|
1409 IB_PORT_CLIENT_REG_SUP
| IB_PORT_SL_MAP_SUP
|
1410 IB_PORT_TRAP_SUP
| IB_PORT_AUTO_MIGR_SUP
|
1411 IB_PORT_DR_NOTICE_SUP
| IB_PORT_CAP_MASK_NOTICE_SUP
|
1412 IB_PORT_OTHER_LOCAL_CHANGES_SUP
;
1413 if (ppd
->dd
->flags
& QIB_HAS_LINK_LATENCY
)
1414 ibp
->rvp
.port_cap_flags
|= IB_PORT_LINK_LATENCY_SUP
;
1415 ibp
->rvp
.pma_counter_select
[0] = IB_PMA_PORT_XMIT_DATA
;
1416 ibp
->rvp
.pma_counter_select
[1] = IB_PMA_PORT_RCV_DATA
;
1417 ibp
->rvp
.pma_counter_select
[2] = IB_PMA_PORT_XMIT_PKTS
;
1418 ibp
->rvp
.pma_counter_select
[3] = IB_PMA_PORT_RCV_PKTS
;
1419 ibp
->rvp
.pma_counter_select
[4] = IB_PMA_PORT_XMIT_WAIT
;
1421 /* Snapshot current HW counters to "clear" them. */
1422 qib_get_counters(ppd
, &cntrs
);
1423 ibp
->z_symbol_error_counter
= cntrs
.symbol_error_counter
;
1424 ibp
->z_link_error_recovery_counter
=
1425 cntrs
.link_error_recovery_counter
;
1426 ibp
->z_link_downed_counter
= cntrs
.link_downed_counter
;
1427 ibp
->z_port_rcv_errors
= cntrs
.port_rcv_errors
;
1428 ibp
->z_port_rcv_remphys_errors
= cntrs
.port_rcv_remphys_errors
;
1429 ibp
->z_port_xmit_discards
= cntrs
.port_xmit_discards
;
1430 ibp
->z_port_xmit_data
= cntrs
.port_xmit_data
;
1431 ibp
->z_port_rcv_data
= cntrs
.port_rcv_data
;
1432 ibp
->z_port_xmit_packets
= cntrs
.port_xmit_packets
;
1433 ibp
->z_port_rcv_packets
= cntrs
.port_rcv_packets
;
1434 ibp
->z_local_link_integrity_errors
=
1435 cntrs
.local_link_integrity_errors
;
1436 ibp
->z_excessive_buffer_overrun_errors
=
1437 cntrs
.excessive_buffer_overrun_errors
;
1438 ibp
->z_vl15_dropped
= cntrs
.vl15_dropped
;
1439 RCU_INIT_POINTER(ibp
->rvp
.qp
[0], NULL
);
1440 RCU_INIT_POINTER(ibp
->rvp
.qp
[1], NULL
);
1444 * qib_fill_device_attr - Fill in rvt dev info device attributes.
1445 * @dd: the device data structure
1447 static void qib_fill_device_attr(struct qib_devdata
*dd
)
1449 struct rvt_dev_info
*rdi
= &dd
->verbs_dev
.rdi
;
1451 memset(&rdi
->dparms
.props
, 0, sizeof(rdi
->dparms
.props
));
1453 rdi
->dparms
.props
.max_pd
= ib_qib_max_pds
;
1454 rdi
->dparms
.props
.max_ah
= ib_qib_max_ahs
;
1455 rdi
->dparms
.props
.device_cap_flags
= IB_DEVICE_BAD_PKEY_CNTR
|
1456 IB_DEVICE_BAD_QKEY_CNTR
| IB_DEVICE_SHUTDOWN_PORT
|
1457 IB_DEVICE_SYS_IMAGE_GUID
| IB_DEVICE_RC_RNR_NAK_GEN
|
1458 IB_DEVICE_PORT_ACTIVE_EVENT
| IB_DEVICE_SRQ_RESIZE
;
1459 rdi
->dparms
.props
.page_size_cap
= PAGE_SIZE
;
1460 rdi
->dparms
.props
.vendor_id
=
1461 QIB_SRC_OUI_1
<< 16 | QIB_SRC_OUI_2
<< 8 | QIB_SRC_OUI_3
;
1462 rdi
->dparms
.props
.vendor_part_id
= dd
->deviceid
;
1463 rdi
->dparms
.props
.hw_ver
= dd
->minrev
;
1464 rdi
->dparms
.props
.sys_image_guid
= ib_qib_sys_image_guid
;
1465 rdi
->dparms
.props
.max_mr_size
= ~0ULL;
1466 rdi
->dparms
.props
.max_qp
= ib_qib_max_qps
;
1467 rdi
->dparms
.props
.max_qp_wr
= ib_qib_max_qp_wrs
;
1468 rdi
->dparms
.props
.max_send_sge
= ib_qib_max_sges
;
1469 rdi
->dparms
.props
.max_recv_sge
= ib_qib_max_sges
;
1470 rdi
->dparms
.props
.max_sge_rd
= ib_qib_max_sges
;
1471 rdi
->dparms
.props
.max_cq
= ib_qib_max_cqs
;
1472 rdi
->dparms
.props
.max_cqe
= ib_qib_max_cqes
;
1473 rdi
->dparms
.props
.max_ah
= ib_qib_max_ahs
;
1474 rdi
->dparms
.props
.max_mr
= rdi
->lkey_table
.max
;
1475 rdi
->dparms
.props
.max_fmr
= rdi
->lkey_table
.max
;
1476 rdi
->dparms
.props
.max_map_per_fmr
= 32767;
1477 rdi
->dparms
.props
.max_qp_rd_atom
= QIB_MAX_RDMA_ATOMIC
;
1478 rdi
->dparms
.props
.max_qp_init_rd_atom
= 255;
1479 rdi
->dparms
.props
.max_srq
= ib_qib_max_srqs
;
1480 rdi
->dparms
.props
.max_srq_wr
= ib_qib_max_srq_wrs
;
1481 rdi
->dparms
.props
.max_srq_sge
= ib_qib_max_srq_sges
;
1482 rdi
->dparms
.props
.atomic_cap
= IB_ATOMIC_GLOB
;
1483 rdi
->dparms
.props
.max_pkeys
= qib_get_npkeys(dd
);
1484 rdi
->dparms
.props
.max_mcast_grp
= ib_qib_max_mcast_grps
;
1485 rdi
->dparms
.props
.max_mcast_qp_attach
= ib_qib_max_mcast_qp_attached
;
1486 rdi
->dparms
.props
.max_total_mcast_qp_attach
=
1487 rdi
->dparms
.props
.max_mcast_qp_attach
*
1488 rdi
->dparms
.props
.max_mcast_grp
;
1489 /* post send table */
1490 dd
->verbs_dev
.rdi
.post_parms
= qib_post_parms
;
1492 /* opcode translation table */
1493 dd
->verbs_dev
.rdi
.wc_opcode
= ib_qib_wc_opcode
;
1496 static const struct ib_device_ops qib_dev_ops
= {
1497 .modify_device
= qib_modify_device
,
1498 .process_mad
= qib_process_mad
,
1502 * qib_register_ib_device - register our device with the infiniband core
1503 * @dd: the device data structure
1504 * Return the allocated qib_ibdev pointer or NULL on error.
1506 int qib_register_ib_device(struct qib_devdata
*dd
)
1508 struct qib_ibdev
*dev
= &dd
->verbs_dev
;
1509 struct ib_device
*ibdev
= &dev
->rdi
.ibdev
;
1510 struct qib_pportdata
*ppd
= dd
->pport
;
1514 get_random_bytes(&dev
->qp_rnd
, sizeof(dev
->qp_rnd
));
1515 for (i
= 0; i
< dd
->num_pports
; i
++)
1516 init_ibport(ppd
+ i
);
1518 /* Only need to initialize non-zero fields. */
1519 timer_setup(&dev
->mem_timer
, mem_timer
, 0);
1521 INIT_LIST_HEAD(&dev
->piowait
);
1522 INIT_LIST_HEAD(&dev
->dmawait
);
1523 INIT_LIST_HEAD(&dev
->txwait
);
1524 INIT_LIST_HEAD(&dev
->memwait
);
1525 INIT_LIST_HEAD(&dev
->txreq_free
);
1527 if (ppd
->sdma_descq_cnt
) {
1528 dev
->pio_hdrs
= dma_alloc_coherent(&dd
->pcidev
->dev
,
1529 ppd
->sdma_descq_cnt
*
1530 sizeof(struct qib_pio_header
),
1531 &dev
->pio_hdrs_phys
,
1533 if (!dev
->pio_hdrs
) {
1539 for (i
= 0; i
< ppd
->sdma_descq_cnt
; i
++) {
1540 struct qib_verbs_txreq
*tx
;
1542 tx
= kzalloc(sizeof(*tx
), GFP_KERNEL
);
1548 list_add(&tx
->txreq
.list
, &dev
->txreq_free
);
1552 * The system image GUID is supposed to be the same for all
1553 * IB HCAs in a single system but since there can be other
1554 * device types in the system, we can't be sure this is unique.
1556 if (!ib_qib_sys_image_guid
)
1557 ib_qib_sys_image_guid
= ppd
->guid
;
1559 ibdev
->owner
= THIS_MODULE
;
1560 ibdev
->node_guid
= ppd
->guid
;
1561 ibdev
->phys_port_cnt
= dd
->num_pports
;
1562 ibdev
->dev
.parent
= &dd
->pcidev
->dev
;
1564 snprintf(ibdev
->node_desc
, sizeof(ibdev
->node_desc
),
1565 "Intel Infiniband HCA %s", init_utsname()->nodename
);
1568 * Fill in rvt info object.
1570 dd
->verbs_dev
.rdi
.driver_f
.port_callback
= qib_create_port_files
;
1571 dd
->verbs_dev
.rdi
.driver_f
.get_pci_dev
= qib_get_pci_dev
;
1572 dd
->verbs_dev
.rdi
.driver_f
.check_ah
= qib_check_ah
;
1573 dd
->verbs_dev
.rdi
.driver_f
.setup_wqe
= qib_check_send_wqe
;
1574 dd
->verbs_dev
.rdi
.driver_f
.notify_new_ah
= qib_notify_new_ah
;
1575 dd
->verbs_dev
.rdi
.driver_f
.alloc_qpn
= qib_alloc_qpn
;
1576 dd
->verbs_dev
.rdi
.driver_f
.qp_priv_alloc
= qib_qp_priv_alloc
;
1577 dd
->verbs_dev
.rdi
.driver_f
.qp_priv_free
= qib_qp_priv_free
;
1578 dd
->verbs_dev
.rdi
.driver_f
.free_all_qps
= qib_free_all_qps
;
1579 dd
->verbs_dev
.rdi
.driver_f
.notify_qp_reset
= qib_notify_qp_reset
;
1580 dd
->verbs_dev
.rdi
.driver_f
.do_send
= qib_do_send
;
1581 dd
->verbs_dev
.rdi
.driver_f
.schedule_send
= qib_schedule_send
;
1582 dd
->verbs_dev
.rdi
.driver_f
.quiesce_qp
= qib_quiesce_qp
;
1583 dd
->verbs_dev
.rdi
.driver_f
.stop_send_queue
= qib_stop_send_queue
;
1584 dd
->verbs_dev
.rdi
.driver_f
.flush_qp_waiters
= qib_flush_qp_waiters
;
1585 dd
->verbs_dev
.rdi
.driver_f
.notify_error_qp
= qib_notify_error_qp
;
1586 dd
->verbs_dev
.rdi
.driver_f
.notify_restart_rc
= qib_restart_rc
;
1587 dd
->verbs_dev
.rdi
.driver_f
.mtu_to_path_mtu
= qib_mtu_to_path_mtu
;
1588 dd
->verbs_dev
.rdi
.driver_f
.mtu_from_qp
= qib_mtu_from_qp
;
1589 dd
->verbs_dev
.rdi
.driver_f
.get_pmtu_from_attr
= qib_get_pmtu_from_attr
;
1590 dd
->verbs_dev
.rdi
.driver_f
.schedule_send_no_lock
= _qib_schedule_send
;
1591 dd
->verbs_dev
.rdi
.driver_f
.query_port_state
= qib_query_port
;
1592 dd
->verbs_dev
.rdi
.driver_f
.shut_down_port
= qib_shut_down_port
;
1593 dd
->verbs_dev
.rdi
.driver_f
.cap_mask_chg
= qib_cap_mask_chg
;
1594 dd
->verbs_dev
.rdi
.driver_f
.notify_create_mad_agent
=
1595 qib_notify_create_mad_agent
;
1596 dd
->verbs_dev
.rdi
.driver_f
.notify_free_mad_agent
=
1597 qib_notify_free_mad_agent
;
1599 dd
->verbs_dev
.rdi
.dparms
.max_rdma_atomic
= QIB_MAX_RDMA_ATOMIC
;
1600 dd
->verbs_dev
.rdi
.driver_f
.get_guid_be
= qib_get_guid_be
;
1601 dd
->verbs_dev
.rdi
.dparms
.lkey_table_size
= qib_lkey_table_size
;
1602 dd
->verbs_dev
.rdi
.dparms
.qp_table_size
= ib_qib_qp_table_size
;
1603 dd
->verbs_dev
.rdi
.dparms
.qpn_start
= 1;
1604 dd
->verbs_dev
.rdi
.dparms
.qpn_res_start
= QIB_KD_QP
;
1605 dd
->verbs_dev
.rdi
.dparms
.qpn_res_end
= QIB_KD_QP
; /* Reserve one QP */
1606 dd
->verbs_dev
.rdi
.dparms
.qpn_inc
= 1;
1607 dd
->verbs_dev
.rdi
.dparms
.qos_shift
= 1;
1608 dd
->verbs_dev
.rdi
.dparms
.psn_mask
= QIB_PSN_MASK
;
1609 dd
->verbs_dev
.rdi
.dparms
.psn_shift
= QIB_PSN_SHIFT
;
1610 dd
->verbs_dev
.rdi
.dparms
.psn_modify_mask
= QIB_PSN_MASK
;
1611 dd
->verbs_dev
.rdi
.dparms
.nports
= dd
->num_pports
;
1612 dd
->verbs_dev
.rdi
.dparms
.npkeys
= qib_get_npkeys(dd
);
1613 dd
->verbs_dev
.rdi
.dparms
.node
= dd
->assigned_node_id
;
1614 dd
->verbs_dev
.rdi
.dparms
.core_cap_flags
= RDMA_CORE_PORT_IBA_IB
;
1615 dd
->verbs_dev
.rdi
.dparms
.max_mad_size
= IB_MGMT_MAD_SIZE
;
1616 dd
->verbs_dev
.rdi
.dparms
.sge_copy_mode
= RVT_SGE_COPY_MEMCPY
;
1618 qib_fill_device_attr(dd
);
1621 for (i
= 0; i
< dd
->num_pports
; i
++, ppd
++) {
1622 ctxt
= ppd
->hw_pidx
;
1623 rvt_init_port(&dd
->verbs_dev
.rdi
,
1624 &ppd
->ibport_data
.rvp
,
1626 dd
->rcd
[ctxt
]->pkeys
);
1628 rdma_set_device_sysfs_group(&dd
->verbs_dev
.rdi
.ibdev
, &qib_attr_group
);
1630 ib_set_device_ops(ibdev
, &qib_dev_ops
);
1631 ret
= rvt_register_device(&dd
->verbs_dev
.rdi
, RDMA_DRIVER_QIB
);
1638 while (!list_empty(&dev
->txreq_free
)) {
1639 struct list_head
*l
= dev
->txreq_free
.next
;
1640 struct qib_verbs_txreq
*tx
;
1643 tx
= list_entry(l
, struct qib_verbs_txreq
, txreq
.list
);
1646 if (ppd
->sdma_descq_cnt
)
1647 dma_free_coherent(&dd
->pcidev
->dev
,
1648 ppd
->sdma_descq_cnt
*
1649 sizeof(struct qib_pio_header
),
1650 dev
->pio_hdrs
, dev
->pio_hdrs_phys
);
1652 qib_dev_err(dd
, "cannot register verbs: %d!\n", -ret
);
1656 void qib_unregister_ib_device(struct qib_devdata
*dd
)
1658 struct qib_ibdev
*dev
= &dd
->verbs_dev
;
1660 qib_verbs_unregister_sysfs(dd
);
1662 rvt_unregister_device(&dd
->verbs_dev
.rdi
);
1664 if (!list_empty(&dev
->piowait
))
1665 qib_dev_err(dd
, "piowait list not empty!\n");
1666 if (!list_empty(&dev
->dmawait
))
1667 qib_dev_err(dd
, "dmawait list not empty!\n");
1668 if (!list_empty(&dev
->txwait
))
1669 qib_dev_err(dd
, "txwait list not empty!\n");
1670 if (!list_empty(&dev
->memwait
))
1671 qib_dev_err(dd
, "memwait list not empty!\n");
1673 del_timer_sync(&dev
->mem_timer
);
1674 while (!list_empty(&dev
->txreq_free
)) {
1675 struct list_head
*l
= dev
->txreq_free
.next
;
1676 struct qib_verbs_txreq
*tx
;
1679 tx
= list_entry(l
, struct qib_verbs_txreq
, txreq
.list
);
1682 if (dd
->pport
->sdma_descq_cnt
)
1683 dma_free_coherent(&dd
->pcidev
->dev
,
1684 dd
->pport
->sdma_descq_cnt
*
1685 sizeof(struct qib_pio_header
),
1686 dev
->pio_hdrs
, dev
->pio_hdrs_phys
);
1690 * _qib_schedule_send - schedule progress
1693 * This schedules progress w/o regard to the s_flags.
1695 * It is only used in post send, which doesn't hold
1698 bool _qib_schedule_send(struct rvt_qp
*qp
)
1700 struct qib_ibport
*ibp
=
1701 to_iport(qp
->ibqp
.device
, qp
->port_num
);
1702 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
1703 struct qib_qp_priv
*priv
= qp
->priv
;
1705 return queue_work(ppd
->qib_wq
, &priv
->s_work
);
1709 * qib_schedule_send - schedule progress
1712 * This schedules qp progress. The s_lock
1715 bool qib_schedule_send(struct rvt_qp
*qp
)
1717 if (qib_send_ok(qp
))
1718 return _qib_schedule_send(qp
);