2 * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <rdma/ib_mad.h>
35 #include <rdma/ib_user_verbs.h>
37 #include <linux/utsname.h>
39 #include "ipath_kernel.h"
40 #include "ipath_verbs.h"
41 #include "ipath_common.h"
43 static unsigned int ib_ipath_qp_table_size
= 251;
44 module_param_named(qp_table_size
, ib_ipath_qp_table_size
, uint
, S_IRUGO
);
45 MODULE_PARM_DESC(qp_table_size
, "QP table size");
47 unsigned int ib_ipath_lkey_table_size
= 12;
48 module_param_named(lkey_table_size
, ib_ipath_lkey_table_size
, uint
,
50 MODULE_PARM_DESC(lkey_table_size
,
51 "LKEY table size in bits (2^n, 1 <= n <= 23)");
53 static unsigned int ib_ipath_max_pds
= 0xFFFF;
54 module_param_named(max_pds
, ib_ipath_max_pds
, uint
, S_IWUSR
| S_IRUGO
);
55 MODULE_PARM_DESC(max_pds
,
56 "Maximum number of protection domains to support");
58 static unsigned int ib_ipath_max_ahs
= 0xFFFF;
59 module_param_named(max_ahs
, ib_ipath_max_ahs
, uint
, S_IWUSR
| S_IRUGO
);
60 MODULE_PARM_DESC(max_ahs
, "Maximum number of address handles to support");
62 unsigned int ib_ipath_max_cqes
= 0x2FFFF;
63 module_param_named(max_cqes
, ib_ipath_max_cqes
, uint
, S_IWUSR
| S_IRUGO
);
64 MODULE_PARM_DESC(max_cqes
,
65 "Maximum number of completion queue entries to support");
67 unsigned int ib_ipath_max_cqs
= 0x1FFFF;
68 module_param_named(max_cqs
, ib_ipath_max_cqs
, uint
, S_IWUSR
| S_IRUGO
);
69 MODULE_PARM_DESC(max_cqs
, "Maximum number of completion queues to support");
71 unsigned int ib_ipath_max_qp_wrs
= 0x3FFF;
72 module_param_named(max_qp_wrs
, ib_ipath_max_qp_wrs
, uint
,
74 MODULE_PARM_DESC(max_qp_wrs
, "Maximum number of QP WRs to support");
76 unsigned int ib_ipath_max_qps
= 16384;
77 module_param_named(max_qps
, ib_ipath_max_qps
, uint
, S_IWUSR
| S_IRUGO
);
78 MODULE_PARM_DESC(max_qps
, "Maximum number of QPs to support");
80 unsigned int ib_ipath_max_sges
= 0x60;
81 module_param_named(max_sges
, ib_ipath_max_sges
, uint
, S_IWUSR
| S_IRUGO
);
82 MODULE_PARM_DESC(max_sges
, "Maximum number of SGEs to support");
84 unsigned int ib_ipath_max_mcast_grps
= 16384;
85 module_param_named(max_mcast_grps
, ib_ipath_max_mcast_grps
, uint
,
87 MODULE_PARM_DESC(max_mcast_grps
,
88 "Maximum number of multicast groups to support");
90 unsigned int ib_ipath_max_mcast_qp_attached
= 16;
91 module_param_named(max_mcast_qp_attached
, ib_ipath_max_mcast_qp_attached
,
92 uint
, S_IWUSR
| S_IRUGO
);
93 MODULE_PARM_DESC(max_mcast_qp_attached
,
94 "Maximum number of attached QPs to support");
96 unsigned int ib_ipath_max_srqs
= 1024;
97 module_param_named(max_srqs
, ib_ipath_max_srqs
, uint
, S_IWUSR
| S_IRUGO
);
98 MODULE_PARM_DESC(max_srqs
, "Maximum number of SRQs to support");
100 unsigned int ib_ipath_max_srq_sges
= 128;
101 module_param_named(max_srq_sges
, ib_ipath_max_srq_sges
,
102 uint
, S_IWUSR
| S_IRUGO
);
103 MODULE_PARM_DESC(max_srq_sges
, "Maximum number of SRQ SGEs to support");
105 unsigned int ib_ipath_max_srq_wrs
= 0x1FFFF;
106 module_param_named(max_srq_wrs
, ib_ipath_max_srq_wrs
,
107 uint
, S_IWUSR
| S_IRUGO
);
108 MODULE_PARM_DESC(max_srq_wrs
, "Maximum number of SRQ WRs support");
110 static unsigned int ib_ipath_disable_sma
;
111 module_param_named(disable_sma
, ib_ipath_disable_sma
, uint
, S_IWUSR
| S_IRUGO
);
112 MODULE_PARM_DESC(disable_sma
, "Disable the SMA");
115 * Note that it is OK to post send work requests in the SQE and ERR
116 * states; ipath_do_send() will process them and generate error
117 * completions as per IB 1.2 C10-96.
119 const int ib_ipath_state_ops
[IB_QPS_ERR
+ 1] = {
121 [IB_QPS_INIT
] = IPATH_POST_RECV_OK
,
122 [IB_QPS_RTR
] = IPATH_POST_RECV_OK
| IPATH_PROCESS_RECV_OK
,
123 [IB_QPS_RTS
] = IPATH_POST_RECV_OK
| IPATH_PROCESS_RECV_OK
|
124 IPATH_POST_SEND_OK
| IPATH_PROCESS_SEND_OK
|
125 IPATH_PROCESS_NEXT_SEND_OK
,
126 [IB_QPS_SQD
] = IPATH_POST_RECV_OK
| IPATH_PROCESS_RECV_OK
|
127 IPATH_POST_SEND_OK
| IPATH_PROCESS_SEND_OK
,
128 [IB_QPS_SQE
] = IPATH_POST_RECV_OK
| IPATH_PROCESS_RECV_OK
|
129 IPATH_POST_SEND_OK
| IPATH_FLUSH_SEND
,
130 [IB_QPS_ERR
] = IPATH_POST_RECV_OK
| IPATH_FLUSH_RECV
|
131 IPATH_POST_SEND_OK
| IPATH_FLUSH_SEND
,
134 struct ipath_ucontext
{
135 struct ib_ucontext ibucontext
;
138 static inline struct ipath_ucontext
*to_iucontext(struct ib_ucontext
141 return container_of(ibucontext
, struct ipath_ucontext
, ibucontext
);
145 * Translate ib_wr_opcode into ib_wc_opcode.
147 const enum ib_wc_opcode ib_ipath_wc_opcode
[] = {
148 [IB_WR_RDMA_WRITE
] = IB_WC_RDMA_WRITE
,
149 [IB_WR_RDMA_WRITE_WITH_IMM
] = IB_WC_RDMA_WRITE
,
150 [IB_WR_SEND
] = IB_WC_SEND
,
151 [IB_WR_SEND_WITH_IMM
] = IB_WC_SEND
,
152 [IB_WR_RDMA_READ
] = IB_WC_RDMA_READ
,
153 [IB_WR_ATOMIC_CMP_AND_SWP
] = IB_WC_COMP_SWAP
,
154 [IB_WR_ATOMIC_FETCH_AND_ADD
] = IB_WC_FETCH_ADD
160 static __be64 sys_image_guid
;
163 * ipath_copy_sge - copy data to SGE memory
165 * @data: the data to copy
166 * @length: the length of the data
168 void ipath_copy_sge(struct ipath_sge_state
*ss
, void *data
, u32 length
)
170 struct ipath_sge
*sge
= &ss
->sge
;
173 u32 len
= sge
->length
;
177 if (len
> sge
->sge_length
)
178 len
= sge
->sge_length
;
180 memcpy(sge
->vaddr
, data
, len
);
183 sge
->sge_length
-= len
;
184 if (sge
->sge_length
== 0) {
186 *sge
= *ss
->sg_list
++;
187 } else if (sge
->length
== 0 && sge
->mr
!= NULL
) {
188 if (++sge
->n
>= IPATH_SEGSZ
) {
189 if (++sge
->m
>= sge
->mr
->mapsz
)
194 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
196 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
204 * ipath_skip_sge - skip over SGE memory - XXX almost dup of prev func
206 * @length: the number of bytes to skip
208 void ipath_skip_sge(struct ipath_sge_state
*ss
, u32 length
)
210 struct ipath_sge
*sge
= &ss
->sge
;
213 u32 len
= sge
->length
;
217 if (len
> sge
->sge_length
)
218 len
= sge
->sge_length
;
222 sge
->sge_length
-= len
;
223 if (sge
->sge_length
== 0) {
225 *sge
= *ss
->sg_list
++;
226 } else if (sge
->length
== 0 && sge
->mr
!= NULL
) {
227 if (++sge
->n
>= IPATH_SEGSZ
) {
228 if (++sge
->m
>= sge
->mr
->mapsz
)
233 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
235 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
242 * Count the number of DMA descriptors needed to send length bytes of data.
243 * Don't modify the ipath_sge_state to get the count.
244 * Return zero if any of the segments is not aligned.
246 static u32
ipath_count_sge(struct ipath_sge_state
*ss
, u32 length
)
248 struct ipath_sge
*sg_list
= ss
->sg_list
;
249 struct ipath_sge sge
= ss
->sge
;
250 u8 num_sge
= ss
->num_sge
;
251 u32 ndesc
= 1; /* count the header */
254 u32 len
= sge
.length
;
258 if (len
> sge
.sge_length
)
259 len
= sge
.sge_length
;
261 if (((long) sge
.vaddr
& (sizeof(u32
) - 1)) ||
262 (len
!= length
&& (len
& (sizeof(u32
) - 1)))) {
269 sge
.sge_length
-= len
;
270 if (sge
.sge_length
== 0) {
273 } else if (sge
.length
== 0 && sge
.mr
!= NULL
) {
274 if (++sge
.n
>= IPATH_SEGSZ
) {
275 if (++sge
.m
>= sge
.mr
->mapsz
)
280 sge
.mr
->map
[sge
.m
]->segs
[sge
.n
].vaddr
;
282 sge
.mr
->map
[sge
.m
]->segs
[sge
.n
].length
;
290 * Copy from the SGEs to the data buffer.
292 static void ipath_copy_from_sge(void *data
, struct ipath_sge_state
*ss
,
295 struct ipath_sge
*sge
= &ss
->sge
;
298 u32 len
= sge
->length
;
302 if (len
> sge
->sge_length
)
303 len
= sge
->sge_length
;
305 memcpy(data
, sge
->vaddr
, len
);
308 sge
->sge_length
-= len
;
309 if (sge
->sge_length
== 0) {
311 *sge
= *ss
->sg_list
++;
312 } else if (sge
->length
== 0 && sge
->mr
!= NULL
) {
313 if (++sge
->n
>= IPATH_SEGSZ
) {
314 if (++sge
->m
>= sge
->mr
->mapsz
)
319 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
321 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
329 * ipath_post_one_send - post one RC, UC, or UD send work request
330 * @qp: the QP to post on
331 * @wr: the work request to send
333 static int ipath_post_one_send(struct ipath_qp
*qp
, struct ib_send_wr
*wr
)
335 struct ipath_swqe
*wqe
;
343 spin_lock_irqsave(&qp
->s_lock
, flags
);
345 /* Check that state is OK to post send. */
346 if (unlikely(!(ib_ipath_state_ops
[qp
->state
] & IPATH_POST_SEND_OK
)))
349 /* IB spec says that num_sge == 0 is OK. */
350 if (wr
->num_sge
> qp
->s_max_sge
)
354 * Don't allow RDMA reads or atomic operations on UC or
355 * undefined operations.
356 * Make sure buffer is large enough to hold the result for atomics.
358 if (qp
->ibqp
.qp_type
== IB_QPT_UC
) {
359 if ((unsigned) wr
->opcode
>= IB_WR_RDMA_READ
)
361 } else if (qp
->ibqp
.qp_type
== IB_QPT_UD
) {
362 /* Check UD opcode */
363 if (wr
->opcode
!= IB_WR_SEND
&&
364 wr
->opcode
!= IB_WR_SEND_WITH_IMM
)
366 /* Check UD destination address PD */
367 if (qp
->ibqp
.pd
!= wr
->wr
.ud
.ah
->pd
)
369 } else if ((unsigned) wr
->opcode
> IB_WR_ATOMIC_FETCH_AND_ADD
)
371 else if (wr
->opcode
>= IB_WR_ATOMIC_CMP_AND_SWP
&&
373 wr
->sg_list
[0].length
< sizeof(u64
) ||
374 wr
->sg_list
[0].addr
& (sizeof(u64
) - 1)))
376 else if (wr
->opcode
>= IB_WR_RDMA_READ
&& !qp
->s_max_rd_atomic
)
379 next
= qp
->s_head
+ 1;
380 if (next
>= qp
->s_size
)
382 if (next
== qp
->s_last
) {
387 wqe
= get_swqe_ptr(qp
, qp
->s_head
);
391 acc
= wr
->opcode
>= IB_WR_RDMA_READ
?
392 IB_ACCESS_LOCAL_WRITE
: 0;
393 for (i
= 0, j
= 0; i
< wr
->num_sge
; i
++) {
394 u32 length
= wr
->sg_list
[i
].length
;
399 ok
= ipath_lkey_ok(qp
, &wqe
->sg_list
[j
],
400 &wr
->sg_list
[i
], acc
);
403 wqe
->length
+= length
;
408 if (qp
->ibqp
.qp_type
== IB_QPT_UC
||
409 qp
->ibqp
.qp_type
== IB_QPT_RC
) {
410 if (wqe
->length
> 0x80000000U
)
412 } else if (wqe
->length
> to_idev(qp
->ibqp
.device
)->dd
->ipath_ibmtu
)
414 wqe
->ssn
= qp
->s_ssn
++;
423 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
428 * ipath_post_send - post a send on a QP
429 * @ibqp: the QP to post the send on
430 * @wr: the list of work requests to post
431 * @bad_wr: the first bad WR is put here
433 * This may be called from interrupt context.
435 static int ipath_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
436 struct ib_send_wr
**bad_wr
)
438 struct ipath_qp
*qp
= to_iqp(ibqp
);
441 for (; wr
; wr
= wr
->next
) {
442 err
= ipath_post_one_send(qp
, wr
);
449 /* Try to do the send work in the caller's context. */
450 ipath_do_send((unsigned long) qp
);
457 * ipath_post_receive - post a receive on a QP
458 * @ibqp: the QP to post the receive on
459 * @wr: the WR to post
460 * @bad_wr: the first bad WR is put here
462 * This may be called from interrupt context.
464 static int ipath_post_receive(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
465 struct ib_recv_wr
**bad_wr
)
467 struct ipath_qp
*qp
= to_iqp(ibqp
);
468 struct ipath_rwq
*wq
= qp
->r_rq
.wq
;
472 /* Check that state is OK to post receive. */
473 if (!(ib_ipath_state_ops
[qp
->state
] & IPATH_POST_RECV_OK
) || !wq
) {
479 for (; wr
; wr
= wr
->next
) {
480 struct ipath_rwqe
*wqe
;
484 if ((unsigned) wr
->num_sge
> qp
->r_rq
.max_sge
) {
490 spin_lock_irqsave(&qp
->r_rq
.lock
, flags
);
492 if (next
>= qp
->r_rq
.size
)
494 if (next
== wq
->tail
) {
495 spin_unlock_irqrestore(&qp
->r_rq
.lock
, flags
);
501 wqe
= get_rwqe_ptr(&qp
->r_rq
, wq
->head
);
502 wqe
->wr_id
= wr
->wr_id
;
503 wqe
->num_sge
= wr
->num_sge
;
504 for (i
= 0; i
< wr
->num_sge
; i
++)
505 wqe
->sg_list
[i
] = wr
->sg_list
[i
];
506 /* Make sure queue entry is written before the head index. */
509 spin_unlock_irqrestore(&qp
->r_rq
.lock
, flags
);
518 * ipath_qp_rcv - processing an incoming packet on a QP
519 * @dev: the device the packet came on
520 * @hdr: the packet header
521 * @has_grh: true if the packet has a GRH
522 * @data: the packet data
523 * @tlen: the packet length
524 * @qp: the QP the packet came on
526 * This is called from ipath_ib_rcv() to process an incoming packet
528 * Called at interrupt level.
530 static void ipath_qp_rcv(struct ipath_ibdev
*dev
,
531 struct ipath_ib_header
*hdr
, int has_grh
,
532 void *data
, u32 tlen
, struct ipath_qp
*qp
)
534 /* Check for valid receive state. */
535 if (!(ib_ipath_state_ops
[qp
->state
] & IPATH_PROCESS_RECV_OK
)) {
540 switch (qp
->ibqp
.qp_type
) {
543 if (ib_ipath_disable_sma
)
547 ipath_ud_rcv(dev
, hdr
, has_grh
, data
, tlen
, qp
);
551 ipath_rc_rcv(dev
, hdr
, has_grh
, data
, tlen
, qp
);
555 ipath_uc_rcv(dev
, hdr
, has_grh
, data
, tlen
, qp
);
564 * ipath_ib_rcv - process an incoming packet
565 * @arg: the device pointer
566 * @rhdr: the header of the packet
567 * @data: the packet data
568 * @tlen: the packet length
570 * This is called from ipath_kreceive() to process an incoming packet at
571 * interrupt level. Tlen is the length of the header + data + CRC in bytes.
573 void ipath_ib_rcv(struct ipath_ibdev
*dev
, void *rhdr
, void *data
,
576 struct ipath_ib_header
*hdr
= rhdr
;
577 struct ipath_other_headers
*ohdr
;
584 if (unlikely(dev
== NULL
))
587 if (unlikely(tlen
< 24)) { /* LRH+BTH+CRC */
592 /* Check for a valid destination LID (see ch. 7.11.1). */
593 lid
= be16_to_cpu(hdr
->lrh
[1]);
594 if (lid
< IPATH_MULTICAST_LID_BASE
) {
595 lid
&= ~((1 << dev
->dd
->ipath_lmc
) - 1);
596 if (unlikely(lid
!= dev
->dd
->ipath_lid
)) {
603 lnh
= be16_to_cpu(hdr
->lrh
[0]) & 3;
604 if (lnh
== IPATH_LRH_BTH
)
606 else if (lnh
== IPATH_LRH_GRH
)
607 ohdr
= &hdr
->u
.l
.oth
;
613 opcode
= be32_to_cpu(ohdr
->bth
[0]) >> 24;
614 dev
->opstats
[opcode
].n_bytes
+= tlen
;
615 dev
->opstats
[opcode
].n_packets
++;
617 /* Get the destination QP number. */
618 qp_num
= be32_to_cpu(ohdr
->bth
[1]) & IPATH_QPN_MASK
;
619 if (qp_num
== IPATH_MULTICAST_QPN
) {
620 struct ipath_mcast
*mcast
;
621 struct ipath_mcast_qp
*p
;
623 if (lnh
!= IPATH_LRH_GRH
) {
627 mcast
= ipath_mcast_find(&hdr
->u
.l
.grh
.dgid
);
632 dev
->n_multicast_rcv
++;
633 list_for_each_entry_rcu(p
, &mcast
->qp_list
, list
)
634 ipath_qp_rcv(dev
, hdr
, 1, data
, tlen
, p
->qp
);
636 * Notify ipath_multicast_detach() if it is waiting for us
639 if (atomic_dec_return(&mcast
->refcount
) <= 1)
640 wake_up(&mcast
->wait
);
642 qp
= ipath_lookup_qpn(&dev
->qp_table
, qp_num
);
644 dev
->n_unicast_rcv
++;
645 ipath_qp_rcv(dev
, hdr
, lnh
== IPATH_LRH_GRH
, data
,
648 * Notify ipath_destroy_qp() if it is waiting
651 if (atomic_dec_and_test(&qp
->refcount
))
661 * ipath_ib_timer - verbs timer
662 * @arg: the device pointer
664 * This is called from ipath_do_rcv_timer() at interrupt level to check for
665 * QPs which need retransmits and to collect performance numbers.
667 static void ipath_ib_timer(struct ipath_ibdev
*dev
)
669 struct ipath_qp
*resend
= NULL
;
670 struct ipath_qp
*rnr
= NULL
;
671 struct list_head
*last
;
678 spin_lock_irqsave(&dev
->pending_lock
, flags
);
679 /* Start filling the next pending queue. */
680 if (++dev
->pending_index
>= ARRAY_SIZE(dev
->pending
))
681 dev
->pending_index
= 0;
682 /* Save any requests still in the new queue, they have timed out. */
683 last
= &dev
->pending
[dev
->pending_index
];
684 while (!list_empty(last
)) {
685 qp
= list_entry(last
->next
, struct ipath_qp
, timerwait
);
686 list_del_init(&qp
->timerwait
);
687 qp
->timer_next
= resend
;
689 atomic_inc(&qp
->refcount
);
691 last
= &dev
->rnrwait
;
692 if (!list_empty(last
)) {
693 qp
= list_entry(last
->next
, struct ipath_qp
, timerwait
);
694 if (--qp
->s_rnr_timeout
== 0) {
696 list_del_init(&qp
->timerwait
);
697 qp
->timer_next
= rnr
;
699 atomic_inc(&qp
->refcount
);
700 if (list_empty(last
))
702 qp
= list_entry(last
->next
, struct ipath_qp
,
704 } while (qp
->s_rnr_timeout
== 0);
708 * We should only be in the started state if pma_sample_start != 0
710 if (dev
->pma_sample_status
== IB_PMA_SAMPLE_STATUS_STARTED
&&
711 --dev
->pma_sample_start
== 0) {
712 dev
->pma_sample_status
= IB_PMA_SAMPLE_STATUS_RUNNING
;
713 ipath_snapshot_counters(dev
->dd
, &dev
->ipath_sword
,
717 &dev
->ipath_xmit_wait
);
719 if (dev
->pma_sample_status
== IB_PMA_SAMPLE_STATUS_RUNNING
) {
720 if (dev
->pma_sample_interval
== 0) {
721 u64 ta
, tb
, tc
, td
, te
;
723 dev
->pma_sample_status
= IB_PMA_SAMPLE_STATUS_DONE
;
724 ipath_snapshot_counters(dev
->dd
, &ta
, &tb
,
727 dev
->ipath_sword
= ta
- dev
->ipath_sword
;
728 dev
->ipath_rword
= tb
- dev
->ipath_rword
;
729 dev
->ipath_spkts
= tc
- dev
->ipath_spkts
;
730 dev
->ipath_rpkts
= td
- dev
->ipath_rpkts
;
731 dev
->ipath_xmit_wait
= te
- dev
->ipath_xmit_wait
;
734 dev
->pma_sample_interval
--;
736 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
738 /* XXX What if timer fires again while this is running? */
739 while (resend
!= NULL
) {
741 resend
= qp
->timer_next
;
743 spin_lock_irqsave(&qp
->s_lock
, flags
);
744 if (qp
->s_last
!= qp
->s_tail
&&
745 ib_ipath_state_ops
[qp
->state
] & IPATH_PROCESS_SEND_OK
) {
747 ipath_restart_rc(qp
, qp
->s_last_psn
+ 1);
749 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
751 /* Notify ipath_destroy_qp() if it is waiting. */
752 if (atomic_dec_and_test(&qp
->refcount
))
755 while (rnr
!= NULL
) {
757 rnr
= qp
->timer_next
;
759 spin_lock_irqsave(&qp
->s_lock
, flags
);
760 if (ib_ipath_state_ops
[qp
->state
] & IPATH_PROCESS_SEND_OK
)
761 ipath_schedule_send(qp
);
762 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
764 /* Notify ipath_destroy_qp() if it is waiting. */
765 if (atomic_dec_and_test(&qp
->refcount
))
770 static void update_sge(struct ipath_sge_state
*ss
, u32 length
)
772 struct ipath_sge
*sge
= &ss
->sge
;
774 sge
->vaddr
+= length
;
775 sge
->length
-= length
;
776 sge
->sge_length
-= length
;
777 if (sge
->sge_length
== 0) {
779 *sge
= *ss
->sg_list
++;
780 } else if (sge
->length
== 0 && sge
->mr
!= NULL
) {
781 if (++sge
->n
>= IPATH_SEGSZ
) {
782 if (++sge
->m
>= sge
->mr
->mapsz
)
786 sge
->vaddr
= sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
787 sge
->length
= sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
791 #ifdef __LITTLE_ENDIAN
792 static inline u32
get_upper_bits(u32 data
, u32 shift
)
794 return data
>> shift
;
797 static inline u32
set_upper_bits(u32 data
, u32 shift
)
799 return data
<< shift
;
802 static inline u32
clear_upper_bytes(u32 data
, u32 n
, u32 off
)
804 data
<<= ((sizeof(u32
) - n
) * BITS_PER_BYTE
);
805 data
>>= ((sizeof(u32
) - n
- off
) * BITS_PER_BYTE
);
809 static inline u32
get_upper_bits(u32 data
, u32 shift
)
811 return data
<< shift
;
814 static inline u32
set_upper_bits(u32 data
, u32 shift
)
816 return data
>> shift
;
819 static inline u32
clear_upper_bytes(u32 data
, u32 n
, u32 off
)
821 data
>>= ((sizeof(u32
) - n
) * BITS_PER_BYTE
);
822 data
<<= ((sizeof(u32
) - n
- off
) * BITS_PER_BYTE
);
827 static void copy_io(u32 __iomem
*piobuf
, struct ipath_sge_state
*ss
,
828 u32 length
, unsigned flush_wc
)
835 u32 len
= ss
->sge
.length
;
840 if (len
> ss
->sge
.sge_length
)
841 len
= ss
->sge
.sge_length
;
843 /* If the source address is not aligned, try to align it. */
844 off
= (unsigned long)ss
->sge
.vaddr
& (sizeof(u32
) - 1);
846 u32
*addr
= (u32
*)((unsigned long)ss
->sge
.vaddr
&
848 u32 v
= get_upper_bits(*addr
, off
* BITS_PER_BYTE
);
851 y
= sizeof(u32
) - off
;
854 if (len
+ extra
>= sizeof(u32
)) {
855 data
|= set_upper_bits(v
, extra
*
857 len
= sizeof(u32
) - extra
;
862 __raw_writel(data
, piobuf
);
867 /* Clear unused upper bytes */
868 data
|= clear_upper_bytes(v
, len
, extra
);
876 /* Source address is aligned. */
877 u32
*addr
= (u32
*) ss
->sge
.vaddr
;
878 int shift
= extra
* BITS_PER_BYTE
;
879 int ushift
= 32 - shift
;
882 while (l
>= sizeof(u32
)) {
885 data
|= set_upper_bits(v
, shift
);
886 __raw_writel(data
, piobuf
);
887 data
= get_upper_bits(v
, ushift
);
893 * We still have 'extra' number of bytes leftover.
898 if (l
+ extra
>= sizeof(u32
)) {
899 data
|= set_upper_bits(v
, shift
);
900 len
-= l
+ extra
- sizeof(u32
);
905 __raw_writel(data
, piobuf
);
910 /* Clear unused upper bytes */
911 data
|= clear_upper_bytes(v
, l
,
919 } else if (len
== length
) {
923 } else if (len
== length
) {
927 * Need to round up for the last dword in the
931 __iowrite32_copy(piobuf
, ss
->sge
.vaddr
, w
- 1);
933 last
= ((u32
*) ss
->sge
.vaddr
)[w
- 1];
938 __iowrite32_copy(piobuf
, ss
->sge
.vaddr
, w
);
941 extra
= len
& (sizeof(u32
) - 1);
943 u32 v
= ((u32
*) ss
->sge
.vaddr
)[w
];
945 /* Clear unused upper bytes */
946 data
= clear_upper_bytes(v
, extra
, 0);
952 /* Update address before sending packet. */
953 update_sge(ss
, length
);
955 /* must flush early everything before trigger word */
957 __raw_writel(last
, piobuf
);
958 /* be sure trigger word is written */
961 __raw_writel(last
, piobuf
);
965 * Convert IB rate to delay multiplier.
967 unsigned ipath_ib_rate_to_mult(enum ib_rate rate
)
970 case IB_RATE_2_5_GBPS
: return 8;
971 case IB_RATE_5_GBPS
: return 4;
972 case IB_RATE_10_GBPS
: return 2;
973 case IB_RATE_20_GBPS
: return 1;
979 * Convert delay multiplier to IB rate
981 static enum ib_rate
ipath_mult_to_ib_rate(unsigned mult
)
984 case 8: return IB_RATE_2_5_GBPS
;
985 case 4: return IB_RATE_5_GBPS
;
986 case 2: return IB_RATE_10_GBPS
;
987 case 1: return IB_RATE_20_GBPS
;
988 default: return IB_RATE_PORT_CURRENT
;
992 static inline struct ipath_verbs_txreq
*get_txreq(struct ipath_ibdev
*dev
)
994 struct ipath_verbs_txreq
*tx
= NULL
;
997 spin_lock_irqsave(&dev
->pending_lock
, flags
);
998 if (!list_empty(&dev
->txreq_free
)) {
999 struct list_head
*l
= dev
->txreq_free
.next
;
1002 tx
= list_entry(l
, struct ipath_verbs_txreq
, txreq
.list
);
1004 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
1008 static inline void put_txreq(struct ipath_ibdev
*dev
,
1009 struct ipath_verbs_txreq
*tx
)
1011 unsigned long flags
;
1013 spin_lock_irqsave(&dev
->pending_lock
, flags
);
1014 list_add(&tx
->txreq
.list
, &dev
->txreq_free
);
1015 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
1018 static void sdma_complete(void *cookie
, int status
)
1020 struct ipath_verbs_txreq
*tx
= cookie
;
1021 struct ipath_qp
*qp
= tx
->qp
;
1022 struct ipath_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
1024 enum ib_wc_status ibs
= status
== IPATH_SDMA_TXREQ_S_OK
?
1025 IB_WC_SUCCESS
: IB_WC_WR_FLUSH_ERR
;
1027 if (atomic_dec_and_test(&qp
->s_dma_busy
)) {
1028 spin_lock_irqsave(&qp
->s_lock
, flags
);
1030 ipath_send_complete(qp
, tx
->wqe
, ibs
);
1031 if ((ib_ipath_state_ops
[qp
->state
] & IPATH_FLUSH_SEND
&&
1032 qp
->s_last
!= qp
->s_head
) ||
1033 (qp
->s_flags
& IPATH_S_WAIT_DMA
))
1034 ipath_schedule_send(qp
);
1035 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1036 wake_up(&qp
->wait_dma
);
1037 } else if (tx
->wqe
) {
1038 spin_lock_irqsave(&qp
->s_lock
, flags
);
1039 ipath_send_complete(qp
, tx
->wqe
, ibs
);
1040 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1043 if (tx
->txreq
.flags
& IPATH_SDMA_TXREQ_F_FREEBUF
)
1044 kfree(tx
->txreq
.map_addr
);
1047 if (atomic_dec_and_test(&qp
->refcount
))
1051 static void decrement_dma_busy(struct ipath_qp
*qp
)
1055 if (atomic_dec_and_test(&qp
->s_dma_busy
)) {
1056 spin_lock_irqsave(&qp
->s_lock
, flags
);
1057 if ((ib_ipath_state_ops
[qp
->state
] & IPATH_FLUSH_SEND
&&
1058 qp
->s_last
!= qp
->s_head
) ||
1059 (qp
->s_flags
& IPATH_S_WAIT_DMA
))
1060 ipath_schedule_send(qp
);
1061 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1062 wake_up(&qp
->wait_dma
);
1067 * Compute the number of clock cycles of delay before sending the next packet.
1068 * The multipliers reflect the number of clocks for the fastest rate so
1069 * one tick at 4xDDR is 8 ticks at 1xSDR.
1070 * If the destination port will take longer to receive a packet than
1071 * the outgoing link can send it, we need to delay sending the next packet
1072 * by the difference in time it takes the receiver to receive and the sender
1073 * to send this packet.
1074 * Note that this delay is always correct for UC and RC but not always
1075 * optimal for UD. For UD, the destination HCA can be different for each
1076 * packet, in which case, we could send packets to a different destination
1077 * while "waiting" for the delay. The overhead for doing this without
1078 * HW support is more than just paying the cost of delaying some packets
1081 static inline unsigned ipath_pkt_delay(u32 plen
, u8 snd_mult
, u8 rcv_mult
)
1083 return (rcv_mult
> snd_mult
) ?
1084 (plen
* (rcv_mult
- snd_mult
) + 1) >> 1 : 0;
1087 static int ipath_verbs_send_dma(struct ipath_qp
*qp
,
1088 struct ipath_ib_header
*hdr
, u32 hdrwords
,
1089 struct ipath_sge_state
*ss
, u32 len
,
1090 u32 plen
, u32 dwords
)
1092 struct ipath_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
1093 struct ipath_devdata
*dd
= dev
->dd
;
1094 struct ipath_verbs_txreq
*tx
;
1103 /* resend previously constructed packet */
1104 atomic_inc(&qp
->s_dma_busy
);
1105 ret
= ipath_sdma_verbs_send(dd
, tx
->ss
, tx
->len
, tx
);
1108 decrement_dma_busy(qp
);
1113 tx
= get_txreq(dev
);
1120 * Get the saved delay count we computed for the previous packet
1121 * and save the delay count for this packet to be used next time
1124 control
= qp
->s_pkt_delay
;
1125 qp
->s_pkt_delay
= ipath_pkt_delay(plen
, dd
->delay_mult
, qp
->s_dmult
);
1128 atomic_inc(&qp
->refcount
);
1129 tx
->wqe
= qp
->s_wqe
;
1130 tx
->txreq
.callback
= sdma_complete
;
1131 tx
->txreq
.callback_cookie
= tx
;
1132 tx
->txreq
.flags
= IPATH_SDMA_TXREQ_F_HEADTOHOST
|
1133 IPATH_SDMA_TXREQ_F_INTREQ
| IPATH_SDMA_TXREQ_F_FREEDESC
;
1134 if (plen
+ 1 >= IPATH_SMALLBUF_DWORDS
)
1135 tx
->txreq
.flags
|= IPATH_SDMA_TXREQ_F_USELARGEBUF
;
1137 /* VL15 packets bypass credit check */
1138 if ((be16_to_cpu(hdr
->lrh
[0]) >> 12) == 15) {
1139 control
|= 1ULL << 31;
1140 tx
->txreq
.flags
|= IPATH_SDMA_TXREQ_F_VL15
;
1145 * Don't try to DMA if it takes more descriptors than
1148 ndesc
= ipath_count_sge(ss
, len
);
1149 if (ndesc
>= dd
->ipath_sdma_descq_cnt
)
1154 tx
->hdr
.pbc
[0] = cpu_to_le32(plen
);
1155 tx
->hdr
.pbc
[1] = cpu_to_le32(control
);
1156 memcpy(&tx
->hdr
.hdr
, hdr
, hdrwords
<< 2);
1157 tx
->txreq
.sg_count
= ndesc
;
1158 tx
->map_len
= (hdrwords
+ 2) << 2;
1159 tx
->txreq
.map_addr
= &tx
->hdr
;
1160 atomic_inc(&qp
->s_dma_busy
);
1161 ret
= ipath_sdma_verbs_send(dd
, ss
, dwords
, tx
);
1163 /* save ss and length in dwords */
1167 decrement_dma_busy(qp
);
1172 /* Allocate a buffer and copy the header and payload to it. */
1173 tx
->map_len
= (plen
+ 1) << 2;
1174 piobuf
= kmalloc(tx
->map_len
, GFP_ATOMIC
);
1175 if (unlikely(piobuf
== NULL
)) {
1179 tx
->txreq
.map_addr
= piobuf
;
1180 tx
->txreq
.flags
|= IPATH_SDMA_TXREQ_F_FREEBUF
;
1181 tx
->txreq
.sg_count
= 1;
1183 *piobuf
++ = (__force u32
) cpu_to_le32(plen
);
1184 *piobuf
++ = (__force u32
) cpu_to_le32(control
);
1185 memcpy(piobuf
, hdr
, hdrwords
<< 2);
1186 ipath_copy_from_sge(piobuf
+ hdrwords
, ss
, len
);
1188 atomic_inc(&qp
->s_dma_busy
);
1189 ret
= ipath_sdma_verbs_send(dd
, NULL
, 0, tx
);
1191 * If we couldn't queue the DMA request, save the info
1192 * and try again later rather than destroying the
1193 * buffer and undoing the side effects of the copy.
1199 decrement_dma_busy(qp
);
1205 if (atomic_dec_and_test(&qp
->refcount
))
1212 static int ipath_verbs_send_pio(struct ipath_qp
*qp
,
1213 struct ipath_ib_header
*ibhdr
, u32 hdrwords
,
1214 struct ipath_sge_state
*ss
, u32 len
,
1215 u32 plen
, u32 dwords
)
1217 struct ipath_devdata
*dd
= to_idev(qp
->ibqp
.device
)->dd
;
1218 u32
*hdr
= (u32
*) ibhdr
;
1219 u32 __iomem
*piobuf
;
1225 piobuf
= ipath_getpiobuf(dd
, plen
, NULL
);
1226 if (unlikely(piobuf
== NULL
)) {
1232 * Get the saved delay count we computed for the previous packet
1233 * and save the delay count for this packet to be used next time
1236 control
= qp
->s_pkt_delay
;
1237 qp
->s_pkt_delay
= ipath_pkt_delay(plen
, dd
->delay_mult
, qp
->s_dmult
);
1239 /* VL15 packets bypass credit check */
1240 if ((be16_to_cpu(ibhdr
->lrh
[0]) >> 12) == 15)
1241 control
|= 1ULL << 31;
1244 * Write the length to the control qword plus any needed flags.
1245 * We have to flush after the PBC for correctness on some cpus
1246 * or WC buffer can be written out of order.
1248 writeq(((u64
) control
<< 32) | plen
, piobuf
);
1251 flush_wc
= dd
->ipath_flags
& IPATH_PIO_FLUSH_WC
;
1254 * If there is just the header portion, must flush before
1255 * writing last word of header for correctness, and after
1256 * the last header word (trigger word).
1260 __iowrite32_copy(piobuf
, hdr
, hdrwords
- 1);
1262 __raw_writel(hdr
[hdrwords
- 1], piobuf
+ hdrwords
- 1);
1265 __iowrite32_copy(piobuf
, hdr
, hdrwords
);
1271 __iowrite32_copy(piobuf
, hdr
, hdrwords
);
1274 /* The common case is aligned and contained in one segment. */
1275 if (likely(ss
->num_sge
== 1 && len
<= ss
->sge
.length
&&
1276 !((unsigned long)ss
->sge
.vaddr
& (sizeof(u32
) - 1)))) {
1277 u32
*addr
= (u32
*) ss
->sge
.vaddr
;
1279 /* Update address before sending packet. */
1280 update_sge(ss
, len
);
1282 __iowrite32_copy(piobuf
, addr
, dwords
- 1);
1283 /* must flush early everything before trigger word */
1285 __raw_writel(addr
[dwords
- 1], piobuf
+ dwords
- 1);
1286 /* be sure trigger word is written */
1289 __iowrite32_copy(piobuf
, addr
, dwords
);
1292 copy_io(piobuf
, ss
, len
, flush_wc
);
1295 spin_lock_irqsave(&qp
->s_lock
, flags
);
1296 ipath_send_complete(qp
, qp
->s_wqe
, IB_WC_SUCCESS
);
1297 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1305 * ipath_verbs_send - send a packet
1306 * @qp: the QP to send on
1307 * @hdr: the packet header
1308 * @hdrwords: the number of 32-bit words in the header
1309 * @ss: the SGE to send
1310 * @len: the length of the packet in bytes
1312 int ipath_verbs_send(struct ipath_qp
*qp
, struct ipath_ib_header
*hdr
,
1313 u32 hdrwords
, struct ipath_sge_state
*ss
, u32 len
)
1315 struct ipath_devdata
*dd
= to_idev(qp
->ibqp
.device
)->dd
;
1318 u32 dwords
= (len
+ 3) >> 2;
1321 * Calculate the send buffer trigger address.
1322 * The +1 counts for the pbc control dword following the pbc length.
1324 plen
= hdrwords
+ dwords
+ 1;
1327 * VL15 packets (IB_QPT_SMI) will always use PIO, so we
1328 * can defer SDMA restart until link goes ACTIVE without
1329 * worrying about just how we got there.
1331 if (qp
->ibqp
.qp_type
== IB_QPT_SMI
||
1332 !(dd
->ipath_flags
& IPATH_HAS_SEND_DMA
))
1333 ret
= ipath_verbs_send_pio(qp
, hdr
, hdrwords
, ss
, len
,
1336 ret
= ipath_verbs_send_dma(qp
, hdr
, hdrwords
, ss
, len
,
1342 int ipath_snapshot_counters(struct ipath_devdata
*dd
, u64
*swords
,
1343 u64
*rwords
, u64
*spkts
, u64
*rpkts
,
1348 if (!(dd
->ipath_flags
& IPATH_INITTED
)) {
1349 /* no hardware, freeze, etc. */
1353 *swords
= ipath_snap_cntr(dd
, dd
->ipath_cregs
->cr_wordsendcnt
);
1354 *rwords
= ipath_snap_cntr(dd
, dd
->ipath_cregs
->cr_wordrcvcnt
);
1355 *spkts
= ipath_snap_cntr(dd
, dd
->ipath_cregs
->cr_pktsendcnt
);
1356 *rpkts
= ipath_snap_cntr(dd
, dd
->ipath_cregs
->cr_pktrcvcnt
);
1357 *xmit_wait
= ipath_snap_cntr(dd
, dd
->ipath_cregs
->cr_sendstallcnt
);
1366 * ipath_get_counters - get various chip counters
1367 * @dd: the infinipath device
1368 * @cntrs: counters are placed here
1370 * Return the counters needed by recv_pma_get_portcounters().
1372 int ipath_get_counters(struct ipath_devdata
*dd
,
1373 struct ipath_verbs_counters
*cntrs
)
1375 struct ipath_cregs
const *crp
= dd
->ipath_cregs
;
1378 if (!(dd
->ipath_flags
& IPATH_INITTED
)) {
1379 /* no hardware, freeze, etc. */
1383 cntrs
->symbol_error_counter
=
1384 ipath_snap_cntr(dd
, crp
->cr_ibsymbolerrcnt
);
1385 cntrs
->link_error_recovery_counter
=
1386 ipath_snap_cntr(dd
, crp
->cr_iblinkerrrecovcnt
);
1388 * The link downed counter counts when the other side downs the
1389 * connection. We add in the number of times we downed the link
1390 * due to local link integrity errors to compensate.
1392 cntrs
->link_downed_counter
=
1393 ipath_snap_cntr(dd
, crp
->cr_iblinkdowncnt
);
1394 cntrs
->port_rcv_errors
=
1395 ipath_snap_cntr(dd
, crp
->cr_rxdroppktcnt
) +
1396 ipath_snap_cntr(dd
, crp
->cr_rcvovflcnt
) +
1397 ipath_snap_cntr(dd
, crp
->cr_portovflcnt
) +
1398 ipath_snap_cntr(dd
, crp
->cr_err_rlencnt
) +
1399 ipath_snap_cntr(dd
, crp
->cr_invalidrlencnt
) +
1400 ipath_snap_cntr(dd
, crp
->cr_errlinkcnt
) +
1401 ipath_snap_cntr(dd
, crp
->cr_erricrccnt
) +
1402 ipath_snap_cntr(dd
, crp
->cr_errvcrccnt
) +
1403 ipath_snap_cntr(dd
, crp
->cr_errlpcrccnt
) +
1404 ipath_snap_cntr(dd
, crp
->cr_badformatcnt
) +
1405 dd
->ipath_rxfc_unsupvl_errs
;
1406 if (crp
->cr_rxotherlocalphyerrcnt
)
1407 cntrs
->port_rcv_errors
+=
1408 ipath_snap_cntr(dd
, crp
->cr_rxotherlocalphyerrcnt
);
1409 if (crp
->cr_rxvlerrcnt
)
1410 cntrs
->port_rcv_errors
+=
1411 ipath_snap_cntr(dd
, crp
->cr_rxvlerrcnt
);
1412 cntrs
->port_rcv_remphys_errors
=
1413 ipath_snap_cntr(dd
, crp
->cr_rcvebpcnt
);
1414 cntrs
->port_xmit_discards
= ipath_snap_cntr(dd
, crp
->cr_unsupvlcnt
);
1415 cntrs
->port_xmit_data
= ipath_snap_cntr(dd
, crp
->cr_wordsendcnt
);
1416 cntrs
->port_rcv_data
= ipath_snap_cntr(dd
, crp
->cr_wordrcvcnt
);
1417 cntrs
->port_xmit_packets
= ipath_snap_cntr(dd
, crp
->cr_pktsendcnt
);
1418 cntrs
->port_rcv_packets
= ipath_snap_cntr(dd
, crp
->cr_pktrcvcnt
);
1419 cntrs
->local_link_integrity_errors
=
1420 crp
->cr_locallinkintegrityerrcnt
?
1421 ipath_snap_cntr(dd
, crp
->cr_locallinkintegrityerrcnt
) :
1422 ((dd
->ipath_flags
& IPATH_GPIO_ERRINTRS
) ?
1423 dd
->ipath_lli_errs
: dd
->ipath_lli_errors
);
1424 cntrs
->excessive_buffer_overrun_errors
=
1425 crp
->cr_excessbufferovflcnt
?
1426 ipath_snap_cntr(dd
, crp
->cr_excessbufferovflcnt
) :
1427 dd
->ipath_overrun_thresh_errs
;
1428 cntrs
->vl15_dropped
= crp
->cr_vl15droppedpktcnt
?
1429 ipath_snap_cntr(dd
, crp
->cr_vl15droppedpktcnt
) : 0;
1438 * ipath_ib_piobufavail - callback when a PIO buffer is available
1439 * @arg: the device pointer
1441 * This is called from ipath_intr() at interrupt level when a PIO buffer is
1442 * available after ipath_verbs_send() returned an error that no buffers were
1443 * available. Return 1 if we consumed all the PIO buffers and we still have
1444 * QPs waiting for buffers (for now, just restart the send tasklet and
1447 int ipath_ib_piobufavail(struct ipath_ibdev
*dev
)
1449 struct list_head
*list
;
1450 struct ipath_qp
*qplist
;
1451 struct ipath_qp
*qp
;
1452 unsigned long flags
;
1457 list
= &dev
->piowait
;
1460 spin_lock_irqsave(&dev
->pending_lock
, flags
);
1461 while (!list_empty(list
)) {
1462 qp
= list_entry(list
->next
, struct ipath_qp
, piowait
);
1463 list_del_init(&qp
->piowait
);
1464 qp
->pio_next
= qplist
;
1466 atomic_inc(&qp
->refcount
);
1468 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
1470 while (qplist
!= NULL
) {
1472 qplist
= qp
->pio_next
;
1474 spin_lock_irqsave(&qp
->s_lock
, flags
);
1475 if (ib_ipath_state_ops
[qp
->state
] & IPATH_PROCESS_SEND_OK
)
1476 ipath_schedule_send(qp
);
1477 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1479 /* Notify ipath_destroy_qp() if it is waiting. */
1480 if (atomic_dec_and_test(&qp
->refcount
))
1488 static int ipath_query_device(struct ib_device
*ibdev
,
1489 struct ib_device_attr
*props
)
1491 struct ipath_ibdev
*dev
= to_idev(ibdev
);
1493 memset(props
, 0, sizeof(*props
));
1495 props
->device_cap_flags
= IB_DEVICE_BAD_PKEY_CNTR
|
1496 IB_DEVICE_BAD_QKEY_CNTR
| IB_DEVICE_SHUTDOWN_PORT
|
1497 IB_DEVICE_SYS_IMAGE_GUID
| IB_DEVICE_RC_RNR_NAK_GEN
|
1498 IB_DEVICE_PORT_ACTIVE_EVENT
| IB_DEVICE_SRQ_RESIZE
;
1499 props
->page_size_cap
= PAGE_SIZE
;
1500 props
->vendor_id
= dev
->dd
->ipath_vendorid
;
1501 props
->vendor_part_id
= dev
->dd
->ipath_deviceid
;
1502 props
->hw_ver
= dev
->dd
->ipath_pcirev
;
1504 props
->sys_image_guid
= dev
->sys_image_guid
;
1506 props
->max_mr_size
= ~0ull;
1507 props
->max_qp
= ib_ipath_max_qps
;
1508 props
->max_qp_wr
= ib_ipath_max_qp_wrs
;
1509 props
->max_sge
= ib_ipath_max_sges
;
1510 props
->max_cq
= ib_ipath_max_cqs
;
1511 props
->max_ah
= ib_ipath_max_ahs
;
1512 props
->max_cqe
= ib_ipath_max_cqes
;
1513 props
->max_mr
= dev
->lk_table
.max
;
1514 props
->max_fmr
= dev
->lk_table
.max
;
1515 props
->max_map_per_fmr
= 32767;
1516 props
->max_pd
= ib_ipath_max_pds
;
1517 props
->max_qp_rd_atom
= IPATH_MAX_RDMA_ATOMIC
;
1518 props
->max_qp_init_rd_atom
= 255;
1519 /* props->max_res_rd_atom */
1520 props
->max_srq
= ib_ipath_max_srqs
;
1521 props
->max_srq_wr
= ib_ipath_max_srq_wrs
;
1522 props
->max_srq_sge
= ib_ipath_max_srq_sges
;
1523 /* props->local_ca_ack_delay */
1524 props
->atomic_cap
= IB_ATOMIC_GLOB
;
1525 props
->max_pkeys
= ipath_get_npkeys(dev
->dd
);
1526 props
->max_mcast_grp
= ib_ipath_max_mcast_grps
;
1527 props
->max_mcast_qp_attach
= ib_ipath_max_mcast_qp_attached
;
1528 props
->max_total_mcast_qp_attach
= props
->max_mcast_qp_attach
*
1529 props
->max_mcast_grp
;
1534 const u8 ipath_cvt_physportstate
[32] = {
1535 [INFINIPATH_IBCS_LT_STATE_DISABLED
] = IB_PHYSPORTSTATE_DISABLED
,
1536 [INFINIPATH_IBCS_LT_STATE_LINKUP
] = IB_PHYSPORTSTATE_LINKUP
,
1537 [INFINIPATH_IBCS_LT_STATE_POLLACTIVE
] = IB_PHYSPORTSTATE_POLL
,
1538 [INFINIPATH_IBCS_LT_STATE_POLLQUIET
] = IB_PHYSPORTSTATE_POLL
,
1539 [INFINIPATH_IBCS_LT_STATE_SLEEPDELAY
] = IB_PHYSPORTSTATE_SLEEP
,
1540 [INFINIPATH_IBCS_LT_STATE_SLEEPQUIET
] = IB_PHYSPORTSTATE_SLEEP
,
1541 [INFINIPATH_IBCS_LT_STATE_CFGDEBOUNCE
] =
1542 IB_PHYSPORTSTATE_CFG_TRAIN
,
1543 [INFINIPATH_IBCS_LT_STATE_CFGRCVFCFG
] =
1544 IB_PHYSPORTSTATE_CFG_TRAIN
,
1545 [INFINIPATH_IBCS_LT_STATE_CFGWAITRMT
] =
1546 IB_PHYSPORTSTATE_CFG_TRAIN
,
1547 [INFINIPATH_IBCS_LT_STATE_CFGIDLE
] = IB_PHYSPORTSTATE_CFG_TRAIN
,
1548 [INFINIPATH_IBCS_LT_STATE_RECOVERRETRAIN
] =
1549 IB_PHYSPORTSTATE_LINK_ERR_RECOVER
,
1550 [INFINIPATH_IBCS_LT_STATE_RECOVERWAITRMT
] =
1551 IB_PHYSPORTSTATE_LINK_ERR_RECOVER
,
1552 [INFINIPATH_IBCS_LT_STATE_RECOVERIDLE
] =
1553 IB_PHYSPORTSTATE_LINK_ERR_RECOVER
,
1554 [0x10] = IB_PHYSPORTSTATE_CFG_TRAIN
,
1555 [0x11] = IB_PHYSPORTSTATE_CFG_TRAIN
,
1556 [0x12] = IB_PHYSPORTSTATE_CFG_TRAIN
,
1557 [0x13] = IB_PHYSPORTSTATE_CFG_TRAIN
,
1558 [0x14] = IB_PHYSPORTSTATE_CFG_TRAIN
,
1559 [0x15] = IB_PHYSPORTSTATE_CFG_TRAIN
,
1560 [0x16] = IB_PHYSPORTSTATE_CFG_TRAIN
,
1561 [0x17] = IB_PHYSPORTSTATE_CFG_TRAIN
1564 u32
ipath_get_cr_errpkey(struct ipath_devdata
*dd
)
1566 return ipath_read_creg32(dd
, dd
->ipath_cregs
->cr_errpkey
);
1569 static int ipath_query_port(struct ib_device
*ibdev
,
1570 u8 port
, struct ib_port_attr
*props
)
1572 struct ipath_ibdev
*dev
= to_idev(ibdev
);
1573 struct ipath_devdata
*dd
= dev
->dd
;
1575 u16 lid
= dd
->ipath_lid
;
1578 memset(props
, 0, sizeof(*props
));
1579 props
->lid
= lid
? lid
: __constant_be16_to_cpu(IB_LID_PERMISSIVE
);
1580 props
->lmc
= dd
->ipath_lmc
;
1581 props
->sm_lid
= dev
->sm_lid
;
1582 props
->sm_sl
= dev
->sm_sl
;
1583 ibcstat
= dd
->ipath_lastibcstat
;
1584 /* map LinkState to IB portinfo values. */
1585 props
->state
= ipath_ib_linkstate(dd
, ibcstat
) + 1;
1587 /* See phys_state_show() */
1588 props
->phys_state
= /* MEA: assumes shift == 0 */
1589 ipath_cvt_physportstate
[dd
->ipath_lastibcstat
&
1591 props
->port_cap_flags
= dev
->port_cap_flags
;
1592 props
->gid_tbl_len
= 1;
1593 props
->max_msg_sz
= 0x80000000;
1594 props
->pkey_tbl_len
= ipath_get_npkeys(dd
);
1595 props
->bad_pkey_cntr
= ipath_get_cr_errpkey(dd
) -
1596 dev
->z_pkey_violations
;
1597 props
->qkey_viol_cntr
= dev
->qkey_violations
;
1598 props
->active_width
= dd
->ipath_link_width_active
;
1599 /* See rate_show() */
1600 props
->active_speed
= dd
->ipath_link_speed_active
;
1601 props
->max_vl_num
= 1; /* VLCap = VL0 */
1602 props
->init_type_reply
= 0;
1604 props
->max_mtu
= ipath_mtu4096
? IB_MTU_4096
: IB_MTU_2048
;
1605 switch (dd
->ipath_ibmtu
) {
1624 props
->active_mtu
= mtu
;
1625 props
->subnet_timeout
= dev
->subnet_timeout
;
1630 static int ipath_modify_device(struct ib_device
*device
,
1631 int device_modify_mask
,
1632 struct ib_device_modify
*device_modify
)
1636 if (device_modify_mask
& ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID
|
1637 IB_DEVICE_MODIFY_NODE_DESC
)) {
1642 if (device_modify_mask
& IB_DEVICE_MODIFY_NODE_DESC
)
1643 memcpy(device
->node_desc
, device_modify
->node_desc
, 64);
1645 if (device_modify_mask
& IB_DEVICE_MODIFY_SYS_IMAGE_GUID
)
1646 to_idev(device
)->sys_image_guid
=
1647 cpu_to_be64(device_modify
->sys_image_guid
);
1655 static int ipath_modify_port(struct ib_device
*ibdev
,
1656 u8 port
, int port_modify_mask
,
1657 struct ib_port_modify
*props
)
1659 struct ipath_ibdev
*dev
= to_idev(ibdev
);
1661 dev
->port_cap_flags
|= props
->set_port_cap_mask
;
1662 dev
->port_cap_flags
&= ~props
->clr_port_cap_mask
;
1663 if (port_modify_mask
& IB_PORT_SHUTDOWN
)
1664 ipath_set_linkstate(dev
->dd
, IPATH_IB_LINKDOWN
);
1665 if (port_modify_mask
& IB_PORT_RESET_QKEY_CNTR
)
1666 dev
->qkey_violations
= 0;
1670 static int ipath_query_gid(struct ib_device
*ibdev
, u8 port
,
1671 int index
, union ib_gid
*gid
)
1673 struct ipath_ibdev
*dev
= to_idev(ibdev
);
1680 gid
->global
.subnet_prefix
= dev
->gid_prefix
;
1681 gid
->global
.interface_id
= dev
->dd
->ipath_guid
;
1689 static struct ib_pd
*ipath_alloc_pd(struct ib_device
*ibdev
,
1690 struct ib_ucontext
*context
,
1691 struct ib_udata
*udata
)
1693 struct ipath_ibdev
*dev
= to_idev(ibdev
);
1694 struct ipath_pd
*pd
;
1698 * This is actually totally arbitrary. Some correctness tests
1699 * assume there's a maximum number of PDs that can be allocated.
1700 * We don't actually have this limit, but we fail the test if
1701 * we allow allocations of more than we report for this value.
1704 pd
= kmalloc(sizeof *pd
, GFP_KERNEL
);
1706 ret
= ERR_PTR(-ENOMEM
);
1710 spin_lock(&dev
->n_pds_lock
);
1711 if (dev
->n_pds_allocated
== ib_ipath_max_pds
) {
1712 spin_unlock(&dev
->n_pds_lock
);
1714 ret
= ERR_PTR(-ENOMEM
);
1718 dev
->n_pds_allocated
++;
1719 spin_unlock(&dev
->n_pds_lock
);
1721 /* ib_alloc_pd() will initialize pd->ibpd. */
1722 pd
->user
= udata
!= NULL
;
1730 static int ipath_dealloc_pd(struct ib_pd
*ibpd
)
1732 struct ipath_pd
*pd
= to_ipd(ibpd
);
1733 struct ipath_ibdev
*dev
= to_idev(ibpd
->device
);
1735 spin_lock(&dev
->n_pds_lock
);
1736 dev
->n_pds_allocated
--;
1737 spin_unlock(&dev
->n_pds_lock
);
1745 * ipath_create_ah - create an address handle
1746 * @pd: the protection domain
1747 * @ah_attr: the attributes of the AH
1749 * This may be called from interrupt context.
1751 static struct ib_ah
*ipath_create_ah(struct ib_pd
*pd
,
1752 struct ib_ah_attr
*ah_attr
)
1754 struct ipath_ah
*ah
;
1756 struct ipath_ibdev
*dev
= to_idev(pd
->device
);
1757 unsigned long flags
;
1759 /* A multicast address requires a GRH (see ch. 8.4.1). */
1760 if (ah_attr
->dlid
>= IPATH_MULTICAST_LID_BASE
&&
1761 ah_attr
->dlid
!= IPATH_PERMISSIVE_LID
&&
1762 !(ah_attr
->ah_flags
& IB_AH_GRH
)) {
1763 ret
= ERR_PTR(-EINVAL
);
1767 if (ah_attr
->dlid
== 0) {
1768 ret
= ERR_PTR(-EINVAL
);
1772 if (ah_attr
->port_num
< 1 ||
1773 ah_attr
->port_num
> pd
->device
->phys_port_cnt
) {
1774 ret
= ERR_PTR(-EINVAL
);
1778 ah
= kmalloc(sizeof *ah
, GFP_ATOMIC
);
1780 ret
= ERR_PTR(-ENOMEM
);
1784 spin_lock_irqsave(&dev
->n_ahs_lock
, flags
);
1785 if (dev
->n_ahs_allocated
== ib_ipath_max_ahs
) {
1786 spin_unlock_irqrestore(&dev
->n_ahs_lock
, flags
);
1788 ret
= ERR_PTR(-ENOMEM
);
1792 dev
->n_ahs_allocated
++;
1793 spin_unlock_irqrestore(&dev
->n_ahs_lock
, flags
);
1795 /* ib_create_ah() will initialize ah->ibah. */
1796 ah
->attr
= *ah_attr
;
1797 ah
->attr
.static_rate
= ipath_ib_rate_to_mult(ah_attr
->static_rate
);
1806 * ipath_destroy_ah - destroy an address handle
1807 * @ibah: the AH to destroy
1809 * This may be called from interrupt context.
1811 static int ipath_destroy_ah(struct ib_ah
*ibah
)
1813 struct ipath_ibdev
*dev
= to_idev(ibah
->device
);
1814 struct ipath_ah
*ah
= to_iah(ibah
);
1815 unsigned long flags
;
1817 spin_lock_irqsave(&dev
->n_ahs_lock
, flags
);
1818 dev
->n_ahs_allocated
--;
1819 spin_unlock_irqrestore(&dev
->n_ahs_lock
, flags
);
1826 static int ipath_query_ah(struct ib_ah
*ibah
, struct ib_ah_attr
*ah_attr
)
1828 struct ipath_ah
*ah
= to_iah(ibah
);
1830 *ah_attr
= ah
->attr
;
1831 ah_attr
->static_rate
= ipath_mult_to_ib_rate(ah
->attr
.static_rate
);
1837 * ipath_get_npkeys - return the size of the PKEY table for port 0
1838 * @dd: the infinipath device
1840 unsigned ipath_get_npkeys(struct ipath_devdata
*dd
)
1842 return ARRAY_SIZE(dd
->ipath_pd
[0]->port_pkeys
);
1846 * ipath_get_pkey - return the indexed PKEY from the port 0 PKEY table
1847 * @dd: the infinipath device
1848 * @index: the PKEY index
1850 unsigned ipath_get_pkey(struct ipath_devdata
*dd
, unsigned index
)
1854 if (index
>= ARRAY_SIZE(dd
->ipath_pd
[0]->port_pkeys
))
1857 ret
= dd
->ipath_pd
[0]->port_pkeys
[index
];
1862 static int ipath_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
,
1865 struct ipath_ibdev
*dev
= to_idev(ibdev
);
1868 if (index
>= ipath_get_npkeys(dev
->dd
)) {
1873 *pkey
= ipath_get_pkey(dev
->dd
, index
);
1881 * ipath_alloc_ucontext - allocate a ucontest
1882 * @ibdev: the infiniband device
1883 * @udata: not used by the InfiniPath driver
1886 static struct ib_ucontext
*ipath_alloc_ucontext(struct ib_device
*ibdev
,
1887 struct ib_udata
*udata
)
1889 struct ipath_ucontext
*context
;
1890 struct ib_ucontext
*ret
;
1892 context
= kmalloc(sizeof *context
, GFP_KERNEL
);
1894 ret
= ERR_PTR(-ENOMEM
);
1898 ret
= &context
->ibucontext
;
1904 static int ipath_dealloc_ucontext(struct ib_ucontext
*context
)
1906 kfree(to_iucontext(context
));
1910 static int ipath_verbs_register_sysfs(struct ib_device
*dev
);
1912 static void __verbs_timer(unsigned long arg
)
1914 struct ipath_devdata
*dd
= (struct ipath_devdata
*) arg
;
1916 /* Handle verbs layer timeouts. */
1917 ipath_ib_timer(dd
->verbs_dev
);
1919 mod_timer(&dd
->verbs_timer
, jiffies
+ 1);
1922 static int enable_timer(struct ipath_devdata
*dd
)
1925 * Early chips had a design flaw where the chip and kernel idea
1926 * of the tail register don't always agree, and therefore we won't
1927 * get an interrupt on the next packet received.
1928 * If the board supports per packet receive interrupts, use it.
1929 * Otherwise, the timer function periodically checks for packets
1930 * to cover this case.
1931 * Either way, the timer is needed for verbs layer related
1934 if (dd
->ipath_flags
& IPATH_GPIO_INTR
) {
1935 ipath_write_kreg(dd
, dd
->ipath_kregs
->kr_debugportselect
,
1936 0x2074076542310ULL
);
1937 /* Enable GPIO bit 2 interrupt */
1938 dd
->ipath_gpio_mask
|= (u64
) (1 << IPATH_GPIO_PORT0_BIT
);
1939 ipath_write_kreg(dd
, dd
->ipath_kregs
->kr_gpio_mask
,
1940 dd
->ipath_gpio_mask
);
1943 init_timer(&dd
->verbs_timer
);
1944 dd
->verbs_timer
.function
= __verbs_timer
;
1945 dd
->verbs_timer
.data
= (unsigned long)dd
;
1946 dd
->verbs_timer
.expires
= jiffies
+ 1;
1947 add_timer(&dd
->verbs_timer
);
1952 static int disable_timer(struct ipath_devdata
*dd
)
1954 /* Disable GPIO bit 2 interrupt */
1955 if (dd
->ipath_flags
& IPATH_GPIO_INTR
) {
1956 /* Disable GPIO bit 2 interrupt */
1957 dd
->ipath_gpio_mask
&= ~((u64
) (1 << IPATH_GPIO_PORT0_BIT
));
1958 ipath_write_kreg(dd
, dd
->ipath_kregs
->kr_gpio_mask
,
1959 dd
->ipath_gpio_mask
);
1961 * We might want to undo changes to debugportselect,
1966 del_timer_sync(&dd
->verbs_timer
);
1972 * ipath_register_ib_device - register our device with the infiniband core
1973 * @dd: the device data structure
1974 * Return the allocated ipath_ibdev pointer or NULL on error.
1976 int ipath_register_ib_device(struct ipath_devdata
*dd
)
1978 struct ipath_verbs_counters cntrs
;
1979 struct ipath_ibdev
*idev
;
1980 struct ib_device
*dev
;
1981 struct ipath_verbs_txreq
*tx
;
1985 idev
= (struct ipath_ibdev
*)ib_alloc_device(sizeof *idev
);
1993 if (dd
->ipath_sdma_descq_cnt
) {
1994 tx
= kmalloc(dd
->ipath_sdma_descq_cnt
* sizeof *tx
,
2002 idev
->txreq_bufs
= tx
;
2004 /* Only need to initialize non-zero fields. */
2005 spin_lock_init(&idev
->n_pds_lock
);
2006 spin_lock_init(&idev
->n_ahs_lock
);
2007 spin_lock_init(&idev
->n_cqs_lock
);
2008 spin_lock_init(&idev
->n_qps_lock
);
2009 spin_lock_init(&idev
->n_srqs_lock
);
2010 spin_lock_init(&idev
->n_mcast_grps_lock
);
2012 spin_lock_init(&idev
->qp_table
.lock
);
2013 spin_lock_init(&idev
->lk_table
.lock
);
2014 idev
->sm_lid
= __constant_be16_to_cpu(IB_LID_PERMISSIVE
);
2015 /* Set the prefix to the default value (see ch. 4.1.1) */
2016 idev
->gid_prefix
= __constant_cpu_to_be64(0xfe80000000000000ULL
);
2018 ret
= ipath_init_qp_table(idev
, ib_ipath_qp_table_size
);
2023 * The top ib_ipath_lkey_table_size bits are used to index the
2024 * table. The lower 8 bits can be owned by the user (copied from
2025 * the LKEY). The remaining bits act as a generation number or tag.
2027 idev
->lk_table
.max
= 1 << ib_ipath_lkey_table_size
;
2028 idev
->lk_table
.table
= kzalloc(idev
->lk_table
.max
*
2029 sizeof(*idev
->lk_table
.table
),
2031 if (idev
->lk_table
.table
== NULL
) {
2035 INIT_LIST_HEAD(&idev
->pending_mmaps
);
2036 spin_lock_init(&idev
->pending_lock
);
2037 idev
->mmap_offset
= PAGE_SIZE
;
2038 spin_lock_init(&idev
->mmap_offset_lock
);
2039 INIT_LIST_HEAD(&idev
->pending
[0]);
2040 INIT_LIST_HEAD(&idev
->pending
[1]);
2041 INIT_LIST_HEAD(&idev
->pending
[2]);
2042 INIT_LIST_HEAD(&idev
->piowait
);
2043 INIT_LIST_HEAD(&idev
->rnrwait
);
2044 INIT_LIST_HEAD(&idev
->txreq_free
);
2045 idev
->pending_index
= 0;
2046 idev
->port_cap_flags
=
2047 IB_PORT_SYS_IMAGE_GUID_SUP
| IB_PORT_CLIENT_REG_SUP
;
2048 if (dd
->ipath_flags
& IPATH_HAS_LINK_LATENCY
)
2049 idev
->port_cap_flags
|= IB_PORT_LINK_LATENCY_SUP
;
2050 idev
->pma_counter_select
[0] = IB_PMA_PORT_XMIT_DATA
;
2051 idev
->pma_counter_select
[1] = IB_PMA_PORT_RCV_DATA
;
2052 idev
->pma_counter_select
[2] = IB_PMA_PORT_XMIT_PKTS
;
2053 idev
->pma_counter_select
[3] = IB_PMA_PORT_RCV_PKTS
;
2054 idev
->pma_counter_select
[4] = IB_PMA_PORT_XMIT_WAIT
;
2056 /* Snapshot current HW counters to "clear" them. */
2057 ipath_get_counters(dd
, &cntrs
);
2058 idev
->z_symbol_error_counter
= cntrs
.symbol_error_counter
;
2059 idev
->z_link_error_recovery_counter
=
2060 cntrs
.link_error_recovery_counter
;
2061 idev
->z_link_downed_counter
= cntrs
.link_downed_counter
;
2062 idev
->z_port_rcv_errors
= cntrs
.port_rcv_errors
;
2063 idev
->z_port_rcv_remphys_errors
=
2064 cntrs
.port_rcv_remphys_errors
;
2065 idev
->z_port_xmit_discards
= cntrs
.port_xmit_discards
;
2066 idev
->z_port_xmit_data
= cntrs
.port_xmit_data
;
2067 idev
->z_port_rcv_data
= cntrs
.port_rcv_data
;
2068 idev
->z_port_xmit_packets
= cntrs
.port_xmit_packets
;
2069 idev
->z_port_rcv_packets
= cntrs
.port_rcv_packets
;
2070 idev
->z_local_link_integrity_errors
=
2071 cntrs
.local_link_integrity_errors
;
2072 idev
->z_excessive_buffer_overrun_errors
=
2073 cntrs
.excessive_buffer_overrun_errors
;
2074 idev
->z_vl15_dropped
= cntrs
.vl15_dropped
;
2076 for (i
= 0; i
< dd
->ipath_sdma_descq_cnt
; i
++, tx
++)
2077 list_add(&tx
->txreq
.list
, &idev
->txreq_free
);
2080 * The system image GUID is supposed to be the same for all
2081 * IB HCAs in a single system but since there can be other
2082 * device types in the system, we can't be sure this is unique.
2084 if (!sys_image_guid
)
2085 sys_image_guid
= dd
->ipath_guid
;
2086 idev
->sys_image_guid
= sys_image_guid
;
2087 idev
->ib_unit
= dd
->ipath_unit
;
2090 strlcpy(dev
->name
, "ipath%d", IB_DEVICE_NAME_MAX
);
2091 dev
->owner
= THIS_MODULE
;
2092 dev
->node_guid
= dd
->ipath_guid
;
2093 dev
->uverbs_abi_ver
= IPATH_UVERBS_ABI_VERSION
;
2094 dev
->uverbs_cmd_mask
=
2095 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
2096 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
2097 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
2098 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
2099 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
2100 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
2101 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
2102 (1ull << IB_USER_VERBS_CMD_QUERY_AH
) |
2103 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
2104 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
2105 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
2106 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
2107 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ
) |
2108 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
2109 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
2110 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
2111 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
2112 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
2113 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
2114 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
2115 (1ull << IB_USER_VERBS_CMD_POST_SEND
) |
2116 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
2117 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST
) |
2118 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST
) |
2119 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ
) |
2120 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ
) |
2121 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ
) |
2122 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ
) |
2123 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV
);
2124 dev
->node_type
= RDMA_NODE_IB_CA
;
2125 dev
->phys_port_cnt
= 1;
2126 dev
->num_comp_vectors
= 1;
2127 dev
->dma_device
= &dd
->pcidev
->dev
;
2128 dev
->query_device
= ipath_query_device
;
2129 dev
->modify_device
= ipath_modify_device
;
2130 dev
->query_port
= ipath_query_port
;
2131 dev
->modify_port
= ipath_modify_port
;
2132 dev
->query_pkey
= ipath_query_pkey
;
2133 dev
->query_gid
= ipath_query_gid
;
2134 dev
->alloc_ucontext
= ipath_alloc_ucontext
;
2135 dev
->dealloc_ucontext
= ipath_dealloc_ucontext
;
2136 dev
->alloc_pd
= ipath_alloc_pd
;
2137 dev
->dealloc_pd
= ipath_dealloc_pd
;
2138 dev
->create_ah
= ipath_create_ah
;
2139 dev
->destroy_ah
= ipath_destroy_ah
;
2140 dev
->query_ah
= ipath_query_ah
;
2141 dev
->create_srq
= ipath_create_srq
;
2142 dev
->modify_srq
= ipath_modify_srq
;
2143 dev
->query_srq
= ipath_query_srq
;
2144 dev
->destroy_srq
= ipath_destroy_srq
;
2145 dev
->create_qp
= ipath_create_qp
;
2146 dev
->modify_qp
= ipath_modify_qp
;
2147 dev
->query_qp
= ipath_query_qp
;
2148 dev
->destroy_qp
= ipath_destroy_qp
;
2149 dev
->post_send
= ipath_post_send
;
2150 dev
->post_recv
= ipath_post_receive
;
2151 dev
->post_srq_recv
= ipath_post_srq_receive
;
2152 dev
->create_cq
= ipath_create_cq
;
2153 dev
->destroy_cq
= ipath_destroy_cq
;
2154 dev
->resize_cq
= ipath_resize_cq
;
2155 dev
->poll_cq
= ipath_poll_cq
;
2156 dev
->req_notify_cq
= ipath_req_notify_cq
;
2157 dev
->get_dma_mr
= ipath_get_dma_mr
;
2158 dev
->reg_phys_mr
= ipath_reg_phys_mr
;
2159 dev
->reg_user_mr
= ipath_reg_user_mr
;
2160 dev
->dereg_mr
= ipath_dereg_mr
;
2161 dev
->alloc_fmr
= ipath_alloc_fmr
;
2162 dev
->map_phys_fmr
= ipath_map_phys_fmr
;
2163 dev
->unmap_fmr
= ipath_unmap_fmr
;
2164 dev
->dealloc_fmr
= ipath_dealloc_fmr
;
2165 dev
->attach_mcast
= ipath_multicast_attach
;
2166 dev
->detach_mcast
= ipath_multicast_detach
;
2167 dev
->process_mad
= ipath_process_mad
;
2168 dev
->mmap
= ipath_mmap
;
2169 dev
->dma_ops
= &ipath_dma_mapping_ops
;
2171 snprintf(dev
->node_desc
, sizeof(dev
->node_desc
),
2172 IPATH_IDSTR
" %s", init_utsname()->nodename
);
2174 ret
= ib_register_device(dev
);
2178 if (ipath_verbs_register_sysfs(dev
))
2186 ib_unregister_device(dev
);
2188 kfree(idev
->lk_table
.table
);
2190 kfree(idev
->qp_table
.table
);
2192 kfree(idev
->txreq_bufs
);
2194 ib_dealloc_device(dev
);
2195 ipath_dev_err(dd
, "cannot register verbs: %d!\n", -ret
);
2199 dd
->verbs_dev
= idev
;
2203 void ipath_unregister_ib_device(struct ipath_ibdev
*dev
)
2205 struct ib_device
*ibdev
= &dev
->ibdev
;
2208 ib_unregister_device(ibdev
);
2210 disable_timer(dev
->dd
);
2212 if (!list_empty(&dev
->pending
[0]) ||
2213 !list_empty(&dev
->pending
[1]) ||
2214 !list_empty(&dev
->pending
[2]))
2215 ipath_dev_err(dev
->dd
, "pending list not empty!\n");
2216 if (!list_empty(&dev
->piowait
))
2217 ipath_dev_err(dev
->dd
, "piowait list not empty!\n");
2218 if (!list_empty(&dev
->rnrwait
))
2219 ipath_dev_err(dev
->dd
, "rnrwait list not empty!\n");
2220 if (!ipath_mcast_tree_empty())
2221 ipath_dev_err(dev
->dd
, "multicast table memory leak!\n");
2223 * Note that ipath_unregister_ib_device() can be called before all
2224 * the QPs are destroyed!
2226 qps_inuse
= ipath_free_all_qps(&dev
->qp_table
);
2228 ipath_dev_err(dev
->dd
, "QP memory leak! %u still in use\n",
2230 kfree(dev
->qp_table
.table
);
2231 kfree(dev
->lk_table
.table
);
2232 kfree(dev
->txreq_bufs
);
2233 ib_dealloc_device(ibdev
);
2236 static ssize_t
show_rev(struct device
*device
, struct device_attribute
*attr
,
2239 struct ipath_ibdev
*dev
=
2240 container_of(device
, struct ipath_ibdev
, ibdev
.dev
);
2242 return sprintf(buf
, "%x\n", dev
->dd
->ipath_pcirev
);
2245 static ssize_t
show_hca(struct device
*device
, struct device_attribute
*attr
,
2248 struct ipath_ibdev
*dev
=
2249 container_of(device
, struct ipath_ibdev
, ibdev
.dev
);
2252 ret
= dev
->dd
->ipath_f_get_boardname(dev
->dd
, buf
, 128);
2262 static ssize_t
show_stats(struct device
*device
, struct device_attribute
*attr
,
2265 struct ipath_ibdev
*dev
=
2266 container_of(device
, struct ipath_ibdev
, ibdev
.dev
);
2284 dev
->n_rc_resends
, dev
->n_rc_qacks
, dev
->n_rc_acks
,
2285 dev
->n_seq_naks
, dev
->n_rdma_seq
, dev
->n_rnr_naks
,
2286 dev
->n_other_naks
, dev
->n_timeouts
,
2287 dev
->n_rdma_dup_busy
, dev
->n_piowait
, dev
->n_unaligned
,
2288 dev
->n_pkt_drops
, dev
->n_wqe_errs
);
2289 for (i
= 0; i
< ARRAY_SIZE(dev
->opstats
); i
++) {
2290 const struct ipath_opcode_stats
*si
= &dev
->opstats
[i
];
2292 if (!si
->n_packets
&& !si
->n_bytes
)
2294 len
+= sprintf(buf
+ len
, "%02x %llu/%llu\n", i
,
2295 (unsigned long long) si
->n_packets
,
2296 (unsigned long long) si
->n_bytes
);
2301 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
2302 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
2303 static DEVICE_ATTR(board_id
, S_IRUGO
, show_hca
, NULL
);
2304 static DEVICE_ATTR(stats
, S_IRUGO
, show_stats
, NULL
);
2306 static struct device_attribute
*ipath_class_attributes
[] = {
2313 static int ipath_verbs_register_sysfs(struct ib_device
*dev
)
2318 for (i
= 0; i
< ARRAY_SIZE(ipath_class_attributes
); ++i
)
2319 if (device_create_file(&dev
->dev
,
2320 ipath_class_attributes
[i
])) {