2 * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <rdma/ib_smi.h>
35 #include <rdma/ib_verbs.h>
41 * qib_ud_loopback - handle send on loopback QPs
42 * @sqp: the sending QP
43 * @swqe: the send work request
45 * This is called from qib_make_ud_req() to forward a WQE addressed
47 * Note that the receive interrupt handler may be calling qib_ud_rcv()
48 * while this is being called.
50 static void qib_ud_loopback(struct rvt_qp
*sqp
, struct rvt_swqe
*swqe
)
52 struct qib_ibport
*ibp
= to_iport(sqp
->ibqp
.device
, sqp
->port_num
);
53 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
54 struct qib_devdata
*dd
= ppd
->dd
;
55 struct rvt_dev_info
*rdi
= &dd
->verbs_dev
.rdi
;
57 struct rdma_ah_attr
*ah_attr
;
59 struct rvt_sge_state ssge
;
63 enum ib_qp_type sqptype
, dqptype
;
66 qp
= rvt_lookup_qpn(rdi
, &ibp
->rvp
, swqe
->ud_wr
.remote_qpn
);
68 ibp
->rvp
.n_pkt_drops
++;
72 sqptype
= sqp
->ibqp
.qp_type
== IB_QPT_GSI
?
73 IB_QPT_UD
: sqp
->ibqp
.qp_type
;
74 dqptype
= qp
->ibqp
.qp_type
== IB_QPT_GSI
?
75 IB_QPT_UD
: qp
->ibqp
.qp_type
;
77 if (dqptype
!= sqptype
||
78 !(ib_rvt_state_ops
[qp
->state
] & RVT_PROCESS_RECV_OK
)) {
79 ibp
->rvp
.n_pkt_drops
++;
83 ah_attr
= &ibah_to_rvtah(swqe
->ud_wr
.ah
)->attr
;
84 ppd
= ppd_from_ibp(ibp
);
86 if (qp
->ibqp
.qp_num
> 1) {
91 pkey1
= qib_get_pkey(ibp
, sqp
->s_pkey_index
);
92 pkey2
= qib_get_pkey(ibp
, qp
->s_pkey_index
);
93 if (unlikely(!qib_pkey_ok(pkey1
, pkey2
))) {
94 lid
= ppd
->lid
| (rdma_ah_get_path_bits(ah_attr
) &
95 ((1 << ppd
->lmc
) - 1));
96 qib_bad_pkey(ibp
, pkey1
,
97 rdma_ah_get_sl(ah_attr
),
98 sqp
->ibqp
.qp_num
, qp
->ibqp
.qp_num
,
100 cpu_to_be16(rdma_ah_get_dlid(ah_attr
)));
106 * Check that the qkey matches (except for QP0, see 9.6.1.4.1).
107 * Qkeys with the high order bit set mean use the
108 * qkey from the QP context instead of the WR (see 10.2.5).
110 if (qp
->ibqp
.qp_num
) {
113 qkey
= (int)swqe
->ud_wr
.remote_qkey
< 0 ?
114 sqp
->qkey
: swqe
->ud_wr
.remote_qkey
;
115 if (unlikely(qkey
!= qp
->qkey
))
120 * A GRH is expected to precede the data even if not
121 * present on the wire.
123 length
= swqe
->length
;
124 memset(&wc
, 0, sizeof(wc
));
125 wc
.byte_len
= length
+ sizeof(struct ib_grh
);
127 if (swqe
->wr
.opcode
== IB_WR_SEND_WITH_IMM
) {
128 wc
.wc_flags
= IB_WC_WITH_IMM
;
129 wc
.ex
.imm_data
= swqe
->wr
.ex
.imm_data
;
132 spin_lock_irqsave(&qp
->r_lock
, flags
);
135 * Get the next work request entry to find where to put the data.
137 if (qp
->r_flags
& RVT_R_REUSE_SGE
)
138 qp
->r_flags
&= ~RVT_R_REUSE_SGE
;
142 ret
= rvt_get_rwqe(qp
, false);
144 rvt_rc_error(qp
, IB_WC_LOC_QP_OP_ERR
);
148 if (qp
->ibqp
.qp_num
== 0)
149 ibp
->rvp
.n_vl15_dropped
++;
153 /* Silently drop packets which are too big. */
154 if (unlikely(wc
.byte_len
> qp
->r_len
)) {
155 qp
->r_flags
|= RVT_R_REUSE_SGE
;
156 ibp
->rvp
.n_pkt_drops
++;
160 if (rdma_ah_get_ah_flags(ah_attr
) & IB_AH_GRH
) {
162 const struct ib_global_route
*grd
= rdma_ah_read_grh(ah_attr
);
164 qib_make_grh(ibp
, &grh
, grd
, 0, 0);
165 rvt_copy_sge(qp
, &qp
->r_sge
, &grh
,
166 sizeof(grh
), true, false);
167 wc
.wc_flags
|= IB_WC_GRH
;
169 rvt_skip_sge(&qp
->r_sge
, sizeof(struct ib_grh
), true);
170 ssge
.sg_list
= swqe
->sg_list
+ 1;
171 ssge
.sge
= *swqe
->sg_list
;
172 ssge
.num_sge
= swqe
->wr
.num_sge
;
175 u32 len
= sge
->length
;
179 if (len
> sge
->sge_length
)
180 len
= sge
->sge_length
;
181 rvt_copy_sge(qp
, &qp
->r_sge
, sge
->vaddr
, len
, true, false);
184 sge
->sge_length
-= len
;
185 if (sge
->sge_length
== 0) {
187 *sge
= *ssge
.sg_list
++;
188 } else if (sge
->length
== 0 && sge
->mr
->lkey
) {
189 if (++sge
->n
>= RVT_SEGSZ
) {
190 if (++sge
->m
>= sge
->mr
->mapsz
)
195 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
197 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
201 rvt_put_ss(&qp
->r_sge
);
202 if (!test_and_clear_bit(RVT_R_WRID_VALID
, &qp
->r_aflags
))
204 wc
.wr_id
= qp
->r_wr_id
;
205 wc
.status
= IB_WC_SUCCESS
;
206 wc
.opcode
= IB_WC_RECV
;
208 wc
.src_qp
= sqp
->ibqp
.qp_num
;
209 wc
.pkey_index
= qp
->ibqp
.qp_type
== IB_QPT_GSI
?
210 swqe
->ud_wr
.pkey_index
: 0;
211 wc
.slid
= ppd
->lid
| (rdma_ah_get_path_bits(ah_attr
) &
212 ((1 << ppd
->lmc
) - 1));
213 wc
.sl
= rdma_ah_get_sl(ah_attr
);
214 wc
.dlid_path_bits
= rdma_ah_get_dlid(ah_attr
) & ((1 << ppd
->lmc
) - 1);
215 wc
.port_num
= qp
->port_num
;
216 /* Signal completion event if the solicited bit is set. */
217 rvt_cq_enter(ibcq_to_rvtcq(qp
->ibqp
.recv_cq
), &wc
,
218 swqe
->wr
.send_flags
& IB_SEND_SOLICITED
);
219 ibp
->rvp
.n_loop_pkts
++;
221 spin_unlock_irqrestore(&qp
->r_lock
, flags
);
227 * qib_make_ud_req - construct a UD request packet
230 * Assumes the s_lock is held.
232 * Return 1 if constructed; otherwise, return 0.
234 int qib_make_ud_req(struct rvt_qp
*qp
, unsigned long *flags
)
236 struct qib_qp_priv
*priv
= qp
->priv
;
237 struct ib_other_headers
*ohdr
;
238 struct rdma_ah_attr
*ah_attr
;
239 struct qib_pportdata
*ppd
;
240 struct qib_ibport
*ibp
;
241 struct rvt_swqe
*wqe
;
250 if (!(ib_rvt_state_ops
[qp
->state
] & RVT_PROCESS_NEXT_SEND_OK
)) {
251 if (!(ib_rvt_state_ops
[qp
->state
] & RVT_FLUSH_SEND
))
253 /* We are in the error state, flush the work request. */
254 if (qp
->s_last
== READ_ONCE(qp
->s_head
))
256 /* If DMAs are in progress, we can't flush immediately. */
257 if (atomic_read(&priv
->s_dma_busy
)) {
258 qp
->s_flags
|= RVT_S_WAIT_DMA
;
261 wqe
= rvt_get_swqe_ptr(qp
, qp
->s_last
);
262 rvt_send_complete(qp
, wqe
, IB_WC_WR_FLUSH_ERR
);
266 /* see post_one_send() */
267 if (qp
->s_cur
== READ_ONCE(qp
->s_head
))
270 wqe
= rvt_get_swqe_ptr(qp
, qp
->s_cur
);
271 next_cur
= qp
->s_cur
+ 1;
272 if (next_cur
>= qp
->s_size
)
275 /* Construct the header. */
276 ibp
= to_iport(qp
->ibqp
.device
, qp
->port_num
);
277 ppd
= ppd_from_ibp(ibp
);
278 ah_attr
= &ibah_to_rvtah(wqe
->ud_wr
.ah
)->attr
;
279 if (rdma_ah_get_dlid(ah_attr
) >= be16_to_cpu(IB_MULTICAST_LID_BASE
)) {
280 if (rdma_ah_get_dlid(ah_attr
) !=
281 be16_to_cpu(IB_LID_PERMISSIVE
))
282 this_cpu_inc(ibp
->pmastats
->n_multicast_xmit
);
284 this_cpu_inc(ibp
->pmastats
->n_unicast_xmit
);
286 this_cpu_inc(ibp
->pmastats
->n_unicast_xmit
);
287 lid
= rdma_ah_get_dlid(ah_attr
) & ~((1 << ppd
->lmc
) - 1);
288 if (unlikely(lid
== ppd
->lid
)) {
289 unsigned long tflags
= *flags
;
291 * If DMAs are in progress, we can't generate
292 * a completion for the loopback packet since
293 * it would be out of order.
294 * XXX Instead of waiting, we could queue a
295 * zero length descriptor so we get a callback.
297 if (atomic_read(&priv
->s_dma_busy
)) {
298 qp
->s_flags
|= RVT_S_WAIT_DMA
;
301 qp
->s_cur
= next_cur
;
302 spin_unlock_irqrestore(&qp
->s_lock
, tflags
);
303 qib_ud_loopback(qp
, wqe
);
304 spin_lock_irqsave(&qp
->s_lock
, tflags
);
306 rvt_send_complete(qp
, wqe
, IB_WC_SUCCESS
);
311 qp
->s_cur
= next_cur
;
312 extra_bytes
= -wqe
->length
& 3;
313 nwords
= (wqe
->length
+ extra_bytes
) >> 2;
315 /* header size in 32-bit words LRH+BTH+DETH = (8+12+8)/4. */
317 qp
->s_cur_size
= wqe
->length
;
318 qp
->s_cur_sge
= &qp
->s_sge
;
319 qp
->s_srate
= rdma_ah_get_static_rate(ah_attr
);
321 qp
->s_sge
.sge
= wqe
->sg_list
[0];
322 qp
->s_sge
.sg_list
= wqe
->sg_list
+ 1;
323 qp
->s_sge
.num_sge
= wqe
->wr
.num_sge
;
324 qp
->s_sge
.total_len
= wqe
->length
;
326 if (rdma_ah_get_ah_flags(ah_attr
) & IB_AH_GRH
) {
327 /* Header size in 32-bit words. */
328 qp
->s_hdrwords
+= qib_make_grh(ibp
, &priv
->s_hdr
->u
.l
.grh
,
329 rdma_ah_read_grh(ah_attr
),
330 qp
->s_hdrwords
, nwords
);
332 ohdr
= &priv
->s_hdr
->u
.l
.oth
;
334 * Don't worry about sending to locally attached multicast
335 * QPs. It is unspecified by the spec. what happens.
338 /* Header size in 32-bit words. */
340 ohdr
= &priv
->s_hdr
->u
.oth
;
342 if (wqe
->wr
.opcode
== IB_WR_SEND_WITH_IMM
) {
344 ohdr
->u
.ud
.imm_data
= wqe
->wr
.ex
.imm_data
;
345 bth0
= IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE
<< 24;
347 bth0
= IB_OPCODE_UD_SEND_ONLY
<< 24;
348 lrh0
|= rdma_ah_get_sl(ah_attr
) << 4;
349 if (qp
->ibqp
.qp_type
== IB_QPT_SMI
)
350 lrh0
|= 0xF000; /* Set VL (see ch. 13.5.3.1) */
352 lrh0
|= ibp
->sl_to_vl
[rdma_ah_get_sl(ah_attr
)] << 12;
353 priv
->s_hdr
->lrh
[0] = cpu_to_be16(lrh0
);
354 priv
->s_hdr
->lrh
[1] =
355 cpu_to_be16(rdma_ah_get_dlid(ah_attr
)); /* DEST LID */
356 priv
->s_hdr
->lrh
[2] =
357 cpu_to_be16(qp
->s_hdrwords
+ nwords
+ SIZE_OF_CRC
);
360 lid
|= rdma_ah_get_path_bits(ah_attr
) &
361 ((1 << ppd
->lmc
) - 1);
362 priv
->s_hdr
->lrh
[3] = cpu_to_be16(lid
);
364 priv
->s_hdr
->lrh
[3] = IB_LID_PERMISSIVE
;
365 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
366 bth0
|= IB_BTH_SOLICITED
;
367 bth0
|= extra_bytes
<< 20;
368 bth0
|= qp
->ibqp
.qp_type
== IB_QPT_SMI
? QIB_DEFAULT_P_KEY
:
369 qib_get_pkey(ibp
, qp
->ibqp
.qp_type
== IB_QPT_GSI
?
370 wqe
->ud_wr
.pkey_index
: qp
->s_pkey_index
);
371 ohdr
->bth
[0] = cpu_to_be32(bth0
);
373 * Use the multicast QP if the destination LID is a multicast LID.
375 ohdr
->bth
[1] = rdma_ah_get_dlid(ah_attr
) >=
376 be16_to_cpu(IB_MULTICAST_LID_BASE
) &&
377 rdma_ah_get_dlid(ah_attr
) != be16_to_cpu(IB_LID_PERMISSIVE
) ?
378 cpu_to_be32(QIB_MULTICAST_QPN
) :
379 cpu_to_be32(wqe
->ud_wr
.remote_qpn
);
380 ohdr
->bth
[2] = cpu_to_be32(wqe
->psn
& QIB_PSN_MASK
);
382 * Qkeys with the high order bit set mean use the
383 * qkey from the QP context instead of the WR (see 10.2.5).
385 ohdr
->u
.ud
.deth
[0] = cpu_to_be32((int)wqe
->ud_wr
.remote_qkey
< 0 ?
386 qp
->qkey
: wqe
->ud_wr
.remote_qkey
);
387 ohdr
->u
.ud
.deth
[1] = cpu_to_be32(qp
->ibqp
.qp_num
);
392 qp
->s_flags
&= ~RVT_S_BUSY
;
396 static unsigned qib_lookup_pkey(struct qib_ibport
*ibp
, u16 pkey
)
398 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
399 struct qib_devdata
*dd
= ppd
->dd
;
400 unsigned ctxt
= ppd
->hw_pidx
;
403 pkey
&= 0x7fff; /* remove limited/full membership bit */
405 for (i
= 0; i
< ARRAY_SIZE(dd
->rcd
[ctxt
]->pkeys
); ++i
)
406 if ((dd
->rcd
[ctxt
]->pkeys
[i
] & 0x7fff) == pkey
)
410 * Should not get here, this means hardware failed to validate pkeys.
411 * Punt and return index 0.
417 * qib_ud_rcv - receive an incoming UD packet
418 * @ibp: the port the packet came in on
419 * @hdr: the packet header
420 * @has_grh: true if the packet has a GRH
421 * @data: the packet data
422 * @tlen: the packet length
423 * @qp: the QP the packet came on
425 * This is called from qib_qp_rcv() to process an incoming UD packet
427 * Called at interrupt level.
429 void qib_ud_rcv(struct qib_ibport
*ibp
, struct ib_header
*hdr
,
430 int has_grh
, void *data
, u32 tlen
, struct rvt_qp
*qp
)
432 struct ib_other_headers
*ohdr
;
444 hdrsize
= 8 + 12 + 8; /* LRH + BTH + DETH */
446 ohdr
= &hdr
->u
.l
.oth
;
447 hdrsize
= 8 + 40 + 12 + 8; /* LRH + GRH + BTH + DETH */
449 qkey
= be32_to_cpu(ohdr
->u
.ud
.deth
[0]);
450 src_qp
= be32_to_cpu(ohdr
->u
.ud
.deth
[1]) & RVT_QPN_MASK
;
453 * Get the number of bytes the message was padded by
454 * and drop incomplete packets.
456 pad
= (be32_to_cpu(ohdr
->bth
[0]) >> 20) & 3;
457 if (unlikely(tlen
< (hdrsize
+ pad
+ 4)))
460 tlen
-= hdrsize
+ pad
+ 4;
463 * Check that the permissive LID is only used on QP0
464 * and the QKEY matches (see 9.6.1.4.1 and 9.6.1.5.1).
466 if (qp
->ibqp
.qp_num
) {
467 if (unlikely(hdr
->lrh
[1] == IB_LID_PERMISSIVE
||
468 hdr
->lrh
[3] == IB_LID_PERMISSIVE
))
470 if (qp
->ibqp
.qp_num
> 1) {
473 pkey1
= be32_to_cpu(ohdr
->bth
[0]);
474 pkey2
= qib_get_pkey(ibp
, qp
->s_pkey_index
);
475 if (unlikely(!qib_pkey_ok(pkey1
, pkey2
))) {
478 (be16_to_cpu(hdr
->lrh
[0]) >> 4) &
480 src_qp
, qp
->ibqp
.qp_num
,
481 hdr
->lrh
[3], hdr
->lrh
[1]);
485 if (unlikely(qkey
!= qp
->qkey
))
488 /* Drop invalid MAD packets (see 13.5.3.1). */
489 if (unlikely(qp
->ibqp
.qp_num
== 1 &&
491 (be16_to_cpu(hdr
->lrh
[0]) >> 12) == 15)))
496 /* Drop invalid MAD packets (see 13.5.3.1). */
497 if (tlen
!= 256 || (be16_to_cpu(hdr
->lrh
[0]) >> 12) != 15)
499 smp
= (struct ib_smp
*) data
;
500 if ((hdr
->lrh
[1] == IB_LID_PERMISSIVE
||
501 hdr
->lrh
[3] == IB_LID_PERMISSIVE
) &&
502 smp
->mgmt_class
!= IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
)
507 * The opcode is in the low byte when its in network order
508 * (top byte when in host order).
510 opcode
= be32_to_cpu(ohdr
->bth
[0]) >> 24;
511 if (qp
->ibqp
.qp_num
> 1 &&
512 opcode
== IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE
) {
513 wc
.ex
.imm_data
= ohdr
->u
.ud
.imm_data
;
514 wc
.wc_flags
= IB_WC_WITH_IMM
;
515 } else if (opcode
== IB_OPCODE_UD_SEND_ONLY
) {
522 * A GRH is expected to precede the data even if not
523 * present on the wire.
525 wc
.byte_len
= tlen
+ sizeof(struct ib_grh
);
528 * Get the next work request entry to find where to put the data.
530 if (qp
->r_flags
& RVT_R_REUSE_SGE
)
531 qp
->r_flags
&= ~RVT_R_REUSE_SGE
;
535 ret
= rvt_get_rwqe(qp
, false);
537 rvt_rc_error(qp
, IB_WC_LOC_QP_OP_ERR
);
541 if (qp
->ibqp
.qp_num
== 0)
542 ibp
->rvp
.n_vl15_dropped
++;
546 /* Silently drop packets which are too big. */
547 if (unlikely(wc
.byte_len
> qp
->r_len
)) {
548 qp
->r_flags
|= RVT_R_REUSE_SGE
;
552 rvt_copy_sge(qp
, &qp
->r_sge
, &hdr
->u
.l
.grh
,
553 sizeof(struct ib_grh
), true, false);
554 wc
.wc_flags
|= IB_WC_GRH
;
556 rvt_skip_sge(&qp
->r_sge
, sizeof(struct ib_grh
), true);
557 rvt_copy_sge(qp
, &qp
->r_sge
, data
, wc
.byte_len
- sizeof(struct ib_grh
),
559 rvt_put_ss(&qp
->r_sge
);
560 if (!test_and_clear_bit(RVT_R_WRID_VALID
, &qp
->r_aflags
))
562 wc
.wr_id
= qp
->r_wr_id
;
563 wc
.status
= IB_WC_SUCCESS
;
564 wc
.opcode
= IB_WC_RECV
;
568 wc
.pkey_index
= qp
->ibqp
.qp_type
== IB_QPT_GSI
?
569 qib_lookup_pkey(ibp
, be32_to_cpu(ohdr
->bth
[0])) : 0;
570 wc
.slid
= be16_to_cpu(hdr
->lrh
[3]);
571 wc
.sl
= (be16_to_cpu(hdr
->lrh
[0]) >> 4) & 0xF;
572 dlid
= be16_to_cpu(hdr
->lrh
[1]);
574 * Save the LMC lower bits if the destination LID is a unicast LID.
576 wc
.dlid_path_bits
= dlid
>= be16_to_cpu(IB_MULTICAST_LID_BASE
) ? 0 :
577 dlid
& ((1 << ppd_from_ibp(ibp
)->lmc
) - 1);
578 wc
.port_num
= qp
->port_num
;
579 /* Signal completion event if the solicited bit is set. */
580 rvt_cq_enter(ibcq_to_rvtcq(qp
->ibqp
.recv_cq
), &wc
,
581 ib_bth_is_solicited(ohdr
));
585 ibp
->rvp
.n_pkt_drops
++;