2 * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
4 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37 /* cut down ridiculously long IB macro names */
38 #define OP(x) IB_OPCODE_UC_##x
41 * qib_make_uc_req - construct a request packet (SEND, RDMA write)
42 * @qp: a pointer to the QP
44 * Assumes the s_lock is held.
46 * Return 1 if constructed; otherwise, return 0.
48 int qib_make_uc_req(struct rvt_qp
*qp
, unsigned long *flags
)
50 struct qib_qp_priv
*priv
= qp
->priv
;
51 struct ib_other_headers
*ohdr
;
59 if (!(ib_rvt_state_ops
[qp
->state
] & RVT_PROCESS_SEND_OK
)) {
60 if (!(ib_rvt_state_ops
[qp
->state
] & RVT_FLUSH_SEND
))
62 /* We are in the error state, flush the work request. */
63 smp_read_barrier_depends(); /* see post_one_send() */
64 if (qp
->s_last
== ACCESS_ONCE(qp
->s_head
))
66 /* If DMAs are in progress, we can't flush immediately. */
67 if (atomic_read(&priv
->s_dma_busy
)) {
68 qp
->s_flags
|= RVT_S_WAIT_DMA
;
71 wqe
= rvt_get_swqe_ptr(qp
, qp
->s_last
);
72 qib_send_complete(qp
, wqe
, IB_WC_WR_FLUSH_ERR
);
76 ohdr
= &priv
->s_hdr
->u
.oth
;
77 if (rdma_ah_get_ah_flags(&qp
->remote_ah_attr
) & IB_AH_GRH
)
78 ohdr
= &priv
->s_hdr
->u
.l
.oth
;
80 /* header size in 32-bit words LRH+BTH = (8+12)/4. */
84 /* Get the next send request. */
85 wqe
= rvt_get_swqe_ptr(qp
, qp
->s_cur
);
87 switch (qp
->s_state
) {
89 if (!(ib_rvt_state_ops
[qp
->state
] &
90 RVT_PROCESS_NEXT_SEND_OK
))
92 /* Check if send work queue is empty. */
93 smp_read_barrier_depends(); /* see post_one_send() */
94 if (qp
->s_cur
== ACCESS_ONCE(qp
->s_head
))
97 * Start a new request.
100 qp
->s_sge
.sge
= wqe
->sg_list
[0];
101 qp
->s_sge
.sg_list
= wqe
->sg_list
+ 1;
102 qp
->s_sge
.num_sge
= wqe
->wr
.num_sge
;
103 qp
->s_sge
.total_len
= wqe
->length
;
106 switch (wqe
->wr
.opcode
) {
108 case IB_WR_SEND_WITH_IMM
:
110 qp
->s_state
= OP(SEND_FIRST
);
114 if (wqe
->wr
.opcode
== IB_WR_SEND
)
115 qp
->s_state
= OP(SEND_ONLY
);
118 OP(SEND_ONLY_WITH_IMMEDIATE
);
119 /* Immediate data comes after the BTH */
120 ohdr
->u
.imm_data
= wqe
->wr
.ex
.imm_data
;
123 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
124 bth0
|= IB_BTH_SOLICITED
;
126 if (++qp
->s_cur
>= qp
->s_size
)
130 case IB_WR_RDMA_WRITE
:
131 case IB_WR_RDMA_WRITE_WITH_IMM
:
132 ohdr
->u
.rc
.reth
.vaddr
=
133 cpu_to_be64(wqe
->rdma_wr
.remote_addr
);
134 ohdr
->u
.rc
.reth
.rkey
=
135 cpu_to_be32(wqe
->rdma_wr
.rkey
);
136 ohdr
->u
.rc
.reth
.length
= cpu_to_be32(len
);
137 hwords
+= sizeof(struct ib_reth
) / 4;
139 qp
->s_state
= OP(RDMA_WRITE_FIRST
);
143 if (wqe
->wr
.opcode
== IB_WR_RDMA_WRITE
)
144 qp
->s_state
= OP(RDMA_WRITE_ONLY
);
147 OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
);
148 /* Immediate data comes after the RETH */
149 ohdr
->u
.rc
.imm_data
= wqe
->wr
.ex
.imm_data
;
151 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
152 bth0
|= IB_BTH_SOLICITED
;
155 if (++qp
->s_cur
>= qp
->s_size
)
165 qp
->s_state
= OP(SEND_MIDDLE
);
167 case OP(SEND_MIDDLE
):
173 if (wqe
->wr
.opcode
== IB_WR_SEND
)
174 qp
->s_state
= OP(SEND_LAST
);
176 qp
->s_state
= OP(SEND_LAST_WITH_IMMEDIATE
);
177 /* Immediate data comes after the BTH */
178 ohdr
->u
.imm_data
= wqe
->wr
.ex
.imm_data
;
181 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
182 bth0
|= IB_BTH_SOLICITED
;
184 if (++qp
->s_cur
>= qp
->s_size
)
188 case OP(RDMA_WRITE_FIRST
):
189 qp
->s_state
= OP(RDMA_WRITE_MIDDLE
);
191 case OP(RDMA_WRITE_MIDDLE
):
197 if (wqe
->wr
.opcode
== IB_WR_RDMA_WRITE
)
198 qp
->s_state
= OP(RDMA_WRITE_LAST
);
201 OP(RDMA_WRITE_LAST_WITH_IMMEDIATE
);
202 /* Immediate data comes after the BTH */
203 ohdr
->u
.imm_data
= wqe
->wr
.ex
.imm_data
;
205 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
206 bth0
|= IB_BTH_SOLICITED
;
209 if (++qp
->s_cur
>= qp
->s_size
)
214 qp
->s_hdrwords
= hwords
;
215 qp
->s_cur_sge
= &qp
->s_sge
;
216 qp
->s_cur_size
= len
;
217 qib_make_ruc_header(qp
, ohdr
, bth0
| (qp
->s_state
<< 24),
218 qp
->s_psn
++ & QIB_PSN_MASK
);
222 qp
->s_flags
&= ~RVT_S_BUSY
;
227 * qib_uc_rcv - handle an incoming UC packet
228 * @ibp: the port the packet came in on
229 * @hdr: the header of the packet
230 * @has_grh: true if the packet has a GRH
231 * @data: the packet data
232 * @tlen: the length of the packet
233 * @qp: the QP for this packet.
235 * This is called from qib_qp_rcv() to process an incoming UC packet
237 * Called at interrupt level.
239 void qib_uc_rcv(struct qib_ibport
*ibp
, struct ib_header
*hdr
,
240 int has_grh
, void *data
, u32 tlen
, struct rvt_qp
*qp
)
242 struct ib_other_headers
*ohdr
;
249 struct ib_reth
*reth
;
255 hdrsize
= 8 + 12; /* LRH + BTH */
257 ohdr
= &hdr
->u
.l
.oth
;
258 hdrsize
= 8 + 40 + 12; /* LRH + GRH + BTH */
261 opcode
= be32_to_cpu(ohdr
->bth
[0]);
262 if (qib_ruc_check_hdr(ibp
, hdr
, has_grh
, qp
, opcode
))
265 psn
= be32_to_cpu(ohdr
->bth
[2]);
268 /* Compare the PSN verses the expected PSN. */
269 if (unlikely(qib_cmp24(psn
, qp
->r_psn
) != 0)) {
271 * Handle a sequence error.
272 * Silently drop any current message.
276 if (qp
->r_state
== OP(SEND_FIRST
) ||
277 qp
->r_state
== OP(SEND_MIDDLE
)) {
278 set_bit(RVT_R_REWIND_SGE
, &qp
->r_aflags
);
279 qp
->r_sge
.num_sge
= 0;
281 rvt_put_ss(&qp
->r_sge
);
282 qp
->r_state
= OP(SEND_LAST
);
286 case OP(SEND_ONLY_WITH_IMMEDIATE
):
289 case OP(RDMA_WRITE_FIRST
):
290 case OP(RDMA_WRITE_ONLY
):
291 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
):
299 /* Check for opcode sequence errors. */
300 switch (qp
->r_state
) {
302 case OP(SEND_MIDDLE
):
303 if (opcode
== OP(SEND_MIDDLE
) ||
304 opcode
== OP(SEND_LAST
) ||
305 opcode
== OP(SEND_LAST_WITH_IMMEDIATE
))
309 case OP(RDMA_WRITE_FIRST
):
310 case OP(RDMA_WRITE_MIDDLE
):
311 if (opcode
== OP(RDMA_WRITE_MIDDLE
) ||
312 opcode
== OP(RDMA_WRITE_LAST
) ||
313 opcode
== OP(RDMA_WRITE_LAST_WITH_IMMEDIATE
))
318 if (opcode
== OP(SEND_FIRST
) ||
319 opcode
== OP(SEND_ONLY
) ||
320 opcode
== OP(SEND_ONLY_WITH_IMMEDIATE
) ||
321 opcode
== OP(RDMA_WRITE_FIRST
) ||
322 opcode
== OP(RDMA_WRITE_ONLY
) ||
323 opcode
== OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
))
328 if (qp
->state
== IB_QPS_RTR
&& !(qp
->r_flags
& RVT_R_COMM_EST
))
331 /* OK, process the packet. */
335 case OP(SEND_ONLY_WITH_IMMEDIATE
):
337 if (test_and_clear_bit(RVT_R_REWIND_SGE
, &qp
->r_aflags
))
338 qp
->r_sge
= qp
->s_rdma_read_sge
;
340 ret
= qib_get_rwqe(qp
, 0);
346 * qp->s_rdma_read_sge will be the owner
347 * of the mr references.
349 qp
->s_rdma_read_sge
= qp
->r_sge
;
352 if (opcode
== OP(SEND_ONLY
))
353 goto no_immediate_data
;
354 else if (opcode
== OP(SEND_ONLY_WITH_IMMEDIATE
))
357 case OP(SEND_MIDDLE
):
358 /* Check for invalid length PMTU or posted rwqe len. */
359 if (unlikely(tlen
!= (hdrsize
+ pmtu
+ 4)))
361 qp
->r_rcv_len
+= pmtu
;
362 if (unlikely(qp
->r_rcv_len
> qp
->r_len
))
364 qib_copy_sge(&qp
->r_sge
, data
, pmtu
, 0);
367 case OP(SEND_LAST_WITH_IMMEDIATE
):
369 wc
.ex
.imm_data
= ohdr
->u
.imm_data
;
371 wc
.wc_flags
= IB_WC_WITH_IMM
;
378 /* Get the number of bytes the message was padded by. */
379 pad
= (be32_to_cpu(ohdr
->bth
[0]) >> 20) & 3;
380 /* Check for invalid length. */
381 /* XXX LAST len should be >= 1 */
382 if (unlikely(tlen
< (hdrsize
+ pad
+ 4)))
384 /* Don't count the CRC. */
385 tlen
-= (hdrsize
+ pad
+ 4);
386 wc
.byte_len
= tlen
+ qp
->r_rcv_len
;
387 if (unlikely(wc
.byte_len
> qp
->r_len
))
389 wc
.opcode
= IB_WC_RECV
;
390 qib_copy_sge(&qp
->r_sge
, data
, tlen
, 0);
391 rvt_put_ss(&qp
->s_rdma_read_sge
);
393 wc
.wr_id
= qp
->r_wr_id
;
394 wc
.status
= IB_WC_SUCCESS
;
396 wc
.src_qp
= qp
->remote_qpn
;
397 wc
.slid
= rdma_ah_get_dlid(&qp
->remote_ah_attr
);
398 wc
.sl
= rdma_ah_get_sl(&qp
->remote_ah_attr
);
399 /* zero fields that are N/A */
402 wc
.dlid_path_bits
= 0;
404 /* Signal completion event if the solicited bit is set. */
405 rvt_cq_enter(ibcq_to_rvtcq(qp
->ibqp
.recv_cq
), &wc
,
407 cpu_to_be32(IB_BTH_SOLICITED
)) != 0);
410 case OP(RDMA_WRITE_FIRST
):
411 case OP(RDMA_WRITE_ONLY
):
412 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
): /* consume RWQE */
414 if (unlikely(!(qp
->qp_access_flags
&
415 IB_ACCESS_REMOTE_WRITE
))) {
418 reth
= &ohdr
->u
.rc
.reth
;
419 hdrsize
+= sizeof(*reth
);
420 qp
->r_len
= be32_to_cpu(reth
->length
);
422 qp
->r_sge
.sg_list
= NULL
;
423 if (qp
->r_len
!= 0) {
424 u32 rkey
= be32_to_cpu(reth
->rkey
);
425 u64 vaddr
= be64_to_cpu(reth
->vaddr
);
429 ok
= rvt_rkey_ok(qp
, &qp
->r_sge
.sge
, qp
->r_len
,
430 vaddr
, rkey
, IB_ACCESS_REMOTE_WRITE
);
433 qp
->r_sge
.num_sge
= 1;
435 qp
->r_sge
.num_sge
= 0;
436 qp
->r_sge
.sge
.mr
= NULL
;
437 qp
->r_sge
.sge
.vaddr
= NULL
;
438 qp
->r_sge
.sge
.length
= 0;
439 qp
->r_sge
.sge
.sge_length
= 0;
441 if (opcode
== OP(RDMA_WRITE_ONLY
))
443 else if (opcode
== OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
)) {
444 wc
.ex
.imm_data
= ohdr
->u
.rc
.imm_data
;
448 case OP(RDMA_WRITE_MIDDLE
):
449 /* Check for invalid length PMTU or posted rwqe len. */
450 if (unlikely(tlen
!= (hdrsize
+ pmtu
+ 4)))
452 qp
->r_rcv_len
+= pmtu
;
453 if (unlikely(qp
->r_rcv_len
> qp
->r_len
))
455 qib_copy_sge(&qp
->r_sge
, data
, pmtu
, 1);
458 case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE
):
459 wc
.ex
.imm_data
= ohdr
->u
.imm_data
;
462 wc
.wc_flags
= IB_WC_WITH_IMM
;
464 /* Get the number of bytes the message was padded by. */
465 pad
= (be32_to_cpu(ohdr
->bth
[0]) >> 20) & 3;
466 /* Check for invalid length. */
467 /* XXX LAST len should be >= 1 */
468 if (unlikely(tlen
< (hdrsize
+ pad
+ 4)))
470 /* Don't count the CRC. */
471 tlen
-= (hdrsize
+ pad
+ 4);
472 if (unlikely(tlen
+ qp
->r_rcv_len
!= qp
->r_len
))
474 if (test_and_clear_bit(RVT_R_REWIND_SGE
, &qp
->r_aflags
))
475 rvt_put_ss(&qp
->s_rdma_read_sge
);
477 ret
= qib_get_rwqe(qp
, 1);
483 wc
.byte_len
= qp
->r_len
;
484 wc
.opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
485 qib_copy_sge(&qp
->r_sge
, data
, tlen
, 1);
486 rvt_put_ss(&qp
->r_sge
);
489 case OP(RDMA_WRITE_LAST
):
491 /* Get the number of bytes the message was padded by. */
492 pad
= (be32_to_cpu(ohdr
->bth
[0]) >> 20) & 3;
493 /* Check for invalid length. */
494 /* XXX LAST len should be >= 1 */
495 if (unlikely(tlen
< (hdrsize
+ pad
+ 4)))
497 /* Don't count the CRC. */
498 tlen
-= (hdrsize
+ pad
+ 4);
499 if (unlikely(tlen
+ qp
->r_rcv_len
!= qp
->r_len
))
501 qib_copy_sge(&qp
->r_sge
, data
, tlen
, 1);
502 rvt_put_ss(&qp
->r_sge
);
506 /* Drop packet for unknown opcodes. */
510 qp
->r_state
= opcode
;
514 set_bit(RVT_R_REWIND_SGE
, &qp
->r_aflags
);
515 qp
->r_sge
.num_sge
= 0;
517 ibp
->rvp
.n_pkt_drops
++;
521 rvt_rc_error(qp
, IB_WC_LOC_QP_OP_ERR
);