2 * Copyright(c) 2015, 2016 Intel Corporation.
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
49 #include "verbs_txreq.h"
52 /* cut down ridiculously long IB macro names */
53 #define OP(x) UC_OP(x)
56 * hfi1_make_uc_req - construct a request packet (SEND, RDMA write)
57 * @qp: a pointer to the QP
59 * Assume s_lock is held.
61 * Return 1 if constructed; otherwise, return 0.
63 int hfi1_make_uc_req(struct rvt_qp
*qp
, struct hfi1_pkt_state
*ps
)
65 struct hfi1_qp_priv
*priv
= qp
->priv
;
66 struct ib_other_headers
*ohdr
;
74 ps
->s_txreq
= get_txreq(ps
->dev
, qp
);
75 if (IS_ERR(ps
->s_txreq
))
78 if (!(ib_rvt_state_ops
[qp
->state
] & RVT_PROCESS_SEND_OK
)) {
79 if (!(ib_rvt_state_ops
[qp
->state
] & RVT_FLUSH_SEND
))
81 /* We are in the error state, flush the work request. */
82 smp_read_barrier_depends(); /* see post_one_send() */
83 if (qp
->s_last
== ACCESS_ONCE(qp
->s_head
))
85 /* If DMAs are in progress, we can't flush immediately. */
86 if (iowait_sdma_pending(&priv
->s_iowait
)) {
87 qp
->s_flags
|= RVT_S_WAIT_DMA
;
91 wqe
= rvt_get_swqe_ptr(qp
, qp
->s_last
);
92 hfi1_send_complete(qp
, wqe
, IB_WC_WR_FLUSH_ERR
);
96 if (priv
->hdr_type
== HFI1_PKT_TYPE_9B
) {
97 /* header size in 32-bit words LRH+BTH = (8+12)/4. */
99 if (rdma_ah_get_ah_flags(&qp
->remote_ah_attr
) & IB_AH_GRH
)
100 ohdr
= &ps
->s_txreq
->phdr
.hdr
.ibh
.u
.l
.oth
;
102 ohdr
= &ps
->s_txreq
->phdr
.hdr
.ibh
.u
.oth
;
104 /* header size in 32-bit words 16B LRH+BTH = (16+12)/4. */
106 if ((rdma_ah_get_ah_flags(&qp
->remote_ah_attr
) & IB_AH_GRH
) &&
107 (hfi1_check_mcast(rdma_ah_get_dlid(&qp
->remote_ah_attr
))))
108 ohdr
= &ps
->s_txreq
->phdr
.hdr
.opah
.u
.l
.oth
;
110 ohdr
= &ps
->s_txreq
->phdr
.hdr
.opah
.u
.oth
;
113 /* Get the next send request. */
114 wqe
= rvt_get_swqe_ptr(qp
, qp
->s_cur
);
116 switch (qp
->s_state
) {
118 if (!(ib_rvt_state_ops
[qp
->state
] &
119 RVT_PROCESS_NEXT_SEND_OK
))
121 /* Check if send work queue is empty. */
122 smp_read_barrier_depends(); /* see post_one_send() */
123 if (qp
->s_cur
== ACCESS_ONCE(qp
->s_head
)) {
128 * Local operations are processed immediately
129 * after all prior requests have completed.
131 if (wqe
->wr
.opcode
== IB_WR_REG_MR
||
132 wqe
->wr
.opcode
== IB_WR_LOCAL_INV
) {
136 if (qp
->s_last
!= qp
->s_cur
)
138 if (++qp
->s_cur
== qp
->s_size
)
140 if (!(wqe
->wr
.send_flags
& RVT_SEND_COMPLETION_ONLY
)) {
141 err
= rvt_invalidate_rkey(
142 qp
, wqe
->wr
.ex
.invalidate_rkey
);
145 hfi1_send_complete(qp
, wqe
, err
? IB_WC_LOC_PROT_ERR
148 atomic_dec(&qp
->local_ops_pending
);
153 * Start a new request.
155 qp
->s_psn
= wqe
->psn
;
156 qp
->s_sge
.sge
= wqe
->sg_list
[0];
157 qp
->s_sge
.sg_list
= wqe
->sg_list
+ 1;
158 qp
->s_sge
.num_sge
= wqe
->wr
.num_sge
;
159 qp
->s_sge
.total_len
= wqe
->length
;
162 switch (wqe
->wr
.opcode
) {
164 case IB_WR_SEND_WITH_IMM
:
166 qp
->s_state
= OP(SEND_FIRST
);
170 if (wqe
->wr
.opcode
== IB_WR_SEND
) {
171 qp
->s_state
= OP(SEND_ONLY
);
174 OP(SEND_ONLY_WITH_IMMEDIATE
);
175 /* Immediate data comes after the BTH */
176 ohdr
->u
.imm_data
= wqe
->wr
.ex
.imm_data
;
179 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
180 bth0
|= IB_BTH_SOLICITED
;
182 if (++qp
->s_cur
>= qp
->s_size
)
186 case IB_WR_RDMA_WRITE
:
187 case IB_WR_RDMA_WRITE_WITH_IMM
:
188 ohdr
->u
.rc
.reth
.vaddr
=
189 cpu_to_be64(wqe
->rdma_wr
.remote_addr
);
190 ohdr
->u
.rc
.reth
.rkey
=
191 cpu_to_be32(wqe
->rdma_wr
.rkey
);
192 ohdr
->u
.rc
.reth
.length
= cpu_to_be32(len
);
193 hwords
+= sizeof(struct ib_reth
) / 4;
195 qp
->s_state
= OP(RDMA_WRITE_FIRST
);
199 if (wqe
->wr
.opcode
== IB_WR_RDMA_WRITE
) {
200 qp
->s_state
= OP(RDMA_WRITE_ONLY
);
203 OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
);
204 /* Immediate data comes after the RETH */
205 ohdr
->u
.rc
.imm_data
= wqe
->wr
.ex
.imm_data
;
207 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
208 bth0
|= IB_BTH_SOLICITED
;
211 if (++qp
->s_cur
>= qp
->s_size
)
221 qp
->s_state
= OP(SEND_MIDDLE
);
223 case OP(SEND_MIDDLE
):
227 middle
= HFI1_CAP_IS_KSET(SDMA_AHG
);
230 if (wqe
->wr
.opcode
== IB_WR_SEND
) {
231 qp
->s_state
= OP(SEND_LAST
);
233 qp
->s_state
= OP(SEND_LAST_WITH_IMMEDIATE
);
234 /* Immediate data comes after the BTH */
235 ohdr
->u
.imm_data
= wqe
->wr
.ex
.imm_data
;
238 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
239 bth0
|= IB_BTH_SOLICITED
;
241 if (++qp
->s_cur
>= qp
->s_size
)
245 case OP(RDMA_WRITE_FIRST
):
246 qp
->s_state
= OP(RDMA_WRITE_MIDDLE
);
248 case OP(RDMA_WRITE_MIDDLE
):
252 middle
= HFI1_CAP_IS_KSET(SDMA_AHG
);
255 if (wqe
->wr
.opcode
== IB_WR_RDMA_WRITE
) {
256 qp
->s_state
= OP(RDMA_WRITE_LAST
);
259 OP(RDMA_WRITE_LAST_WITH_IMMEDIATE
);
260 /* Immediate data comes after the BTH */
261 ohdr
->u
.imm_data
= wqe
->wr
.ex
.imm_data
;
263 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
264 bth0
|= IB_BTH_SOLICITED
;
267 if (++qp
->s_cur
>= qp
->s_size
)
272 qp
->s_hdrwords
= hwords
;
273 ps
->s_txreq
->sde
= priv
->s_sde
;
274 ps
->s_txreq
->ss
= &qp
->s_sge
;
275 ps
->s_txreq
->s_cur_size
= len
;
276 hfi1_make_ruc_header(qp
, ohdr
, bth0
| (qp
->s_state
<< 24),
277 mask_psn(qp
->s_psn
++), middle
, ps
);
279 ps
->s_txreq
->hdr_dwords
= qp
->s_hdrwords
+ 2;
283 hfi1_put_txreq(ps
->s_txreq
);
288 hfi1_put_txreq(ps
->s_txreq
);
292 qp
->s_flags
&= ~RVT_S_BUSY
;
298 * hfi1_uc_rcv - handle an incoming UC packet
299 * @ibp: the port the packet came in on
300 * @hdr: the header of the packet
301 * @rcv_flags: flags relevant to rcv processing
302 * @data: the packet data
303 * @tlen: the length of the packet
304 * @qp: the QP for this packet.
306 * This is called from qp_rcv() to process an incoming UC packet
308 * Called at interrupt level.
310 void hfi1_uc_rcv(struct hfi1_packet
*packet
)
312 struct hfi1_ibport
*ibp
= rcd_to_iport(packet
->rcd
);
313 void *data
= packet
->payload
;
314 u32 tlen
= packet
->tlen
;
315 struct rvt_qp
*qp
= packet
->qp
;
316 struct ib_other_headers
*ohdr
= packet
->ohdr
;
317 u32 opcode
= packet
->opcode
;
318 u32 hdrsize
= packet
->hlen
;
320 u32 pad
= packet
->pad
;
323 struct ib_reth
*reth
;
325 u8 extra_bytes
= pad
+ packet
->extra_byte
+ (SIZE_OF_CRC
<< 2);
327 if (hfi1_ruc_check_hdr(ibp
, packet
))
330 process_ecn(qp
, packet
, true);
332 psn
= ib_bth_get_psn(ohdr
);
333 /* Compare the PSN verses the expected PSN. */
334 if (unlikely(cmp_psn(psn
, qp
->r_psn
) != 0)) {
336 * Handle a sequence error.
337 * Silently drop any current message.
341 if (qp
->r_state
== OP(SEND_FIRST
) ||
342 qp
->r_state
== OP(SEND_MIDDLE
)) {
343 set_bit(RVT_R_REWIND_SGE
, &qp
->r_aflags
);
344 qp
->r_sge
.num_sge
= 0;
346 rvt_put_ss(&qp
->r_sge
);
348 qp
->r_state
= OP(SEND_LAST
);
352 case OP(SEND_ONLY_WITH_IMMEDIATE
):
355 case OP(RDMA_WRITE_FIRST
):
356 case OP(RDMA_WRITE_ONLY
):
357 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
):
365 /* Check for opcode sequence errors. */
366 switch (qp
->r_state
) {
368 case OP(SEND_MIDDLE
):
369 if (opcode
== OP(SEND_MIDDLE
) ||
370 opcode
== OP(SEND_LAST
) ||
371 opcode
== OP(SEND_LAST_WITH_IMMEDIATE
))
375 case OP(RDMA_WRITE_FIRST
):
376 case OP(RDMA_WRITE_MIDDLE
):
377 if (opcode
== OP(RDMA_WRITE_MIDDLE
) ||
378 opcode
== OP(RDMA_WRITE_LAST
) ||
379 opcode
== OP(RDMA_WRITE_LAST_WITH_IMMEDIATE
))
384 if (opcode
== OP(SEND_FIRST
) ||
385 opcode
== OP(SEND_ONLY
) ||
386 opcode
== OP(SEND_ONLY_WITH_IMMEDIATE
) ||
387 opcode
== OP(RDMA_WRITE_FIRST
) ||
388 opcode
== OP(RDMA_WRITE_ONLY
) ||
389 opcode
== OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
))
394 if (qp
->state
== IB_QPS_RTR
&& !(qp
->r_flags
& RVT_R_COMM_EST
))
397 /* OK, process the packet. */
401 case OP(SEND_ONLY_WITH_IMMEDIATE
):
403 if (test_and_clear_bit(RVT_R_REWIND_SGE
, &qp
->r_aflags
)) {
404 qp
->r_sge
= qp
->s_rdma_read_sge
;
406 ret
= hfi1_rvt_get_rwqe(qp
, 0);
412 * qp->s_rdma_read_sge will be the owner
413 * of the mr references.
415 qp
->s_rdma_read_sge
= qp
->r_sge
;
418 if (opcode
== OP(SEND_ONLY
))
419 goto no_immediate_data
;
420 else if (opcode
== OP(SEND_ONLY_WITH_IMMEDIATE
))
423 case OP(SEND_MIDDLE
):
424 /* Check for invalid length PMTU or posted rwqe len. */
426 * There will be no padding for 9B packet but 16B packets
427 * will come in with some padding since we always add
428 * CRC and LT bytes which will need to be flit aligned
430 if (unlikely(tlen
!= (hdrsize
+ pmtu
+ extra_bytes
)))
432 qp
->r_rcv_len
+= pmtu
;
433 if (unlikely(qp
->r_rcv_len
> qp
->r_len
))
435 hfi1_copy_sge(&qp
->r_sge
, data
, pmtu
, false, false);
438 case OP(SEND_LAST_WITH_IMMEDIATE
):
440 wc
.ex
.imm_data
= ohdr
->u
.imm_data
;
441 wc
.wc_flags
= IB_WC_WITH_IMM
;
448 /* Check for invalid length. */
449 /* LAST len should be >= 1 */
450 if (unlikely(tlen
< (hdrsize
+ extra_bytes
)))
452 /* Don't count the CRC. */
453 tlen
-= (hdrsize
+ extra_bytes
);
454 wc
.byte_len
= tlen
+ qp
->r_rcv_len
;
455 if (unlikely(wc
.byte_len
> qp
->r_len
))
457 wc
.opcode
= IB_WC_RECV
;
458 hfi1_copy_sge(&qp
->r_sge
, data
, tlen
, false, false);
459 rvt_put_ss(&qp
->s_rdma_read_sge
);
461 wc
.wr_id
= qp
->r_wr_id
;
462 wc
.status
= IB_WC_SUCCESS
;
464 wc
.src_qp
= qp
->remote_qpn
;
465 wc
.slid
= rdma_ah_get_dlid(&qp
->remote_ah_attr
) & U16_MAX
;
467 * It seems that IB mandates the presence of an SL in a
468 * work completion only for the UD transport (see section
469 * 11.4.2 of IBTA Vol. 1).
471 * However, the way the SL is chosen below is consistent
472 * with the way that IB/qib works and is trying avoid
473 * introducing incompatibilities.
475 * See also OPA Vol. 1, section 9.7.6, and table 9-17.
477 wc
.sl
= rdma_ah_get_sl(&qp
->remote_ah_attr
);
478 /* zero fields that are N/A */
481 wc
.dlid_path_bits
= 0;
483 /* Signal completion event if the solicited bit is set. */
484 rvt_cq_enter(ibcq_to_rvtcq(qp
->ibqp
.recv_cq
), &wc
,
486 cpu_to_be32(IB_BTH_SOLICITED
)) != 0);
489 case OP(RDMA_WRITE_FIRST
):
490 case OP(RDMA_WRITE_ONLY
):
491 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
): /* consume RWQE */
493 if (unlikely(!(qp
->qp_access_flags
&
494 IB_ACCESS_REMOTE_WRITE
))) {
497 reth
= &ohdr
->u
.rc
.reth
;
498 qp
->r_len
= be32_to_cpu(reth
->length
);
500 qp
->r_sge
.sg_list
= NULL
;
501 if (qp
->r_len
!= 0) {
502 u32 rkey
= be32_to_cpu(reth
->rkey
);
503 u64 vaddr
= be64_to_cpu(reth
->vaddr
);
507 ok
= rvt_rkey_ok(qp
, &qp
->r_sge
.sge
, qp
->r_len
,
508 vaddr
, rkey
, IB_ACCESS_REMOTE_WRITE
);
511 qp
->r_sge
.num_sge
= 1;
513 qp
->r_sge
.num_sge
= 0;
514 qp
->r_sge
.sge
.mr
= NULL
;
515 qp
->r_sge
.sge
.vaddr
= NULL
;
516 qp
->r_sge
.sge
.length
= 0;
517 qp
->r_sge
.sge
.sge_length
= 0;
519 if (opcode
== OP(RDMA_WRITE_ONLY
)) {
521 } else if (opcode
== OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE
)) {
522 wc
.ex
.imm_data
= ohdr
->u
.rc
.imm_data
;
526 case OP(RDMA_WRITE_MIDDLE
):
527 /* Check for invalid length PMTU or posted rwqe len. */
528 if (unlikely(tlen
!= (hdrsize
+ pmtu
+ 4)))
530 qp
->r_rcv_len
+= pmtu
;
531 if (unlikely(qp
->r_rcv_len
> qp
->r_len
))
533 hfi1_copy_sge(&qp
->r_sge
, data
, pmtu
, true, false);
536 case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE
):
537 wc
.ex
.imm_data
= ohdr
->u
.imm_data
;
539 wc
.wc_flags
= IB_WC_WITH_IMM
;
541 /* Check for invalid length. */
542 /* LAST len should be >= 1 */
543 if (unlikely(tlen
< (hdrsize
+ pad
+ 4)))
545 /* Don't count the CRC. */
546 tlen
-= (hdrsize
+ extra_bytes
);
547 if (unlikely(tlen
+ qp
->r_rcv_len
!= qp
->r_len
))
549 if (test_and_clear_bit(RVT_R_REWIND_SGE
, &qp
->r_aflags
)) {
550 rvt_put_ss(&qp
->s_rdma_read_sge
);
552 ret
= hfi1_rvt_get_rwqe(qp
, 1);
558 wc
.byte_len
= qp
->r_len
;
559 wc
.opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
560 hfi1_copy_sge(&qp
->r_sge
, data
, tlen
, true, false);
561 rvt_put_ss(&qp
->r_sge
);
564 case OP(RDMA_WRITE_LAST
):
566 /* Check for invalid length. */
567 /* LAST len should be >= 1 */
568 if (unlikely(tlen
< (hdrsize
+ pad
+ 4)))
570 /* Don't count the CRC. */
571 tlen
-= (hdrsize
+ extra_bytes
);
572 if (unlikely(tlen
+ qp
->r_rcv_len
!= qp
->r_len
))
574 hfi1_copy_sge(&qp
->r_sge
, data
, tlen
, true, false);
575 rvt_put_ss(&qp
->r_sge
);
579 /* Drop packet for unknown opcodes. */
583 qp
->r_state
= opcode
;
587 set_bit(RVT_R_REWIND_SGE
, &qp
->r_aflags
);
588 qp
->r_sge
.num_sge
= 0;
590 ibp
->rvp
.n_pkt_drops
++;
594 rvt_rc_error(qp
, IB_WC_LOC_QP_OP_ERR
);