]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/infiniband/hw/hfi1/ruc.c
IB/hfi1: Use defines from common headers
[mirror_ubuntu-jammy-kernel.git] / drivers / infiniband / hw / hfi1 / ruc.c
CommitLineData
77241056 1/*
05d6ac1d 2 * Copyright(c) 2015, 2016 Intel Corporation.
77241056
MM
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
77241056
MM
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
77241056
MM
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47
48#include <linux/spinlock.h>
49
50#include "hfi.h"
51#include "mad.h"
52#include "qp.h"
45842abb 53#include "verbs_txreq.h"
bb5df5f9 54#include "trace.h"
77241056 55
77241056
MM
56/*
57 * Validate a RWQE and fill in the SGE state.
58 * Return 1 if OK.
59 */
895420dd 60static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe)
77241056
MM
61{
62 int i, j, ret;
63 struct ib_wc wc;
cd4ceee3 64 struct rvt_lkey_table *rkt;
4f87ccfc 65 struct rvt_pd *pd;
895420dd 66 struct rvt_sge_state *ss;
77241056 67
895420dd 68 rkt = &to_idev(qp->ibqp.device)->rdi.lkey_table;
4f87ccfc 69 pd = ibpd_to_rvtpd(qp->ibqp.srq ? qp->ibqp.srq->pd : qp->ibqp.pd);
77241056
MM
70 ss = &qp->r_sge;
71 ss->sg_list = qp->r_sg_list;
72 qp->r_len = 0;
73 for (i = j = 0; i < wqe->num_sge; i++) {
74 if (wqe->sg_list[i].length == 0)
75 continue;
76 /* Check LKEY */
895420dd
DD
77 if (!rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge,
78 &wqe->sg_list[i], IB_ACCESS_LOCAL_WRITE))
77241056
MM
79 goto bad_lkey;
80 qp->r_len += wqe->sg_list[i].length;
81 j++;
82 }
83 ss->num_sge = j;
84 ss->total_len = qp->r_len;
85 ret = 1;
86 goto bail;
87
88bad_lkey:
89 while (j) {
895420dd 90 struct rvt_sge *sge = --j ? &ss->sg_list[j - 1] : &ss->sge;
77241056 91
895420dd 92 rvt_put_mr(sge->mr);
77241056
MM
93 }
94 ss->num_sge = 0;
95 memset(&wc, 0, sizeof(wc));
96 wc.wr_id = wqe->wr_id;
97 wc.status = IB_WC_LOC_PROT_ERR;
98 wc.opcode = IB_WC_RECV;
99 wc.qp = &qp->ibqp;
100 /* Signal solicited completion event. */
abd712da 101 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1);
77241056
MM
102 ret = 0;
103bail:
104 return ret;
105}
106
107/**
ec4274f1 108 * hfi1_rvt_get_rwqe - copy the next RWQE into the QP's RWQE
77241056
MM
109 * @qp: the QP
110 * @wr_id_only: update qp->r_wr_id only, not qp->r_sge
111 *
112 * Return -1 if there is a local error, 0 if no RWQE is available,
113 * otherwise return 1.
114 *
115 * Can be called from interrupt level.
116 */
ec4274f1 117int hfi1_rvt_get_rwqe(struct rvt_qp *qp, int wr_id_only)
77241056
MM
118{
119 unsigned long flags;
895420dd
DD
120 struct rvt_rq *rq;
121 struct rvt_rwq *wq;
39db3e66 122 struct rvt_srq *srq;
895420dd 123 struct rvt_rwqe *wqe;
77241056
MM
124 void (*handler)(struct ib_event *, void *);
125 u32 tail;
126 int ret;
127
128 if (qp->ibqp.srq) {
39db3e66 129 srq = ibsrq_to_rvtsrq(qp->ibqp.srq);
77241056
MM
130 handler = srq->ibsrq.event_handler;
131 rq = &srq->rq;
132 } else {
133 srq = NULL;
134 handler = NULL;
135 rq = &qp->r_rq;
136 }
137
138 spin_lock_irqsave(&rq->lock, flags);
83693bd1 139 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
77241056
MM
140 ret = 0;
141 goto unlock;
142 }
143
144 wq = rq->wq;
145 tail = wq->tail;
146 /* Validate tail before using it since it is user writable. */
147 if (tail >= rq->size)
148 tail = 0;
149 if (unlikely(tail == wq->head)) {
150 ret = 0;
151 goto unlock;
152 }
153 /* Make sure entry is read after head index is read. */
154 smp_rmb();
ec4274f1 155 wqe = rvt_get_rwqe_ptr(rq, tail);
77241056
MM
156 /*
157 * Even though we update the tail index in memory, the verbs
158 * consumer is not supposed to post more entries until a
159 * completion is generated.
160 */
161 if (++tail >= rq->size)
162 tail = 0;
163 wq->tail = tail;
164 if (!wr_id_only && !init_sge(qp, wqe)) {
165 ret = -1;
166 goto unlock;
167 }
168 qp->r_wr_id = wqe->wr_id;
169
170 ret = 1;
54d10c1e 171 set_bit(RVT_R_WRID_VALID, &qp->r_aflags);
77241056
MM
172 if (handler) {
173 u32 n;
174
175 /*
176 * Validate head pointer value and compute
177 * the number of remaining WQEs.
178 */
179 n = wq->head;
180 if (n >= rq->size)
181 n = 0;
182 if (n < tail)
183 n += rq->size - tail;
184 else
185 n -= tail;
186 if (n < srq->limit) {
187 struct ib_event ev;
188
189 srq->limit = 0;
190 spin_unlock_irqrestore(&rq->lock, flags);
191 ev.device = qp->ibqp.device;
192 ev.element.srq = qp->ibqp.srq;
193 ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
194 handler(&ev, srq->ibsrq.srq_context);
195 goto bail;
196 }
197 }
198unlock:
199 spin_unlock_irqrestore(&rq->lock, flags);
200bail:
201 return ret;
202}
203
77241056
MM
204static int gid_ok(union ib_gid *gid, __be64 gid_prefix, __be64 id)
205{
206 return (gid->global.interface_id == id &&
207 (gid->global.subnet_prefix == gid_prefix ||
208 gid->global.subnet_prefix == IB_DEFAULT_GID_PREFIX));
209}
210
211/*
212 *
213 * This should be called with the QP r_lock held.
214 *
215 * The s_lock will be acquired around the hfi1_migrate_qp() call.
216 */
261a4351 217int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct ib_header *hdr,
895420dd 218 int has_grh, struct rvt_qp *qp, u32 bth0)
77241056
MM
219{
220 __be64 guid;
221 unsigned long flags;
222 u8 sc5 = ibp->sl_to_sc[qp->remote_ah_attr.sl];
223
224 if (qp->s_mig_state == IB_MIG_ARMED && (bth0 & IB_BTH_MIG_REQ)) {
225 if (!has_grh) {
226 if (qp->alt_ah_attr.ah_flags & IB_AH_GRH)
227 goto err;
228 } else {
229 if (!(qp->alt_ah_attr.ah_flags & IB_AH_GRH))
230 goto err;
231 guid = get_sguid(ibp, qp->alt_ah_attr.grh.sgid_index);
4eb06882
DD
232 if (!gid_ok(&hdr->u.l.grh.dgid, ibp->rvp.gid_prefix,
233 guid))
77241056 234 goto err;
17fb4f29
JJ
235 if (!gid_ok(
236 &hdr->u.l.grh.sgid,
237 qp->alt_ah_attr.grh.dgid.global.subnet_prefix,
238 qp->alt_ah_attr.grh.dgid.global.interface_id))
77241056
MM
239 goto err;
240 }
cb427057
DH
241 if (unlikely(rcv_pkey_check(ppd_from_ibp(ibp), (u16)bth0, sc5,
242 ib_get_slid(hdr)))) {
5cd24119 243 hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_P_KEY,
77241056 244 (u16)bth0,
cb427057 245 ib_get_sl(hdr),
77241056 246 0, qp->ibqp.qp_num,
cb427057
DH
247 ib_get_slid(hdr),
248 ib_get_dlid(hdr));
77241056
MM
249 goto err;
250 }
251 /* Validate the SLID. See Ch. 9.6.1.5 and 17.2.8 */
cb427057 252 if (ib_get_slid(hdr) != qp->alt_ah_attr.dlid ||
77241056
MM
253 ppd_from_ibp(ibp)->port != qp->alt_ah_attr.port_num)
254 goto err;
255 spin_lock_irqsave(&qp->s_lock, flags);
256 hfi1_migrate_qp(qp);
257 spin_unlock_irqrestore(&qp->s_lock, flags);
258 } else {
259 if (!has_grh) {
260 if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
261 goto err;
262 } else {
263 if (!(qp->remote_ah_attr.ah_flags & IB_AH_GRH))
264 goto err;
265 guid = get_sguid(ibp,
266 qp->remote_ah_attr.grh.sgid_index);
4eb06882
DD
267 if (!gid_ok(&hdr->u.l.grh.dgid, ibp->rvp.gid_prefix,
268 guid))
77241056 269 goto err;
17fb4f29
JJ
270 if (!gid_ok(
271 &hdr->u.l.grh.sgid,
272 qp->remote_ah_attr.grh.dgid.global.subnet_prefix,
273 qp->remote_ah_attr.grh.dgid.global.interface_id))
77241056
MM
274 goto err;
275 }
cb427057
DH
276 if (unlikely(rcv_pkey_check(ppd_from_ibp(ibp), (u16)bth0, sc5,
277 ib_get_slid(hdr)))) {
5cd24119 278 hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_P_KEY,
77241056 279 (u16)bth0,
cb427057 280 ib_get_sl(hdr),
77241056 281 0, qp->ibqp.qp_num,
cb427057
DH
282 ib_get_slid(hdr),
283 ib_get_dlid(hdr));
77241056
MM
284 goto err;
285 }
286 /* Validate the SLID. See Ch. 9.6.1.5 */
cb427057 287 if (ib_get_slid(hdr) != qp->remote_ah_attr.dlid ||
77241056
MM
288 ppd_from_ibp(ibp)->port != qp->port_num)
289 goto err;
290 if (qp->s_mig_state == IB_MIG_REARM &&
291 !(bth0 & IB_BTH_MIG_REQ))
292 qp->s_mig_state = IB_MIG_ARMED;
293 }
294
295 return 0;
296
297err:
298 return 1;
299}
300
301/**
302 * ruc_loopback - handle UC and RC loopback requests
303 * @sqp: the sending QP
304 *
305 * This is called from hfi1_do_send() to
306 * forward a WQE addressed to the same HFI.
ca00c62b 307 * Note that although we are single threaded due to the send engine, we still
77241056
MM
308 * have to protect against post_send(). We don't have to worry about
309 * receive interrupts since this is a connected protocol and all packets
310 * will pass through here.
311 */
895420dd 312static void ruc_loopback(struct rvt_qp *sqp)
77241056
MM
313{
314 struct hfi1_ibport *ibp = to_iport(sqp->ibqp.device, sqp->port_num);
895420dd
DD
315 struct rvt_qp *qp;
316 struct rvt_swqe *wqe;
317 struct rvt_sge *sge;
77241056
MM
318 unsigned long flags;
319 struct ib_wc wc;
320 u64 sdata;
321 atomic64_t *maddr;
322 enum ib_wc_status send_status;
0128fcea 323 bool release;
77241056 324 int ret;
0128fcea 325 bool copy_last = false;
0db3dfa0 326 int local_ops = 0;
77241056
MM
327
328 rcu_read_lock();
329
330 /*
331 * Note that we check the responder QP state after
332 * checking the requester's state.
333 */
ec4274f1
DD
334 qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), &ibp->rvp,
335 sqp->remote_qpn);
77241056
MM
336
337 spin_lock_irqsave(&sqp->s_lock, flags);
338
339 /* Return if we are already busy processing a work request. */
54d10c1e 340 if ((sqp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT)) ||
83693bd1 341 !(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_OR_FLUSH_SEND))
77241056
MM
342 goto unlock;
343
54d10c1e 344 sqp->s_flags |= RVT_S_BUSY;
77241056
MM
345
346again:
46a80d62
MM
347 smp_read_barrier_depends(); /* see post_one_send() */
348 if (sqp->s_last == ACCESS_ONCE(sqp->s_head))
77241056 349 goto clr_busy;
83693bd1 350 wqe = rvt_get_swqe_ptr(sqp, sqp->s_last);
77241056
MM
351
352 /* Return if it is not OK to start a new work request. */
83693bd1
DD
353 if (!(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_NEXT_SEND_OK)) {
354 if (!(ib_rvt_state_ops[sqp->state] & RVT_FLUSH_SEND))
77241056
MM
355 goto clr_busy;
356 /* We are in the error state, flush the work request. */
357 send_status = IB_WC_WR_FLUSH_ERR;
358 goto flush_send;
359 }
360
361 /*
362 * We can rely on the entry not changing without the s_lock
363 * being held until we update s_last.
364 * We increment s_cur to indicate s_last is in progress.
365 */
366 if (sqp->s_last == sqp->s_cur) {
367 if (++sqp->s_cur >= sqp->s_size)
368 sqp->s_cur = 0;
369 }
370 spin_unlock_irqrestore(&sqp->s_lock, flags);
371
83693bd1 372 if (!qp || !(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) ||
77241056 373 qp->ibqp.qp_type != sqp->ibqp.qp_type) {
4eb06882 374 ibp->rvp.n_pkt_drops++;
77241056
MM
375 /*
376 * For RC, the requester would timeout and retry so
377 * shortcut the timeouts and just signal too many retries.
378 */
379 if (sqp->ibqp.qp_type == IB_QPT_RC)
380 send_status = IB_WC_RETRY_EXC_ERR;
381 else
382 send_status = IB_WC_SUCCESS;
383 goto serr;
384 }
385
386 memset(&wc, 0, sizeof(wc));
387 send_status = IB_WC_SUCCESS;
388
0128fcea 389 release = true;
77241056
MM
390 sqp->s_sge.sge = wqe->sg_list[0];
391 sqp->s_sge.sg_list = wqe->sg_list + 1;
392 sqp->s_sge.num_sge = wqe->wr.num_sge;
393 sqp->s_len = wqe->length;
394 switch (wqe->wr.opcode) {
0db3dfa0 395 case IB_WR_REG_MR:
0db3dfa0
JX
396 goto send_comp;
397
398 case IB_WR_LOCAL_INV:
d9b13c20
JX
399 if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {
400 if (rvt_invalidate_rkey(sqp,
401 wqe->wr.ex.invalidate_rkey))
402 send_status = IB_WC_LOC_PROT_ERR;
403 local_ops = 1;
404 }
0db3dfa0
JX
405 goto send_comp;
406
407 case IB_WR_SEND_WITH_INV:
408 if (!rvt_invalidate_rkey(qp, wqe->wr.ex.invalidate_rkey)) {
409 wc.wc_flags = IB_WC_WITH_INVALIDATE;
410 wc.ex.invalidate_rkey = wqe->wr.ex.invalidate_rkey;
411 }
412 goto send;
413
77241056
MM
414 case IB_WR_SEND_WITH_IMM:
415 wc.wc_flags = IB_WC_WITH_IMM;
416 wc.ex.imm_data = wqe->wr.ex.imm_data;
417 /* FALLTHROUGH */
418 case IB_WR_SEND:
0db3dfa0 419send:
ec4274f1 420 ret = hfi1_rvt_get_rwqe(qp, 0);
77241056
MM
421 if (ret < 0)
422 goto op_err;
423 if (!ret)
424 goto rnr_nak;
425 break;
426
427 case IB_WR_RDMA_WRITE_WITH_IMM:
428 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
429 goto inv_err;
430 wc.wc_flags = IB_WC_WITH_IMM;
431 wc.ex.imm_data = wqe->wr.ex.imm_data;
ec4274f1 432 ret = hfi1_rvt_get_rwqe(qp, 1);
77241056
MM
433 if (ret < 0)
434 goto op_err;
435 if (!ret)
436 goto rnr_nak;
7b0b01aa
DL
437 /* skip copy_last set and qp_access_flags recheck */
438 goto do_write;
77241056 439 case IB_WR_RDMA_WRITE:
0128fcea 440 copy_last = rvt_is_user_qp(qp);
77241056
MM
441 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
442 goto inv_err;
7b0b01aa 443do_write:
77241056 444 if (wqe->length == 0)
42d6ec19 445 break;
895420dd
DD
446 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, wqe->length,
447 wqe->rdma_wr.remote_addr,
448 wqe->rdma_wr.rkey,
449 IB_ACCESS_REMOTE_WRITE)))
77241056
MM
450 goto acc_err;
451 qp->r_sge.sg_list = NULL;
452 qp->r_sge.num_sge = 1;
453 qp->r_sge.total_len = wqe->length;
454 break;
455
456 case IB_WR_RDMA_READ:
457 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
458 goto inv_err;
895420dd
DD
459 if (unlikely(!rvt_rkey_ok(qp, &sqp->s_sge.sge, wqe->length,
460 wqe->rdma_wr.remote_addr,
461 wqe->rdma_wr.rkey,
462 IB_ACCESS_REMOTE_READ)))
77241056 463 goto acc_err;
0128fcea 464 release = false;
77241056
MM
465 sqp->s_sge.sg_list = NULL;
466 sqp->s_sge.num_sge = 1;
467 qp->r_sge.sge = wqe->sg_list[0];
468 qp->r_sge.sg_list = wqe->sg_list + 1;
469 qp->r_sge.num_sge = wqe->wr.num_sge;
470 qp->r_sge.total_len = wqe->length;
471 break;
472
473 case IB_WR_ATOMIC_CMP_AND_SWP:
474 case IB_WR_ATOMIC_FETCH_AND_ADD:
475 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
476 goto inv_err;
895420dd
DD
477 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
478 wqe->atomic_wr.remote_addr,
479 wqe->atomic_wr.rkey,
480 IB_ACCESS_REMOTE_ATOMIC)))
77241056
MM
481 goto acc_err;
482 /* Perform atomic OP and save result. */
50e5dcbe 483 maddr = (atomic64_t *)qp->r_sge.sge.vaddr;
e622f2f4 484 sdata = wqe->atomic_wr.compare_add;
50e5dcbe 485 *(u64 *)sqp->s_sge.sge.vaddr =
77241056 486 (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ?
50e5dcbe
JJ
487 (u64)atomic64_add_return(sdata, maddr) - sdata :
488 (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr,
e622f2f4 489 sdata, wqe->atomic_wr.swap);
895420dd 490 rvt_put_mr(qp->r_sge.sge.mr);
77241056
MM
491 qp->r_sge.num_sge = 0;
492 goto send_comp;
493
494 default:
495 send_status = IB_WC_LOC_QP_OP_ERR;
496 goto serr;
497 }
498
499 sge = &sqp->s_sge.sge;
500 while (sqp->s_len) {
501 u32 len = sqp->s_len;
502
503 if (len > sge->length)
504 len = sge->length;
505 if (len > sge->sge_length)
506 len = sge->sge_length;
507 WARN_ON_ONCE(len == 0);
7b0b01aa 508 hfi1_copy_sge(&qp->r_sge, sge->vaddr, len, release, copy_last);
77241056
MM
509 sge->vaddr += len;
510 sge->length -= len;
511 sge->sge_length -= len;
512 if (sge->sge_length == 0) {
513 if (!release)
895420dd 514 rvt_put_mr(sge->mr);
77241056
MM
515 if (--sqp->s_sge.num_sge)
516 *sge = *sqp->s_sge.sg_list++;
517 } else if (sge->length == 0 && sge->mr->lkey) {
cd4ceee3 518 if (++sge->n >= RVT_SEGSZ) {
77241056
MM
519 if (++sge->m >= sge->mr->mapsz)
520 break;
521 sge->n = 0;
522 }
523 sge->vaddr =
524 sge->mr->map[sge->m]->segs[sge->n].vaddr;
525 sge->length =
526 sge->mr->map[sge->m]->segs[sge->n].length;
527 }
528 sqp->s_len -= len;
529 }
530 if (release)
ec4274f1 531 rvt_put_ss(&qp->r_sge);
77241056 532
54d10c1e 533 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
77241056
MM
534 goto send_comp;
535
536 if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM)
537 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
538 else
539 wc.opcode = IB_WC_RECV;
540 wc.wr_id = qp->r_wr_id;
541 wc.status = IB_WC_SUCCESS;
542 wc.byte_len = wqe->length;
543 wc.qp = &qp->ibqp;
544 wc.src_qp = qp->remote_qpn;
545 wc.slid = qp->remote_ah_attr.dlid;
546 wc.sl = qp->remote_ah_attr.sl;
547 wc.port_num = 1;
548 /* Signal completion event if the solicited bit is set. */
abd712da
DD
549 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
550 wqe->wr.send_flags & IB_SEND_SOLICITED);
77241056
MM
551
552send_comp:
553 spin_lock_irqsave(&sqp->s_lock, flags);
4eb06882 554 ibp->rvp.n_loop_pkts++;
77241056
MM
555flush_send:
556 sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
557 hfi1_send_complete(sqp, wqe, send_status);
0db3dfa0
JX
558 if (local_ops) {
559 atomic_dec(&sqp->local_ops_pending);
560 local_ops = 0;
561 }
77241056
MM
562 goto again;
563
564rnr_nak:
565 /* Handle RNR NAK */
566 if (qp->ibqp.qp_type == IB_QPT_UC)
567 goto send_comp;
4eb06882 568 ibp->rvp.n_rnr_naks++;
77241056
MM
569 /*
570 * Note: we don't need the s_lock held since the BUSY flag
571 * makes this single threaded.
572 */
573 if (sqp->s_rnr_retry == 0) {
574 send_status = IB_WC_RNR_RETRY_EXC_ERR;
575 goto serr;
576 }
577 if (sqp->s_rnr_retry_cnt < 7)
578 sqp->s_rnr_retry--;
579 spin_lock_irqsave(&sqp->s_lock, flags);
83693bd1 580 if (!(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_RECV_OK))
77241056 581 goto clr_busy;
56acbbfb 582 rvt_add_rnr_timer(sqp, qp->r_min_rnr_timer <<
832666c1 583 IB_AETH_CREDIT_SHIFT);
77241056
MM
584 goto clr_busy;
585
586op_err:
587 send_status = IB_WC_REM_OP_ERR;
588 wc.status = IB_WC_LOC_QP_OP_ERR;
589 goto err;
590
591inv_err:
592 send_status = IB_WC_REM_INV_REQ_ERR;
593 wc.status = IB_WC_LOC_QP_OP_ERR;
594 goto err;
595
596acc_err:
597 send_status = IB_WC_REM_ACCESS_ERR;
598 wc.status = IB_WC_LOC_PROT_ERR;
599err:
600 /* responder goes to error state */
beb5a042 601 rvt_rc_error(qp, wc.status);
77241056
MM
602
603serr:
604 spin_lock_irqsave(&sqp->s_lock, flags);
605 hfi1_send_complete(sqp, wqe, send_status);
606 if (sqp->ibqp.qp_type == IB_QPT_RC) {
ec4274f1 607 int lastwqe = rvt_error_qp(sqp, IB_WC_WR_FLUSH_ERR);
77241056 608
54d10c1e 609 sqp->s_flags &= ~RVT_S_BUSY;
77241056
MM
610 spin_unlock_irqrestore(&sqp->s_lock, flags);
611 if (lastwqe) {
612 struct ib_event ev;
613
614 ev.device = sqp->ibqp.device;
615 ev.element.qp = &sqp->ibqp;
616 ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
617 sqp->ibqp.event_handler(&ev, sqp->ibqp.qp_context);
618 }
619 goto done;
620 }
621clr_busy:
54d10c1e 622 sqp->s_flags &= ~RVT_S_BUSY;
77241056
MM
623unlock:
624 spin_unlock_irqrestore(&sqp->s_lock, flags);
625done:
626 rcu_read_unlock();
627}
628
629/**
630 * hfi1_make_grh - construct a GRH header
631 * @ibp: a pointer to the IB port
632 * @hdr: a pointer to the GRH header being constructed
633 * @grh: the global route address to send to
634 * @hwords: the number of 32 bit words of header being sent
635 * @nwords: the number of 32 bit words of data being sent
636 *
637 * Return the size of the header in 32 bit words.
638 */
639u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,
640 struct ib_global_route *grh, u32 hwords, u32 nwords)
641{
642 hdr->version_tclass_flow =
643 cpu_to_be32((IB_GRH_VERSION << IB_GRH_VERSION_SHIFT) |
644 (grh->traffic_class << IB_GRH_TCLASS_SHIFT) |
645 (grh->flow_label << IB_GRH_FLOW_SHIFT));
646 hdr->paylen = cpu_to_be16((hwords - 2 + nwords + SIZE_OF_CRC) << 2);
647 /* next_hdr is defined by C8-7 in ch. 8.4.1 */
648 hdr->next_hdr = IB_GRH_NEXT_HDR;
649 hdr->hop_limit = grh->hop_limit;
650 /* The SGID is 32-bit aligned. */
4eb06882 651 hdr->sgid.global.subnet_prefix = ibp->rvp.gid_prefix;
77241056 652 hdr->sgid.global.interface_id =
a6cd5f08
JP
653 grh->sgid_index < HFI1_GUIDS_PER_PORT ?
654 get_sguid(ibp, grh->sgid_index) :
655 get_sguid(ibp, HFI1_PORT_GUID_INDEX);
77241056
MM
656 hdr->dgid = grh->dgid;
657
658 /* GRH header size in 32-bit words. */
659 return sizeof(struct ib_grh) / sizeof(u32);
660}
661
d4d602e9 662#define BTH2_OFFSET (offsetof(struct hfi1_sdma_header, hdr.u.oth.bth[2]) / 4)
77241056
MM
663
664/**
a9b6b3bc 665 * build_ahg - create ahg in s_ahg
77241056
MM
666 * @qp: a pointer to QP
667 * @npsn: the next PSN for the request/response
668 *
669 * This routine handles the AHG by allocating an ahg entry and causing the
670 * copy of the first middle.
671 *
672 * Subsequent middles use the copied entry, editing the
673 * PSN with 1 or 2 edits.
674 */
895420dd 675static inline void build_ahg(struct rvt_qp *qp, u32 npsn)
77241056 676{
4c6829c5 677 struct hfi1_qp_priv *priv = qp->priv;
bb5df5f9 678
54d10c1e 679 if (unlikely(qp->s_flags & RVT_S_AHG_CLEAR))
77241056 680 clear_ahg(qp);
54d10c1e 681 if (!(qp->s_flags & RVT_S_AHG_VALID)) {
77241056 682 /* first middle that needs copy */
d7b8ba51 683 if (qp->s_ahgidx < 0)
4c6829c5 684 qp->s_ahgidx = sdma_ahg_alloc(priv->s_sde);
77241056
MM
685 if (qp->s_ahgidx >= 0) {
686 qp->s_ahgpsn = npsn;
a9b6b3bc 687 priv->s_ahg->tx_flags |= SDMA_TXREQ_F_AHG_COPY;
77241056 688 /* save to protect a change in another thread */
a9b6b3bc 689 priv->s_ahg->ahgidx = qp->s_ahgidx;
54d10c1e 690 qp->s_flags |= RVT_S_AHG_VALID;
77241056
MM
691 }
692 } else {
693 /* subsequent middle after valid */
694 if (qp->s_ahgidx >= 0) {
a9b6b3bc
DC
695 priv->s_ahg->tx_flags |= SDMA_TXREQ_F_USE_AHG;
696 priv->s_ahg->ahgidx = qp->s_ahgidx;
697 priv->s_ahg->ahgcount++;
698 priv->s_ahg->ahgdesc[0] =
77241056
MM
699 sdma_build_ahg_descriptor(
700 (__force u16)cpu_to_be16((u16)npsn),
701 BTH2_OFFSET,
702 16,
703 16);
704 if ((npsn & 0xffff0000) !=
705 (qp->s_ahgpsn & 0xffff0000)) {
a9b6b3bc
DC
706 priv->s_ahg->ahgcount++;
707 priv->s_ahg->ahgdesc[1] =
77241056
MM
708 sdma_build_ahg_descriptor(
709 (__force u16)cpu_to_be16(
710 (u16)(npsn >> 16)),
711 BTH2_OFFSET,
712 0,
713 16);
714 }
715 }
716 }
717}
718
261a4351 719void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
bb5df5f9
DD
720 u32 bth0, u32 bth2, int middle,
721 struct hfi1_pkt_state *ps)
77241056 722{
4c6829c5 723 struct hfi1_qp_priv *priv = qp->priv;
bb5df5f9 724 struct hfi1_ibport *ibp = ps->ibp;
77241056
MM
725 u16 lrh0;
726 u32 nwords;
727 u32 extra_bytes;
77241056
MM
728 u32 bth1;
729
730 /* Construct the header. */
e922ae06
DH
731 extra_bytes = -ps->s_txreq->s_cur_size & 3;
732 nwords = (ps->s_txreq->s_cur_size + extra_bytes) >> 2;
77241056
MM
733 lrh0 = HFI1_LRH_BTH;
734 if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
bb5df5f9
DD
735 qp->s_hdrwords += hfi1_make_grh(ibp,
736 &ps->s_txreq->phdr.hdr.u.l.grh,
4c6829c5
DD
737 &qp->remote_ah_attr.grh,
738 qp->s_hdrwords, nwords);
77241056
MM
739 lrh0 = HFI1_LRH_GRH;
740 middle = 0;
741 }
4c6829c5 742 lrh0 |= (priv->s_sc & 0xf) << 12 | (qp->remote_ah_attr.sl & 0xf) << 4;
77241056 743 /*
a9b6b3bc 744 * reset s_ahg/AHG fields
77241056
MM
745 *
746 * This insures that the ahgentry/ahgcount
747 * are at a non-AHG default to protect
748 * build_verbs_tx_desc() from using
749 * an include ahgidx.
750 *
751 * build_ahg() will modify as appropriate
752 * to use the AHG feature.
753 */
a9b6b3bc
DC
754 priv->s_ahg->tx_flags = 0;
755 priv->s_ahg->ahgcount = 0;
756 priv->s_ahg->ahgidx = 0;
77241056
MM
757 if (qp->s_mig_state == IB_MIG_MIGRATED)
758 bth0 |= IB_BTH_MIG_REQ;
759 else
760 middle = 0;
761 if (middle)
762 build_ahg(qp, bth2);
763 else
54d10c1e 764 qp->s_flags &= ~RVT_S_AHG_VALID;
bb5df5f9
DD
765 ps->s_txreq->phdr.hdr.lrh[0] = cpu_to_be16(lrh0);
766 ps->s_txreq->phdr.hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
767 ps->s_txreq->phdr.hdr.lrh[2] =
77241056 768 cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
bb5df5f9 769 ps->s_txreq->phdr.hdr.lrh[3] = cpu_to_be16(ppd_from_ibp(ibp)->lid |
77241056
MM
770 qp->remote_ah_attr.src_path_bits);
771 bth0 |= hfi1_get_pkey(ibp, qp->s_pkey_index);
772 bth0 |= extra_bytes << 20;
773 ohdr->bth[0] = cpu_to_be32(bth0);
774 bth1 = qp->remote_qpn;
54d10c1e
DD
775 if (qp->s_flags & RVT_S_ECN) {
776 qp->s_flags &= ~RVT_S_ECN;
77241056 777 /* we recently received a FECN, so return a BECN */
3d591099 778 bth1 |= (IB_BECN_MASK << IB_BECN_SHIFT);
77241056
MM
779 }
780 ohdr->bth[1] = cpu_to_be32(bth1);
781 ohdr->bth[2] = cpu_to_be32(bth2);
782}
783
b421922e
DL
784/* when sending, force a reschedule every one of these periods */
785#define SEND_RESCHED_TIMEOUT (5 * HZ) /* 5s in jiffies */
786
83693bd1
DD
787void _hfi1_do_send(struct work_struct *work)
788{
789 struct iowait *wait = container_of(work, struct iowait, iowork);
790 struct rvt_qp *qp = iowait_to_qp(wait);
791
792 hfi1_do_send(qp);
793}
794
77241056
MM
795/**
796 * hfi1_do_send - perform a send on a QP
797 * @work: contains a pointer to the QP
798 *
799 * Process entries in the send work queue until credit or queue is
ca00c62b 800 * exhausted. Only allow one CPU to send a packet per QP.
77241056
MM
801 * Otherwise, two threads could send packets out of order.
802 */
83693bd1 803void hfi1_do_send(struct rvt_qp *qp)
77241056 804{
d46e5144 805 struct hfi1_pkt_state ps;
23cd4716 806 struct hfi1_qp_priv *priv = qp->priv;
bb5df5f9 807 int (*make_req)(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
b421922e 808 unsigned long timeout;
23cd4716
VM
809 unsigned long timeout_int;
810 int cpu;
77241056 811
d46e5144
DD
812 ps.dev = to_idev(qp->ibqp.device);
813 ps.ibp = to_iport(qp->ibqp.device, qp->port_num);
814 ps.ppd = ppd_from_ibp(ps.ibp);
815
23cd4716
VM
816 switch (qp->ibqp.qp_type) {
817 case IB_QPT_RC:
818 if (!loopback && ((qp->remote_ah_attr.dlid & ~((1 << ps.ppd->lmc
819 ) - 1)) ==
820 ps.ppd->lid)) {
821 ruc_loopback(qp);
822 return;
823 }
77241056 824 make_req = hfi1_make_rc_req;
23cd4716
VM
825 timeout_int = (qp->timeout_jiffies);
826 break;
827 case IB_QPT_UC:
828 if (!loopback && ((qp->remote_ah_attr.dlid & ~((1 << ps.ppd->lmc
829 ) - 1)) ==
830 ps.ppd->lid)) {
831 ruc_loopback(qp);
832 return;
833 }
77241056 834 make_req = hfi1_make_uc_req;
23cd4716
VM
835 timeout_int = SEND_RESCHED_TIMEOUT;
836 break;
837 default:
77241056 838 make_req = hfi1_make_ud_req;
23cd4716
VM
839 timeout_int = SEND_RESCHED_TIMEOUT;
840 }
77241056 841
747f4d7a 842 spin_lock_irqsave(&qp->s_lock, ps.flags);
77241056
MM
843
844 /* Return if we are already busy processing a work request. */
845 if (!hfi1_send_ok(qp)) {
747f4d7a 846 spin_unlock_irqrestore(&qp->s_lock, ps.flags);
77241056
MM
847 return;
848 }
849
54d10c1e 850 qp->s_flags |= RVT_S_BUSY;
77241056 851
23cd4716
VM
852 timeout = jiffies + (timeout_int) / 8;
853 cpu = priv->s_sde ? priv->s_sde->cpu :
854 cpumask_first(cpumask_of_node(ps.ppd->dd->node));
711e104d
MM
855 /* insure a pre-built packet is handled */
856 ps.s_txreq = get_waiting_verbs_txreq(qp);
77241056
MM
857 do {
858 /* Check for a constructed packet to be sent. */
859 if (qp->s_hdrwords != 0) {
747f4d7a 860 spin_unlock_irqrestore(&qp->s_lock, ps.flags);
77241056
MM
861 /*
862 * If the packet cannot be sent now, return and
ca00c62b 863 * the send engine will be woken up later.
77241056 864 */
d46e5144 865 if (hfi1_verbs_send(qp, &ps))
46a80d62 866 return;
a9b6b3bc 867 /* Record that s_ahg is empty. */
77241056 868 qp->s_hdrwords = 0;
46a80d62
MM
869 /* allow other tasks to run */
870 if (unlikely(time_after(jiffies, timeout))) {
871 if (workqueue_congested(cpu,
872 ps.ppd->hfi1_wq)) {
747f4d7a
MM
873 spin_lock_irqsave(
874 &qp->s_lock,
875 ps.flags);
46a80d62
MM
876 qp->s_flags &= ~RVT_S_BUSY;
877 hfi1_schedule_send(qp);
747f4d7a
MM
878 spin_unlock_irqrestore(
879 &qp->s_lock,
880 ps.flags);
46a80d62
MM
881 this_cpu_inc(
882 *ps.ppd->dd->send_schedule);
883 return;
884 }
831464ce
KW
885 if (!irqs_disabled()) {
886 cond_resched();
887 this_cpu_inc(
888 *ps.ppd->dd->send_schedule);
889 }
46a80d62 890 timeout = jiffies + (timeout_int) / 8;
23cd4716 891 }
747f4d7a 892 spin_lock_irqsave(&qp->s_lock, ps.flags);
b421922e 893 }
bb5df5f9 894 } while (make_req(qp, &ps));
46a80d62 895
747f4d7a 896 spin_unlock_irqrestore(&qp->s_lock, ps.flags);
77241056
MM
897}
898
899/*
900 * This should be called with s_lock held.
901 */
895420dd 902void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
77241056
MM
903 enum ib_wc_status status)
904{
905 u32 old_last, last;
77241056 906
83693bd1 907 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND))
77241056
MM
908 return;
909
6c2ab0b8
MM
910 last = qp->s_last;
911 old_last = last;
9260b354 912 trace_hfi1_qp_send_completion(qp, wqe, last);
6c2ab0b8
MM
913 if (++last >= qp->s_size)
914 last = 0;
9260b354 915 trace_hfi1_qp_send_completion(qp, wqe, last);
6c2ab0b8
MM
916 qp->s_last = last;
917 /* See post_send() */
918 barrier();
c64607aa 919 rvt_put_swqe(wqe);
77241056
MM
920 if (qp->ibqp.qp_type == IB_QPT_UD ||
921 qp->ibqp.qp_type == IB_QPT_SMI ||
922 qp->ibqp.qp_type == IB_QPT_GSI)
15723f06 923 atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount);
77241056 924
43a474aa
MM
925 rvt_qp_swqe_complete(qp,
926 wqe,
927 ib_hfi1_wc_opcode[wqe->wr.opcode],
928 status);
77241056 929
77241056
MM
930 if (qp->s_acked == old_last)
931 qp->s_acked = last;
932 if (qp->s_cur == old_last)
933 qp->s_cur = last;
934 if (qp->s_tail == old_last)
935 qp->s_tail = last;
936 if (qp->state == IB_QPS_SQD && last == qp->s_cur)
937 qp->s_draining = 0;
938}