]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/infiniband/hw/ehca/ehca_reqs.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'for-linus' of git://git390.osdl.marist.edu/pub/scm/linux-2.6
[mirror_ubuntu-bionic-kernel.git] / drivers / infiniband / hw / ehca / ehca_reqs.c
CommitLineData
fab97220
HS		 1/*
2 * IBM eServer eHCA Infiniband device driver for Linux on POWER
3 *
4 * post_send/recv, poll_cq, req_notify
5 *
a6a12947
JF		 6 * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
7 * Waleri Fomin <fomin@de.ibm.com>
8 * Joachim Fenkes <fenkes@de.ibm.com>
fab97220
HS		 9 * Reinhard Ernst <rernst@de.ibm.com>
10 *
11 * Copyright (c) 2005 IBM Corporation
12 *
13 * All rights reserved.
14 *
15 * This source code is distributed under a dual license of GPL v2.0 and OpenIB
16 * BSD.
17 *
18 * OpenIB BSD License
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions are met:
22 *
23 * Redistributions of source code must retain the above copyright notice, this
24 * list of conditions and the following disclaimer.
25 *
26 * Redistributions in binary form must reproduce the above copyright notice,
27 * this list of conditions and the following disclaimer in the documentation
28 * and/or other materials
29 * provided with the distribution.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
32 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
35 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
36 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
37 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
38 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
39 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
41 * POSSIBILITY OF SUCH DAMAGE.
42 */
43
44
b8b572e1 45#include <asm/system.h>
fab97220
HS		 46#include "ehca_classes.h"
47#include "ehca_tools.h"
48#include "ehca_qes.h"
49#include "ehca_iverbs.h"
50#include "hcp_if.h"
51#include "hipz_fns.h"
52
2ec8e662
JF		 53/* in RC traffic, insert an empty RDMA READ every this many packets */
54#define ACK_CIRC_THRESHOLD 2000000
55
fab97220
HS		 56static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
57 struct ehca_wqe *wqe_p,
58 struct ib_recv_wr *recv_wr)
59{
60 u8 cnt_ds;
61 if (unlikely((recv_wr->num_sge < 0) ||
62 (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
63 ehca_gen_err("Invalid number of WQE SGE. "
64 "num_sqe=%x max_nr_of_sg=%x",
65 recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
66 return -EINVAL; /* invalid SG list length */
67 }
68
69 /* clear wqe header until sglist */
70 memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
71
72 wqe_p->work_request_id = recv_wr->wr_id;
73 wqe_p->nr_of_data_seg = recv_wr->num_sge;
74
75 for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
76 wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
77 recv_wr->sg_list[cnt_ds].addr;
78 wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
79 recv_wr->sg_list[cnt_ds].lkey;
80 wqe_p->u.all_rcv.sg_list[cnt_ds].length =
81 recv_wr->sg_list[cnt_ds].length;
82 }
83
4da27d6d 84 if (ehca_debug_level >= 3) {
2b94397a
HNN		 85 ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
86 ipz_rqueue);
2ec8e662 87 ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
fab97220
HS		 88 }
89
90 return 0;
91}
92
93#if defined(DEBUG_GSI_SEND_WR)
94
95/* need ib_mad struct */
96#include <rdma/ib_mad.h>
97
98static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
99{
100 int idx;
101 int j;
102 while (send_wr) {
103 struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
104 struct ib_sge *sge = send_wr->sg_list;
105 ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
2b94397a 106 "send_flags=%x opcode=%x", idx, send_wr->wr_id,
fab97220
HS		 107 send_wr->num_sge, send_wr->send_flags,
108 send_wr->opcode);
109 if (mad_hdr) {
110 ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
111 "mgmt_class=%x class_version=%x method=%x "
112 "status=%x class_specific=%x tid=%lx "
113 "attr_id=%x resv=%x attr_mod=%x",
114 idx, mad_hdr->base_version,
115 mad_hdr->mgmt_class,
116 mad_hdr->class_version, mad_hdr->method,
117 mad_hdr->status, mad_hdr->class_specific,
118 mad_hdr->tid, mad_hdr->attr_id,
119 mad_hdr->resv,
120 mad_hdr->attr_mod);
121 }
122 for (j = 0; j < send_wr->num_sge; j++) {
2b94397a 123 u8 *data = (u8 *)abs_to_virt(sge->addr);
fab97220
HS		 124 ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
125 "lkey=%x",
126 idx, j, data, sge->length, sge->lkey);
127 /* assume length is n*16 */
128 ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
129 idx, j);
130 sge++;
131 } /* eof for j */
132 idx++;
133 send_wr = send_wr->next;
134 } /* eof while send_wr */
135}
136
137#endif /* DEBUG_GSI_SEND_WR */
138
139static inline int ehca_write_swqe(struct ehca_qp *qp,
140 struct ehca_wqe *wqe_p,
2ec8e662 141 const struct ib_send_wr *send_wr,
6773f079 142 u32 sq_map_idx,
2ec8e662 143 int hidden)
fab97220
HS		 144{
145 u32 idx;
146 u64 dma_length;
147 struct ehca_av *my_av;
148 u32 remote_qkey = send_wr->wr.ud.remote_qkey;
149
150 if (unlikely((send_wr->num_sge < 0) ||
151 (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
152 ehca_gen_err("Invalid number of WQE SGE. "
153 "num_sqe=%x max_nr_of_sg=%x",
154 send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
155 return -EINVAL; /* invalid SG list length */
156 }
157
158 /* clear wqe header until sglist */
159 memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
160
6773f079
AS		 161 wqe_p->work_request_id = send_wr->wr_id & ~QMAP_IDX_MASK;
162 wqe_p->work_request_id |= sq_map_idx & QMAP_IDX_MASK;
163
164 qp->sq_map[sq_map_idx].app_wr_id = send_wr->wr_id & QMAP_IDX_MASK;
165 qp->sq_map[sq_map_idx].reported = 0;
fab97220
HS		 166
167 switch (send_wr->opcode) {
168 case IB_WR_SEND:
169 case IB_WR_SEND_WITH_IMM:
170 wqe_p->optype = WQE_OPTYPE_SEND;
171 break;
172 case IB_WR_RDMA_WRITE:
173 case IB_WR_RDMA_WRITE_WITH_IMM:
174 wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
175 break;
176 case IB_WR_RDMA_READ:
177 wqe_p->optype = WQE_OPTYPE_RDMAREAD;
178 break;
179 default:
180 ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
181 return -EINVAL; /* invalid opcode */
182 }
183
184 wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
185
186 wqe_p->wr_flag = 0;
187
2ec8e662
JF		 188 if ((send_wr->send_flags & IB_SEND_SIGNALED ||
189 qp->init_attr.sq_sig_type == IB_SIGNAL_ALL_WR)
190 && !hidden)
fab97220
HS		 191 wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
192
193 if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
194 send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
195 /* this might not work as long as HW does not support it */
0f39cf3d 196 wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data);
fab97220
HS		 197 wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
198 }
199
200 wqe_p->nr_of_data_seg = send_wr->num_sge;
201
202 switch (qp->qp_type) {
203 case IB_QPT_SMI:
204 case IB_QPT_GSI:
205 /* no break is intential here */
206 case IB_QPT_UD:
207 /* IB 1.2 spec C10-15 compliance */
208 if (send_wr->wr.ud.remote_qkey & 0x80000000)
209 remote_qkey = qp->qkey;
210
211 wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
212 wqe_p->local_ee_context_qkey = remote_qkey;
2ec8e662 213 if (unlikely(!send_wr->wr.ud.ah)) {
fab97220
HS		 214 ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
215 return -EINVAL;
2b7274c3
JF		 216 }
217 if (unlikely(send_wr->wr.ud.remote_qpn == 0)) {
218 ehca_gen_err("dest QP# is 0. qp=%x", qp->real_qp_num);
219 return -EINVAL;
fab97220
HS		 220 }
221 my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
222 wqe_p->u.ud_av.ud_av = my_av->av;
223
224 /*
225 * omitted check of IB_SEND_INLINE
226 * since HW does not support it
227 */
228 for (idx = 0; idx < send_wr->num_sge; idx++) {
229 wqe_p->u.ud_av.sg_list[idx].vaddr =
230 send_wr->sg_list[idx].addr;
231 wqe_p->u.ud_av.sg_list[idx].lkey =
232 send_wr->sg_list[idx].lkey;
233 wqe_p->u.ud_av.sg_list[idx].length =
234 send_wr->sg_list[idx].length;
235 } /* eof for idx */
236 if (qp->qp_type == IB_QPT_SMI ||
237 qp->qp_type == IB_QPT_GSI)
238 wqe_p->u.ud_av.ud_av.pmtu = 1;
239 if (qp->qp_type == IB_QPT_GSI) {
240 wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
241#ifdef DEBUG_GSI_SEND_WR
242 trace_send_wr_ud(send_wr);
243#endif /* DEBUG_GSI_SEND_WR */
244 }
245 break;
246
247 case IB_QPT_UC:
248 if (send_wr->send_flags & IB_SEND_FENCE)
249 wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
250 /* no break is intentional here */
251 case IB_QPT_RC:
252 /* TODO: atomic not implemented */
253 wqe_p->u.nud.remote_virtual_adress =
254 send_wr->wr.rdma.remote_addr;
255 wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
256
257 /*
258 * omitted checking of IB_SEND_INLINE
259 * since HW does not support it
260 */
261 dma_length = 0;
262 for (idx = 0; idx < send_wr->num_sge; idx++) {
263 wqe_p->u.nud.sg_list[idx].vaddr =
264 send_wr->sg_list[idx].addr;
265 wqe_p->u.nud.sg_list[idx].lkey =
266 send_wr->sg_list[idx].lkey;
267 wqe_p->u.nud.sg_list[idx].length =
268 send_wr->sg_list[idx].length;
269 dma_length += send_wr->sg_list[idx].length;
270 } /* eof idx */
271 wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
272
2ec8e662
JF		 273 /* unsolicited ack circumvention */
274 if (send_wr->opcode == IB_WR_RDMA_READ) {
275 /* on RDMA read, switch on and reset counters */
276 qp->message_count = qp->packet_count = 0;
277 qp->unsol_ack_circ = 1;
278 } else
279 /* else estimate #packets */
280 qp->packet_count += (dma_length >> qp->mtu_shift) + 1;
281
fab97220
HS		 282 break;
283
284 default:
285 ehca_gen_err("Invalid qptype=%x", qp->qp_type);
286 return -EINVAL;
287 }
288
4da27d6d 289 if (ehca_debug_level >= 3) {
fab97220
HS		 290 ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
291 ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
292 }
293 return 0;
294}
295
296/* map_ib_wc_status converts raw cqe_status to ib_wc_status */
297static inline void map_ib_wc_status(u32 cqe_status,
298 enum ib_wc_status *wc_status)
299{
300 if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
301 switch (cqe_status & 0x3F) {
302 case 0x01:
303 case 0x21:
304 *wc_status = IB_WC_LOC_LEN_ERR;
305 break;
306 case 0x02:
307 case 0x22:
308 *wc_status = IB_WC_LOC_QP_OP_ERR;
309 break;
310 case 0x03:
311 case 0x23:
312 *wc_status = IB_WC_LOC_EEC_OP_ERR;
313 break;
314 case 0x04:
315 case 0x24:
316 *wc_status = IB_WC_LOC_PROT_ERR;
317 break;
318 case 0x05:
319 case 0x25:
320 *wc_status = IB_WC_WR_FLUSH_ERR;
321 break;
322 case 0x06:
323 *wc_status = IB_WC_MW_BIND_ERR;
324 break;
325 case 0x07: /* remote error - look into bits 20:24 */
326 switch ((cqe_status
327 & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
328 case 0x0:
329 /*
330 * PSN Sequence Error!
331 * couldn't find a matching status!
332 */
333 *wc_status = IB_WC_GENERAL_ERR;
334 break;
335 case 0x1:
336 *wc_status = IB_WC_REM_INV_REQ_ERR;
337 break;
338 case 0x2:
339 *wc_status = IB_WC_REM_ACCESS_ERR;
340 break;
341 case 0x3:
342 *wc_status = IB_WC_REM_OP_ERR;
343 break;
344 case 0x4:
345 *wc_status = IB_WC_REM_INV_RD_REQ_ERR;
346 break;
347 }
348 break;
349 case 0x08:
350 *wc_status = IB_WC_RETRY_EXC_ERR;
351 break;
352 case 0x09:
353 *wc_status = IB_WC_RNR_RETRY_EXC_ERR;
354 break;
355 case 0x0A:
356 case 0x2D:
357 *wc_status = IB_WC_REM_ABORT_ERR;
358 break;
359 case 0x0B:
360 case 0x2E:
361 *wc_status = IB_WC_INV_EECN_ERR;
362 break;
363 case 0x0C:
364 case 0x2F:
365 *wc_status = IB_WC_INV_EEC_STATE_ERR;
366 break;
367 case 0x0D:
368 *wc_status = IB_WC_BAD_RESP_ERR;
369 break;
370 case 0x10:
371 /* WQE purged */
372 *wc_status = IB_WC_WR_FLUSH_ERR;
373 break;
374 default:
375 *wc_status = IB_WC_FATAL_ERR;
376
377 }
378 } else
379 *wc_status = IB_WC_SUCCESS;
380}
381
2ec8e662
JF		 382static inline int post_one_send(struct ehca_qp *my_qp,
383 struct ib_send_wr *cur_send_wr,
384 struct ib_send_wr **bad_send_wr,
385 int hidden)
386{
387 struct ehca_wqe *wqe_p;
388 int ret;
6773f079 389 u32 sq_map_idx;
2ec8e662
JF		 390 u64 start_offset = my_qp->ipz_squeue.current_q_offset;
391
392 /* get pointer next to free WQE */
393 wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
394 if (unlikely(!wqe_p)) {
395 /* too many posted work requests: queue overflow */
396 if (bad_send_wr)
397 *bad_send_wr = cur_send_wr;
398 ehca_err(my_qp->ib_qp.device, "Too many posted WQEs "
399 "qp_num=%x", my_qp->ib_qp.qp_num);
400 return -ENOMEM;
401 }
6773f079
AS		 402
403 /*
404 * Get the index of the WQE in the send queue. The same index is used
405 * for writing into the sq_map.
406 */
407 sq_map_idx = start_offset / my_qp->ipz_squeue.qe_size;
408
2ec8e662 409 /* write a SEND WQE into the QUEUE */
6773f079 410 ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr, sq_map_idx, hidden);
2ec8e662
JF		 411 /*
412 * if something failed,
413 * reset the free entry pointer to the start value
414 */
415 if (unlikely(ret)) {
416 my_qp->ipz_squeue.current_q_offset = start_offset;
417 if (bad_send_wr)
418 *bad_send_wr = cur_send_wr;
419 ehca_err(my_qp->ib_qp.device, "Could not write WQE "
420 "qp_num=%x", my_qp->ib_qp.qp_num);
421 return -EINVAL;
422 }
423
424 return 0;
425}
426
fab97220
HS		 427int ehca_post_send(struct ib_qp *qp,
428 struct ib_send_wr *send_wr,
429 struct ib_send_wr **bad_send_wr)
430{
431 struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
432 struct ib_send_wr *cur_send_wr;
fab97220
HS		 433 int wqe_cnt = 0;
434 int ret = 0;
9844b71b 435 unsigned long flags;
fab97220 436
088af154
JF		 437 /* Reject WR if QP is in RESET, INIT or RTR state */
438 if (unlikely(my_qp->state < IB_QPS_RTS)) {
439 ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
440 my_qp->state, qp->qp_num);
863fb09f
JF		 441 return -EINVAL;
442 }
443
fab97220 444 /* LOCK the QUEUE */
9844b71b 445 spin_lock_irqsave(&my_qp->spinlock_s, flags);
fab97220 446
2ec8e662
JF		 447 /* Send an empty extra RDMA read if:
448 * 1) there has been an RDMA read on this connection before
449 * 2) no RDMA read occurred for ACK_CIRC_THRESHOLD link packets
450 * 3) we can be sure that any previous extra RDMA read has been
451 * processed so we don't overflow the SQ
452 */
453 if (unlikely(my_qp->unsol_ack_circ &&
454 my_qp->packet_count > ACK_CIRC_THRESHOLD &&
455 my_qp->message_count > my_qp->init_attr.cap.max_send_wr)) {
456 /* insert an empty RDMA READ to fix up the remote QP state */
457 struct ib_send_wr circ_wr;
458 memset(&circ_wr, 0, sizeof(circ_wr));
459 circ_wr.opcode = IB_WR_RDMA_READ;
460 post_one_send(my_qp, &circ_wr, NULL, 1); /* ignore retcode */
461 wqe_cnt++;
462 ehca_dbg(qp->device, "posted circ wr qp_num=%x", qp->qp_num);
463 my_qp->message_count = my_qp->packet_count = 0;
464 }
465
fab97220
HS		 466 /* loop processes list of send reqs */
467 for (cur_send_wr = send_wr; cur_send_wr != NULL;
468 cur_send_wr = cur_send_wr->next) {
2ec8e662 469 ret = post_one_send(my_qp, cur_send_wr, bad_send_wr, 0);
fab97220 470 if (unlikely(ret)) {
2ec8e662
JF		 471 /* if one or more WQEs were successful, don't fail */
472 if (wqe_cnt)
473 ret = 0;
fab97220
HS		 474 goto post_send_exit0;
475 }
476 wqe_cnt++;
fab97220
HS		 477 } /* eof for cur_send_wr */
478
479post_send_exit0:
fab97220
HS		 480 iosync(); /* serialize GAL register access */
481 hipz_update_sqa(my_qp, wqe_cnt);
4da27d6d
JF		 482 if (unlikely(ret || ehca_debug_level >= 2))
483 ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
484 my_qp, qp->qp_num, wqe_cnt, ret);
2ec8e662 485 my_qp->message_count += wqe_cnt;
f72d2f08 486 spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
fab97220
HS		 487 return ret;
488}
489
a6a12947
JF		 490static int internal_post_recv(struct ehca_qp *my_qp,
491 struct ib_device *dev,
492 struct ib_recv_wr *recv_wr,
493 struct ib_recv_wr **bad_recv_wr)
fab97220 494{
fab97220
HS		 495 struct ib_recv_wr *cur_recv_wr;
496 struct ehca_wqe *wqe_p;
497 int wqe_cnt = 0;
498 int ret = 0;
9844b71b 499 unsigned long flags;
fab97220 500
a6a12947
JF		 501 if (unlikely(!HAS_RQ(my_qp))) {
502 ehca_err(dev, "QP has no RQ ehca_qp=%p qp_num=%x ext_type=%d",
503 my_qp, my_qp->real_qp_num, my_qp->ext_type);
504 return -ENODEV;
505 }
506
fab97220 507 /* LOCK the QUEUE */
9844b71b 508 spin_lock_irqsave(&my_qp->spinlock_r, flags);
fab97220
HS		 509
510 /* loop processes list of send reqs */
511 for (cur_recv_wr = recv_wr; cur_recv_wr != NULL;
512 cur_recv_wr = cur_recv_wr->next) {
513 u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
514 /* get pointer next to free WQE */
515 wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
516 if (unlikely(!wqe_p)) {
517 /* too many posted work requests: queue overflow */
518 if (bad_recv_wr)
519 *bad_recv_wr = cur_recv_wr;
520 if (wqe_cnt == 0) {
521 ret = -ENOMEM;
a6a12947
JF		 522 ehca_err(dev, "Too many posted WQEs "
523 "qp_num=%x", my_qp->real_qp_num);
fab97220
HS		 524 }
525 goto post_recv_exit0;
526 }
527 /* write a RECV WQE into the QUEUE */
528 ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, cur_recv_wr);
529 /*
530 * if something failed,
531 * reset the free entry pointer to the start value
532 */
533 if (unlikely(ret)) {
534 my_qp->ipz_rqueue.current_q_offset = start_offset;
535 *bad_recv_wr = cur_recv_wr;
536 if (wqe_cnt == 0) {
537 ret = -EINVAL;
a6a12947
JF		 538 ehca_err(dev, "Could not write WQE "
539 "qp_num=%x", my_qp->real_qp_num);
fab97220
HS		 540 }
541 goto post_recv_exit0;
542 }
543 wqe_cnt++;
fab97220
HS		 544 } /* eof for cur_recv_wr */
545
546post_recv_exit0:
fab97220
HS		 547 iosync(); /* serialize GAL register access */
548 hipz_update_rqa(my_qp, wqe_cnt);
4da27d6d
JF		 549 if (unlikely(ret || ehca_debug_level >= 2))
550 ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
551 my_qp, my_qp->real_qp_num, wqe_cnt, ret);
f72d2f08 552 spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
fab97220
HS		 553 return ret;
554}
555
a6a12947
JF		 556int ehca_post_recv(struct ib_qp *qp,
557 struct ib_recv_wr *recv_wr,
558 struct ib_recv_wr **bad_recv_wr)
559{
3e255eac
JF		 560 struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
561
562 /* Reject WR if QP is in RESET state */
563 if (unlikely(my_qp->state == IB_QPS_RESET)) {
564 ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
565 my_qp->state, qp->qp_num);
566 return -EINVAL;
567 }
568
569 return internal_post_recv(my_qp, qp->device, recv_wr, bad_recv_wr);
a6a12947
JF		 570}
571
572int ehca_post_srq_recv(struct ib_srq *srq,
573 struct ib_recv_wr *recv_wr,
574 struct ib_recv_wr **bad_recv_wr)
575{
576 return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
577 srq->device, recv_wr, bad_recv_wr);
578}
579
fab97220
HS		 580/*
581 * ib_wc_opcode table converts ehca wc opcode to ib
582 * Since we use zero to indicate invalid opcode, the actual ib opcode must
583 * be decremented!!!
584 */
585static const u8 ib_wc_opcode[255] = {
586 [0x01] = IB_WC_RECV+1,
587 [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
588 [0x04] = IB_WC_BIND_MW+1,
589 [0x08] = IB_WC_FETCH_ADD+1,
590 [0x10] = IB_WC_COMP_SWAP+1,
591 [0x20] = IB_WC_RDMA_WRITE+1,
592 [0x40] = IB_WC_RDMA_READ+1,
593 [0x80] = IB_WC_SEND+1
594};
595
596/* internal function to poll one entry of cq */
597static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
598{
599 int ret = 0;
600 struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
601 struct ehca_cqe *cqe;
b1cfe43d 602 struct ehca_qp *my_qp;
4da27d6d 603 int cqe_count = 0, is_error;
fab97220 604
6c02eed9 605repoll:
fab97220
HS		 606 cqe = (struct ehca_cqe *)
607 ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
608 if (!cqe) {
609 ret = -EAGAIN;
4da27d6d
JF		 610 if (ehca_debug_level >= 3)
611 ehca_dbg(cq->device, "Completion queue is empty "
612 "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number);
613 goto poll_cq_one_exit0;
fab97220
HS		 614 }
615
616 /* prevents loads being reordered across this point */
617 rmb();
618
619 cqe_count++;
620 if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
2b94397a 621 struct ehca_qp *qp;
fab97220 622 int purgeflag;
9844b71b 623 unsigned long flags;
2b94397a
HNN		 624
625 qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number);
fab97220
HS		 626 if (!qp) {
627 ehca_err(cq->device, "cq_num=%x qp_num=%x "
628 "could not find qp -> ignore cqe",
629 my_cq->cq_number, cqe->local_qp_number);
630 ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
631 my_cq->cq_number, cqe->local_qp_number);
632 /* ignore this purged cqe */
6c02eed9 633 goto repoll;
fab97220 634 }
9844b71b 635 spin_lock_irqsave(&qp->spinlock_s, flags);
fab97220 636 purgeflag = qp->sqerr_purgeflag;
9844b71b 637 spin_unlock_irqrestore(&qp->spinlock_s, flags);
fab97220
HS		 638
639 if (purgeflag) {
2b94397a
HNN		 640 ehca_dbg(cq->device,
641 "Got CQE with purged bit qp_num=%x src_qp=%x",
fab97220 642 cqe->local_qp_number, cqe->remote_qp_number);
4da27d6d 643 if (ehca_debug_level >= 2)
fab97220
HS		 644 ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
645 cqe->local_qp_number,
646 cqe->remote_qp_number);
647 /*
648 * ignore this to avoid double cqes of bad wqe
649 * that caused sqe and turn off purge flag
650 */
651 qp->sqerr_purgeflag = 0;
6c02eed9 652 goto repoll;
fab97220
HS		 653 }
654 }
655
4da27d6d
JF		 656 is_error = cqe->status & WC_STATUS_ERROR_BIT;
657
658 /* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */
659 if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) {
fab97220 660 ehca_dbg(cq->device,
4da27d6d
JF		 661 "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----",
662 is_error ? "ERROR " : "", my_cq, my_cq->cq_number);
fab97220
HS		 663 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
664 my_cq, my_cq->cq_number);
665 ehca_dbg(cq->device,
666 "ehca_cq=%p cq_num=%x -------------------------",
667 my_cq, my_cq->cq_number);
668 }
669
6773f079
AS		 670 read_lock(&ehca_qp_idr_lock);
671 my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
672 read_unlock(&ehca_qp_idr_lock);
673 if (!my_qp)
674 goto repoll;
675 wc->qp = &my_qp->ib_qp;
676
677 if (!(cqe->w_completion_flags & WC_SEND_RECEIVE_BIT)) {
678 struct ehca_qmap_entry *qmap_entry;
679 /*
680 * We got a send completion and need to restore the original
681 * wr_id.
682 */
683 qmap_entry = &my_qp->sq_map[cqe->work_request_id &
684 QMAP_IDX_MASK];
685
686 if (qmap_entry->reported) {
687 ehca_warn(cq->device, "Double cqe on qp_num=%#x",
688 my_qp->real_qp_num);
689 /* found a double cqe, discard it and read next one */
690 goto repoll;
691 }
692 wc->wr_id = cqe->work_request_id & ~QMAP_IDX_MASK;
693 wc->wr_id |= qmap_entry->app_wr_id;
694 qmap_entry->reported = 1;
695 } else
696 /* We got a receive completion. */
697 wc->wr_id = cqe->work_request_id;
fab97220
HS		 698
699 /* eval ib_wc_opcode */
700 wc->opcode = ib_wc_opcode[cqe->optype]-1;
701 if (unlikely(wc->opcode == -1)) {
702 ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
703 "ehca_cq=%p cq_num=%x",
704 cqe->optype, cqe->status, my_cq, my_cq->cq_number);
705 /* dump cqe for other infos */
706 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
707 my_cq, my_cq->cq_number);
708 /* update also queue adder to throw away this entry!!! */
17c2b53a 709 goto repoll;
fab97220 710 }
4da27d6d 711
fab97220 712 /* eval ib_wc_status */
4da27d6d 713 if (unlikely(is_error)) {
fab97220
HS		 714 /* complete with errors */
715 map_ib_wc_status(cqe->status, &wc->status);
716 wc->vendor_err = wc->status;
717 } else
718 wc->status = IB_WC_SUCCESS;
719
fab97220
HS		 720 wc->byte_len = cqe->nr_bytes_transferred;
721 wc->pkey_index = cqe->pkey_index;
722 wc->slid = cqe->rlid;
723 wc->dlid_path_bits = cqe->dlid;
724 wc->src_qp = cqe->remote_qp_number;
725 wc->wc_flags = cqe->w_completion_flags;
00f7ec36 726 wc->ex.imm_data = cpu_to_be32(cqe->immediate_data);
fab97220
HS		 727 wc->sl = cqe->service_level;
728
fab97220
HS		 729poll_cq_one_exit0:
730 if (cqe_count > 0)
731 hipz_update_feca(my_cq, cqe_count);
732
733 return ret;
734}
735
736int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
737{
738 struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
739 int nr;
740 struct ib_wc *current_wc = wc;
741 int ret = 0;
9844b71b 742 unsigned long flags;
fab97220
HS		 743
744 if (num_entries < 1) {
745 ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
746 "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
747 ret = -EINVAL;
748 goto poll_cq_exit0;
749 }
750
9844b71b 751 spin_lock_irqsave(&my_cq->spinlock, flags);
fab97220
HS		 752 for (nr = 0; nr < num_entries; nr++) {
753 ret = ehca_poll_cq_one(cq, current_wc);
754 if (ret)
755 break;
756 current_wc++;
757 } /* eof for nr */
9844b71b 758 spin_unlock_irqrestore(&my_cq->spinlock, flags);
fab97220
HS		 759 if (ret == -EAGAIN || !ret)
760 ret = nr;
761
762poll_cq_exit0:
763 return ret;
764}
765
ed23a727 766int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
fab97220
HS		 767{
768 struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
ed23a727 769 int ret = 0;
fab97220 770
ed23a727 771 switch (notify_flags & IB_CQ_SOLICITED_MASK) {
fab97220
HS		 772 case IB_CQ_SOLICITED:
773 hipz_set_cqx_n0(my_cq, 1);
774 break;
775 case IB_CQ_NEXT_COMP:
776 hipz_set_cqx_n1(my_cq, 1);
777 break;
778 default:
779 return -EINVAL;
780 }
781
ed23a727 782 if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
fffba373 783 unsigned long spl_flags;
ed23a727
RD		 784 spin_lock_irqsave(&my_cq->spinlock, spl_flags);
785 ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
786 spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
787 }
788
789 return ret;
fab97220 790}
ubuntu-bionic-kernel mirror