]>
Commit | Line | Data |
---|---|---|
fab97220 HS |
1 | /* |
2 | * IBM eServer eHCA Infiniband device driver for Linux on POWER | |
3 | * | |
4 | * post_send/recv, poll_cq, req_notify | |
5 | * | |
a6a12947 JF |
6 | * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com> |
7 | * Waleri Fomin <fomin@de.ibm.com> | |
8 | * Joachim Fenkes <fenkes@de.ibm.com> | |
fab97220 HS |
9 | * Reinhard Ernst <rernst@de.ibm.com> |
10 | * | |
11 | * Copyright (c) 2005 IBM Corporation | |
12 | * | |
13 | * All rights reserved. | |
14 | * | |
15 | * This source code is distributed under a dual license of GPL v2.0 and OpenIB | |
16 | * BSD. | |
17 | * | |
18 | * OpenIB BSD License | |
19 | * | |
20 | * Redistribution and use in source and binary forms, with or without | |
21 | * modification, are permitted provided that the following conditions are met: | |
22 | * | |
23 | * Redistributions of source code must retain the above copyright notice, this | |
24 | * list of conditions and the following disclaimer. | |
25 | * | |
26 | * Redistributions in binary form must reproduce the above copyright notice, | |
27 | * this list of conditions and the following disclaimer in the documentation | |
28 | * and/or other materials | |
29 | * provided with the distribution. | |
30 | * | |
31 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |
32 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
33 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
34 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | |
35 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
36 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
37 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
38 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER | |
39 | * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
40 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | |
41 | * POSSIBILITY OF SUCH DAMAGE. | |
42 | */ | |
43 | ||
44 | ||
45 | #include <asm-powerpc/system.h> | |
46 | #include "ehca_classes.h" | |
47 | #include "ehca_tools.h" | |
48 | #include "ehca_qes.h" | |
49 | #include "ehca_iverbs.h" | |
50 | #include "hcp_if.h" | |
51 | #include "hipz_fns.h" | |
52 | ||
2ec8e662 JF |
53 | /* in RC traffic, insert an empty RDMA READ every this many packets */ |
54 | #define ACK_CIRC_THRESHOLD 2000000 | |
55 | ||
fab97220 HS |
56 | static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue, |
57 | struct ehca_wqe *wqe_p, | |
58 | struct ib_recv_wr *recv_wr) | |
59 | { | |
60 | u8 cnt_ds; | |
61 | if (unlikely((recv_wr->num_sge < 0) || | |
62 | (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) { | |
63 | ehca_gen_err("Invalid number of WQE SGE. " | |
64 | "num_sqe=%x max_nr_of_sg=%x", | |
65 | recv_wr->num_sge, ipz_rqueue->act_nr_of_sg); | |
66 | return -EINVAL; /* invalid SG list length */ | |
67 | } | |
68 | ||
69 | /* clear wqe header until sglist */ | |
70 | memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list)); | |
71 | ||
72 | wqe_p->work_request_id = recv_wr->wr_id; | |
73 | wqe_p->nr_of_data_seg = recv_wr->num_sge; | |
74 | ||
75 | for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) { | |
76 | wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr = | |
77 | recv_wr->sg_list[cnt_ds].addr; | |
78 | wqe_p->u.all_rcv.sg_list[cnt_ds].lkey = | |
79 | recv_wr->sg_list[cnt_ds].lkey; | |
80 | wqe_p->u.all_rcv.sg_list[cnt_ds].length = | |
81 | recv_wr->sg_list[cnt_ds].length; | |
82 | } | |
83 | ||
4da27d6d | 84 | if (ehca_debug_level >= 3) { |
2b94397a HNN |
85 | ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p", |
86 | ipz_rqueue); | |
2ec8e662 | 87 | ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe"); |
fab97220 HS |
88 | } |
89 | ||
90 | return 0; | |
91 | } | |
92 | ||
93 | #if defined(DEBUG_GSI_SEND_WR) | |
94 | ||
95 | /* need ib_mad struct */ | |
96 | #include <rdma/ib_mad.h> | |
97 | ||
98 | static void trace_send_wr_ud(const struct ib_send_wr *send_wr) | |
99 | { | |
100 | int idx; | |
101 | int j; | |
102 | while (send_wr) { | |
103 | struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr; | |
104 | struct ib_sge *sge = send_wr->sg_list; | |
105 | ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x " | |
2b94397a | 106 | "send_flags=%x opcode=%x", idx, send_wr->wr_id, |
fab97220 HS |
107 | send_wr->num_sge, send_wr->send_flags, |
108 | send_wr->opcode); | |
109 | if (mad_hdr) { | |
110 | ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x " | |
111 | "mgmt_class=%x class_version=%x method=%x " | |
112 | "status=%x class_specific=%x tid=%lx " | |
113 | "attr_id=%x resv=%x attr_mod=%x", | |
114 | idx, mad_hdr->base_version, | |
115 | mad_hdr->mgmt_class, | |
116 | mad_hdr->class_version, mad_hdr->method, | |
117 | mad_hdr->status, mad_hdr->class_specific, | |
118 | mad_hdr->tid, mad_hdr->attr_id, | |
119 | mad_hdr->resv, | |
120 | mad_hdr->attr_mod); | |
121 | } | |
122 | for (j = 0; j < send_wr->num_sge; j++) { | |
2b94397a | 123 | u8 *data = (u8 *)abs_to_virt(sge->addr); |
fab97220 HS |
124 | ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x " |
125 | "lkey=%x", | |
126 | idx, j, data, sge->length, sge->lkey); | |
127 | /* assume length is n*16 */ | |
128 | ehca_dmp(data, sge->length, "send_wr#%x sge#%x", | |
129 | idx, j); | |
130 | sge++; | |
131 | } /* eof for j */ | |
132 | idx++; | |
133 | send_wr = send_wr->next; | |
134 | } /* eof while send_wr */ | |
135 | } | |
136 | ||
137 | #endif /* DEBUG_GSI_SEND_WR */ | |
138 | ||
139 | static inline int ehca_write_swqe(struct ehca_qp *qp, | |
140 | struct ehca_wqe *wqe_p, | |
2ec8e662 JF |
141 | const struct ib_send_wr *send_wr, |
142 | int hidden) | |
fab97220 HS |
143 | { |
144 | u32 idx; | |
145 | u64 dma_length; | |
146 | struct ehca_av *my_av; | |
147 | u32 remote_qkey = send_wr->wr.ud.remote_qkey; | |
148 | ||
149 | if (unlikely((send_wr->num_sge < 0) || | |
150 | (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) { | |
151 | ehca_gen_err("Invalid number of WQE SGE. " | |
152 | "num_sqe=%x max_nr_of_sg=%x", | |
153 | send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg); | |
154 | return -EINVAL; /* invalid SG list length */ | |
155 | } | |
156 | ||
157 | /* clear wqe header until sglist */ | |
158 | memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list)); | |
159 | ||
160 | wqe_p->work_request_id = send_wr->wr_id; | |
161 | ||
162 | switch (send_wr->opcode) { | |
163 | case IB_WR_SEND: | |
164 | case IB_WR_SEND_WITH_IMM: | |
165 | wqe_p->optype = WQE_OPTYPE_SEND; | |
166 | break; | |
167 | case IB_WR_RDMA_WRITE: | |
168 | case IB_WR_RDMA_WRITE_WITH_IMM: | |
169 | wqe_p->optype = WQE_OPTYPE_RDMAWRITE; | |
170 | break; | |
171 | case IB_WR_RDMA_READ: | |
172 | wqe_p->optype = WQE_OPTYPE_RDMAREAD; | |
173 | break; | |
174 | default: | |
175 | ehca_gen_err("Invalid opcode=%x", send_wr->opcode); | |
176 | return -EINVAL; /* invalid opcode */ | |
177 | } | |
178 | ||
179 | wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE; | |
180 | ||
181 | wqe_p->wr_flag = 0; | |
182 | ||
2ec8e662 JF |
183 | if ((send_wr->send_flags & IB_SEND_SIGNALED || |
184 | qp->init_attr.sq_sig_type == IB_SIGNAL_ALL_WR) | |
185 | && !hidden) | |
fab97220 HS |
186 | wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM; |
187 | ||
188 | if (send_wr->opcode == IB_WR_SEND_WITH_IMM || | |
189 | send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { | |
190 | /* this might not work as long as HW does not support it */ | |
0f39cf3d | 191 | wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data); |
fab97220 HS |
192 | wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT; |
193 | } | |
194 | ||
195 | wqe_p->nr_of_data_seg = send_wr->num_sge; | |
196 | ||
197 | switch (qp->qp_type) { | |
198 | case IB_QPT_SMI: | |
199 | case IB_QPT_GSI: | |
200 | /* no break is intential here */ | |
201 | case IB_QPT_UD: | |
202 | /* IB 1.2 spec C10-15 compliance */ | |
203 | if (send_wr->wr.ud.remote_qkey & 0x80000000) | |
204 | remote_qkey = qp->qkey; | |
205 | ||
206 | wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8; | |
207 | wqe_p->local_ee_context_qkey = remote_qkey; | |
2ec8e662 | 208 | if (unlikely(!send_wr->wr.ud.ah)) { |
fab97220 HS |
209 | ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp); |
210 | return -EINVAL; | |
2b7274c3 JF |
211 | } |
212 | if (unlikely(send_wr->wr.ud.remote_qpn == 0)) { | |
213 | ehca_gen_err("dest QP# is 0. qp=%x", qp->real_qp_num); | |
214 | return -EINVAL; | |
fab97220 HS |
215 | } |
216 | my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah); | |
217 | wqe_p->u.ud_av.ud_av = my_av->av; | |
218 | ||
219 | /* | |
220 | * omitted check of IB_SEND_INLINE | |
221 | * since HW does not support it | |
222 | */ | |
223 | for (idx = 0; idx < send_wr->num_sge; idx++) { | |
224 | wqe_p->u.ud_av.sg_list[idx].vaddr = | |
225 | send_wr->sg_list[idx].addr; | |
226 | wqe_p->u.ud_av.sg_list[idx].lkey = | |
227 | send_wr->sg_list[idx].lkey; | |
228 | wqe_p->u.ud_av.sg_list[idx].length = | |
229 | send_wr->sg_list[idx].length; | |
230 | } /* eof for idx */ | |
231 | if (qp->qp_type == IB_QPT_SMI || | |
232 | qp->qp_type == IB_QPT_GSI) | |
233 | wqe_p->u.ud_av.ud_av.pmtu = 1; | |
234 | if (qp->qp_type == IB_QPT_GSI) { | |
235 | wqe_p->pkeyi = send_wr->wr.ud.pkey_index; | |
236 | #ifdef DEBUG_GSI_SEND_WR | |
237 | trace_send_wr_ud(send_wr); | |
238 | #endif /* DEBUG_GSI_SEND_WR */ | |
239 | } | |
240 | break; | |
241 | ||
242 | case IB_QPT_UC: | |
243 | if (send_wr->send_flags & IB_SEND_FENCE) | |
244 | wqe_p->wr_flag |= WQE_WRFLAG_FENCE; | |
245 | /* no break is intentional here */ | |
246 | case IB_QPT_RC: | |
247 | /* TODO: atomic not implemented */ | |
248 | wqe_p->u.nud.remote_virtual_adress = | |
249 | send_wr->wr.rdma.remote_addr; | |
250 | wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey; | |
251 | ||
252 | /* | |
253 | * omitted checking of IB_SEND_INLINE | |
254 | * since HW does not support it | |
255 | */ | |
256 | dma_length = 0; | |
257 | for (idx = 0; idx < send_wr->num_sge; idx++) { | |
258 | wqe_p->u.nud.sg_list[idx].vaddr = | |
259 | send_wr->sg_list[idx].addr; | |
260 | wqe_p->u.nud.sg_list[idx].lkey = | |
261 | send_wr->sg_list[idx].lkey; | |
262 | wqe_p->u.nud.sg_list[idx].length = | |
263 | send_wr->sg_list[idx].length; | |
264 | dma_length += send_wr->sg_list[idx].length; | |
265 | } /* eof idx */ | |
266 | wqe_p->u.nud.atomic_1st_op_dma_len = dma_length; | |
267 | ||
2ec8e662 JF |
268 | /* unsolicited ack circumvention */ |
269 | if (send_wr->opcode == IB_WR_RDMA_READ) { | |
270 | /* on RDMA read, switch on and reset counters */ | |
271 | qp->message_count = qp->packet_count = 0; | |
272 | qp->unsol_ack_circ = 1; | |
273 | } else | |
274 | /* else estimate #packets */ | |
275 | qp->packet_count += (dma_length >> qp->mtu_shift) + 1; | |
276 | ||
fab97220 HS |
277 | break; |
278 | ||
279 | default: | |
280 | ehca_gen_err("Invalid qptype=%x", qp->qp_type); | |
281 | return -EINVAL; | |
282 | } | |
283 | ||
4da27d6d | 284 | if (ehca_debug_level >= 3) { |
fab97220 HS |
285 | ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp); |
286 | ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe"); | |
287 | } | |
288 | return 0; | |
289 | } | |
290 | ||
291 | /* map_ib_wc_status converts raw cqe_status to ib_wc_status */ | |
292 | static inline void map_ib_wc_status(u32 cqe_status, | |
293 | enum ib_wc_status *wc_status) | |
294 | { | |
295 | if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) { | |
296 | switch (cqe_status & 0x3F) { | |
297 | case 0x01: | |
298 | case 0x21: | |
299 | *wc_status = IB_WC_LOC_LEN_ERR; | |
300 | break; | |
301 | case 0x02: | |
302 | case 0x22: | |
303 | *wc_status = IB_WC_LOC_QP_OP_ERR; | |
304 | break; | |
305 | case 0x03: | |
306 | case 0x23: | |
307 | *wc_status = IB_WC_LOC_EEC_OP_ERR; | |
308 | break; | |
309 | case 0x04: | |
310 | case 0x24: | |
311 | *wc_status = IB_WC_LOC_PROT_ERR; | |
312 | break; | |
313 | case 0x05: | |
314 | case 0x25: | |
315 | *wc_status = IB_WC_WR_FLUSH_ERR; | |
316 | break; | |
317 | case 0x06: | |
318 | *wc_status = IB_WC_MW_BIND_ERR; | |
319 | break; | |
320 | case 0x07: /* remote error - look into bits 20:24 */ | |
321 | switch ((cqe_status | |
322 | & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) { | |
323 | case 0x0: | |
324 | /* | |
325 | * PSN Sequence Error! | |
326 | * couldn't find a matching status! | |
327 | */ | |
328 | *wc_status = IB_WC_GENERAL_ERR; | |
329 | break; | |
330 | case 0x1: | |
331 | *wc_status = IB_WC_REM_INV_REQ_ERR; | |
332 | break; | |
333 | case 0x2: | |
334 | *wc_status = IB_WC_REM_ACCESS_ERR; | |
335 | break; | |
336 | case 0x3: | |
337 | *wc_status = IB_WC_REM_OP_ERR; | |
338 | break; | |
339 | case 0x4: | |
340 | *wc_status = IB_WC_REM_INV_RD_REQ_ERR; | |
341 | break; | |
342 | } | |
343 | break; | |
344 | case 0x08: | |
345 | *wc_status = IB_WC_RETRY_EXC_ERR; | |
346 | break; | |
347 | case 0x09: | |
348 | *wc_status = IB_WC_RNR_RETRY_EXC_ERR; | |
349 | break; | |
350 | case 0x0A: | |
351 | case 0x2D: | |
352 | *wc_status = IB_WC_REM_ABORT_ERR; | |
353 | break; | |
354 | case 0x0B: | |
355 | case 0x2E: | |
356 | *wc_status = IB_WC_INV_EECN_ERR; | |
357 | break; | |
358 | case 0x0C: | |
359 | case 0x2F: | |
360 | *wc_status = IB_WC_INV_EEC_STATE_ERR; | |
361 | break; | |
362 | case 0x0D: | |
363 | *wc_status = IB_WC_BAD_RESP_ERR; | |
364 | break; | |
365 | case 0x10: | |
366 | /* WQE purged */ | |
367 | *wc_status = IB_WC_WR_FLUSH_ERR; | |
368 | break; | |
369 | default: | |
370 | *wc_status = IB_WC_FATAL_ERR; | |
371 | ||
372 | } | |
373 | } else | |
374 | *wc_status = IB_WC_SUCCESS; | |
375 | } | |
376 | ||
2ec8e662 JF |
377 | static inline int post_one_send(struct ehca_qp *my_qp, |
378 | struct ib_send_wr *cur_send_wr, | |
379 | struct ib_send_wr **bad_send_wr, | |
380 | int hidden) | |
381 | { | |
382 | struct ehca_wqe *wqe_p; | |
383 | int ret; | |
384 | u64 start_offset = my_qp->ipz_squeue.current_q_offset; | |
385 | ||
386 | /* get pointer next to free WQE */ | |
387 | wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue); | |
388 | if (unlikely(!wqe_p)) { | |
389 | /* too many posted work requests: queue overflow */ | |
390 | if (bad_send_wr) | |
391 | *bad_send_wr = cur_send_wr; | |
392 | ehca_err(my_qp->ib_qp.device, "Too many posted WQEs " | |
393 | "qp_num=%x", my_qp->ib_qp.qp_num); | |
394 | return -ENOMEM; | |
395 | } | |
396 | /* write a SEND WQE into the QUEUE */ | |
397 | ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr, hidden); | |
398 | /* | |
399 | * if something failed, | |
400 | * reset the free entry pointer to the start value | |
401 | */ | |
402 | if (unlikely(ret)) { | |
403 | my_qp->ipz_squeue.current_q_offset = start_offset; | |
404 | if (bad_send_wr) | |
405 | *bad_send_wr = cur_send_wr; | |
406 | ehca_err(my_qp->ib_qp.device, "Could not write WQE " | |
407 | "qp_num=%x", my_qp->ib_qp.qp_num); | |
408 | return -EINVAL; | |
409 | } | |
410 | ||
411 | return 0; | |
412 | } | |
413 | ||
fab97220 HS |
414 | int ehca_post_send(struct ib_qp *qp, |
415 | struct ib_send_wr *send_wr, | |
416 | struct ib_send_wr **bad_send_wr) | |
417 | { | |
418 | struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp); | |
419 | struct ib_send_wr *cur_send_wr; | |
fab97220 HS |
420 | int wqe_cnt = 0; |
421 | int ret = 0; | |
9844b71b | 422 | unsigned long flags; |
fab97220 | 423 | |
088af154 JF |
424 | /* Reject WR if QP is in RESET, INIT or RTR state */ |
425 | if (unlikely(my_qp->state < IB_QPS_RTS)) { | |
426 | ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x", | |
427 | my_qp->state, qp->qp_num); | |
863fb09f JF |
428 | return -EINVAL; |
429 | } | |
430 | ||
fab97220 | 431 | /* LOCK the QUEUE */ |
9844b71b | 432 | spin_lock_irqsave(&my_qp->spinlock_s, flags); |
fab97220 | 433 | |
2ec8e662 JF |
434 | /* Send an empty extra RDMA read if: |
435 | * 1) there has been an RDMA read on this connection before | |
436 | * 2) no RDMA read occurred for ACK_CIRC_THRESHOLD link packets | |
437 | * 3) we can be sure that any previous extra RDMA read has been | |
438 | * processed so we don't overflow the SQ | |
439 | */ | |
440 | if (unlikely(my_qp->unsol_ack_circ && | |
441 | my_qp->packet_count > ACK_CIRC_THRESHOLD && | |
442 | my_qp->message_count > my_qp->init_attr.cap.max_send_wr)) { | |
443 | /* insert an empty RDMA READ to fix up the remote QP state */ | |
444 | struct ib_send_wr circ_wr; | |
445 | memset(&circ_wr, 0, sizeof(circ_wr)); | |
446 | circ_wr.opcode = IB_WR_RDMA_READ; | |
447 | post_one_send(my_qp, &circ_wr, NULL, 1); /* ignore retcode */ | |
448 | wqe_cnt++; | |
449 | ehca_dbg(qp->device, "posted circ wr qp_num=%x", qp->qp_num); | |
450 | my_qp->message_count = my_qp->packet_count = 0; | |
451 | } | |
452 | ||
fab97220 HS |
453 | /* loop processes list of send reqs */ |
454 | for (cur_send_wr = send_wr; cur_send_wr != NULL; | |
455 | cur_send_wr = cur_send_wr->next) { | |
2ec8e662 | 456 | ret = post_one_send(my_qp, cur_send_wr, bad_send_wr, 0); |
fab97220 | 457 | if (unlikely(ret)) { |
2ec8e662 JF |
458 | /* if one or more WQEs were successful, don't fail */ |
459 | if (wqe_cnt) | |
460 | ret = 0; | |
fab97220 HS |
461 | goto post_send_exit0; |
462 | } | |
463 | wqe_cnt++; | |
fab97220 HS |
464 | } /* eof for cur_send_wr */ |
465 | ||
466 | post_send_exit0: | |
fab97220 HS |
467 | iosync(); /* serialize GAL register access */ |
468 | hipz_update_sqa(my_qp, wqe_cnt); | |
4da27d6d JF |
469 | if (unlikely(ret || ehca_debug_level >= 2)) |
470 | ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i", | |
471 | my_qp, qp->qp_num, wqe_cnt, ret); | |
2ec8e662 | 472 | my_qp->message_count += wqe_cnt; |
f72d2f08 | 473 | spin_unlock_irqrestore(&my_qp->spinlock_s, flags); |
fab97220 HS |
474 | return ret; |
475 | } | |
476 | ||
a6a12947 JF |
477 | static int internal_post_recv(struct ehca_qp *my_qp, |
478 | struct ib_device *dev, | |
479 | struct ib_recv_wr *recv_wr, | |
480 | struct ib_recv_wr **bad_recv_wr) | |
fab97220 | 481 | { |
fab97220 HS |
482 | struct ib_recv_wr *cur_recv_wr; |
483 | struct ehca_wqe *wqe_p; | |
484 | int wqe_cnt = 0; | |
485 | int ret = 0; | |
9844b71b | 486 | unsigned long flags; |
fab97220 | 487 | |
a6a12947 JF |
488 | if (unlikely(!HAS_RQ(my_qp))) { |
489 | ehca_err(dev, "QP has no RQ ehca_qp=%p qp_num=%x ext_type=%d", | |
490 | my_qp, my_qp->real_qp_num, my_qp->ext_type); | |
491 | return -ENODEV; | |
492 | } | |
493 | ||
fab97220 | 494 | /* LOCK the QUEUE */ |
9844b71b | 495 | spin_lock_irqsave(&my_qp->spinlock_r, flags); |
fab97220 HS |
496 | |
497 | /* loop processes list of send reqs */ | |
498 | for (cur_recv_wr = recv_wr; cur_recv_wr != NULL; | |
499 | cur_recv_wr = cur_recv_wr->next) { | |
500 | u64 start_offset = my_qp->ipz_rqueue.current_q_offset; | |
501 | /* get pointer next to free WQE */ | |
502 | wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue); | |
503 | if (unlikely(!wqe_p)) { | |
504 | /* too many posted work requests: queue overflow */ | |
505 | if (bad_recv_wr) | |
506 | *bad_recv_wr = cur_recv_wr; | |
507 | if (wqe_cnt == 0) { | |
508 | ret = -ENOMEM; | |
a6a12947 JF |
509 | ehca_err(dev, "Too many posted WQEs " |
510 | "qp_num=%x", my_qp->real_qp_num); | |
fab97220 HS |
511 | } |
512 | goto post_recv_exit0; | |
513 | } | |
514 | /* write a RECV WQE into the QUEUE */ | |
515 | ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, cur_recv_wr); | |
516 | /* | |
517 | * if something failed, | |
518 | * reset the free entry pointer to the start value | |
519 | */ | |
520 | if (unlikely(ret)) { | |
521 | my_qp->ipz_rqueue.current_q_offset = start_offset; | |
522 | *bad_recv_wr = cur_recv_wr; | |
523 | if (wqe_cnt == 0) { | |
524 | ret = -EINVAL; | |
a6a12947 JF |
525 | ehca_err(dev, "Could not write WQE " |
526 | "qp_num=%x", my_qp->real_qp_num); | |
fab97220 HS |
527 | } |
528 | goto post_recv_exit0; | |
529 | } | |
530 | wqe_cnt++; | |
fab97220 HS |
531 | } /* eof for cur_recv_wr */ |
532 | ||
533 | post_recv_exit0: | |
fab97220 HS |
534 | iosync(); /* serialize GAL register access */ |
535 | hipz_update_rqa(my_qp, wqe_cnt); | |
4da27d6d JF |
536 | if (unlikely(ret || ehca_debug_level >= 2)) |
537 | ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i", | |
538 | my_qp, my_qp->real_qp_num, wqe_cnt, ret); | |
f72d2f08 | 539 | spin_unlock_irqrestore(&my_qp->spinlock_r, flags); |
fab97220 HS |
540 | return ret; |
541 | } | |
542 | ||
a6a12947 JF |
543 | int ehca_post_recv(struct ib_qp *qp, |
544 | struct ib_recv_wr *recv_wr, | |
545 | struct ib_recv_wr **bad_recv_wr) | |
546 | { | |
547 | return internal_post_recv(container_of(qp, struct ehca_qp, ib_qp), | |
548 | qp->device, recv_wr, bad_recv_wr); | |
549 | } | |
550 | ||
551 | int ehca_post_srq_recv(struct ib_srq *srq, | |
552 | struct ib_recv_wr *recv_wr, | |
553 | struct ib_recv_wr **bad_recv_wr) | |
554 | { | |
555 | return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq), | |
556 | srq->device, recv_wr, bad_recv_wr); | |
557 | } | |
558 | ||
fab97220 HS |
559 | /* |
560 | * ib_wc_opcode table converts ehca wc opcode to ib | |
561 | * Since we use zero to indicate invalid opcode, the actual ib opcode must | |
562 | * be decremented!!! | |
563 | */ | |
564 | static const u8 ib_wc_opcode[255] = { | |
565 | [0x01] = IB_WC_RECV+1, | |
566 | [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1, | |
567 | [0x04] = IB_WC_BIND_MW+1, | |
568 | [0x08] = IB_WC_FETCH_ADD+1, | |
569 | [0x10] = IB_WC_COMP_SWAP+1, | |
570 | [0x20] = IB_WC_RDMA_WRITE+1, | |
571 | [0x40] = IB_WC_RDMA_READ+1, | |
572 | [0x80] = IB_WC_SEND+1 | |
573 | }; | |
574 | ||
575 | /* internal function to poll one entry of cq */ | |
576 | static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc) | |
577 | { | |
578 | int ret = 0; | |
579 | struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq); | |
580 | struct ehca_cqe *cqe; | |
b1cfe43d | 581 | struct ehca_qp *my_qp; |
4da27d6d | 582 | int cqe_count = 0, is_error; |
fab97220 HS |
583 | |
584 | poll_cq_one_read_cqe: | |
585 | cqe = (struct ehca_cqe *) | |
586 | ipz_qeit_get_inc_valid(&my_cq->ipz_queue); | |
587 | if (!cqe) { | |
588 | ret = -EAGAIN; | |
4da27d6d JF |
589 | if (ehca_debug_level >= 3) |
590 | ehca_dbg(cq->device, "Completion queue is empty " | |
591 | "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number); | |
592 | goto poll_cq_one_exit0; | |
fab97220 HS |
593 | } |
594 | ||
595 | /* prevents loads being reordered across this point */ | |
596 | rmb(); | |
597 | ||
598 | cqe_count++; | |
599 | if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) { | |
2b94397a | 600 | struct ehca_qp *qp; |
fab97220 | 601 | int purgeflag; |
9844b71b | 602 | unsigned long flags; |
2b94397a HNN |
603 | |
604 | qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number); | |
fab97220 HS |
605 | if (!qp) { |
606 | ehca_err(cq->device, "cq_num=%x qp_num=%x " | |
607 | "could not find qp -> ignore cqe", | |
608 | my_cq->cq_number, cqe->local_qp_number); | |
609 | ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x", | |
610 | my_cq->cq_number, cqe->local_qp_number); | |
611 | /* ignore this purged cqe */ | |
612 | goto poll_cq_one_read_cqe; | |
613 | } | |
9844b71b | 614 | spin_lock_irqsave(&qp->spinlock_s, flags); |
fab97220 | 615 | purgeflag = qp->sqerr_purgeflag; |
9844b71b | 616 | spin_unlock_irqrestore(&qp->spinlock_s, flags); |
fab97220 HS |
617 | |
618 | if (purgeflag) { | |
2b94397a HNN |
619 | ehca_dbg(cq->device, |
620 | "Got CQE with purged bit qp_num=%x src_qp=%x", | |
fab97220 | 621 | cqe->local_qp_number, cqe->remote_qp_number); |
4da27d6d | 622 | if (ehca_debug_level >= 2) |
fab97220 HS |
623 | ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x", |
624 | cqe->local_qp_number, | |
625 | cqe->remote_qp_number); | |
626 | /* | |
627 | * ignore this to avoid double cqes of bad wqe | |
628 | * that caused sqe and turn off purge flag | |
629 | */ | |
630 | qp->sqerr_purgeflag = 0; | |
631 | goto poll_cq_one_read_cqe; | |
632 | } | |
633 | } | |
634 | ||
4da27d6d JF |
635 | is_error = cqe->status & WC_STATUS_ERROR_BIT; |
636 | ||
637 | /* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */ | |
638 | if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) { | |
fab97220 | 639 | ehca_dbg(cq->device, |
4da27d6d JF |
640 | "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----", |
641 | is_error ? "ERROR " : "", my_cq, my_cq->cq_number); | |
fab97220 HS |
642 | ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x", |
643 | my_cq, my_cq->cq_number); | |
644 | ehca_dbg(cq->device, | |
645 | "ehca_cq=%p cq_num=%x -------------------------", | |
646 | my_cq, my_cq->cq_number); | |
647 | } | |
648 | ||
649 | /* we got a completion! */ | |
650 | wc->wr_id = cqe->work_request_id; | |
651 | ||
652 | /* eval ib_wc_opcode */ | |
653 | wc->opcode = ib_wc_opcode[cqe->optype]-1; | |
654 | if (unlikely(wc->opcode == -1)) { | |
655 | ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x " | |
656 | "ehca_cq=%p cq_num=%x", | |
657 | cqe->optype, cqe->status, my_cq, my_cq->cq_number); | |
658 | /* dump cqe for other infos */ | |
659 | ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x", | |
660 | my_cq, my_cq->cq_number); | |
661 | /* update also queue adder to throw away this entry!!! */ | |
662 | goto poll_cq_one_exit0; | |
663 | } | |
4da27d6d | 664 | |
fab97220 | 665 | /* eval ib_wc_status */ |
4da27d6d | 666 | if (unlikely(is_error)) { |
fab97220 HS |
667 | /* complete with errors */ |
668 | map_ib_wc_status(cqe->status, &wc->status); | |
669 | wc->vendor_err = wc->status; | |
670 | } else | |
671 | wc->status = IB_WC_SUCCESS; | |
672 | ||
b1cfe43d JF |
673 | read_lock(&ehca_qp_idr_lock); |
674 | my_qp = idr_find(&ehca_qp_idr, cqe->qp_token); | |
675 | wc->qp = &my_qp->ib_qp; | |
676 | read_unlock(&ehca_qp_idr_lock); | |
677 | ||
fab97220 HS |
678 | wc->byte_len = cqe->nr_bytes_transferred; |
679 | wc->pkey_index = cqe->pkey_index; | |
680 | wc->slid = cqe->rlid; | |
681 | wc->dlid_path_bits = cqe->dlid; | |
682 | wc->src_qp = cqe->remote_qp_number; | |
683 | wc->wc_flags = cqe->w_completion_flags; | |
00f7ec36 | 684 | wc->ex.imm_data = cpu_to_be32(cqe->immediate_data); |
fab97220 HS |
685 | wc->sl = cqe->service_level; |
686 | ||
fab97220 HS |
687 | poll_cq_one_exit0: |
688 | if (cqe_count > 0) | |
689 | hipz_update_feca(my_cq, cqe_count); | |
690 | ||
691 | return ret; | |
692 | } | |
693 | ||
694 | int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc) | |
695 | { | |
696 | struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq); | |
697 | int nr; | |
698 | struct ib_wc *current_wc = wc; | |
699 | int ret = 0; | |
9844b71b | 700 | unsigned long flags; |
fab97220 HS |
701 | |
702 | if (num_entries < 1) { | |
703 | ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p " | |
704 | "cq_num=%x", num_entries, my_cq, my_cq->cq_number); | |
705 | ret = -EINVAL; | |
706 | goto poll_cq_exit0; | |
707 | } | |
708 | ||
9844b71b | 709 | spin_lock_irqsave(&my_cq->spinlock, flags); |
fab97220 HS |
710 | for (nr = 0; nr < num_entries; nr++) { |
711 | ret = ehca_poll_cq_one(cq, current_wc); | |
712 | if (ret) | |
713 | break; | |
714 | current_wc++; | |
715 | } /* eof for nr */ | |
9844b71b | 716 | spin_unlock_irqrestore(&my_cq->spinlock, flags); |
fab97220 HS |
717 | if (ret == -EAGAIN || !ret) |
718 | ret = nr; | |
719 | ||
720 | poll_cq_exit0: | |
721 | return ret; | |
722 | } | |
723 | ||
ed23a727 | 724 | int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags) |
fab97220 HS |
725 | { |
726 | struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq); | |
ed23a727 | 727 | int ret = 0; |
fab97220 | 728 | |
ed23a727 | 729 | switch (notify_flags & IB_CQ_SOLICITED_MASK) { |
fab97220 HS |
730 | case IB_CQ_SOLICITED: |
731 | hipz_set_cqx_n0(my_cq, 1); | |
732 | break; | |
733 | case IB_CQ_NEXT_COMP: | |
734 | hipz_set_cqx_n1(my_cq, 1); | |
735 | break; | |
736 | default: | |
737 | return -EINVAL; | |
738 | } | |
739 | ||
ed23a727 | 740 | if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) { |
fffba373 | 741 | unsigned long spl_flags; |
ed23a727 RD |
742 | spin_lock_irqsave(&my_cq->spinlock, spl_flags); |
743 | ret = ipz_qeit_is_valid(&my_cq->ipz_queue); | |
744 | spin_unlock_irqrestore(&my_cq->spinlock, spl_flags); | |
745 | } | |
746 | ||
747 | return ret; | |
fab97220 | 748 | } |