]>
Commit | Line | Data |
---|---|---|
8700e3e7 MS |
1 | /* |
2 | * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. | |
3 | * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. | |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | */ | |
33 | ||
34 | #include <linux/skbuff.h> | |
35 | ||
36 | #include "rxe.h" | |
37 | #include "rxe_loc.h" | |
38 | #include "rxe_queue.h" | |
39 | #include "rxe_task.h" | |
40 | ||
41 | enum comp_state { | |
42 | COMPST_GET_ACK, | |
43 | COMPST_GET_WQE, | |
44 | COMPST_COMP_WQE, | |
45 | COMPST_COMP_ACK, | |
46 | COMPST_CHECK_PSN, | |
47 | COMPST_CHECK_ACK, | |
48 | COMPST_READ, | |
49 | COMPST_ATOMIC, | |
50 | COMPST_WRITE_SEND, | |
51 | COMPST_UPDATE_COMP, | |
52 | COMPST_ERROR_RETRY, | |
53 | COMPST_RNR_RETRY, | |
54 | COMPST_ERROR, | |
55 | COMPST_EXIT, /* We have an issue, and we want to rerun the completer */ | |
56 | COMPST_DONE, /* The completer finished successflly */ | |
57 | }; | |
58 | ||
59 | static char *comp_state_name[] = { | |
60 | [COMPST_GET_ACK] = "GET ACK", | |
61 | [COMPST_GET_WQE] = "GET WQE", | |
62 | [COMPST_COMP_WQE] = "COMP WQE", | |
63 | [COMPST_COMP_ACK] = "COMP ACK", | |
64 | [COMPST_CHECK_PSN] = "CHECK PSN", | |
65 | [COMPST_CHECK_ACK] = "CHECK ACK", | |
66 | [COMPST_READ] = "READ", | |
67 | [COMPST_ATOMIC] = "ATOMIC", | |
68 | [COMPST_WRITE_SEND] = "WRITE/SEND", | |
69 | [COMPST_UPDATE_COMP] = "UPDATE COMP", | |
70 | [COMPST_ERROR_RETRY] = "ERROR RETRY", | |
71 | [COMPST_RNR_RETRY] = "RNR RETRY", | |
72 | [COMPST_ERROR] = "ERROR", | |
73 | [COMPST_EXIT] = "EXIT", | |
74 | [COMPST_DONE] = "DONE", | |
75 | }; | |
76 | ||
77 | static unsigned long rnrnak_usec[32] = { | |
78 | [IB_RNR_TIMER_655_36] = 655360, | |
79 | [IB_RNR_TIMER_000_01] = 10, | |
80 | [IB_RNR_TIMER_000_02] = 20, | |
81 | [IB_RNR_TIMER_000_03] = 30, | |
82 | [IB_RNR_TIMER_000_04] = 40, | |
83 | [IB_RNR_TIMER_000_06] = 60, | |
84 | [IB_RNR_TIMER_000_08] = 80, | |
85 | [IB_RNR_TIMER_000_12] = 120, | |
86 | [IB_RNR_TIMER_000_16] = 160, | |
87 | [IB_RNR_TIMER_000_24] = 240, | |
88 | [IB_RNR_TIMER_000_32] = 320, | |
89 | [IB_RNR_TIMER_000_48] = 480, | |
90 | [IB_RNR_TIMER_000_64] = 640, | |
91 | [IB_RNR_TIMER_000_96] = 960, | |
92 | [IB_RNR_TIMER_001_28] = 1280, | |
93 | [IB_RNR_TIMER_001_92] = 1920, | |
94 | [IB_RNR_TIMER_002_56] = 2560, | |
95 | [IB_RNR_TIMER_003_84] = 3840, | |
96 | [IB_RNR_TIMER_005_12] = 5120, | |
97 | [IB_RNR_TIMER_007_68] = 7680, | |
98 | [IB_RNR_TIMER_010_24] = 10240, | |
99 | [IB_RNR_TIMER_015_36] = 15360, | |
100 | [IB_RNR_TIMER_020_48] = 20480, | |
101 | [IB_RNR_TIMER_030_72] = 30720, | |
102 | [IB_RNR_TIMER_040_96] = 40960, | |
103 | [IB_RNR_TIMER_061_44] = 61410, | |
104 | [IB_RNR_TIMER_081_92] = 81920, | |
105 | [IB_RNR_TIMER_122_88] = 122880, | |
106 | [IB_RNR_TIMER_163_84] = 163840, | |
107 | [IB_RNR_TIMER_245_76] = 245760, | |
108 | [IB_RNR_TIMER_327_68] = 327680, | |
109 | [IB_RNR_TIMER_491_52] = 491520, | |
110 | }; | |
111 | ||
112 | static inline unsigned long rnrnak_jiffies(u8 timeout) | |
113 | { | |
114 | return max_t(unsigned long, | |
115 | usecs_to_jiffies(rnrnak_usec[timeout]), 1); | |
116 | } | |
117 | ||
118 | static enum ib_wc_opcode wr_to_wc_opcode(enum ib_wr_opcode opcode) | |
119 | { | |
120 | switch (opcode) { | |
121 | case IB_WR_RDMA_WRITE: return IB_WC_RDMA_WRITE; | |
122 | case IB_WR_RDMA_WRITE_WITH_IMM: return IB_WC_RDMA_WRITE; | |
123 | case IB_WR_SEND: return IB_WC_SEND; | |
124 | case IB_WR_SEND_WITH_IMM: return IB_WC_SEND; | |
125 | case IB_WR_RDMA_READ: return IB_WC_RDMA_READ; | |
126 | case IB_WR_ATOMIC_CMP_AND_SWP: return IB_WC_COMP_SWAP; | |
127 | case IB_WR_ATOMIC_FETCH_AND_ADD: return IB_WC_FETCH_ADD; | |
128 | case IB_WR_LSO: return IB_WC_LSO; | |
129 | case IB_WR_SEND_WITH_INV: return IB_WC_SEND; | |
130 | case IB_WR_RDMA_READ_WITH_INV: return IB_WC_RDMA_READ; | |
131 | case IB_WR_LOCAL_INV: return IB_WC_LOCAL_INV; | |
132 | case IB_WR_REG_MR: return IB_WC_REG_MR; | |
133 | ||
134 | default: | |
135 | return 0xff; | |
136 | } | |
137 | } | |
138 | ||
3bfbea74 | 139 | void retransmit_timer(struct timer_list *t) |
8700e3e7 | 140 | { |
3bfbea74 | 141 | struct rxe_qp *qp = from_timer(qp, t, retrans_timer); |
8700e3e7 MS |
142 | |
143 | if (qp->valid) { | |
144 | qp->comp.timeout = 1; | |
145 | rxe_run_task(&qp->comp.task, 1); | |
146 | } | |
147 | } | |
148 | ||
9802c335 | 149 | void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) |
8700e3e7 MS |
150 | { |
151 | int must_sched; | |
152 | ||
153 | skb_queue_tail(&qp->resp_pkts, skb); | |
154 | ||
155 | must_sched = skb_queue_len(&qp->resp_pkts) > 1; | |
0b1e5b99 | 156 | if (must_sched != 0) |
9802c335 ZY |
157 | rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED); |
158 | ||
8700e3e7 MS |
159 | rxe_run_task(&qp->comp.task, must_sched); |
160 | } | |
161 | ||
162 | static inline enum comp_state get_wqe(struct rxe_qp *qp, | |
163 | struct rxe_pkt_info *pkt, | |
164 | struct rxe_send_wqe **wqe_p) | |
165 | { | |
166 | struct rxe_send_wqe *wqe; | |
167 | ||
168 | /* we come here whether or not we found a response packet to see if | |
169 | * there are any posted WQEs | |
170 | */ | |
171 | wqe = queue_head(qp->sq.queue); | |
172 | *wqe_p = wqe; | |
173 | ||
174 | /* no WQE or requester has not started it yet */ | |
175 | if (!wqe || wqe->state == wqe_state_posted) | |
176 | return pkt ? COMPST_DONE : COMPST_EXIT; | |
177 | ||
178 | /* WQE does not require an ack */ | |
179 | if (wqe->state == wqe_state_done) | |
180 | return COMPST_COMP_WQE; | |
181 | ||
182 | /* WQE caused an error */ | |
183 | if (wqe->state == wqe_state_error) | |
184 | return COMPST_ERROR; | |
185 | ||
186 | /* we have a WQE, if we also have an ack check its PSN */ | |
187 | return pkt ? COMPST_CHECK_PSN : COMPST_EXIT; | |
188 | } | |
189 | ||
190 | static inline void reset_retry_counters(struct rxe_qp *qp) | |
191 | { | |
192 | qp->comp.retry_cnt = qp->attr.retry_cnt; | |
193 | qp->comp.rnr_retry = qp->attr.rnr_retry; | |
4e4c53df | 194 | qp->comp.started_retry = 0; |
8700e3e7 MS |
195 | } |
196 | ||
197 | static inline enum comp_state check_psn(struct rxe_qp *qp, | |
198 | struct rxe_pkt_info *pkt, | |
199 | struct rxe_send_wqe *wqe) | |
200 | { | |
201 | s32 diff; | |
202 | ||
203 | /* check to see if response is past the oldest WQE. if it is, complete | |
204 | * send/write or error read/atomic | |
205 | */ | |
206 | diff = psn_compare(pkt->psn, wqe->last_psn); | |
207 | if (diff > 0) { | |
208 | if (wqe->state == wqe_state_pending) { | |
209 | if (wqe->mask & WR_ATOMIC_OR_READ_MASK) | |
210 | return COMPST_ERROR_RETRY; | |
211 | ||
212 | reset_retry_counters(qp); | |
213 | return COMPST_COMP_WQE; | |
214 | } else { | |
215 | return COMPST_DONE; | |
216 | } | |
217 | } | |
218 | ||
219 | /* compare response packet to expected response */ | |
220 | diff = psn_compare(pkt->psn, qp->comp.psn); | |
221 | if (diff < 0) { | |
222 | /* response is most likely a retried packet if it matches an | |
223 | * uncompleted WQE go complete it else ignore it | |
224 | */ | |
225 | if (pkt->psn == wqe->last_psn) | |
226 | return COMPST_COMP_ACK; | |
227 | else | |
228 | return COMPST_DONE; | |
229 | } else if ((diff > 0) && (wqe->mask & WR_ATOMIC_OR_READ_MASK)) { | |
cbf1f9a4 | 230 | return COMPST_DONE; |
8700e3e7 MS |
231 | } else { |
232 | return COMPST_CHECK_ACK; | |
233 | } | |
234 | } | |
235 | ||
236 | static inline enum comp_state check_ack(struct rxe_qp *qp, | |
237 | struct rxe_pkt_info *pkt, | |
238 | struct rxe_send_wqe *wqe) | |
239 | { | |
240 | unsigned int mask = pkt->mask; | |
241 | u8 syn; | |
0b1e5b99 | 242 | struct rxe_dev *rxe = to_rdev(qp->ibqp.device); |
8700e3e7 MS |
243 | |
244 | /* Check the sequence only */ | |
245 | switch (qp->comp.opcode) { | |
246 | case -1: | |
247 | /* Will catch all *_ONLY cases. */ | |
248 | if (!(mask & RXE_START_MASK)) | |
249 | return COMPST_ERROR; | |
250 | ||
251 | break; | |
252 | ||
253 | case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST: | |
254 | case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE: | |
255 | if (pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE && | |
256 | pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST) { | |
030e46e4 VI |
257 | /* read retries of partial data may restart from |
258 | * read response first or response only. | |
259 | */ | |
260 | if ((pkt->psn == wqe->first_psn && | |
261 | pkt->opcode == | |
262 | IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST) || | |
263 | (wqe->first_psn == wqe->last_psn && | |
264 | pkt->opcode == | |
265 | IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY)) | |
266 | break; | |
267 | ||
8700e3e7 MS |
268 | return COMPST_ERROR; |
269 | } | |
270 | break; | |
271 | default: | |
43553b47 | 272 | WARN_ON_ONCE(1); |
8700e3e7 MS |
273 | } |
274 | ||
275 | /* Check operation validity. */ | |
276 | switch (pkt->opcode) { | |
277 | case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST: | |
278 | case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST: | |
279 | case IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY: | |
280 | syn = aeth_syn(pkt); | |
281 | ||
282 | if ((syn & AETH_TYPE_MASK) != AETH_ACK) | |
283 | return COMPST_ERROR; | |
284 | ||
df561f66 | 285 | fallthrough; |
ea6ee93b | 286 | /* (IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE doesn't have an AETH) |
8700e3e7 MS |
287 | */ |
288 | case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE: | |
289 | if (wqe->wr.opcode != IB_WR_RDMA_READ && | |
290 | wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV) { | |
61b717d0 | 291 | wqe->status = IB_WC_FATAL_ERR; |
8700e3e7 MS |
292 | return COMPST_ERROR; |
293 | } | |
294 | reset_retry_counters(qp); | |
295 | return COMPST_READ; | |
296 | ||
297 | case IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE: | |
298 | syn = aeth_syn(pkt); | |
299 | ||
300 | if ((syn & AETH_TYPE_MASK) != AETH_ACK) | |
301 | return COMPST_ERROR; | |
302 | ||
303 | if (wqe->wr.opcode != IB_WR_ATOMIC_CMP_AND_SWP && | |
304 | wqe->wr.opcode != IB_WR_ATOMIC_FETCH_AND_ADD) | |
305 | return COMPST_ERROR; | |
306 | reset_retry_counters(qp); | |
307 | return COMPST_ATOMIC; | |
308 | ||
309 | case IB_OPCODE_RC_ACKNOWLEDGE: | |
310 | syn = aeth_syn(pkt); | |
311 | switch (syn & AETH_TYPE_MASK) { | |
312 | case AETH_ACK: | |
313 | reset_retry_counters(qp); | |
314 | return COMPST_WRITE_SEND; | |
315 | ||
316 | case AETH_RNR_NAK: | |
0b1e5b99 | 317 | rxe_counter_inc(rxe, RXE_CNT_RCV_RNR); |
8700e3e7 MS |
318 | return COMPST_RNR_RETRY; |
319 | ||
320 | case AETH_NAK: | |
321 | switch (syn) { | |
322 | case AETH_NAK_PSN_SEQ_ERROR: | |
323 | /* a nak implicitly acks all packets with psns | |
324 | * before | |
325 | */ | |
326 | if (psn_compare(pkt->psn, qp->comp.psn) > 0) { | |
0b1e5b99 YC |
327 | rxe_counter_inc(rxe, |
328 | RXE_CNT_RCV_SEQ_ERR); | |
8700e3e7 MS |
329 | qp->comp.psn = pkt->psn; |
330 | if (qp->req.wait_psn) { | |
331 | qp->req.wait_psn = 0; | |
8ac0e664 | 332 | rxe_run_task(&qp->req.task, 0); |
8700e3e7 MS |
333 | } |
334 | } | |
335 | return COMPST_ERROR_RETRY; | |
336 | ||
337 | case AETH_NAK_INVALID_REQ: | |
338 | wqe->status = IB_WC_REM_INV_REQ_ERR; | |
339 | return COMPST_ERROR; | |
340 | ||
341 | case AETH_NAK_REM_ACC_ERR: | |
342 | wqe->status = IB_WC_REM_ACCESS_ERR; | |
343 | return COMPST_ERROR; | |
344 | ||
345 | case AETH_NAK_REM_OP_ERR: | |
346 | wqe->status = IB_WC_REM_OP_ERR; | |
347 | return COMPST_ERROR; | |
348 | ||
349 | default: | |
350 | pr_warn("unexpected nak %x\n", syn); | |
351 | wqe->status = IB_WC_REM_OP_ERR; | |
352 | return COMPST_ERROR; | |
353 | } | |
354 | ||
355 | default: | |
356 | return COMPST_ERROR; | |
357 | } | |
358 | break; | |
359 | ||
360 | default: | |
361 | pr_warn("unexpected opcode\n"); | |
362 | } | |
363 | ||
364 | return COMPST_ERROR; | |
365 | } | |
366 | ||
367 | static inline enum comp_state do_read(struct rxe_qp *qp, | |
368 | struct rxe_pkt_info *pkt, | |
369 | struct rxe_send_wqe *wqe) | |
370 | { | |
8700e3e7 MS |
371 | int ret; |
372 | ||
e12ee8ce | 373 | ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, |
8700e3e7 MS |
374 | &wqe->dma, payload_addr(pkt), |
375 | payload_size(pkt), to_mem_obj, NULL); | |
376 | if (ret) | |
377 | return COMPST_ERROR; | |
378 | ||
379 | if (wqe->dma.resid == 0 && (pkt->mask & RXE_END_MASK)) | |
380 | return COMPST_COMP_ACK; | |
381 | else | |
382 | return COMPST_UPDATE_COMP; | |
383 | } | |
384 | ||
385 | static inline enum comp_state do_atomic(struct rxe_qp *qp, | |
386 | struct rxe_pkt_info *pkt, | |
387 | struct rxe_send_wqe *wqe) | |
388 | { | |
8700e3e7 MS |
389 | int ret; |
390 | ||
391 | u64 atomic_orig = atmack_orig(pkt); | |
392 | ||
e12ee8ce | 393 | ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, |
8700e3e7 MS |
394 | &wqe->dma, &atomic_orig, |
395 | sizeof(u64), to_mem_obj, NULL); | |
396 | if (ret) | |
397 | return COMPST_ERROR; | |
398 | else | |
399 | return COMPST_COMP_ACK; | |
400 | } | |
401 | ||
402 | static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe, | |
403 | struct rxe_cqe *cqe) | |
404 | { | |
405 | memset(cqe, 0, sizeof(*cqe)); | |
406 | ||
407 | if (!qp->is_user) { | |
408 | struct ib_wc *wc = &cqe->ibwc; | |
409 | ||
410 | wc->wr_id = wqe->wr.wr_id; | |
411 | wc->status = wqe->status; | |
412 | wc->opcode = wr_to_wc_opcode(wqe->wr.opcode); | |
413 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM || | |
414 | wqe->wr.opcode == IB_WR_SEND_WITH_IMM) | |
415 | wc->wc_flags = IB_WC_WITH_IMM; | |
416 | wc->byte_len = wqe->dma.length; | |
417 | wc->qp = &qp->ibqp; | |
418 | } else { | |
419 | struct ib_uverbs_wc *uwc = &cqe->uibwc; | |
420 | ||
421 | uwc->wr_id = wqe->wr.wr_id; | |
422 | uwc->status = wqe->status; | |
423 | uwc->opcode = wr_to_wc_opcode(wqe->wr.opcode); | |
424 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM || | |
425 | wqe->wr.opcode == IB_WR_SEND_WITH_IMM) | |
426 | uwc->wc_flags = IB_WC_WITH_IMM; | |
427 | uwc->byte_len = wqe->dma.length; | |
428 | uwc->qp_num = qp->ibqp.qp_num; | |
429 | } | |
430 | } | |
431 | ||
18d3451c BVA |
432 | /* |
433 | * IBA Spec. Section 10.7.3.1 SIGNALED COMPLETIONS | |
434 | * ---------8<---------8<------------- | |
435 | * ...Note that if a completion error occurs, a Work Completion | |
436 | * will always be generated, even if the signaling | |
437 | * indicator requests an Unsignaled Completion. | |
438 | * ---------8<---------8<------------- | |
439 | */ | |
8700e3e7 MS |
440 | static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe) |
441 | { | |
6e5559b2 | 442 | struct rxe_dev *rxe = to_rdev(qp->ibqp.device); |
8700e3e7 MS |
443 | struct rxe_cqe cqe; |
444 | ||
445 | if ((qp->sq_sig_type == IB_SIGNAL_ALL_WR) || | |
446 | (wqe->wr.send_flags & IB_SEND_SIGNALED) || | |
18d3451c | 447 | wqe->status != IB_WC_SUCCESS) { |
8700e3e7 | 448 | make_send_cqe(qp, wqe, &cqe); |
dd753d87 | 449 | advance_consumer(qp->sq.queue); |
8700e3e7 | 450 | rxe_cq_post(qp->scq, &cqe, 0); |
dd753d87 AB |
451 | } else { |
452 | advance_consumer(qp->sq.queue); | |
8700e3e7 MS |
453 | } |
454 | ||
6e5559b2 AB |
455 | if (wqe->wr.opcode == IB_WR_SEND || |
456 | wqe->wr.opcode == IB_WR_SEND_WITH_IMM || | |
457 | wqe->wr.opcode == IB_WR_SEND_WITH_INV) | |
458 | rxe_counter_inc(rxe, RXE_CNT_RDMA_SEND); | |
459 | ||
8700e3e7 MS |
460 | /* |
461 | * we completed something so let req run again | |
462 | * if it is trying to fence | |
463 | */ | |
464 | if (qp->req.wait_fence) { | |
465 | qp->req.wait_fence = 0; | |
8ac0e664 | 466 | rxe_run_task(&qp->req.task, 0); |
8700e3e7 MS |
467 | } |
468 | } | |
469 | ||
470 | static inline enum comp_state complete_ack(struct rxe_qp *qp, | |
471 | struct rxe_pkt_info *pkt, | |
472 | struct rxe_send_wqe *wqe) | |
473 | { | |
474 | unsigned long flags; | |
475 | ||
476 | if (wqe->has_rd_atomic) { | |
477 | wqe->has_rd_atomic = 0; | |
478 | atomic_inc(&qp->req.rd_atomic); | |
479 | if (qp->req.need_rd_atomic) { | |
480 | qp->comp.timeout_retry = 0; | |
481 | qp->req.need_rd_atomic = 0; | |
8ac0e664 | 482 | rxe_run_task(&qp->req.task, 0); |
8700e3e7 MS |
483 | } |
484 | } | |
485 | ||
486 | if (unlikely(qp->req.state == QP_STATE_DRAIN)) { | |
487 | /* state_lock used by requester & completer */ | |
488 | spin_lock_irqsave(&qp->state_lock, flags); | |
489 | if ((qp->req.state == QP_STATE_DRAIN) && | |
490 | (qp->comp.psn == qp->req.psn)) { | |
491 | qp->req.state = QP_STATE_DRAINED; | |
492 | spin_unlock_irqrestore(&qp->state_lock, flags); | |
493 | ||
494 | if (qp->ibqp.event_handler) { | |
495 | struct ib_event ev; | |
496 | ||
497 | ev.device = qp->ibqp.device; | |
498 | ev.element.qp = &qp->ibqp; | |
499 | ev.event = IB_EVENT_SQ_DRAINED; | |
500 | qp->ibqp.event_handler(&ev, | |
501 | qp->ibqp.qp_context); | |
502 | } | |
503 | } else { | |
504 | spin_unlock_irqrestore(&qp->state_lock, flags); | |
505 | } | |
506 | } | |
507 | ||
508 | do_complete(qp, wqe); | |
509 | ||
510 | if (psn_compare(pkt->psn, qp->comp.psn) >= 0) | |
511 | return COMPST_UPDATE_COMP; | |
512 | else | |
513 | return COMPST_DONE; | |
514 | } | |
515 | ||
516 | static inline enum comp_state complete_wqe(struct rxe_qp *qp, | |
517 | struct rxe_pkt_info *pkt, | |
518 | struct rxe_send_wqe *wqe) | |
519 | { | |
030e46e4 VI |
520 | if (pkt && wqe->state == wqe_state_pending) { |
521 | if (psn_compare(wqe->last_psn, qp->comp.psn) >= 0) { | |
522 | qp->comp.psn = (wqe->last_psn + 1) & BTH_PSN_MASK; | |
523 | qp->comp.opcode = -1; | |
524 | } | |
8700e3e7 MS |
525 | |
526 | if (qp->req.wait_psn) { | |
527 | qp->req.wait_psn = 0; | |
528 | rxe_run_task(&qp->req.task, 1); | |
529 | } | |
530 | } | |
531 | ||
532 | do_complete(qp, wqe); | |
533 | ||
534 | return COMPST_GET_WQE; | |
535 | } | |
536 | ||
723ec9ae | 537 | static void rxe_drain_resp_pkts(struct rxe_qp *qp, bool notify) |
8700e3e7 | 538 | { |
723ec9ae BVA |
539 | struct sk_buff *skb; |
540 | struct rxe_send_wqe *wqe; | |
8700e3e7 | 541 | |
723ec9ae BVA |
542 | while ((skb = skb_dequeue(&qp->resp_pkts))) { |
543 | rxe_drop_ref(qp); | |
544 | kfree_skb(skb); | |
8700e3e7 MS |
545 | } |
546 | ||
723ec9ae BVA |
547 | while ((wqe = queue_head(qp->sq.queue))) { |
548 | if (notify) { | |
8700e3e7 MS |
549 | wqe->status = IB_WC_WR_FLUSH_ERR; |
550 | do_complete(qp, wqe); | |
723ec9ae BVA |
551 | } else { |
552 | advance_consumer(qp->sq.queue); | |
8700e3e7 | 553 | } |
8700e3e7 | 554 | } |
723ec9ae | 555 | } |
8700e3e7 | 556 | |
723ec9ae BVA |
557 | int rxe_completer(void *arg) |
558 | { | |
559 | struct rxe_qp *qp = (struct rxe_qp *)arg; | |
0b1e5b99 | 560 | struct rxe_dev *rxe = to_rdev(qp->ibqp.device); |
d3e53971 | 561 | struct rxe_send_wqe *wqe = NULL; |
723ec9ae BVA |
562 | struct sk_buff *skb = NULL; |
563 | struct rxe_pkt_info *pkt = NULL; | |
564 | enum comp_state state; | |
8700e3e7 | 565 | |
723ec9ae | 566 | rxe_add_ref(qp); |
8700e3e7 | 567 | |
723ec9ae BVA |
568 | if (!qp->valid || qp->req.state == QP_STATE_ERROR || |
569 | qp->req.state == QP_STATE_RESET) { | |
570 | rxe_drain_resp_pkts(qp, qp->valid && | |
571 | qp->req.state == QP_STATE_ERROR); | |
8700e3e7 MS |
572 | goto exit; |
573 | } | |
574 | ||
575 | if (qp->comp.timeout) { | |
576 | qp->comp.timeout_retry = 1; | |
577 | qp->comp.timeout = 0; | |
578 | } else { | |
579 | qp->comp.timeout_retry = 0; | |
580 | } | |
581 | ||
582 | if (qp->req.need_retry) | |
583 | goto exit; | |
584 | ||
585 | state = COMPST_GET_ACK; | |
586 | ||
587 | while (1) { | |
e404f945 PP |
588 | pr_debug("qp#%d state = %s\n", qp_num(qp), |
589 | comp_state_name[state]); | |
8700e3e7 MS |
590 | switch (state) { |
591 | case COMPST_GET_ACK: | |
592 | skb = skb_dequeue(&qp->resp_pkts); | |
593 | if (skb) { | |
594 | pkt = SKB_TO_PKT(skb); | |
595 | qp->comp.timeout_retry = 0; | |
596 | } | |
597 | state = COMPST_GET_WQE; | |
598 | break; | |
599 | ||
600 | case COMPST_GET_WQE: | |
601 | state = get_wqe(qp, pkt, &wqe); | |
602 | break; | |
603 | ||
604 | case COMPST_CHECK_PSN: | |
605 | state = check_psn(qp, pkt, wqe); | |
606 | break; | |
607 | ||
608 | case COMPST_CHECK_ACK: | |
609 | state = check_ack(qp, pkt, wqe); | |
610 | break; | |
611 | ||
612 | case COMPST_READ: | |
613 | state = do_read(qp, pkt, wqe); | |
614 | break; | |
615 | ||
616 | case COMPST_ATOMIC: | |
617 | state = do_atomic(qp, pkt, wqe); | |
618 | break; | |
619 | ||
620 | case COMPST_WRITE_SEND: | |
621 | if (wqe->state == wqe_state_pending && | |
622 | wqe->last_psn == pkt->psn) | |
623 | state = COMPST_COMP_ACK; | |
624 | else | |
625 | state = COMPST_UPDATE_COMP; | |
626 | break; | |
627 | ||
628 | case COMPST_COMP_ACK: | |
629 | state = complete_ack(qp, pkt, wqe); | |
630 | break; | |
631 | ||
632 | case COMPST_COMP_WQE: | |
633 | state = complete_wqe(qp, pkt, wqe); | |
634 | break; | |
635 | ||
636 | case COMPST_UPDATE_COMP: | |
637 | if (pkt->mask & RXE_END_MASK) | |
638 | qp->comp.opcode = -1; | |
639 | else | |
640 | qp->comp.opcode = pkt->opcode; | |
641 | ||
642 | if (psn_compare(pkt->psn, qp->comp.psn) >= 0) | |
643 | qp->comp.psn = (pkt->psn + 1) & BTH_PSN_MASK; | |
644 | ||
645 | if (qp->req.wait_psn) { | |
646 | qp->req.wait_psn = 0; | |
647 | rxe_run_task(&qp->req.task, 1); | |
648 | } | |
649 | ||
650 | state = COMPST_DONE; | |
651 | break; | |
652 | ||
653 | case COMPST_DONE: | |
654 | if (pkt) { | |
655 | rxe_drop_ref(pkt->qp); | |
656 | kfree_skb(skb); | |
c5540a01 | 657 | skb = NULL; |
8700e3e7 MS |
658 | } |
659 | goto done; | |
660 | ||
661 | case COMPST_EXIT: | |
662 | if (qp->comp.timeout_retry && wqe) { | |
663 | state = COMPST_ERROR_RETRY; | |
664 | break; | |
665 | } | |
666 | ||
667 | /* re reset the timeout counter if | |
668 | * (1) QP is type RC | |
669 | * (2) the QP is alive | |
670 | * (3) there is a packet sent by the requester that | |
671 | * might be acked (we still might get spurious | |
672 | * timeouts but try to keep them as few as possible) | |
673 | * (4) the timeout parameter is set | |
674 | */ | |
675 | if ((qp_type(qp) == IB_QPT_RC) && | |
676 | (qp->req.state == QP_STATE_READY) && | |
677 | (psn_compare(qp->req.psn, qp->comp.psn) > 0) && | |
678 | qp->qp_timeout_jiffies) | |
679 | mod_timer(&qp->retrans_timer, | |
680 | jiffies + qp->qp_timeout_jiffies); | |
681 | goto exit; | |
682 | ||
683 | case COMPST_ERROR_RETRY: | |
684 | /* we come here if the retry timer fired and we did | |
685 | * not receive a response packet. try to retry the send | |
686 | * queue if that makes sense and the limits have not | |
687 | * been exceeded. remember that some timeouts are | |
688 | * spurious since we do not reset the timer but kick | |
689 | * it down the road or let it expire | |
690 | */ | |
691 | ||
692 | /* there is nothing to retry in this case */ | |
c5540a01 | 693 | if (!wqe || (wqe->state == wqe_state_posted)) { |
8700e3e7 | 694 | goto exit; |
c5540a01 | 695 | } |
8700e3e7 | 696 | |
4e4c53df VI |
697 | /* if we've started a retry, don't start another |
698 | * retry sequence, unless this is a timeout. | |
699 | */ | |
700 | if (qp->comp.started_retry && | |
701 | !qp->comp.timeout_retry) { | |
702 | if (pkt) { | |
703 | rxe_drop_ref(pkt->qp); | |
704 | kfree_skb(skb); | |
705 | skb = NULL; | |
706 | } | |
707 | ||
708 | goto done; | |
709 | } | |
710 | ||
8700e3e7 MS |
711 | if (qp->comp.retry_cnt > 0) { |
712 | if (qp->comp.retry_cnt != 7) | |
713 | qp->comp.retry_cnt--; | |
714 | ||
715 | /* no point in retrying if we have already | |
716 | * seen the last ack that the requester could | |
717 | * have caused | |
718 | */ | |
719 | if (psn_compare(qp->req.psn, | |
720 | qp->comp.psn) > 0) { | |
721 | /* tell the requester to retry the | |
0b1e5b99 | 722 | * send queue next time around |
8700e3e7 | 723 | */ |
0b1e5b99 YC |
724 | rxe_counter_inc(rxe, |
725 | RXE_CNT_COMP_RETRY); | |
8700e3e7 | 726 | qp->req.need_retry = 1; |
4e4c53df | 727 | qp->comp.started_retry = 1; |
8ac0e664 | 728 | rxe_run_task(&qp->req.task, 0); |
8700e3e7 | 729 | } |
c1cc72cb YC |
730 | |
731 | if (pkt) { | |
732 | rxe_drop_ref(pkt->qp); | |
733 | kfree_skb(skb); | |
c5540a01 | 734 | skb = NULL; |
c1cc72cb YC |
735 | } |
736 | ||
4e4c53df | 737 | goto done; |
c1cc72cb | 738 | |
8700e3e7 | 739 | } else { |
0b1e5b99 | 740 | rxe_counter_inc(rxe, RXE_CNT_RETRY_EXCEEDED); |
8700e3e7 MS |
741 | wqe->status = IB_WC_RETRY_EXC_ERR; |
742 | state = COMPST_ERROR; | |
743 | } | |
744 | break; | |
745 | ||
746 | case COMPST_RNR_RETRY: | |
747 | if (qp->comp.rnr_retry > 0) { | |
748 | if (qp->comp.rnr_retry != 7) | |
749 | qp->comp.rnr_retry--; | |
750 | ||
751 | qp->req.need_retry = 1; | |
e404f945 PP |
752 | pr_debug("qp#%d set rnr nak timer\n", |
753 | qp_num(qp)); | |
8700e3e7 MS |
754 | mod_timer(&qp->rnr_nak_timer, |
755 | jiffies + rnrnak_jiffies(aeth_syn(pkt) | |
756 | & ~AETH_TYPE_MASK)); | |
c5540a01 BVA |
757 | rxe_drop_ref(pkt->qp); |
758 | kfree_skb(skb); | |
759 | skb = NULL; | |
8700e3e7 MS |
760 | goto exit; |
761 | } else { | |
0b1e5b99 YC |
762 | rxe_counter_inc(rxe, |
763 | RXE_CNT_RNR_RETRY_EXCEEDED); | |
8700e3e7 MS |
764 | wqe->status = IB_WC_RNR_RETRY_EXC_ERR; |
765 | state = COMPST_ERROR; | |
766 | } | |
767 | break; | |
768 | ||
769 | case COMPST_ERROR: | |
18d3451c | 770 | WARN_ON_ONCE(wqe->status == IB_WC_SUCCESS); |
8700e3e7 MS |
771 | do_complete(qp, wqe); |
772 | rxe_qp_error(qp); | |
c1cc72cb YC |
773 | |
774 | if (pkt) { | |
775 | rxe_drop_ref(pkt->qp); | |
776 | kfree_skb(skb); | |
c5540a01 | 777 | skb = NULL; |
c1cc72cb YC |
778 | } |
779 | ||
8700e3e7 MS |
780 | goto exit; |
781 | } | |
782 | } | |
783 | ||
784 | exit: | |
785 | /* we come here if we are done with processing and want the task to | |
786 | * exit from the loop calling us | |
787 | */ | |
c5540a01 | 788 | WARN_ON_ONCE(skb); |
37f69f43 | 789 | rxe_drop_ref(qp); |
8700e3e7 MS |
790 | return -EAGAIN; |
791 | ||
792 | done: | |
793 | /* we come here if we have processed a packet we want the task to call | |
794 | * us again to see if there is anything else to do | |
795 | */ | |
c5540a01 | 796 | WARN_ON_ONCE(skb); |
37f69f43 | 797 | rxe_drop_ref(qp); |
8700e3e7 MS |
798 | return 0; |
799 | } |