]>
Commit | Line | Data |
---|---|---|
74ed6b5e | 1 | /* |
e509be89 | 2 | * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved. |
74ed6b5e BS |
3 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | */ | |
33 | ||
34 | #include "ipath_verbs.h" | |
34b2aafe | 35 | #include "ipath_kernel.h" |
74ed6b5e BS |
36 | |
37 | /* cut down ridiculously long IB macro names */ | |
38 | #define OP(x) IB_OPCODE_UC_##x | |
39 | ||
74ed6b5e | 40 | /** |
ddd4bb22 BS |
41 | * ipath_make_uc_req - construct a request packet (SEND, RDMA write) |
42 | * @qp: a pointer to the QP | |
74ed6b5e | 43 | * |
ddd4bb22 | 44 | * Return 1 if constructed; otherwise, return 0. |
74ed6b5e | 45 | */ |
4ee97180 | 46 | int ipath_make_uc_req(struct ipath_qp *qp) |
74ed6b5e | 47 | { |
4ee97180 | 48 | struct ipath_other_headers *ohdr; |
74ed6b5e | 49 | struct ipath_swqe *wqe; |
e509be89 | 50 | unsigned long flags; |
74ed6b5e | 51 | u32 hwords; |
74ed6b5e | 52 | u32 bth0; |
74ed6b5e | 53 | u32 len; |
4ee97180 RC |
54 | u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); |
55 | int ret = 0; | |
74ed6b5e | 56 | |
e509be89 RC |
57 | spin_lock_irqsave(&qp->s_lock, flags); |
58 | ||
59 | if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) { | |
60 | if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND)) | |
61 | goto bail; | |
62 | /* We are in the error state, flush the work request. */ | |
63 | if (qp->s_last == qp->s_head) | |
64 | goto bail; | |
65 | /* If DMAs are in progress, we can't flush immediately. */ | |
66 | if (atomic_read(&qp->s_dma_busy)) { | |
67 | qp->s_flags |= IPATH_S_WAIT_DMA; | |
68 | goto bail; | |
69 | } | |
70 | wqe = get_swqe_ptr(qp, qp->s_last); | |
71 | ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); | |
74ed6b5e | 72 | goto done; |
e509be89 | 73 | } |
74ed6b5e | 74 | |
4ee97180 RC |
75 | ohdr = &qp->s_hdr.u.oth; |
76 | if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) | |
77 | ohdr = &qp->s_hdr.u.l.oth; | |
78 | ||
ddd4bb22 BS |
79 | /* header size in 32-bit words LRH+BTH = (8+12)/4. */ |
80 | hwords = 5; | |
1dd6a1be | 81 | bth0 = 1 << 22; /* Set M bit */ |
74ed6b5e | 82 | |
ddd4bb22 | 83 | /* Get the next send request. */ |
4ee97180 RC |
84 | wqe = get_swqe_ptr(qp, qp->s_cur); |
85 | qp->s_wqe = NULL; | |
74ed6b5e BS |
86 | switch (qp->s_state) { |
87 | default: | |
e509be89 RC |
88 | if (!(ib_ipath_state_ops[qp->state] & |
89 | IPATH_PROCESS_NEXT_SEND_OK)) | |
90 | goto bail; | |
74ed6b5e | 91 | /* Check if send work queue is empty. */ |
4ee97180 | 92 | if (qp->s_cur == qp->s_head) |
e509be89 | 93 | goto bail; |
74ed6b5e BS |
94 | /* |
95 | * Start a new request. | |
96 | */ | |
97 | qp->s_psn = wqe->psn = qp->s_next_psn; | |
98 | qp->s_sge.sge = wqe->sg_list[0]; | |
99 | qp->s_sge.sg_list = wqe->sg_list + 1; | |
100 | qp->s_sge.num_sge = wqe->wr.num_sge; | |
101 | qp->s_len = len = wqe->length; | |
102 | switch (wqe->wr.opcode) { | |
103 | case IB_WR_SEND: | |
104 | case IB_WR_SEND_WITH_IMM: | |
105 | if (len > pmtu) { | |
106 | qp->s_state = OP(SEND_FIRST); | |
107 | len = pmtu; | |
108 | break; | |
109 | } | |
110 | if (wqe->wr.opcode == IB_WR_SEND) | |
111 | qp->s_state = OP(SEND_ONLY); | |
112 | else { | |
113 | qp->s_state = | |
114 | OP(SEND_ONLY_WITH_IMMEDIATE); | |
115 | /* Immediate data comes after the BTH */ | |
0f39cf3d | 116 | ohdr->u.imm_data = wqe->wr.ex.imm_data; |
74ed6b5e BS |
117 | hwords += 1; |
118 | } | |
119 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
120 | bth0 |= 1 << 23; | |
4ee97180 RC |
121 | qp->s_wqe = wqe; |
122 | if (++qp->s_cur >= qp->s_size) | |
123 | qp->s_cur = 0; | |
74ed6b5e BS |
124 | break; |
125 | ||
126 | case IB_WR_RDMA_WRITE: | |
127 | case IB_WR_RDMA_WRITE_WITH_IMM: | |
128 | ohdr->u.rc.reth.vaddr = | |
e622f2f4 | 129 | cpu_to_be64(wqe->rdma_wr.remote_addr); |
74ed6b5e | 130 | ohdr->u.rc.reth.rkey = |
e622f2f4 | 131 | cpu_to_be32(wqe->rdma_wr.rkey); |
74ed6b5e BS |
132 | ohdr->u.rc.reth.length = cpu_to_be32(len); |
133 | hwords += sizeof(struct ib_reth) / 4; | |
134 | if (len > pmtu) { | |
135 | qp->s_state = OP(RDMA_WRITE_FIRST); | |
136 | len = pmtu; | |
137 | break; | |
138 | } | |
139 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE) | |
140 | qp->s_state = OP(RDMA_WRITE_ONLY); | |
141 | else { | |
142 | qp->s_state = | |
143 | OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); | |
144 | /* Immediate data comes after the RETH */ | |
0f39cf3d | 145 | ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; |
74ed6b5e BS |
146 | hwords += 1; |
147 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
148 | bth0 |= 1 << 23; | |
149 | } | |
4ee97180 RC |
150 | qp->s_wqe = wqe; |
151 | if (++qp->s_cur >= qp->s_size) | |
152 | qp->s_cur = 0; | |
74ed6b5e BS |
153 | break; |
154 | ||
155 | default: | |
e509be89 | 156 | goto bail; |
74ed6b5e | 157 | } |
74ed6b5e BS |
158 | break; |
159 | ||
160 | case OP(SEND_FIRST): | |
161 | qp->s_state = OP(SEND_MIDDLE); | |
162 | /* FALLTHROUGH */ | |
163 | case OP(SEND_MIDDLE): | |
164 | len = qp->s_len; | |
165 | if (len > pmtu) { | |
166 | len = pmtu; | |
167 | break; | |
168 | } | |
169 | if (wqe->wr.opcode == IB_WR_SEND) | |
170 | qp->s_state = OP(SEND_LAST); | |
171 | else { | |
172 | qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); | |
173 | /* Immediate data comes after the BTH */ | |
0f39cf3d | 174 | ohdr->u.imm_data = wqe->wr.ex.imm_data; |
74ed6b5e BS |
175 | hwords += 1; |
176 | } | |
177 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
178 | bth0 |= 1 << 23; | |
4ee97180 RC |
179 | qp->s_wqe = wqe; |
180 | if (++qp->s_cur >= qp->s_size) | |
181 | qp->s_cur = 0; | |
74ed6b5e BS |
182 | break; |
183 | ||
184 | case OP(RDMA_WRITE_FIRST): | |
185 | qp->s_state = OP(RDMA_WRITE_MIDDLE); | |
186 | /* FALLTHROUGH */ | |
187 | case OP(RDMA_WRITE_MIDDLE): | |
188 | len = qp->s_len; | |
189 | if (len > pmtu) { | |
190 | len = pmtu; | |
191 | break; | |
192 | } | |
193 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE) | |
194 | qp->s_state = OP(RDMA_WRITE_LAST); | |
195 | else { | |
196 | qp->s_state = | |
197 | OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); | |
198 | /* Immediate data comes after the BTH */ | |
0f39cf3d | 199 | ohdr->u.imm_data = wqe->wr.ex.imm_data; |
74ed6b5e BS |
200 | hwords += 1; |
201 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
202 | bth0 |= 1 << 23; | |
203 | } | |
4ee97180 RC |
204 | qp->s_wqe = wqe; |
205 | if (++qp->s_cur >= qp->s_size) | |
206 | qp->s_cur = 0; | |
74ed6b5e BS |
207 | break; |
208 | } | |
74ed6b5e | 209 | qp->s_len -= len; |
74ed6b5e BS |
210 | qp->s_hdrwords = hwords; |
211 | qp->s_cur_sge = &qp->s_sge; | |
212 | qp->s_cur_size = len; | |
4ee97180 RC |
213 | ipath_make_ruc_header(to_idev(qp->ibqp.device), |
214 | qp, ohdr, bth0 | (qp->s_state << 24), | |
215 | qp->s_next_psn++ & IPATH_PSN_MASK); | |
e509be89 | 216 | done: |
4ee97180 | 217 | ret = 1; |
e509be89 | 218 | goto unlock; |
74ed6b5e | 219 | |
e509be89 RC |
220 | bail: |
221 | qp->s_flags &= ~IPATH_S_BUSY; | |
222 | unlock: | |
223 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
4ee97180 | 224 | return ret; |
74ed6b5e BS |
225 | } |
226 | ||
227 | /** | |
228 | * ipath_uc_rcv - handle an incoming UC packet | |
229 | * @dev: the device the packet came in on | |
230 | * @hdr: the header of the packet | |
231 | * @has_grh: true if the packet has a GRH | |
232 | * @data: the packet data | |
233 | * @tlen: the length of the packet | |
234 | * @qp: the QP for this packet. | |
235 | * | |
236 | * This is called from ipath_qp_rcv() to process an incoming UC packet | |
237 | * for the given QP. | |
238 | * Called at interrupt level. | |
239 | */ | |
240 | void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, | |
241 | int has_grh, void *data, u32 tlen, struct ipath_qp *qp) | |
242 | { | |
243 | struct ipath_other_headers *ohdr; | |
244 | int opcode; | |
245 | u32 hdrsize; | |
246 | u32 psn; | |
247 | u32 pad; | |
74ed6b5e BS |
248 | struct ib_wc wc; |
249 | u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); | |
250 | struct ib_reth *reth; | |
251 | int header_in_data; | |
252 | ||
10aeb0e6 BS |
253 | /* Validate the SLID. See Ch. 9.6.1.5 */ |
254 | if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid)) | |
255 | goto done; | |
256 | ||
74ed6b5e BS |
257 | /* Check for GRH */ |
258 | if (!has_grh) { | |
259 | ohdr = &hdr->u.oth; | |
260 | hdrsize = 8 + 12; /* LRH + BTH */ | |
261 | psn = be32_to_cpu(ohdr->bth[2]); | |
262 | header_in_data = 0; | |
263 | } else { | |
264 | ohdr = &hdr->u.l.oth; | |
265 | hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ | |
266 | /* | |
267 | * The header with GRH is 60 bytes and the | |
268 | * core driver sets the eager header buffer | |
269 | * size to 56 bytes so the last 4 bytes of | |
270 | * the BTH header (PSN) is in the data buffer. | |
271 | */ | |
34b2aafe | 272 | header_in_data = dev->dd->ipath_rcvhdrentsize == 16; |
74ed6b5e BS |
273 | if (header_in_data) { |
274 | psn = be32_to_cpu(((__be32 *) data)[0]); | |
275 | data += sizeof(__be32); | |
276 | } else | |
277 | psn = be32_to_cpu(ohdr->bth[2]); | |
278 | } | |
279 | /* | |
280 | * The opcode is in the low byte when its in network order | |
281 | * (top byte when in host order). | |
282 | */ | |
283 | opcode = be32_to_cpu(ohdr->bth[0]) >> 24; | |
284 | ||
e509be89 | 285 | memset(&wc, 0, sizeof wc); |
74ed6b5e | 286 | |
74ed6b5e BS |
287 | /* Compare the PSN verses the expected PSN. */ |
288 | if (unlikely(ipath_cmp24(psn, qp->r_psn) != 0)) { | |
289 | /* | |
290 | * Handle a sequence error. | |
291 | * Silently drop any current message. | |
292 | */ | |
293 | qp->r_psn = psn; | |
294 | inv: | |
295 | qp->r_state = OP(SEND_LAST); | |
296 | switch (opcode) { | |
297 | case OP(SEND_FIRST): | |
298 | case OP(SEND_ONLY): | |
299 | case OP(SEND_ONLY_WITH_IMMEDIATE): | |
300 | goto send_first; | |
301 | ||
302 | case OP(RDMA_WRITE_FIRST): | |
303 | case OP(RDMA_WRITE_ONLY): | |
304 | case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): | |
305 | goto rdma_first; | |
306 | ||
307 | default: | |
308 | dev->n_pkt_drops++; | |
309 | goto done; | |
310 | } | |
311 | } | |
312 | ||
313 | /* Check for opcode sequence errors. */ | |
314 | switch (qp->r_state) { | |
315 | case OP(SEND_FIRST): | |
316 | case OP(SEND_MIDDLE): | |
317 | if (opcode == OP(SEND_MIDDLE) || | |
318 | opcode == OP(SEND_LAST) || | |
319 | opcode == OP(SEND_LAST_WITH_IMMEDIATE)) | |
320 | break; | |
321 | goto inv; | |
322 | ||
323 | case OP(RDMA_WRITE_FIRST): | |
324 | case OP(RDMA_WRITE_MIDDLE): | |
325 | if (opcode == OP(RDMA_WRITE_MIDDLE) || | |
326 | opcode == OP(RDMA_WRITE_LAST) || | |
327 | opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) | |
328 | break; | |
329 | goto inv; | |
330 | ||
331 | default: | |
332 | if (opcode == OP(SEND_FIRST) || | |
333 | opcode == OP(SEND_ONLY) || | |
334 | opcode == OP(SEND_ONLY_WITH_IMMEDIATE) || | |
335 | opcode == OP(RDMA_WRITE_FIRST) || | |
336 | opcode == OP(RDMA_WRITE_ONLY) || | |
337 | opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) | |
338 | break; | |
339 | goto inv; | |
340 | } | |
341 | ||
342 | /* OK, process the packet. */ | |
343 | switch (opcode) { | |
344 | case OP(SEND_FIRST): | |
345 | case OP(SEND_ONLY): | |
346 | case OP(SEND_ONLY_WITH_IMMEDIATE): | |
347 | send_first: | |
e509be89 RC |
348 | if (qp->r_flags & IPATH_R_REUSE_SGE) { |
349 | qp->r_flags &= ~IPATH_R_REUSE_SGE; | |
3859e39d | 350 | qp->r_sge = qp->s_rdma_read_sge; |
74ed6b5e BS |
351 | } else if (!ipath_get_rwqe(qp, 0)) { |
352 | dev->n_pkt_drops++; | |
353 | goto done; | |
354 | } | |
355 | /* Save the WQE so we can reuse it in case of an error. */ | |
3859e39d | 356 | qp->s_rdma_read_sge = qp->r_sge; |
74ed6b5e BS |
357 | qp->r_rcv_len = 0; |
358 | if (opcode == OP(SEND_ONLY)) | |
359 | goto send_last; | |
360 | else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE)) | |
361 | goto send_last_imm; | |
362 | /* FALLTHROUGH */ | |
363 | case OP(SEND_MIDDLE): | |
364 | /* Check for invalid length PMTU or posted rwqe len. */ | |
365 | if (unlikely(tlen != (hdrsize + pmtu + 4))) { | |
e509be89 | 366 | qp->r_flags |= IPATH_R_REUSE_SGE; |
74ed6b5e BS |
367 | dev->n_pkt_drops++; |
368 | goto done; | |
369 | } | |
370 | qp->r_rcv_len += pmtu; | |
371 | if (unlikely(qp->r_rcv_len > qp->r_len)) { | |
e509be89 | 372 | qp->r_flags |= IPATH_R_REUSE_SGE; |
74ed6b5e BS |
373 | dev->n_pkt_drops++; |
374 | goto done; | |
375 | } | |
376 | ipath_copy_sge(&qp->r_sge, data, pmtu); | |
377 | break; | |
378 | ||
379 | case OP(SEND_LAST_WITH_IMMEDIATE): | |
380 | send_last_imm: | |
381 | if (header_in_data) { | |
00f7ec36 | 382 | wc.ex.imm_data = *(__be32 *) data; |
74ed6b5e BS |
383 | data += sizeof(__be32); |
384 | } else { | |
385 | /* Immediate data comes after BTH */ | |
00f7ec36 | 386 | wc.ex.imm_data = ohdr->u.imm_data; |
74ed6b5e BS |
387 | } |
388 | hdrsize += 4; | |
389 | wc.wc_flags = IB_WC_WITH_IMM; | |
390 | /* FALLTHROUGH */ | |
391 | case OP(SEND_LAST): | |
392 | send_last: | |
393 | /* Get the number of bytes the message was padded by. */ | |
394 | pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; | |
395 | /* Check for invalid length. */ | |
396 | /* XXX LAST len should be >= 1 */ | |
397 | if (unlikely(tlen < (hdrsize + pad + 4))) { | |
e509be89 | 398 | qp->r_flags |= IPATH_R_REUSE_SGE; |
74ed6b5e BS |
399 | dev->n_pkt_drops++; |
400 | goto done; | |
401 | } | |
402 | /* Don't count the CRC. */ | |
403 | tlen -= (hdrsize + pad + 4); | |
404 | wc.byte_len = tlen + qp->r_rcv_len; | |
405 | if (unlikely(wc.byte_len > qp->r_len)) { | |
e509be89 | 406 | qp->r_flags |= IPATH_R_REUSE_SGE; |
74ed6b5e BS |
407 | dev->n_pkt_drops++; |
408 | goto done; | |
409 | } | |
df3f0da8 | 410 | wc.opcode = IB_WC_RECV; |
74ed6b5e BS |
411 | last_imm: |
412 | ipath_copy_sge(&qp->r_sge, data, tlen); | |
413 | wc.wr_id = qp->r_wr_id; | |
414 | wc.status = IB_WC_SUCCESS; | |
062dbb69 | 415 | wc.qp = &qp->ibqp; |
74ed6b5e | 416 | wc.src_qp = qp->remote_qpn; |
74ed6b5e BS |
417 | wc.slid = qp->remote_ah_attr.dlid; |
418 | wc.sl = qp->remote_ah_attr.sl; | |
74ed6b5e BS |
419 | /* Signal completion event if the solicited bit is set. */ |
420 | ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, | |
421 | (ohdr->bth[0] & | |
9c3da099 | 422 | cpu_to_be32(1 << 23)) != 0); |
74ed6b5e BS |
423 | break; |
424 | ||
425 | case OP(RDMA_WRITE_FIRST): | |
426 | case OP(RDMA_WRITE_ONLY): | |
427 | case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): /* consume RWQE */ | |
428 | rdma_first: | |
429 | /* RETH comes after BTH */ | |
430 | if (!header_in_data) | |
431 | reth = &ohdr->u.rc.reth; | |
432 | else { | |
433 | reth = (struct ib_reth *)data; | |
434 | data += sizeof(*reth); | |
435 | } | |
436 | hdrsize += sizeof(*reth); | |
437 | qp->r_len = be32_to_cpu(reth->length); | |
438 | qp->r_rcv_len = 0; | |
439 | if (qp->r_len != 0) { | |
440 | u32 rkey = be32_to_cpu(reth->rkey); | |
441 | u64 vaddr = be64_to_cpu(reth->vaddr); | |
ddd4bb22 | 442 | int ok; |
74ed6b5e BS |
443 | |
444 | /* Check rkey */ | |
6a553af2 | 445 | ok = ipath_rkey_ok(qp, &qp->r_sge, qp->r_len, |
ddd4bb22 BS |
446 | vaddr, rkey, |
447 | IB_ACCESS_REMOTE_WRITE); | |
448 | if (unlikely(!ok)) { | |
74ed6b5e BS |
449 | dev->n_pkt_drops++; |
450 | goto done; | |
451 | } | |
452 | } else { | |
453 | qp->r_sge.sg_list = NULL; | |
454 | qp->r_sge.sge.mr = NULL; | |
455 | qp->r_sge.sge.vaddr = NULL; | |
456 | qp->r_sge.sge.length = 0; | |
457 | qp->r_sge.sge.sge_length = 0; | |
458 | } | |
459 | if (unlikely(!(qp->qp_access_flags & | |
460 | IB_ACCESS_REMOTE_WRITE))) { | |
461 | dev->n_pkt_drops++; | |
462 | goto done; | |
463 | } | |
464 | if (opcode == OP(RDMA_WRITE_ONLY)) | |
465 | goto rdma_last; | |
ddd4bb22 | 466 | else if (opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) |
74ed6b5e BS |
467 | goto rdma_last_imm; |
468 | /* FALLTHROUGH */ | |
469 | case OP(RDMA_WRITE_MIDDLE): | |
470 | /* Check for invalid length PMTU or posted rwqe len. */ | |
471 | if (unlikely(tlen != (hdrsize + pmtu + 4))) { | |
472 | dev->n_pkt_drops++; | |
473 | goto done; | |
474 | } | |
475 | qp->r_rcv_len += pmtu; | |
476 | if (unlikely(qp->r_rcv_len > qp->r_len)) { | |
477 | dev->n_pkt_drops++; | |
478 | goto done; | |
479 | } | |
480 | ipath_copy_sge(&qp->r_sge, data, pmtu); | |
481 | break; | |
482 | ||
483 | case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): | |
484 | rdma_last_imm: | |
55046698 | 485 | if (header_in_data) { |
00f7ec36 | 486 | wc.ex.imm_data = *(__be32 *) data; |
55046698 RC |
487 | data += sizeof(__be32); |
488 | } else { | |
489 | /* Immediate data comes after BTH */ | |
00f7ec36 | 490 | wc.ex.imm_data = ohdr->u.imm_data; |
55046698 RC |
491 | } |
492 | hdrsize += 4; | |
493 | wc.wc_flags = IB_WC_WITH_IMM; | |
494 | ||
74ed6b5e BS |
495 | /* Get the number of bytes the message was padded by. */ |
496 | pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; | |
497 | /* Check for invalid length. */ | |
498 | /* XXX LAST len should be >= 1 */ | |
499 | if (unlikely(tlen < (hdrsize + pad + 4))) { | |
500 | dev->n_pkt_drops++; | |
501 | goto done; | |
502 | } | |
503 | /* Don't count the CRC. */ | |
504 | tlen -= (hdrsize + pad + 4); | |
505 | if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) { | |
506 | dev->n_pkt_drops++; | |
507 | goto done; | |
508 | } | |
e509be89 RC |
509 | if (qp->r_flags & IPATH_R_REUSE_SGE) |
510 | qp->r_flags &= ~IPATH_R_REUSE_SGE; | |
ddd4bb22 | 511 | else if (!ipath_get_rwqe(qp, 1)) { |
74ed6b5e BS |
512 | dev->n_pkt_drops++; |
513 | goto done; | |
514 | } | |
55046698 | 515 | wc.byte_len = qp->r_len; |
df3f0da8 | 516 | wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; |
74ed6b5e BS |
517 | goto last_imm; |
518 | ||
519 | case OP(RDMA_WRITE_LAST): | |
520 | rdma_last: | |
521 | /* Get the number of bytes the message was padded by. */ | |
522 | pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; | |
523 | /* Check for invalid length. */ | |
524 | /* XXX LAST len should be >= 1 */ | |
525 | if (unlikely(tlen < (hdrsize + pad + 4))) { | |
526 | dev->n_pkt_drops++; | |
527 | goto done; | |
528 | } | |
529 | /* Don't count the CRC. */ | |
530 | tlen -= (hdrsize + pad + 4); | |
531 | if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) { | |
532 | dev->n_pkt_drops++; | |
533 | goto done; | |
534 | } | |
535 | ipath_copy_sge(&qp->r_sge, data, tlen); | |
536 | break; | |
537 | ||
538 | default: | |
539 | /* Drop packet for unknown opcodes. */ | |
74ed6b5e | 540 | dev->n_pkt_drops++; |
12eef41f | 541 | goto done; |
74ed6b5e BS |
542 | } |
543 | qp->r_psn++; | |
544 | qp->r_state = opcode; | |
545 | done: | |
74ed6b5e BS |
546 | return; |
547 | } |