]>
Commit | Line | Data |
---|---|---|
b038ced7 SW |
1 | /* |
2 | * Copyright (c) 2006 Chelsio, Inc. All rights reserved. | |
b038ced7 SW |
3 | * |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | #include "iwch_provider.h" | |
33 | #include "iwch.h" | |
34 | #include "iwch_cm.h" | |
35 | #include "cxio_hal.h" | |
36 | ||
37 | #define NO_SUPPORT -1 | |
38 | ||
2b540355 AB |
39 | static int iwch_build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr, |
40 | u8 * flit_cnt) | |
b038ced7 SW |
41 | { |
42 | int i; | |
43 | u32 plen; | |
44 | ||
45 | switch (wr->opcode) { | |
46 | case IB_WR_SEND: | |
47 | case IB_WR_SEND_WITH_IMM: | |
48 | if (wr->send_flags & IB_SEND_SOLICITED) | |
49 | wqe->send.rdmaop = T3_SEND_WITH_SE; | |
50 | else | |
51 | wqe->send.rdmaop = T3_SEND; | |
52 | wqe->send.rem_stag = 0; | |
53 | break; | |
54 | #if 0 /* Not currently supported */ | |
55 | case TYPE_SEND_INVALIDATE: | |
56 | case TYPE_SEND_INVALIDATE_IMMEDIATE: | |
57 | wqe->send.rdmaop = T3_SEND_WITH_INV; | |
58 | wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); | |
59 | break; | |
60 | case TYPE_SEND_SE_INVALIDATE: | |
61 | wqe->send.rdmaop = T3_SEND_WITH_SE_INV; | |
62 | wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); | |
63 | break; | |
64 | #endif | |
65 | default: | |
66 | break; | |
67 | } | |
68 | if (wr->num_sge > T3_MAX_SGE) | |
69 | return -EINVAL; | |
70 | wqe->send.reserved[0] = 0; | |
71 | wqe->send.reserved[1] = 0; | |
72 | wqe->send.reserved[2] = 0; | |
73 | if (wr->opcode == IB_WR_SEND_WITH_IMM) { | |
74 | plen = 4; | |
0f39cf3d | 75 | wqe->send.sgl[0].stag = wr->ex.imm_data; |
b038ced7 SW |
76 | wqe->send.sgl[0].len = __constant_cpu_to_be32(0); |
77 | wqe->send.num_sgle = __constant_cpu_to_be32(0); | |
78 | *flit_cnt = 5; | |
79 | } else { | |
80 | plen = 0; | |
81 | for (i = 0; i < wr->num_sge; i++) { | |
82 | if ((plen + wr->sg_list[i].length) < plen) { | |
83 | return -EMSGSIZE; | |
84 | } | |
85 | plen += wr->sg_list[i].length; | |
86 | wqe->send.sgl[i].stag = | |
87 | cpu_to_be32(wr->sg_list[i].lkey); | |
88 | wqe->send.sgl[i].len = | |
89 | cpu_to_be32(wr->sg_list[i].length); | |
90 | wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); | |
91 | } | |
92 | wqe->send.num_sgle = cpu_to_be32(wr->num_sge); | |
93 | *flit_cnt = 4 + ((wr->num_sge) << 1); | |
94 | } | |
95 | wqe->send.plen = cpu_to_be32(plen); | |
96 | return 0; | |
97 | } | |
98 | ||
2b540355 AB |
99 | static int iwch_build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr, |
100 | u8 *flit_cnt) | |
b038ced7 SW |
101 | { |
102 | int i; | |
103 | u32 plen; | |
104 | if (wr->num_sge > T3_MAX_SGE) | |
105 | return -EINVAL; | |
106 | wqe->write.rdmaop = T3_RDMA_WRITE; | |
107 | wqe->write.reserved[0] = 0; | |
108 | wqe->write.reserved[1] = 0; | |
109 | wqe->write.reserved[2] = 0; | |
110 | wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); | |
111 | wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); | |
112 | ||
113 | if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { | |
114 | plen = 4; | |
0f39cf3d | 115 | wqe->write.sgl[0].stag = wr->ex.imm_data; |
b038ced7 SW |
116 | wqe->write.sgl[0].len = __constant_cpu_to_be32(0); |
117 | wqe->write.num_sgle = __constant_cpu_to_be32(0); | |
118 | *flit_cnt = 6; | |
119 | } else { | |
120 | plen = 0; | |
121 | for (i = 0; i < wr->num_sge; i++) { | |
122 | if ((plen + wr->sg_list[i].length) < plen) { | |
123 | return -EMSGSIZE; | |
124 | } | |
125 | plen += wr->sg_list[i].length; | |
126 | wqe->write.sgl[i].stag = | |
127 | cpu_to_be32(wr->sg_list[i].lkey); | |
128 | wqe->write.sgl[i].len = | |
129 | cpu_to_be32(wr->sg_list[i].length); | |
130 | wqe->write.sgl[i].to = | |
131 | cpu_to_be64(wr->sg_list[i].addr); | |
132 | } | |
133 | wqe->write.num_sgle = cpu_to_be32(wr->num_sge); | |
134 | *flit_cnt = 5 + ((wr->num_sge) << 1); | |
135 | } | |
136 | wqe->write.plen = cpu_to_be32(plen); | |
137 | return 0; | |
138 | } | |
139 | ||
2b540355 AB |
140 | static int iwch_build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr, |
141 | u8 *flit_cnt) | |
b038ced7 SW |
142 | { |
143 | if (wr->num_sge > 1) | |
144 | return -EINVAL; | |
145 | wqe->read.rdmaop = T3_READ_REQ; | |
146 | wqe->read.reserved[0] = 0; | |
147 | wqe->read.reserved[1] = 0; | |
148 | wqe->read.reserved[2] = 0; | |
149 | wqe->read.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); | |
150 | wqe->read.rem_to = cpu_to_be64(wr->wr.rdma.remote_addr); | |
151 | wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey); | |
152 | wqe->read.local_len = cpu_to_be32(wr->sg_list[0].length); | |
153 | wqe->read.local_to = cpu_to_be64(wr->sg_list[0].addr); | |
154 | *flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; | |
155 | return 0; | |
156 | } | |
157 | ||
158 | /* | |
159 | * TBD: this is going to be moved to firmware. Missing pdid/qpid check for now. | |
160 | */ | |
2b540355 AB |
161 | static int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list, |
162 | u32 num_sgle, u32 * pbl_addr, u8 * page_size) | |
b038ced7 SW |
163 | { |
164 | int i; | |
165 | struct iwch_mr *mhp; | |
166 | u32 offset; | |
167 | for (i = 0; i < num_sgle; i++) { | |
168 | ||
169 | mhp = get_mhp(rhp, (sg_list[i].lkey) >> 8); | |
170 | if (!mhp) { | |
33718363 | 171 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
172 | return -EIO; |
173 | } | |
174 | if (!mhp->attr.state) { | |
33718363 | 175 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
176 | return -EIO; |
177 | } | |
178 | if (mhp->attr.zbva) { | |
33718363 | 179 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
180 | return -EIO; |
181 | } | |
182 | ||
183 | if (sg_list[i].addr < mhp->attr.va_fbo) { | |
33718363 | 184 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
185 | return -EINVAL; |
186 | } | |
187 | if (sg_list[i].addr + ((u64) sg_list[i].length) < | |
188 | sg_list[i].addr) { | |
33718363 | 189 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
190 | return -EINVAL; |
191 | } | |
192 | if (sg_list[i].addr + ((u64) sg_list[i].length) > | |
193 | mhp->attr.va_fbo + ((u64) mhp->attr.len)) { | |
33718363 | 194 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
195 | return -EINVAL; |
196 | } | |
197 | offset = sg_list[i].addr - mhp->attr.va_fbo; | |
198 | offset += ((u32) mhp->attr.va_fbo) % | |
199 | (1UL << (12 + mhp->attr.page_size)); | |
200 | pbl_addr[i] = ((mhp->attr.pbl_addr - | |
201 | rhp->rdev.rnic_info.pbl_base) >> 3) + | |
202 | (offset >> (12 + mhp->attr.page_size)); | |
203 | page_size[i] = mhp->attr.page_size; | |
204 | } | |
205 | return 0; | |
206 | } | |
207 | ||
2b540355 AB |
208 | static int iwch_build_rdma_recv(struct iwch_dev *rhp, union t3_wr *wqe, |
209 | struct ib_recv_wr *wr) | |
b038ced7 | 210 | { |
457fe7b8 | 211 | int i; |
b038ced7 SW |
212 | if (wr->num_sge > T3_MAX_SGE) |
213 | return -EINVAL; | |
b038ced7 SW |
214 | wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); |
215 | for (i = 0; i < wr->num_sge; i++) { | |
216 | wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); | |
217 | wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); | |
457fe7b8 | 218 | wqe->recv.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); |
b038ced7 SW |
219 | } |
220 | for (; i < T3_MAX_SGE; i++) { | |
221 | wqe->recv.sgl[i].stag = 0; | |
222 | wqe->recv.sgl[i].len = 0; | |
223 | wqe->recv.sgl[i].to = 0; | |
b038ced7 SW |
224 | } |
225 | return 0; | |
226 | } | |
227 | ||
228 | int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, | |
229 | struct ib_send_wr **bad_wr) | |
230 | { | |
231 | int err = 0; | |
232 | u8 t3_wr_flit_cnt; | |
233 | enum t3_wr_opcode t3_wr_opcode = 0; | |
234 | enum t3_wr_flags t3_wr_flags; | |
235 | struct iwch_qp *qhp; | |
236 | u32 idx; | |
237 | union t3_wr *wqe; | |
238 | u32 num_wrs; | |
239 | unsigned long flag; | |
240 | struct t3_swsq *sqp; | |
241 | ||
242 | qhp = to_iwch_qp(ibqp); | |
243 | spin_lock_irqsave(&qhp->lock, flag); | |
244 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
245 | spin_unlock_irqrestore(&qhp->lock, flag); | |
246 | return -EINVAL; | |
247 | } | |
248 | num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, | |
249 | qhp->wq.sq_size_log2); | |
250 | if (num_wrs <= 0) { | |
251 | spin_unlock_irqrestore(&qhp->lock, flag); | |
252 | return -ENOMEM; | |
253 | } | |
254 | while (wr) { | |
255 | if (num_wrs == 0) { | |
256 | err = -ENOMEM; | |
257 | *bad_wr = wr; | |
258 | break; | |
259 | } | |
260 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
261 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
262 | t3_wr_flags = 0; | |
263 | if (wr->send_flags & IB_SEND_SOLICITED) | |
264 | t3_wr_flags |= T3_SOLICITED_EVENT_FLAG; | |
265 | if (wr->send_flags & IB_SEND_FENCE) | |
266 | t3_wr_flags |= T3_READ_FENCE_FLAG; | |
267 | if (wr->send_flags & IB_SEND_SIGNALED) | |
268 | t3_wr_flags |= T3_COMPLETION_FLAG; | |
269 | sqp = qhp->wq.sq + | |
270 | Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); | |
271 | switch (wr->opcode) { | |
272 | case IB_WR_SEND: | |
273 | case IB_WR_SEND_WITH_IMM: | |
274 | t3_wr_opcode = T3_WR_SEND; | |
275 | err = iwch_build_rdma_send(wqe, wr, &t3_wr_flit_cnt); | |
276 | break; | |
277 | case IB_WR_RDMA_WRITE: | |
278 | case IB_WR_RDMA_WRITE_WITH_IMM: | |
279 | t3_wr_opcode = T3_WR_WRITE; | |
280 | err = iwch_build_rdma_write(wqe, wr, &t3_wr_flit_cnt); | |
281 | break; | |
282 | case IB_WR_RDMA_READ: | |
283 | t3_wr_opcode = T3_WR_READ; | |
284 | t3_wr_flags = 0; /* T3 reads are always signaled */ | |
285 | err = iwch_build_rdma_read(wqe, wr, &t3_wr_flit_cnt); | |
286 | if (err) | |
287 | break; | |
288 | sqp->read_len = wqe->read.local_len; | |
289 | if (!qhp->wq.oldest_read) | |
290 | qhp->wq.oldest_read = sqp; | |
291 | break; | |
292 | default: | |
33718363 | 293 | PDBG("%s post of type=%d TBD!\n", __func__, |
b038ced7 SW |
294 | wr->opcode); |
295 | err = -EINVAL; | |
296 | } | |
297 | if (err) { | |
298 | *bad_wr = wr; | |
299 | break; | |
300 | } | |
301 | wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; | |
302 | sqp->wr_id = wr->wr_id; | |
303 | sqp->opcode = wr2opcode(t3_wr_opcode); | |
304 | sqp->sq_wptr = qhp->wq.sq_wptr; | |
305 | sqp->complete = 0; | |
306 | sqp->signaled = (wr->send_flags & IB_SEND_SIGNALED); | |
307 | ||
308 | build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, | |
309 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), | |
310 | 0, t3_wr_flit_cnt); | |
311 | PDBG("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n", | |
33718363 | 312 | __func__, (unsigned long long) wr->wr_id, idx, |
b038ced7 SW |
313 | Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2), |
314 | sqp->opcode); | |
315 | wr = wr->next; | |
316 | num_wrs--; | |
317 | ++(qhp->wq.wptr); | |
318 | ++(qhp->wq.sq_wptr); | |
319 | } | |
320 | spin_unlock_irqrestore(&qhp->lock, flag); | |
321 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
322 | return err; | |
323 | } | |
324 | ||
325 | int iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, | |
326 | struct ib_recv_wr **bad_wr) | |
327 | { | |
328 | int err = 0; | |
329 | struct iwch_qp *qhp; | |
330 | u32 idx; | |
331 | union t3_wr *wqe; | |
332 | u32 num_wrs; | |
333 | unsigned long flag; | |
334 | ||
335 | qhp = to_iwch_qp(ibqp); | |
336 | spin_lock_irqsave(&qhp->lock, flag); | |
337 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
338 | spin_unlock_irqrestore(&qhp->lock, flag); | |
339 | return -EINVAL; | |
340 | } | |
341 | num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, | |
342 | qhp->wq.rq_size_log2) - 1; | |
343 | if (!wr) { | |
344 | spin_unlock_irqrestore(&qhp->lock, flag); | |
345 | return -EINVAL; | |
346 | } | |
347 | while (wr) { | |
348 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
349 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
350 | if (num_wrs) | |
351 | err = iwch_build_rdma_recv(qhp->rhp, wqe, wr); | |
352 | else | |
353 | err = -ENOMEM; | |
354 | if (err) { | |
355 | *bad_wr = wr; | |
356 | break; | |
357 | } | |
358 | qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, qhp->wq.rq_size_log2)] = | |
359 | wr->wr_id; | |
360 | build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, | |
361 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), | |
362 | 0, sizeof(struct t3_receive_wr) >> 3); | |
363 | PDBG("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x " | |
33718363 | 364 | "wqe %p \n", __func__, (unsigned long long) wr->wr_id, |
b038ced7 SW |
365 | idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); |
366 | ++(qhp->wq.rq_wptr); | |
367 | ++(qhp->wq.wptr); | |
368 | wr = wr->next; | |
369 | num_wrs--; | |
370 | } | |
371 | spin_unlock_irqrestore(&qhp->lock, flag); | |
372 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
373 | return err; | |
374 | } | |
375 | ||
376 | int iwch_bind_mw(struct ib_qp *qp, | |
377 | struct ib_mw *mw, | |
378 | struct ib_mw_bind *mw_bind) | |
379 | { | |
380 | struct iwch_dev *rhp; | |
381 | struct iwch_mw *mhp; | |
382 | struct iwch_qp *qhp; | |
383 | union t3_wr *wqe; | |
384 | u32 pbl_addr; | |
385 | u8 page_size; | |
386 | u32 num_wrs; | |
387 | unsigned long flag; | |
388 | struct ib_sge sgl; | |
389 | int err=0; | |
390 | enum t3_wr_flags t3_wr_flags; | |
391 | u32 idx; | |
392 | struct t3_swsq *sqp; | |
393 | ||
394 | qhp = to_iwch_qp(qp); | |
395 | mhp = to_iwch_mw(mw); | |
396 | rhp = qhp->rhp; | |
397 | ||
398 | spin_lock_irqsave(&qhp->lock, flag); | |
399 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
400 | spin_unlock_irqrestore(&qhp->lock, flag); | |
401 | return -EINVAL; | |
402 | } | |
403 | num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, | |
404 | qhp->wq.sq_size_log2); | |
405 | if ((num_wrs) <= 0) { | |
406 | spin_unlock_irqrestore(&qhp->lock, flag); | |
407 | return -ENOMEM; | |
408 | } | |
409 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
33718363 | 410 | PDBG("%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p\n", __func__, idx, |
b038ced7 SW |
411 | mw, mw_bind); |
412 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
413 | ||
414 | t3_wr_flags = 0; | |
415 | if (mw_bind->send_flags & IB_SEND_SIGNALED) | |
416 | t3_wr_flags = T3_COMPLETION_FLAG; | |
417 | ||
418 | sgl.addr = mw_bind->addr; | |
419 | sgl.lkey = mw_bind->mr->lkey; | |
420 | sgl.length = mw_bind->length; | |
421 | wqe->bind.reserved = 0; | |
422 | wqe->bind.type = T3_VA_BASED_TO; | |
423 | ||
424 | /* TBD: check perms */ | |
e64518f3 | 425 | wqe->bind.perms = iwch_ib_to_mwbind_access(mw_bind->mw_access_flags); |
b038ced7 SW |
426 | wqe->bind.mr_stag = cpu_to_be32(mw_bind->mr->lkey); |
427 | wqe->bind.mw_stag = cpu_to_be32(mw->rkey); | |
428 | wqe->bind.mw_len = cpu_to_be32(mw_bind->length); | |
429 | wqe->bind.mw_va = cpu_to_be64(mw_bind->addr); | |
430 | err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size); | |
431 | if (err) { | |
432 | spin_unlock_irqrestore(&qhp->lock, flag); | |
433 | return err; | |
434 | } | |
435 | wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; | |
436 | sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); | |
437 | sqp->wr_id = mw_bind->wr_id; | |
438 | sqp->opcode = T3_BIND_MW; | |
439 | sqp->sq_wptr = qhp->wq.sq_wptr; | |
440 | sqp->complete = 0; | |
441 | sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED); | |
442 | wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr); | |
443 | wqe->bind.mr_pagesz = page_size; | |
444 | wqe->flit[T3_SQ_COOKIE_FLIT] = mw_bind->wr_id; | |
445 | build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags, | |
446 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0, | |
447 | sizeof(struct t3_bind_mw_wr) >> 3); | |
448 | ++(qhp->wq.wptr); | |
449 | ++(qhp->wq.sq_wptr); | |
450 | spin_unlock_irqrestore(&qhp->lock, flag); | |
451 | ||
452 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
453 | ||
454 | return err; | |
455 | } | |
456 | ||
4a97d47e SW |
457 | static inline void build_term_codes(struct respQ_msg_t *rsp_msg, |
458 | u8 *layer_type, u8 *ecode) | |
b038ced7 | 459 | { |
4a97d47e SW |
460 | int status = TPT_ERR_INTERNAL_ERR; |
461 | int tagged = 0; | |
462 | int opcode = -1; | |
463 | int rqtype = 0; | |
464 | int send_inv = 0; | |
465 | ||
466 | if (rsp_msg) { | |
467 | status = CQE_STATUS(rsp_msg->cqe); | |
468 | opcode = CQE_OPCODE(rsp_msg->cqe); | |
469 | rqtype = RQ_TYPE(rsp_msg->cqe); | |
470 | send_inv = (opcode == T3_SEND_WITH_INV) || | |
471 | (opcode == T3_SEND_WITH_SE_INV); | |
472 | tagged = (opcode == T3_RDMA_WRITE) || | |
473 | (rqtype && (opcode == T3_READ_RESP)); | |
474 | } | |
475 | ||
476 | switch (status) { | |
b038ced7 | 477 | case TPT_ERR_STAG: |
4a97d47e SW |
478 | if (send_inv) { |
479 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
480 | *ecode = RDMAP_CANT_INV_STAG; | |
481 | } else { | |
b038ced7 SW |
482 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
483 | *ecode = RDMAP_INV_STAG; | |
484 | } | |
485 | break; | |
486 | case TPT_ERR_PDID: | |
4a97d47e SW |
487 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
488 | if ((opcode == T3_SEND_WITH_INV) || | |
489 | (opcode == T3_SEND_WITH_SE_INV)) | |
490 | *ecode = RDMAP_CANT_INV_STAG; | |
491 | else | |
492 | *ecode = RDMAP_STAG_NOT_ASSOC; | |
493 | break; | |
b038ced7 | 494 | case TPT_ERR_QPID: |
4a97d47e SW |
495 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
496 | *ecode = RDMAP_STAG_NOT_ASSOC; | |
497 | break; | |
b038ced7 | 498 | case TPT_ERR_ACCESS: |
4a97d47e SW |
499 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
500 | *ecode = RDMAP_ACC_VIOL; | |
b038ced7 SW |
501 | break; |
502 | case TPT_ERR_WRAP: | |
503 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; | |
504 | *ecode = RDMAP_TO_WRAP; | |
505 | break; | |
506 | case TPT_ERR_BOUND: | |
4a97d47e | 507 | if (tagged) { |
b038ced7 SW |
508 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; |
509 | *ecode = DDPT_BASE_BOUNDS; | |
4a97d47e | 510 | } else { |
b038ced7 SW |
511 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
512 | *ecode = RDMAP_BASE_BOUNDS; | |
b038ced7 SW |
513 | } |
514 | break; | |
515 | case TPT_ERR_INVALIDATE_SHARED_MR: | |
516 | case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: | |
517 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
518 | *ecode = RDMAP_CANT_INV_STAG; | |
519 | break; | |
520 | case TPT_ERR_ECC: | |
521 | case TPT_ERR_ECC_PSTAG: | |
522 | case TPT_ERR_INTERNAL_ERR: | |
523 | *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; | |
524 | *ecode = 0; | |
525 | break; | |
526 | case TPT_ERR_OUT_OF_RQE: | |
527 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
528 | *ecode = DDPU_INV_MSN_NOBUF; | |
529 | break; | |
530 | case TPT_ERR_PBL_ADDR_BOUND: | |
531 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
532 | *ecode = DDPT_BASE_BOUNDS; | |
533 | break; | |
534 | case TPT_ERR_CRC: | |
535 | *layer_type = LAYER_MPA|DDP_LLP; | |
536 | *ecode = MPA_CRC_ERR; | |
537 | break; | |
538 | case TPT_ERR_MARKER: | |
539 | *layer_type = LAYER_MPA|DDP_LLP; | |
540 | *ecode = MPA_MARKER_ERR; | |
541 | break; | |
542 | case TPT_ERR_PDU_LEN_ERR: | |
543 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
544 | *ecode = DDPU_MSG_TOOBIG; | |
545 | break; | |
546 | case TPT_ERR_DDP_VERSION: | |
547 | if (tagged) { | |
548 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
549 | *ecode = DDPT_INV_VERS; | |
550 | } else { | |
551 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
552 | *ecode = DDPU_INV_VERS; | |
553 | } | |
554 | break; | |
555 | case TPT_ERR_RDMA_VERSION: | |
556 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
557 | *ecode = RDMAP_INV_VERS; | |
558 | break; | |
559 | case TPT_ERR_OPCODE: | |
560 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
561 | *ecode = RDMAP_INV_OPCODE; | |
562 | break; | |
563 | case TPT_ERR_DDP_QUEUE_NUM: | |
564 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
565 | *ecode = DDPU_INV_QN; | |
566 | break; | |
567 | case TPT_ERR_MSN: | |
568 | case TPT_ERR_MSN_GAP: | |
569 | case TPT_ERR_MSN_RANGE: | |
570 | case TPT_ERR_IRD_OVERFLOW: | |
571 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
572 | *ecode = DDPU_INV_MSN_RANGE; | |
573 | break; | |
574 | case TPT_ERR_TBIT: | |
575 | *layer_type = LAYER_DDP|DDP_LOCAL_CATA; | |
576 | *ecode = 0; | |
577 | break; | |
578 | case TPT_ERR_MO: | |
579 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
580 | *ecode = DDPU_INV_MO; | |
581 | break; | |
582 | default: | |
583 | *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; | |
584 | *ecode = 0; | |
585 | break; | |
586 | } | |
587 | } | |
588 | ||
f8b0dfd1 SW |
589 | int iwch_post_zb_read(struct iwch_qp *qhp) |
590 | { | |
591 | union t3_wr *wqe; | |
592 | struct sk_buff *skb; | |
593 | u8 flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; | |
594 | ||
595 | PDBG("%s enter\n", __func__); | |
596 | skb = alloc_skb(40, GFP_KERNEL); | |
597 | if (!skb) { | |
598 | printk(KERN_ERR "%s cannot send zb_read!!\n", __func__); | |
599 | return -ENOMEM; | |
600 | } | |
601 | wqe = (union t3_wr *)skb_put(skb, sizeof(struct t3_rdma_read_wr)); | |
602 | memset(wqe, 0, sizeof(struct t3_rdma_read_wr)); | |
603 | wqe->read.rdmaop = T3_READ_REQ; | |
604 | wqe->read.reserved[0] = 0; | |
605 | wqe->read.reserved[1] = 0; | |
606 | wqe->read.reserved[2] = 0; | |
607 | wqe->read.rem_stag = cpu_to_be32(1); | |
608 | wqe->read.rem_to = cpu_to_be64(1); | |
609 | wqe->read.local_stag = cpu_to_be32(1); | |
610 | wqe->read.local_len = cpu_to_be32(0); | |
611 | wqe->read.local_to = cpu_to_be64(1); | |
612 | wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_READ)); | |
613 | wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid)| | |
614 | V_FW_RIWR_LEN(flit_cnt)); | |
615 | skb->priority = CPL_PRIORITY_DATA; | |
616 | return cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); | |
617 | } | |
618 | ||
b038ced7 SW |
619 | /* |
620 | * This posts a TERMINATE with layer=RDMA, type=catastrophic. | |
621 | */ | |
622 | int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg) | |
623 | { | |
624 | union t3_wr *wqe; | |
625 | struct terminate_message *term; | |
b038ced7 SW |
626 | struct sk_buff *skb; |
627 | ||
33718363 | 628 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
629 | skb = alloc_skb(40, GFP_ATOMIC); |
630 | if (!skb) { | |
33718363 | 631 | printk(KERN_ERR "%s cannot send TERMINATE!\n", __func__); |
b038ced7 SW |
632 | return -ENOMEM; |
633 | } | |
634 | wqe = (union t3_wr *)skb_put(skb, 40); | |
635 | memset(wqe, 0, 40); | |
636 | wqe->send.rdmaop = T3_TERMINATE; | |
637 | ||
638 | /* immediate data length */ | |
639 | wqe->send.plen = htonl(4); | |
640 | ||
641 | /* immediate data starts here. */ | |
642 | term = (struct terminate_message *)wqe->send.sgl; | |
4a97d47e | 643 | build_term_codes(rsp_msg, &term->layer_etype, &term->ecode); |
fb497d72 SW |
644 | wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_SEND) | |
645 | V_FW_RIWR_FLAGS(T3_COMPLETION_FLAG | T3_NOTIFY_FLAG)); | |
646 | wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid)); | |
b038ced7 SW |
647 | skb->priority = CPL_PRIORITY_DATA; |
648 | return cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); | |
649 | } | |
650 | ||
651 | /* | |
652 | * Assumes qhp lock is held. | |
653 | */ | |
654 | static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag) | |
655 | { | |
656 | struct iwch_cq *rchp, *schp; | |
657 | int count; | |
c8286944 | 658 | int flushed; |
b038ced7 SW |
659 | |
660 | rchp = get_chp(qhp->rhp, qhp->attr.rcq); | |
661 | schp = get_chp(qhp->rhp, qhp->attr.scq); | |
662 | ||
33718363 | 663 | PDBG("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp); |
b038ced7 SW |
664 | /* take a ref on the qhp since we must release the lock */ |
665 | atomic_inc(&qhp->refcnt); | |
666 | spin_unlock_irqrestore(&qhp->lock, *flag); | |
667 | ||
668 | /* locking heirarchy: cq lock first, then qp lock. */ | |
669 | spin_lock_irqsave(&rchp->lock, *flag); | |
670 | spin_lock(&qhp->lock); | |
671 | cxio_flush_hw_cq(&rchp->cq); | |
672 | cxio_count_rcqes(&rchp->cq, &qhp->wq, &count); | |
c8286944 | 673 | flushed = cxio_flush_rq(&qhp->wq, &rchp->cq, count); |
b038ced7 SW |
674 | spin_unlock(&qhp->lock); |
675 | spin_unlock_irqrestore(&rchp->lock, *flag); | |
c8286944 SW |
676 | if (flushed) |
677 | (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); | |
b038ced7 SW |
678 | |
679 | /* locking heirarchy: cq lock first, then qp lock. */ | |
680 | spin_lock_irqsave(&schp->lock, *flag); | |
681 | spin_lock(&qhp->lock); | |
682 | cxio_flush_hw_cq(&schp->cq); | |
683 | cxio_count_scqes(&schp->cq, &qhp->wq, &count); | |
c8286944 | 684 | flushed = cxio_flush_sq(&qhp->wq, &schp->cq, count); |
b038ced7 SW |
685 | spin_unlock(&qhp->lock); |
686 | spin_unlock_irqrestore(&schp->lock, *flag); | |
c8286944 SW |
687 | if (flushed) |
688 | (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); | |
b038ced7 SW |
689 | |
690 | /* deref */ | |
691 | if (atomic_dec_and_test(&qhp->refcnt)) | |
692 | wake_up(&qhp->wait); | |
693 | ||
694 | spin_lock_irqsave(&qhp->lock, *flag); | |
695 | } | |
696 | ||
2b540355 | 697 | static void flush_qp(struct iwch_qp *qhp, unsigned long *flag) |
b038ced7 | 698 | { |
856b5925 | 699 | if (qhp->ibqp.uobject) |
b038ced7 SW |
700 | cxio_set_wq_in_error(&qhp->wq); |
701 | else | |
702 | __flush_qp(qhp, flag); | |
703 | } | |
704 | ||
705 | ||
706 | /* | |
f8b0dfd1 | 707 | * Return count of RECV WRs posted |
b038ced7 | 708 | */ |
f8b0dfd1 | 709 | u16 iwch_rqes_posted(struct iwch_qp *qhp) |
b038ced7 | 710 | { |
f8b0dfd1 SW |
711 | union t3_wr *wqe = qhp->wq.queue; |
712 | u16 count = 0; | |
713 | while ((count+1) != 0 && fw_riwrh_opcode((struct fw_riwrh *)wqe) == T3_WR_RCV) { | |
714 | count++; | |
715 | wqe++; | |
716 | } | |
717 | PDBG("%s qhp %p count %u\n", __func__, qhp, count); | |
718 | return count; | |
b038ced7 SW |
719 | } |
720 | ||
721 | static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, | |
722 | enum iwch_qp_attr_mask mask, | |
723 | struct iwch_qp_attributes *attrs) | |
724 | { | |
725 | struct t3_rdma_init_attr init_attr; | |
726 | int ret; | |
727 | ||
728 | init_attr.tid = qhp->ep->hwtid; | |
729 | init_attr.qpid = qhp->wq.qpid; | |
730 | init_attr.pdid = qhp->attr.pd; | |
731 | init_attr.scqid = qhp->attr.scq; | |
732 | init_attr.rcqid = qhp->attr.rcq; | |
733 | init_attr.rq_addr = qhp->wq.rq_addr; | |
734 | init_attr.rq_size = 1 << qhp->wq.rq_size_log2; | |
735 | init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE | | |
736 | qhp->attr.mpa_attr.recv_marker_enabled | | |
737 | (qhp->attr.mpa_attr.xmit_marker_enabled << 1) | | |
738 | (qhp->attr.mpa_attr.crc_enabled << 2); | |
739 | ||
740 | /* | |
741 | * XXX - The IWCM doesn't quite handle getting these | |
742 | * attrs set before going into RTS. For now, just turn | |
743 | * them on always... | |
744 | */ | |
745 | #if 0 | |
746 | init_attr.qpcaps = qhp->attr.enableRdmaRead | | |
747 | (qhp->attr.enableRdmaWrite << 1) | | |
748 | (qhp->attr.enableBind << 2) | | |
749 | (qhp->attr.enable_stag0_fastreg << 3) | | |
750 | (qhp->attr.enable_stag0_fastreg << 4); | |
751 | #else | |
752 | init_attr.qpcaps = 0x1f; | |
753 | #endif | |
754 | init_attr.tcp_emss = qhp->ep->emss; | |
755 | init_attr.ord = qhp->attr.max_ord; | |
756 | init_attr.ird = qhp->attr.max_ird; | |
757 | init_attr.qp_dma_addr = qhp->wq.dma_addr; | |
758 | init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); | |
f8b0dfd1 SW |
759 | init_attr.rqe_count = iwch_rqes_posted(qhp); |
760 | init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0; | |
c6b5b504 | 761 | init_attr.flags |= capable(CAP_NET_BIND_SERVICE) ? PRIV_QP : 0; |
f8b0dfd1 SW |
762 | if (peer2peer) { |
763 | init_attr.rtr_type = RTR_READ; | |
764 | if (init_attr.ord == 0 && qhp->attr.mpa_attr.initiator) | |
765 | init_attr.ord = 1; | |
766 | if (init_attr.ird == 0 && !qhp->attr.mpa_attr.initiator) | |
767 | init_attr.ird = 1; | |
768 | } else | |
769 | init_attr.rtr_type = 0; | |
de3d3530 | 770 | init_attr.irs = qhp->ep->rcv_seq; |
b038ced7 | 771 | PDBG("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d " |
33718363 | 772 | "flags 0x%x qpcaps 0x%x\n", __func__, |
b038ced7 SW |
773 | init_attr.rq_addr, init_attr.rq_size, |
774 | init_attr.flags, init_attr.qpcaps); | |
775 | ret = cxio_rdma_init(&rhp->rdev, &init_attr); | |
33718363 | 776 | PDBG("%s ret %d\n", __func__, ret); |
b038ced7 SW |
777 | return ret; |
778 | } | |
779 | ||
780 | int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp, | |
781 | enum iwch_qp_attr_mask mask, | |
782 | struct iwch_qp_attributes *attrs, | |
783 | int internal) | |
784 | { | |
785 | int ret = 0; | |
786 | struct iwch_qp_attributes newattr = qhp->attr; | |
787 | unsigned long flag; | |
788 | int disconnect = 0; | |
789 | int terminate = 0; | |
790 | int abort = 0; | |
791 | int free = 0; | |
792 | struct iwch_ep *ep = NULL; | |
793 | ||
33718363 | 794 | PDBG("%s qhp %p qpid 0x%x ep %p state %d -> %d\n", __func__, |
b038ced7 SW |
795 | qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state, |
796 | (mask & IWCH_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); | |
797 | ||
798 | spin_lock_irqsave(&qhp->lock, flag); | |
799 | ||
800 | /* Process attr changes if in IDLE */ | |
801 | if (mask & IWCH_QP_ATTR_VALID_MODIFY) { | |
802 | if (qhp->attr.state != IWCH_QP_STATE_IDLE) { | |
803 | ret = -EIO; | |
804 | goto out; | |
805 | } | |
806 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_READ) | |
807 | newattr.enable_rdma_read = attrs->enable_rdma_read; | |
808 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_WRITE) | |
809 | newattr.enable_rdma_write = attrs->enable_rdma_write; | |
810 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_BIND) | |
811 | newattr.enable_bind = attrs->enable_bind; | |
812 | if (mask & IWCH_QP_ATTR_MAX_ORD) { | |
813 | if (attrs->max_ord > | |
814 | rhp->attr.max_rdma_read_qp_depth) { | |
815 | ret = -EINVAL; | |
816 | goto out; | |
817 | } | |
818 | newattr.max_ord = attrs->max_ord; | |
819 | } | |
820 | if (mask & IWCH_QP_ATTR_MAX_IRD) { | |
821 | if (attrs->max_ird > | |
822 | rhp->attr.max_rdma_reads_per_qp) { | |
823 | ret = -EINVAL; | |
824 | goto out; | |
825 | } | |
826 | newattr.max_ird = attrs->max_ird; | |
827 | } | |
828 | qhp->attr = newattr; | |
829 | } | |
830 | ||
831 | if (!(mask & IWCH_QP_ATTR_NEXT_STATE)) | |
832 | goto out; | |
833 | if (qhp->attr.state == attrs->next_state) | |
834 | goto out; | |
835 | ||
836 | switch (qhp->attr.state) { | |
837 | case IWCH_QP_STATE_IDLE: | |
838 | switch (attrs->next_state) { | |
839 | case IWCH_QP_STATE_RTS: | |
840 | if (!(mask & IWCH_QP_ATTR_LLP_STREAM_HANDLE)) { | |
841 | ret = -EINVAL; | |
842 | goto out; | |
843 | } | |
844 | if (!(mask & IWCH_QP_ATTR_MPA_ATTR)) { | |
845 | ret = -EINVAL; | |
846 | goto out; | |
847 | } | |
848 | qhp->attr.mpa_attr = attrs->mpa_attr; | |
849 | qhp->attr.llp_stream_handle = attrs->llp_stream_handle; | |
850 | qhp->ep = qhp->attr.llp_stream_handle; | |
851 | qhp->attr.state = IWCH_QP_STATE_RTS; | |
852 | ||
853 | /* | |
854 | * Ref the endpoint here and deref when we | |
855 | * disassociate the endpoint from the QP. This | |
856 | * happens in CLOSING->IDLE transition or *->ERROR | |
857 | * transition. | |
858 | */ | |
859 | get_ep(&qhp->ep->com); | |
860 | spin_unlock_irqrestore(&qhp->lock, flag); | |
861 | ret = rdma_init(rhp, qhp, mask, attrs); | |
862 | spin_lock_irqsave(&qhp->lock, flag); | |
863 | if (ret) | |
864 | goto err; | |
865 | break; | |
866 | case IWCH_QP_STATE_ERROR: | |
867 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
868 | flush_qp(qhp, &flag); | |
869 | break; | |
870 | default: | |
871 | ret = -EINVAL; | |
872 | goto out; | |
873 | } | |
874 | break; | |
875 | case IWCH_QP_STATE_RTS: | |
876 | switch (attrs->next_state) { | |
877 | case IWCH_QP_STATE_CLOSING: | |
878 | BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); | |
879 | qhp->attr.state = IWCH_QP_STATE_CLOSING; | |
880 | if (!internal) { | |
881 | abort=0; | |
882 | disconnect = 1; | |
883 | ep = qhp->ep; | |
989a1780 | 884 | get_ep(&ep->com); |
b038ced7 SW |
885 | } |
886 | break; | |
887 | case IWCH_QP_STATE_TERMINATE: | |
888 | qhp->attr.state = IWCH_QP_STATE_TERMINATE; | |
856b5925 | 889 | if (qhp->ibqp.uobject) |
a1a75052 | 890 | cxio_set_wq_in_error(&qhp->wq); |
b038ced7 SW |
891 | if (!internal) |
892 | terminate = 1; | |
893 | break; | |
894 | case IWCH_QP_STATE_ERROR: | |
895 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
896 | if (!internal) { | |
897 | abort=1; | |
898 | disconnect = 1; | |
899 | ep = qhp->ep; | |
989a1780 | 900 | get_ep(&ep->com); |
b038ced7 SW |
901 | } |
902 | goto err; | |
903 | break; | |
904 | default: | |
905 | ret = -EINVAL; | |
906 | goto out; | |
907 | } | |
908 | break; | |
909 | case IWCH_QP_STATE_CLOSING: | |
910 | if (!internal) { | |
911 | ret = -EINVAL; | |
912 | goto out; | |
913 | } | |
914 | switch (attrs->next_state) { | |
915 | case IWCH_QP_STATE_IDLE: | |
c8286944 | 916 | flush_qp(qhp, &flag); |
b038ced7 SW |
917 | qhp->attr.state = IWCH_QP_STATE_IDLE; |
918 | qhp->attr.llp_stream_handle = NULL; | |
919 | put_ep(&qhp->ep->com); | |
920 | qhp->ep = NULL; | |
921 | wake_up(&qhp->wait); | |
922 | break; | |
923 | case IWCH_QP_STATE_ERROR: | |
924 | goto err; | |
925 | default: | |
926 | ret = -EINVAL; | |
927 | goto err; | |
928 | } | |
929 | break; | |
930 | case IWCH_QP_STATE_ERROR: | |
931 | if (attrs->next_state != IWCH_QP_STATE_IDLE) { | |
932 | ret = -EINVAL; | |
933 | goto out; | |
934 | } | |
935 | ||
936 | if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) || | |
937 | !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) { | |
938 | ret = -EINVAL; | |
939 | goto out; | |
940 | } | |
941 | qhp->attr.state = IWCH_QP_STATE_IDLE; | |
942 | memset(&qhp->attr, 0, sizeof(qhp->attr)); | |
943 | break; | |
944 | case IWCH_QP_STATE_TERMINATE: | |
945 | if (!internal) { | |
946 | ret = -EINVAL; | |
947 | goto out; | |
948 | } | |
949 | goto err; | |
950 | break; | |
951 | default: | |
952 | printk(KERN_ERR "%s in a bad state %d\n", | |
33718363 | 953 | __func__, qhp->attr.state); |
b038ced7 SW |
954 | ret = -EINVAL; |
955 | goto err; | |
956 | break; | |
957 | } | |
958 | goto out; | |
959 | err: | |
33718363 | 960 | PDBG("%s disassociating ep %p qpid 0x%x\n", __func__, qhp->ep, |
b038ced7 SW |
961 | qhp->wq.qpid); |
962 | ||
963 | /* disassociate the LLP connection */ | |
964 | qhp->attr.llp_stream_handle = NULL; | |
965 | ep = qhp->ep; | |
966 | qhp->ep = NULL; | |
967 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
968 | free=1; | |
969 | wake_up(&qhp->wait); | |
970 | BUG_ON(!ep); | |
971 | flush_qp(qhp, &flag); | |
972 | out: | |
973 | spin_unlock_irqrestore(&qhp->lock, flag); | |
974 | ||
975 | if (terminate) | |
976 | iwch_post_terminate(qhp, NULL); | |
977 | ||
978 | /* | |
979 | * If disconnect is 1, then we need to initiate a disconnect | |
980 | * on the EP. This can be a normal close (RTS->CLOSING) or | |
981 | * an abnormal close (RTS/CLOSING->ERROR). | |
982 | */ | |
989a1780 | 983 | if (disconnect) { |
b038ced7 | 984 | iwch_ep_disconnect(ep, abort, GFP_KERNEL); |
989a1780 SW |
985 | put_ep(&ep->com); |
986 | } | |
b038ced7 SW |
987 | |
988 | /* | |
989 | * If free is 1, then we've disassociated the EP from the QP | |
990 | * and we need to dereference the EP. | |
991 | */ | |
992 | if (free) | |
993 | put_ep(&ep->com); | |
994 | ||
33718363 | 995 | PDBG("%s exit state %d\n", __func__, qhp->attr.state); |
b038ced7 SW |
996 | return ret; |
997 | } | |
998 | ||
999 | static int quiesce_qp(struct iwch_qp *qhp) | |
1000 | { | |
1001 | spin_lock_irq(&qhp->lock); | |
1002 | iwch_quiesce_tid(qhp->ep); | |
1003 | qhp->flags |= QP_QUIESCED; | |
1004 | spin_unlock_irq(&qhp->lock); | |
1005 | return 0; | |
1006 | } | |
1007 | ||
1008 | static int resume_qp(struct iwch_qp *qhp) | |
1009 | { | |
1010 | spin_lock_irq(&qhp->lock); | |
1011 | iwch_resume_tid(qhp->ep); | |
1012 | qhp->flags &= ~QP_QUIESCED; | |
1013 | spin_unlock_irq(&qhp->lock); | |
1014 | return 0; | |
1015 | } | |
1016 | ||
1017 | int iwch_quiesce_qps(struct iwch_cq *chp) | |
1018 | { | |
1019 | int i; | |
1020 | struct iwch_qp *qhp; | |
1021 | ||
1022 | for (i=0; i < T3_MAX_NUM_QP; i++) { | |
1023 | qhp = get_qhp(chp->rhp, i); | |
1024 | if (!qhp) | |
1025 | continue; | |
1026 | if ((qhp->attr.rcq == chp->cq.cqid) && !qp_quiesced(qhp)) { | |
1027 | quiesce_qp(qhp); | |
1028 | continue; | |
1029 | } | |
1030 | if ((qhp->attr.scq == chp->cq.cqid) && !qp_quiesced(qhp)) | |
1031 | quiesce_qp(qhp); | |
1032 | } | |
1033 | return 0; | |
1034 | } | |
1035 | ||
1036 | int iwch_resume_qps(struct iwch_cq *chp) | |
1037 | { | |
1038 | int i; | |
1039 | struct iwch_qp *qhp; | |
1040 | ||
1041 | for (i=0; i < T3_MAX_NUM_QP; i++) { | |
1042 | qhp = get_qhp(chp->rhp, i); | |
1043 | if (!qhp) | |
1044 | continue; | |
1045 | if ((qhp->attr.rcq == chp->cq.cqid) && qp_quiesced(qhp)) { | |
1046 | resume_qp(qhp); | |
1047 | continue; | |
1048 | } | |
1049 | if ((qhp->attr.scq == chp->cq.cqid) && qp_quiesced(qhp)) | |
1050 | resume_qp(qhp); | |
1051 | } | |
1052 | return 0; | |
1053 | } |