]>
Commit | Line | Data |
---|---|---|
f94b533d TT |
1 | /* |
2 | * Copyright (c) 2004 Topspin Communications. All rights reserved. | |
3 | * Copyright (c) 2005 Cisco Systems. All rights reserved. | |
4 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. | |
5 | * Copyright (c) 2004 Voltaire, Inc. All rights reserved. | |
6 | * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. | |
7 | * | |
8 | * This software is available to you under a choice of one of two | |
9 | * licenses. You may choose to be licensed under the terms of the GNU | |
10 | * General Public License (GPL) Version 2, available from the file | |
11 | * COPYING in the main directory of this source tree, or the | |
12 | * OpenIB.org BSD license below: | |
13 | * | |
14 | * Redistribution and use in source and binary forms, with or | |
15 | * without modification, are permitted provided that the following | |
16 | * conditions are met: | |
17 | * | |
18 | * - Redistributions of source code must retain the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer. | |
21 | * | |
22 | * - Redistributions in binary form must reproduce the above | |
23 | * copyright notice, this list of conditions and the following | |
24 | * disclaimer in the documentation and/or other materials | |
25 | * provided with the distribution. | |
26 | * | |
27 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
28 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
29 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
30 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
31 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
32 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
33 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
34 | * SOFTWARE. | |
35 | * | |
36 | */ | |
37 | ||
e52e6080 | 38 | #include <linux/delay.h> |
5a0e3ad6 | 39 | #include <linux/gfp.h> |
e52e6080 | 40 | |
f94b533d TT |
41 | #include "c2.h" |
42 | #include "c2_vq.h" | |
43 | #include "c2_status.h" | |
44 | ||
45 | #define C2_MAX_ORD_PER_QP 128 | |
46 | #define C2_MAX_IRD_PER_QP 128 | |
47 | ||
48 | #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count) | |
49 | #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16) | |
50 | #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF) | |
51 | ||
52 | #define NO_SUPPORT -1 | |
53 | static const u8 c2_opcode[] = { | |
54 | [IB_WR_SEND] = C2_WR_TYPE_SEND, | |
55 | [IB_WR_SEND_WITH_IMM] = NO_SUPPORT, | |
56 | [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE, | |
57 | [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT, | |
58 | [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ, | |
59 | [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT, | |
60 | [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT, | |
61 | }; | |
62 | ||
63 | static int to_c2_state(enum ib_qp_state ib_state) | |
64 | { | |
65 | switch (ib_state) { | |
66 | case IB_QPS_RESET: | |
67 | return C2_QP_STATE_IDLE; | |
68 | case IB_QPS_RTS: | |
69 | return C2_QP_STATE_RTS; | |
70 | case IB_QPS_SQD: | |
71 | return C2_QP_STATE_CLOSING; | |
72 | case IB_QPS_SQE: | |
73 | return C2_QP_STATE_CLOSING; | |
74 | case IB_QPS_ERR: | |
75 | return C2_QP_STATE_ERROR; | |
76 | default: | |
77 | return -1; | |
78 | } | |
79 | } | |
80 | ||
81 | static int to_ib_state(enum c2_qp_state c2_state) | |
82 | { | |
83 | switch (c2_state) { | |
84 | case C2_QP_STATE_IDLE: | |
85 | return IB_QPS_RESET; | |
86 | case C2_QP_STATE_CONNECTING: | |
87 | return IB_QPS_RTR; | |
88 | case C2_QP_STATE_RTS: | |
89 | return IB_QPS_RTS; | |
90 | case C2_QP_STATE_CLOSING: | |
91 | return IB_QPS_SQD; | |
92 | case C2_QP_STATE_ERROR: | |
93 | return IB_QPS_ERR; | |
94 | case C2_QP_STATE_TERMINATE: | |
95 | return IB_QPS_SQE; | |
96 | default: | |
97 | return -1; | |
98 | } | |
99 | } | |
100 | ||
101 | static const char *to_ib_state_str(int ib_state) | |
102 | { | |
103 | static const char *state_str[] = { | |
104 | "IB_QPS_RESET", | |
105 | "IB_QPS_INIT", | |
106 | "IB_QPS_RTR", | |
107 | "IB_QPS_RTS", | |
108 | "IB_QPS_SQD", | |
109 | "IB_QPS_SQE", | |
110 | "IB_QPS_ERR" | |
111 | }; | |
112 | if (ib_state < IB_QPS_RESET || | |
113 | ib_state > IB_QPS_ERR) | |
114 | return "<invalid IB QP state>"; | |
115 | ||
116 | ib_state -= IB_QPS_RESET; | |
117 | return state_str[ib_state]; | |
118 | } | |
119 | ||
120 | void c2_set_qp_state(struct c2_qp *qp, int c2_state) | |
121 | { | |
122 | int new_state = to_ib_state(c2_state); | |
123 | ||
124 | pr_debug("%s: qp[%p] state modify %s --> %s\n", | |
33718363 | 125 | __func__, |
f94b533d TT |
126 | qp, |
127 | to_ib_state_str(qp->state), | |
128 | to_ib_state_str(new_state)); | |
129 | qp->state = new_state; | |
130 | } | |
131 | ||
132 | #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF | |
133 | ||
134 | int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp, | |
135 | struct ib_qp_attr *attr, int attr_mask) | |
136 | { | |
137 | struct c2wr_qp_modify_req wr; | |
138 | struct c2wr_qp_modify_rep *reply; | |
139 | struct c2_vq_req *vq_req; | |
140 | unsigned long flags; | |
141 | u8 next_state; | |
142 | int err; | |
143 | ||
144 | pr_debug("%s:%d qp=%p, %s --> %s\n", | |
33718363 | 145 | __func__, __LINE__, |
f94b533d TT |
146 | qp, |
147 | to_ib_state_str(qp->state), | |
148 | to_ib_state_str(attr->qp_state)); | |
149 | ||
150 | vq_req = vq_req_alloc(c2dev); | |
151 | if (!vq_req) | |
152 | return -ENOMEM; | |
153 | ||
154 | c2_wr_set_id(&wr, CCWR_QP_MODIFY); | |
155 | wr.hdr.context = (unsigned long) vq_req; | |
156 | wr.rnic_handle = c2dev->adapter_handle; | |
157 | wr.qp_handle = qp->adapter_handle; | |
158 | wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
159 | wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
160 | wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
161 | wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
162 | ||
163 | if (attr_mask & IB_QP_STATE) { | |
164 | /* Ensure the state is valid */ | |
ad1f9791 KK |
165 | if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) { |
166 | err = -EINVAL; | |
167 | goto bail0; | |
168 | } | |
f94b533d TT |
169 | |
170 | wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state)); | |
171 | ||
172 | if (attr->qp_state == IB_QPS_ERR) { | |
173 | spin_lock_irqsave(&qp->lock, flags); | |
174 | if (qp->cm_id && qp->state == IB_QPS_RTS) { | |
175 | pr_debug("Generating CLOSE event for QP-->ERR, " | |
176 | "qp=%p, cm_id=%p\n",qp,qp->cm_id); | |
177 | /* Generate an CLOSE event */ | |
178 | vq_req->cm_id = qp->cm_id; | |
179 | vq_req->event = IW_CM_EVENT_CLOSE; | |
180 | } | |
181 | spin_unlock_irqrestore(&qp->lock, flags); | |
182 | } | |
183 | next_state = attr->qp_state; | |
184 | ||
185 | } else if (attr_mask & IB_QP_CUR_STATE) { | |
186 | ||
187 | if (attr->cur_qp_state != IB_QPS_RTR && | |
188 | attr->cur_qp_state != IB_QPS_RTS && | |
189 | attr->cur_qp_state != IB_QPS_SQD && | |
ad1f9791 KK |
190 | attr->cur_qp_state != IB_QPS_SQE) { |
191 | err = -EINVAL; | |
192 | goto bail0; | |
193 | } else | |
f94b533d TT |
194 | wr.next_qp_state = |
195 | cpu_to_be32(to_c2_state(attr->cur_qp_state)); | |
196 | ||
197 | next_state = attr->cur_qp_state; | |
198 | ||
199 | } else { | |
200 | err = 0; | |
201 | goto bail0; | |
202 | } | |
203 | ||
204 | /* reference the request struct */ | |
205 | vq_req_get(c2dev, vq_req); | |
206 | ||
207 | err = vq_send_wr(c2dev, (union c2wr *) & wr); | |
208 | if (err) { | |
209 | vq_req_put(c2dev, vq_req); | |
210 | goto bail0; | |
211 | } | |
212 | ||
213 | err = vq_wait_for_reply(c2dev, vq_req); | |
214 | if (err) | |
215 | goto bail0; | |
216 | ||
217 | reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg; | |
218 | if (!reply) { | |
219 | err = -ENOMEM; | |
220 | goto bail0; | |
221 | } | |
222 | ||
223 | err = c2_errno(reply); | |
224 | if (!err) | |
225 | qp->state = next_state; | |
226 | #ifdef DEBUG | |
227 | else | |
33718363 | 228 | pr_debug("%s: c2_errno=%d\n", __func__, err); |
f94b533d TT |
229 | #endif |
230 | /* | |
231 | * If we're going to error and generating the event here, then | |
232 | * we need to remove the reference because there will be no | |
233 | * close event generated by the adapter | |
234 | */ | |
235 | spin_lock_irqsave(&qp->lock, flags); | |
236 | if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) { | |
237 | qp->cm_id->rem_ref(qp->cm_id); | |
238 | qp->cm_id = NULL; | |
239 | } | |
240 | spin_unlock_irqrestore(&qp->lock, flags); | |
241 | ||
242 | vq_repbuf_free(c2dev, reply); | |
243 | bail0: | |
244 | vq_req_free(c2dev, vq_req); | |
245 | ||
246 | pr_debug("%s:%d qp=%p, cur_state=%s\n", | |
33718363 | 247 | __func__, __LINE__, |
f94b533d TT |
248 | qp, |
249 | to_ib_state_str(qp->state)); | |
250 | return err; | |
251 | } | |
252 | ||
253 | int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp, | |
254 | int ord, int ird) | |
255 | { | |
256 | struct c2wr_qp_modify_req wr; | |
257 | struct c2wr_qp_modify_rep *reply; | |
258 | struct c2_vq_req *vq_req; | |
259 | int err; | |
260 | ||
261 | vq_req = vq_req_alloc(c2dev); | |
262 | if (!vq_req) | |
263 | return -ENOMEM; | |
264 | ||
265 | c2_wr_set_id(&wr, CCWR_QP_MODIFY); | |
266 | wr.hdr.context = (unsigned long) vq_req; | |
267 | wr.rnic_handle = c2dev->adapter_handle; | |
268 | wr.qp_handle = qp->adapter_handle; | |
269 | wr.ord = cpu_to_be32(ord); | |
270 | wr.ird = cpu_to_be32(ird); | |
271 | wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
272 | wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
273 | wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
274 | ||
275 | /* reference the request struct */ | |
276 | vq_req_get(c2dev, vq_req); | |
277 | ||
278 | err = vq_send_wr(c2dev, (union c2wr *) & wr); | |
279 | if (err) { | |
280 | vq_req_put(c2dev, vq_req); | |
281 | goto bail0; | |
282 | } | |
283 | ||
284 | err = vq_wait_for_reply(c2dev, vq_req); | |
285 | if (err) | |
286 | goto bail0; | |
287 | ||
288 | reply = (struct c2wr_qp_modify_rep *) (unsigned long) | |
289 | vq_req->reply_msg; | |
290 | if (!reply) { | |
291 | err = -ENOMEM; | |
292 | goto bail0; | |
293 | } | |
294 | ||
295 | err = c2_errno(reply); | |
296 | vq_repbuf_free(c2dev, reply); | |
297 | bail0: | |
298 | vq_req_free(c2dev, vq_req); | |
299 | return err; | |
300 | } | |
301 | ||
302 | static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp) | |
303 | { | |
304 | struct c2_vq_req *vq_req; | |
305 | struct c2wr_qp_destroy_req wr; | |
306 | struct c2wr_qp_destroy_rep *reply; | |
307 | unsigned long flags; | |
308 | int err; | |
309 | ||
310 | /* | |
311 | * Allocate a verb request message | |
312 | */ | |
313 | vq_req = vq_req_alloc(c2dev); | |
314 | if (!vq_req) { | |
315 | return -ENOMEM; | |
316 | } | |
317 | ||
318 | /* | |
319 | * Initialize the WR | |
320 | */ | |
321 | c2_wr_set_id(&wr, CCWR_QP_DESTROY); | |
322 | wr.hdr.context = (unsigned long) vq_req; | |
323 | wr.rnic_handle = c2dev->adapter_handle; | |
324 | wr.qp_handle = qp->adapter_handle; | |
325 | ||
326 | /* | |
327 | * reference the request struct. dereferenced in the int handler. | |
328 | */ | |
329 | vq_req_get(c2dev, vq_req); | |
330 | ||
331 | spin_lock_irqsave(&qp->lock, flags); | |
332 | if (qp->cm_id && qp->state == IB_QPS_RTS) { | |
333 | pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, " | |
334 | "qp=%p, cm_id=%p\n",qp,qp->cm_id); | |
335 | /* Generate an CLOSE event */ | |
336 | vq_req->qp = qp; | |
337 | vq_req->cm_id = qp->cm_id; | |
338 | vq_req->event = IW_CM_EVENT_CLOSE; | |
339 | } | |
340 | spin_unlock_irqrestore(&qp->lock, flags); | |
341 | ||
342 | /* | |
343 | * Send WR to adapter | |
344 | */ | |
345 | err = vq_send_wr(c2dev, (union c2wr *) & wr); | |
346 | if (err) { | |
347 | vq_req_put(c2dev, vq_req); | |
348 | goto bail0; | |
349 | } | |
350 | ||
351 | /* | |
352 | * Wait for reply from adapter | |
353 | */ | |
354 | err = vq_wait_for_reply(c2dev, vq_req); | |
355 | if (err) { | |
356 | goto bail0; | |
357 | } | |
358 | ||
359 | /* | |
360 | * Process reply | |
361 | */ | |
362 | reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg); | |
363 | if (!reply) { | |
364 | err = -ENOMEM; | |
365 | goto bail0; | |
366 | } | |
367 | ||
368 | spin_lock_irqsave(&qp->lock, flags); | |
369 | if (qp->cm_id) { | |
370 | qp->cm_id->rem_ref(qp->cm_id); | |
371 | qp->cm_id = NULL; | |
372 | } | |
373 | spin_unlock_irqrestore(&qp->lock, flags); | |
374 | ||
375 | vq_repbuf_free(c2dev, reply); | |
376 | bail0: | |
377 | vq_req_free(c2dev, vq_req); | |
378 | return err; | |
379 | } | |
380 | ||
381 | static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp) | |
382 | { | |
383 | int ret; | |
384 | ||
385 | do { | |
386 | spin_lock_irq(&c2dev->qp_table.lock); | |
387 | ret = idr_get_new_above(&c2dev->qp_table.idr, qp, | |
388 | c2dev->qp_table.last++, &qp->qpn); | |
389 | spin_unlock_irq(&c2dev->qp_table.lock); | |
390 | } while ((ret == -EAGAIN) && | |
391 | idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL)); | |
392 | return ret; | |
393 | } | |
394 | ||
395 | static void c2_free_qpn(struct c2_dev *c2dev, int qpn) | |
396 | { | |
397 | spin_lock_irq(&c2dev->qp_table.lock); | |
398 | idr_remove(&c2dev->qp_table.idr, qpn); | |
399 | spin_unlock_irq(&c2dev->qp_table.lock); | |
400 | } | |
401 | ||
402 | struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn) | |
403 | { | |
404 | unsigned long flags; | |
405 | struct c2_qp *qp; | |
406 | ||
407 | spin_lock_irqsave(&c2dev->qp_table.lock, flags); | |
408 | qp = idr_find(&c2dev->qp_table.idr, qpn); | |
409 | spin_unlock_irqrestore(&c2dev->qp_table.lock, flags); | |
410 | return qp; | |
411 | } | |
412 | ||
413 | int c2_alloc_qp(struct c2_dev *c2dev, | |
414 | struct c2_pd *pd, | |
415 | struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp) | |
416 | { | |
417 | struct c2wr_qp_create_req wr; | |
418 | struct c2wr_qp_create_rep *reply; | |
419 | struct c2_vq_req *vq_req; | |
420 | struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq); | |
421 | struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq); | |
422 | unsigned long peer_pa; | |
423 | u32 q_size, msg_size, mmap_size; | |
424 | void __iomem *mmap; | |
425 | int err; | |
426 | ||
427 | err = c2_alloc_qpn(c2dev, qp); | |
428 | if (err) | |
429 | return err; | |
430 | qp->ibqp.qp_num = qp->qpn; | |
431 | qp->ibqp.qp_type = IB_QPT_RC; | |
432 | ||
433 | /* Allocate the SQ and RQ shared pointers */ | |
434 | qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, | |
435 | &qp->sq_mq.shared_dma, GFP_KERNEL); | |
436 | if (!qp->sq_mq.shared) { | |
437 | err = -ENOMEM; | |
438 | goto bail0; | |
439 | } | |
440 | ||
441 | qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, | |
442 | &qp->rq_mq.shared_dma, GFP_KERNEL); | |
443 | if (!qp->rq_mq.shared) { | |
444 | err = -ENOMEM; | |
445 | goto bail1; | |
446 | } | |
447 | ||
448 | /* Allocate the verbs request */ | |
449 | vq_req = vq_req_alloc(c2dev); | |
450 | if (vq_req == NULL) { | |
451 | err = -ENOMEM; | |
452 | goto bail2; | |
453 | } | |
454 | ||
455 | /* Initialize the work request */ | |
456 | memset(&wr, 0, sizeof(wr)); | |
457 | c2_wr_set_id(&wr, CCWR_QP_CREATE); | |
458 | wr.hdr.context = (unsigned long) vq_req; | |
459 | wr.rnic_handle = c2dev->adapter_handle; | |
460 | wr.sq_cq_handle = send_cq->adapter_handle; | |
461 | wr.rq_cq_handle = recv_cq->adapter_handle; | |
462 | wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1); | |
463 | wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1); | |
464 | wr.srq_handle = 0; | |
465 | wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND | | |
466 | QP_ZERO_STAG | QP_RDMA_READ_RESPONSE); | |
467 | wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge); | |
468 | wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge); | |
469 | wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge); | |
470 | wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma); | |
471 | wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma); | |
472 | wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP); | |
473 | wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP); | |
474 | wr.pd_id = pd->pd_id; | |
475 | wr.user_context = (unsigned long) qp; | |
476 | ||
477 | vq_req_get(c2dev, vq_req); | |
478 | ||
479 | /* Send the WR to the adapter */ | |
480 | err = vq_send_wr(c2dev, (union c2wr *) & wr); | |
481 | if (err) { | |
482 | vq_req_put(c2dev, vq_req); | |
483 | goto bail3; | |
484 | } | |
485 | ||
486 | /* Wait for the verb reply */ | |
487 | err = vq_wait_for_reply(c2dev, vq_req); | |
488 | if (err) { | |
489 | goto bail3; | |
490 | } | |
491 | ||
492 | /* Process the reply */ | |
493 | reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg); | |
494 | if (!reply) { | |
495 | err = -ENOMEM; | |
496 | goto bail3; | |
497 | } | |
498 | ||
499 | if ((err = c2_wr_get_result(reply)) != 0) { | |
500 | goto bail4; | |
501 | } | |
502 | ||
503 | /* Fill in the kernel QP struct */ | |
504 | atomic_set(&qp->refcount, 1); | |
505 | qp->adapter_handle = reply->qp_handle; | |
506 | qp->state = IB_QPS_RESET; | |
507 | qp->send_sgl_depth = qp_attrs->cap.max_send_sge; | |
508 | qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge; | |
509 | qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge; | |
4e8e6ee3 | 510 | init_waitqueue_head(&qp->wait); |
f94b533d TT |
511 | |
512 | /* Initialize the SQ MQ */ | |
513 | q_size = be32_to_cpu(reply->sq_depth); | |
514 | msg_size = be32_to_cpu(reply->sq_msg_size); | |
515 | peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start); | |
516 | mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size); | |
517 | mmap = ioremap_nocache(peer_pa, mmap_size); | |
518 | if (!mmap) { | |
519 | err = -ENOMEM; | |
520 | goto bail5; | |
521 | } | |
522 | ||
523 | c2_mq_req_init(&qp->sq_mq, | |
524 | be32_to_cpu(reply->sq_mq_index), | |
525 | q_size, | |
526 | msg_size, | |
527 | mmap + sizeof(struct c2_mq_shared), /* pool start */ | |
528 | mmap, /* peer */ | |
529 | C2_MQ_ADAPTER_TARGET); | |
530 | ||
531 | /* Initialize the RQ mq */ | |
532 | q_size = be32_to_cpu(reply->rq_depth); | |
533 | msg_size = be32_to_cpu(reply->rq_msg_size); | |
534 | peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start); | |
535 | mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size); | |
536 | mmap = ioremap_nocache(peer_pa, mmap_size); | |
537 | if (!mmap) { | |
538 | err = -ENOMEM; | |
539 | goto bail6; | |
540 | } | |
541 | ||
542 | c2_mq_req_init(&qp->rq_mq, | |
543 | be32_to_cpu(reply->rq_mq_index), | |
544 | q_size, | |
545 | msg_size, | |
546 | mmap + sizeof(struct c2_mq_shared), /* pool start */ | |
547 | mmap, /* peer */ | |
548 | C2_MQ_ADAPTER_TARGET); | |
549 | ||
550 | vq_repbuf_free(c2dev, reply); | |
551 | vq_req_free(c2dev, vq_req); | |
552 | ||
553 | return 0; | |
554 | ||
555 | bail6: | |
556 | iounmap(qp->sq_mq.peer); | |
557 | bail5: | |
558 | destroy_qp(c2dev, qp); | |
559 | bail4: | |
560 | vq_repbuf_free(c2dev, reply); | |
561 | bail3: | |
562 | vq_req_free(c2dev, vq_req); | |
563 | bail2: | |
564 | c2_free_mqsp(qp->rq_mq.shared); | |
565 | bail1: | |
566 | c2_free_mqsp(qp->sq_mq.shared); | |
567 | bail0: | |
568 | c2_free_qpn(c2dev, qp->qpn); | |
569 | return err; | |
570 | } | |
571 | ||
c9edea29 KK |
572 | static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq) |
573 | { | |
574 | if (send_cq == recv_cq) | |
575 | spin_lock_irq(&send_cq->lock); | |
576 | else if (send_cq > recv_cq) { | |
577 | spin_lock_irq(&send_cq->lock); | |
578 | spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); | |
579 | } else { | |
580 | spin_lock_irq(&recv_cq->lock); | |
581 | spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); | |
582 | } | |
583 | } | |
584 | ||
585 | static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq) | |
586 | { | |
587 | if (send_cq == recv_cq) | |
588 | spin_unlock_irq(&send_cq->lock); | |
589 | else if (send_cq > recv_cq) { | |
590 | spin_unlock(&recv_cq->lock); | |
591 | spin_unlock_irq(&send_cq->lock); | |
592 | } else { | |
593 | spin_unlock(&send_cq->lock); | |
594 | spin_unlock_irq(&recv_cq->lock); | |
595 | } | |
596 | } | |
597 | ||
f94b533d TT |
598 | void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp) |
599 | { | |
600 | struct c2_cq *send_cq; | |
601 | struct c2_cq *recv_cq; | |
602 | ||
603 | send_cq = to_c2cq(qp->ibqp.send_cq); | |
604 | recv_cq = to_c2cq(qp->ibqp.recv_cq); | |
605 | ||
606 | /* | |
607 | * Lock CQs here, so that CQ polling code can do QP lookup | |
608 | * without taking a lock. | |
609 | */ | |
c9edea29 | 610 | c2_lock_cqs(send_cq, recv_cq); |
f94b533d | 611 | c2_free_qpn(c2dev, qp->qpn); |
c9edea29 | 612 | c2_unlock_cqs(send_cq, recv_cq); |
f94b533d TT |
613 | |
614 | /* | |
25985edc | 615 | * Destroy qp in the rnic... |
f94b533d TT |
616 | */ |
617 | destroy_qp(c2dev, qp); | |
618 | ||
619 | /* | |
620 | * Mark any unreaped CQEs as null and void. | |
621 | */ | |
622 | c2_cq_clean(c2dev, qp, send_cq->cqn); | |
623 | if (send_cq != recv_cq) | |
624 | c2_cq_clean(c2dev, qp, recv_cq->cqn); | |
625 | /* | |
626 | * Unmap the MQs and return the shared pointers | |
627 | * to the message pool. | |
628 | */ | |
629 | iounmap(qp->sq_mq.peer); | |
630 | iounmap(qp->rq_mq.peer); | |
631 | c2_free_mqsp(qp->sq_mq.shared); | |
632 | c2_free_mqsp(qp->rq_mq.shared); | |
633 | ||
634 | atomic_dec(&qp->refcount); | |
635 | wait_event(qp->wait, !atomic_read(&qp->refcount)); | |
636 | } | |
637 | ||
638 | /* | |
639 | * Function: move_sgl | |
640 | * | |
641 | * Description: | |
642 | * Move an SGL from the user's work request struct into a CCIL Work Request | |
643 | * message, swapping to WR byte order and ensure the total length doesn't | |
644 | * overflow. | |
645 | * | |
646 | * IN: | |
647 | * dst - ptr to CCIL Work Request message SGL memory. | |
648 | * src - ptr to the consumers SGL memory. | |
649 | * | |
650 | * OUT: none | |
651 | * | |
652 | * Return: | |
653 | * CCIL status codes. | |
654 | */ | |
655 | static int | |
656 | move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len, | |
657 | u8 * actual_count) | |
658 | { | |
659 | u32 tot = 0; /* running total */ | |
660 | u8 acount = 0; /* running total non-0 len sge's */ | |
661 | ||
662 | while (count > 0) { | |
663 | /* | |
664 | * If the addition of this SGE causes the | |
665 | * total SGL length to exceed 2^32-1, then | |
666 | * fail-n-bail. | |
667 | * | |
668 | * If the current total plus the next element length | |
669 | * wraps, then it will go negative and be less than the | |
670 | * current total... | |
671 | */ | |
672 | if ((tot + src->length) < tot) { | |
673 | return -EINVAL; | |
674 | } | |
675 | /* | |
676 | * Bug: 1456 (as well as 1498 & 1643) | |
677 | * Skip over any sge's supplied with len=0 | |
678 | */ | |
679 | if (src->length) { | |
680 | tot += src->length; | |
681 | dst->stag = cpu_to_be32(src->lkey); | |
682 | dst->to = cpu_to_be64(src->addr); | |
683 | dst->length = cpu_to_be32(src->length); | |
684 | dst++; | |
685 | acount++; | |
686 | } | |
687 | src++; | |
688 | count--; | |
689 | } | |
690 | ||
691 | if (acount == 0) { | |
692 | /* | |
693 | * Bug: 1476 (as well as 1498, 1456 and 1643) | |
694 | * Setup the SGL in the WR to make it easier for the RNIC. | |
695 | * This way, the FW doesn't have to deal with special cases. | |
696 | * Setting length=0 should be sufficient. | |
697 | */ | |
698 | dst->stag = 0; | |
699 | dst->to = 0; | |
700 | dst->length = 0; | |
701 | } | |
702 | ||
703 | *p_len = tot; | |
704 | *actual_count = acount; | |
705 | return 0; | |
706 | } | |
707 | ||
708 | /* | |
709 | * Function: c2_activity (private function) | |
710 | * | |
711 | * Description: | |
712 | * Post an mq index to the host->adapter activity fifo. | |
713 | * | |
714 | * IN: | |
715 | * c2dev - ptr to c2dev structure | |
716 | * mq_index - mq index to post | |
717 | * shared - value most recently written to shared | |
718 | * | |
719 | * OUT: | |
720 | * | |
721 | * Return: | |
722 | * none | |
723 | */ | |
724 | static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared) | |
725 | { | |
726 | /* | |
727 | * First read the register to see if the FIFO is full, and if so, | |
728 | * spin until it's not. This isn't perfect -- there is no | |
729 | * synchronization among the clients of the register, but in | |
730 | * practice it prevents multiple CPU from hammering the bus | |
731 | * with PCI RETRY. Note that when this does happen, the card | |
732 | * cannot get on the bus and the card and system hang in a | |
733 | * deadlock -- thus the need for this code. [TOT] | |
734 | */ | |
e52e6080 TT |
735 | while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000) |
736 | udelay(10); | |
f94b533d TT |
737 | |
738 | __raw_writel(C2_HINT_MAKE(mq_index, shared), | |
739 | c2dev->regs + PCI_BAR0_ADAPTER_HINT); | |
740 | } | |
741 | ||
742 | /* | |
743 | * Function: qp_wr_post | |
744 | * | |
745 | * Description: | |
746 | * This in-line function allocates a MQ msg, then moves the host-copy of | |
747 | * the completed WR into msg. Then it posts the message. | |
748 | * | |
749 | * IN: | |
750 | * q - ptr to user MQ. | |
751 | * wr - ptr to host-copy of the WR. | |
752 | * qp - ptr to user qp | |
753 | * size - Number of bytes to post. Assumed to be divisible by 4. | |
754 | * | |
755 | * OUT: none | |
756 | * | |
757 | * Return: | |
758 | * CCIL status codes. | |
759 | */ | |
760 | static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size) | |
761 | { | |
762 | union c2wr *msg; | |
763 | ||
764 | msg = c2_mq_alloc(q); | |
765 | if (msg == NULL) { | |
766 | return -EINVAL; | |
767 | } | |
768 | #ifdef CCMSGMAGIC | |
769 | ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC); | |
770 | #endif | |
771 | ||
772 | /* | |
773 | * Since all header fields in the WR are the same as the | |
774 | * CQE, set the following so the adapter need not. | |
775 | */ | |
776 | c2_wr_set_result(wr, CCERR_PENDING); | |
777 | ||
778 | /* | |
779 | * Copy the wr down to the adapter | |
780 | */ | |
781 | memcpy((void *) msg, (void *) wr, size); | |
782 | ||
783 | c2_mq_produce(q); | |
784 | return 0; | |
785 | } | |
786 | ||
787 | ||
788 | int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr, | |
789 | struct ib_send_wr **bad_wr) | |
790 | { | |
791 | struct c2_dev *c2dev = to_c2dev(ibqp->device); | |
792 | struct c2_qp *qp = to_c2qp(ibqp); | |
793 | union c2wr wr; | |
e52e6080 | 794 | unsigned long lock_flags; |
f94b533d TT |
795 | int err = 0; |
796 | ||
797 | u32 flags; | |
798 | u32 tot_len; | |
799 | u8 actual_sge_count; | |
800 | u32 msg_size; | |
801 | ||
c597b024 FZ |
802 | if (qp->state > IB_QPS_RTS) { |
803 | err = -EINVAL; | |
804 | goto out; | |
805 | } | |
f94b533d TT |
806 | |
807 | while (ib_wr) { | |
808 | ||
809 | flags = 0; | |
810 | wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id; | |
811 | if (ib_wr->send_flags & IB_SEND_SIGNALED) { | |
812 | flags |= SQ_SIGNALED; | |
813 | } | |
814 | ||
815 | switch (ib_wr->opcode) { | |
816 | case IB_WR_SEND: | |
139b2db7 RD |
817 | case IB_WR_SEND_WITH_INV: |
818 | if (ib_wr->opcode == IB_WR_SEND) { | |
819 | if (ib_wr->send_flags & IB_SEND_SOLICITED) | |
820 | c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE); | |
821 | else | |
822 | c2_wr_set_id(&wr, C2_WR_TYPE_SEND); | |
823 | wr.sqwr.send.remote_stag = 0; | |
f94b533d | 824 | } else { |
139b2db7 RD |
825 | if (ib_wr->send_flags & IB_SEND_SOLICITED) |
826 | c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV); | |
827 | else | |
828 | c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV); | |
829 | wr.sqwr.send.remote_stag = | |
830 | cpu_to_be32(ib_wr->ex.invalidate_rkey); | |
f94b533d TT |
831 | } |
832 | ||
139b2db7 RD |
833 | msg_size = sizeof(struct c2wr_send_req) + |
834 | sizeof(struct c2_data_addr) * ib_wr->num_sge; | |
f94b533d TT |
835 | if (ib_wr->num_sge > qp->send_sgl_depth) { |
836 | err = -EINVAL; | |
837 | break; | |
838 | } | |
839 | if (ib_wr->send_flags & IB_SEND_FENCE) { | |
840 | flags |= SQ_READ_FENCE; | |
841 | } | |
842 | err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data), | |
843 | ib_wr->sg_list, | |
844 | ib_wr->num_sge, | |
845 | &tot_len, &actual_sge_count); | |
846 | wr.sqwr.send.sge_len = cpu_to_be32(tot_len); | |
847 | c2_wr_set_sge_count(&wr, actual_sge_count); | |
848 | break; | |
849 | case IB_WR_RDMA_WRITE: | |
850 | c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE); | |
851 | msg_size = sizeof(struct c2wr_rdma_write_req) + | |
852 | (sizeof(struct c2_data_addr) * ib_wr->num_sge); | |
853 | if (ib_wr->num_sge > qp->rdma_write_sgl_depth) { | |
854 | err = -EINVAL; | |
855 | break; | |
856 | } | |
857 | if (ib_wr->send_flags & IB_SEND_FENCE) { | |
858 | flags |= SQ_READ_FENCE; | |
859 | } | |
860 | wr.sqwr.rdma_write.remote_stag = | |
861 | cpu_to_be32(ib_wr->wr.rdma.rkey); | |
862 | wr.sqwr.rdma_write.remote_to = | |
863 | cpu_to_be64(ib_wr->wr.rdma.remote_addr); | |
864 | err = move_sgl((struct c2_data_addr *) | |
865 | & (wr.sqwr.rdma_write.data), | |
866 | ib_wr->sg_list, | |
867 | ib_wr->num_sge, | |
868 | &tot_len, &actual_sge_count); | |
869 | wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len); | |
870 | c2_wr_set_sge_count(&wr, actual_sge_count); | |
871 | break; | |
872 | case IB_WR_RDMA_READ: | |
873 | c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ); | |
874 | msg_size = sizeof(struct c2wr_rdma_read_req); | |
875 | ||
876 | /* IWarp only suppots 1 sge for RDMA reads */ | |
877 | if (ib_wr->num_sge > 1) { | |
878 | err = -EINVAL; | |
879 | break; | |
880 | } | |
881 | ||
882 | /* | |
883 | * Move the local and remote stag/to/len into the WR. | |
884 | */ | |
885 | wr.sqwr.rdma_read.local_stag = | |
886 | cpu_to_be32(ib_wr->sg_list->lkey); | |
887 | wr.sqwr.rdma_read.local_to = | |
888 | cpu_to_be64(ib_wr->sg_list->addr); | |
889 | wr.sqwr.rdma_read.remote_stag = | |
890 | cpu_to_be32(ib_wr->wr.rdma.rkey); | |
891 | wr.sqwr.rdma_read.remote_to = | |
892 | cpu_to_be64(ib_wr->wr.rdma.remote_addr); | |
893 | wr.sqwr.rdma_read.length = | |
894 | cpu_to_be32(ib_wr->sg_list->length); | |
895 | break; | |
896 | default: | |
897 | /* error */ | |
898 | msg_size = 0; | |
899 | err = -EINVAL; | |
900 | break; | |
901 | } | |
902 | ||
903 | /* | |
904 | * If we had an error on the last wr build, then | |
905 | * break out. Possible errors include bogus WR | |
906 | * type, and a bogus SGL length... | |
907 | */ | |
908 | if (err) { | |
909 | break; | |
910 | } | |
911 | ||
912 | /* | |
913 | * Store flags | |
914 | */ | |
915 | c2_wr_set_flags(&wr, flags); | |
916 | ||
917 | /* | |
918 | * Post the puppy! | |
919 | */ | |
e52e6080 | 920 | spin_lock_irqsave(&qp->lock, lock_flags); |
f94b533d TT |
921 | err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size); |
922 | if (err) { | |
e52e6080 | 923 | spin_unlock_irqrestore(&qp->lock, lock_flags); |
f94b533d TT |
924 | break; |
925 | } | |
926 | ||
927 | /* | |
928 | * Enqueue mq index to activity FIFO. | |
929 | */ | |
930 | c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count); | |
e52e6080 | 931 | spin_unlock_irqrestore(&qp->lock, lock_flags); |
f94b533d TT |
932 | |
933 | ib_wr = ib_wr->next; | |
934 | } | |
935 | ||
c597b024 | 936 | out: |
f94b533d TT |
937 | if (err) |
938 | *bad_wr = ib_wr; | |
939 | return err; | |
940 | } | |
941 | ||
942 | int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr, | |
943 | struct ib_recv_wr **bad_wr) | |
944 | { | |
945 | struct c2_dev *c2dev = to_c2dev(ibqp->device); | |
946 | struct c2_qp *qp = to_c2qp(ibqp); | |
947 | union c2wr wr; | |
e52e6080 | 948 | unsigned long lock_flags; |
f94b533d TT |
949 | int err = 0; |
950 | ||
c597b024 FZ |
951 | if (qp->state > IB_QPS_RTS) { |
952 | err = -EINVAL; | |
953 | goto out; | |
954 | } | |
f94b533d TT |
955 | |
956 | /* | |
957 | * Try and post each work request | |
958 | */ | |
959 | while (ib_wr) { | |
960 | u32 tot_len; | |
961 | u8 actual_sge_count; | |
962 | ||
963 | if (ib_wr->num_sge > qp->recv_sgl_depth) { | |
964 | err = -EINVAL; | |
965 | break; | |
966 | } | |
967 | ||
968 | /* | |
969 | * Create local host-copy of the WR | |
970 | */ | |
971 | wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id; | |
972 | c2_wr_set_id(&wr, CCWR_RECV); | |
973 | c2_wr_set_flags(&wr, 0); | |
974 | ||
975 | /* sge_count is limited to eight bits. */ | |
976 | BUG_ON(ib_wr->num_sge >= 256); | |
977 | err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data), | |
978 | ib_wr->sg_list, | |
979 | ib_wr->num_sge, &tot_len, &actual_sge_count); | |
980 | c2_wr_set_sge_count(&wr, actual_sge_count); | |
981 | ||
982 | /* | |
983 | * If we had an error on the last wr build, then | |
984 | * break out. Possible errors include bogus WR | |
985 | * type, and a bogus SGL length... | |
986 | */ | |
987 | if (err) { | |
988 | break; | |
989 | } | |
990 | ||
e52e6080 | 991 | spin_lock_irqsave(&qp->lock, lock_flags); |
f94b533d TT |
992 | err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size); |
993 | if (err) { | |
e52e6080 | 994 | spin_unlock_irqrestore(&qp->lock, lock_flags); |
f94b533d TT |
995 | break; |
996 | } | |
997 | ||
998 | /* | |
999 | * Enqueue mq index to activity FIFO | |
1000 | */ | |
1001 | c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count); | |
e52e6080 | 1002 | spin_unlock_irqrestore(&qp->lock, lock_flags); |
f94b533d TT |
1003 | |
1004 | ib_wr = ib_wr->next; | |
1005 | } | |
1006 | ||
c597b024 | 1007 | out: |
f94b533d TT |
1008 | if (err) |
1009 | *bad_wr = ib_wr; | |
1010 | return err; | |
1011 | } | |
1012 | ||
1013 | void __devinit c2_init_qp_table(struct c2_dev *c2dev) | |
1014 | { | |
1015 | spin_lock_init(&c2dev->qp_table.lock); | |
1016 | idr_init(&c2dev->qp_table.idr); | |
1017 | } | |
1018 | ||
1019 | void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev) | |
1020 | { | |
1021 | idr_destroy(&c2dev->qp_table.idr); | |
1022 | } |