]>
Commit | Line | Data |
---|---|---|
fab97220 HS |
1 | /* |
2 | * IBM eServer eHCA Infiniband device driver for Linux on POWER | |
3 | * | |
4 | * Completion queue handling | |
5 | * | |
6 | * Authors: Waleri Fomin <fomin@de.ibm.com> | |
7 | * Khadija Souissi <souissi@de.ibm.com> | |
8 | * Reinhard Ernst <rernst@de.ibm.com> | |
9 | * Heiko J Schick <schickhj@de.ibm.com> | |
10 | * Hoang-Nam Nguyen <hnguyen@de.ibm.com> | |
11 | * | |
12 | * | |
13 | * Copyright (c) 2005 IBM Corporation | |
14 | * | |
15 | * All rights reserved. | |
16 | * | |
17 | * This source code is distributed under a dual license of GPL v2.0 and OpenIB | |
18 | * BSD. | |
19 | * | |
20 | * OpenIB BSD License | |
21 | * | |
22 | * Redistribution and use in source and binary forms, with or without | |
23 | * modification, are permitted provided that the following conditions are met: | |
24 | * | |
25 | * Redistributions of source code must retain the above copyright notice, this | |
26 | * list of conditions and the following disclaimer. | |
27 | * | |
28 | * Redistributions in binary form must reproduce the above copyright notice, | |
29 | * this list of conditions and the following disclaimer in the documentation | |
30 | * and/or other materials | |
31 | * provided with the distribution. | |
32 | * | |
33 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |
34 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
35 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
36 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | |
37 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
38 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
39 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
40 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER | |
41 | * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
42 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | |
43 | * POSSIBILITY OF SUCH DAMAGE. | |
44 | */ | |
45 | ||
5a0e3ad6 TH |
46 | #include <linux/slab.h> |
47 | ||
fab97220 HS |
48 | #include "ehca_iverbs.h" |
49 | #include "ehca_classes.h" | |
50 | #include "ehca_irq.h" | |
51 | #include "hcp_if.h" | |
52 | ||
53 | static struct kmem_cache *cq_cache; | |
54 | ||
55 | int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp) | |
56 | { | |
57 | unsigned int qp_num = qp->real_qp_num; | |
58 | unsigned int key = qp_num & (QP_HASHTAB_LEN-1); | |
9844b71b | 59 | unsigned long flags; |
fab97220 | 60 | |
9844b71b | 61 | spin_lock_irqsave(&cq->spinlock, flags); |
fab97220 | 62 | hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]); |
9844b71b | 63 | spin_unlock_irqrestore(&cq->spinlock, flags); |
fab97220 HS |
64 | |
65 | ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x", | |
66 | cq->cq_number, qp_num); | |
67 | ||
68 | return 0; | |
69 | } | |
70 | ||
71 | int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num) | |
72 | { | |
73 | int ret = -EINVAL; | |
74 | unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1); | |
75 | struct hlist_node *iter; | |
76 | struct ehca_qp *qp; | |
9844b71b | 77 | unsigned long flags; |
fab97220 | 78 | |
9844b71b | 79 | spin_lock_irqsave(&cq->spinlock, flags); |
fab97220 HS |
80 | hlist_for_each(iter, &cq->qp_hashtab[key]) { |
81 | qp = hlist_entry(iter, struct ehca_qp, list_entries); | |
82 | if (qp->real_qp_num == real_qp_num) { | |
83 | hlist_del(iter); | |
84 | ehca_dbg(cq->ib_cq.device, | |
85 | "removed qp from cq .cq_num=%x real_qp_num=%x", | |
86 | cq->cq_number, real_qp_num); | |
87 | ret = 0; | |
88 | break; | |
89 | } | |
90 | } | |
9844b71b | 91 | spin_unlock_irqrestore(&cq->spinlock, flags); |
fab97220 HS |
92 | if (ret) |
93 | ehca_err(cq->ib_cq.device, | |
94 | "qp not found cq_num=%x real_qp_num=%x", | |
95 | cq->cq_number, real_qp_num); | |
96 | ||
97 | return ret; | |
98 | } | |
99 | ||
2b94397a | 100 | struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num) |
fab97220 HS |
101 | { |
102 | struct ehca_qp *ret = NULL; | |
103 | unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1); | |
104 | struct hlist_node *iter; | |
105 | struct ehca_qp *qp; | |
106 | hlist_for_each(iter, &cq->qp_hashtab[key]) { | |
107 | qp = hlist_entry(iter, struct ehca_qp, list_entries); | |
108 | if (qp->real_qp_num == real_qp_num) { | |
109 | ret = qp; | |
110 | break; | |
111 | } | |
112 | } | |
113 | return ret; | |
114 | } | |
115 | ||
f4fd0b22 | 116 | struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector, |
fab97220 HS |
117 | struct ib_ucontext *context, |
118 | struct ib_udata *udata) | |
119 | { | |
120 | static const u32 additional_cqe = 20; | |
121 | struct ib_cq *cq; | |
122 | struct ehca_cq *my_cq; | |
123 | struct ehca_shca *shca = | |
124 | container_of(device, struct ehca_shca, ib_device); | |
125 | struct ipz_adapter_handle adapter_handle; | |
126 | struct ehca_alloc_cq_parms param; /* h_call's out parameters */ | |
127 | struct h_galpa gal; | |
128 | void *vpage; | |
129 | u32 counter; | |
130 | u64 rpage, cqx_fec, h_ret; | |
131 | int ipz_rc, ret, i; | |
132 | unsigned long flags; | |
133 | ||
134 | if (cqe >= 0xFFFFFFFF - 64 - additional_cqe) | |
135 | return ERR_PTR(-EINVAL); | |
136 | ||
19f42821 | 137 | if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) { |
d227fa72 | 138 | ehca_err(device, "Unable to create CQ, max number of %i " |
19f42821 | 139 | "CQs reached.", shca->max_num_cqs); |
d227fa72 SR |
140 | ehca_err(device, "To increase the maximum number of CQs " |
141 | "use the number_of_cqs module parameter.\n"); | |
142 | return ERR_PTR(-ENOSPC); | |
143 | } | |
144 | ||
c3762229 | 145 | my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL); |
fab97220 HS |
146 | if (!my_cq) { |
147 | ehca_err(device, "Out of memory for ehca_cq struct device=%p", | |
148 | device); | |
d227fa72 | 149 | atomic_dec(&shca->num_cqs); |
fab97220 HS |
150 | return ERR_PTR(-ENOMEM); |
151 | } | |
152 | ||
fab97220 HS |
153 | memset(¶m, 0, sizeof(struct ehca_alloc_cq_parms)); |
154 | ||
155 | spin_lock_init(&my_cq->spinlock); | |
156 | spin_lock_init(&my_cq->cb_lock); | |
157 | spin_lock_init(&my_cq->task_lock); | |
28db6beb | 158 | atomic_set(&my_cq->nr_events, 0); |
31726798 | 159 | init_waitqueue_head(&my_cq->wait_completion); |
fab97220 HS |
160 | |
161 | cq = &my_cq->ib_cq; | |
162 | ||
163 | adapter_handle = shca->ipz_hca_handle; | |
164 | param.eq_handle = shca->eq.ipz_eq_handle; | |
165 | ||
166 | do { | |
167 | if (!idr_pre_get(&ehca_cq_idr, GFP_KERNEL)) { | |
168 | cq = ERR_PTR(-ENOMEM); | |
169 | ehca_err(device, "Can't reserve idr nr. device=%p", | |
170 | device); | |
171 | goto create_cq_exit1; | |
172 | } | |
173 | ||
26ed687f | 174 | write_lock_irqsave(&ehca_cq_idr_lock, flags); |
fab97220 | 175 | ret = idr_get_new(&ehca_cq_idr, my_cq, &my_cq->token); |
26ed687f | 176 | write_unlock_irqrestore(&ehca_cq_idr_lock, flags); |
fab97220 HS |
177 | } while (ret == -EAGAIN); |
178 | ||
179 | if (ret) { | |
180 | cq = ERR_PTR(-ENOMEM); | |
181 | ehca_err(device, "Can't allocate new idr entry. device=%p", | |
182 | device); | |
183 | goto create_cq_exit1; | |
184 | } | |
185 | ||
5281a4b8 SR |
186 | if (my_cq->token > 0x1FFFFFF) { |
187 | cq = ERR_PTR(-ENOMEM); | |
188 | ehca_err(device, "Invalid number of cq. device=%p", device); | |
189 | goto create_cq_exit2; | |
190 | } | |
191 | ||
fab97220 HS |
192 | /* |
193 | * CQs maximum depth is 4GB-64, but we need additional 20 as buffer | |
194 | * for receiving errors CQEs. | |
195 | */ | |
196 | param.nr_cqe = cqe + additional_cqe; | |
197 | h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, ¶m); | |
198 | ||
199 | if (h_ret != H_SUCCESS) { | |
200 | ehca_err(device, "hipz_h_alloc_resource_cq() failed " | |
3750f605 | 201 | "h_ret=%lli device=%p", h_ret, device); |
fab97220 HS |
202 | cq = ERR_PTR(ehca2ib_return_code(h_ret)); |
203 | goto create_cq_exit2; | |
204 | } | |
205 | ||
e2f81daf SR |
206 | ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages, |
207 | EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0); | |
fab97220 | 208 | if (!ipz_rc) { |
e3722192 | 209 | ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p", |
fab97220 HS |
210 | ipz_rc, device); |
211 | cq = ERR_PTR(-EINVAL); | |
212 | goto create_cq_exit3; | |
213 | } | |
214 | ||
215 | for (counter = 0; counter < param.act_pages; counter++) { | |
216 | vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue); | |
217 | if (!vpage) { | |
218 | ehca_err(device, "ipz_qpageit_get_inc() " | |
219 | "returns NULL device=%p", device); | |
220 | cq = ERR_PTR(-EAGAIN); | |
221 | goto create_cq_exit4; | |
222 | } | |
223 | rpage = virt_to_abs(vpage); | |
224 | ||
225 | h_ret = hipz_h_register_rpage_cq(adapter_handle, | |
226 | my_cq->ipz_cq_handle, | |
227 | &my_cq->pf, | |
228 | 0, | |
229 | 0, | |
230 | rpage, | |
231 | 1, | |
232 | my_cq->galpas. | |
233 | kernel); | |
234 | ||
235 | if (h_ret < H_SUCCESS) { | |
236 | ehca_err(device, "hipz_h_register_rpage_cq() failed " | |
3750f605 | 237 | "ehca_cq=%p cq_num=%x h_ret=%lli counter=%i " |
fab97220 HS |
238 | "act_pages=%i", my_cq, my_cq->cq_number, |
239 | h_ret, counter, param.act_pages); | |
240 | cq = ERR_PTR(-EINVAL); | |
241 | goto create_cq_exit4; | |
242 | } | |
243 | ||
244 | if (counter == (param.act_pages - 1)) { | |
245 | vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue); | |
246 | if ((h_ret != H_SUCCESS) || vpage) { | |
247 | ehca_err(device, "Registration of pages not " | |
248 | "complete ehca_cq=%p cq_num=%x " | |
3750f605 | 249 | "h_ret=%lli", my_cq, my_cq->cq_number, |
fab97220 HS |
250 | h_ret); |
251 | cq = ERR_PTR(-EAGAIN); | |
252 | goto create_cq_exit4; | |
253 | } | |
254 | } else { | |
255 | if (h_ret != H_PAGE_REGISTERED) { | |
256 | ehca_err(device, "Registration of page failed " | |
3750f605 | 257 | "ehca_cq=%p cq_num=%x h_ret=%lli " |
fab97220 HS |
258 | "counter=%i act_pages=%i", |
259 | my_cq, my_cq->cq_number, | |
260 | h_ret, counter, param.act_pages); | |
261 | cq = ERR_PTR(-ENOMEM); | |
262 | goto create_cq_exit4; | |
263 | } | |
264 | } | |
265 | } | |
266 | ||
267 | ipz_qeit_reset(&my_cq->ipz_queue); | |
268 | ||
269 | gal = my_cq->galpas.kernel; | |
270 | cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec)); | |
3750f605 | 271 | ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%llx", |
fab97220 HS |
272 | my_cq, my_cq->cq_number, cqx_fec); |
273 | ||
274 | my_cq->ib_cq.cqe = my_cq->nr_of_entries = | |
275 | param.act_nr_of_entries - additional_cqe; | |
276 | my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff; | |
277 | ||
278 | for (i = 0; i < QP_HASHTAB_LEN; i++) | |
279 | INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]); | |
280 | ||
b9012e0a AS |
281 | INIT_LIST_HEAD(&my_cq->sqp_err_list); |
282 | INIT_LIST_HEAD(&my_cq->rqp_err_list); | |
283 | ||
fab97220 HS |
284 | if (context) { |
285 | struct ipz_queue *ipz_queue = &my_cq->ipz_queue; | |
286 | struct ehca_create_cq_resp resp; | |
fab97220 HS |
287 | memset(&resp, 0, sizeof(resp)); |
288 | resp.cq_number = my_cq->cq_number; | |
289 | resp.token = my_cq->token; | |
290 | resp.ipz_queue.qe_size = ipz_queue->qe_size; | |
291 | resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg; | |
292 | resp.ipz_queue.queue_length = ipz_queue->queue_length; | |
293 | resp.ipz_queue.pagesize = ipz_queue->pagesize; | |
294 | resp.ipz_queue.toggle_state = ipz_queue->toggle_state; | |
e390d3b5 HNN |
295 | resp.fw_handle_ofs = (u32) |
296 | (my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1)); | |
fab97220 HS |
297 | if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { |
298 | ehca_err(device, "Copy to udata failed."); | |
4c34bdf5 | 299 | goto create_cq_exit4; |
fab97220 HS |
300 | } |
301 | } | |
302 | ||
303 | return cq; | |
304 | ||
fab97220 | 305 | create_cq_exit4: |
e2f81daf | 306 | ipz_queue_dtor(NULL, &my_cq->ipz_queue); |
fab97220 HS |
307 | |
308 | create_cq_exit3: | |
309 | h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); | |
310 | if (h_ret != H_SUCCESS) | |
311 | ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p " | |
3750f605 | 312 | "cq_num=%x h_ret=%lli", my_cq, my_cq->cq_number, h_ret); |
fab97220 HS |
313 | |
314 | create_cq_exit2: | |
26ed687f | 315 | write_lock_irqsave(&ehca_cq_idr_lock, flags); |
fab97220 | 316 | idr_remove(&ehca_cq_idr, my_cq->token); |
26ed687f | 317 | write_unlock_irqrestore(&ehca_cq_idr_lock, flags); |
fab97220 HS |
318 | |
319 | create_cq_exit1: | |
320 | kmem_cache_free(cq_cache, my_cq); | |
321 | ||
d227fa72 | 322 | atomic_dec(&shca->num_cqs); |
fab97220 HS |
323 | return cq; |
324 | } | |
325 | ||
326 | int ehca_destroy_cq(struct ib_cq *cq) | |
327 | { | |
328 | u64 h_ret; | |
fab97220 HS |
329 | struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq); |
330 | int cq_num = my_cq->cq_number; | |
331 | struct ib_device *device = cq->device; | |
332 | struct ehca_shca *shca = container_of(device, struct ehca_shca, | |
333 | ib_device); | |
334 | struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle; | |
fab97220 HS |
335 | unsigned long flags; |
336 | ||
4c34bdf5 HNN |
337 | if (cq->uobject) { |
338 | if (my_cq->mm_count_galpa || my_cq->mm_count_queue) { | |
339 | ehca_err(device, "Resources still referenced in " | |
340 | "user space cq_num=%x", my_cq->cq_number); | |
341 | return -EINVAL; | |
342 | } | |
4c34bdf5 HNN |
343 | } |
344 | ||
28db6beb JF |
345 | /* |
346 | * remove the CQ from the idr first to make sure | |
347 | * no more interrupt tasklets will touch this CQ | |
348 | */ | |
26ed687f | 349 | write_lock_irqsave(&ehca_cq_idr_lock, flags); |
fab97220 | 350 | idr_remove(&ehca_cq_idr, my_cq->token); |
26ed687f | 351 | write_unlock_irqrestore(&ehca_cq_idr_lock, flags); |
fab97220 | 352 | |
28db6beb JF |
353 | /* now wait until all pending events have completed */ |
354 | wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events)); | |
355 | ||
356 | /* nobody's using our CQ any longer -- we can destroy it */ | |
fab97220 HS |
357 | h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0); |
358 | if (h_ret == H_R_STATE) { | |
359 | /* cq in err: read err data and destroy it forcibly */ | |
3750f605 | 360 | ehca_dbg(device, "ehca_cq=%p cq_num=%x resource=%llx in err " |
fab97220 HS |
361 | "state. Try to delete it forcibly.", |
362 | my_cq, cq_num, my_cq->ipz_cq_handle.handle); | |
363 | ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle); | |
364 | h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); | |
365 | if (h_ret == H_SUCCESS) | |
366 | ehca_dbg(device, "cq_num=%x deleted successfully.", | |
367 | cq_num); | |
368 | } | |
369 | if (h_ret != H_SUCCESS) { | |
3750f605 | 370 | ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lli " |
fab97220 HS |
371 | "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num); |
372 | return ehca2ib_return_code(h_ret); | |
373 | } | |
e2f81daf | 374 | ipz_queue_dtor(NULL, &my_cq->ipz_queue); |
fab97220 HS |
375 | kmem_cache_free(cq_cache, my_cq); |
376 | ||
d227fa72 | 377 | atomic_dec(&shca->num_cqs); |
fab97220 HS |
378 | return 0; |
379 | } | |
380 | ||
381 | int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata) | |
382 | { | |
fab97220 HS |
383 | /* TODO: proper resize needs to be done */ |
384 | ehca_err(cq->device, "not implemented yet"); | |
385 | ||
386 | return -EFAULT; | |
387 | } | |
388 | ||
389 | int ehca_init_cq_cache(void) | |
390 | { | |
391 | cq_cache = kmem_cache_create("ehca_cache_cq", | |
392 | sizeof(struct ehca_cq), 0, | |
393 | SLAB_HWCACHE_ALIGN, | |
20c2df83 | 394 | NULL); |
fab97220 HS |
395 | if (!cq_cache) |
396 | return -ENOMEM; | |
397 | return 0; | |
398 | } | |
399 | ||
400 | void ehca_cleanup_cq_cache(void) | |
401 | { | |
402 | if (cq_cache) | |
403 | kmem_cache_destroy(cq_cache); | |
404 | } |