]>
Commit | Line | Data |
---|---|---|
1ac5a404 SX |
1 | /* |
2 | * Broadcom NetXtreme-E RoCE driver. | |
3 | * | |
4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
6 | * | |
7 | * This software is available to you under a choice of one of two | |
8 | * licenses. You may choose to be licensed under the terms of the GNU | |
9 | * General Public License (GPL) Version 2, available from the file | |
10 | * COPYING in the main directory of this source tree, or the | |
11 | * BSD license below: | |
12 | * | |
13 | * Redistribution and use in source and binary forms, with or without | |
14 | * modification, are permitted provided that the following conditions | |
15 | * are met: | |
16 | * | |
17 | * 1. Redistributions of source code must retain the above copyright | |
18 | * notice, this list of conditions and the following disclaimer. | |
19 | * 2. Redistributions in binary form must reproduce the above copyright | |
20 | * notice, this list of conditions and the following disclaimer in | |
21 | * the documentation and/or other materials provided with the | |
22 | * distribution. | |
23 | * | |
24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
35 | * | |
36 | * Description: Fast Path Operators | |
37 | */ | |
38 | ||
39 | #include <linux/interrupt.h> | |
40 | #include <linux/spinlock.h> | |
41 | #include <linux/sched.h> | |
42 | #include <linux/slab.h> | |
43 | #include <linux/pci.h> | |
44 | #include <linux/prefetch.h> | |
45 | ||
46 | #include "roce_hsi.h" | |
47 | ||
48 | #include "qplib_res.h" | |
49 | #include "qplib_rcfw.h" | |
50 | #include "qplib_sp.h" | |
51 | #include "qplib_fp.h" | |
52 | ||
53 | static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq); | |
54 | ||
55 | static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res, | |
56 | struct bnxt_qplib_qp *qp) | |
57 | { | |
58 | struct bnxt_qplib_q *rq = &qp->rq; | |
59 | struct bnxt_qplib_q *sq = &qp->sq; | |
60 | ||
61 | if (qp->rq_hdr_buf) | |
62 | dma_free_coherent(&res->pdev->dev, | |
63 | rq->hwq.max_elements * qp->rq_hdr_buf_size, | |
64 | qp->rq_hdr_buf, qp->rq_hdr_buf_map); | |
65 | if (qp->sq_hdr_buf) | |
66 | dma_free_coherent(&res->pdev->dev, | |
67 | sq->hwq.max_elements * qp->sq_hdr_buf_size, | |
68 | qp->sq_hdr_buf, qp->sq_hdr_buf_map); | |
69 | qp->rq_hdr_buf = NULL; | |
70 | qp->sq_hdr_buf = NULL; | |
71 | qp->rq_hdr_buf_map = 0; | |
72 | qp->sq_hdr_buf_map = 0; | |
73 | qp->sq_hdr_buf_size = 0; | |
74 | qp->rq_hdr_buf_size = 0; | |
75 | } | |
76 | ||
77 | static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, | |
78 | struct bnxt_qplib_qp *qp) | |
79 | { | |
80 | struct bnxt_qplib_q *rq = &qp->rq; | |
81 | struct bnxt_qplib_q *sq = &qp->rq; | |
82 | int rc = 0; | |
83 | ||
84 | if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { | |
85 | qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, | |
86 | sq->hwq.max_elements * | |
87 | qp->sq_hdr_buf_size, | |
88 | &qp->sq_hdr_buf_map, GFP_KERNEL); | |
89 | if (!qp->sq_hdr_buf) { | |
90 | rc = -ENOMEM; | |
91 | dev_err(&res->pdev->dev, | |
92 | "QPLIB: Failed to create sq_hdr_buf"); | |
93 | goto fail; | |
94 | } | |
95 | } | |
96 | ||
97 | if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { | |
98 | qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, | |
99 | rq->hwq.max_elements * | |
100 | qp->rq_hdr_buf_size, | |
101 | &qp->rq_hdr_buf_map, | |
102 | GFP_KERNEL); | |
103 | if (!qp->rq_hdr_buf) { | |
104 | rc = -ENOMEM; | |
105 | dev_err(&res->pdev->dev, | |
106 | "QPLIB: Failed to create rq_hdr_buf"); | |
107 | goto fail; | |
108 | } | |
109 | } | |
110 | return 0; | |
111 | ||
112 | fail: | |
113 | bnxt_qplib_free_qp_hdr_buf(res, qp); | |
114 | return rc; | |
115 | } | |
116 | ||
117 | static void bnxt_qplib_service_nq(unsigned long data) | |
118 | { | |
119 | struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; | |
120 | struct bnxt_qplib_hwq *hwq = &nq->hwq; | |
121 | struct nq_base *nqe, **nq_ptr; | |
122 | int num_cqne_processed = 0; | |
123 | u32 sw_cons, raw_cons; | |
124 | u16 type; | |
125 | int budget = nq->budget; | |
126 | u64 q_handle; | |
127 | ||
128 | /* Service the NQ until empty */ | |
129 | raw_cons = hwq->cons; | |
130 | while (budget--) { | |
131 | sw_cons = HWQ_CMP(raw_cons, hwq); | |
132 | nq_ptr = (struct nq_base **)hwq->pbl_ptr; | |
133 | nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]; | |
134 | if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) | |
135 | break; | |
136 | ||
137 | type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; | |
138 | switch (type) { | |
139 | case NQ_BASE_TYPE_CQ_NOTIFICATION: | |
140 | { | |
141 | struct nq_cn *nqcne = (struct nq_cn *)nqe; | |
142 | ||
143 | q_handle = le32_to_cpu(nqcne->cq_handle_low); | |
144 | q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) | |
145 | << 32; | |
146 | bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *) | |
147 | ((unsigned long)q_handle)); | |
148 | if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *) | |
149 | ((unsigned long)q_handle))) | |
150 | num_cqne_processed++; | |
151 | else | |
152 | dev_warn(&nq->pdev->dev, | |
153 | "QPLIB: cqn - type 0x%x not handled", | |
154 | type); | |
155 | break; | |
156 | } | |
157 | case NQ_BASE_TYPE_DBQ_EVENT: | |
158 | break; | |
159 | default: | |
160 | dev_warn(&nq->pdev->dev, | |
161 | "QPLIB: nqe with type = 0x%x not handled", | |
162 | type); | |
163 | break; | |
164 | } | |
165 | raw_cons++; | |
166 | } | |
167 | if (hwq->cons != raw_cons) { | |
168 | hwq->cons = raw_cons; | |
169 | NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements); | |
170 | } | |
171 | } | |
172 | ||
173 | static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) | |
174 | { | |
175 | struct bnxt_qplib_nq *nq = dev_instance; | |
176 | struct bnxt_qplib_hwq *hwq = &nq->hwq; | |
177 | struct nq_base **nq_ptr; | |
178 | u32 sw_cons; | |
179 | ||
180 | /* Prefetch the NQ element */ | |
181 | sw_cons = HWQ_CMP(hwq->cons, hwq); | |
182 | nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr; | |
183 | prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]); | |
184 | ||
185 | /* Fan out to CPU affinitized kthreads? */ | |
186 | tasklet_schedule(&nq->worker); | |
187 | ||
188 | return IRQ_HANDLED; | |
189 | } | |
190 | ||
191 | void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) | |
192 | { | |
193 | /* Make sure the HW is stopped! */ | |
194 | synchronize_irq(nq->vector); | |
195 | tasklet_disable(&nq->worker); | |
196 | tasklet_kill(&nq->worker); | |
197 | ||
198 | if (nq->requested) { | |
199 | free_irq(nq->vector, nq); | |
200 | nq->requested = false; | |
201 | } | |
202 | if (nq->bar_reg_iomem) | |
203 | iounmap(nq->bar_reg_iomem); | |
204 | nq->bar_reg_iomem = NULL; | |
205 | ||
206 | nq->cqn_handler = NULL; | |
207 | nq->srqn_handler = NULL; | |
208 | nq->vector = 0; | |
209 | } | |
210 | ||
211 | int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, | |
212 | int msix_vector, int bar_reg_offset, | |
213 | int (*cqn_handler)(struct bnxt_qplib_nq *nq, | |
214 | struct bnxt_qplib_cq *), | |
215 | int (*srqn_handler)(struct bnxt_qplib_nq *nq, | |
216 | void *, u8 event)) | |
217 | { | |
218 | resource_size_t nq_base; | |
219 | int rc; | |
220 | ||
221 | nq->pdev = pdev; | |
222 | nq->vector = msix_vector; | |
223 | ||
224 | nq->cqn_handler = cqn_handler; | |
225 | ||
226 | nq->srqn_handler = srqn_handler; | |
227 | ||
228 | tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq); | |
229 | ||
230 | nq->requested = false; | |
231 | rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq); | |
232 | if (rc) { | |
233 | dev_err(&nq->pdev->dev, | |
234 | "Failed to request IRQ for NQ: %#x", rc); | |
235 | bnxt_qplib_disable_nq(nq); | |
236 | goto fail; | |
237 | } | |
238 | nq->requested = true; | |
239 | nq->bar_reg = NQ_CONS_PCI_BAR_REGION; | |
240 | nq->bar_reg_off = bar_reg_offset; | |
241 | nq_base = pci_resource_start(pdev, nq->bar_reg); | |
242 | if (!nq_base) { | |
243 | rc = -ENOMEM; | |
244 | goto fail; | |
245 | } | |
246 | nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4); | |
247 | if (!nq->bar_reg_iomem) { | |
248 | rc = -ENOMEM; | |
249 | goto fail; | |
250 | } | |
251 | NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements); | |
252 | ||
253 | return 0; | |
254 | fail: | |
255 | bnxt_qplib_disable_nq(nq); | |
256 | return rc; | |
257 | } | |
258 | ||
259 | void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq) | |
260 | { | |
261 | if (nq->hwq.max_elements) | |
262 | bnxt_qplib_free_hwq(nq->pdev, &nq->hwq); | |
263 | } | |
264 | ||
265 | int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq) | |
266 | { | |
267 | nq->pdev = pdev; | |
268 | if (!nq->hwq.max_elements || | |
269 | nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) | |
270 | nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; | |
271 | ||
272 | if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0, | |
273 | &nq->hwq.max_elements, | |
274 | BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0, | |
275 | PAGE_SIZE, HWQ_TYPE_L2_CMPL)) | |
276 | return -ENOMEM; | |
277 | ||
278 | nq->budget = 8; | |
279 | return 0; | |
280 | } | |
281 | ||
282 | /* QP */ | |
283 | int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | |
284 | { | |
285 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
286 | struct cmdq_create_qp1 req; | |
cc1ec769 | 287 | struct creq_create_qp1_resp resp; |
1ac5a404 SX |
288 | struct bnxt_qplib_pbl *pbl; |
289 | struct bnxt_qplib_q *sq = &qp->sq; | |
290 | struct bnxt_qplib_q *rq = &qp->rq; | |
291 | int rc; | |
292 | u16 cmd_flags = 0; | |
293 | u32 qp_flags = 0; | |
294 | ||
295 | RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags); | |
296 | ||
297 | /* General */ | |
298 | req.type = qp->type; | |
299 | req.dpi = cpu_to_le32(qp->dpi->dpi); | |
300 | req.qp_handle = cpu_to_le64(qp->qp_handle); | |
301 | ||
302 | /* SQ */ | |
303 | sq->hwq.max_elements = sq->max_wqe; | |
304 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0, | |
305 | &sq->hwq.max_elements, | |
306 | BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0, | |
307 | PAGE_SIZE, HWQ_TYPE_QUEUE); | |
308 | if (rc) | |
309 | goto exit; | |
310 | ||
311 | sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); | |
312 | if (!sq->swq) { | |
313 | rc = -ENOMEM; | |
314 | goto fail_sq; | |
315 | } | |
316 | pbl = &sq->hwq.pbl[PBL_LVL_0]; | |
317 | req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
318 | req.sq_pg_size_sq_lvl = | |
319 | ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) | |
320 | << CMDQ_CREATE_QP1_SQ_LVL_SFT) | | |
321 | (pbl->pg_size == ROCE_PG_SIZE_4K ? | |
322 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K : | |
323 | pbl->pg_size == ROCE_PG_SIZE_8K ? | |
324 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K : | |
325 | pbl->pg_size == ROCE_PG_SIZE_64K ? | |
326 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K : | |
327 | pbl->pg_size == ROCE_PG_SIZE_2M ? | |
328 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M : | |
329 | pbl->pg_size == ROCE_PG_SIZE_8M ? | |
330 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M : | |
331 | pbl->pg_size == ROCE_PG_SIZE_1G ? | |
332 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G : | |
333 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K); | |
334 | ||
335 | if (qp->scq) | |
336 | req.scq_cid = cpu_to_le32(qp->scq->id); | |
337 | ||
338 | qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; | |
339 | ||
340 | /* RQ */ | |
341 | if (rq->max_wqe) { | |
342 | rq->hwq.max_elements = qp->rq.max_wqe; | |
343 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0, | |
344 | &rq->hwq.max_elements, | |
345 | BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, | |
346 | PAGE_SIZE, HWQ_TYPE_QUEUE); | |
347 | if (rc) | |
348 | goto fail_sq; | |
349 | ||
350 | rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), | |
351 | GFP_KERNEL); | |
352 | if (!rq->swq) { | |
353 | rc = -ENOMEM; | |
354 | goto fail_rq; | |
355 | } | |
356 | pbl = &rq->hwq.pbl[PBL_LVL_0]; | |
357 | req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
358 | req.rq_pg_size_rq_lvl = | |
359 | ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << | |
360 | CMDQ_CREATE_QP1_RQ_LVL_SFT) | | |
361 | (pbl->pg_size == ROCE_PG_SIZE_4K ? | |
362 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K : | |
363 | pbl->pg_size == ROCE_PG_SIZE_8K ? | |
364 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K : | |
365 | pbl->pg_size == ROCE_PG_SIZE_64K ? | |
366 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K : | |
367 | pbl->pg_size == ROCE_PG_SIZE_2M ? | |
368 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M : | |
369 | pbl->pg_size == ROCE_PG_SIZE_8M ? | |
370 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M : | |
371 | pbl->pg_size == ROCE_PG_SIZE_1G ? | |
372 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G : | |
373 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K); | |
374 | if (qp->rcq) | |
375 | req.rcq_cid = cpu_to_le32(qp->rcq->id); | |
376 | } | |
377 | ||
378 | /* Header buffer - allow hdr_buf pass in */ | |
379 | rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); | |
380 | if (rc) { | |
381 | rc = -ENOMEM; | |
382 | goto fail; | |
383 | } | |
384 | req.qp_flags = cpu_to_le32(qp_flags); | |
385 | req.sq_size = cpu_to_le32(sq->hwq.max_elements); | |
386 | req.rq_size = cpu_to_le32(rq->hwq.max_elements); | |
387 | ||
388 | req.sq_fwo_sq_sge = | |
389 | cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << | |
390 | CMDQ_CREATE_QP1_SQ_SGE_SFT); | |
391 | req.rq_fwo_rq_sge = | |
392 | cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) << | |
393 | CMDQ_CREATE_QP1_RQ_SGE_SFT); | |
394 | ||
395 | req.pd_id = cpu_to_le32(qp->pd->id); | |
396 | ||
cc1ec769 DS |
397 | rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, |
398 | (void *)&resp, NULL, 0); | |
399 | if (rc) | |
1ac5a404 | 400 | goto fail; |
cc1ec769 DS |
401 | |
402 | qp->id = le32_to_cpu(resp.xid); | |
1ac5a404 SX |
403 | qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; |
404 | sq->flush_in_progress = false; | |
405 | rq->flush_in_progress = false; | |
406 | ||
407 | return 0; | |
408 | ||
409 | fail: | |
410 | bnxt_qplib_free_qp_hdr_buf(res, qp); | |
411 | fail_rq: | |
412 | bnxt_qplib_free_hwq(res->pdev, &rq->hwq); | |
413 | kfree(rq->swq); | |
414 | fail_sq: | |
415 | bnxt_qplib_free_hwq(res->pdev, &sq->hwq); | |
416 | kfree(sq->swq); | |
417 | exit: | |
418 | return rc; | |
419 | } | |
420 | ||
421 | int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | |
422 | { | |
423 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
424 | struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; | |
425 | struct cmdq_create_qp req; | |
cc1ec769 | 426 | struct creq_create_qp_resp resp; |
1ac5a404 SX |
427 | struct bnxt_qplib_pbl *pbl; |
428 | struct sq_psn_search **psn_search_ptr; | |
429 | unsigned long int psn_search, poff = 0; | |
430 | struct bnxt_qplib_q *sq = &qp->sq; | |
431 | struct bnxt_qplib_q *rq = &qp->rq; | |
432 | struct bnxt_qplib_hwq *xrrq; | |
433 | int i, rc, req_size, psn_sz; | |
434 | u16 cmd_flags = 0, max_ssge; | |
435 | u32 sw_prod, qp_flags = 0; | |
436 | ||
437 | RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); | |
438 | ||
439 | /* General */ | |
440 | req.type = qp->type; | |
441 | req.dpi = cpu_to_le32(qp->dpi->dpi); | |
442 | req.qp_handle = cpu_to_le64(qp->qp_handle); | |
443 | ||
444 | /* SQ */ | |
445 | psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ? | |
446 | sizeof(struct sq_psn_search) : 0; | |
447 | sq->hwq.max_elements = sq->max_wqe; | |
448 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist, | |
449 | sq->nmap, &sq->hwq.max_elements, | |
450 | BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, | |
451 | psn_sz, | |
452 | PAGE_SIZE, HWQ_TYPE_QUEUE); | |
453 | if (rc) | |
454 | goto exit; | |
455 | ||
456 | sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); | |
457 | if (!sq->swq) { | |
458 | rc = -ENOMEM; | |
459 | goto fail_sq; | |
460 | } | |
461 | hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; | |
462 | if (psn_sz) { | |
463 | psn_search_ptr = (struct sq_psn_search **) | |
464 | &hw_sq_send_ptr[get_sqe_pg | |
465 | (sq->hwq.max_elements)]; | |
466 | psn_search = (unsigned long int) | |
467 | &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)] | |
468 | [get_sqe_idx(sq->hwq.max_elements)]; | |
469 | if (psn_search & ~PAGE_MASK) { | |
470 | /* If the psn_search does not start on a page boundary, | |
471 | * then calculate the offset | |
472 | */ | |
473 | poff = (psn_search & ~PAGE_MASK) / | |
474 | BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE; | |
475 | } | |
476 | for (i = 0; i < sq->hwq.max_elements; i++) | |
477 | sq->swq[i].psn_search = | |
478 | &psn_search_ptr[get_psne_pg(i + poff)] | |
479 | [get_psne_idx(i + poff)]; | |
480 | } | |
481 | pbl = &sq->hwq.pbl[PBL_LVL_0]; | |
482 | req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
483 | req.sq_pg_size_sq_lvl = | |
484 | ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) | |
485 | << CMDQ_CREATE_QP_SQ_LVL_SFT) | | |
486 | (pbl->pg_size == ROCE_PG_SIZE_4K ? | |
487 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K : | |
488 | pbl->pg_size == ROCE_PG_SIZE_8K ? | |
489 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K : | |
490 | pbl->pg_size == ROCE_PG_SIZE_64K ? | |
491 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K : | |
492 | pbl->pg_size == ROCE_PG_SIZE_2M ? | |
493 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M : | |
494 | pbl->pg_size == ROCE_PG_SIZE_8M ? | |
495 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M : | |
496 | pbl->pg_size == ROCE_PG_SIZE_1G ? | |
497 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G : | |
498 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K); | |
499 | ||
500 | /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */ | |
501 | hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; | |
502 | for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) { | |
503 | hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] | |
504 | [get_sqe_idx(sw_prod)]; | |
505 | hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID; | |
506 | } | |
507 | ||
508 | if (qp->scq) | |
509 | req.scq_cid = cpu_to_le32(qp->scq->id); | |
510 | ||
511 | qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; | |
512 | qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; | |
513 | if (qp->sig_type) | |
514 | qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; | |
515 | ||
516 | /* RQ */ | |
517 | if (rq->max_wqe) { | |
518 | rq->hwq.max_elements = rq->max_wqe; | |
519 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist, | |
520 | rq->nmap, &rq->hwq.max_elements, | |
521 | BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, | |
522 | PAGE_SIZE, HWQ_TYPE_QUEUE); | |
523 | if (rc) | |
524 | goto fail_sq; | |
525 | ||
526 | rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), | |
527 | GFP_KERNEL); | |
528 | if (!rq->swq) { | |
529 | rc = -ENOMEM; | |
530 | goto fail_rq; | |
531 | } | |
532 | pbl = &rq->hwq.pbl[PBL_LVL_0]; | |
533 | req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
534 | req.rq_pg_size_rq_lvl = | |
535 | ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << | |
536 | CMDQ_CREATE_QP_RQ_LVL_SFT) | | |
537 | (pbl->pg_size == ROCE_PG_SIZE_4K ? | |
538 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K : | |
539 | pbl->pg_size == ROCE_PG_SIZE_8K ? | |
540 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K : | |
541 | pbl->pg_size == ROCE_PG_SIZE_64K ? | |
542 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K : | |
543 | pbl->pg_size == ROCE_PG_SIZE_2M ? | |
544 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M : | |
545 | pbl->pg_size == ROCE_PG_SIZE_8M ? | |
546 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M : | |
547 | pbl->pg_size == ROCE_PG_SIZE_1G ? | |
548 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G : | |
549 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K); | |
550 | } | |
551 | ||
552 | if (qp->rcq) | |
553 | req.rcq_cid = cpu_to_le32(qp->rcq->id); | |
554 | req.qp_flags = cpu_to_le32(qp_flags); | |
555 | req.sq_size = cpu_to_le32(sq->hwq.max_elements); | |
556 | req.rq_size = cpu_to_le32(rq->hwq.max_elements); | |
557 | qp->sq_hdr_buf = NULL; | |
558 | qp->rq_hdr_buf = NULL; | |
559 | ||
560 | rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); | |
561 | if (rc) | |
562 | goto fail_rq; | |
563 | ||
564 | /* CTRL-22434: Irrespective of the requested SGE count on the SQ | |
565 | * always create the QP with max send sges possible if the requested | |
566 | * inline size is greater than 0. | |
567 | */ | |
568 | max_ssge = qp->max_inline_data ? 6 : sq->max_sge; | |
569 | req.sq_fwo_sq_sge = cpu_to_le16( | |
570 | ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) | |
571 | << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); | |
572 | req.rq_fwo_rq_sge = cpu_to_le16( | |
573 | ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK) | |
574 | << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); | |
575 | /* ORRQ and IRRQ */ | |
576 | if (psn_sz) { | |
577 | xrrq = &qp->orrq; | |
578 | xrrq->max_elements = | |
579 | ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); | |
580 | req_size = xrrq->max_elements * | |
581 | BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; | |
582 | req_size &= ~(PAGE_SIZE - 1); | |
583 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, | |
584 | &xrrq->max_elements, | |
585 | BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE, | |
586 | 0, req_size, HWQ_TYPE_CTX); | |
587 | if (rc) | |
588 | goto fail_buf_free; | |
589 | pbl = &xrrq->pbl[PBL_LVL_0]; | |
590 | req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); | |
591 | ||
592 | xrrq = &qp->irrq; | |
593 | xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( | |
594 | qp->max_dest_rd_atomic); | |
595 | req_size = xrrq->max_elements * | |
596 | BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; | |
597 | req_size &= ~(PAGE_SIZE - 1); | |
598 | ||
599 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, | |
600 | &xrrq->max_elements, | |
601 | BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE, | |
602 | 0, req_size, HWQ_TYPE_CTX); | |
603 | if (rc) | |
604 | goto fail_orrq; | |
605 | ||
606 | pbl = &xrrq->pbl[PBL_LVL_0]; | |
607 | req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); | |
608 | } | |
609 | req.pd_id = cpu_to_le32(qp->pd->id); | |
610 | ||
cc1ec769 DS |
611 | rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, |
612 | (void *)&resp, NULL, 0); | |
613 | if (rc) | |
1ac5a404 | 614 | goto fail; |
cc1ec769 DS |
615 | |
616 | qp->id = le32_to_cpu(resp.xid); | |
1ac5a404 SX |
617 | qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; |
618 | sq->flush_in_progress = false; | |
619 | rq->flush_in_progress = false; | |
620 | ||
621 | return 0; | |
622 | ||
623 | fail: | |
624 | if (qp->irrq.max_elements) | |
625 | bnxt_qplib_free_hwq(res->pdev, &qp->irrq); | |
626 | fail_orrq: | |
627 | if (qp->orrq.max_elements) | |
628 | bnxt_qplib_free_hwq(res->pdev, &qp->orrq); | |
629 | fail_buf_free: | |
630 | bnxt_qplib_free_qp_hdr_buf(res, qp); | |
631 | fail_rq: | |
632 | bnxt_qplib_free_hwq(res->pdev, &rq->hwq); | |
633 | kfree(rq->swq); | |
634 | fail_sq: | |
635 | bnxt_qplib_free_hwq(res->pdev, &sq->hwq); | |
636 | kfree(sq->swq); | |
637 | exit: | |
638 | return rc; | |
639 | } | |
640 | ||
641 | static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp) | |
642 | { | |
643 | switch (qp->state) { | |
644 | case CMDQ_MODIFY_QP_NEW_STATE_RTR: | |
645 | /* INIT->RTR, configure the path_mtu to the default | |
646 | * 2048 if not being requested | |
647 | */ | |
648 | if (!(qp->modify_flags & | |
649 | CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { | |
650 | qp->modify_flags |= | |
651 | CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; | |
652 | qp->path_mtu = | |
653 | CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; | |
654 | } | |
655 | qp->modify_flags &= | |
656 | ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; | |
657 | /* Bono FW require the max_dest_rd_atomic to be >= 1 */ | |
658 | if (qp->max_dest_rd_atomic < 1) | |
659 | qp->max_dest_rd_atomic = 1; | |
660 | qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; | |
661 | /* Bono FW 20.6.5 requires SGID_INDEX configuration */ | |
662 | if (!(qp->modify_flags & | |
663 | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { | |
664 | qp->modify_flags |= | |
665 | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; | |
666 | qp->ah.sgid_index = 0; | |
667 | } | |
668 | break; | |
669 | default: | |
670 | break; | |
671 | } | |
672 | } | |
673 | ||
674 | static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp) | |
675 | { | |
676 | switch (qp->state) { | |
677 | case CMDQ_MODIFY_QP_NEW_STATE_RTS: | |
678 | /* Bono FW requires the max_rd_atomic to be >= 1 */ | |
679 | if (qp->max_rd_atomic < 1) | |
680 | qp->max_rd_atomic = 1; | |
681 | /* Bono FW does not allow PKEY_INDEX, | |
682 | * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT, | |
683 | * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN, | |
684 | * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID | |
685 | * modification | |
686 | */ | |
687 | qp->modify_flags &= | |
688 | ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | | |
689 | CMDQ_MODIFY_QP_MODIFY_MASK_DGID | | |
690 | CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | | |
691 | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | | |
692 | CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | | |
693 | CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | | |
694 | CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | | |
695 | CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | | |
696 | CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | | |
697 | CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | | |
698 | CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | | |
699 | CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); | |
700 | break; | |
701 | default: | |
702 | break; | |
703 | } | |
704 | } | |
705 | ||
706 | static void __filter_modify_flags(struct bnxt_qplib_qp *qp) | |
707 | { | |
708 | switch (qp->cur_qp_state) { | |
709 | case CMDQ_MODIFY_QP_NEW_STATE_RESET: | |
710 | break; | |
711 | case CMDQ_MODIFY_QP_NEW_STATE_INIT: | |
712 | __modify_flags_from_init_state(qp); | |
713 | break; | |
714 | case CMDQ_MODIFY_QP_NEW_STATE_RTR: | |
715 | __modify_flags_from_rtr_state(qp); | |
716 | break; | |
717 | case CMDQ_MODIFY_QP_NEW_STATE_RTS: | |
718 | break; | |
719 | case CMDQ_MODIFY_QP_NEW_STATE_SQD: | |
720 | break; | |
721 | case CMDQ_MODIFY_QP_NEW_STATE_SQE: | |
722 | break; | |
723 | case CMDQ_MODIFY_QP_NEW_STATE_ERR: | |
724 | break; | |
725 | default: | |
726 | break; | |
727 | } | |
728 | } | |
729 | ||
730 | int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | |
731 | { | |
732 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
733 | struct cmdq_modify_qp req; | |
cc1ec769 | 734 | struct creq_modify_qp_resp resp; |
1ac5a404 SX |
735 | u16 cmd_flags = 0, pkey; |
736 | u32 temp32[4]; | |
737 | u32 bmask; | |
cc1ec769 | 738 | int rc; |
1ac5a404 SX |
739 | |
740 | RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags); | |
741 | ||
742 | /* Filter out the qp_attr_mask based on the state->new transition */ | |
743 | __filter_modify_flags(qp); | |
744 | bmask = qp->modify_flags; | |
745 | req.modify_mask = cpu_to_le32(qp->modify_flags); | |
746 | req.qp_cid = cpu_to_le32(qp->id); | |
747 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { | |
748 | req.network_type_en_sqd_async_notify_new_state = | |
749 | (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | | |
750 | (qp->en_sqd_async_notify ? | |
751 | CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); | |
752 | } | |
753 | req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; | |
754 | ||
755 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) | |
756 | req.access = qp->access; | |
757 | ||
758 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) { | |
759 | if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl, | |
760 | qp->pkey_index, &pkey)) | |
761 | req.pkey = cpu_to_le16(pkey); | |
762 | } | |
763 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) | |
764 | req.qkey = cpu_to_le32(qp->qkey); | |
765 | ||
766 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { | |
767 | memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); | |
768 | req.dgid[0] = cpu_to_le32(temp32[0]); | |
769 | req.dgid[1] = cpu_to_le32(temp32[1]); | |
770 | req.dgid[2] = cpu_to_le32(temp32[2]); | |
771 | req.dgid[3] = cpu_to_le32(temp32[3]); | |
772 | } | |
773 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) | |
774 | req.flow_label = cpu_to_le32(qp->ah.flow_label); | |
775 | ||
776 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) | |
777 | req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id | |
778 | [qp->ah.sgid_index]); | |
779 | ||
780 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) | |
781 | req.hop_limit = qp->ah.hop_limit; | |
782 | ||
783 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) | |
784 | req.traffic_class = qp->ah.traffic_class; | |
785 | ||
786 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) | |
787 | memcpy(req.dest_mac, qp->ah.dmac, 6); | |
788 | ||
789 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) | |
790 | req.path_mtu = qp->path_mtu; | |
791 | ||
792 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) | |
793 | req.timeout = qp->timeout; | |
794 | ||
795 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) | |
796 | req.retry_cnt = qp->retry_cnt; | |
797 | ||
798 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) | |
799 | req.rnr_retry = qp->rnr_retry; | |
800 | ||
801 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) | |
802 | req.min_rnr_timer = qp->min_rnr_timer; | |
803 | ||
804 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) | |
805 | req.rq_psn = cpu_to_le32(qp->rq.psn); | |
806 | ||
807 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) | |
808 | req.sq_psn = cpu_to_le32(qp->sq.psn); | |
809 | ||
810 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) | |
811 | req.max_rd_atomic = | |
812 | ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); | |
813 | ||
814 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) | |
815 | req.max_dest_rd_atomic = | |
816 | IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); | |
817 | ||
818 | req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); | |
819 | req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); | |
820 | req.sq_sge = cpu_to_le16(qp->sq.max_sge); | |
821 | req.rq_sge = cpu_to_le16(qp->rq.max_sge); | |
822 | req.max_inline_data = cpu_to_le32(qp->max_inline_data); | |
823 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) | |
824 | req.dest_qp_id = cpu_to_le32(qp->dest_qpn); | |
825 | ||
826 | req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); | |
827 | ||
cc1ec769 DS |
828 | rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, |
829 | (void *)&resp, NULL, 0); | |
830 | if (rc) | |
831 | return rc; | |
1ac5a404 SX |
832 | qp->cur_qp_state = qp->state; |
833 | return 0; | |
834 | } | |
835 | ||
836 | int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | |
837 | { | |
838 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
839 | struct cmdq_query_qp req; | |
cc1ec769 DS |
840 | struct creq_query_qp_resp resp; |
841 | struct bnxt_qplib_rcfw_sbuf *sbuf; | |
1ac5a404 SX |
842 | struct creq_query_qp_resp_sb *sb; |
843 | u16 cmd_flags = 0; | |
844 | u32 temp32[4]; | |
cc1ec769 | 845 | int i, rc = 0; |
1ac5a404 SX |
846 | |
847 | RCFW_CMD_PREP(req, QUERY_QP, cmd_flags); | |
848 | ||
cc1ec769 DS |
849 | sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); |
850 | if (!sbuf) | |
851 | return -ENOMEM; | |
852 | sb = sbuf->sb; | |
853 | ||
1ac5a404 SX |
854 | req.qp_cid = cpu_to_le32(qp->id); |
855 | req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; | |
cc1ec769 DS |
856 | rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, |
857 | (void *)sbuf, 0); | |
858 | if (rc) | |
859 | goto bail; | |
1ac5a404 SX |
860 | /* Extract the context from the side buffer */ |
861 | qp->state = sb->en_sqd_async_notify_state & | |
862 | CREQ_QUERY_QP_RESP_SB_STATE_MASK; | |
863 | qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & | |
864 | CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? | |
865 | true : false; | |
866 | qp->access = sb->access; | |
867 | qp->pkey_index = le16_to_cpu(sb->pkey); | |
868 | qp->qkey = le32_to_cpu(sb->qkey); | |
869 | ||
870 | temp32[0] = le32_to_cpu(sb->dgid[0]); | |
871 | temp32[1] = le32_to_cpu(sb->dgid[1]); | |
872 | temp32[2] = le32_to_cpu(sb->dgid[2]); | |
873 | temp32[3] = le32_to_cpu(sb->dgid[3]); | |
874 | memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); | |
875 | ||
876 | qp->ah.flow_label = le32_to_cpu(sb->flow_label); | |
877 | ||
878 | qp->ah.sgid_index = 0; | |
879 | for (i = 0; i < res->sgid_tbl.max; i++) { | |
880 | if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { | |
881 | qp->ah.sgid_index = i; | |
882 | break; | |
883 | } | |
884 | } | |
885 | if (i == res->sgid_tbl.max) | |
886 | dev_warn(&res->pdev->dev, "QPLIB: SGID not found??"); | |
887 | ||
888 | qp->ah.hop_limit = sb->hop_limit; | |
889 | qp->ah.traffic_class = sb->traffic_class; | |
890 | memcpy(qp->ah.dmac, sb->dest_mac, 6); | |
891 | qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & | |
892 | CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >> | |
893 | CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; | |
894 | qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & | |
895 | CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >> | |
896 | CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT; | |
897 | qp->timeout = sb->timeout; | |
898 | qp->retry_cnt = sb->retry_cnt; | |
899 | qp->rnr_retry = sb->rnr_retry; | |
900 | qp->min_rnr_timer = sb->min_rnr_timer; | |
901 | qp->rq.psn = le32_to_cpu(sb->rq_psn); | |
902 | qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); | |
903 | qp->sq.psn = le32_to_cpu(sb->sq_psn); | |
904 | qp->max_dest_rd_atomic = | |
905 | IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); | |
906 | qp->sq.max_wqe = qp->sq.hwq.max_elements; | |
907 | qp->rq.max_wqe = qp->rq.hwq.max_elements; | |
908 | qp->sq.max_sge = le16_to_cpu(sb->sq_sge); | |
909 | qp->rq.max_sge = le16_to_cpu(sb->rq_sge); | |
910 | qp->max_inline_data = le32_to_cpu(sb->max_inline_data); | |
911 | qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); | |
912 | memcpy(qp->smac, sb->src_mac, 6); | |
913 | qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); | |
cc1ec769 DS |
914 | bail: |
915 | bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); | |
916 | return rc; | |
1ac5a404 SX |
917 | } |
918 | ||
919 | static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) | |
920 | { | |
921 | struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; | |
922 | struct cq_base *hw_cqe, **hw_cqe_ptr; | |
923 | int i; | |
924 | ||
925 | for (i = 0; i < cq_hwq->max_elements; i++) { | |
926 | hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr; | |
927 | hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)]; | |
928 | if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements)) | |
929 | continue; | |
930 | switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { | |
931 | case CQ_BASE_CQE_TYPE_REQ: | |
932 | case CQ_BASE_CQE_TYPE_TERMINAL: | |
933 | { | |
934 | struct cq_req *cqe = (struct cq_req *)hw_cqe; | |
935 | ||
936 | if (qp == le64_to_cpu(cqe->qp_handle)) | |
937 | cqe->qp_handle = 0; | |
938 | break; | |
939 | } | |
940 | case CQ_BASE_CQE_TYPE_RES_RC: | |
941 | case CQ_BASE_CQE_TYPE_RES_UD: | |
942 | case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: | |
943 | { | |
944 | struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; | |
945 | ||
946 | if (qp == le64_to_cpu(cqe->qp_handle)) | |
947 | cqe->qp_handle = 0; | |
948 | break; | |
949 | } | |
950 | default: | |
951 | break; | |
952 | } | |
953 | } | |
954 | } | |
955 | ||
956 | int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, | |
957 | struct bnxt_qplib_qp *qp) | |
958 | { | |
959 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
960 | struct cmdq_destroy_qp req; | |
cc1ec769 | 961 | struct creq_destroy_qp_resp resp; |
1ac5a404 SX |
962 | unsigned long flags; |
963 | u16 cmd_flags = 0; | |
cc1ec769 | 964 | int rc; |
1ac5a404 SX |
965 | |
966 | RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags); | |
967 | ||
968 | req.qp_cid = cpu_to_le32(qp->id); | |
cc1ec769 DS |
969 | rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, |
970 | (void *)&resp, NULL, 0); | |
971 | if (rc) | |
972 | return rc; | |
1ac5a404 SX |
973 | |
974 | /* Must walk the associated CQs to nullified the QP ptr */ | |
975 | spin_lock_irqsave(&qp->scq->hwq.lock, flags); | |
976 | ||
977 | __clean_cq(qp->scq, (u64)(unsigned long)qp); | |
978 | ||
979 | if (qp->rcq && qp->rcq != qp->scq) { | |
980 | spin_lock(&qp->rcq->hwq.lock); | |
981 | __clean_cq(qp->rcq, (u64)(unsigned long)qp); | |
982 | spin_unlock(&qp->rcq->hwq.lock); | |
983 | } | |
984 | ||
985 | spin_unlock_irqrestore(&qp->scq->hwq.lock, flags); | |
986 | ||
987 | bnxt_qplib_free_qp_hdr_buf(res, qp); | |
988 | bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq); | |
989 | kfree(qp->sq.swq); | |
990 | ||
991 | bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq); | |
992 | kfree(qp->rq.swq); | |
993 | ||
994 | if (qp->irrq.max_elements) | |
995 | bnxt_qplib_free_hwq(res->pdev, &qp->irrq); | |
996 | if (qp->orrq.max_elements) | |
997 | bnxt_qplib_free_hwq(res->pdev, &qp->orrq); | |
998 | ||
999 | return 0; | |
1000 | } | |
1001 | ||
1002 | void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, | |
1003 | struct bnxt_qplib_sge *sge) | |
1004 | { | |
1005 | struct bnxt_qplib_q *sq = &qp->sq; | |
1006 | u32 sw_prod; | |
1007 | ||
1008 | memset(sge, 0, sizeof(*sge)); | |
1009 | ||
1010 | if (qp->sq_hdr_buf) { | |
1011 | sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
1012 | sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + | |
1013 | sw_prod * qp->sq_hdr_buf_size); | |
1014 | sge->lkey = 0xFFFFFFFF; | |
1015 | sge->size = qp->sq_hdr_buf_size; | |
1016 | return qp->sq_hdr_buf + sw_prod * sge->size; | |
1017 | } | |
1018 | return NULL; | |
1019 | } | |
1020 | ||
1021 | u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) | |
1022 | { | |
1023 | struct bnxt_qplib_q *rq = &qp->rq; | |
1024 | ||
1025 | return HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
1026 | } | |
1027 | ||
1028 | dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) | |
1029 | { | |
1030 | return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size); | |
1031 | } | |
1032 | ||
1033 | void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, | |
1034 | struct bnxt_qplib_sge *sge) | |
1035 | { | |
1036 | struct bnxt_qplib_q *rq = &qp->rq; | |
1037 | u32 sw_prod; | |
1038 | ||
1039 | memset(sge, 0, sizeof(*sge)); | |
1040 | ||
1041 | if (qp->rq_hdr_buf) { | |
1042 | sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
1043 | sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + | |
1044 | sw_prod * qp->rq_hdr_buf_size); | |
1045 | sge->lkey = 0xFFFFFFFF; | |
1046 | sge->size = qp->rq_hdr_buf_size; | |
1047 | return qp->rq_hdr_buf + sw_prod * sge->size; | |
1048 | } | |
1049 | return NULL; | |
1050 | } | |
1051 | ||
1052 | void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) | |
1053 | { | |
1054 | struct bnxt_qplib_q *sq = &qp->sq; | |
1055 | struct dbr_dbr db_msg = { 0 }; | |
1056 | u32 sw_prod; | |
1057 | ||
1058 | sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
1059 | ||
1060 | db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & | |
1061 | DBR_DBR_INDEX_MASK); | |
1062 | db_msg.type_xid = | |
1063 | cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | |
1064 | DBR_DBR_TYPE_SQ); | |
1065 | /* Flush all the WQE writes to HW */ | |
1066 | wmb(); | |
1067 | __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); | |
1068 | } | |
1069 | ||
1070 | int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, | |
1071 | struct bnxt_qplib_swqe *wqe) | |
1072 | { | |
1073 | struct bnxt_qplib_q *sq = &qp->sq; | |
1074 | struct bnxt_qplib_swq *swq; | |
1075 | struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; | |
1076 | struct sq_sge *hw_sge; | |
1077 | u32 sw_prod; | |
1078 | u8 wqe_size16; | |
1079 | int i, rc = 0, data_len = 0, pkt_num = 0; | |
1080 | __le32 temp32; | |
1081 | ||
1082 | if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) { | |
1083 | rc = -EINVAL; | |
1084 | goto done; | |
1085 | } | |
9152e0b7 EW |
1086 | |
1087 | if (bnxt_qplib_queue_full(sq)) { | |
1088 | dev_err(&sq->hwq.pdev->dev, | |
1089 | "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x", | |
1090 | sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements, | |
1091 | sq->q_full_delta); | |
1ac5a404 SX |
1092 | rc = -ENOMEM; |
1093 | goto done; | |
1094 | } | |
1095 | sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
1096 | swq = &sq->swq[sw_prod]; | |
1097 | swq->wr_id = wqe->wr_id; | |
1098 | swq->type = wqe->type; | |
1099 | swq->flags = wqe->flags; | |
1100 | if (qp->sig_type) | |
1101 | swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; | |
1102 | swq->start_psn = sq->psn & BTH_PSN_MASK; | |
1103 | ||
1104 | hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; | |
1105 | hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] | |
1106 | [get_sqe_idx(sw_prod)]; | |
1107 | ||
1108 | memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); | |
1109 | ||
1110 | if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { | |
1111 | /* Copy the inline data */ | |
1112 | if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { | |
1113 | dev_warn(&sq->hwq.pdev->dev, | |
1114 | "QPLIB: Inline data length > 96 detected"); | |
1115 | data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH; | |
1116 | } else { | |
1117 | data_len = wqe->inline_len; | |
1118 | } | |
1119 | memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len); | |
1120 | wqe_size16 = (data_len + 15) >> 4; | |
1121 | } else { | |
1122 | for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data; | |
1123 | i < wqe->num_sge; i++, hw_sge++) { | |
1124 | hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); | |
1125 | hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); | |
1126 | hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); | |
1127 | data_len += wqe->sg_list[i].size; | |
1128 | } | |
1129 | /* Each SGE entry = 1 WQE size16 */ | |
1130 | wqe_size16 = wqe->num_sge; | |
1131 | } | |
1132 | ||
1133 | /* Specifics */ | |
1134 | switch (wqe->type) { | |
1135 | case BNXT_QPLIB_SWQE_TYPE_SEND: | |
1136 | if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { | |
1137 | /* Assemble info for Raw Ethertype QPs */ | |
1138 | struct sq_send_raweth_qp1 *sqe = | |
1139 | (struct sq_send_raweth_qp1 *)hw_sq_send_hdr; | |
1140 | ||
1141 | sqe->wqe_type = wqe->type; | |
1142 | sqe->flags = wqe->flags; | |
1143 | sqe->wqe_size = wqe_size16 + | |
1144 | ((offsetof(typeof(*sqe), data) + 15) >> 4); | |
1145 | sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); | |
1146 | sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); | |
1147 | sqe->length = cpu_to_le32(data_len); | |
1148 | sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & | |
1149 | SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << | |
1150 | SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); | |
1151 | ||
1152 | break; | |
1153 | } | |
1154 | /* else, just fall thru */ | |
1155 | case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: | |
1156 | case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: | |
1157 | { | |
1158 | struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr; | |
1159 | ||
1160 | sqe->wqe_type = wqe->type; | |
1161 | sqe->flags = wqe->flags; | |
1162 | sqe->wqe_size = wqe_size16 + | |
1163 | ((offsetof(typeof(*sqe), data) + 15) >> 4); | |
1164 | sqe->inv_key_or_imm_data = cpu_to_le32( | |
1165 | wqe->send.inv_key); | |
1166 | if (qp->type == CMDQ_CREATE_QP_TYPE_UD) { | |
1167 | sqe->q_key = cpu_to_le32(wqe->send.q_key); | |
1168 | sqe->dst_qp = cpu_to_le32( | |
1169 | wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); | |
1170 | sqe->length = cpu_to_le32(data_len); | |
1171 | sqe->avid = cpu_to_le32(wqe->send.avid & | |
1172 | SQ_SEND_AVID_MASK); | |
1173 | sq->psn = (sq->psn + 1) & BTH_PSN_MASK; | |
1174 | } else { | |
1175 | sqe->length = cpu_to_le32(data_len); | |
1176 | sqe->dst_qp = 0; | |
1177 | sqe->avid = 0; | |
1178 | if (qp->mtu) | |
1179 | pkt_num = (data_len + qp->mtu - 1) / qp->mtu; | |
1180 | if (!pkt_num) | |
1181 | pkt_num = 1; | |
1182 | sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; | |
1183 | } | |
1184 | break; | |
1185 | } | |
1186 | case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: | |
1187 | case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: | |
1188 | case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: | |
1189 | { | |
1190 | struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr; | |
1191 | ||
1192 | sqe->wqe_type = wqe->type; | |
1193 | sqe->flags = wqe->flags; | |
1194 | sqe->wqe_size = wqe_size16 + | |
1195 | ((offsetof(typeof(*sqe), data) + 15) >> 4); | |
1196 | sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); | |
1197 | sqe->length = cpu_to_le32((u32)data_len); | |
1198 | sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); | |
1199 | sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); | |
1200 | if (qp->mtu) | |
1201 | pkt_num = (data_len + qp->mtu - 1) / qp->mtu; | |
1202 | if (!pkt_num) | |
1203 | pkt_num = 1; | |
1204 | sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; | |
1205 | break; | |
1206 | } | |
1207 | case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: | |
1208 | case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: | |
1209 | { | |
1210 | struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr; | |
1211 | ||
1212 | sqe->wqe_type = wqe->type; | |
1213 | sqe->flags = wqe->flags; | |
1214 | sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); | |
1215 | sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); | |
1216 | sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); | |
1217 | sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); | |
1218 | if (qp->mtu) | |
1219 | pkt_num = (data_len + qp->mtu - 1) / qp->mtu; | |
1220 | if (!pkt_num) | |
1221 | pkt_num = 1; | |
1222 | sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; | |
1223 | break; | |
1224 | } | |
1225 | case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: | |
1226 | { | |
1227 | struct sq_localinvalidate *sqe = | |
1228 | (struct sq_localinvalidate *)hw_sq_send_hdr; | |
1229 | ||
1230 | sqe->wqe_type = wqe->type; | |
1231 | sqe->flags = wqe->flags; | |
1232 | sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); | |
1233 | ||
1234 | break; | |
1235 | } | |
1236 | case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: | |
1237 | { | |
1238 | struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr; | |
1239 | ||
1240 | sqe->wqe_type = wqe->type; | |
1241 | sqe->flags = wqe->flags; | |
1242 | sqe->access_cntl = wqe->frmr.access_cntl | | |
1243 | SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; | |
1244 | sqe->zero_based_page_size_log = | |
1245 | (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << | |
1246 | SQ_FR_PMR_PAGE_SIZE_LOG_SFT | | |
1247 | (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0); | |
1248 | sqe->l_key = cpu_to_le32(wqe->frmr.l_key); | |
1249 | temp32 = cpu_to_le32(wqe->frmr.length); | |
1250 | memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); | |
1251 | sqe->numlevels_pbl_page_size_log = | |
1252 | ((wqe->frmr.pbl_pg_sz_log << | |
1253 | SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & | |
1254 | SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | | |
1255 | ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & | |
1256 | SQ_FR_PMR_NUMLEVELS_MASK); | |
1257 | ||
1258 | for (i = 0; i < wqe->frmr.page_list_len; i++) | |
1259 | wqe->frmr.pbl_ptr[i] = cpu_to_le64( | |
1260 | wqe->frmr.page_list[i] | | |
1261 | PTU_PTE_VALID); | |
1262 | sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); | |
1263 | sqe->va = cpu_to_le64(wqe->frmr.va); | |
1264 | ||
1265 | break; | |
1266 | } | |
1267 | case BNXT_QPLIB_SWQE_TYPE_BIND_MW: | |
1268 | { | |
1269 | struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr; | |
1270 | ||
1271 | sqe->wqe_type = wqe->type; | |
1272 | sqe->flags = wqe->flags; | |
1273 | sqe->access_cntl = wqe->bind.access_cntl; | |
1274 | sqe->mw_type_zero_based = wqe->bind.mw_type | | |
1275 | (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); | |
1276 | sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); | |
1277 | sqe->l_key = cpu_to_le32(wqe->bind.r_key); | |
1278 | sqe->va = cpu_to_le64(wqe->bind.va); | |
1279 | temp32 = cpu_to_le32(wqe->bind.length); | |
1280 | memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length)); | |
1281 | break; | |
1282 | } | |
1283 | default: | |
1284 | /* Bad wqe, return error */ | |
1285 | rc = -EINVAL; | |
1286 | goto done; | |
1287 | } | |
1288 | swq->next_psn = sq->psn & BTH_PSN_MASK; | |
1289 | if (swq->psn_search) { | |
1290 | swq->psn_search->opcode_start_psn = cpu_to_le32( | |
1291 | ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & | |
1292 | SQ_PSN_SEARCH_START_PSN_MASK) | | |
1293 | ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & | |
1294 | SQ_PSN_SEARCH_OPCODE_MASK)); | |
1295 | swq->psn_search->flags_next_psn = cpu_to_le32( | |
1296 | ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & | |
1297 | SQ_PSN_SEARCH_NEXT_PSN_MASK)); | |
1298 | } | |
1299 | ||
1300 | sq->hwq.prod++; | |
3fb755b3 SK |
1301 | |
1302 | qp->wqe_cnt++; | |
1303 | ||
1ac5a404 SX |
1304 | done: |
1305 | return rc; | |
1306 | } | |
1307 | ||
1308 | void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) | |
1309 | { | |
1310 | struct bnxt_qplib_q *rq = &qp->rq; | |
1311 | struct dbr_dbr db_msg = { 0 }; | |
1312 | u32 sw_prod; | |
1313 | ||
1314 | sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
1315 | db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & | |
1316 | DBR_DBR_INDEX_MASK); | |
1317 | db_msg.type_xid = | |
1318 | cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | |
1319 | DBR_DBR_TYPE_RQ); | |
1320 | ||
1321 | /* Flush the writes to HW Rx WQE before the ringing Rx DB */ | |
1322 | wmb(); | |
1323 | __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); | |
1324 | } | |
1325 | ||
1326 | int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, | |
1327 | struct bnxt_qplib_swqe *wqe) | |
1328 | { | |
1329 | struct bnxt_qplib_q *rq = &qp->rq; | |
1330 | struct rq_wqe *rqe, **rqe_ptr; | |
1331 | struct sq_sge *hw_sge; | |
1332 | u32 sw_prod; | |
1333 | int i, rc = 0; | |
1334 | ||
1335 | if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
1336 | dev_err(&rq->hwq.pdev->dev, | |
1337 | "QPLIB: FP: QP (0x%x) is in the 0x%x state", | |
1338 | qp->id, qp->state); | |
1339 | rc = -EINVAL; | |
1340 | goto done; | |
1341 | } | |
9152e0b7 | 1342 | if (bnxt_qplib_queue_full(rq)) { |
1ac5a404 SX |
1343 | dev_err(&rq->hwq.pdev->dev, |
1344 | "QPLIB: FP: QP (0x%x) RQ is full!", qp->id); | |
1345 | rc = -EINVAL; | |
1346 | goto done; | |
1347 | } | |
1348 | sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
1349 | rq->swq[sw_prod].wr_id = wqe->wr_id; | |
1350 | ||
1351 | rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr; | |
1352 | rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)]; | |
1353 | ||
1354 | memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); | |
1355 | ||
1356 | /* Calculate wqe_size16 and data_len */ | |
1357 | for (i = 0, hw_sge = (struct sq_sge *)rqe->data; | |
1358 | i < wqe->num_sge; i++, hw_sge++) { | |
1359 | hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); | |
1360 | hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); | |
1361 | hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); | |
1362 | } | |
1363 | rqe->wqe_type = wqe->type; | |
1364 | rqe->flags = wqe->flags; | |
1365 | rqe->wqe_size = wqe->num_sge + | |
1366 | ((offsetof(typeof(*rqe), data) + 15) >> 4); | |
1367 | ||
1368 | /* Supply the rqe->wr_id index to the wr_id_tbl for now */ | |
1369 | rqe->wr_id[0] = cpu_to_le32(sw_prod); | |
1370 | ||
1371 | rq->hwq.prod++; | |
1372 | done: | |
1373 | return rc; | |
1374 | } | |
1375 | ||
1376 | /* CQ */ | |
1377 | ||
1378 | /* Spinlock must be held */ | |
1379 | static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq) | |
1380 | { | |
1381 | struct dbr_dbr db_msg = { 0 }; | |
1382 | ||
1383 | db_msg.type_xid = | |
1384 | cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | |
1385 | DBR_DBR_TYPE_CQ_ARMENA); | |
1386 | /* Flush memory writes before enabling the CQ */ | |
1387 | wmb(); | |
1388 | __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64)); | |
1389 | } | |
1390 | ||
1391 | static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type) | |
1392 | { | |
1393 | struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; | |
1394 | struct dbr_dbr db_msg = { 0 }; | |
1395 | u32 sw_cons; | |
1396 | ||
1397 | /* Ring DB */ | |
1398 | sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq); | |
1399 | db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) & | |
1400 | DBR_DBR_INDEX_MASK); | |
1401 | db_msg.type_xid = | |
1402 | cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | |
1403 | arm_type); | |
1404 | /* flush memory writes before arming the CQ */ | |
1405 | wmb(); | |
1406 | __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); | |
1407 | } | |
1408 | ||
1409 | int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) | |
1410 | { | |
1411 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
1412 | struct cmdq_create_cq req; | |
cc1ec769 | 1413 | struct creq_create_cq_resp resp; |
1ac5a404 SX |
1414 | struct bnxt_qplib_pbl *pbl; |
1415 | u16 cmd_flags = 0; | |
1416 | int rc; | |
1417 | ||
1418 | cq->hwq.max_elements = cq->max_wqe; | |
1419 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead, | |
1420 | cq->nmap, &cq->hwq.max_elements, | |
1421 | BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0, | |
1422 | PAGE_SIZE, HWQ_TYPE_QUEUE); | |
1423 | if (rc) | |
1424 | goto exit; | |
1425 | ||
1426 | RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags); | |
1427 | ||
1428 | if (!cq->dpi) { | |
1429 | dev_err(&rcfw->pdev->dev, | |
1430 | "QPLIB: FP: CREATE_CQ failed due to NULL DPI"); | |
1431 | return -EINVAL; | |
1432 | } | |
1433 | req.dpi = cpu_to_le32(cq->dpi->dpi); | |
1434 | req.cq_handle = cpu_to_le64(cq->cq_handle); | |
1435 | ||
1436 | req.cq_size = cpu_to_le32(cq->hwq.max_elements); | |
1437 | pbl = &cq->hwq.pbl[PBL_LVL_0]; | |
1438 | req.pg_size_lvl = cpu_to_le32( | |
1439 | ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << | |
1440 | CMDQ_CREATE_CQ_LVL_SFT) | | |
1441 | (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K : | |
1442 | pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K : | |
1443 | pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K : | |
1444 | pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M : | |
1445 | pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M : | |
1446 | pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G : | |
1447 | CMDQ_CREATE_CQ_PG_SIZE_PG_4K)); | |
1448 | ||
1449 | req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
1450 | ||
1451 | req.cq_fco_cnq_id = cpu_to_le32( | |
1452 | (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << | |
1453 | CMDQ_CREATE_CQ_CNQ_ID_SFT); | |
1454 | ||
cc1ec769 DS |
1455 | rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, |
1456 | (void *)&resp, NULL, 0); | |
1457 | if (rc) | |
1ac5a404 | 1458 | goto fail; |
cc1ec769 DS |
1459 | |
1460 | cq->id = le32_to_cpu(resp.xid); | |
1ac5a404 SX |
1461 | cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem; |
1462 | cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; | |
1463 | init_waitqueue_head(&cq->waitq); | |
1464 | ||
1465 | bnxt_qplib_arm_cq_enable(cq); | |
1466 | return 0; | |
1467 | ||
1468 | fail: | |
1469 | bnxt_qplib_free_hwq(res->pdev, &cq->hwq); | |
1470 | exit: | |
1471 | return rc; | |
1472 | } | |
1473 | ||
1474 | int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) | |
1475 | { | |
1476 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
1477 | struct cmdq_destroy_cq req; | |
cc1ec769 | 1478 | struct creq_destroy_cq_resp resp; |
1ac5a404 | 1479 | u16 cmd_flags = 0; |
cc1ec769 | 1480 | int rc; |
1ac5a404 SX |
1481 | |
1482 | RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags); | |
1483 | ||
1484 | req.cq_cid = cpu_to_le32(cq->id); | |
cc1ec769 DS |
1485 | rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, |
1486 | (void *)&resp, NULL, 0); | |
1487 | if (rc) | |
1488 | return rc; | |
1ac5a404 SX |
1489 | bnxt_qplib_free_hwq(res->pdev, &cq->hwq); |
1490 | return 0; | |
1491 | } | |
1492 | ||
1493 | static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, | |
1494 | struct bnxt_qplib_cqe **pcqe, int *budget) | |
1495 | { | |
1496 | u32 sw_prod, sw_cons; | |
1497 | struct bnxt_qplib_cqe *cqe; | |
1498 | int rc = 0; | |
1499 | ||
1500 | /* Now complete all outstanding SQEs with FLUSHED_ERR */ | |
1501 | sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
1502 | cqe = *pcqe; | |
1503 | while (*budget) { | |
1504 | sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | |
1505 | if (sw_cons == sw_prod) { | |
1506 | sq->flush_in_progress = false; | |
1507 | break; | |
1508 | } | |
1509 | memset(cqe, 0, sizeof(*cqe)); | |
1510 | cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; | |
1511 | cqe->opcode = CQ_BASE_CQE_TYPE_REQ; | |
1512 | cqe->qp_handle = (u64)(unsigned long)qp; | |
1513 | cqe->wr_id = sq->swq[sw_cons].wr_id; | |
1514 | cqe->src_qp = qp->id; | |
1515 | cqe->type = sq->swq[sw_cons].type; | |
1516 | cqe++; | |
1517 | (*budget)--; | |
1518 | sq->hwq.cons++; | |
1519 | } | |
1520 | *pcqe = cqe; | |
1521 | if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod) | |
1522 | /* Out of budget */ | |
1523 | rc = -EAGAIN; | |
1524 | ||
1525 | return rc; | |
1526 | } | |
1527 | ||
1528 | static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, | |
1529 | int opcode, struct bnxt_qplib_cqe **pcqe, int *budget) | |
1530 | { | |
1531 | struct bnxt_qplib_cqe *cqe; | |
1532 | u32 sw_prod, sw_cons; | |
1533 | int rc = 0; | |
1534 | ||
1535 | /* Flush the rest of the RQ */ | |
1536 | sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
1537 | cqe = *pcqe; | |
1538 | while (*budget) { | |
1539 | sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq); | |
1540 | if (sw_cons == sw_prod) | |
1541 | break; | |
1542 | memset(cqe, 0, sizeof(*cqe)); | |
1543 | cqe->status = | |
1544 | CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; | |
1545 | cqe->opcode = opcode; | |
1546 | cqe->qp_handle = (unsigned long)qp; | |
1547 | cqe->wr_id = rq->swq[sw_cons].wr_id; | |
1548 | cqe++; | |
1549 | (*budget)--; | |
1550 | rq->hwq.cons++; | |
1551 | } | |
1552 | *pcqe = cqe; | |
1553 | if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod) | |
1554 | /* Out of budget */ | |
1555 | rc = -EAGAIN; | |
1556 | ||
1557 | return rc; | |
1558 | } | |
1559 | ||
9152e0b7 EW |
1560 | /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive) |
1561 | * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 | |
1562 | */ | |
1563 | static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, | |
1564 | u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons) | |
1565 | { | |
1566 | struct bnxt_qplib_q *sq = &qp->sq; | |
1567 | struct bnxt_qplib_swq *swq; | |
1568 | u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx; | |
1569 | struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr; | |
1570 | struct cq_req *peek_req_hwcqe; | |
1571 | struct bnxt_qplib_qp *peek_qp; | |
1572 | struct bnxt_qplib_q *peek_sq; | |
1573 | int i, rc = 0; | |
1574 | ||
1575 | /* Normal mode */ | |
1576 | /* Check for the psn_search marking before completing */ | |
1577 | swq = &sq->swq[sw_sq_cons]; | |
1578 | if (swq->psn_search && | |
1579 | le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { | |
1580 | /* Unmark */ | |
1581 | swq->psn_search->flags_next_psn = cpu_to_le32 | |
1582 | (le32_to_cpu(swq->psn_search->flags_next_psn) | |
1583 | & ~0x80000000); | |
1584 | dev_dbg(&cq->hwq.pdev->dev, | |
1585 | "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", | |
1586 | cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); | |
1587 | sq->condition = true; | |
1588 | sq->send_phantom = true; | |
1589 | ||
1590 | /* TODO: Only ARM if the previous SQE is ARMALL */ | |
1591 | bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL); | |
1592 | ||
1593 | rc = -EAGAIN; | |
1594 | goto out; | |
1595 | } | |
1596 | if (sq->condition) { | |
1597 | /* Peek at the completions */ | |
1598 | peek_raw_cq_cons = cq->hwq.cons; | |
1599 | peek_sw_cq_cons = cq_cons; | |
1600 | i = cq->hwq.max_elements; | |
1601 | while (i--) { | |
1602 | peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq); | |
1603 | peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; | |
1604 | peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)] | |
1605 | [CQE_IDX(peek_sw_cq_cons)]; | |
1606 | /* If the next hwcqe is VALID */ | |
1607 | if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons, | |
1608 | cq->hwq.max_elements)) { | |
1609 | /* If the next hwcqe is a REQ */ | |
1610 | if ((peek_hwcqe->cqe_type_toggle & | |
1611 | CQ_BASE_CQE_TYPE_MASK) == | |
1612 | CQ_BASE_CQE_TYPE_REQ) { | |
1613 | peek_req_hwcqe = (struct cq_req *) | |
1614 | peek_hwcqe; | |
1615 | peek_qp = (struct bnxt_qplib_qp *) | |
1616 | ((unsigned long) | |
1617 | le64_to_cpu | |
1618 | (peek_req_hwcqe->qp_handle)); | |
1619 | peek_sq = &peek_qp->sq; | |
1620 | peek_sq_cons_idx = HWQ_CMP(le16_to_cpu( | |
1621 | peek_req_hwcqe->sq_cons_idx) - 1 | |
1622 | , &sq->hwq); | |
1623 | /* If the hwcqe's sq's wr_id matches */ | |
1624 | if (peek_sq == sq && | |
1625 | sq->swq[peek_sq_cons_idx].wr_id == | |
1626 | BNXT_QPLIB_FENCE_WRID) { | |
1627 | /* | |
1628 | * Unbreak only if the phantom | |
1629 | * comes back | |
1630 | */ | |
1631 | dev_dbg(&cq->hwq.pdev->dev, | |
1632 | "FP:Got Phantom CQE"); | |
1633 | sq->condition = false; | |
1634 | sq->single = true; | |
1635 | rc = 0; | |
1636 | goto out; | |
1637 | } | |
1638 | } | |
1639 | /* Valid but not the phantom, so keep looping */ | |
1640 | } else { | |
1641 | /* Not valid yet, just exit and wait */ | |
1642 | rc = -EINVAL; | |
1643 | goto out; | |
1644 | } | |
1645 | peek_sw_cq_cons++; | |
1646 | peek_raw_cq_cons++; | |
1647 | } | |
1648 | dev_err(&cq->hwq.pdev->dev, | |
1649 | "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x", | |
1650 | cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); | |
1651 | rc = -EINVAL; | |
1652 | } | |
1653 | out: | |
1654 | return rc; | |
1655 | } | |
1656 | ||
1ac5a404 SX |
1657 | static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, |
1658 | struct cq_req *hwcqe, | |
9152e0b7 EW |
1659 | struct bnxt_qplib_cqe **pcqe, int *budget, |
1660 | u32 cq_cons, struct bnxt_qplib_qp **lib_qp) | |
1ac5a404 SX |
1661 | { |
1662 | struct bnxt_qplib_qp *qp; | |
1663 | struct bnxt_qplib_q *sq; | |
1664 | struct bnxt_qplib_cqe *cqe; | |
9152e0b7 EW |
1665 | u32 sw_sq_cons, cqe_sq_cons; |
1666 | struct bnxt_qplib_swq *swq; | |
1ac5a404 SX |
1667 | int rc = 0; |
1668 | ||
1669 | qp = (struct bnxt_qplib_qp *)((unsigned long) | |
1670 | le64_to_cpu(hwcqe->qp_handle)); | |
1671 | if (!qp) { | |
1672 | dev_err(&cq->hwq.pdev->dev, | |
1673 | "QPLIB: FP: Process Req qp is NULL"); | |
1674 | return -EINVAL; | |
1675 | } | |
1676 | sq = &qp->sq; | |
1677 | ||
9152e0b7 EW |
1678 | cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq); |
1679 | if (cqe_sq_cons > sq->hwq.max_elements) { | |
1ac5a404 SX |
1680 | dev_err(&cq->hwq.pdev->dev, |
1681 | "QPLIB: FP: CQ Process req reported "); | |
1682 | dev_err(&cq->hwq.pdev->dev, | |
1683 | "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", | |
9152e0b7 | 1684 | cqe_sq_cons, sq->hwq.max_elements); |
1ac5a404 SX |
1685 | return -EINVAL; |
1686 | } | |
1687 | /* If we were in the middle of flushing the SQ, continue */ | |
1688 | if (sq->flush_in_progress) | |
1689 | goto flush; | |
1690 | ||
1691 | /* Require to walk the sq's swq to fabricate CQEs for all previously | |
1692 | * signaled SWQEs due to CQE aggregation from the current sq cons | |
9152e0b7 | 1693 | * to the cqe_sq_cons |
1ac5a404 SX |
1694 | */ |
1695 | cqe = *pcqe; | |
1696 | while (*budget) { | |
9152e0b7 EW |
1697 | sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); |
1698 | if (sw_sq_cons == cqe_sq_cons) | |
1699 | /* Done */ | |
1ac5a404 | 1700 | break; |
9152e0b7 EW |
1701 | |
1702 | swq = &sq->swq[sw_sq_cons]; | |
1ac5a404 SX |
1703 | memset(cqe, 0, sizeof(*cqe)); |
1704 | cqe->opcode = CQ_BASE_CQE_TYPE_REQ; | |
1705 | cqe->qp_handle = (u64)(unsigned long)qp; | |
1706 | cqe->src_qp = qp->id; | |
9152e0b7 EW |
1707 | cqe->wr_id = swq->wr_id; |
1708 | if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) | |
1709 | goto skip; | |
1710 | cqe->type = swq->type; | |
1ac5a404 SX |
1711 | |
1712 | /* For the last CQE, check for status. For errors, regardless | |
1713 | * of the request being signaled or not, it must complete with | |
1714 | * the hwcqe error status | |
1715 | */ | |
9152e0b7 | 1716 | if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons && |
1ac5a404 SX |
1717 | hwcqe->status != CQ_REQ_STATUS_OK) { |
1718 | cqe->status = hwcqe->status; | |
1719 | dev_err(&cq->hwq.pdev->dev, | |
1720 | "QPLIB: FP: CQ Processed Req "); | |
1721 | dev_err(&cq->hwq.pdev->dev, | |
1722 | "QPLIB: wr_id[%d] = 0x%llx with status 0x%x", | |
9152e0b7 | 1723 | sw_sq_cons, cqe->wr_id, cqe->status); |
1ac5a404 SX |
1724 | cqe++; |
1725 | (*budget)--; | |
1726 | sq->flush_in_progress = true; | |
1727 | /* Must block new posting of SQ and RQ */ | |
1728 | qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; | |
9152e0b7 EW |
1729 | sq->condition = false; |
1730 | sq->single = false; | |
1ac5a404 | 1731 | } else { |
9152e0b7 EW |
1732 | if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { |
1733 | /* Before we complete, do WA 9060 */ | |
1734 | if (do_wa9060(qp, cq, cq_cons, sw_sq_cons, | |
1735 | cqe_sq_cons)) { | |
1736 | *lib_qp = qp; | |
1737 | goto out; | |
1738 | } | |
1ac5a404 SX |
1739 | cqe->status = CQ_REQ_STATUS_OK; |
1740 | cqe++; | |
1741 | (*budget)--; | |
1742 | } | |
1743 | } | |
9152e0b7 | 1744 | skip: |
1ac5a404 | 1745 | sq->hwq.cons++; |
9152e0b7 EW |
1746 | if (sq->single) |
1747 | break; | |
1ac5a404 | 1748 | } |
9152e0b7 | 1749 | out: |
1ac5a404 | 1750 | *pcqe = cqe; |
9152e0b7 | 1751 | if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) { |
1ac5a404 SX |
1752 | /* Out of budget */ |
1753 | rc = -EAGAIN; | |
1754 | goto done; | |
1755 | } | |
9152e0b7 EW |
1756 | /* |
1757 | * Back to normal completion mode only after it has completed all of | |
1758 | * the WC for this CQE | |
1759 | */ | |
1760 | sq->single = false; | |
1ac5a404 SX |
1761 | if (!sq->flush_in_progress) |
1762 | goto done; | |
1763 | flush: | |
1764 | /* Require to walk the sq's swq to fabricate CQEs for all | |
1765 | * previously posted SWQEs due to the error CQE received | |
1766 | */ | |
1767 | rc = __flush_sq(sq, qp, pcqe, budget); | |
1768 | if (!rc) | |
1769 | sq->flush_in_progress = false; | |
1770 | done: | |
1771 | return rc; | |
1772 | } | |
1773 | ||
1774 | static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, | |
1775 | struct cq_res_rc *hwcqe, | |
1776 | struct bnxt_qplib_cqe **pcqe, | |
1777 | int *budget) | |
1778 | { | |
1779 | struct bnxt_qplib_qp *qp; | |
1780 | struct bnxt_qplib_q *rq; | |
1781 | struct bnxt_qplib_cqe *cqe; | |
1782 | u32 wr_id_idx; | |
1783 | int rc = 0; | |
1784 | ||
1785 | qp = (struct bnxt_qplib_qp *)((unsigned long) | |
1786 | le64_to_cpu(hwcqe->qp_handle)); | |
1787 | if (!qp) { | |
1788 | dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL"); | |
1789 | return -EINVAL; | |
1790 | } | |
1791 | cqe = *pcqe; | |
1792 | cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; | |
1793 | cqe->length = le32_to_cpu(hwcqe->length); | |
1794 | cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); | |
1795 | cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); | |
1796 | cqe->flags = le16_to_cpu(hwcqe->flags); | |
1797 | cqe->status = hwcqe->status; | |
1798 | cqe->qp_handle = (u64)(unsigned long)qp; | |
1799 | ||
1800 | wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & | |
1801 | CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; | |
1802 | rq = &qp->rq; | |
1803 | if (wr_id_idx > rq->hwq.max_elements) { | |
1804 | dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC "); | |
1805 | dev_err(&cq->hwq.pdev->dev, | |
1806 | "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x", | |
1807 | wr_id_idx, rq->hwq.max_elements); | |
1808 | return -EINVAL; | |
1809 | } | |
1810 | if (rq->flush_in_progress) | |
1811 | goto flush_rq; | |
1812 | ||
1813 | cqe->wr_id = rq->swq[wr_id_idx].wr_id; | |
1814 | cqe++; | |
1815 | (*budget)--; | |
1816 | rq->hwq.cons++; | |
1817 | *pcqe = cqe; | |
1818 | ||
1819 | if (hwcqe->status != CQ_RES_RC_STATUS_OK) { | |
1820 | rq->flush_in_progress = true; | |
1821 | flush_rq: | |
1822 | rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget); | |
1823 | if (!rc) | |
1824 | rq->flush_in_progress = false; | |
1825 | } | |
1826 | return rc; | |
1827 | } | |
1828 | ||
1829 | static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, | |
1830 | struct cq_res_ud *hwcqe, | |
1831 | struct bnxt_qplib_cqe **pcqe, | |
1832 | int *budget) | |
1833 | { | |
1834 | struct bnxt_qplib_qp *qp; | |
1835 | struct bnxt_qplib_q *rq; | |
1836 | struct bnxt_qplib_cqe *cqe; | |
1837 | u32 wr_id_idx; | |
1838 | int rc = 0; | |
1839 | ||
1840 | qp = (struct bnxt_qplib_qp *)((unsigned long) | |
1841 | le64_to_cpu(hwcqe->qp_handle)); | |
1842 | if (!qp) { | |
1843 | dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL"); | |
1844 | return -EINVAL; | |
1845 | } | |
1846 | cqe = *pcqe; | |
1847 | cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; | |
1848 | cqe->length = le32_to_cpu(hwcqe->length); | |
1849 | cqe->invrkey = le32_to_cpu(hwcqe->imm_data); | |
1850 | cqe->flags = le16_to_cpu(hwcqe->flags); | |
1851 | cqe->status = hwcqe->status; | |
1852 | cqe->qp_handle = (u64)(unsigned long)qp; | |
1853 | memcpy(cqe->smac, hwcqe->src_mac, 6); | |
1854 | wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) | |
1855 | & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK; | |
1856 | cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | | |
1857 | ((le32_to_cpu( | |
1858 | hwcqe->src_qp_high_srq_or_rq_wr_id) & | |
1859 | CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8); | |
1860 | ||
1861 | rq = &qp->rq; | |
1862 | if (wr_id_idx > rq->hwq.max_elements) { | |
1863 | dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD "); | |
1864 | dev_err(&cq->hwq.pdev->dev, | |
1865 | "QPLIB: wr_id idx %#x exceeded RQ max %#x", | |
1866 | wr_id_idx, rq->hwq.max_elements); | |
1867 | return -EINVAL; | |
1868 | } | |
1869 | if (rq->flush_in_progress) | |
1870 | goto flush_rq; | |
1871 | ||
1872 | cqe->wr_id = rq->swq[wr_id_idx].wr_id; | |
1873 | cqe++; | |
1874 | (*budget)--; | |
1875 | rq->hwq.cons++; | |
1876 | *pcqe = cqe; | |
1877 | ||
1878 | if (hwcqe->status != CQ_RES_RC_STATUS_OK) { | |
1879 | rq->flush_in_progress = true; | |
1880 | flush_rq: | |
1881 | rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget); | |
1882 | if (!rc) | |
1883 | rq->flush_in_progress = false; | |
1884 | } | |
1885 | return rc; | |
1886 | } | |
1887 | ||
1888 | static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, | |
1889 | struct cq_res_raweth_qp1 *hwcqe, | |
1890 | struct bnxt_qplib_cqe **pcqe, | |
1891 | int *budget) | |
1892 | { | |
1893 | struct bnxt_qplib_qp *qp; | |
1894 | struct bnxt_qplib_q *rq; | |
1895 | struct bnxt_qplib_cqe *cqe; | |
1896 | u32 wr_id_idx; | |
1897 | int rc = 0; | |
1898 | ||
1899 | qp = (struct bnxt_qplib_qp *)((unsigned long) | |
1900 | le64_to_cpu(hwcqe->qp_handle)); | |
1901 | if (!qp) { | |
1902 | dev_err(&cq->hwq.pdev->dev, | |
1903 | "QPLIB: process_cq Raw/QP1 qp is NULL"); | |
1904 | return -EINVAL; | |
1905 | } | |
1906 | cqe = *pcqe; | |
1907 | cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; | |
1908 | cqe->flags = le16_to_cpu(hwcqe->flags); | |
1909 | cqe->qp_handle = (u64)(unsigned long)qp; | |
1910 | ||
1911 | wr_id_idx = | |
1912 | le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) | |
1913 | & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; | |
1914 | cqe->src_qp = qp->id; | |
1915 | if (qp->id == 1 && !cqe->length) { | |
1916 | /* Add workaround for the length misdetection */ | |
1917 | cqe->length = 296; | |
1918 | } else { | |
1919 | cqe->length = le16_to_cpu(hwcqe->length); | |
1920 | } | |
1921 | cqe->pkey_index = qp->pkey_index; | |
1922 | memcpy(cqe->smac, qp->smac, 6); | |
1923 | ||
1924 | cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); | |
1925 | cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); | |
1926 | ||
1927 | rq = &qp->rq; | |
1928 | if (wr_id_idx > rq->hwq.max_elements) { | |
1929 | dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id "); | |
1930 | dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x", | |
1931 | wr_id_idx, rq->hwq.max_elements); | |
1932 | return -EINVAL; | |
1933 | } | |
1934 | if (rq->flush_in_progress) | |
1935 | goto flush_rq; | |
1936 | ||
1937 | cqe->wr_id = rq->swq[wr_id_idx].wr_id; | |
1938 | cqe++; | |
1939 | (*budget)--; | |
1940 | rq->hwq.cons++; | |
1941 | *pcqe = cqe; | |
1942 | ||
1943 | if (hwcqe->status != CQ_RES_RC_STATUS_OK) { | |
1944 | rq->flush_in_progress = true; | |
1945 | flush_rq: | |
1946 | rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe, | |
1947 | budget); | |
1948 | if (!rc) | |
1949 | rq->flush_in_progress = false; | |
1950 | } | |
1951 | return rc; | |
1952 | } | |
1953 | ||
1954 | static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, | |
1955 | struct cq_terminal *hwcqe, | |
1956 | struct bnxt_qplib_cqe **pcqe, | |
1957 | int *budget) | |
1958 | { | |
1959 | struct bnxt_qplib_qp *qp; | |
1960 | struct bnxt_qplib_q *sq, *rq; | |
1961 | struct bnxt_qplib_cqe *cqe; | |
1962 | u32 sw_cons = 0, cqe_cons; | |
1963 | int rc = 0; | |
1964 | u8 opcode = 0; | |
1965 | ||
1966 | /* Check the Status */ | |
1967 | if (hwcqe->status != CQ_TERMINAL_STATUS_OK) | |
1968 | dev_warn(&cq->hwq.pdev->dev, | |
1969 | "QPLIB: FP: CQ Process Terminal Error status = 0x%x", | |
1970 | hwcqe->status); | |
1971 | ||
1972 | qp = (struct bnxt_qplib_qp *)((unsigned long) | |
1973 | le64_to_cpu(hwcqe->qp_handle)); | |
1974 | if (!qp) { | |
1975 | dev_err(&cq->hwq.pdev->dev, | |
1976 | "QPLIB: FP: CQ Process terminal qp is NULL"); | |
1977 | return -EINVAL; | |
1978 | } | |
1979 | /* Must block new posting of SQ and RQ */ | |
1980 | qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; | |
1981 | ||
1982 | sq = &qp->sq; | |
1983 | rq = &qp->rq; | |
1984 | ||
1985 | cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); | |
1986 | if (cqe_cons == 0xFFFF) | |
1987 | goto do_rq; | |
1988 | ||
1989 | if (cqe_cons > sq->hwq.max_elements) { | |
1990 | dev_err(&cq->hwq.pdev->dev, | |
1991 | "QPLIB: FP: CQ Process terminal reported "); | |
1992 | dev_err(&cq->hwq.pdev->dev, | |
1993 | "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", | |
1994 | cqe_cons, sq->hwq.max_elements); | |
1995 | goto do_rq; | |
1996 | } | |
1997 | /* If we were in the middle of flushing, continue */ | |
1998 | if (sq->flush_in_progress) | |
1999 | goto flush_sq; | |
2000 | ||
2001 | /* Terminal CQE can also include aggregated successful CQEs prior. | |
2002 | * So we must complete all CQEs from the current sq's cons to the | |
2003 | * cq_cons with status OK | |
2004 | */ | |
2005 | cqe = *pcqe; | |
2006 | while (*budget) { | |
2007 | sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | |
2008 | if (sw_cons == cqe_cons) | |
2009 | break; | |
2010 | if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { | |
2011 | memset(cqe, 0, sizeof(*cqe)); | |
2012 | cqe->status = CQ_REQ_STATUS_OK; | |
2013 | cqe->opcode = CQ_BASE_CQE_TYPE_REQ; | |
2014 | cqe->qp_handle = (u64)(unsigned long)qp; | |
2015 | cqe->src_qp = qp->id; | |
2016 | cqe->wr_id = sq->swq[sw_cons].wr_id; | |
2017 | cqe->type = sq->swq[sw_cons].type; | |
2018 | cqe++; | |
2019 | (*budget)--; | |
2020 | } | |
2021 | sq->hwq.cons++; | |
2022 | } | |
2023 | *pcqe = cqe; | |
2024 | if (!(*budget) && sw_cons != cqe_cons) { | |
2025 | /* Out of budget */ | |
2026 | rc = -EAGAIN; | |
2027 | goto sq_done; | |
2028 | } | |
2029 | sq->flush_in_progress = true; | |
2030 | flush_sq: | |
2031 | rc = __flush_sq(sq, qp, pcqe, budget); | |
2032 | if (!rc) | |
2033 | sq->flush_in_progress = false; | |
2034 | sq_done: | |
2035 | if (rc) | |
2036 | return rc; | |
2037 | do_rq: | |
2038 | cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); | |
2039 | if (cqe_cons == 0xFFFF) { | |
2040 | goto done; | |
2041 | } else if (cqe_cons > rq->hwq.max_elements) { | |
2042 | dev_err(&cq->hwq.pdev->dev, | |
2043 | "QPLIB: FP: CQ Processed terminal "); | |
2044 | dev_err(&cq->hwq.pdev->dev, | |
2045 | "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x", | |
2046 | cqe_cons, rq->hwq.max_elements); | |
2047 | goto done; | |
2048 | } | |
2049 | /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR | |
2050 | * from the current rq->cons to the rq->prod regardless what the | |
2051 | * rq->cons the terminal CQE indicates | |
2052 | */ | |
2053 | rq->flush_in_progress = true; | |
2054 | switch (qp->type) { | |
2055 | case CMDQ_CREATE_QP1_TYPE_GSI: | |
2056 | opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; | |
2057 | break; | |
2058 | case CMDQ_CREATE_QP_TYPE_RC: | |
2059 | opcode = CQ_BASE_CQE_TYPE_RES_RC; | |
2060 | break; | |
2061 | case CMDQ_CREATE_QP_TYPE_UD: | |
2062 | opcode = CQ_BASE_CQE_TYPE_RES_UD; | |
2063 | break; | |
2064 | } | |
2065 | ||
2066 | rc = __flush_rq(rq, qp, opcode, pcqe, budget); | |
2067 | if (!rc) | |
2068 | rq->flush_in_progress = false; | |
2069 | done: | |
2070 | return rc; | |
2071 | } | |
2072 | ||
2073 | static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, | |
2074 | struct cq_cutoff *hwcqe) | |
2075 | { | |
2076 | /* Check the Status */ | |
2077 | if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { | |
2078 | dev_err(&cq->hwq.pdev->dev, | |
2079 | "QPLIB: FP: CQ Process Cutoff Error status = 0x%x", | |
2080 | hwcqe->status); | |
2081 | return -EINVAL; | |
2082 | } | |
2083 | clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); | |
2084 | wake_up_interruptible(&cq->waitq); | |
2085 | ||
2086 | return 0; | |
2087 | } | |
2088 | ||
2089 | int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, | |
9152e0b7 | 2090 | int num_cqes, struct bnxt_qplib_qp **lib_qp) |
1ac5a404 SX |
2091 | { |
2092 | struct cq_base *hw_cqe, **hw_cqe_ptr; | |
2093 | unsigned long flags; | |
2094 | u32 sw_cons, raw_cons; | |
2095 | int budget, rc = 0; | |
2096 | ||
2097 | spin_lock_irqsave(&cq->hwq.lock, flags); | |
2098 | raw_cons = cq->hwq.cons; | |
2099 | budget = num_cqes; | |
2100 | ||
2101 | while (budget) { | |
2102 | sw_cons = HWQ_CMP(raw_cons, &cq->hwq); | |
2103 | hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; | |
2104 | hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)]; | |
2105 | ||
2106 | /* Check for Valid bit */ | |
2107 | if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements)) | |
2108 | break; | |
2109 | ||
2110 | /* From the device's respective CQE format to qplib_wc*/ | |
2111 | switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { | |
2112 | case CQ_BASE_CQE_TYPE_REQ: | |
2113 | rc = bnxt_qplib_cq_process_req(cq, | |
2114 | (struct cq_req *)hw_cqe, | |
9152e0b7 EW |
2115 | &cqe, &budget, |
2116 | sw_cons, lib_qp); | |
1ac5a404 SX |
2117 | break; |
2118 | case CQ_BASE_CQE_TYPE_RES_RC: | |
2119 | rc = bnxt_qplib_cq_process_res_rc(cq, | |
2120 | (struct cq_res_rc *) | |
2121 | hw_cqe, &cqe, | |
2122 | &budget); | |
2123 | break; | |
2124 | case CQ_BASE_CQE_TYPE_RES_UD: | |
2125 | rc = bnxt_qplib_cq_process_res_ud | |
2126 | (cq, (struct cq_res_ud *)hw_cqe, &cqe, | |
2127 | &budget); | |
2128 | break; | |
2129 | case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: | |
2130 | rc = bnxt_qplib_cq_process_res_raweth_qp1 | |
2131 | (cq, (struct cq_res_raweth_qp1 *) | |
2132 | hw_cqe, &cqe, &budget); | |
2133 | break; | |
2134 | case CQ_BASE_CQE_TYPE_TERMINAL: | |
2135 | rc = bnxt_qplib_cq_process_terminal | |
2136 | (cq, (struct cq_terminal *)hw_cqe, | |
2137 | &cqe, &budget); | |
2138 | break; | |
2139 | case CQ_BASE_CQE_TYPE_CUT_OFF: | |
2140 | bnxt_qplib_cq_process_cutoff | |
2141 | (cq, (struct cq_cutoff *)hw_cqe); | |
2142 | /* Done processing this CQ */ | |
2143 | goto exit; | |
2144 | default: | |
2145 | dev_err(&cq->hwq.pdev->dev, | |
2146 | "QPLIB: process_cq unknown type 0x%lx", | |
2147 | hw_cqe->cqe_type_toggle & | |
2148 | CQ_BASE_CQE_TYPE_MASK); | |
2149 | rc = -EINVAL; | |
2150 | break; | |
2151 | } | |
2152 | if (rc < 0) { | |
2153 | if (rc == -EAGAIN) | |
2154 | break; | |
2155 | /* Error while processing the CQE, just skip to the | |
2156 | * next one | |
2157 | */ | |
2158 | dev_err(&cq->hwq.pdev->dev, | |
2159 | "QPLIB: process_cqe error rc = 0x%x", rc); | |
2160 | } | |
2161 | raw_cons++; | |
2162 | } | |
2163 | if (cq->hwq.cons != raw_cons) { | |
2164 | cq->hwq.cons = raw_cons; | |
2165 | bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ); | |
2166 | } | |
2167 | exit: | |
2168 | spin_unlock_irqrestore(&cq->hwq.lock, flags); | |
2169 | return num_cqes - budget; | |
2170 | } | |
2171 | ||
2172 | void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) | |
2173 | { | |
2174 | unsigned long flags; | |
2175 | ||
2176 | spin_lock_irqsave(&cq->hwq.lock, flags); | |
2177 | if (arm_type) | |
2178 | bnxt_qplib_arm_cq(cq, arm_type); | |
2179 | ||
2180 | spin_unlock_irqrestore(&cq->hwq.lock, flags); | |
2181 | } |