]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/infiniband/hw/bnxt_re/qplib_fp.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:


61764f7aa79b5eeb9515027dcf04e4d5cb09b76b
[mirror_ubuntu-bionic-kernel.git] / drivers / infiniband / hw / bnxt_re / qplib_fp.c
1 /*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Fast Path Operators
37 */
38
39 #include <linux/interrupt.h>
40 #include <linux/spinlock.h>
41 #include <linux/sched.h>
42 #include <linux/slab.h>
43 #include <linux/pci.h>
44 #include <linux/prefetch.h>
45
46 #include "roce_hsi.h"
47
48 #include "qplib_res.h"
49 #include "qplib_rcfw.h"
50 #include "qplib_sp.h"
51 #include "qplib_fp.h"
52
53 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
54 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
55
56 static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
57 {
58 qp->sq.condition = false;
59 qp->sq.send_phantom = false;
60 qp->sq.single = false;
61 }
62
63 /* Flush list */
64 static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
65 {
66 struct bnxt_qplib_cq *scq, *rcq;
67
68 scq = qp->scq;
69 rcq = qp->rcq;
70
71 if (!qp->sq.flushed) {
72 dev_dbg(&scq->hwq.pdev->dev,
73 "QPLIB: FP: Adding to SQ Flush list = %p",
74 qp);
75 bnxt_qplib_cancel_phantom_processing(qp);
76 list_add_tail(&qp->sq_flush, &scq->sqf_head);
77 qp->sq.flushed = true;
78 }
79 if (!qp->srq) {
80 if (!qp->rq.flushed) {
81 dev_dbg(&rcq->hwq.pdev->dev,
82 "QPLIB: FP: Adding to RQ Flush list = %p",
83 qp);
84 list_add_tail(&qp->rq_flush, &rcq->rqf_head);
85 qp->rq.flushed = true;
86 }
87 }
88 }
89
90 void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp,
91 unsigned long *flags)
92 __acquires(&qp->scq->hwq.lock) __acquires(&qp->rcq->hwq.lock)
93 {
94 spin_lock_irqsave(&qp->scq->hwq.lock, *flags);
95 if (qp->scq == qp->rcq)
96 __acquire(&qp->rcq->hwq.lock);
97 else
98 spin_lock(&qp->rcq->hwq.lock);
99 }
100
101 void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp,
102 unsigned long *flags)
103 __releases(&qp->scq->hwq.lock) __releases(&qp->rcq->hwq.lock)
104 {
105 if (qp->scq == qp->rcq)
106 __release(&qp->rcq->hwq.lock);
107 else
108 spin_unlock(&qp->rcq->hwq.lock);
109 spin_unlock_irqrestore(&qp->scq->hwq.lock, *flags);
110 }
111
112 static struct bnxt_qplib_cq *bnxt_qplib_find_buddy_cq(struct bnxt_qplib_qp *qp,
113 struct bnxt_qplib_cq *cq)
114 {
115 struct bnxt_qplib_cq *buddy_cq = NULL;
116
117 if (qp->scq == qp->rcq)
118 buddy_cq = NULL;
119 else if (qp->scq == cq)
120 buddy_cq = qp->rcq;
121 else
122 buddy_cq = qp->scq;
123 return buddy_cq;
124 }
125
126 static void bnxt_qplib_lock_buddy_cq(struct bnxt_qplib_qp *qp,
127 struct bnxt_qplib_cq *cq)
128 __acquires(&buddy_cq->hwq.lock)
129 {
130 struct bnxt_qplib_cq *buddy_cq = NULL;
131
132 buddy_cq = bnxt_qplib_find_buddy_cq(qp, cq);
133 if (!buddy_cq)
134 __acquire(&cq->hwq.lock);
135 else
136 spin_lock(&buddy_cq->hwq.lock);
137 }
138
139 static void bnxt_qplib_unlock_buddy_cq(struct bnxt_qplib_qp *qp,
140 struct bnxt_qplib_cq *cq)
141 __releases(&buddy_cq->hwq.lock)
142 {
143 struct bnxt_qplib_cq *buddy_cq = NULL;
144
145 buddy_cq = bnxt_qplib_find_buddy_cq(qp, cq);
146 if (!buddy_cq)
147 __release(&cq->hwq.lock);
148 else
149 spin_unlock(&buddy_cq->hwq.lock);
150 }
151
152 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
153 {
154 unsigned long flags;
155
156 bnxt_qplib_acquire_cq_locks(qp, &flags);
157 __bnxt_qplib_add_flush_qp(qp);
158 bnxt_qplib_release_cq_locks(qp, &flags);
159 }
160
161 static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
162 {
163 if (qp->sq.flushed) {
164 qp->sq.flushed = false;
165 list_del(&qp->sq_flush);
166 }
167 if (!qp->srq) {
168 if (qp->rq.flushed) {
169 qp->rq.flushed = false;
170 list_del(&qp->rq_flush);
171 }
172 }
173 }
174
175 void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
176 {
177 unsigned long flags;
178
179 bnxt_qplib_acquire_cq_locks(qp, &flags);
180 __clean_cq(qp->scq, (u64)(unsigned long)qp);
181 qp->sq.hwq.prod = 0;
182 qp->sq.hwq.cons = 0;
183 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
184 qp->rq.hwq.prod = 0;
185 qp->rq.hwq.cons = 0;
186
187 __bnxt_qplib_del_flush_qp(qp);
188 bnxt_qplib_release_cq_locks(qp, &flags);
189 }
190
191 static void bnxt_qpn_cqn_sched_task(struct work_struct *work)
192 {
193 struct bnxt_qplib_nq_work *nq_work =
194 container_of(work, struct bnxt_qplib_nq_work, work);
195
196 struct bnxt_qplib_cq *cq = nq_work->cq;
197 struct bnxt_qplib_nq *nq = nq_work->nq;
198
199 if (cq && nq) {
200 spin_lock_bh(&cq->compl_lock);
201 if (atomic_read(&cq->arm_state) && nq->cqn_handler) {
202 dev_dbg(&nq->pdev->dev,
203 "%s:Trigger cq = %p event nq = %p\n",
204 __func__, cq, nq);
205 nq->cqn_handler(nq, cq);
206 }
207 spin_unlock_bh(&cq->compl_lock);
208 }
209 kfree(nq_work);
210 }
211
212 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
213 struct bnxt_qplib_qp *qp)
214 {
215 struct bnxt_qplib_q *rq = &qp->rq;
216 struct bnxt_qplib_q *sq = &qp->sq;
217
218 if (qp->rq_hdr_buf)
219 dma_free_coherent(&res->pdev->dev,
220 rq->hwq.max_elements * qp->rq_hdr_buf_size,
221 qp->rq_hdr_buf, qp->rq_hdr_buf_map);
222 if (qp->sq_hdr_buf)
223 dma_free_coherent(&res->pdev->dev,
224 sq->hwq.max_elements * qp->sq_hdr_buf_size,
225 qp->sq_hdr_buf, qp->sq_hdr_buf_map);
226 qp->rq_hdr_buf = NULL;
227 qp->sq_hdr_buf = NULL;
228 qp->rq_hdr_buf_map = 0;
229 qp->sq_hdr_buf_map = 0;
230 qp->sq_hdr_buf_size = 0;
231 qp->rq_hdr_buf_size = 0;
232 }
233
234 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
235 struct bnxt_qplib_qp *qp)
236 {
237 struct bnxt_qplib_q *rq = &qp->rq;
238 struct bnxt_qplib_q *sq = &qp->rq;
239 int rc = 0;
240
241 if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
242 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
243 sq->hwq.max_elements *
244 qp->sq_hdr_buf_size,
245 &qp->sq_hdr_buf_map, GFP_KERNEL);
246 if (!qp->sq_hdr_buf) {
247 rc = -ENOMEM;
248 dev_err(&res->pdev->dev,
249 "QPLIB: Failed to create sq_hdr_buf");
250 goto fail;
251 }
252 }
253
254 if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
255 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
256 rq->hwq.max_elements *
257 qp->rq_hdr_buf_size,
258 &qp->rq_hdr_buf_map,
259 GFP_KERNEL);
260 if (!qp->rq_hdr_buf) {
261 rc = -ENOMEM;
262 dev_err(&res->pdev->dev,
263 "QPLIB: Failed to create rq_hdr_buf");
264 goto fail;
265 }
266 }
267 return 0;
268
269 fail:
270 bnxt_qplib_free_qp_hdr_buf(res, qp);
271 return rc;
272 }
273
274 static void bnxt_qplib_service_nq(unsigned long data)
275 {
276 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
277 struct bnxt_qplib_hwq *hwq = &nq->hwq;
278 struct nq_base *nqe, **nq_ptr;
279 struct bnxt_qplib_cq *cq;
280 int num_cqne_processed = 0;
281 u32 sw_cons, raw_cons;
282 u16 type;
283 int budget = nq->budget;
284 u64 q_handle;
285
286 /* Service the NQ until empty */
287 raw_cons = hwq->cons;
288 while (budget--) {
289 sw_cons = HWQ_CMP(raw_cons, hwq);
290 nq_ptr = (struct nq_base **)hwq->pbl_ptr;
291 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
292 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
293 break;
294
295 /*
296 * The valid test of the entry must be done first before
297 * reading any further.
298 */
299 dma_rmb();
300
301 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
302 switch (type) {
303 case NQ_BASE_TYPE_CQ_NOTIFICATION:
304 {
305 struct nq_cn *nqcne = (struct nq_cn *)nqe;
306
307 q_handle = le32_to_cpu(nqcne->cq_handle_low);
308 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
309 << 32;
310 cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle;
311 bnxt_qplib_arm_cq_enable(cq);
312 spin_lock_bh(&cq->compl_lock);
313 atomic_set(&cq->arm_state, 0);
314 if (!nq->cqn_handler(nq, (cq)))
315 num_cqne_processed++;
316 else
317 dev_warn(&nq->pdev->dev,
318 "QPLIB: cqn - type 0x%x not handled",
319 type);
320 spin_unlock_bh(&cq->compl_lock);
321 break;
322 }
323 case NQ_BASE_TYPE_DBQ_EVENT:
324 break;
325 default:
326 dev_warn(&nq->pdev->dev,
327 "QPLIB: nqe with type = 0x%x not handled",
328 type);
329 break;
330 }
331 raw_cons++;
332 }
333 if (hwq->cons != raw_cons) {
334 hwq->cons = raw_cons;
335 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
336 }
337 }
338
339 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
340 {
341 struct bnxt_qplib_nq *nq = dev_instance;
342 struct bnxt_qplib_hwq *hwq = &nq->hwq;
343 struct nq_base **nq_ptr;
344 u32 sw_cons;
345
346 /* Prefetch the NQ element */
347 sw_cons = HWQ_CMP(hwq->cons, hwq);
348 nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
349 prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
350
351 /* Fan out to CPU affinitized kthreads? */
352 tasklet_schedule(&nq->worker);
353
354 return IRQ_HANDLED;
355 }
356
357 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
358 {
359 if (nq->cqn_wq) {
360 destroy_workqueue(nq->cqn_wq);
361 nq->cqn_wq = NULL;
362 }
363 /* Make sure the HW is stopped! */
364 synchronize_irq(nq->vector);
365 tasklet_disable(&nq->worker);
366 tasklet_kill(&nq->worker);
367
368 if (nq->requested) {
369 irq_set_affinity_hint(nq->vector, NULL);
370 free_irq(nq->vector, nq);
371 nq->requested = false;
372 }
373 if (nq->bar_reg_iomem)
374 iounmap(nq->bar_reg_iomem);
375 nq->bar_reg_iomem = NULL;
376
377 nq->cqn_handler = NULL;
378 nq->srqn_handler = NULL;
379 nq->vector = 0;
380 }
381
382 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
383 int nq_idx, int msix_vector, int bar_reg_offset,
384 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
385 struct bnxt_qplib_cq *),
386 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
387 void *, u8 event))
388 {
389 resource_size_t nq_base;
390 int rc = -1;
391
392 nq->pdev = pdev;
393 nq->vector = msix_vector;
394
395 nq->cqn_handler = cqn_handler;
396
397 nq->srqn_handler = srqn_handler;
398
399 tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
400
401 /* Have a task to schedule CQ notifiers in post send case */
402 nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq");
403 if (!nq->cqn_wq)
404 goto fail;
405
406 nq->requested = false;
407 memset(nq->name, 0, 32);
408 sprintf(nq->name, "bnxt_qplib_nq-%d", nq_idx);
409 rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq);
410 if (rc) {
411 dev_err(&nq->pdev->dev,
412 "Failed to request IRQ for NQ: %#x", rc);
413 bnxt_qplib_disable_nq(nq);
414 goto fail;
415 }
416
417 cpumask_clear(&nq->mask);
418 cpumask_set_cpu(nq_idx, &nq->mask);
419 rc = irq_set_affinity_hint(nq->vector, &nq->mask);
420 if (rc) {
421 dev_warn(&nq->pdev->dev,
422 "QPLIB: set affinity failed; vector: %d nq_idx: %d\n",
423 nq->vector, nq_idx);
424 }
425
426 nq->requested = true;
427 nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
428 nq->bar_reg_off = bar_reg_offset;
429 nq_base = pci_resource_start(pdev, nq->bar_reg);
430 if (!nq_base) {
431 rc = -ENOMEM;
432 goto fail;
433 }
434 nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
435 if (!nq->bar_reg_iomem) {
436 rc = -ENOMEM;
437 goto fail;
438 }
439 NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
440
441 return 0;
442 fail:
443 bnxt_qplib_disable_nq(nq);
444 return rc;
445 }
446
447 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
448 {
449 if (nq->hwq.max_elements) {
450 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
451 nq->hwq.max_elements = 0;
452 }
453 }
454
455 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
456 {
457 nq->pdev = pdev;
458 if (!nq->hwq.max_elements ||
459 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
460 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
461
462 if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
463 &nq->hwq.max_elements,
464 BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
465 PAGE_SIZE, HWQ_TYPE_L2_CMPL))
466 return -ENOMEM;
467
468 nq->budget = 8;
469 return 0;
470 }
471
472 /* QP */
473 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
474 {
475 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
476 struct cmdq_create_qp1 req;
477 struct creq_create_qp1_resp resp;
478 struct bnxt_qplib_pbl *pbl;
479 struct bnxt_qplib_q *sq = &qp->sq;
480 struct bnxt_qplib_q *rq = &qp->rq;
481 int rc;
482 u16 cmd_flags = 0;
483 u32 qp_flags = 0;
484
485 RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
486
487 /* General */
488 req.type = qp->type;
489 req.dpi = cpu_to_le32(qp->dpi->dpi);
490 req.qp_handle = cpu_to_le64(qp->qp_handle);
491
492 /* SQ */
493 sq->hwq.max_elements = sq->max_wqe;
494 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
495 &sq->hwq.max_elements,
496 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
497 PAGE_SIZE, HWQ_TYPE_QUEUE);
498 if (rc)
499 goto exit;
500
501 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
502 if (!sq->swq) {
503 rc = -ENOMEM;
504 goto fail_sq;
505 }
506 pbl = &sq->hwq.pbl[PBL_LVL_0];
507 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
508 req.sq_pg_size_sq_lvl =
509 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
510 << CMDQ_CREATE_QP1_SQ_LVL_SFT) |
511 (pbl->pg_size == ROCE_PG_SIZE_4K ?
512 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
513 pbl->pg_size == ROCE_PG_SIZE_8K ?
514 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
515 pbl->pg_size == ROCE_PG_SIZE_64K ?
516 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
517 pbl->pg_size == ROCE_PG_SIZE_2M ?
518 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
519 pbl->pg_size == ROCE_PG_SIZE_8M ?
520 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
521 pbl->pg_size == ROCE_PG_SIZE_1G ?
522 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
523 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
524
525 if (qp->scq)
526 req.scq_cid = cpu_to_le32(qp->scq->id);
527
528 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
529
530 /* RQ */
531 if (rq->max_wqe) {
532 rq->hwq.max_elements = qp->rq.max_wqe;
533 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
534 &rq->hwq.max_elements,
535 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
536 PAGE_SIZE, HWQ_TYPE_QUEUE);
537 if (rc)
538 goto fail_sq;
539
540 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
541 GFP_KERNEL);
542 if (!rq->swq) {
543 rc = -ENOMEM;
544 goto fail_rq;
545 }
546 pbl = &rq->hwq.pbl[PBL_LVL_0];
547 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
548 req.rq_pg_size_rq_lvl =
549 ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
550 CMDQ_CREATE_QP1_RQ_LVL_SFT) |
551 (pbl->pg_size == ROCE_PG_SIZE_4K ?
552 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
553 pbl->pg_size == ROCE_PG_SIZE_8K ?
554 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
555 pbl->pg_size == ROCE_PG_SIZE_64K ?
556 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
557 pbl->pg_size == ROCE_PG_SIZE_2M ?
558 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
559 pbl->pg_size == ROCE_PG_SIZE_8M ?
560 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
561 pbl->pg_size == ROCE_PG_SIZE_1G ?
562 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
563 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
564 if (qp->rcq)
565 req.rcq_cid = cpu_to_le32(qp->rcq->id);
566 }
567
568 /* Header buffer - allow hdr_buf pass in */
569 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
570 if (rc) {
571 rc = -ENOMEM;
572 goto fail;
573 }
574 req.qp_flags = cpu_to_le32(qp_flags);
575 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
576 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
577
578 req.sq_fwo_sq_sge =
579 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
580 CMDQ_CREATE_QP1_SQ_SGE_SFT);
581 req.rq_fwo_rq_sge =
582 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
583 CMDQ_CREATE_QP1_RQ_SGE_SFT);
584
585 req.pd_id = cpu_to_le32(qp->pd->id);
586
587 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
588 (void *)&resp, NULL, 0);
589 if (rc)
590 goto fail;
591
592 qp->id = le32_to_cpu(resp.xid);
593 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
594 rcfw->qp_tbl[qp->id].qp_id = qp->id;
595 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
596
597 return 0;
598
599 fail:
600 bnxt_qplib_free_qp_hdr_buf(res, qp);
601 fail_rq:
602 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
603 kfree(rq->swq);
604 fail_sq:
605 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
606 kfree(sq->swq);
607 exit:
608 return rc;
609 }
610
611 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
612 {
613 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
614 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
615 struct cmdq_create_qp req;
616 struct creq_create_qp_resp resp;
617 struct bnxt_qplib_pbl *pbl;
618 struct sq_psn_search **psn_search_ptr;
619 unsigned long int psn_search, poff = 0;
620 struct bnxt_qplib_q *sq = &qp->sq;
621 struct bnxt_qplib_q *rq = &qp->rq;
622 struct bnxt_qplib_hwq *xrrq;
623 int i, rc, req_size, psn_sz;
624 u16 cmd_flags = 0, max_ssge;
625 u32 sw_prod, qp_flags = 0;
626
627 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
628
629 /* General */
630 req.type = qp->type;
631 req.dpi = cpu_to_le32(qp->dpi->dpi);
632 req.qp_handle = cpu_to_le64(qp->qp_handle);
633
634 /* SQ */
635 psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
636 sizeof(struct sq_psn_search) : 0;
637 sq->hwq.max_elements = sq->max_wqe;
638 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
639 sq->nmap, &sq->hwq.max_elements,
640 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
641 psn_sz,
642 PAGE_SIZE, HWQ_TYPE_QUEUE);
643 if (rc)
644 goto exit;
645
646 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
647 if (!sq->swq) {
648 rc = -ENOMEM;
649 goto fail_sq;
650 }
651 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
652 if (psn_sz) {
653 psn_search_ptr = (struct sq_psn_search **)
654 &hw_sq_send_ptr[get_sqe_pg
655 (sq->hwq.max_elements)];
656 psn_search = (unsigned long int)
657 &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
658 [get_sqe_idx(sq->hwq.max_elements)];
659 if (psn_search & ~PAGE_MASK) {
660 /* If the psn_search does not start on a page boundary,
661 * then calculate the offset
662 */
663 poff = (psn_search & ~PAGE_MASK) /
664 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
665 }
666 for (i = 0; i < sq->hwq.max_elements; i++)
667 sq->swq[i].psn_search =
668 &psn_search_ptr[get_psne_pg(i + poff)]
669 [get_psne_idx(i + poff)];
670 }
671 pbl = &sq->hwq.pbl[PBL_LVL_0];
672 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
673 req.sq_pg_size_sq_lvl =
674 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
675 << CMDQ_CREATE_QP_SQ_LVL_SFT) |
676 (pbl->pg_size == ROCE_PG_SIZE_4K ?
677 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
678 pbl->pg_size == ROCE_PG_SIZE_8K ?
679 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
680 pbl->pg_size == ROCE_PG_SIZE_64K ?
681 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
682 pbl->pg_size == ROCE_PG_SIZE_2M ?
683 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
684 pbl->pg_size == ROCE_PG_SIZE_8M ?
685 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
686 pbl->pg_size == ROCE_PG_SIZE_1G ?
687 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
688 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
689
690 /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
691 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
692 for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
693 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
694 [get_sqe_idx(sw_prod)];
695 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
696 }
697
698 if (qp->scq)
699 req.scq_cid = cpu_to_le32(qp->scq->id);
700
701 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
702 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
703 if (qp->sig_type)
704 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
705
706 /* RQ */
707 if (rq->max_wqe) {
708 rq->hwq.max_elements = rq->max_wqe;
709 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
710 rq->nmap, &rq->hwq.max_elements,
711 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
712 PAGE_SIZE, HWQ_TYPE_QUEUE);
713 if (rc)
714 goto fail_sq;
715
716 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
717 GFP_KERNEL);
718 if (!rq->swq) {
719 rc = -ENOMEM;
720 goto fail_rq;
721 }
722 pbl = &rq->hwq.pbl[PBL_LVL_0];
723 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
724 req.rq_pg_size_rq_lvl =
725 ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
726 CMDQ_CREATE_QP_RQ_LVL_SFT) |
727 (pbl->pg_size == ROCE_PG_SIZE_4K ?
728 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
729 pbl->pg_size == ROCE_PG_SIZE_8K ?
730 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
731 pbl->pg_size == ROCE_PG_SIZE_64K ?
732 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
733 pbl->pg_size == ROCE_PG_SIZE_2M ?
734 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
735 pbl->pg_size == ROCE_PG_SIZE_8M ?
736 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
737 pbl->pg_size == ROCE_PG_SIZE_1G ?
738 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
739 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
740 }
741
742 if (qp->rcq)
743 req.rcq_cid = cpu_to_le32(qp->rcq->id);
744 req.qp_flags = cpu_to_le32(qp_flags);
745 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
746 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
747 qp->sq_hdr_buf = NULL;
748 qp->rq_hdr_buf = NULL;
749
750 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
751 if (rc)
752 goto fail_rq;
753
754 /* CTRL-22434: Irrespective of the requested SGE count on the SQ
755 * always create the QP with max send sges possible if the requested
756 * inline size is greater than 0.
757 */
758 max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
759 req.sq_fwo_sq_sge = cpu_to_le16(
760 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
761 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
762 req.rq_fwo_rq_sge = cpu_to_le16(
763 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
764 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
765 /* ORRQ and IRRQ */
766 if (psn_sz) {
767 xrrq = &qp->orrq;
768 xrrq->max_elements =
769 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
770 req_size = xrrq->max_elements *
771 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
772 req_size &= ~(PAGE_SIZE - 1);
773 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
774 &xrrq->max_elements,
775 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
776 0, req_size, HWQ_TYPE_CTX);
777 if (rc)
778 goto fail_buf_free;
779 pbl = &xrrq->pbl[PBL_LVL_0];
780 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
781
782 xrrq = &qp->irrq;
783 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
784 qp->max_dest_rd_atomic);
785 req_size = xrrq->max_elements *
786 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
787 req_size &= ~(PAGE_SIZE - 1);
788
789 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
790 &xrrq->max_elements,
791 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
792 0, req_size, HWQ_TYPE_CTX);
793 if (rc)
794 goto fail_orrq;
795
796 pbl = &xrrq->pbl[PBL_LVL_0];
797 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
798 }
799 req.pd_id = cpu_to_le32(qp->pd->id);
800
801 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
802 (void *)&resp, NULL, 0);
803 if (rc)
804 goto fail;
805
806 qp->id = le32_to_cpu(resp.xid);
807 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
808 INIT_LIST_HEAD(&qp->sq_flush);
809 INIT_LIST_HEAD(&qp->rq_flush);
810 rcfw->qp_tbl[qp->id].qp_id = qp->id;
811 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
812
813 return 0;
814
815 fail:
816 if (qp->irrq.max_elements)
817 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
818 fail_orrq:
819 if (qp->orrq.max_elements)
820 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
821 fail_buf_free:
822 bnxt_qplib_free_qp_hdr_buf(res, qp);
823 fail_rq:
824 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
825 kfree(rq->swq);
826 fail_sq:
827 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
828 kfree(sq->swq);
829 exit:
830 return rc;
831 }
832
833 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
834 {
835 switch (qp->state) {
836 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
837 /* INIT->RTR, configure the path_mtu to the default
838 * 2048 if not being requested
839 */
840 if (!(qp->modify_flags &
841 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
842 qp->modify_flags |=
843 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
844 qp->path_mtu =
845 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
846 }
847 qp->modify_flags &=
848 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
849 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
850 if (qp->max_dest_rd_atomic < 1)
851 qp->max_dest_rd_atomic = 1;
852 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
853 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
854 if (!(qp->modify_flags &
855 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
856 qp->modify_flags |=
857 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
858 qp->ah.sgid_index = 0;
859 }
860 break;
861 default:
862 break;
863 }
864 }
865
866 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
867 {
868 switch (qp->state) {
869 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
870 /* Bono FW requires the max_rd_atomic to be >= 1 */
871 if (qp->max_rd_atomic < 1)
872 qp->max_rd_atomic = 1;
873 /* Bono FW does not allow PKEY_INDEX,
874 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
875 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
876 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
877 * modification
878 */
879 qp->modify_flags &=
880 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
881 CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
882 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
883 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
884 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
885 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
886 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
887 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
888 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
889 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
890 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
891 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
892 break;
893 default:
894 break;
895 }
896 }
897
898 static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
899 {
900 switch (qp->cur_qp_state) {
901 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
902 break;
903 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
904 __modify_flags_from_init_state(qp);
905 break;
906 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
907 __modify_flags_from_rtr_state(qp);
908 break;
909 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
910 break;
911 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
912 break;
913 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
914 break;
915 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
916 break;
917 default:
918 break;
919 }
920 }
921
922 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
923 {
924 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
925 struct cmdq_modify_qp req;
926 struct creq_modify_qp_resp resp;
927 u16 cmd_flags = 0, pkey;
928 u32 temp32[4];
929 u32 bmask;
930 int rc;
931
932 RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
933
934 /* Filter out the qp_attr_mask based on the state->new transition */
935 __filter_modify_flags(qp);
936 bmask = qp->modify_flags;
937 req.modify_mask = cpu_to_le32(qp->modify_flags);
938 req.qp_cid = cpu_to_le32(qp->id);
939 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
940 req.network_type_en_sqd_async_notify_new_state =
941 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
942 (qp->en_sqd_async_notify ?
943 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
944 }
945 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
946
947 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
948 req.access = qp->access;
949
950 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
951 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
952 qp->pkey_index, &pkey))
953 req.pkey = cpu_to_le16(pkey);
954 }
955 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
956 req.qkey = cpu_to_le32(qp->qkey);
957
958 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
959 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
960 req.dgid[0] = cpu_to_le32(temp32[0]);
961 req.dgid[1] = cpu_to_le32(temp32[1]);
962 req.dgid[2] = cpu_to_le32(temp32[2]);
963 req.dgid[3] = cpu_to_le32(temp32[3]);
964 }
965 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
966 req.flow_label = cpu_to_le32(qp->ah.flow_label);
967
968 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
969 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
970 [qp->ah.sgid_index]);
971
972 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
973 req.hop_limit = qp->ah.hop_limit;
974
975 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
976 req.traffic_class = qp->ah.traffic_class;
977
978 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
979 memcpy(req.dest_mac, qp->ah.dmac, 6);
980
981 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
982 req.path_mtu = qp->path_mtu;
983
984 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
985 req.timeout = qp->timeout;
986
987 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
988 req.retry_cnt = qp->retry_cnt;
989
990 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
991 req.rnr_retry = qp->rnr_retry;
992
993 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
994 req.min_rnr_timer = qp->min_rnr_timer;
995
996 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
997 req.rq_psn = cpu_to_le32(qp->rq.psn);
998
999 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
1000 req.sq_psn = cpu_to_le32(qp->sq.psn);
1001
1002 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
1003 req.max_rd_atomic =
1004 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
1005
1006 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
1007 req.max_dest_rd_atomic =
1008 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
1009
1010 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
1011 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
1012 req.sq_sge = cpu_to_le16(qp->sq.max_sge);
1013 req.rq_sge = cpu_to_le16(qp->rq.max_sge);
1014 req.max_inline_data = cpu_to_le32(qp->max_inline_data);
1015 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
1016 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
1017
1018 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
1019
1020 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1021 (void *)&resp, NULL, 0);
1022 if (rc)
1023 return rc;
1024 qp->cur_qp_state = qp->state;
1025 return 0;
1026 }
1027
1028 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1029 {
1030 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1031 struct cmdq_query_qp req;
1032 struct creq_query_qp_resp resp;
1033 struct bnxt_qplib_rcfw_sbuf *sbuf;
1034 struct creq_query_qp_resp_sb *sb;
1035 u16 cmd_flags = 0;
1036 u32 temp32[4];
1037 int i, rc = 0;
1038
1039 RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
1040
1041 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
1042 if (!sbuf)
1043 return -ENOMEM;
1044 sb = sbuf->sb;
1045
1046 req.qp_cid = cpu_to_le32(qp->id);
1047 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
1048 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
1049 (void *)sbuf, 0);
1050 if (rc)
1051 goto bail;
1052 /* Extract the context from the side buffer */
1053 qp->state = sb->en_sqd_async_notify_state &
1054 CREQ_QUERY_QP_RESP_SB_STATE_MASK;
1055 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
1056 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
1057 true : false;
1058 qp->access = sb->access;
1059 qp->pkey_index = le16_to_cpu(sb->pkey);
1060 qp->qkey = le32_to_cpu(sb->qkey);
1061
1062 temp32[0] = le32_to_cpu(sb->dgid[0]);
1063 temp32[1] = le32_to_cpu(sb->dgid[1]);
1064 temp32[2] = le32_to_cpu(sb->dgid[2]);
1065 temp32[3] = le32_to_cpu(sb->dgid[3]);
1066 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
1067
1068 qp->ah.flow_label = le32_to_cpu(sb->flow_label);
1069
1070 qp->ah.sgid_index = 0;
1071 for (i = 0; i < res->sgid_tbl.max; i++) {
1072 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
1073 qp->ah.sgid_index = i;
1074 break;
1075 }
1076 }
1077 if (i == res->sgid_tbl.max)
1078 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
1079
1080 qp->ah.hop_limit = sb->hop_limit;
1081 qp->ah.traffic_class = sb->traffic_class;
1082 memcpy(qp->ah.dmac, sb->dest_mac, 6);
1083 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1084 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
1085 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
1086 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1087 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
1088 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
1089 qp->timeout = sb->timeout;
1090 qp->retry_cnt = sb->retry_cnt;
1091 qp->rnr_retry = sb->rnr_retry;
1092 qp->min_rnr_timer = sb->min_rnr_timer;
1093 qp->rq.psn = le32_to_cpu(sb->rq_psn);
1094 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
1095 qp->sq.psn = le32_to_cpu(sb->sq_psn);
1096 qp->max_dest_rd_atomic =
1097 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
1098 qp->sq.max_wqe = qp->sq.hwq.max_elements;
1099 qp->rq.max_wqe = qp->rq.hwq.max_elements;
1100 qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
1101 qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
1102 qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
1103 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
1104 memcpy(qp->smac, sb->src_mac, 6);
1105 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
1106 bail:
1107 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
1108 return rc;
1109 }
1110
1111 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
1112 {
1113 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1114 struct cq_base *hw_cqe, **hw_cqe_ptr;
1115 int i;
1116
1117 for (i = 0; i < cq_hwq->max_elements; i++) {
1118 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
1119 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
1120 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
1121 continue;
1122 /*
1123 * The valid test of the entry must be done first before
1124 * reading any further.
1125 */
1126 dma_rmb();
1127 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
1128 case CQ_BASE_CQE_TYPE_REQ:
1129 case CQ_BASE_CQE_TYPE_TERMINAL:
1130 {
1131 struct cq_req *cqe = (struct cq_req *)hw_cqe;
1132
1133 if (qp == le64_to_cpu(cqe->qp_handle))
1134 cqe->qp_handle = 0;
1135 break;
1136 }
1137 case CQ_BASE_CQE_TYPE_RES_RC:
1138 case CQ_BASE_CQE_TYPE_RES_UD:
1139 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
1140 {
1141 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
1142
1143 if (qp == le64_to_cpu(cqe->qp_handle))
1144 cqe->qp_handle = 0;
1145 break;
1146 }
1147 default:
1148 break;
1149 }
1150 }
1151 }
1152
1153 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
1154 struct bnxt_qplib_qp *qp)
1155 {
1156 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1157 struct cmdq_destroy_qp req;
1158 struct creq_destroy_qp_resp resp;
1159 unsigned long flags;
1160 u16 cmd_flags = 0;
1161 int rc;
1162
1163 rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID;
1164 rcfw->qp_tbl[qp->id].qp_handle = NULL;
1165
1166 RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1167
1168 req.qp_cid = cpu_to_le32(qp->id);
1169 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1170 (void *)&resp, NULL, 0);
1171 if (rc) {
1172 rcfw->qp_tbl[qp->id].qp_id = qp->id;
1173 rcfw->qp_tbl[qp->id].qp_handle = qp;
1174 return rc;
1175 }
1176
1177 /* Must walk the associated CQs to nullified the QP ptr */
1178 spin_lock_irqsave(&qp->scq->hwq.lock, flags);
1179
1180 __clean_cq(qp->scq, (u64)(unsigned long)qp);
1181
1182 if (qp->rcq && qp->rcq != qp->scq) {
1183 spin_lock(&qp->rcq->hwq.lock);
1184 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
1185 spin_unlock(&qp->rcq->hwq.lock);
1186 }
1187
1188 spin_unlock_irqrestore(&qp->scq->hwq.lock, flags);
1189
1190 bnxt_qplib_free_qp_hdr_buf(res, qp);
1191 bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
1192 kfree(qp->sq.swq);
1193
1194 bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
1195 kfree(qp->rq.swq);
1196
1197 if (qp->irrq.max_elements)
1198 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
1199 if (qp->orrq.max_elements)
1200 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1201
1202 return 0;
1203 }
1204
1205 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1206 struct bnxt_qplib_sge *sge)
1207 {
1208 struct bnxt_qplib_q *sq = &qp->sq;
1209 u32 sw_prod;
1210
1211 memset(sge, 0, sizeof(*sge));
1212
1213 if (qp->sq_hdr_buf) {
1214 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1215 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1216 sw_prod * qp->sq_hdr_buf_size);
1217 sge->lkey = 0xFFFFFFFF;
1218 sge->size = qp->sq_hdr_buf_size;
1219 return qp->sq_hdr_buf + sw_prod * sge->size;
1220 }
1221 return NULL;
1222 }
1223
1224 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1225 {
1226 struct bnxt_qplib_q *rq = &qp->rq;
1227
1228 return HWQ_CMP(rq->hwq.prod, &rq->hwq);
1229 }
1230
1231 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1232 {
1233 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1234 }
1235
1236 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1237 struct bnxt_qplib_sge *sge)
1238 {
1239 struct bnxt_qplib_q *rq = &qp->rq;
1240 u32 sw_prod;
1241
1242 memset(sge, 0, sizeof(*sge));
1243
1244 if (qp->rq_hdr_buf) {
1245 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1246 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1247 sw_prod * qp->rq_hdr_buf_size);
1248 sge->lkey = 0xFFFFFFFF;
1249 sge->size = qp->rq_hdr_buf_size;
1250 return qp->rq_hdr_buf + sw_prod * sge->size;
1251 }
1252 return NULL;
1253 }
1254
1255 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1256 {
1257 struct bnxt_qplib_q *sq = &qp->sq;
1258 struct dbr_dbr db_msg = { 0 };
1259 u32 sw_prod;
1260
1261 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1262
1263 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1264 DBR_DBR_INDEX_MASK);
1265 db_msg.type_xid =
1266 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1267 DBR_DBR_TYPE_SQ);
1268 /* Flush all the WQE writes to HW */
1269 wmb();
1270 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1271 }
1272
1273 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1274 struct bnxt_qplib_swqe *wqe)
1275 {
1276 struct bnxt_qplib_q *sq = &qp->sq;
1277 struct bnxt_qplib_swq *swq;
1278 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
1279 struct sq_sge *hw_sge;
1280 struct bnxt_qplib_nq_work *nq_work = NULL;
1281 bool sch_handler = false;
1282 u32 sw_prod;
1283 u8 wqe_size16;
1284 int i, rc = 0, data_len = 0, pkt_num = 0;
1285 __le32 temp32;
1286
1287 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1288 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1289 sch_handler = true;
1290 dev_dbg(&sq->hwq.pdev->dev,
1291 "%s Error QP. Scheduling for poll_cq\n",
1292 __func__);
1293 goto queue_err;
1294 }
1295 }
1296
1297 if (bnxt_qplib_queue_full(sq)) {
1298 dev_err(&sq->hwq.pdev->dev,
1299 "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x",
1300 sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,
1301 sq->q_full_delta);
1302 rc = -ENOMEM;
1303 goto done;
1304 }
1305 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1306 swq = &sq->swq[sw_prod];
1307 swq->wr_id = wqe->wr_id;
1308 swq->type = wqe->type;
1309 swq->flags = wqe->flags;
1310 if (qp->sig_type)
1311 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1312 swq->start_psn = sq->psn & BTH_PSN_MASK;
1313
1314 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1315 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1316 [get_sqe_idx(sw_prod)];
1317
1318 memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1319
1320 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1321 /* Copy the inline data */
1322 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1323 dev_warn(&sq->hwq.pdev->dev,
1324 "QPLIB: Inline data length > 96 detected");
1325 data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1326 } else {
1327 data_len = wqe->inline_len;
1328 }
1329 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1330 wqe_size16 = (data_len + 15) >> 4;
1331 } else {
1332 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1333 i < wqe->num_sge; i++, hw_sge++) {
1334 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1335 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1336 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1337 data_len += wqe->sg_list[i].size;
1338 }
1339 /* Each SGE entry = 1 WQE size16 */
1340 wqe_size16 = wqe->num_sge;
1341 /* HW requires wqe size has room for atleast one SGE even if
1342 * none was supplied by ULP
1343 */
1344 if (!wqe->num_sge)
1345 wqe_size16++;
1346 }
1347
1348 /* Specifics */
1349 switch (wqe->type) {
1350 case BNXT_QPLIB_SWQE_TYPE_SEND:
1351 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1352 /* Assemble info for Raw Ethertype QPs */
1353 struct sq_send_raweth_qp1 *sqe =
1354 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1355
1356 sqe->wqe_type = wqe->type;
1357 sqe->flags = wqe->flags;
1358 sqe->wqe_size = wqe_size16 +
1359 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1360 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1361 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1362 sqe->length = cpu_to_le32(data_len);
1363 sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1364 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1365 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1366
1367 break;
1368 }
1369 /* fall thru */
1370 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1371 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1372 {
1373 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
1374
1375 sqe->wqe_type = wqe->type;
1376 sqe->flags = wqe->flags;
1377 sqe->wqe_size = wqe_size16 +
1378 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1379 sqe->inv_key_or_imm_data = cpu_to_le32(
1380 wqe->send.inv_key);
1381 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
1382 sqe->q_key = cpu_to_le32(wqe->send.q_key);
1383 sqe->dst_qp = cpu_to_le32(
1384 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1385 sqe->length = cpu_to_le32(data_len);
1386 sqe->avid = cpu_to_le32(wqe->send.avid &
1387 SQ_SEND_AVID_MASK);
1388 sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1389 } else {
1390 sqe->length = cpu_to_le32(data_len);
1391 sqe->dst_qp = 0;
1392 sqe->avid = 0;
1393 if (qp->mtu)
1394 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1395 if (!pkt_num)
1396 pkt_num = 1;
1397 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1398 }
1399 break;
1400 }
1401 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1402 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1403 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1404 {
1405 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
1406
1407 sqe->wqe_type = wqe->type;
1408 sqe->flags = wqe->flags;
1409 sqe->wqe_size = wqe_size16 +
1410 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1411 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1412 sqe->length = cpu_to_le32((u32)data_len);
1413 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1414 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1415 if (qp->mtu)
1416 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1417 if (!pkt_num)
1418 pkt_num = 1;
1419 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1420 break;
1421 }
1422 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1423 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1424 {
1425 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
1426
1427 sqe->wqe_type = wqe->type;
1428 sqe->flags = wqe->flags;
1429 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1430 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1431 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1432 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1433 if (qp->mtu)
1434 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1435 if (!pkt_num)
1436 pkt_num = 1;
1437 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1438 break;
1439 }
1440 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1441 {
1442 struct sq_localinvalidate *sqe =
1443 (struct sq_localinvalidate *)hw_sq_send_hdr;
1444
1445 sqe->wqe_type = wqe->type;
1446 sqe->flags = wqe->flags;
1447 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1448
1449 break;
1450 }
1451 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1452 {
1453 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
1454
1455 sqe->wqe_type = wqe->type;
1456 sqe->flags = wqe->flags;
1457 sqe->access_cntl = wqe->frmr.access_cntl |
1458 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1459 sqe->zero_based_page_size_log =
1460 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1461 SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1462 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1463 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1464 temp32 = cpu_to_le32(wqe->frmr.length);
1465 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1466 sqe->numlevels_pbl_page_size_log =
1467 ((wqe->frmr.pbl_pg_sz_log <<
1468 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1469 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1470 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1471 SQ_FR_PMR_NUMLEVELS_MASK);
1472
1473 for (i = 0; i < wqe->frmr.page_list_len; i++)
1474 wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1475 wqe->frmr.page_list[i] |
1476 PTU_PTE_VALID);
1477 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1478 sqe->va = cpu_to_le64(wqe->frmr.va);
1479
1480 break;
1481 }
1482 case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1483 {
1484 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
1485
1486 sqe->wqe_type = wqe->type;
1487 sqe->flags = wqe->flags;
1488 sqe->access_cntl = wqe->bind.access_cntl;
1489 sqe->mw_type_zero_based = wqe->bind.mw_type |
1490 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1491 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1492 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1493 sqe->va = cpu_to_le64(wqe->bind.va);
1494 temp32 = cpu_to_le32(wqe->bind.length);
1495 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1496 break;
1497 }
1498 default:
1499 /* Bad wqe, return error */
1500 rc = -EINVAL;
1501 goto done;
1502 }
1503 swq->next_psn = sq->psn & BTH_PSN_MASK;
1504 if (swq->psn_search) {
1505 swq->psn_search->opcode_start_psn = cpu_to_le32(
1506 ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1507 SQ_PSN_SEARCH_START_PSN_MASK) |
1508 ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1509 SQ_PSN_SEARCH_OPCODE_MASK));
1510 swq->psn_search->flags_next_psn = cpu_to_le32(
1511 ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1512 SQ_PSN_SEARCH_NEXT_PSN_MASK));
1513 }
1514 queue_err:
1515 if (sch_handler) {
1516 /* Store the ULP info in the software structures */
1517 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1518 swq = &sq->swq[sw_prod];
1519 swq->wr_id = wqe->wr_id;
1520 swq->type = wqe->type;
1521 swq->flags = wqe->flags;
1522 if (qp->sig_type)
1523 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1524 swq->start_psn = sq->psn & BTH_PSN_MASK;
1525 }
1526 sq->hwq.prod++;
1527 qp->wqe_cnt++;
1528
1529 done:
1530 if (sch_handler) {
1531 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1532 if (nq_work) {
1533 nq_work->cq = qp->scq;
1534 nq_work->nq = qp->scq->nq;
1535 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1536 queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
1537 } else {
1538 dev_err(&sq->hwq.pdev->dev,
1539 "QPLIB: FP: Failed to allocate SQ nq_work!");
1540 rc = -ENOMEM;
1541 }
1542 }
1543 return rc;
1544 }
1545
1546 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1547 {
1548 struct bnxt_qplib_q *rq = &qp->rq;
1549 struct dbr_dbr db_msg = { 0 };
1550 u32 sw_prod;
1551
1552 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1553 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1554 DBR_DBR_INDEX_MASK);
1555 db_msg.type_xid =
1556 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1557 DBR_DBR_TYPE_RQ);
1558
1559 /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1560 wmb();
1561 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1562 }
1563
1564 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1565 struct bnxt_qplib_swqe *wqe)
1566 {
1567 struct bnxt_qplib_q *rq = &qp->rq;
1568 struct rq_wqe *rqe, **rqe_ptr;
1569 struct sq_sge *hw_sge;
1570 struct bnxt_qplib_nq_work *nq_work = NULL;
1571 bool sch_handler = false;
1572 u32 sw_prod;
1573 int i, rc = 0;
1574
1575 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1576 sch_handler = true;
1577 dev_dbg(&rq->hwq.pdev->dev,
1578 "%s Error QP. Scheduling for poll_cq\n",
1579 __func__);
1580 goto queue_err;
1581 }
1582 if (bnxt_qplib_queue_full(rq)) {
1583 dev_err(&rq->hwq.pdev->dev,
1584 "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
1585 rc = -EINVAL;
1586 goto done;
1587 }
1588 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1589 rq->swq[sw_prod].wr_id = wqe->wr_id;
1590
1591 rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1592 rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1593
1594 memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1595
1596 /* Calculate wqe_size16 and data_len */
1597 for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1598 i < wqe->num_sge; i++, hw_sge++) {
1599 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1600 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1601 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1602 }
1603 rqe->wqe_type = wqe->type;
1604 rqe->flags = wqe->flags;
1605 rqe->wqe_size = wqe->num_sge +
1606 ((offsetof(typeof(*rqe), data) + 15) >> 4);
1607 /* HW requires wqe size has room for atleast one SGE even if none
1608 * was supplied by ULP
1609 */
1610 if (!wqe->num_sge)
1611 rqe->wqe_size++;
1612
1613 /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1614 rqe->wr_id[0] = cpu_to_le32(sw_prod);
1615
1616 queue_err:
1617 if (sch_handler) {
1618 /* Store the ULP info in the software structures */
1619 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1620 rq->swq[sw_prod].wr_id = wqe->wr_id;
1621 }
1622
1623 rq->hwq.prod++;
1624 if (sch_handler) {
1625 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1626 if (nq_work) {
1627 nq_work->cq = qp->rcq;
1628 nq_work->nq = qp->rcq->nq;
1629 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1630 queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
1631 } else {
1632 dev_err(&rq->hwq.pdev->dev,
1633 "QPLIB: FP: Failed to allocate RQ nq_work!");
1634 rc = -ENOMEM;
1635 }
1636 }
1637 done:
1638 return rc;
1639 }
1640
1641 /* CQ */
1642
1643 /* Spinlock must be held */
1644 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1645 {
1646 struct dbr_dbr db_msg = { 0 };
1647
1648 db_msg.type_xid =
1649 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1650 DBR_DBR_TYPE_CQ_ARMENA);
1651 /* Flush memory writes before enabling the CQ */
1652 wmb();
1653 __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1654 }
1655
1656 static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1657 {
1658 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1659 struct dbr_dbr db_msg = { 0 };
1660 u32 sw_cons;
1661
1662 /* Ring DB */
1663 sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1664 db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1665 DBR_DBR_INDEX_MASK);
1666 db_msg.type_xid =
1667 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1668 arm_type);
1669 /* flush memory writes before arming the CQ */
1670 wmb();
1671 __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1672 }
1673
1674 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1675 {
1676 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1677 struct cmdq_create_cq req;
1678 struct creq_create_cq_resp resp;
1679 struct bnxt_qplib_pbl *pbl;
1680 u16 cmd_flags = 0;
1681 int rc;
1682
1683 cq->hwq.max_elements = cq->max_wqe;
1684 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1685 cq->nmap, &cq->hwq.max_elements,
1686 BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1687 PAGE_SIZE, HWQ_TYPE_QUEUE);
1688 if (rc)
1689 goto exit;
1690
1691 RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1692
1693 if (!cq->dpi) {
1694 dev_err(&rcfw->pdev->dev,
1695 "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1696 return -EINVAL;
1697 }
1698 req.dpi = cpu_to_le32(cq->dpi->dpi);
1699 req.cq_handle = cpu_to_le64(cq->cq_handle);
1700
1701 req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1702 pbl = &cq->hwq.pbl[PBL_LVL_0];
1703 req.pg_size_lvl = cpu_to_le32(
1704 ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1705 CMDQ_CREATE_CQ_LVL_SFT) |
1706 (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1707 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1708 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1709 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1710 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1711 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1712 CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1713
1714 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1715
1716 req.cq_fco_cnq_id = cpu_to_le32(
1717 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1718 CMDQ_CREATE_CQ_CNQ_ID_SFT);
1719
1720 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1721 (void *)&resp, NULL, 0);
1722 if (rc)
1723 goto fail;
1724
1725 cq->id = le32_to_cpu(resp.xid);
1726 cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1727 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1728 init_waitqueue_head(&cq->waitq);
1729 INIT_LIST_HEAD(&cq->sqf_head);
1730 INIT_LIST_HEAD(&cq->rqf_head);
1731 spin_lock_init(&cq->compl_lock);
1732
1733 bnxt_qplib_arm_cq_enable(cq);
1734 return 0;
1735
1736 fail:
1737 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1738 exit:
1739 return rc;
1740 }
1741
1742 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1743 {
1744 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1745 struct cmdq_destroy_cq req;
1746 struct creq_destroy_cq_resp resp;
1747 u16 cmd_flags = 0;
1748 int rc;
1749
1750 RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
1751
1752 req.cq_cid = cpu_to_le32(cq->id);
1753 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1754 (void *)&resp, NULL, 0);
1755 if (rc)
1756 return rc;
1757 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1758 return 0;
1759 }
1760
1761 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
1762 struct bnxt_qplib_cqe **pcqe, int *budget)
1763 {
1764 u32 sw_prod, sw_cons;
1765 struct bnxt_qplib_cqe *cqe;
1766 int rc = 0;
1767
1768 /* Now complete all outstanding SQEs with FLUSHED_ERR */
1769 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1770 cqe = *pcqe;
1771 while (*budget) {
1772 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1773 if (sw_cons == sw_prod) {
1774 break;
1775 }
1776 /* Skip the FENCE WQE completions */
1777 if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) {
1778 bnxt_qplib_cancel_phantom_processing(qp);
1779 goto skip_compl;
1780 }
1781 memset(cqe, 0, sizeof(*cqe));
1782 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
1783 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1784 cqe->qp_handle = (u64)(unsigned long)qp;
1785 cqe->wr_id = sq->swq[sw_cons].wr_id;
1786 cqe->src_qp = qp->id;
1787 cqe->type = sq->swq[sw_cons].type;
1788 cqe++;
1789 (*budget)--;
1790 skip_compl:
1791 sq->hwq.cons++;
1792 }
1793 *pcqe = cqe;
1794 if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
1795 /* Out of budget */
1796 rc = -EAGAIN;
1797
1798 return rc;
1799 }
1800
1801 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
1802 struct bnxt_qplib_cqe **pcqe, int *budget)
1803 {
1804 struct bnxt_qplib_cqe *cqe;
1805 u32 sw_prod, sw_cons;
1806 int rc = 0;
1807 int opcode = 0;
1808
1809 switch (qp->type) {
1810 case CMDQ_CREATE_QP1_TYPE_GSI:
1811 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
1812 break;
1813 case CMDQ_CREATE_QP_TYPE_RC:
1814 opcode = CQ_BASE_CQE_TYPE_RES_RC;
1815 break;
1816 case CMDQ_CREATE_QP_TYPE_UD:
1817 opcode = CQ_BASE_CQE_TYPE_RES_UD;
1818 break;
1819 }
1820
1821 /* Flush the rest of the RQ */
1822 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1823 cqe = *pcqe;
1824 while (*budget) {
1825 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
1826 if (sw_cons == sw_prod)
1827 break;
1828 memset(cqe, 0, sizeof(*cqe));
1829 cqe->status =
1830 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
1831 cqe->opcode = opcode;
1832 cqe->qp_handle = (unsigned long)qp;
1833 cqe->wr_id = rq->swq[sw_cons].wr_id;
1834 cqe++;
1835 (*budget)--;
1836 rq->hwq.cons++;
1837 }
1838 *pcqe = cqe;
1839 if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
1840 /* Out of budget */
1841 rc = -EAGAIN;
1842
1843 return rc;
1844 }
1845
1846 void bnxt_qplib_mark_qp_error(void *qp_handle)
1847 {
1848 struct bnxt_qplib_qp *qp = qp_handle;
1849
1850 if (!qp)
1851 return;
1852
1853 /* Must block new posting of SQ and RQ */
1854 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1855 bnxt_qplib_cancel_phantom_processing(qp);
1856
1857 /* Add qp to flush list of the CQ */
1858 __bnxt_qplib_add_flush_qp(qp);
1859 }
1860
1861 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
1862 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1
1863 */
1864 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
1865 u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
1866 {
1867 struct bnxt_qplib_q *sq = &qp->sq;
1868 struct bnxt_qplib_swq *swq;
1869 u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
1870 struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr;
1871 struct cq_req *peek_req_hwcqe;
1872 struct bnxt_qplib_qp *peek_qp;
1873 struct bnxt_qplib_q *peek_sq;
1874 int i, rc = 0;
1875
1876 /* Normal mode */
1877 /* Check for the psn_search marking before completing */
1878 swq = &sq->swq[sw_sq_cons];
1879 if (swq->psn_search &&
1880 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
1881 /* Unmark */
1882 swq->psn_search->flags_next_psn = cpu_to_le32
1883 (le32_to_cpu(swq->psn_search->flags_next_psn)
1884 & ~0x80000000);
1885 dev_dbg(&cq->hwq.pdev->dev,
1886 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
1887 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
1888 sq->condition = true;
1889 sq->send_phantom = true;
1890
1891 /* TODO: Only ARM if the previous SQE is ARMALL */
1892 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL);
1893
1894 rc = -EAGAIN;
1895 goto out;
1896 }
1897 if (sq->condition) {
1898 /* Peek at the completions */
1899 peek_raw_cq_cons = cq->hwq.cons;
1900 peek_sw_cq_cons = cq_cons;
1901 i = cq->hwq.max_elements;
1902 while (i--) {
1903 peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
1904 peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
1905 peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]
1906 [CQE_IDX(peek_sw_cq_cons)];
1907 /* If the next hwcqe is VALID */
1908 if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
1909 cq->hwq.max_elements)) {
1910 /*
1911 * The valid test of the entry must be done first before
1912 * reading any further.
1913 */
1914 dma_rmb();
1915 /* If the next hwcqe is a REQ */
1916 if ((peek_hwcqe->cqe_type_toggle &
1917 CQ_BASE_CQE_TYPE_MASK) ==
1918 CQ_BASE_CQE_TYPE_REQ) {
1919 peek_req_hwcqe = (struct cq_req *)
1920 peek_hwcqe;
1921 peek_qp = (struct bnxt_qplib_qp *)
1922 ((unsigned long)
1923 le64_to_cpu
1924 (peek_req_hwcqe->qp_handle));
1925 peek_sq = &peek_qp->sq;
1926 peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(
1927 peek_req_hwcqe->sq_cons_idx) - 1
1928 , &sq->hwq);
1929 /* If the hwcqe's sq's wr_id matches */
1930 if (peek_sq == sq &&
1931 sq->swq[peek_sq_cons_idx].wr_id ==
1932 BNXT_QPLIB_FENCE_WRID) {
1933 /*
1934 * Unbreak only if the phantom
1935 * comes back
1936 */
1937 dev_dbg(&cq->hwq.pdev->dev,
1938 "FP:Got Phantom CQE");
1939 sq->condition = false;
1940 sq->single = true;
1941 rc = 0;
1942 goto out;
1943 }
1944 }
1945 /* Valid but not the phantom, so keep looping */
1946 } else {
1947 /* Not valid yet, just exit and wait */
1948 rc = -EINVAL;
1949 goto out;
1950 }
1951 peek_sw_cq_cons++;
1952 peek_raw_cq_cons++;
1953 }
1954 dev_err(&cq->hwq.pdev->dev,
1955 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x",
1956 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
1957 rc = -EINVAL;
1958 }
1959 out:
1960 return rc;
1961 }
1962
1963 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
1964 struct cq_req *hwcqe,
1965 struct bnxt_qplib_cqe **pcqe, int *budget,
1966 u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
1967 {
1968 struct bnxt_qplib_qp *qp;
1969 struct bnxt_qplib_q *sq;
1970 struct bnxt_qplib_cqe *cqe;
1971 u32 sw_sq_cons, cqe_sq_cons;
1972 struct bnxt_qplib_swq *swq;
1973 int rc = 0;
1974
1975 qp = (struct bnxt_qplib_qp *)((unsigned long)
1976 le64_to_cpu(hwcqe->qp_handle));
1977 if (!qp) {
1978 dev_err(&cq->hwq.pdev->dev,
1979 "QPLIB: FP: Process Req qp is NULL");
1980 return -EINVAL;
1981 }
1982 sq = &qp->sq;
1983
1984 cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
1985 if (cqe_sq_cons > sq->hwq.max_elements) {
1986 dev_err(&cq->hwq.pdev->dev,
1987 "QPLIB: FP: CQ Process req reported ");
1988 dev_err(&cq->hwq.pdev->dev,
1989 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1990 cqe_sq_cons, sq->hwq.max_elements);
1991 return -EINVAL;
1992 }
1993
1994 if (qp->sq.flushed) {
1995 dev_dbg(&cq->hwq.pdev->dev,
1996 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
1997 goto done;
1998 }
1999 /* Require to walk the sq's swq to fabricate CQEs for all previously
2000 * signaled SWQEs due to CQE aggregation from the current sq cons
2001 * to the cqe_sq_cons
2002 */
2003 cqe = *pcqe;
2004 while (*budget) {
2005 sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2006 if (sw_sq_cons == cqe_sq_cons)
2007 /* Done */
2008 break;
2009
2010 swq = &sq->swq[sw_sq_cons];
2011 memset(cqe, 0, sizeof(*cqe));
2012 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2013 cqe->qp_handle = (u64)(unsigned long)qp;
2014 cqe->src_qp = qp->id;
2015 cqe->wr_id = swq->wr_id;
2016 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
2017 goto skip;
2018 cqe->type = swq->type;
2019
2020 /* For the last CQE, check for status. For errors, regardless
2021 * of the request being signaled or not, it must complete with
2022 * the hwcqe error status
2023 */
2024 if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&
2025 hwcqe->status != CQ_REQ_STATUS_OK) {
2026 cqe->status = hwcqe->status;
2027 dev_err(&cq->hwq.pdev->dev,
2028 "QPLIB: FP: CQ Processed Req ");
2029 dev_err(&cq->hwq.pdev->dev,
2030 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
2031 sw_sq_cons, cqe->wr_id, cqe->status);
2032 cqe++;
2033 (*budget)--;
2034 bnxt_qplib_lock_buddy_cq(qp, cq);
2035 bnxt_qplib_mark_qp_error(qp);
2036 bnxt_qplib_unlock_buddy_cq(qp, cq);
2037 } else {
2038 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2039 /* Before we complete, do WA 9060 */
2040 if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,
2041 cqe_sq_cons)) {
2042 *lib_qp = qp;
2043 goto out;
2044 }
2045 cqe->status = CQ_REQ_STATUS_OK;
2046 cqe++;
2047 (*budget)--;
2048 }
2049 }
2050 skip:
2051 sq->hwq.cons++;
2052 if (sq->single)
2053 break;
2054 }
2055 out:
2056 *pcqe = cqe;
2057 if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {
2058 /* Out of budget */
2059 rc = -EAGAIN;
2060 goto done;
2061 }
2062 /*
2063 * Back to normal completion mode only after it has completed all of
2064 * the WC for this CQE
2065 */
2066 sq->single = false;
2067 done:
2068 return rc;
2069 }
2070
2071 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
2072 struct cq_res_rc *hwcqe,
2073 struct bnxt_qplib_cqe **pcqe,
2074 int *budget)
2075 {
2076 struct bnxt_qplib_qp *qp;
2077 struct bnxt_qplib_q *rq;
2078 struct bnxt_qplib_cqe *cqe;
2079 u32 wr_id_idx;
2080 int rc = 0;
2081
2082 qp = (struct bnxt_qplib_qp *)((unsigned long)
2083 le64_to_cpu(hwcqe->qp_handle));
2084 if (!qp) {
2085 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
2086 return -EINVAL;
2087 }
2088 if (qp->rq.flushed) {
2089 dev_dbg(&cq->hwq.pdev->dev,
2090 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2091 goto done;
2092 }
2093
2094 cqe = *pcqe;
2095 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2096 cqe->length = le32_to_cpu(hwcqe->length);
2097 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
2098 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
2099 cqe->flags = le16_to_cpu(hwcqe->flags);
2100 cqe->status = hwcqe->status;
2101 cqe->qp_handle = (u64)(unsigned long)qp;
2102
2103 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
2104 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
2105 rq = &qp->rq;
2106 if (wr_id_idx > rq->hwq.max_elements) {
2107 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC ");
2108 dev_err(&cq->hwq.pdev->dev,
2109 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
2110 wr_id_idx, rq->hwq.max_elements);
2111 return -EINVAL;
2112 }
2113
2114 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
2115 cqe++;
2116 (*budget)--;
2117 rq->hwq.cons++;
2118 *pcqe = cqe;
2119
2120 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2121 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2122 /* Add qp to flush list of the CQ */
2123 bnxt_qplib_lock_buddy_cq(qp, cq);
2124 __bnxt_qplib_add_flush_qp(qp);
2125 bnxt_qplib_unlock_buddy_cq(qp, cq);
2126 }
2127
2128 done:
2129 return rc;
2130 }
2131
2132 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
2133 struct cq_res_ud *hwcqe,
2134 struct bnxt_qplib_cqe **pcqe,
2135 int *budget)
2136 {
2137 struct bnxt_qplib_qp *qp;
2138 struct bnxt_qplib_q *rq;
2139 struct bnxt_qplib_cqe *cqe;
2140 u32 wr_id_idx;
2141 int rc = 0;
2142
2143 qp = (struct bnxt_qplib_qp *)((unsigned long)
2144 le64_to_cpu(hwcqe->qp_handle));
2145 if (!qp) {
2146 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
2147 return -EINVAL;
2148 }
2149 if (qp->rq.flushed) {
2150 dev_dbg(&cq->hwq.pdev->dev,
2151 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2152 goto done;
2153 }
2154 cqe = *pcqe;
2155 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2156 cqe->length = le32_to_cpu(hwcqe->length);
2157 cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
2158 cqe->flags = le16_to_cpu(hwcqe->flags);
2159 cqe->status = hwcqe->status;
2160 cqe->qp_handle = (u64)(unsigned long)qp;
2161 memcpy(cqe->smac, hwcqe->src_mac, 6);
2162 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
2163 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
2164 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
2165 ((le32_to_cpu(
2166 hwcqe->src_qp_high_srq_or_rq_wr_id) &
2167 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
2168
2169 rq = &qp->rq;
2170 if (wr_id_idx > rq->hwq.max_elements) {
2171 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD ");
2172 dev_err(&cq->hwq.pdev->dev,
2173 "QPLIB: wr_id idx %#x exceeded RQ max %#x",
2174 wr_id_idx, rq->hwq.max_elements);
2175 return -EINVAL;
2176 }
2177
2178 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
2179 cqe++;
2180 (*budget)--;
2181 rq->hwq.cons++;
2182 *pcqe = cqe;
2183
2184 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2185 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2186 /* Add qp to flush list of the CQ */
2187 bnxt_qplib_lock_buddy_cq(qp, cq);
2188 __bnxt_qplib_add_flush_qp(qp);
2189 bnxt_qplib_unlock_buddy_cq(qp, cq);
2190 }
2191 done:
2192 return rc;
2193 }
2194
2195 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
2196 {
2197 struct cq_base *hw_cqe, **hw_cqe_ptr;
2198 unsigned long flags;
2199 u32 sw_cons, raw_cons;
2200 bool rc = true;
2201
2202 spin_lock_irqsave(&cq->hwq.lock, flags);
2203 raw_cons = cq->hwq.cons;
2204 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2205 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2206 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2207
2208 /* Check for Valid bit. If the CQE is valid, return false */
2209 rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
2210 spin_unlock_irqrestore(&cq->hwq.lock, flags);
2211 return rc;
2212 }
2213
2214 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
2215 struct cq_res_raweth_qp1 *hwcqe,
2216 struct bnxt_qplib_cqe **pcqe,
2217 int *budget)
2218 {
2219 struct bnxt_qplib_qp *qp;
2220 struct bnxt_qplib_q *rq;
2221 struct bnxt_qplib_cqe *cqe;
2222 u32 wr_id_idx;
2223 int rc = 0;
2224
2225 qp = (struct bnxt_qplib_qp *)((unsigned long)
2226 le64_to_cpu(hwcqe->qp_handle));
2227 if (!qp) {
2228 dev_err(&cq->hwq.pdev->dev,
2229 "QPLIB: process_cq Raw/QP1 qp is NULL");
2230 return -EINVAL;
2231 }
2232 if (qp->rq.flushed) {
2233 dev_dbg(&cq->hwq.pdev->dev,
2234 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2235 goto done;
2236 }
2237 cqe = *pcqe;
2238 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2239 cqe->flags = le16_to_cpu(hwcqe->flags);
2240 cqe->qp_handle = (u64)(unsigned long)qp;
2241
2242 wr_id_idx =
2243 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
2244 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
2245 cqe->src_qp = qp->id;
2246 if (qp->id == 1 && !cqe->length) {
2247 /* Add workaround for the length misdetection */
2248 cqe->length = 296;
2249 } else {
2250 cqe->length = le16_to_cpu(hwcqe->length);
2251 }
2252 cqe->pkey_index = qp->pkey_index;
2253 memcpy(cqe->smac, qp->smac, 6);
2254
2255 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
2256 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
2257 cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata);
2258
2259 rq = &qp->rq;
2260 if (wr_id_idx > rq->hwq.max_elements) {
2261 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
2262 dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x",
2263 wr_id_idx, rq->hwq.max_elements);
2264 return -EINVAL;
2265 }
2266
2267 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
2268 cqe++;
2269 (*budget)--;
2270 rq->hwq.cons++;
2271 *pcqe = cqe;
2272
2273 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2274 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2275 /* Add qp to flush list of the CQ */
2276 bnxt_qplib_lock_buddy_cq(qp, cq);
2277 __bnxt_qplib_add_flush_qp(qp);
2278 bnxt_qplib_unlock_buddy_cq(qp, cq);
2279 }
2280
2281 done:
2282 return rc;
2283 }
2284
2285 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
2286 struct cq_terminal *hwcqe,
2287 struct bnxt_qplib_cqe **pcqe,
2288 int *budget)
2289 {
2290 struct bnxt_qplib_qp *qp;
2291 struct bnxt_qplib_q *sq, *rq;
2292 struct bnxt_qplib_cqe *cqe;
2293 u32 sw_cons = 0, cqe_cons;
2294 int rc = 0;
2295
2296 /* Check the Status */
2297 if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
2298 dev_warn(&cq->hwq.pdev->dev,
2299 "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
2300 hwcqe->status);
2301
2302 qp = (struct bnxt_qplib_qp *)((unsigned long)
2303 le64_to_cpu(hwcqe->qp_handle));
2304 if (!qp) {
2305 dev_err(&cq->hwq.pdev->dev,
2306 "QPLIB: FP: CQ Process terminal qp is NULL");
2307 return -EINVAL;
2308 }
2309
2310 /* Must block new posting of SQ and RQ */
2311 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2312
2313 sq = &qp->sq;
2314 rq = &qp->rq;
2315
2316 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
2317 if (cqe_cons == 0xFFFF)
2318 goto do_rq;
2319
2320 if (cqe_cons > sq->hwq.max_elements) {
2321 dev_err(&cq->hwq.pdev->dev,
2322 "QPLIB: FP: CQ Process terminal reported ");
2323 dev_err(&cq->hwq.pdev->dev,
2324 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
2325 cqe_cons, sq->hwq.max_elements);
2326 goto do_rq;
2327 }
2328
2329 if (qp->sq.flushed) {
2330 dev_dbg(&cq->hwq.pdev->dev,
2331 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2332 goto sq_done;
2333 }
2334
2335 /* Terminal CQE can also include aggregated successful CQEs prior.
2336 * So we must complete all CQEs from the current sq's cons to the
2337 * cq_cons with status OK
2338 */
2339 cqe = *pcqe;
2340 while (*budget) {
2341 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2342 if (sw_cons == cqe_cons)
2343 break;
2344 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2345 memset(cqe, 0, sizeof(*cqe));
2346 cqe->status = CQ_REQ_STATUS_OK;
2347 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2348 cqe->qp_handle = (u64)(unsigned long)qp;
2349 cqe->src_qp = qp->id;
2350 cqe->wr_id = sq->swq[sw_cons].wr_id;
2351 cqe->type = sq->swq[sw_cons].type;
2352 cqe++;
2353 (*budget)--;
2354 }
2355 sq->hwq.cons++;
2356 }
2357 *pcqe = cqe;
2358 if (!(*budget) && sw_cons != cqe_cons) {
2359 /* Out of budget */
2360 rc = -EAGAIN;
2361 goto sq_done;
2362 }
2363 sq_done:
2364 if (rc)
2365 return rc;
2366 do_rq:
2367 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2368 if (cqe_cons == 0xFFFF) {
2369 goto done;
2370 } else if (cqe_cons > rq->hwq.max_elements) {
2371 dev_err(&cq->hwq.pdev->dev,
2372 "QPLIB: FP: CQ Processed terminal ");
2373 dev_err(&cq->hwq.pdev->dev,
2374 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2375 cqe_cons, rq->hwq.max_elements);
2376 goto done;
2377 }
2378
2379 if (qp->rq.flushed) {
2380 dev_dbg(&cq->hwq.pdev->dev,
2381 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2382 rc = 0;
2383 goto done;
2384 }
2385
2386 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2387 * from the current rq->cons to the rq->prod regardless what the
2388 * rq->cons the terminal CQE indicates
2389 */
2390
2391 /* Add qp to flush list of the CQ */
2392 bnxt_qplib_lock_buddy_cq(qp, cq);
2393 __bnxt_qplib_add_flush_qp(qp);
2394 bnxt_qplib_unlock_buddy_cq(qp, cq);
2395 done:
2396 return rc;
2397 }
2398
2399 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2400 struct cq_cutoff *hwcqe)
2401 {
2402 /* Check the Status */
2403 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2404 dev_err(&cq->hwq.pdev->dev,
2405 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2406 hwcqe->status);
2407 return -EINVAL;
2408 }
2409 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2410 wake_up_interruptible(&cq->waitq);
2411
2412 return 0;
2413 }
2414
2415 int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
2416 struct bnxt_qplib_cqe *cqe,
2417 int num_cqes)
2418 {
2419 struct bnxt_qplib_qp *qp = NULL;
2420 u32 budget = num_cqes;
2421 unsigned long flags;
2422
2423 spin_lock_irqsave(&cq->hwq.lock, flags);
2424 list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
2425 dev_dbg(&cq->hwq.pdev->dev,
2426 "QPLIB: FP: Flushing SQ QP= %p",
2427 qp);
2428 __flush_sq(&qp->sq, qp, &cqe, &budget);
2429 }
2430
2431 list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
2432 dev_dbg(&cq->hwq.pdev->dev,
2433 "QPLIB: FP: Flushing RQ QP= %p",
2434 qp);
2435 __flush_rq(&qp->rq, qp, &cqe, &budget);
2436 }
2437 spin_unlock_irqrestore(&cq->hwq.lock, flags);
2438
2439 return num_cqes - budget;
2440 }
2441
2442 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
2443 int num_cqes, struct bnxt_qplib_qp **lib_qp)
2444 {
2445 struct cq_base *hw_cqe, **hw_cqe_ptr;
2446 unsigned long flags;
2447 u32 sw_cons, raw_cons;
2448 int budget, rc = 0;
2449
2450 spin_lock_irqsave(&cq->hwq.lock, flags);
2451 raw_cons = cq->hwq.cons;
2452 budget = num_cqes;
2453
2454 while (budget) {
2455 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2456 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2457 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2458
2459 /* Check for Valid bit */
2460 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2461 break;
2462
2463 /*
2464 * The valid test of the entry must be done first before
2465 * reading any further.
2466 */
2467 dma_rmb();
2468 /* From the device's respective CQE format to qplib_wc*/
2469 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2470 case CQ_BASE_CQE_TYPE_REQ:
2471 rc = bnxt_qplib_cq_process_req(cq,
2472 (struct cq_req *)hw_cqe,
2473 &cqe, &budget,
2474 sw_cons, lib_qp);
2475 break;
2476 case CQ_BASE_CQE_TYPE_RES_RC:
2477 rc = bnxt_qplib_cq_process_res_rc(cq,
2478 (struct cq_res_rc *)
2479 hw_cqe, &cqe,
2480 &budget);
2481 break;
2482 case CQ_BASE_CQE_TYPE_RES_UD:
2483 rc = bnxt_qplib_cq_process_res_ud
2484 (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2485 &budget);
2486 break;
2487 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2488 rc = bnxt_qplib_cq_process_res_raweth_qp1
2489 (cq, (struct cq_res_raweth_qp1 *)
2490 hw_cqe, &cqe, &budget);
2491 break;
2492 case CQ_BASE_CQE_TYPE_TERMINAL:
2493 rc = bnxt_qplib_cq_process_terminal
2494 (cq, (struct cq_terminal *)hw_cqe,
2495 &cqe, &budget);
2496 break;
2497 case CQ_BASE_CQE_TYPE_CUT_OFF:
2498 bnxt_qplib_cq_process_cutoff
2499 (cq, (struct cq_cutoff *)hw_cqe);
2500 /* Done processing this CQ */
2501 goto exit;
2502 default:
2503 dev_err(&cq->hwq.pdev->dev,
2504 "QPLIB: process_cq unknown type 0x%lx",
2505 hw_cqe->cqe_type_toggle &
2506 CQ_BASE_CQE_TYPE_MASK);
2507 rc = -EINVAL;
2508 break;
2509 }
2510 if (rc < 0) {
2511 if (rc == -EAGAIN)
2512 break;
2513 /* Error while processing the CQE, just skip to the
2514 * next one
2515 */
2516 dev_err(&cq->hwq.pdev->dev,
2517 "QPLIB: process_cqe error rc = 0x%x", rc);
2518 }
2519 raw_cons++;
2520 }
2521 if (cq->hwq.cons != raw_cons) {
2522 cq->hwq.cons = raw_cons;
2523 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
2524 }
2525 exit:
2526 spin_unlock_irqrestore(&cq->hwq.lock, flags);
2527 return num_cqes - budget;
2528 }
2529
2530 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2531 {
2532 unsigned long flags;
2533
2534 spin_lock_irqsave(&cq->hwq.lock, flags);
2535 if (arm_type)
2536 bnxt_qplib_arm_cq(cq, arm_type);
2537 /* Using cq->arm_state variable to track whether to issue cq handler */
2538 atomic_set(&cq->arm_state, 1);
2539 spin_unlock_irqrestore(&cq->hwq.lock, flags);
2540 }
2541
2542 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp)
2543 {
2544 flush_workqueue(qp->scq->nq->cqn_wq);
2545 if (qp->scq != qp->rcq)
2546 flush_workqueue(qp->rcq->nq->cqn_wq);
2547 }
ubuntu-bionic-kernel mirror