2 * Broadcom NetXtreme-E RoCE driver.
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Description: Fast Path Operators
39 #include <linux/interrupt.h>
40 #include <linux/spinlock.h>
41 #include <linux/sched.h>
42 #include <linux/slab.h>
43 #include <linux/pci.h>
44 #include <linux/prefetch.h>
48 #include "qplib_res.h"
49 #include "qplib_rcfw.h"
53 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq
*cq
);
55 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res
*res
,
56 struct bnxt_qplib_qp
*qp
)
58 struct bnxt_qplib_q
*rq
= &qp
->rq
;
59 struct bnxt_qplib_q
*sq
= &qp
->sq
;
62 dma_free_coherent(&res
->pdev
->dev
,
63 rq
->hwq
.max_elements
* qp
->rq_hdr_buf_size
,
64 qp
->rq_hdr_buf
, qp
->rq_hdr_buf_map
);
66 dma_free_coherent(&res
->pdev
->dev
,
67 sq
->hwq
.max_elements
* qp
->sq_hdr_buf_size
,
68 qp
->sq_hdr_buf
, qp
->sq_hdr_buf_map
);
69 qp
->rq_hdr_buf
= NULL
;
70 qp
->sq_hdr_buf
= NULL
;
71 qp
->rq_hdr_buf_map
= 0;
72 qp
->sq_hdr_buf_map
= 0;
73 qp
->sq_hdr_buf_size
= 0;
74 qp
->rq_hdr_buf_size
= 0;
77 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res
*res
,
78 struct bnxt_qplib_qp
*qp
)
80 struct bnxt_qplib_q
*rq
= &qp
->rq
;
81 struct bnxt_qplib_q
*sq
= &qp
->rq
;
84 if (qp
->sq_hdr_buf_size
&& sq
->hwq
.max_elements
) {
85 qp
->sq_hdr_buf
= dma_alloc_coherent(&res
->pdev
->dev
,
86 sq
->hwq
.max_elements
*
88 &qp
->sq_hdr_buf_map
, GFP_KERNEL
);
89 if (!qp
->sq_hdr_buf
) {
91 dev_err(&res
->pdev
->dev
,
92 "QPLIB: Failed to create sq_hdr_buf");
97 if (qp
->rq_hdr_buf_size
&& rq
->hwq
.max_elements
) {
98 qp
->rq_hdr_buf
= dma_alloc_coherent(&res
->pdev
->dev
,
99 rq
->hwq
.max_elements
*
103 if (!qp
->rq_hdr_buf
) {
105 dev_err(&res
->pdev
->dev
,
106 "QPLIB: Failed to create rq_hdr_buf");
113 bnxt_qplib_free_qp_hdr_buf(res
, qp
);
117 static void bnxt_qplib_service_nq(unsigned long data
)
119 struct bnxt_qplib_nq
*nq
= (struct bnxt_qplib_nq
*)data
;
120 struct bnxt_qplib_hwq
*hwq
= &nq
->hwq
;
121 struct nq_base
*nqe
, **nq_ptr
;
122 int num_cqne_processed
= 0;
123 u32 sw_cons
, raw_cons
;
125 int budget
= nq
->budget
;
128 /* Service the NQ until empty */
129 raw_cons
= hwq
->cons
;
131 sw_cons
= HWQ_CMP(raw_cons
, hwq
);
132 nq_ptr
= (struct nq_base
**)hwq
->pbl_ptr
;
133 nqe
= &nq_ptr
[NQE_PG(sw_cons
)][NQE_IDX(sw_cons
)];
134 if (!NQE_CMP_VALID(nqe
, raw_cons
, hwq
->max_elements
))
137 type
= le16_to_cpu(nqe
->info10_type
) & NQ_BASE_TYPE_MASK
;
139 case NQ_BASE_TYPE_CQ_NOTIFICATION
:
141 struct nq_cn
*nqcne
= (struct nq_cn
*)nqe
;
143 q_handle
= le32_to_cpu(nqcne
->cq_handle_low
);
144 q_handle
|= (u64
)le32_to_cpu(nqcne
->cq_handle_high
)
146 bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq
*)
147 ((unsigned long)q_handle
));
148 if (!nq
->cqn_handler(nq
, (struct bnxt_qplib_cq
*)
149 ((unsigned long)q_handle
)))
150 num_cqne_processed
++;
152 dev_warn(&nq
->pdev
->dev
,
153 "QPLIB: cqn - type 0x%x not handled",
157 case NQ_BASE_TYPE_DBQ_EVENT
:
160 dev_warn(&nq
->pdev
->dev
,
161 "QPLIB: nqe with type = 0x%x not handled",
167 if (hwq
->cons
!= raw_cons
) {
168 hwq
->cons
= raw_cons
;
169 NQ_DB_REARM(nq
->bar_reg_iomem
, hwq
->cons
, hwq
->max_elements
);
173 static irqreturn_t
bnxt_qplib_nq_irq(int irq
, void *dev_instance
)
175 struct bnxt_qplib_nq
*nq
= dev_instance
;
176 struct bnxt_qplib_hwq
*hwq
= &nq
->hwq
;
177 struct nq_base
**nq_ptr
;
180 /* Prefetch the NQ element */
181 sw_cons
= HWQ_CMP(hwq
->cons
, hwq
);
182 nq_ptr
= (struct nq_base
**)nq
->hwq
.pbl_ptr
;
183 prefetch(&nq_ptr
[NQE_PG(sw_cons
)][NQE_IDX(sw_cons
)]);
185 /* Fan out to CPU affinitized kthreads? */
186 tasklet_schedule(&nq
->worker
);
191 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq
*nq
)
193 /* Make sure the HW is stopped! */
194 synchronize_irq(nq
->vector
);
195 tasklet_disable(&nq
->worker
);
196 tasklet_kill(&nq
->worker
);
199 free_irq(nq
->vector
, nq
);
200 nq
->requested
= false;
202 if (nq
->bar_reg_iomem
)
203 iounmap(nq
->bar_reg_iomem
);
204 nq
->bar_reg_iomem
= NULL
;
206 nq
->cqn_handler
= NULL
;
207 nq
->srqn_handler
= NULL
;
211 int bnxt_qplib_enable_nq(struct pci_dev
*pdev
, struct bnxt_qplib_nq
*nq
,
212 int msix_vector
, int bar_reg_offset
,
213 int (*cqn_handler
)(struct bnxt_qplib_nq
*nq
,
214 struct bnxt_qplib_cq
*),
215 int (*srqn_handler
)(struct bnxt_qplib_nq
*nq
,
218 resource_size_t nq_base
;
222 nq
->vector
= msix_vector
;
224 nq
->cqn_handler
= cqn_handler
;
226 nq
->srqn_handler
= srqn_handler
;
228 tasklet_init(&nq
->worker
, bnxt_qplib_service_nq
, (unsigned long)nq
);
230 nq
->requested
= false;
231 rc
= request_irq(nq
->vector
, bnxt_qplib_nq_irq
, 0, "bnxt_qplib_nq", nq
);
233 dev_err(&nq
->pdev
->dev
,
234 "Failed to request IRQ for NQ: %#x", rc
);
235 bnxt_qplib_disable_nq(nq
);
238 nq
->requested
= true;
239 nq
->bar_reg
= NQ_CONS_PCI_BAR_REGION
;
240 nq
->bar_reg_off
= bar_reg_offset
;
241 nq_base
= pci_resource_start(pdev
, nq
->bar_reg
);
246 nq
->bar_reg_iomem
= ioremap_nocache(nq_base
+ nq
->bar_reg_off
, 4);
247 if (!nq
->bar_reg_iomem
) {
251 NQ_DB_REARM(nq
->bar_reg_iomem
, nq
->hwq
.cons
, nq
->hwq
.max_elements
);
255 bnxt_qplib_disable_nq(nq
);
259 void bnxt_qplib_free_nq(struct bnxt_qplib_nq
*nq
)
261 if (nq
->hwq
.max_elements
)
262 bnxt_qplib_free_hwq(nq
->pdev
, &nq
->hwq
);
265 int bnxt_qplib_alloc_nq(struct pci_dev
*pdev
, struct bnxt_qplib_nq
*nq
)
268 if (!nq
->hwq
.max_elements
||
269 nq
->hwq
.max_elements
> BNXT_QPLIB_NQE_MAX_CNT
)
270 nq
->hwq
.max_elements
= BNXT_QPLIB_NQE_MAX_CNT
;
272 if (bnxt_qplib_alloc_init_hwq(nq
->pdev
, &nq
->hwq
, NULL
, 0,
273 &nq
->hwq
.max_elements
,
274 BNXT_QPLIB_MAX_NQE_ENTRY_SIZE
, 0,
275 PAGE_SIZE
, HWQ_TYPE_L2_CMPL
))
283 int bnxt_qplib_create_qp1(struct bnxt_qplib_res
*res
, struct bnxt_qplib_qp
*qp
)
285 struct bnxt_qplib_rcfw
*rcfw
= res
->rcfw
;
286 struct cmdq_create_qp1 req
;
287 struct creq_create_qp1_resp resp
;
288 struct bnxt_qplib_pbl
*pbl
;
289 struct bnxt_qplib_q
*sq
= &qp
->sq
;
290 struct bnxt_qplib_q
*rq
= &qp
->rq
;
295 RCFW_CMD_PREP(req
, CREATE_QP1
, cmd_flags
);
299 req
.dpi
= cpu_to_le32(qp
->dpi
->dpi
);
300 req
.qp_handle
= cpu_to_le64(qp
->qp_handle
);
303 sq
->hwq
.max_elements
= sq
->max_wqe
;
304 rc
= bnxt_qplib_alloc_init_hwq(res
->pdev
, &sq
->hwq
, NULL
, 0,
305 &sq
->hwq
.max_elements
,
306 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE
, 0,
307 PAGE_SIZE
, HWQ_TYPE_QUEUE
);
311 sq
->swq
= kcalloc(sq
->hwq
.max_elements
, sizeof(*sq
->swq
), GFP_KERNEL
);
316 pbl
= &sq
->hwq
.pbl
[PBL_LVL_0
];
317 req
.sq_pbl
= cpu_to_le64(pbl
->pg_map_arr
[0]);
318 req
.sq_pg_size_sq_lvl
=
319 ((sq
->hwq
.level
& CMDQ_CREATE_QP1_SQ_LVL_MASK
)
320 << CMDQ_CREATE_QP1_SQ_LVL_SFT
) |
321 (pbl
->pg_size
== ROCE_PG_SIZE_4K
?
322 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K
:
323 pbl
->pg_size
== ROCE_PG_SIZE_8K
?
324 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K
:
325 pbl
->pg_size
== ROCE_PG_SIZE_64K
?
326 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K
:
327 pbl
->pg_size
== ROCE_PG_SIZE_2M
?
328 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M
:
329 pbl
->pg_size
== ROCE_PG_SIZE_8M
?
330 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M
:
331 pbl
->pg_size
== ROCE_PG_SIZE_1G
?
332 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G
:
333 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K
);
336 req
.scq_cid
= cpu_to_le32(qp
->scq
->id
);
338 qp_flags
|= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE
;
342 rq
->hwq
.max_elements
= qp
->rq
.max_wqe
;
343 rc
= bnxt_qplib_alloc_init_hwq(res
->pdev
, &rq
->hwq
, NULL
, 0,
344 &rq
->hwq
.max_elements
,
345 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE
, 0,
346 PAGE_SIZE
, HWQ_TYPE_QUEUE
);
350 rq
->swq
= kcalloc(rq
->hwq
.max_elements
, sizeof(*rq
->swq
),
356 pbl
= &rq
->hwq
.pbl
[PBL_LVL_0
];
357 req
.rq_pbl
= cpu_to_le64(pbl
->pg_map_arr
[0]);
358 req
.rq_pg_size_rq_lvl
=
359 ((rq
->hwq
.level
& CMDQ_CREATE_QP1_RQ_LVL_MASK
) <<
360 CMDQ_CREATE_QP1_RQ_LVL_SFT
) |
361 (pbl
->pg_size
== ROCE_PG_SIZE_4K
?
362 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K
:
363 pbl
->pg_size
== ROCE_PG_SIZE_8K
?
364 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K
:
365 pbl
->pg_size
== ROCE_PG_SIZE_64K
?
366 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K
:
367 pbl
->pg_size
== ROCE_PG_SIZE_2M
?
368 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M
:
369 pbl
->pg_size
== ROCE_PG_SIZE_8M
?
370 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M
:
371 pbl
->pg_size
== ROCE_PG_SIZE_1G
?
372 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G
:
373 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K
);
375 req
.rcq_cid
= cpu_to_le32(qp
->rcq
->id
);
378 /* Header buffer - allow hdr_buf pass in */
379 rc
= bnxt_qplib_alloc_qp_hdr_buf(res
, qp
);
384 req
.qp_flags
= cpu_to_le32(qp_flags
);
385 req
.sq_size
= cpu_to_le32(sq
->hwq
.max_elements
);
386 req
.rq_size
= cpu_to_le32(rq
->hwq
.max_elements
);
389 cpu_to_le16((sq
->max_sge
& CMDQ_CREATE_QP1_SQ_SGE_MASK
) <<
390 CMDQ_CREATE_QP1_SQ_SGE_SFT
);
392 cpu_to_le16((rq
->max_sge
& CMDQ_CREATE_QP1_RQ_SGE_MASK
) <<
393 CMDQ_CREATE_QP1_RQ_SGE_SFT
);
395 req
.pd_id
= cpu_to_le32(qp
->pd
->id
);
397 rc
= bnxt_qplib_rcfw_send_message(rcfw
, (void *)&req
,
398 (void *)&resp
, NULL
, 0);
402 qp
->id
= le32_to_cpu(resp
.xid
);
403 qp
->cur_qp_state
= CMDQ_MODIFY_QP_NEW_STATE_RESET
;
404 sq
->flush_in_progress
= false;
405 rq
->flush_in_progress
= false;
410 bnxt_qplib_free_qp_hdr_buf(res
, qp
);
412 bnxt_qplib_free_hwq(res
->pdev
, &rq
->hwq
);
415 bnxt_qplib_free_hwq(res
->pdev
, &sq
->hwq
);
421 int bnxt_qplib_create_qp(struct bnxt_qplib_res
*res
, struct bnxt_qplib_qp
*qp
)
423 struct bnxt_qplib_rcfw
*rcfw
= res
->rcfw
;
424 struct sq_send
*hw_sq_send_hdr
, **hw_sq_send_ptr
;
425 struct cmdq_create_qp req
;
426 struct creq_create_qp_resp resp
;
427 struct bnxt_qplib_pbl
*pbl
;
428 struct sq_psn_search
**psn_search_ptr
;
429 unsigned long int psn_search
, poff
= 0;
430 struct bnxt_qplib_q
*sq
= &qp
->sq
;
431 struct bnxt_qplib_q
*rq
= &qp
->rq
;
432 struct bnxt_qplib_hwq
*xrrq
;
433 int i
, rc
, req_size
, psn_sz
;
434 u16 cmd_flags
= 0, max_ssge
;
435 u32 sw_prod
, qp_flags
= 0;
437 RCFW_CMD_PREP(req
, CREATE_QP
, cmd_flags
);
441 req
.dpi
= cpu_to_le32(qp
->dpi
->dpi
);
442 req
.qp_handle
= cpu_to_le64(qp
->qp_handle
);
445 psn_sz
= (qp
->type
== CMDQ_CREATE_QP_TYPE_RC
) ?
446 sizeof(struct sq_psn_search
) : 0;
447 sq
->hwq
.max_elements
= sq
->max_wqe
;
448 rc
= bnxt_qplib_alloc_init_hwq(res
->pdev
, &sq
->hwq
, sq
->sglist
,
449 sq
->nmap
, &sq
->hwq
.max_elements
,
450 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE
,
452 PAGE_SIZE
, HWQ_TYPE_QUEUE
);
456 sq
->swq
= kcalloc(sq
->hwq
.max_elements
, sizeof(*sq
->swq
), GFP_KERNEL
);
461 hw_sq_send_ptr
= (struct sq_send
**)sq
->hwq
.pbl_ptr
;
463 psn_search_ptr
= (struct sq_psn_search
**)
464 &hw_sq_send_ptr
[get_sqe_pg
465 (sq
->hwq
.max_elements
)];
466 psn_search
= (unsigned long int)
467 &hw_sq_send_ptr
[get_sqe_pg(sq
->hwq
.max_elements
)]
468 [get_sqe_idx(sq
->hwq
.max_elements
)];
469 if (psn_search
& ~PAGE_MASK
) {
470 /* If the psn_search does not start on a page boundary,
471 * then calculate the offset
473 poff
= (psn_search
& ~PAGE_MASK
) /
474 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE
;
476 for (i
= 0; i
< sq
->hwq
.max_elements
; i
++)
477 sq
->swq
[i
].psn_search
=
478 &psn_search_ptr
[get_psne_pg(i
+ poff
)]
479 [get_psne_idx(i
+ poff
)];
481 pbl
= &sq
->hwq
.pbl
[PBL_LVL_0
];
482 req
.sq_pbl
= cpu_to_le64(pbl
->pg_map_arr
[0]);
483 req
.sq_pg_size_sq_lvl
=
484 ((sq
->hwq
.level
& CMDQ_CREATE_QP_SQ_LVL_MASK
)
485 << CMDQ_CREATE_QP_SQ_LVL_SFT
) |
486 (pbl
->pg_size
== ROCE_PG_SIZE_4K
?
487 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K
:
488 pbl
->pg_size
== ROCE_PG_SIZE_8K
?
489 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K
:
490 pbl
->pg_size
== ROCE_PG_SIZE_64K
?
491 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K
:
492 pbl
->pg_size
== ROCE_PG_SIZE_2M
?
493 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M
:
494 pbl
->pg_size
== ROCE_PG_SIZE_8M
?
495 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M
:
496 pbl
->pg_size
== ROCE_PG_SIZE_1G
?
497 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G
:
498 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K
);
500 /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
501 hw_sq_send_ptr
= (struct sq_send
**)sq
->hwq
.pbl_ptr
;
502 for (sw_prod
= 0; sw_prod
< sq
->hwq
.max_elements
; sw_prod
++) {
503 hw_sq_send_hdr
= &hw_sq_send_ptr
[get_sqe_pg(sw_prod
)]
504 [get_sqe_idx(sw_prod
)];
505 hw_sq_send_hdr
->wqe_type
= SQ_BASE_WQE_TYPE_LOCAL_INVALID
;
509 req
.scq_cid
= cpu_to_le32(qp
->scq
->id
);
511 qp_flags
|= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE
;
512 qp_flags
|= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED
;
514 qp_flags
|= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION
;
518 rq
->hwq
.max_elements
= rq
->max_wqe
;
519 rc
= bnxt_qplib_alloc_init_hwq(res
->pdev
, &rq
->hwq
, rq
->sglist
,
520 rq
->nmap
, &rq
->hwq
.max_elements
,
521 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE
, 0,
522 PAGE_SIZE
, HWQ_TYPE_QUEUE
);
526 rq
->swq
= kcalloc(rq
->hwq
.max_elements
, sizeof(*rq
->swq
),
532 pbl
= &rq
->hwq
.pbl
[PBL_LVL_0
];
533 req
.rq_pbl
= cpu_to_le64(pbl
->pg_map_arr
[0]);
534 req
.rq_pg_size_rq_lvl
=
535 ((rq
->hwq
.level
& CMDQ_CREATE_QP_RQ_LVL_MASK
) <<
536 CMDQ_CREATE_QP_RQ_LVL_SFT
) |
537 (pbl
->pg_size
== ROCE_PG_SIZE_4K
?
538 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K
:
539 pbl
->pg_size
== ROCE_PG_SIZE_8K
?
540 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K
:
541 pbl
->pg_size
== ROCE_PG_SIZE_64K
?
542 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K
:
543 pbl
->pg_size
== ROCE_PG_SIZE_2M
?
544 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M
:
545 pbl
->pg_size
== ROCE_PG_SIZE_8M
?
546 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M
:
547 pbl
->pg_size
== ROCE_PG_SIZE_1G
?
548 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G
:
549 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K
);
553 req
.rcq_cid
= cpu_to_le32(qp
->rcq
->id
);
554 req
.qp_flags
= cpu_to_le32(qp_flags
);
555 req
.sq_size
= cpu_to_le32(sq
->hwq
.max_elements
);
556 req
.rq_size
= cpu_to_le32(rq
->hwq
.max_elements
);
557 qp
->sq_hdr_buf
= NULL
;
558 qp
->rq_hdr_buf
= NULL
;
560 rc
= bnxt_qplib_alloc_qp_hdr_buf(res
, qp
);
564 /* CTRL-22434: Irrespective of the requested SGE count on the SQ
565 * always create the QP with max send sges possible if the requested
566 * inline size is greater than 0.
568 max_ssge
= qp
->max_inline_data
? 6 : sq
->max_sge
;
569 req
.sq_fwo_sq_sge
= cpu_to_le16(
570 ((max_ssge
& CMDQ_CREATE_QP_SQ_SGE_MASK
)
571 << CMDQ_CREATE_QP_SQ_SGE_SFT
) | 0);
572 req
.rq_fwo_rq_sge
= cpu_to_le16(
573 ((rq
->max_sge
& CMDQ_CREATE_QP_RQ_SGE_MASK
)
574 << CMDQ_CREATE_QP_RQ_SGE_SFT
) | 0);
579 ORD_LIMIT_TO_ORRQ_SLOTS(qp
->max_rd_atomic
);
580 req_size
= xrrq
->max_elements
*
581 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE
+ PAGE_SIZE
- 1;
582 req_size
&= ~(PAGE_SIZE
- 1);
583 rc
= bnxt_qplib_alloc_init_hwq(res
->pdev
, xrrq
, NULL
, 0,
585 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE
,
586 0, req_size
, HWQ_TYPE_CTX
);
589 pbl
= &xrrq
->pbl
[PBL_LVL_0
];
590 req
.orrq_addr
= cpu_to_le64(pbl
->pg_map_arr
[0]);
593 xrrq
->max_elements
= IRD_LIMIT_TO_IRRQ_SLOTS(
594 qp
->max_dest_rd_atomic
);
595 req_size
= xrrq
->max_elements
*
596 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE
+ PAGE_SIZE
- 1;
597 req_size
&= ~(PAGE_SIZE
- 1);
599 rc
= bnxt_qplib_alloc_init_hwq(res
->pdev
, xrrq
, NULL
, 0,
601 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE
,
602 0, req_size
, HWQ_TYPE_CTX
);
606 pbl
= &xrrq
->pbl
[PBL_LVL_0
];
607 req
.irrq_addr
= cpu_to_le64(pbl
->pg_map_arr
[0]);
609 req
.pd_id
= cpu_to_le32(qp
->pd
->id
);
611 rc
= bnxt_qplib_rcfw_send_message(rcfw
, (void *)&req
,
612 (void *)&resp
, NULL
, 0);
616 qp
->id
= le32_to_cpu(resp
.xid
);
617 qp
->cur_qp_state
= CMDQ_MODIFY_QP_NEW_STATE_RESET
;
618 sq
->flush_in_progress
= false;
619 rq
->flush_in_progress
= false;
624 if (qp
->irrq
.max_elements
)
625 bnxt_qplib_free_hwq(res
->pdev
, &qp
->irrq
);
627 if (qp
->orrq
.max_elements
)
628 bnxt_qplib_free_hwq(res
->pdev
, &qp
->orrq
);
630 bnxt_qplib_free_qp_hdr_buf(res
, qp
);
632 bnxt_qplib_free_hwq(res
->pdev
, &rq
->hwq
);
635 bnxt_qplib_free_hwq(res
->pdev
, &sq
->hwq
);
641 static void __modify_flags_from_init_state(struct bnxt_qplib_qp
*qp
)
644 case CMDQ_MODIFY_QP_NEW_STATE_RTR
:
645 /* INIT->RTR, configure the path_mtu to the default
646 * 2048 if not being requested
648 if (!(qp
->modify_flags
&
649 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU
)) {
651 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU
;
653 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048
;
656 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID
;
657 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
658 if (qp
->max_dest_rd_atomic
< 1)
659 qp
->max_dest_rd_atomic
= 1;
660 qp
->modify_flags
&= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC
;
661 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
662 if (!(qp
->modify_flags
&
663 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX
)) {
665 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX
;
666 qp
->ah
.sgid_index
= 0;
674 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp
*qp
)
677 case CMDQ_MODIFY_QP_NEW_STATE_RTS
:
678 /* Bono FW requires the max_rd_atomic to be >= 1 */
679 if (qp
->max_rd_atomic
< 1)
680 qp
->max_rd_atomic
= 1;
681 /* Bono FW does not allow PKEY_INDEX,
682 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
683 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
684 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
688 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY
|
689 CMDQ_MODIFY_QP_MODIFY_MASK_DGID
|
690 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL
|
691 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX
|
692 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT
|
693 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS
|
694 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC
|
695 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU
|
696 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN
|
697 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER
|
698 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC
|
699 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID
);
706 static void __filter_modify_flags(struct bnxt_qplib_qp
*qp
)
708 switch (qp
->cur_qp_state
) {
709 case CMDQ_MODIFY_QP_NEW_STATE_RESET
:
711 case CMDQ_MODIFY_QP_NEW_STATE_INIT
:
712 __modify_flags_from_init_state(qp
);
714 case CMDQ_MODIFY_QP_NEW_STATE_RTR
:
715 __modify_flags_from_rtr_state(qp
);
717 case CMDQ_MODIFY_QP_NEW_STATE_RTS
:
719 case CMDQ_MODIFY_QP_NEW_STATE_SQD
:
721 case CMDQ_MODIFY_QP_NEW_STATE_SQE
:
723 case CMDQ_MODIFY_QP_NEW_STATE_ERR
:
730 int bnxt_qplib_modify_qp(struct bnxt_qplib_res
*res
, struct bnxt_qplib_qp
*qp
)
732 struct bnxt_qplib_rcfw
*rcfw
= res
->rcfw
;
733 struct cmdq_modify_qp req
;
734 struct creq_modify_qp_resp resp
;
735 u16 cmd_flags
= 0, pkey
;
740 RCFW_CMD_PREP(req
, MODIFY_QP
, cmd_flags
);
742 /* Filter out the qp_attr_mask based on the state->new transition */
743 __filter_modify_flags(qp
);
744 bmask
= qp
->modify_flags
;
745 req
.modify_mask
= cpu_to_le32(qp
->modify_flags
);
746 req
.qp_cid
= cpu_to_le32(qp
->id
);
747 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_STATE
) {
748 req
.network_type_en_sqd_async_notify_new_state
=
749 (qp
->state
& CMDQ_MODIFY_QP_NEW_STATE_MASK
) |
750 (qp
->en_sqd_async_notify
?
751 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY
: 0);
753 req
.network_type_en_sqd_async_notify_new_state
|= qp
->nw_type
;
755 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS
)
756 req
.access
= qp
->access
;
758 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_PKEY
) {
759 if (!bnxt_qplib_get_pkey(res
, &res
->pkey_tbl
,
760 qp
->pkey_index
, &pkey
))
761 req
.pkey
= cpu_to_le16(pkey
);
763 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
)
764 req
.qkey
= cpu_to_le32(qp
->qkey
);
766 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_DGID
) {
767 memcpy(temp32
, qp
->ah
.dgid
.data
, sizeof(struct bnxt_qplib_gid
));
768 req
.dgid
[0] = cpu_to_le32(temp32
[0]);
769 req
.dgid
[1] = cpu_to_le32(temp32
[1]);
770 req
.dgid
[2] = cpu_to_le32(temp32
[2]);
771 req
.dgid
[3] = cpu_to_le32(temp32
[3]);
773 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL
)
774 req
.flow_label
= cpu_to_le32(qp
->ah
.flow_label
);
776 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX
)
777 req
.sgid_index
= cpu_to_le16(res
->sgid_tbl
.hw_id
778 [qp
->ah
.sgid_index
]);
780 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT
)
781 req
.hop_limit
= qp
->ah
.hop_limit
;
783 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS
)
784 req
.traffic_class
= qp
->ah
.traffic_class
;
786 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC
)
787 memcpy(req
.dest_mac
, qp
->ah
.dmac
, 6);
789 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU
)
790 req
.path_mtu
= qp
->path_mtu
;
792 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT
)
793 req
.timeout
= qp
->timeout
;
795 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT
)
796 req
.retry_cnt
= qp
->retry_cnt
;
798 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY
)
799 req
.rnr_retry
= qp
->rnr_retry
;
801 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER
)
802 req
.min_rnr_timer
= qp
->min_rnr_timer
;
804 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN
)
805 req
.rq_psn
= cpu_to_le32(qp
->rq
.psn
);
807 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN
)
808 req
.sq_psn
= cpu_to_le32(qp
->sq
.psn
);
810 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC
)
812 ORD_LIMIT_TO_ORRQ_SLOTS(qp
->max_rd_atomic
);
814 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC
)
815 req
.max_dest_rd_atomic
=
816 IRD_LIMIT_TO_IRRQ_SLOTS(qp
->max_dest_rd_atomic
);
818 req
.sq_size
= cpu_to_le32(qp
->sq
.hwq
.max_elements
);
819 req
.rq_size
= cpu_to_le32(qp
->rq
.hwq
.max_elements
);
820 req
.sq_sge
= cpu_to_le16(qp
->sq
.max_sge
);
821 req
.rq_sge
= cpu_to_le16(qp
->rq
.max_sge
);
822 req
.max_inline_data
= cpu_to_le32(qp
->max_inline_data
);
823 if (bmask
& CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID
)
824 req
.dest_qp_id
= cpu_to_le32(qp
->dest_qpn
);
826 req
.vlan_pcp_vlan_dei_vlan_id
= cpu_to_le16(qp
->vlan_id
);
828 rc
= bnxt_qplib_rcfw_send_message(rcfw
, (void *)&req
,
829 (void *)&resp
, NULL
, 0);
832 qp
->cur_qp_state
= qp
->state
;
836 int bnxt_qplib_query_qp(struct bnxt_qplib_res
*res
, struct bnxt_qplib_qp
*qp
)
838 struct bnxt_qplib_rcfw
*rcfw
= res
->rcfw
;
839 struct cmdq_query_qp req
;
840 struct creq_query_qp_resp resp
;
841 struct bnxt_qplib_rcfw_sbuf
*sbuf
;
842 struct creq_query_qp_resp_sb
*sb
;
847 RCFW_CMD_PREP(req
, QUERY_QP
, cmd_flags
);
849 sbuf
= bnxt_qplib_rcfw_alloc_sbuf(rcfw
, sizeof(*sb
));
854 req
.qp_cid
= cpu_to_le32(qp
->id
);
855 req
.resp_size
= sizeof(*sb
) / BNXT_QPLIB_CMDQE_UNITS
;
856 rc
= bnxt_qplib_rcfw_send_message(rcfw
, (void *)&req
, (void *)&resp
,
860 /* Extract the context from the side buffer */
861 qp
->state
= sb
->en_sqd_async_notify_state
&
862 CREQ_QUERY_QP_RESP_SB_STATE_MASK
;
863 qp
->en_sqd_async_notify
= sb
->en_sqd_async_notify_state
&
864 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY
?
866 qp
->access
= sb
->access
;
867 qp
->pkey_index
= le16_to_cpu(sb
->pkey
);
868 qp
->qkey
= le32_to_cpu(sb
->qkey
);
870 temp32
[0] = le32_to_cpu(sb
->dgid
[0]);
871 temp32
[1] = le32_to_cpu(sb
->dgid
[1]);
872 temp32
[2] = le32_to_cpu(sb
->dgid
[2]);
873 temp32
[3] = le32_to_cpu(sb
->dgid
[3]);
874 memcpy(qp
->ah
.dgid
.data
, temp32
, sizeof(qp
->ah
.dgid
.data
));
876 qp
->ah
.flow_label
= le32_to_cpu(sb
->flow_label
);
878 qp
->ah
.sgid_index
= 0;
879 for (i
= 0; i
< res
->sgid_tbl
.max
; i
++) {
880 if (res
->sgid_tbl
.hw_id
[i
] == le16_to_cpu(sb
->sgid_index
)) {
881 qp
->ah
.sgid_index
= i
;
885 if (i
== res
->sgid_tbl
.max
)
886 dev_warn(&res
->pdev
->dev
, "QPLIB: SGID not found??");
888 qp
->ah
.hop_limit
= sb
->hop_limit
;
889 qp
->ah
.traffic_class
= sb
->traffic_class
;
890 memcpy(qp
->ah
.dmac
, sb
->dest_mac
, 6);
891 qp
->ah
.vlan_id
= (le16_to_cpu(sb
->path_mtu_dest_vlan_id
) &
892 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK
) >>
893 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT
;
894 qp
->path_mtu
= (le16_to_cpu(sb
->path_mtu_dest_vlan_id
) &
895 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK
) >>
896 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT
;
897 qp
->timeout
= sb
->timeout
;
898 qp
->retry_cnt
= sb
->retry_cnt
;
899 qp
->rnr_retry
= sb
->rnr_retry
;
900 qp
->min_rnr_timer
= sb
->min_rnr_timer
;
901 qp
->rq
.psn
= le32_to_cpu(sb
->rq_psn
);
902 qp
->max_rd_atomic
= ORRQ_SLOTS_TO_ORD_LIMIT(sb
->max_rd_atomic
);
903 qp
->sq
.psn
= le32_to_cpu(sb
->sq_psn
);
904 qp
->max_dest_rd_atomic
=
905 IRRQ_SLOTS_TO_IRD_LIMIT(sb
->max_dest_rd_atomic
);
906 qp
->sq
.max_wqe
= qp
->sq
.hwq
.max_elements
;
907 qp
->rq
.max_wqe
= qp
->rq
.hwq
.max_elements
;
908 qp
->sq
.max_sge
= le16_to_cpu(sb
->sq_sge
);
909 qp
->rq
.max_sge
= le16_to_cpu(sb
->rq_sge
);
910 qp
->max_inline_data
= le32_to_cpu(sb
->max_inline_data
);
911 qp
->dest_qpn
= le32_to_cpu(sb
->dest_qp_id
);
912 memcpy(qp
->smac
, sb
->src_mac
, 6);
913 qp
->vlan_id
= le16_to_cpu(sb
->vlan_pcp_vlan_dei_vlan_id
);
915 bnxt_qplib_rcfw_free_sbuf(rcfw
, sbuf
);
919 static void __clean_cq(struct bnxt_qplib_cq
*cq
, u64 qp
)
921 struct bnxt_qplib_hwq
*cq_hwq
= &cq
->hwq
;
922 struct cq_base
*hw_cqe
, **hw_cqe_ptr
;
925 for (i
= 0; i
< cq_hwq
->max_elements
; i
++) {
926 hw_cqe_ptr
= (struct cq_base
**)cq_hwq
->pbl_ptr
;
927 hw_cqe
= &hw_cqe_ptr
[CQE_PG(i
)][CQE_IDX(i
)];
928 if (!CQE_CMP_VALID(hw_cqe
, i
, cq_hwq
->max_elements
))
930 switch (hw_cqe
->cqe_type_toggle
& CQ_BASE_CQE_TYPE_MASK
) {
931 case CQ_BASE_CQE_TYPE_REQ
:
932 case CQ_BASE_CQE_TYPE_TERMINAL
:
934 struct cq_req
*cqe
= (struct cq_req
*)hw_cqe
;
936 if (qp
== le64_to_cpu(cqe
->qp_handle
))
940 case CQ_BASE_CQE_TYPE_RES_RC
:
941 case CQ_BASE_CQE_TYPE_RES_UD
:
942 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1
:
944 struct cq_res_rc
*cqe
= (struct cq_res_rc
*)hw_cqe
;
946 if (qp
== le64_to_cpu(cqe
->qp_handle
))
956 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res
*res
,
957 struct bnxt_qplib_qp
*qp
)
959 struct bnxt_qplib_rcfw
*rcfw
= res
->rcfw
;
960 struct cmdq_destroy_qp req
;
961 struct creq_destroy_qp_resp resp
;
966 RCFW_CMD_PREP(req
, DESTROY_QP
, cmd_flags
);
968 req
.qp_cid
= cpu_to_le32(qp
->id
);
969 rc
= bnxt_qplib_rcfw_send_message(rcfw
, (void *)&req
,
970 (void *)&resp
, NULL
, 0);
974 /* Must walk the associated CQs to nullified the QP ptr */
975 spin_lock_irqsave(&qp
->scq
->hwq
.lock
, flags
);
977 __clean_cq(qp
->scq
, (u64
)(unsigned long)qp
);
979 if (qp
->rcq
&& qp
->rcq
!= qp
->scq
) {
980 spin_lock(&qp
->rcq
->hwq
.lock
);
981 __clean_cq(qp
->rcq
, (u64
)(unsigned long)qp
);
982 spin_unlock(&qp
->rcq
->hwq
.lock
);
985 spin_unlock_irqrestore(&qp
->scq
->hwq
.lock
, flags
);
987 bnxt_qplib_free_qp_hdr_buf(res
, qp
);
988 bnxt_qplib_free_hwq(res
->pdev
, &qp
->sq
.hwq
);
991 bnxt_qplib_free_hwq(res
->pdev
, &qp
->rq
.hwq
);
994 if (qp
->irrq
.max_elements
)
995 bnxt_qplib_free_hwq(res
->pdev
, &qp
->irrq
);
996 if (qp
->orrq
.max_elements
)
997 bnxt_qplib_free_hwq(res
->pdev
, &qp
->orrq
);
1002 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp
*qp
,
1003 struct bnxt_qplib_sge
*sge
)
1005 struct bnxt_qplib_q
*sq
= &qp
->sq
;
1008 memset(sge
, 0, sizeof(*sge
));
1010 if (qp
->sq_hdr_buf
) {
1011 sw_prod
= HWQ_CMP(sq
->hwq
.prod
, &sq
->hwq
);
1012 sge
->addr
= (dma_addr_t
)(qp
->sq_hdr_buf_map
+
1013 sw_prod
* qp
->sq_hdr_buf_size
);
1014 sge
->lkey
= 0xFFFFFFFF;
1015 sge
->size
= qp
->sq_hdr_buf_size
;
1016 return qp
->sq_hdr_buf
+ sw_prod
* sge
->size
;
1021 u32
bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp
*qp
)
1023 struct bnxt_qplib_q
*rq
= &qp
->rq
;
1025 return HWQ_CMP(rq
->hwq
.prod
, &rq
->hwq
);
1028 dma_addr_t
bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp
*qp
, u32 index
)
1030 return (qp
->rq_hdr_buf_map
+ index
* qp
->rq_hdr_buf_size
);
1033 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp
*qp
,
1034 struct bnxt_qplib_sge
*sge
)
1036 struct bnxt_qplib_q
*rq
= &qp
->rq
;
1039 memset(sge
, 0, sizeof(*sge
));
1041 if (qp
->rq_hdr_buf
) {
1042 sw_prod
= HWQ_CMP(rq
->hwq
.prod
, &rq
->hwq
);
1043 sge
->addr
= (dma_addr_t
)(qp
->rq_hdr_buf_map
+
1044 sw_prod
* qp
->rq_hdr_buf_size
);
1045 sge
->lkey
= 0xFFFFFFFF;
1046 sge
->size
= qp
->rq_hdr_buf_size
;
1047 return qp
->rq_hdr_buf
+ sw_prod
* sge
->size
;
1052 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp
*qp
)
1054 struct bnxt_qplib_q
*sq
= &qp
->sq
;
1055 struct dbr_dbr db_msg
= { 0 };
1058 sw_prod
= HWQ_CMP(sq
->hwq
.prod
, &sq
->hwq
);
1060 db_msg
.index
= cpu_to_le32((sw_prod
<< DBR_DBR_INDEX_SFT
) &
1061 DBR_DBR_INDEX_MASK
);
1063 cpu_to_le32(((qp
->id
<< DBR_DBR_XID_SFT
) & DBR_DBR_XID_MASK
) |
1065 /* Flush all the WQE writes to HW */
1067 __iowrite64_copy(qp
->dpi
->dbr
, &db_msg
, sizeof(db_msg
) / sizeof(u64
));
1070 int bnxt_qplib_post_send(struct bnxt_qplib_qp
*qp
,
1071 struct bnxt_qplib_swqe
*wqe
)
1073 struct bnxt_qplib_q
*sq
= &qp
->sq
;
1074 struct bnxt_qplib_swq
*swq
;
1075 struct sq_send
*hw_sq_send_hdr
, **hw_sq_send_ptr
;
1076 struct sq_sge
*hw_sge
;
1079 int i
, rc
= 0, data_len
= 0, pkt_num
= 0;
1082 if (qp
->state
!= CMDQ_MODIFY_QP_NEW_STATE_RTS
) {
1087 if (bnxt_qplib_queue_full(sq
)) {
1088 dev_err(&sq
->hwq
.pdev
->dev
,
1089 "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x",
1090 sq
->hwq
.prod
, sq
->hwq
.cons
, sq
->hwq
.max_elements
,
1095 sw_prod
= HWQ_CMP(sq
->hwq
.prod
, &sq
->hwq
);
1096 swq
= &sq
->swq
[sw_prod
];
1097 swq
->wr_id
= wqe
->wr_id
;
1098 swq
->type
= wqe
->type
;
1099 swq
->flags
= wqe
->flags
;
1101 swq
->flags
|= SQ_SEND_FLAGS_SIGNAL_COMP
;
1102 swq
->start_psn
= sq
->psn
& BTH_PSN_MASK
;
1104 hw_sq_send_ptr
= (struct sq_send
**)sq
->hwq
.pbl_ptr
;
1105 hw_sq_send_hdr
= &hw_sq_send_ptr
[get_sqe_pg(sw_prod
)]
1106 [get_sqe_idx(sw_prod
)];
1108 memset(hw_sq_send_hdr
, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE
);
1110 if (wqe
->flags
& BNXT_QPLIB_SWQE_FLAGS_INLINE
) {
1111 /* Copy the inline data */
1112 if (wqe
->inline_len
> BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH
) {
1113 dev_warn(&sq
->hwq
.pdev
->dev
,
1114 "QPLIB: Inline data length > 96 detected");
1115 data_len
= BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH
;
1117 data_len
= wqe
->inline_len
;
1119 memcpy(hw_sq_send_hdr
->data
, wqe
->inline_data
, data_len
);
1120 wqe_size16
= (data_len
+ 15) >> 4;
1122 for (i
= 0, hw_sge
= (struct sq_sge
*)hw_sq_send_hdr
->data
;
1123 i
< wqe
->num_sge
; i
++, hw_sge
++) {
1124 hw_sge
->va_or_pa
= cpu_to_le64(wqe
->sg_list
[i
].addr
);
1125 hw_sge
->l_key
= cpu_to_le32(wqe
->sg_list
[i
].lkey
);
1126 hw_sge
->size
= cpu_to_le32(wqe
->sg_list
[i
].size
);
1127 data_len
+= wqe
->sg_list
[i
].size
;
1129 /* Each SGE entry = 1 WQE size16 */
1130 wqe_size16
= wqe
->num_sge
;
1134 switch (wqe
->type
) {
1135 case BNXT_QPLIB_SWQE_TYPE_SEND
:
1136 if (qp
->type
== CMDQ_CREATE_QP1_TYPE_GSI
) {
1137 /* Assemble info for Raw Ethertype QPs */
1138 struct sq_send_raweth_qp1
*sqe
=
1139 (struct sq_send_raweth_qp1
*)hw_sq_send_hdr
;
1141 sqe
->wqe_type
= wqe
->type
;
1142 sqe
->flags
= wqe
->flags
;
1143 sqe
->wqe_size
= wqe_size16
+
1144 ((offsetof(typeof(*sqe
), data
) + 15) >> 4);
1145 sqe
->cfa_action
= cpu_to_le16(wqe
->rawqp1
.cfa_action
);
1146 sqe
->lflags
= cpu_to_le16(wqe
->rawqp1
.lflags
);
1147 sqe
->length
= cpu_to_le32(data_len
);
1148 sqe
->cfa_meta
= cpu_to_le32((wqe
->rawqp1
.cfa_meta
&
1149 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK
) <<
1150 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT
);
1154 /* else, just fall thru */
1155 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM
:
1156 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV
:
1158 struct sq_send
*sqe
= (struct sq_send
*)hw_sq_send_hdr
;
1160 sqe
->wqe_type
= wqe
->type
;
1161 sqe
->flags
= wqe
->flags
;
1162 sqe
->wqe_size
= wqe_size16
+
1163 ((offsetof(typeof(*sqe
), data
) + 15) >> 4);
1164 sqe
->inv_key_or_imm_data
= cpu_to_le32(
1166 if (qp
->type
== CMDQ_CREATE_QP_TYPE_UD
) {
1167 sqe
->q_key
= cpu_to_le32(wqe
->send
.q_key
);
1168 sqe
->dst_qp
= cpu_to_le32(
1169 wqe
->send
.dst_qp
& SQ_SEND_DST_QP_MASK
);
1170 sqe
->length
= cpu_to_le32(data_len
);
1171 sqe
->avid
= cpu_to_le32(wqe
->send
.avid
&
1173 sq
->psn
= (sq
->psn
+ 1) & BTH_PSN_MASK
;
1175 sqe
->length
= cpu_to_le32(data_len
);
1179 pkt_num
= (data_len
+ qp
->mtu
- 1) / qp
->mtu
;
1182 sq
->psn
= (sq
->psn
+ pkt_num
) & BTH_PSN_MASK
;
1186 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE
:
1187 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM
:
1188 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ
:
1190 struct sq_rdma
*sqe
= (struct sq_rdma
*)hw_sq_send_hdr
;
1192 sqe
->wqe_type
= wqe
->type
;
1193 sqe
->flags
= wqe
->flags
;
1194 sqe
->wqe_size
= wqe_size16
+
1195 ((offsetof(typeof(*sqe
), data
) + 15) >> 4);
1196 sqe
->imm_data
= cpu_to_le32(wqe
->rdma
.inv_key
);
1197 sqe
->length
= cpu_to_le32((u32
)data_len
);
1198 sqe
->remote_va
= cpu_to_le64(wqe
->rdma
.remote_va
);
1199 sqe
->remote_key
= cpu_to_le32(wqe
->rdma
.r_key
);
1201 pkt_num
= (data_len
+ qp
->mtu
- 1) / qp
->mtu
;
1204 sq
->psn
= (sq
->psn
+ pkt_num
) & BTH_PSN_MASK
;
1207 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP
:
1208 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD
:
1210 struct sq_atomic
*sqe
= (struct sq_atomic
*)hw_sq_send_hdr
;
1212 sqe
->wqe_type
= wqe
->type
;
1213 sqe
->flags
= wqe
->flags
;
1214 sqe
->remote_key
= cpu_to_le32(wqe
->atomic
.r_key
);
1215 sqe
->remote_va
= cpu_to_le64(wqe
->atomic
.remote_va
);
1216 sqe
->swap_data
= cpu_to_le64(wqe
->atomic
.swap_data
);
1217 sqe
->cmp_data
= cpu_to_le64(wqe
->atomic
.cmp_data
);
1219 pkt_num
= (data_len
+ qp
->mtu
- 1) / qp
->mtu
;
1222 sq
->psn
= (sq
->psn
+ pkt_num
) & BTH_PSN_MASK
;
1225 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV
:
1227 struct sq_localinvalidate
*sqe
=
1228 (struct sq_localinvalidate
*)hw_sq_send_hdr
;
1230 sqe
->wqe_type
= wqe
->type
;
1231 sqe
->flags
= wqe
->flags
;
1232 sqe
->inv_l_key
= cpu_to_le32(wqe
->local_inv
.inv_l_key
);
1236 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR
:
1238 struct sq_fr_pmr
*sqe
= (struct sq_fr_pmr
*)hw_sq_send_hdr
;
1240 sqe
->wqe_type
= wqe
->type
;
1241 sqe
->flags
= wqe
->flags
;
1242 sqe
->access_cntl
= wqe
->frmr
.access_cntl
|
1243 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE
;
1244 sqe
->zero_based_page_size_log
=
1245 (wqe
->frmr
.pg_sz_log
& SQ_FR_PMR_PAGE_SIZE_LOG_MASK
) <<
1246 SQ_FR_PMR_PAGE_SIZE_LOG_SFT
|
1247 (wqe
->frmr
.zero_based
? SQ_FR_PMR_ZERO_BASED
: 0);
1248 sqe
->l_key
= cpu_to_le32(wqe
->frmr
.l_key
);
1249 temp32
= cpu_to_le32(wqe
->frmr
.length
);
1250 memcpy(sqe
->length
, &temp32
, sizeof(wqe
->frmr
.length
));
1251 sqe
->numlevels_pbl_page_size_log
=
1252 ((wqe
->frmr
.pbl_pg_sz_log
<<
1253 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT
) &
1254 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK
) |
1255 ((wqe
->frmr
.levels
<< SQ_FR_PMR_NUMLEVELS_SFT
) &
1256 SQ_FR_PMR_NUMLEVELS_MASK
);
1258 for (i
= 0; i
< wqe
->frmr
.page_list_len
; i
++)
1259 wqe
->frmr
.pbl_ptr
[i
] = cpu_to_le64(
1260 wqe
->frmr
.page_list
[i
] |
1262 sqe
->pblptr
= cpu_to_le64(wqe
->frmr
.pbl_dma_ptr
);
1263 sqe
->va
= cpu_to_le64(wqe
->frmr
.va
);
1267 case BNXT_QPLIB_SWQE_TYPE_BIND_MW
:
1269 struct sq_bind
*sqe
= (struct sq_bind
*)hw_sq_send_hdr
;
1271 sqe
->wqe_type
= wqe
->type
;
1272 sqe
->flags
= wqe
->flags
;
1273 sqe
->access_cntl
= wqe
->bind
.access_cntl
;
1274 sqe
->mw_type_zero_based
= wqe
->bind
.mw_type
|
1275 (wqe
->bind
.zero_based
? SQ_BIND_ZERO_BASED
: 0);
1276 sqe
->parent_l_key
= cpu_to_le32(wqe
->bind
.parent_l_key
);
1277 sqe
->l_key
= cpu_to_le32(wqe
->bind
.r_key
);
1278 sqe
->va
= cpu_to_le64(wqe
->bind
.va
);
1279 temp32
= cpu_to_le32(wqe
->bind
.length
);
1280 memcpy(&sqe
->length
, &temp32
, sizeof(wqe
->bind
.length
));
1284 /* Bad wqe, return error */
1288 swq
->next_psn
= sq
->psn
& BTH_PSN_MASK
;
1289 if (swq
->psn_search
) {
1290 swq
->psn_search
->opcode_start_psn
= cpu_to_le32(
1291 ((swq
->start_psn
<< SQ_PSN_SEARCH_START_PSN_SFT
) &
1292 SQ_PSN_SEARCH_START_PSN_MASK
) |
1293 ((wqe
->type
<< SQ_PSN_SEARCH_OPCODE_SFT
) &
1294 SQ_PSN_SEARCH_OPCODE_MASK
));
1295 swq
->psn_search
->flags_next_psn
= cpu_to_le32(
1296 ((swq
->next_psn
<< SQ_PSN_SEARCH_NEXT_PSN_SFT
) &
1297 SQ_PSN_SEARCH_NEXT_PSN_MASK
));
1308 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp
*qp
)
1310 struct bnxt_qplib_q
*rq
= &qp
->rq
;
1311 struct dbr_dbr db_msg
= { 0 };
1314 sw_prod
= HWQ_CMP(rq
->hwq
.prod
, &rq
->hwq
);
1315 db_msg
.index
= cpu_to_le32((sw_prod
<< DBR_DBR_INDEX_SFT
) &
1316 DBR_DBR_INDEX_MASK
);
1318 cpu_to_le32(((qp
->id
<< DBR_DBR_XID_SFT
) & DBR_DBR_XID_MASK
) |
1321 /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1323 __iowrite64_copy(qp
->dpi
->dbr
, &db_msg
, sizeof(db_msg
) / sizeof(u64
));
1326 int bnxt_qplib_post_recv(struct bnxt_qplib_qp
*qp
,
1327 struct bnxt_qplib_swqe
*wqe
)
1329 struct bnxt_qplib_q
*rq
= &qp
->rq
;
1330 struct rq_wqe
*rqe
, **rqe_ptr
;
1331 struct sq_sge
*hw_sge
;
1335 if (qp
->state
== CMDQ_MODIFY_QP_NEW_STATE_ERR
) {
1336 dev_err(&rq
->hwq
.pdev
->dev
,
1337 "QPLIB: FP: QP (0x%x) is in the 0x%x state",
1342 if (bnxt_qplib_queue_full(rq
)) {
1343 dev_err(&rq
->hwq
.pdev
->dev
,
1344 "QPLIB: FP: QP (0x%x) RQ is full!", qp
->id
);
1348 sw_prod
= HWQ_CMP(rq
->hwq
.prod
, &rq
->hwq
);
1349 rq
->swq
[sw_prod
].wr_id
= wqe
->wr_id
;
1351 rqe_ptr
= (struct rq_wqe
**)rq
->hwq
.pbl_ptr
;
1352 rqe
= &rqe_ptr
[RQE_PG(sw_prod
)][RQE_IDX(sw_prod
)];
1354 memset(rqe
, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE
);
1356 /* Calculate wqe_size16 and data_len */
1357 for (i
= 0, hw_sge
= (struct sq_sge
*)rqe
->data
;
1358 i
< wqe
->num_sge
; i
++, hw_sge
++) {
1359 hw_sge
->va_or_pa
= cpu_to_le64(wqe
->sg_list
[i
].addr
);
1360 hw_sge
->l_key
= cpu_to_le32(wqe
->sg_list
[i
].lkey
);
1361 hw_sge
->size
= cpu_to_le32(wqe
->sg_list
[i
].size
);
1363 rqe
->wqe_type
= wqe
->type
;
1364 rqe
->flags
= wqe
->flags
;
1365 rqe
->wqe_size
= wqe
->num_sge
+
1366 ((offsetof(typeof(*rqe
), data
) + 15) >> 4);
1368 /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1369 rqe
->wr_id
[0] = cpu_to_le32(sw_prod
);
1378 /* Spinlock must be held */
1379 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq
*cq
)
1381 struct dbr_dbr db_msg
= { 0 };
1384 cpu_to_le32(((cq
->id
<< DBR_DBR_XID_SFT
) & DBR_DBR_XID_MASK
) |
1385 DBR_DBR_TYPE_CQ_ARMENA
);
1386 /* Flush memory writes before enabling the CQ */
1388 __iowrite64_copy(cq
->dbr_base
, &db_msg
, sizeof(db_msg
) / sizeof(u64
));
1391 static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq
*cq
, u32 arm_type
)
1393 struct bnxt_qplib_hwq
*cq_hwq
= &cq
->hwq
;
1394 struct dbr_dbr db_msg
= { 0 };
1398 sw_cons
= HWQ_CMP(cq_hwq
->cons
, cq_hwq
);
1399 db_msg
.index
= cpu_to_le32((sw_cons
<< DBR_DBR_INDEX_SFT
) &
1400 DBR_DBR_INDEX_MASK
);
1402 cpu_to_le32(((cq
->id
<< DBR_DBR_XID_SFT
) & DBR_DBR_XID_MASK
) |
1404 /* flush memory writes before arming the CQ */
1406 __iowrite64_copy(cq
->dpi
->dbr
, &db_msg
, sizeof(db_msg
) / sizeof(u64
));
1409 int bnxt_qplib_create_cq(struct bnxt_qplib_res
*res
, struct bnxt_qplib_cq
*cq
)
1411 struct bnxt_qplib_rcfw
*rcfw
= res
->rcfw
;
1412 struct cmdq_create_cq req
;
1413 struct creq_create_cq_resp resp
;
1414 struct bnxt_qplib_pbl
*pbl
;
1418 cq
->hwq
.max_elements
= cq
->max_wqe
;
1419 rc
= bnxt_qplib_alloc_init_hwq(res
->pdev
, &cq
->hwq
, cq
->sghead
,
1420 cq
->nmap
, &cq
->hwq
.max_elements
,
1421 BNXT_QPLIB_MAX_CQE_ENTRY_SIZE
, 0,
1422 PAGE_SIZE
, HWQ_TYPE_QUEUE
);
1426 RCFW_CMD_PREP(req
, CREATE_CQ
, cmd_flags
);
1429 dev_err(&rcfw
->pdev
->dev
,
1430 "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1433 req
.dpi
= cpu_to_le32(cq
->dpi
->dpi
);
1434 req
.cq_handle
= cpu_to_le64(cq
->cq_handle
);
1436 req
.cq_size
= cpu_to_le32(cq
->hwq
.max_elements
);
1437 pbl
= &cq
->hwq
.pbl
[PBL_LVL_0
];
1438 req
.pg_size_lvl
= cpu_to_le32(
1439 ((cq
->hwq
.level
& CMDQ_CREATE_CQ_LVL_MASK
) <<
1440 CMDQ_CREATE_CQ_LVL_SFT
) |
1441 (pbl
->pg_size
== ROCE_PG_SIZE_4K
? CMDQ_CREATE_CQ_PG_SIZE_PG_4K
:
1442 pbl
->pg_size
== ROCE_PG_SIZE_8K
? CMDQ_CREATE_CQ_PG_SIZE_PG_8K
:
1443 pbl
->pg_size
== ROCE_PG_SIZE_64K
? CMDQ_CREATE_CQ_PG_SIZE_PG_64K
:
1444 pbl
->pg_size
== ROCE_PG_SIZE_2M
? CMDQ_CREATE_CQ_PG_SIZE_PG_2M
:
1445 pbl
->pg_size
== ROCE_PG_SIZE_8M
? CMDQ_CREATE_CQ_PG_SIZE_PG_8M
:
1446 pbl
->pg_size
== ROCE_PG_SIZE_1G
? CMDQ_CREATE_CQ_PG_SIZE_PG_1G
:
1447 CMDQ_CREATE_CQ_PG_SIZE_PG_4K
));
1449 req
.pbl
= cpu_to_le64(pbl
->pg_map_arr
[0]);
1451 req
.cq_fco_cnq_id
= cpu_to_le32(
1452 (cq
->cnq_hw_ring_id
& CMDQ_CREATE_CQ_CNQ_ID_MASK
) <<
1453 CMDQ_CREATE_CQ_CNQ_ID_SFT
);
1455 rc
= bnxt_qplib_rcfw_send_message(rcfw
, (void *)&req
,
1456 (void *)&resp
, NULL
, 0);
1460 cq
->id
= le32_to_cpu(resp
.xid
);
1461 cq
->dbr_base
= res
->dpi_tbl
.dbr_bar_reg_iomem
;
1462 cq
->period
= BNXT_QPLIB_QUEUE_START_PERIOD
;
1463 init_waitqueue_head(&cq
->waitq
);
1465 bnxt_qplib_arm_cq_enable(cq
);
1469 bnxt_qplib_free_hwq(res
->pdev
, &cq
->hwq
);
1474 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res
*res
, struct bnxt_qplib_cq
*cq
)
1476 struct bnxt_qplib_rcfw
*rcfw
= res
->rcfw
;
1477 struct cmdq_destroy_cq req
;
1478 struct creq_destroy_cq_resp resp
;
1482 RCFW_CMD_PREP(req
, DESTROY_CQ
, cmd_flags
);
1484 req
.cq_cid
= cpu_to_le32(cq
->id
);
1485 rc
= bnxt_qplib_rcfw_send_message(rcfw
, (void *)&req
,
1486 (void *)&resp
, NULL
, 0);
1489 bnxt_qplib_free_hwq(res
->pdev
, &cq
->hwq
);
1493 static int __flush_sq(struct bnxt_qplib_q
*sq
, struct bnxt_qplib_qp
*qp
,
1494 struct bnxt_qplib_cqe
**pcqe
, int *budget
)
1496 u32 sw_prod
, sw_cons
;
1497 struct bnxt_qplib_cqe
*cqe
;
1500 /* Now complete all outstanding SQEs with FLUSHED_ERR */
1501 sw_prod
= HWQ_CMP(sq
->hwq
.prod
, &sq
->hwq
);
1504 sw_cons
= HWQ_CMP(sq
->hwq
.cons
, &sq
->hwq
);
1505 if (sw_cons
== sw_prod
) {
1506 sq
->flush_in_progress
= false;
1509 memset(cqe
, 0, sizeof(*cqe
));
1510 cqe
->status
= CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR
;
1511 cqe
->opcode
= CQ_BASE_CQE_TYPE_REQ
;
1512 cqe
->qp_handle
= (u64
)(unsigned long)qp
;
1513 cqe
->wr_id
= sq
->swq
[sw_cons
].wr_id
;
1514 cqe
->src_qp
= qp
->id
;
1515 cqe
->type
= sq
->swq
[sw_cons
].type
;
1521 if (!(*budget
) && HWQ_CMP(sq
->hwq
.cons
, &sq
->hwq
) != sw_prod
)
1528 static int __flush_rq(struct bnxt_qplib_q
*rq
, struct bnxt_qplib_qp
*qp
,
1529 int opcode
, struct bnxt_qplib_cqe
**pcqe
, int *budget
)
1531 struct bnxt_qplib_cqe
*cqe
;
1532 u32 sw_prod
, sw_cons
;
1535 /* Flush the rest of the RQ */
1536 sw_prod
= HWQ_CMP(rq
->hwq
.prod
, &rq
->hwq
);
1539 sw_cons
= HWQ_CMP(rq
->hwq
.cons
, &rq
->hwq
);
1540 if (sw_cons
== sw_prod
)
1542 memset(cqe
, 0, sizeof(*cqe
));
1544 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR
;
1545 cqe
->opcode
= opcode
;
1546 cqe
->qp_handle
= (unsigned long)qp
;
1547 cqe
->wr_id
= rq
->swq
[sw_cons
].wr_id
;
1553 if (!*budget
&& HWQ_CMP(rq
->hwq
.cons
, &rq
->hwq
) != sw_prod
)
1560 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
1561 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1
1563 static int do_wa9060(struct bnxt_qplib_qp
*qp
, struct bnxt_qplib_cq
*cq
,
1564 u32 cq_cons
, u32 sw_sq_cons
, u32 cqe_sq_cons
)
1566 struct bnxt_qplib_q
*sq
= &qp
->sq
;
1567 struct bnxt_qplib_swq
*swq
;
1568 u32 peek_sw_cq_cons
, peek_raw_cq_cons
, peek_sq_cons_idx
;
1569 struct cq_base
*peek_hwcqe
, **peek_hw_cqe_ptr
;
1570 struct cq_req
*peek_req_hwcqe
;
1571 struct bnxt_qplib_qp
*peek_qp
;
1572 struct bnxt_qplib_q
*peek_sq
;
1576 /* Check for the psn_search marking before completing */
1577 swq
= &sq
->swq
[sw_sq_cons
];
1578 if (swq
->psn_search
&&
1579 le32_to_cpu(swq
->psn_search
->flags_next_psn
) & 0x80000000) {
1581 swq
->psn_search
->flags_next_psn
= cpu_to_le32
1582 (le32_to_cpu(swq
->psn_search
->flags_next_psn
)
1584 dev_dbg(&cq
->hwq
.pdev
->dev
,
1585 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
1586 cq_cons
, qp
->id
, sw_sq_cons
, cqe_sq_cons
);
1587 sq
->condition
= true;
1588 sq
->send_phantom
= true;
1590 /* TODO: Only ARM if the previous SQE is ARMALL */
1591 bnxt_qplib_arm_cq(cq
, DBR_DBR_TYPE_CQ_ARMALL
);
1596 if (sq
->condition
) {
1597 /* Peek at the completions */
1598 peek_raw_cq_cons
= cq
->hwq
.cons
;
1599 peek_sw_cq_cons
= cq_cons
;
1600 i
= cq
->hwq
.max_elements
;
1602 peek_sw_cq_cons
= HWQ_CMP((peek_sw_cq_cons
), &cq
->hwq
);
1603 peek_hw_cqe_ptr
= (struct cq_base
**)cq
->hwq
.pbl_ptr
;
1604 peek_hwcqe
= &peek_hw_cqe_ptr
[CQE_PG(peek_sw_cq_cons
)]
1605 [CQE_IDX(peek_sw_cq_cons
)];
1606 /* If the next hwcqe is VALID */
1607 if (CQE_CMP_VALID(peek_hwcqe
, peek_raw_cq_cons
,
1608 cq
->hwq
.max_elements
)) {
1609 /* If the next hwcqe is a REQ */
1610 if ((peek_hwcqe
->cqe_type_toggle
&
1611 CQ_BASE_CQE_TYPE_MASK
) ==
1612 CQ_BASE_CQE_TYPE_REQ
) {
1613 peek_req_hwcqe
= (struct cq_req
*)
1615 peek_qp
= (struct bnxt_qplib_qp
*)
1618 (peek_req_hwcqe
->qp_handle
));
1619 peek_sq
= &peek_qp
->sq
;
1620 peek_sq_cons_idx
= HWQ_CMP(le16_to_cpu(
1621 peek_req_hwcqe
->sq_cons_idx
) - 1
1623 /* If the hwcqe's sq's wr_id matches */
1624 if (peek_sq
== sq
&&
1625 sq
->swq
[peek_sq_cons_idx
].wr_id
==
1626 BNXT_QPLIB_FENCE_WRID
) {
1628 * Unbreak only if the phantom
1631 dev_dbg(&cq
->hwq
.pdev
->dev
,
1632 "FP:Got Phantom CQE");
1633 sq
->condition
= false;
1639 /* Valid but not the phantom, so keep looping */
1641 /* Not valid yet, just exit and wait */
1648 dev_err(&cq
->hwq
.pdev
->dev
,
1649 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x",
1650 cq_cons
, qp
->id
, sw_sq_cons
, cqe_sq_cons
);
1657 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq
*cq
,
1658 struct cq_req
*hwcqe
,
1659 struct bnxt_qplib_cqe
**pcqe
, int *budget
,
1660 u32 cq_cons
, struct bnxt_qplib_qp
**lib_qp
)
1662 struct bnxt_qplib_qp
*qp
;
1663 struct bnxt_qplib_q
*sq
;
1664 struct bnxt_qplib_cqe
*cqe
;
1665 u32 sw_sq_cons
, cqe_sq_cons
;
1666 struct bnxt_qplib_swq
*swq
;
1669 qp
= (struct bnxt_qplib_qp
*)((unsigned long)
1670 le64_to_cpu(hwcqe
->qp_handle
));
1672 dev_err(&cq
->hwq
.pdev
->dev
,
1673 "QPLIB: FP: Process Req qp is NULL");
1678 cqe_sq_cons
= HWQ_CMP(le16_to_cpu(hwcqe
->sq_cons_idx
), &sq
->hwq
);
1679 if (cqe_sq_cons
> sq
->hwq
.max_elements
) {
1680 dev_err(&cq
->hwq
.pdev
->dev
,
1681 "QPLIB: FP: CQ Process req reported ");
1682 dev_err(&cq
->hwq
.pdev
->dev
,
1683 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1684 cqe_sq_cons
, sq
->hwq
.max_elements
);
1687 /* If we were in the middle of flushing the SQ, continue */
1688 if (sq
->flush_in_progress
)
1691 /* Require to walk the sq's swq to fabricate CQEs for all previously
1692 * signaled SWQEs due to CQE aggregation from the current sq cons
1693 * to the cqe_sq_cons
1697 sw_sq_cons
= HWQ_CMP(sq
->hwq
.cons
, &sq
->hwq
);
1698 if (sw_sq_cons
== cqe_sq_cons
)
1702 swq
= &sq
->swq
[sw_sq_cons
];
1703 memset(cqe
, 0, sizeof(*cqe
));
1704 cqe
->opcode
= CQ_BASE_CQE_TYPE_REQ
;
1705 cqe
->qp_handle
= (u64
)(unsigned long)qp
;
1706 cqe
->src_qp
= qp
->id
;
1707 cqe
->wr_id
= swq
->wr_id
;
1708 if (cqe
->wr_id
== BNXT_QPLIB_FENCE_WRID
)
1710 cqe
->type
= swq
->type
;
1712 /* For the last CQE, check for status. For errors, regardless
1713 * of the request being signaled or not, it must complete with
1714 * the hwcqe error status
1716 if (HWQ_CMP((sw_sq_cons
+ 1), &sq
->hwq
) == cqe_sq_cons
&&
1717 hwcqe
->status
!= CQ_REQ_STATUS_OK
) {
1718 cqe
->status
= hwcqe
->status
;
1719 dev_err(&cq
->hwq
.pdev
->dev
,
1720 "QPLIB: FP: CQ Processed Req ");
1721 dev_err(&cq
->hwq
.pdev
->dev
,
1722 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
1723 sw_sq_cons
, cqe
->wr_id
, cqe
->status
);
1726 sq
->flush_in_progress
= true;
1727 /* Must block new posting of SQ and RQ */
1728 qp
->state
= CMDQ_MODIFY_QP_NEW_STATE_ERR
;
1729 sq
->condition
= false;
1732 if (swq
->flags
& SQ_SEND_FLAGS_SIGNAL_COMP
) {
1733 /* Before we complete, do WA 9060 */
1734 if (do_wa9060(qp
, cq
, cq_cons
, sw_sq_cons
,
1739 cqe
->status
= CQ_REQ_STATUS_OK
;
1751 if (HWQ_CMP(sq
->hwq
.cons
, &sq
->hwq
) != cqe_sq_cons
) {
1757 * Back to normal completion mode only after it has completed all of
1758 * the WC for this CQE
1761 if (!sq
->flush_in_progress
)
1764 /* Require to walk the sq's swq to fabricate CQEs for all
1765 * previously posted SWQEs due to the error CQE received
1767 rc
= __flush_sq(sq
, qp
, pcqe
, budget
);
1769 sq
->flush_in_progress
= false;
1774 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq
*cq
,
1775 struct cq_res_rc
*hwcqe
,
1776 struct bnxt_qplib_cqe
**pcqe
,
1779 struct bnxt_qplib_qp
*qp
;
1780 struct bnxt_qplib_q
*rq
;
1781 struct bnxt_qplib_cqe
*cqe
;
1785 qp
= (struct bnxt_qplib_qp
*)((unsigned long)
1786 le64_to_cpu(hwcqe
->qp_handle
));
1788 dev_err(&cq
->hwq
.pdev
->dev
, "QPLIB: process_cq RC qp is NULL");
1792 cqe
->opcode
= hwcqe
->cqe_type_toggle
& CQ_BASE_CQE_TYPE_MASK
;
1793 cqe
->length
= le32_to_cpu(hwcqe
->length
);
1794 cqe
->invrkey
= le32_to_cpu(hwcqe
->imm_data_or_inv_r_key
);
1795 cqe
->mr_handle
= le64_to_cpu(hwcqe
->mr_handle
);
1796 cqe
->flags
= le16_to_cpu(hwcqe
->flags
);
1797 cqe
->status
= hwcqe
->status
;
1798 cqe
->qp_handle
= (u64
)(unsigned long)qp
;
1800 wr_id_idx
= le32_to_cpu(hwcqe
->srq_or_rq_wr_id
) &
1801 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK
;
1803 if (wr_id_idx
> rq
->hwq
.max_elements
) {
1804 dev_err(&cq
->hwq
.pdev
->dev
, "QPLIB: FP: CQ Process RC ");
1805 dev_err(&cq
->hwq
.pdev
->dev
,
1806 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
1807 wr_id_idx
, rq
->hwq
.max_elements
);
1810 if (rq
->flush_in_progress
)
1813 cqe
->wr_id
= rq
->swq
[wr_id_idx
].wr_id
;
1819 if (hwcqe
->status
!= CQ_RES_RC_STATUS_OK
) {
1820 rq
->flush_in_progress
= true;
1822 rc
= __flush_rq(rq
, qp
, CQ_BASE_CQE_TYPE_RES_RC
, pcqe
, budget
);
1824 rq
->flush_in_progress
= false;
1829 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq
*cq
,
1830 struct cq_res_ud
*hwcqe
,
1831 struct bnxt_qplib_cqe
**pcqe
,
1834 struct bnxt_qplib_qp
*qp
;
1835 struct bnxt_qplib_q
*rq
;
1836 struct bnxt_qplib_cqe
*cqe
;
1840 qp
= (struct bnxt_qplib_qp
*)((unsigned long)
1841 le64_to_cpu(hwcqe
->qp_handle
));
1843 dev_err(&cq
->hwq
.pdev
->dev
, "QPLIB: process_cq UD qp is NULL");
1847 cqe
->opcode
= hwcqe
->cqe_type_toggle
& CQ_BASE_CQE_TYPE_MASK
;
1848 cqe
->length
= le32_to_cpu(hwcqe
->length
);
1849 cqe
->invrkey
= le32_to_cpu(hwcqe
->imm_data
);
1850 cqe
->flags
= le16_to_cpu(hwcqe
->flags
);
1851 cqe
->status
= hwcqe
->status
;
1852 cqe
->qp_handle
= (u64
)(unsigned long)qp
;
1853 memcpy(cqe
->smac
, hwcqe
->src_mac
, 6);
1854 wr_id_idx
= le32_to_cpu(hwcqe
->src_qp_high_srq_or_rq_wr_id
)
1855 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK
;
1856 cqe
->src_qp
= le16_to_cpu(hwcqe
->src_qp_low
) |
1858 hwcqe
->src_qp_high_srq_or_rq_wr_id
) &
1859 CQ_RES_UD_SRC_QP_HIGH_MASK
) >> 8);
1862 if (wr_id_idx
> rq
->hwq
.max_elements
) {
1863 dev_err(&cq
->hwq
.pdev
->dev
, "QPLIB: FP: CQ Process UD ");
1864 dev_err(&cq
->hwq
.pdev
->dev
,
1865 "QPLIB: wr_id idx %#x exceeded RQ max %#x",
1866 wr_id_idx
, rq
->hwq
.max_elements
);
1869 if (rq
->flush_in_progress
)
1872 cqe
->wr_id
= rq
->swq
[wr_id_idx
].wr_id
;
1878 if (hwcqe
->status
!= CQ_RES_RC_STATUS_OK
) {
1879 rq
->flush_in_progress
= true;
1881 rc
= __flush_rq(rq
, qp
, CQ_BASE_CQE_TYPE_RES_UD
, pcqe
, budget
);
1883 rq
->flush_in_progress
= false;
1888 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq
*cq
,
1889 struct cq_res_raweth_qp1
*hwcqe
,
1890 struct bnxt_qplib_cqe
**pcqe
,
1893 struct bnxt_qplib_qp
*qp
;
1894 struct bnxt_qplib_q
*rq
;
1895 struct bnxt_qplib_cqe
*cqe
;
1899 qp
= (struct bnxt_qplib_qp
*)((unsigned long)
1900 le64_to_cpu(hwcqe
->qp_handle
));
1902 dev_err(&cq
->hwq
.pdev
->dev
,
1903 "QPLIB: process_cq Raw/QP1 qp is NULL");
1907 cqe
->opcode
= hwcqe
->cqe_type_toggle
& CQ_BASE_CQE_TYPE_MASK
;
1908 cqe
->flags
= le16_to_cpu(hwcqe
->flags
);
1909 cqe
->qp_handle
= (u64
)(unsigned long)qp
;
1912 le32_to_cpu(hwcqe
->raweth_qp1_payload_offset_srq_or_rq_wr_id
)
1913 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK
;
1914 cqe
->src_qp
= qp
->id
;
1915 if (qp
->id
== 1 && !cqe
->length
) {
1916 /* Add workaround for the length misdetection */
1919 cqe
->length
= le16_to_cpu(hwcqe
->length
);
1921 cqe
->pkey_index
= qp
->pkey_index
;
1922 memcpy(cqe
->smac
, qp
->smac
, 6);
1924 cqe
->raweth_qp1_flags
= le16_to_cpu(hwcqe
->raweth_qp1_flags
);
1925 cqe
->raweth_qp1_flags2
= le32_to_cpu(hwcqe
->raweth_qp1_flags2
);
1928 if (wr_id_idx
> rq
->hwq
.max_elements
) {
1929 dev_err(&cq
->hwq
.pdev
->dev
, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
1930 dev_err(&cq
->hwq
.pdev
->dev
, "QPLIB: ix 0x%x exceeded RQ max 0x%x",
1931 wr_id_idx
, rq
->hwq
.max_elements
);
1934 if (rq
->flush_in_progress
)
1937 cqe
->wr_id
= rq
->swq
[wr_id_idx
].wr_id
;
1943 if (hwcqe
->status
!= CQ_RES_RC_STATUS_OK
) {
1944 rq
->flush_in_progress
= true;
1946 rc
= __flush_rq(rq
, qp
, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1
, pcqe
,
1949 rq
->flush_in_progress
= false;
1954 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq
*cq
,
1955 struct cq_terminal
*hwcqe
,
1956 struct bnxt_qplib_cqe
**pcqe
,
1959 struct bnxt_qplib_qp
*qp
;
1960 struct bnxt_qplib_q
*sq
, *rq
;
1961 struct bnxt_qplib_cqe
*cqe
;
1962 u32 sw_cons
= 0, cqe_cons
;
1966 /* Check the Status */
1967 if (hwcqe
->status
!= CQ_TERMINAL_STATUS_OK
)
1968 dev_warn(&cq
->hwq
.pdev
->dev
,
1969 "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
1972 qp
= (struct bnxt_qplib_qp
*)((unsigned long)
1973 le64_to_cpu(hwcqe
->qp_handle
));
1975 dev_err(&cq
->hwq
.pdev
->dev
,
1976 "QPLIB: FP: CQ Process terminal qp is NULL");
1979 /* Must block new posting of SQ and RQ */
1980 qp
->state
= CMDQ_MODIFY_QP_NEW_STATE_ERR
;
1985 cqe_cons
= le16_to_cpu(hwcqe
->sq_cons_idx
);
1986 if (cqe_cons
== 0xFFFF)
1989 if (cqe_cons
> sq
->hwq
.max_elements
) {
1990 dev_err(&cq
->hwq
.pdev
->dev
,
1991 "QPLIB: FP: CQ Process terminal reported ");
1992 dev_err(&cq
->hwq
.pdev
->dev
,
1993 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1994 cqe_cons
, sq
->hwq
.max_elements
);
1997 /* If we were in the middle of flushing, continue */
1998 if (sq
->flush_in_progress
)
2001 /* Terminal CQE can also include aggregated successful CQEs prior.
2002 * So we must complete all CQEs from the current sq's cons to the
2003 * cq_cons with status OK
2007 sw_cons
= HWQ_CMP(sq
->hwq
.cons
, &sq
->hwq
);
2008 if (sw_cons
== cqe_cons
)
2010 if (sq
->swq
[sw_cons
].flags
& SQ_SEND_FLAGS_SIGNAL_COMP
) {
2011 memset(cqe
, 0, sizeof(*cqe
));
2012 cqe
->status
= CQ_REQ_STATUS_OK
;
2013 cqe
->opcode
= CQ_BASE_CQE_TYPE_REQ
;
2014 cqe
->qp_handle
= (u64
)(unsigned long)qp
;
2015 cqe
->src_qp
= qp
->id
;
2016 cqe
->wr_id
= sq
->swq
[sw_cons
].wr_id
;
2017 cqe
->type
= sq
->swq
[sw_cons
].type
;
2024 if (!(*budget
) && sw_cons
!= cqe_cons
) {
2029 sq
->flush_in_progress
= true;
2031 rc
= __flush_sq(sq
, qp
, pcqe
, budget
);
2033 sq
->flush_in_progress
= false;
2038 cqe_cons
= le16_to_cpu(hwcqe
->rq_cons_idx
);
2039 if (cqe_cons
== 0xFFFF) {
2041 } else if (cqe_cons
> rq
->hwq
.max_elements
) {
2042 dev_err(&cq
->hwq
.pdev
->dev
,
2043 "QPLIB: FP: CQ Processed terminal ");
2044 dev_err(&cq
->hwq
.pdev
->dev
,
2045 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2046 cqe_cons
, rq
->hwq
.max_elements
);
2049 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2050 * from the current rq->cons to the rq->prod regardless what the
2051 * rq->cons the terminal CQE indicates
2053 rq
->flush_in_progress
= true;
2055 case CMDQ_CREATE_QP1_TYPE_GSI
:
2056 opcode
= CQ_BASE_CQE_TYPE_RES_RAWETH_QP1
;
2058 case CMDQ_CREATE_QP_TYPE_RC
:
2059 opcode
= CQ_BASE_CQE_TYPE_RES_RC
;
2061 case CMDQ_CREATE_QP_TYPE_UD
:
2062 opcode
= CQ_BASE_CQE_TYPE_RES_UD
;
2066 rc
= __flush_rq(rq
, qp
, opcode
, pcqe
, budget
);
2068 rq
->flush_in_progress
= false;
2073 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq
*cq
,
2074 struct cq_cutoff
*hwcqe
)
2076 /* Check the Status */
2077 if (hwcqe
->status
!= CQ_CUTOFF_STATUS_OK
) {
2078 dev_err(&cq
->hwq
.pdev
->dev
,
2079 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2083 clear_bit(CQ_FLAGS_RESIZE_IN_PROG
, &cq
->flags
);
2084 wake_up_interruptible(&cq
->waitq
);
2089 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq
*cq
, struct bnxt_qplib_cqe
*cqe
,
2090 int num_cqes
, struct bnxt_qplib_qp
**lib_qp
)
2092 struct cq_base
*hw_cqe
, **hw_cqe_ptr
;
2093 unsigned long flags
;
2094 u32 sw_cons
, raw_cons
;
2097 spin_lock_irqsave(&cq
->hwq
.lock
, flags
);
2098 raw_cons
= cq
->hwq
.cons
;
2102 sw_cons
= HWQ_CMP(raw_cons
, &cq
->hwq
);
2103 hw_cqe_ptr
= (struct cq_base
**)cq
->hwq
.pbl_ptr
;
2104 hw_cqe
= &hw_cqe_ptr
[CQE_PG(sw_cons
)][CQE_IDX(sw_cons
)];
2106 /* Check for Valid bit */
2107 if (!CQE_CMP_VALID(hw_cqe
, raw_cons
, cq
->hwq
.max_elements
))
2110 /* From the device's respective CQE format to qplib_wc*/
2111 switch (hw_cqe
->cqe_type_toggle
& CQ_BASE_CQE_TYPE_MASK
) {
2112 case CQ_BASE_CQE_TYPE_REQ
:
2113 rc
= bnxt_qplib_cq_process_req(cq
,
2114 (struct cq_req
*)hw_cqe
,
2118 case CQ_BASE_CQE_TYPE_RES_RC
:
2119 rc
= bnxt_qplib_cq_process_res_rc(cq
,
2120 (struct cq_res_rc
*)
2124 case CQ_BASE_CQE_TYPE_RES_UD
:
2125 rc
= bnxt_qplib_cq_process_res_ud
2126 (cq
, (struct cq_res_ud
*)hw_cqe
, &cqe
,
2129 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1
:
2130 rc
= bnxt_qplib_cq_process_res_raweth_qp1
2131 (cq
, (struct cq_res_raweth_qp1
*)
2132 hw_cqe
, &cqe
, &budget
);
2134 case CQ_BASE_CQE_TYPE_TERMINAL
:
2135 rc
= bnxt_qplib_cq_process_terminal
2136 (cq
, (struct cq_terminal
*)hw_cqe
,
2139 case CQ_BASE_CQE_TYPE_CUT_OFF
:
2140 bnxt_qplib_cq_process_cutoff
2141 (cq
, (struct cq_cutoff
*)hw_cqe
);
2142 /* Done processing this CQ */
2145 dev_err(&cq
->hwq
.pdev
->dev
,
2146 "QPLIB: process_cq unknown type 0x%lx",
2147 hw_cqe
->cqe_type_toggle
&
2148 CQ_BASE_CQE_TYPE_MASK
);
2155 /* Error while processing the CQE, just skip to the
2158 dev_err(&cq
->hwq
.pdev
->dev
,
2159 "QPLIB: process_cqe error rc = 0x%x", rc
);
2163 if (cq
->hwq
.cons
!= raw_cons
) {
2164 cq
->hwq
.cons
= raw_cons
;
2165 bnxt_qplib_arm_cq(cq
, DBR_DBR_TYPE_CQ
);
2168 spin_unlock_irqrestore(&cq
->hwq
.lock
, flags
);
2169 return num_cqes
- budget
;
2172 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq
*cq
, u32 arm_type
)
2174 unsigned long flags
;
2176 spin_lock_irqsave(&cq
->hwq
.lock
, flags
);
2178 bnxt_qplib_arm_cq(cq
, arm_type
);
2180 spin_unlock_irqrestore(&cq
->hwq
.lock
, flags
);