2 * Copyright (c) 2016-2017 Hisilicon Limited.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/acpi.h>
34 #include <linux/etherdevice.h>
35 #include <linux/interrupt.h>
36 #include <linux/kernel.h>
37 #include <rdma/ib_umem.h>
40 #include "hns_roce_common.h"
41 #include "hns_roce_device.h"
42 #include "hns_roce_cmd.h"
43 #include "hns_roce_hem.h"
44 #include "hns_roce_hw_v2.h"
46 static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg
*dseg
,
49 dseg
->lkey
= cpu_to_le32(sg
->lkey
);
50 dseg
->addr
= cpu_to_le64(sg
->addr
);
51 dseg
->len
= cpu_to_le32(sg
->length
);
54 static int hns_roce_v2_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
55 struct ib_send_wr
**bad_wr
)
57 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
58 struct hns_roce_v2_rc_send_wqe
*rc_sq_wqe
;
59 struct hns_roce_qp
*qp
= to_hr_qp(ibqp
);
60 struct hns_roce_v2_wqe_data_seg
*dseg
;
61 struct device
*dev
= hr_dev
->dev
;
62 struct hns_roce_v2_db sq_db
;
63 unsigned int sge_ind
= 0;
64 unsigned int wqe_sz
= 0;
65 unsigned int owner_bit
;
73 if (unlikely(ibqp
->qp_type
!= IB_QPT_RC
)) {
74 dev_err(dev
, "Not supported QP(0x%x)type!\n", ibqp
->qp_type
);
79 if (unlikely(qp
->state
!= IB_QPS_RTS
&& qp
->state
!= IB_QPS_SQD
)) {
80 dev_err(dev
, "Post WQE fail, QP state %d err!\n", qp
->state
);
85 spin_lock_irqsave(&qp
->sq
.lock
, flags
);
86 ind
= qp
->sq_next_wqe
;
87 sge_ind
= qp
->next_sge
;
89 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
90 if (hns_roce_wq_overflow(&qp
->sq
, nreq
, qp
->ibqp
.send_cq
)) {
96 if (unlikely(wr
->num_sge
> qp
->sq
.max_gs
)) {
97 dev_err(dev
, "num_sge=%d > qp->sq.max_gs=%d\n",
98 wr
->num_sge
, qp
->sq
.max_gs
);
104 wqe
= get_send_wqe(qp
, ind
& (qp
->sq
.wqe_cnt
- 1));
105 qp
->sq
.wrid
[(qp
->sq
.head
+ nreq
) & (qp
->sq
.wqe_cnt
- 1)] =
108 owner_bit
= ~(qp
->sq
.head
>> ilog2(qp
->sq
.wqe_cnt
)) & 0x1;
110 memset(rc_sq_wqe
, 0, sizeof(*rc_sq_wqe
));
111 for (i
= 0; i
< wr
->num_sge
; i
++)
112 rc_sq_wqe
->msg_len
+= wr
->sg_list
[i
].length
;
114 rc_sq_wqe
->inv_key_immtdata
= send_ieth(wr
);
116 roce_set_bit(rc_sq_wqe
->byte_4
, V2_RC_SEND_WQE_BYTE_4_FENCE_S
,
117 (wr
->send_flags
& IB_SEND_FENCE
) ? 1 : 0);
119 roce_set_bit(rc_sq_wqe
->byte_4
, V2_RC_SEND_WQE_BYTE_4_SE_S
,
120 (wr
->send_flags
& IB_SEND_SOLICITED
) ? 1 : 0);
122 roce_set_bit(rc_sq_wqe
->byte_4
, V2_RC_SEND_WQE_BYTE_4_CQE_S
,
123 (wr
->send_flags
& IB_SEND_SIGNALED
) ? 1 : 0);
125 roce_set_bit(rc_sq_wqe
->byte_4
, V2_RC_SEND_WQE_BYTE_4_OWNER_S
,
128 switch (wr
->opcode
) {
129 case IB_WR_RDMA_READ
:
130 roce_set_field(rc_sq_wqe
->byte_4
,
131 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
132 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
133 HNS_ROCE_V2_WQE_OP_RDMA_READ
);
134 rc_sq_wqe
->rkey
= cpu_to_le32(rdma_wr(wr
)->rkey
);
135 rc_sq_wqe
->va
= cpu_to_le64(rdma_wr(wr
)->remote_addr
);
137 case IB_WR_RDMA_WRITE
:
138 roce_set_field(rc_sq_wqe
->byte_4
,
139 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
140 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
141 HNS_ROCE_V2_WQE_OP_RDMA_WRITE
);
142 rc_sq_wqe
->rkey
= cpu_to_le32(rdma_wr(wr
)->rkey
);
143 rc_sq_wqe
->va
= cpu_to_le64(rdma_wr(wr
)->remote_addr
);
145 case IB_WR_RDMA_WRITE_WITH_IMM
:
146 roce_set_field(rc_sq_wqe
->byte_4
,
147 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
148 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
149 HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM
);
150 rc_sq_wqe
->rkey
= cpu_to_le32(rdma_wr(wr
)->rkey
);
151 rc_sq_wqe
->va
= cpu_to_le64(rdma_wr(wr
)->remote_addr
);
154 roce_set_field(rc_sq_wqe
->byte_4
,
155 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
156 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
157 HNS_ROCE_V2_WQE_OP_SEND
);
159 case IB_WR_SEND_WITH_INV
:
160 roce_set_field(rc_sq_wqe
->byte_4
,
161 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
162 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
163 HNS_ROCE_V2_WQE_OP_SEND_WITH_INV
);
165 case IB_WR_SEND_WITH_IMM
:
166 roce_set_field(rc_sq_wqe
->byte_4
,
167 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
168 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
169 HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM
);
171 case IB_WR_LOCAL_INV
:
172 roce_set_field(rc_sq_wqe
->byte_4
,
173 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
174 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
175 HNS_ROCE_V2_WQE_OP_LOCAL_INV
);
177 case IB_WR_ATOMIC_CMP_AND_SWP
:
178 roce_set_field(rc_sq_wqe
->byte_4
,
179 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
180 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
181 HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP
);
183 case IB_WR_ATOMIC_FETCH_AND_ADD
:
184 roce_set_field(rc_sq_wqe
->byte_4
,
185 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
186 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
187 HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD
);
189 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP
:
190 roce_set_field(rc_sq_wqe
->byte_4
,
191 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
192 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
193 HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP
);
195 case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD
:
196 roce_set_field(rc_sq_wqe
->byte_4
,
197 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
198 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
199 HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD
);
202 roce_set_field(rc_sq_wqe
->byte_4
,
203 V2_RC_SEND_WQE_BYTE_4_OPCODE_M
,
204 V2_RC_SEND_WQE_BYTE_4_OPCODE_S
,
205 HNS_ROCE_V2_WQE_OP_MASK
);
209 wqe
+= sizeof(struct hns_roce_v2_rc_send_wqe
);
211 if (wr
->send_flags
& IB_SEND_INLINE
&& wr
->num_sge
) {
212 if (rc_sq_wqe
->msg_len
>
213 hr_dev
->caps
.max_sq_inline
) {
216 dev_err(dev
, "inline len(1-%d)=%d, illegal",
218 hr_dev
->caps
.max_sq_inline
);
222 for (i
= 0; i
< wr
->num_sge
; i
++) {
223 memcpy(wqe
, ((void *)wr
->sg_list
[i
].addr
),
224 wr
->sg_list
[i
].length
);
225 wqe
+= wr
->sg_list
[i
].length
;
226 wqe_sz
+= wr
->sg_list
[i
].length
;
229 roce_set_bit(rc_sq_wqe
->byte_4
,
230 V2_RC_SEND_WQE_BYTE_4_INLINE_S
, 1);
232 if (wr
->num_sge
<= 2) {
233 for (i
= 0; i
< wr
->num_sge
; i
++)
234 set_data_seg_v2(dseg
+ i
,
237 roce_set_field(rc_sq_wqe
->byte_20
,
238 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M
,
239 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S
,
240 sge_ind
& (qp
->sge
.sge_cnt
- 1));
242 for (i
= 0; i
< 2; i
++)
243 set_data_seg_v2(dseg
+ i
,
246 dseg
= get_send_extend_sge(qp
,
247 sge_ind
& (qp
->sge
.sge_cnt
- 1));
249 for (i
= 0; i
< wr
->num_sge
- 2; i
++) {
250 set_data_seg_v2(dseg
+ i
,
251 wr
->sg_list
+ 2 + i
);
256 roce_set_field(rc_sq_wqe
->byte_16
,
257 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M
,
258 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S
,
260 wqe_sz
+= wr
->num_sge
*
261 sizeof(struct hns_roce_v2_wqe_data_seg
);
275 roce_set_field(sq_db
.byte_4
, V2_DB_BYTE_4_TAG_M
,
276 V2_DB_BYTE_4_TAG_S
, qp
->doorbell_qpn
);
277 roce_set_field(sq_db
.byte_4
, V2_DB_BYTE_4_CMD_M
,
278 V2_DB_BYTE_4_CMD_S
, HNS_ROCE_V2_SQ_DB
);
279 roce_set_field(sq_db
.parameter
, V2_DB_PARAMETER_CONS_IDX_M
,
280 V2_DB_PARAMETER_CONS_IDX_S
,
281 qp
->sq
.head
& ((qp
->sq
.wqe_cnt
<< 1) - 1));
282 roce_set_field(sq_db
.parameter
, V2_DB_PARAMETER_SL_M
,
283 V2_DB_PARAMETER_SL_S
, qp
->sl
);
285 hns_roce_write64_k((__be32
*)&sq_db
, qp
->sq
.db_reg_l
);
287 qp
->sq_next_wqe
= ind
;
288 qp
->next_sge
= sge_ind
;
291 spin_unlock_irqrestore(&qp
->sq
.lock
, flags
);
296 static int hns_roce_v2_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
297 struct ib_recv_wr
**bad_wr
)
299 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
300 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
301 struct hns_roce_v2_wqe_data_seg
*dseg
;
302 struct device
*dev
= hr_dev
->dev
;
303 struct hns_roce_v2_db rq_db
;
311 spin_lock_irqsave(&hr_qp
->rq
.lock
, flags
);
312 ind
= hr_qp
->rq
.head
& (hr_qp
->rq
.wqe_cnt
- 1);
314 if (hr_qp
->state
== IB_QPS_RESET
|| hr_qp
->state
== IB_QPS_ERR
) {
315 spin_unlock_irqrestore(&hr_qp
->rq
.lock
, flags
);
320 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
321 if (hns_roce_wq_overflow(&hr_qp
->rq
, nreq
,
322 hr_qp
->ibqp
.recv_cq
)) {
328 if (unlikely(wr
->num_sge
> hr_qp
->rq
.max_gs
)) {
329 dev_err(dev
, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
330 wr
->num_sge
, hr_qp
->rq
.max_gs
);
336 wqe
= get_recv_wqe(hr_qp
, ind
);
337 dseg
= (struct hns_roce_v2_wqe_data_seg
*)wqe
;
338 for (i
= 0; i
< wr
->num_sge
; i
++) {
339 if (!wr
->sg_list
[i
].length
)
341 set_data_seg_v2(dseg
, wr
->sg_list
+ i
);
345 if (i
< hr_qp
->rq
.max_gs
) {
346 dseg
[i
].lkey
= cpu_to_be32(HNS_ROCE_INVALID_LKEY
);
350 hr_qp
->rq
.wrid
[ind
] = wr
->wr_id
;
352 ind
= (ind
+ 1) & (hr_qp
->rq
.wqe_cnt
- 1);
357 hr_qp
->rq
.head
+= nreq
;
364 roce_set_field(rq_db
.byte_4
, V2_DB_BYTE_4_TAG_M
,
365 V2_DB_BYTE_4_TAG_S
, hr_qp
->qpn
);
366 roce_set_field(rq_db
.byte_4
, V2_DB_BYTE_4_CMD_M
,
367 V2_DB_BYTE_4_CMD_S
, HNS_ROCE_V2_RQ_DB
);
368 roce_set_field(rq_db
.parameter
, V2_DB_PARAMETER_CONS_IDX_M
,
369 V2_DB_PARAMETER_CONS_IDX_S
, hr_qp
->rq
.head
);
371 hns_roce_write64_k((__be32
*)&rq_db
, hr_qp
->rq
.db_reg_l
);
373 spin_unlock_irqrestore(&hr_qp
->rq
.lock
, flags
);
378 static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring
*ring
)
380 int ntu
= ring
->next_to_use
;
381 int ntc
= ring
->next_to_clean
;
382 int used
= (ntu
- ntc
+ ring
->desc_num
) % ring
->desc_num
;
384 return ring
->desc_num
- used
- 1;
387 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev
*hr_dev
,
388 struct hns_roce_v2_cmq_ring
*ring
)
390 int size
= ring
->desc_num
* sizeof(struct hns_roce_cmq_desc
);
392 ring
->desc
= kzalloc(size
, GFP_KERNEL
);
396 ring
->desc_dma_addr
= dma_map_single(hr_dev
->dev
, ring
->desc
, size
,
398 if (dma_mapping_error(hr_dev
->dev
, ring
->desc_dma_addr
)) {
399 ring
->desc_dma_addr
= 0;
408 static void hns_roce_free_cmq_desc(struct hns_roce_dev
*hr_dev
,
409 struct hns_roce_v2_cmq_ring
*ring
)
411 dma_unmap_single(hr_dev
->dev
, ring
->desc_dma_addr
,
412 ring
->desc_num
* sizeof(struct hns_roce_cmq_desc
),
417 static int hns_roce_init_cmq_ring(struct hns_roce_dev
*hr_dev
, bool ring_type
)
419 struct hns_roce_v2_priv
*priv
= (struct hns_roce_v2_priv
*)hr_dev
->priv
;
420 struct hns_roce_v2_cmq_ring
*ring
= (ring_type
== TYPE_CSQ
) ?
421 &priv
->cmq
.csq
: &priv
->cmq
.crq
;
423 ring
->flag
= ring_type
;
424 ring
->next_to_clean
= 0;
425 ring
->next_to_use
= 0;
427 return hns_roce_alloc_cmq_desc(hr_dev
, ring
);
430 static void hns_roce_cmq_init_regs(struct hns_roce_dev
*hr_dev
, bool ring_type
)
432 struct hns_roce_v2_priv
*priv
= (struct hns_roce_v2_priv
*)hr_dev
->priv
;
433 struct hns_roce_v2_cmq_ring
*ring
= (ring_type
== TYPE_CSQ
) ?
434 &priv
->cmq
.csq
: &priv
->cmq
.crq
;
435 dma_addr_t dma
= ring
->desc_dma_addr
;
437 if (ring_type
== TYPE_CSQ
) {
438 roce_write(hr_dev
, ROCEE_TX_CMQ_BASEADDR_L_REG
, (u32
)dma
);
439 roce_write(hr_dev
, ROCEE_TX_CMQ_BASEADDR_H_REG
,
441 roce_write(hr_dev
, ROCEE_TX_CMQ_DEPTH_REG
,
442 (ring
->desc_num
>> HNS_ROCE_CMQ_DESC_NUM_S
) |
443 HNS_ROCE_CMQ_ENABLE
);
444 roce_write(hr_dev
, ROCEE_TX_CMQ_HEAD_REG
, 0);
445 roce_write(hr_dev
, ROCEE_TX_CMQ_TAIL_REG
, 0);
447 roce_write(hr_dev
, ROCEE_RX_CMQ_BASEADDR_L_REG
, (u32
)dma
);
448 roce_write(hr_dev
, ROCEE_RX_CMQ_BASEADDR_H_REG
,
450 roce_write(hr_dev
, ROCEE_RX_CMQ_DEPTH_REG
,
451 (ring
->desc_num
>> HNS_ROCE_CMQ_DESC_NUM_S
) |
452 HNS_ROCE_CMQ_ENABLE
);
453 roce_write(hr_dev
, ROCEE_RX_CMQ_HEAD_REG
, 0);
454 roce_write(hr_dev
, ROCEE_RX_CMQ_TAIL_REG
, 0);
458 static int hns_roce_v2_cmq_init(struct hns_roce_dev
*hr_dev
)
460 struct hns_roce_v2_priv
*priv
= (struct hns_roce_v2_priv
*)hr_dev
->priv
;
463 /* Setup the queue entries for command queue */
464 priv
->cmq
.csq
.desc_num
= 1024;
465 priv
->cmq
.crq
.desc_num
= 1024;
467 /* Setup the lock for command queue */
468 spin_lock_init(&priv
->cmq
.csq
.lock
);
469 spin_lock_init(&priv
->cmq
.crq
.lock
);
471 /* Setup Tx write back timeout */
472 priv
->cmq
.tx_timeout
= HNS_ROCE_CMQ_TX_TIMEOUT
;
475 ret
= hns_roce_init_cmq_ring(hr_dev
, TYPE_CSQ
);
477 dev_err(hr_dev
->dev
, "Init CSQ error, ret = %d.\n", ret
);
482 ret
= hns_roce_init_cmq_ring(hr_dev
, TYPE_CRQ
);
484 dev_err(hr_dev
->dev
, "Init CRQ error, ret = %d.\n", ret
);
489 hns_roce_cmq_init_regs(hr_dev
, TYPE_CSQ
);
492 hns_roce_cmq_init_regs(hr_dev
, TYPE_CRQ
);
497 hns_roce_free_cmq_desc(hr_dev
, &priv
->cmq
.csq
);
502 static void hns_roce_v2_cmq_exit(struct hns_roce_dev
*hr_dev
)
504 struct hns_roce_v2_priv
*priv
= (struct hns_roce_v2_priv
*)hr_dev
->priv
;
506 hns_roce_free_cmq_desc(hr_dev
, &priv
->cmq
.csq
);
507 hns_roce_free_cmq_desc(hr_dev
, &priv
->cmq
.crq
);
510 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc
*desc
,
511 enum hns_roce_opcode_type opcode
,
514 memset((void *)desc
, 0, sizeof(struct hns_roce_cmq_desc
));
515 desc
->opcode
= cpu_to_le16(opcode
);
517 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR
| HNS_ROCE_CMD_FLAG_IN
);
519 desc
->flag
|= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR
);
521 desc
->flag
&= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR
);
524 static int hns_roce_cmq_csq_done(struct hns_roce_dev
*hr_dev
)
526 struct hns_roce_v2_priv
*priv
= (struct hns_roce_v2_priv
*)hr_dev
->priv
;
527 u32 head
= roce_read(hr_dev
, ROCEE_TX_CMQ_HEAD_REG
);
529 return head
== priv
->cmq
.csq
.next_to_use
;
532 static int hns_roce_cmq_csq_clean(struct hns_roce_dev
*hr_dev
)
534 struct hns_roce_v2_priv
*priv
= (struct hns_roce_v2_priv
*)hr_dev
->priv
;
535 struct hns_roce_v2_cmq_ring
*csq
= &priv
->cmq
.csq
;
536 struct hns_roce_cmq_desc
*desc
;
537 u16 ntc
= csq
->next_to_clean
;
541 desc
= &csq
->desc
[ntc
];
542 head
= roce_read(hr_dev
, ROCEE_TX_CMQ_HEAD_REG
);
543 while (head
!= ntc
) {
544 memset(desc
, 0, sizeof(*desc
));
546 if (ntc
== csq
->desc_num
)
548 desc
= &csq
->desc
[ntc
];
551 csq
->next_to_clean
= ntc
;
556 static int hns_roce_cmq_send(struct hns_roce_dev
*hr_dev
,
557 struct hns_roce_cmq_desc
*desc
, int num
)
559 struct hns_roce_v2_priv
*priv
= (struct hns_roce_v2_priv
*)hr_dev
->priv
;
560 struct hns_roce_v2_cmq_ring
*csq
= &priv
->cmq
.csq
;
561 struct hns_roce_cmq_desc
*desc_to_use
;
562 bool complete
= false;
569 spin_lock_bh(&csq
->lock
);
571 if (num
> hns_roce_cmq_space(csq
)) {
572 spin_unlock_bh(&csq
->lock
);
577 * Record the location of desc in the cmq for this time
578 * which will be use for hardware to write back
580 ntc
= csq
->next_to_use
;
582 while (handle
< num
) {
583 desc_to_use
= &csq
->desc
[csq
->next_to_use
];
584 *desc_to_use
= desc
[handle
];
585 dev_dbg(hr_dev
->dev
, "set cmq desc:\n");
587 if (csq
->next_to_use
== csq
->desc_num
)
588 csq
->next_to_use
= 0;
592 /* Write to hardware */
593 roce_write(hr_dev
, ROCEE_TX_CMQ_TAIL_REG
, csq
->next_to_use
);
596 * If the command is sync, wait for the firmware to write back,
597 * if multi descriptors to be sent, use the first one to check
599 if ((desc
->flag
) & HNS_ROCE_CMD_FLAG_NO_INTR
) {
601 if (hns_roce_cmq_csq_done(hr_dev
))
605 } while (timeout
< priv
->cmq
.tx_timeout
);
608 if (hns_roce_cmq_csq_done(hr_dev
)) {
611 while (handle
< num
) {
612 /* get the result of hardware write back */
613 desc_to_use
= &csq
->desc
[ntc
];
614 desc
[handle
] = *desc_to_use
;
615 dev_dbg(hr_dev
->dev
, "Get cmq desc:\n");
616 desc_ret
= desc
[handle
].retval
;
617 if (desc_ret
== CMD_EXEC_SUCCESS
)
621 priv
->cmq
.last_status
= desc_ret
;
624 if (ntc
== csq
->desc_num
)
632 /* clean the command send queue */
633 handle
= hns_roce_cmq_csq_clean(hr_dev
);
635 dev_warn(hr_dev
->dev
, "Cleaned %d, need to clean %d\n",
638 spin_unlock_bh(&csq
->lock
);
643 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev
*hr_dev
)
645 struct hns_roce_query_version
*resp
;
646 struct hns_roce_cmq_desc desc
;
649 hns_roce_cmq_setup_basic_desc(&desc
, HNS_ROCE_OPC_QUERY_HW_VER
, true);
650 ret
= hns_roce_cmq_send(hr_dev
, &desc
, 1);
654 resp
= (struct hns_roce_query_version
*)desc
.data
;
655 hr_dev
->hw_rev
= le32_to_cpu(resp
->rocee_hw_version
);
656 hr_dev
->vendor_id
= le32_to_cpu(resp
->rocee_vendor_id
);
661 static int hns_roce_config_global_param(struct hns_roce_dev
*hr_dev
)
663 struct hns_roce_cfg_global_param
*req
;
664 struct hns_roce_cmq_desc desc
;
666 hns_roce_cmq_setup_basic_desc(&desc
, HNS_ROCE_OPC_CFG_GLOBAL_PARAM
,
669 req
= (struct hns_roce_cfg_global_param
*)desc
.data
;
670 memset(req
, 0, sizeof(*req
));
671 roce_set_field(req
->time_cfg_udp_port
,
672 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M
,
673 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S
, 0x3e8);
674 roce_set_field(req
->time_cfg_udp_port
,
675 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M
,
676 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S
, 0x12b7);
678 return hns_roce_cmq_send(hr_dev
, &desc
, 1);
681 static int hns_roce_query_pf_resource(struct hns_roce_dev
*hr_dev
)
683 struct hns_roce_cmq_desc desc
[2];
684 struct hns_roce_pf_res
*res
;
688 for (i
= 0; i
< 2; i
++) {
689 hns_roce_cmq_setup_basic_desc(&desc
[i
],
690 HNS_ROCE_OPC_QUERY_PF_RES
, true);
693 desc
[i
].flag
|= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT
);
695 desc
[i
].flag
&= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT
);
698 ret
= hns_roce_cmq_send(hr_dev
, desc
, 2);
702 res
= (struct hns_roce_pf_res
*)desc
[0].data
;
704 hr_dev
->caps
.qpc_bt_num
= roce_get_field(res
->qpc_bt_idx_num
,
705 PF_RES_DATA_1_PF_QPC_BT_NUM_M
,
706 PF_RES_DATA_1_PF_QPC_BT_NUM_S
);
707 hr_dev
->caps
.srqc_bt_num
= roce_get_field(res
->srqc_bt_idx_num
,
708 PF_RES_DATA_2_PF_SRQC_BT_NUM_M
,
709 PF_RES_DATA_2_PF_SRQC_BT_NUM_S
);
710 hr_dev
->caps
.cqc_bt_num
= roce_get_field(res
->cqc_bt_idx_num
,
711 PF_RES_DATA_3_PF_CQC_BT_NUM_M
,
712 PF_RES_DATA_3_PF_CQC_BT_NUM_S
);
713 hr_dev
->caps
.mpt_bt_num
= roce_get_field(res
->mpt_bt_idx_num
,
714 PF_RES_DATA_4_PF_MPT_BT_NUM_M
,
715 PF_RES_DATA_4_PF_MPT_BT_NUM_S
);
720 static int hns_roce_alloc_vf_resource(struct hns_roce_dev
*hr_dev
)
722 struct hns_roce_cmq_desc desc
[2];
723 struct hns_roce_vf_res_a
*req_a
;
724 struct hns_roce_vf_res_b
*req_b
;
727 req_a
= (struct hns_roce_vf_res_a
*)desc
[0].data
;
728 req_b
= (struct hns_roce_vf_res_b
*)desc
[1].data
;
729 memset(req_a
, 0, sizeof(*req_a
));
730 memset(req_b
, 0, sizeof(*req_b
));
731 for (i
= 0; i
< 2; i
++) {
732 hns_roce_cmq_setup_basic_desc(&desc
[i
],
733 HNS_ROCE_OPC_ALLOC_VF_RES
, false);
736 desc
[i
].flag
|= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT
);
738 desc
[i
].flag
&= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT
);
741 roce_set_field(req_a
->vf_qpc_bt_idx_num
,
742 VF_RES_A_DATA_1_VF_QPC_BT_IDX_M
,
743 VF_RES_A_DATA_1_VF_QPC_BT_IDX_S
, 0);
744 roce_set_field(req_a
->vf_qpc_bt_idx_num
,
745 VF_RES_A_DATA_1_VF_QPC_BT_NUM_M
,
746 VF_RES_A_DATA_1_VF_QPC_BT_NUM_S
,
747 HNS_ROCE_VF_QPC_BT_NUM
);
749 roce_set_field(req_a
->vf_srqc_bt_idx_num
,
750 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M
,
751 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S
, 0);
752 roce_set_field(req_a
->vf_srqc_bt_idx_num
,
753 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M
,
754 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S
,
755 HNS_ROCE_VF_SRQC_BT_NUM
);
757 roce_set_field(req_a
->vf_cqc_bt_idx_num
,
758 VF_RES_A_DATA_3_VF_CQC_BT_IDX_M
,
759 VF_RES_A_DATA_3_VF_CQC_BT_IDX_S
, 0);
760 roce_set_field(req_a
->vf_cqc_bt_idx_num
,
761 VF_RES_A_DATA_3_VF_CQC_BT_NUM_M
,
762 VF_RES_A_DATA_3_VF_CQC_BT_NUM_S
,
763 HNS_ROCE_VF_CQC_BT_NUM
);
765 roce_set_field(req_a
->vf_mpt_bt_idx_num
,
766 VF_RES_A_DATA_4_VF_MPT_BT_IDX_M
,
767 VF_RES_A_DATA_4_VF_MPT_BT_IDX_S
, 0);
768 roce_set_field(req_a
->vf_mpt_bt_idx_num
,
769 VF_RES_A_DATA_4_VF_MPT_BT_NUM_M
,
770 VF_RES_A_DATA_4_VF_MPT_BT_NUM_S
,
771 HNS_ROCE_VF_MPT_BT_NUM
);
773 roce_set_field(req_a
->vf_eqc_bt_idx_num
,
774 VF_RES_A_DATA_5_VF_EQC_IDX_M
,
775 VF_RES_A_DATA_5_VF_EQC_IDX_S
, 0);
776 roce_set_field(req_a
->vf_eqc_bt_idx_num
,
777 VF_RES_A_DATA_5_VF_EQC_NUM_M
,
778 VF_RES_A_DATA_5_VF_EQC_NUM_S
,
779 HNS_ROCE_VF_EQC_NUM
);
781 roce_set_field(req_b
->vf_smac_idx_num
,
782 VF_RES_B_DATA_1_VF_SMAC_IDX_M
,
783 VF_RES_B_DATA_1_VF_SMAC_IDX_S
, 0);
784 roce_set_field(req_b
->vf_smac_idx_num
,
785 VF_RES_B_DATA_1_VF_SMAC_NUM_M
,
786 VF_RES_B_DATA_1_VF_SMAC_NUM_S
,
787 HNS_ROCE_VF_SMAC_NUM
);
789 roce_set_field(req_b
->vf_sgid_idx_num
,
790 VF_RES_B_DATA_2_VF_SGID_IDX_M
,
791 VF_RES_B_DATA_2_VF_SGID_IDX_S
, 0);
792 roce_set_field(req_b
->vf_sgid_idx_num
,
793 VF_RES_B_DATA_2_VF_SGID_NUM_M
,
794 VF_RES_B_DATA_2_VF_SGID_NUM_S
,
795 HNS_ROCE_VF_SGID_NUM
);
797 roce_set_field(req_b
->vf_qid_idx_sl_num
,
798 VF_RES_B_DATA_3_VF_QID_IDX_M
,
799 VF_RES_B_DATA_3_VF_QID_IDX_S
, 0);
800 roce_set_field(req_b
->vf_qid_idx_sl_num
,
801 VF_RES_B_DATA_3_VF_SL_NUM_M
,
802 VF_RES_B_DATA_3_VF_SL_NUM_S
,
807 return hns_roce_cmq_send(hr_dev
, desc
, 2);
810 static int hns_roce_v2_set_bt(struct hns_roce_dev
*hr_dev
)
812 u8 srqc_hop_num
= hr_dev
->caps
.srqc_hop_num
;
813 u8 qpc_hop_num
= hr_dev
->caps
.qpc_hop_num
;
814 u8 cqc_hop_num
= hr_dev
->caps
.cqc_hop_num
;
815 u8 mpt_hop_num
= hr_dev
->caps
.mpt_hop_num
;
816 struct hns_roce_cfg_bt_attr
*req
;
817 struct hns_roce_cmq_desc desc
;
819 hns_roce_cmq_setup_basic_desc(&desc
, HNS_ROCE_OPC_CFG_BT_ATTR
, false);
820 req
= (struct hns_roce_cfg_bt_attr
*)desc
.data
;
821 memset(req
, 0, sizeof(*req
));
823 roce_set_field(req
->vf_qpc_cfg
, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M
,
824 CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S
,
825 hr_dev
->caps
.qpc_ba_pg_sz
);
826 roce_set_field(req
->vf_qpc_cfg
, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M
,
827 CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S
,
828 hr_dev
->caps
.qpc_buf_pg_sz
);
829 roce_set_field(req
->vf_qpc_cfg
, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M
,
830 CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S
,
831 qpc_hop_num
== HNS_ROCE_HOP_NUM_0
? 0 : qpc_hop_num
);
833 roce_set_field(req
->vf_srqc_cfg
, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M
,
834 CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S
,
835 hr_dev
->caps
.srqc_ba_pg_sz
);
836 roce_set_field(req
->vf_srqc_cfg
, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M
,
837 CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S
,
838 hr_dev
->caps
.srqc_buf_pg_sz
);
839 roce_set_field(req
->vf_srqc_cfg
, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M
,
840 CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S
,
841 srqc_hop_num
== HNS_ROCE_HOP_NUM_0
? 0 : srqc_hop_num
);
843 roce_set_field(req
->vf_cqc_cfg
, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M
,
844 CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S
,
845 hr_dev
->caps
.cqc_ba_pg_sz
);
846 roce_set_field(req
->vf_cqc_cfg
, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M
,
847 CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S
,
848 hr_dev
->caps
.cqc_buf_pg_sz
);
849 roce_set_field(req
->vf_cqc_cfg
, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M
,
850 CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S
,
851 cqc_hop_num
== HNS_ROCE_HOP_NUM_0
? 0 : cqc_hop_num
);
853 roce_set_field(req
->vf_mpt_cfg
, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M
,
854 CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S
,
855 hr_dev
->caps
.mpt_ba_pg_sz
);
856 roce_set_field(req
->vf_mpt_cfg
, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M
,
857 CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S
,
858 hr_dev
->caps
.mpt_buf_pg_sz
);
859 roce_set_field(req
->vf_mpt_cfg
, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M
,
860 CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S
,
861 mpt_hop_num
== HNS_ROCE_HOP_NUM_0
? 0 : mpt_hop_num
);
863 return hns_roce_cmq_send(hr_dev
, &desc
, 1);
866 static int hns_roce_v2_profile(struct hns_roce_dev
*hr_dev
)
868 struct hns_roce_caps
*caps
= &hr_dev
->caps
;
871 ret
= hns_roce_cmq_query_hw_info(hr_dev
);
873 dev_err(hr_dev
->dev
, "Query firmware version fail, ret = %d.\n",
878 ret
= hns_roce_config_global_param(hr_dev
);
880 dev_err(hr_dev
->dev
, "Configure global param fail, ret = %d.\n",
884 /* Get pf resource owned by every pf */
885 ret
= hns_roce_query_pf_resource(hr_dev
);
887 dev_err(hr_dev
->dev
, "Query pf resource fail, ret = %d.\n",
892 ret
= hns_roce_alloc_vf_resource(hr_dev
);
894 dev_err(hr_dev
->dev
, "Allocate vf resource fail, ret = %d.\n",
899 hr_dev
->vendor_part_id
= 0;
900 hr_dev
->sys_image_guid
= 0;
902 caps
->num_qps
= HNS_ROCE_V2_MAX_QP_NUM
;
903 caps
->max_wqes
= HNS_ROCE_V2_MAX_WQE_NUM
;
904 caps
->num_cqs
= HNS_ROCE_V2_MAX_CQ_NUM
;
905 caps
->max_cqes
= HNS_ROCE_V2_MAX_CQE_NUM
;
906 caps
->max_sq_sg
= HNS_ROCE_V2_MAX_SQ_SGE_NUM
;
907 caps
->max_rq_sg
= HNS_ROCE_V2_MAX_RQ_SGE_NUM
;
908 caps
->max_sq_inline
= HNS_ROCE_V2_MAX_SQ_INLINE
;
909 caps
->num_uars
= HNS_ROCE_V2_UAR_NUM
;
910 caps
->phy_num_uars
= HNS_ROCE_V2_PHY_UAR_NUM
;
911 caps
->num_aeq_vectors
= 1;
912 caps
->num_comp_vectors
= 63;
913 caps
->num_other_vectors
= 0;
914 caps
->num_mtpts
= HNS_ROCE_V2_MAX_MTPT_NUM
;
915 caps
->num_mtt_segs
= HNS_ROCE_V2_MAX_MTT_SEGS
;
916 caps
->num_cqe_segs
= HNS_ROCE_V2_MAX_CQE_SEGS
;
917 caps
->num_pds
= HNS_ROCE_V2_MAX_PD_NUM
;
918 caps
->max_qp_init_rdma
= HNS_ROCE_V2_MAX_QP_INIT_RDMA
;
919 caps
->max_qp_dest_rdma
= HNS_ROCE_V2_MAX_QP_DEST_RDMA
;
920 caps
->max_sq_desc_sz
= HNS_ROCE_V2_MAX_SQ_DESC_SZ
;
921 caps
->max_rq_desc_sz
= HNS_ROCE_V2_MAX_RQ_DESC_SZ
;
922 caps
->max_srq_desc_sz
= HNS_ROCE_V2_MAX_SRQ_DESC_SZ
;
923 caps
->qpc_entry_sz
= HNS_ROCE_V2_QPC_ENTRY_SZ
;
924 caps
->irrl_entry_sz
= HNS_ROCE_V2_IRRL_ENTRY_SZ
;
925 caps
->trrl_entry_sz
= HNS_ROCE_V2_TRRL_ENTRY_SZ
;
926 caps
->cqc_entry_sz
= HNS_ROCE_V2_CQC_ENTRY_SZ
;
927 caps
->mtpt_entry_sz
= HNS_ROCE_V2_MTPT_ENTRY_SZ
;
928 caps
->mtt_entry_sz
= HNS_ROCE_V2_MTT_ENTRY_SZ
;
929 caps
->cq_entry_sz
= HNS_ROCE_V2_CQE_ENTRY_SIZE
;
930 caps
->page_size_cap
= HNS_ROCE_V2_PAGE_SIZE_SUPPORTED
;
931 caps
->reserved_lkey
= 0;
932 caps
->reserved_pds
= 0;
933 caps
->reserved_mrws
= 1;
934 caps
->reserved_uars
= 0;
935 caps
->reserved_cqs
= 0;
937 caps
->qpc_ba_pg_sz
= 0;
938 caps
->qpc_buf_pg_sz
= 0;
939 caps
->qpc_hop_num
= HNS_ROCE_CONTEXT_HOP_NUM
;
940 caps
->srqc_ba_pg_sz
= 0;
941 caps
->srqc_buf_pg_sz
= 0;
942 caps
->srqc_hop_num
= HNS_ROCE_HOP_NUM_0
;
943 caps
->cqc_ba_pg_sz
= 0;
944 caps
->cqc_buf_pg_sz
= 0;
945 caps
->cqc_hop_num
= HNS_ROCE_CONTEXT_HOP_NUM
;
946 caps
->mpt_ba_pg_sz
= 0;
947 caps
->mpt_buf_pg_sz
= 0;
948 caps
->mpt_hop_num
= HNS_ROCE_CONTEXT_HOP_NUM
;
949 caps
->pbl_ba_pg_sz
= 0;
950 caps
->pbl_buf_pg_sz
= 0;
951 caps
->pbl_hop_num
= HNS_ROCE_PBL_HOP_NUM
;
952 caps
->mtt_ba_pg_sz
= 0;
953 caps
->mtt_buf_pg_sz
= 0;
954 caps
->mtt_hop_num
= HNS_ROCE_MTT_HOP_NUM
;
955 caps
->cqe_ba_pg_sz
= 0;
956 caps
->cqe_buf_pg_sz
= 0;
957 caps
->cqe_hop_num
= HNS_ROCE_CQE_HOP_NUM
;
958 caps
->chunk_sz
= HNS_ROCE_V2_TABLE_CHUNK_SIZE
;
960 caps
->flags
= HNS_ROCE_CAP_FLAG_REREG_MR
|
961 HNS_ROCE_CAP_FLAG_ROCE_V1_V2
;
962 caps
->pkey_table_len
[0] = 1;
963 caps
->gid_table_len
[0] = HNS_ROCE_V2_GID_INDEX_NUM
;
964 caps
->local_ca_ack_delay
= 0;
965 caps
->max_mtu
= IB_MTU_4096
;
967 ret
= hns_roce_v2_set_bt(hr_dev
);
969 dev_err(hr_dev
->dev
, "Configure bt attribute fail, ret = %d.\n",
975 static int hns_roce_v2_cmd_pending(struct hns_roce_dev
*hr_dev
)
977 u32 status
= readl(hr_dev
->reg_base
+ ROCEE_VF_MB_STATUS_REG
);
979 return status
>> HNS_ROCE_HW_RUN_BIT_SHIFT
;
982 static int hns_roce_v2_cmd_complete(struct hns_roce_dev
*hr_dev
)
984 u32 status
= readl(hr_dev
->reg_base
+ ROCEE_VF_MB_STATUS_REG
);
986 return status
& HNS_ROCE_HW_MB_STATUS_MASK
;
989 static int hns_roce_v2_post_mbox(struct hns_roce_dev
*hr_dev
, u64 in_param
,
990 u64 out_param
, u32 in_modifier
, u8 op_modifier
,
991 u16 op
, u16 token
, int event
)
993 struct device
*dev
= hr_dev
->dev
;
994 u32 __iomem
*hcr
= (u32 __iomem
*)(hr_dev
->reg_base
+
995 ROCEE_VF_MB_CFG0_REG
);
1000 end
= msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS
) + jiffies
;
1001 while (hns_roce_v2_cmd_pending(hr_dev
)) {
1002 if (time_after(jiffies
, end
)) {
1003 dev_dbg(dev
, "jiffies=%d end=%d\n", (int)jiffies
,
1010 roce_set_field(val0
, HNS_ROCE_VF_MB4_TAG_MASK
,
1011 HNS_ROCE_VF_MB4_TAG_SHIFT
, in_modifier
);
1012 roce_set_field(val0
, HNS_ROCE_VF_MB4_CMD_MASK
,
1013 HNS_ROCE_VF_MB4_CMD_SHIFT
, op
);
1014 roce_set_field(val1
, HNS_ROCE_VF_MB5_EVENT_MASK
,
1015 HNS_ROCE_VF_MB5_EVENT_SHIFT
, event
);
1016 roce_set_field(val1
, HNS_ROCE_VF_MB5_TOKEN_MASK
,
1017 HNS_ROCE_VF_MB5_TOKEN_SHIFT
, token
);
1019 __raw_writeq(cpu_to_le64(in_param
), hcr
+ 0);
1020 __raw_writeq(cpu_to_le64(out_param
), hcr
+ 2);
1022 /* Memory barrier */
1025 __raw_writel(cpu_to_le32(val0
), hcr
+ 4);
1026 __raw_writel(cpu_to_le32(val1
), hcr
+ 5);
1033 static int hns_roce_v2_chk_mbox(struct hns_roce_dev
*hr_dev
,
1034 unsigned long timeout
)
1036 struct device
*dev
= hr_dev
->dev
;
1037 unsigned long end
= 0;
1040 end
= msecs_to_jiffies(timeout
) + jiffies
;
1041 while (hns_roce_v2_cmd_pending(hr_dev
) && time_before(jiffies
, end
))
1044 if (hns_roce_v2_cmd_pending(hr_dev
)) {
1045 dev_err(dev
, "[cmd_poll]hw run cmd TIMEDOUT!\n");
1049 status
= hns_roce_v2_cmd_complete(hr_dev
);
1050 if (status
!= 0x1) {
1051 dev_err(dev
, "mailbox status 0x%x!\n", status
);
1058 static int hns_roce_v2_set_gid(struct hns_roce_dev
*hr_dev
, u8 port
,
1059 int gid_index
, union ib_gid
*gid
,
1060 const struct ib_gid_attr
*attr
)
1062 enum hns_roce_sgid_type sgid_type
= GID_TYPE_FLAG_ROCE_V1
;
1069 if (attr
->gid_type
== IB_GID_TYPE_ROCE
)
1070 sgid_type
= GID_TYPE_FLAG_ROCE_V1
;
1072 if (attr
->gid_type
== IB_GID_TYPE_ROCE_UDP_ENCAP
) {
1073 if (ipv6_addr_v4mapped((void *)gid
))
1074 sgid_type
= GID_TYPE_FLAG_ROCE_V2_IPV4
;
1076 sgid_type
= GID_TYPE_FLAG_ROCE_V2_IPV6
;
1079 p
= (u32
*)&gid
->raw
[0];
1080 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_VF_SGID_CFG0_REG
+
1083 p
= (u32
*)&gid
->raw
[4];
1084 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_VF_SGID_CFG1_REG
+
1087 p
= (u32
*)&gid
->raw
[8];
1088 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_VF_SGID_CFG2_REG
+
1091 p
= (u32
*)&gid
->raw
[0xc];
1092 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_VF_SGID_CFG3_REG
+
1095 val
= roce_read(hr_dev
, ROCEE_VF_SGID_CFG4_REG
+ 0x20 * gid_index
);
1096 roce_set_field(val
, ROCEE_VF_SGID_CFG4_SGID_TYPE_M
,
1097 ROCEE_VF_SGID_CFG4_SGID_TYPE_S
, sgid_type
);
1099 roce_write(hr_dev
, ROCEE_VF_SGID_CFG4_REG
+ 0x20 * gid_index
, val
);
1104 static int hns_roce_v2_set_mac(struct hns_roce_dev
*hr_dev
, u8 phy_port
,
1111 reg_smac_l
= *(u32
*)(&addr
[0]);
1112 roce_raw_write(reg_smac_l
, hr_dev
->reg_base
+ ROCEE_VF_SMAC_CFG0_REG
+
1114 val
= roce_read(hr_dev
, ROCEE_VF_SMAC_CFG1_REG
+ 0x08 * phy_port
);
1116 reg_smac_h
= *(u16
*)(&addr
[4]);
1117 roce_set_field(val
, ROCEE_VF_SMAC_CFG1_VF_SMAC_H_M
,
1118 ROCEE_VF_SMAC_CFG1_VF_SMAC_H_S
, reg_smac_h
);
1119 roce_write(hr_dev
, ROCEE_VF_SMAC_CFG1_REG
+ 0x08 * phy_port
, val
);
1124 static int hns_roce_v2_write_mtpt(void *mb_buf
, struct hns_roce_mr
*mr
,
1125 unsigned long mtpt_idx
)
1127 struct hns_roce_v2_mpt_entry
*mpt_entry
;
1128 struct scatterlist
*sg
;
1134 memset(mpt_entry
, 0, sizeof(*mpt_entry
));
1136 roce_set_field(mpt_entry
->byte_4_pd_hop_st
, V2_MPT_BYTE_4_MPT_ST_M
,
1137 V2_MPT_BYTE_4_MPT_ST_S
, V2_MPT_ST_VALID
);
1138 roce_set_field(mpt_entry
->byte_4_pd_hop_st
, V2_MPT_BYTE_4_PBL_HOP_NUM_M
,
1139 V2_MPT_BYTE_4_PBL_HOP_NUM_S
, mr
->pbl_hop_num
==
1140 HNS_ROCE_HOP_NUM_0
? 0 : mr
->pbl_hop_num
);
1141 roce_set_field(mpt_entry
->byte_4_pd_hop_st
,
1142 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M
,
1143 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S
, mr
->pbl_ba_pg_sz
);
1144 roce_set_field(mpt_entry
->byte_4_pd_hop_st
, V2_MPT_BYTE_4_PD_M
,
1145 V2_MPT_BYTE_4_PD_S
, mr
->pd
);
1146 mpt_entry
->byte_4_pd_hop_st
= cpu_to_le32(mpt_entry
->byte_4_pd_hop_st
);
1148 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_RA_EN_S
, 0);
1149 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_R_INV_EN_S
, 1);
1150 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_L_INV_EN_S
, 0);
1151 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_BIND_EN_S
,
1152 (mr
->access
& IB_ACCESS_MW_BIND
? 1 : 0));
1153 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_ATOMIC_EN_S
, 0);
1154 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_RR_EN_S
,
1155 (mr
->access
& IB_ACCESS_REMOTE_READ
? 1 : 0));
1156 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_RW_EN_S
,
1157 (mr
->access
& IB_ACCESS_REMOTE_WRITE
? 1 : 0));
1158 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_LW_EN_S
,
1159 (mr
->access
& IB_ACCESS_LOCAL_WRITE
? 1 : 0));
1160 mpt_entry
->byte_8_mw_cnt_en
= cpu_to_le32(mpt_entry
->byte_8_mw_cnt_en
);
1162 roce_set_bit(mpt_entry
->byte_12_mw_pa
, V2_MPT_BYTE_12_PA_S
,
1163 mr
->type
== MR_TYPE_MR
? 0 : 1);
1164 mpt_entry
->byte_12_mw_pa
= cpu_to_le32(mpt_entry
->byte_12_mw_pa
);
1166 mpt_entry
->len_l
= cpu_to_le32(lower_32_bits(mr
->size
));
1167 mpt_entry
->len_h
= cpu_to_le32(upper_32_bits(mr
->size
));
1168 mpt_entry
->lkey
= cpu_to_le32(mr
->key
);
1169 mpt_entry
->va_l
= cpu_to_le32(lower_32_bits(mr
->iova
));
1170 mpt_entry
->va_h
= cpu_to_le32(upper_32_bits(mr
->iova
));
1172 if (mr
->type
== MR_TYPE_DMA
)
1175 mpt_entry
->pbl_size
= cpu_to_le32(mr
->pbl_size
);
1177 mpt_entry
->pbl_ba_l
= cpu_to_le32(lower_32_bits(mr
->pbl_ba
>> 3));
1178 roce_set_field(mpt_entry
->byte_48_mode_ba
, V2_MPT_BYTE_48_PBL_BA_H_M
,
1179 V2_MPT_BYTE_48_PBL_BA_H_S
,
1180 upper_32_bits(mr
->pbl_ba
>> 3));
1181 mpt_entry
->byte_48_mode_ba
= cpu_to_le32(mpt_entry
->byte_48_mode_ba
);
1183 pages
= (u64
*)__get_free_page(GFP_KERNEL
);
1188 for_each_sg(mr
->umem
->sg_head
.sgl
, sg
, mr
->umem
->nmap
, entry
) {
1189 pages
[i
] = ((u64
)sg_dma_address(sg
)) >> 6;
1191 /* Record the first 2 entry directly to MTPT table */
1192 if (i
>= HNS_ROCE_V2_MAX_INNER_MTPT_NUM
- 1)
1197 mpt_entry
->pa0_l
= cpu_to_le32(lower_32_bits(pages
[0]));
1198 roce_set_field(mpt_entry
->byte_56_pa0_h
, V2_MPT_BYTE_56_PA0_H_M
,
1199 V2_MPT_BYTE_56_PA0_H_S
,
1200 upper_32_bits(pages
[0]));
1201 mpt_entry
->byte_56_pa0_h
= cpu_to_le32(mpt_entry
->byte_56_pa0_h
);
1203 mpt_entry
->pa1_l
= cpu_to_le32(lower_32_bits(pages
[1]));
1204 roce_set_field(mpt_entry
->byte_64_buf_pa1
, V2_MPT_BYTE_64_PA1_H_M
,
1205 V2_MPT_BYTE_64_PA1_H_S
, upper_32_bits(pages
[1]));
1207 free_page((unsigned long)pages
);
1209 roce_set_field(mpt_entry
->byte_64_buf_pa1
,
1210 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M
,
1211 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S
, mr
->pbl_buf_pg_sz
);
1212 mpt_entry
->byte_64_buf_pa1
= cpu_to_le32(mpt_entry
->byte_64_buf_pa1
);
1217 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev
*hr_dev
,
1218 struct hns_roce_mr
*mr
, int flags
,
1219 u32 pdn
, int mr_access_flags
, u64 iova
,
1220 u64 size
, void *mb_buf
)
1222 struct hns_roce_v2_mpt_entry
*mpt_entry
= mb_buf
;
1224 if (flags
& IB_MR_REREG_PD
) {
1225 roce_set_field(mpt_entry
->byte_4_pd_hop_st
, V2_MPT_BYTE_4_PD_M
,
1226 V2_MPT_BYTE_4_PD_S
, pdn
);
1230 if (flags
& IB_MR_REREG_ACCESS
) {
1231 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
,
1232 V2_MPT_BYTE_8_BIND_EN_S
,
1233 (mr_access_flags
& IB_ACCESS_MW_BIND
? 1 : 0));
1234 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
,
1235 V2_MPT_BYTE_8_ATOMIC_EN_S
,
1236 (mr_access_flags
& IB_ACCESS_REMOTE_ATOMIC
? 1 : 0));
1237 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_RR_EN_S
,
1238 (mr_access_flags
& IB_ACCESS_REMOTE_READ
? 1 : 0));
1239 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_RW_EN_S
,
1240 (mr_access_flags
& IB_ACCESS_REMOTE_WRITE
? 1 : 0));
1241 roce_set_bit(mpt_entry
->byte_8_mw_cnt_en
, V2_MPT_BYTE_8_LW_EN_S
,
1242 (mr_access_flags
& IB_ACCESS_LOCAL_WRITE
? 1 : 0));
1245 if (flags
& IB_MR_REREG_TRANS
) {
1246 mpt_entry
->va_l
= cpu_to_le32(lower_32_bits(iova
));
1247 mpt_entry
->va_h
= cpu_to_le32(upper_32_bits(iova
));
1248 mpt_entry
->len_l
= cpu_to_le32(lower_32_bits(size
));
1249 mpt_entry
->len_h
= cpu_to_le32(upper_32_bits(size
));
1251 mpt_entry
->pbl_size
= cpu_to_le32(mr
->pbl_size
);
1252 mpt_entry
->pbl_ba_l
=
1253 cpu_to_le32(lower_32_bits(mr
->pbl_ba
>> 3));
1254 roce_set_field(mpt_entry
->byte_48_mode_ba
,
1255 V2_MPT_BYTE_48_PBL_BA_H_M
,
1256 V2_MPT_BYTE_48_PBL_BA_H_S
,
1257 upper_32_bits(mr
->pbl_ba
>> 3));
1258 mpt_entry
->byte_48_mode_ba
=
1259 cpu_to_le32(mpt_entry
->byte_48_mode_ba
);
1268 static void *get_cqe_v2(struct hns_roce_cq
*hr_cq
, int n
)
1270 return hns_roce_buf_offset(&hr_cq
->hr_buf
.hr_buf
,
1271 n
* HNS_ROCE_V2_CQE_ENTRY_SIZE
);
1274 static void *get_sw_cqe_v2(struct hns_roce_cq
*hr_cq
, int n
)
1276 struct hns_roce_v2_cqe
*cqe
= get_cqe_v2(hr_cq
, n
& hr_cq
->ib_cq
.cqe
);
1278 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1279 return (roce_get_bit(cqe
->byte_4
, V2_CQE_BYTE_4_OWNER_S
) ^
1280 !!(n
& (hr_cq
->ib_cq
.cqe
+ 1))) ? cqe
: NULL
;
1283 static struct hns_roce_v2_cqe
*next_cqe_sw_v2(struct hns_roce_cq
*hr_cq
)
1285 return get_sw_cqe_v2(hr_cq
, hr_cq
->cons_index
);
1288 static void hns_roce_v2_cq_set_ci(struct hns_roce_cq
*hr_cq
, u32 cons_index
)
1290 struct hns_roce_v2_cq_db cq_db
;
1293 cq_db
.parameter
= 0;
1295 roce_set_field(cq_db
.byte_4
, V2_CQ_DB_BYTE_4_TAG_M
,
1296 V2_CQ_DB_BYTE_4_TAG_S
, hr_cq
->cqn
);
1297 roce_set_field(cq_db
.byte_4
, V2_CQ_DB_BYTE_4_CMD_M
,
1298 V2_CQ_DB_BYTE_4_CMD_S
, HNS_ROCE_V2_CQ_DB_PTR
);
1300 roce_set_field(cq_db
.parameter
, V2_CQ_DB_PARAMETER_CONS_IDX_M
,
1301 V2_CQ_DB_PARAMETER_CONS_IDX_S
,
1302 cons_index
& ((hr_cq
->cq_depth
<< 1) - 1));
1303 roce_set_field(cq_db
.parameter
, V2_CQ_DB_PARAMETER_CMD_SN_M
,
1304 V2_CQ_DB_PARAMETER_CMD_SN_S
, 1);
1306 hns_roce_write64_k((__be32
*)&cq_db
, hr_cq
->cq_db_l
);
1310 static void __hns_roce_v2_cq_clean(struct hns_roce_cq
*hr_cq
, u32 qpn
,
1311 struct hns_roce_srq
*srq
)
1313 struct hns_roce_v2_cqe
*cqe
, *dest
;
1318 for (prod_index
= hr_cq
->cons_index
; get_sw_cqe_v2(hr_cq
, prod_index
);
1320 if (prod_index
== hr_cq
->cons_index
+ hr_cq
->ib_cq
.cqe
)
1325 * Now backwards through the CQ, removing CQ entries
1326 * that match our QP by overwriting them with next entries.
1328 while ((int) --prod_index
- (int) hr_cq
->cons_index
>= 0) {
1329 cqe
= get_cqe_v2(hr_cq
, prod_index
& hr_cq
->ib_cq
.cqe
);
1330 if ((roce_get_field(cqe
->byte_16
, V2_CQE_BYTE_16_LCL_QPN_M
,
1331 V2_CQE_BYTE_16_LCL_QPN_S
) &
1332 HNS_ROCE_V2_CQE_QPN_MASK
) == qpn
) {
1333 /* In v1 engine, not support SRQ */
1335 } else if (nfreed
) {
1336 dest
= get_cqe_v2(hr_cq
, (prod_index
+ nfreed
) &
1338 owner_bit
= roce_get_bit(dest
->byte_4
,
1339 V2_CQE_BYTE_4_OWNER_S
);
1340 memcpy(dest
, cqe
, sizeof(*cqe
));
1341 roce_set_bit(dest
->byte_4
, V2_CQE_BYTE_4_OWNER_S
,
1347 hr_cq
->cons_index
+= nfreed
;
1349 * Make sure update of buffer contents is done before
1350 * updating consumer index.
1353 hns_roce_v2_cq_set_ci(hr_cq
, hr_cq
->cons_index
);
1357 static void hns_roce_v2_cq_clean(struct hns_roce_cq
*hr_cq
, u32 qpn
,
1358 struct hns_roce_srq
*srq
)
1360 spin_lock_irq(&hr_cq
->lock
);
1361 __hns_roce_v2_cq_clean(hr_cq
, qpn
, srq
);
1362 spin_unlock_irq(&hr_cq
->lock
);
1365 static void hns_roce_v2_write_cqc(struct hns_roce_dev
*hr_dev
,
1366 struct hns_roce_cq
*hr_cq
, void *mb_buf
,
1367 u64
*mtts
, dma_addr_t dma_handle
, int nent
,
1370 struct hns_roce_v2_cq_context
*cq_context
;
1372 cq_context
= mb_buf
;
1373 memset(cq_context
, 0, sizeof(*cq_context
));
1375 roce_set_field(cq_context
->byte_4_pg_ceqn
, V2_CQC_BYTE_4_CQ_ST_M
,
1376 V2_CQC_BYTE_4_CQ_ST_S
, V2_CQ_STATE_VALID
);
1377 roce_set_field(cq_context
->byte_4_pg_ceqn
, V2_CQC_BYTE_4_SHIFT_M
,
1378 V2_CQC_BYTE_4_SHIFT_S
, ilog2((unsigned int)nent
));
1379 roce_set_field(cq_context
->byte_4_pg_ceqn
, V2_CQC_BYTE_4_CEQN_M
,
1380 V2_CQC_BYTE_4_CEQN_S
, vector
);
1381 cq_context
->byte_4_pg_ceqn
= cpu_to_le32(cq_context
->byte_4_pg_ceqn
);
1383 roce_set_field(cq_context
->byte_8_cqn
, V2_CQC_BYTE_8_CQN_M
,
1384 V2_CQC_BYTE_8_CQN_S
, hr_cq
->cqn
);
1386 cq_context
->cqe_cur_blk_addr
= (u32
)(mtts
[0] >> PAGE_ADDR_SHIFT
);
1387 cq_context
->cqe_cur_blk_addr
=
1388 cpu_to_le32(cq_context
->cqe_cur_blk_addr
);
1390 roce_set_field(cq_context
->byte_16_hop_addr
,
1391 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M
,
1392 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S
,
1393 cpu_to_le32((mtts
[0]) >> (32 + PAGE_ADDR_SHIFT
)));
1394 roce_set_field(cq_context
->byte_16_hop_addr
,
1395 V2_CQC_BYTE_16_CQE_HOP_NUM_M
,
1396 V2_CQC_BYTE_16_CQE_HOP_NUM_S
, hr_dev
->caps
.cqe_hop_num
==
1397 HNS_ROCE_HOP_NUM_0
? 0 : hr_dev
->caps
.cqe_hop_num
);
1399 cq_context
->cqe_nxt_blk_addr
= (u32
)(mtts
[1] >> PAGE_ADDR_SHIFT
);
1400 roce_set_field(cq_context
->byte_24_pgsz_addr
,
1401 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M
,
1402 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S
,
1403 cpu_to_le32((mtts
[1]) >> (32 + PAGE_ADDR_SHIFT
)));
1404 roce_set_field(cq_context
->byte_24_pgsz_addr
,
1405 V2_CQC_BYTE_24_CQE_BA_PG_SZ_M
,
1406 V2_CQC_BYTE_24_CQE_BA_PG_SZ_S
,
1407 hr_dev
->caps
.cqe_ba_pg_sz
);
1408 roce_set_field(cq_context
->byte_24_pgsz_addr
,
1409 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M
,
1410 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S
,
1411 hr_dev
->caps
.cqe_buf_pg_sz
);
1413 cq_context
->cqe_ba
= (u32
)(dma_handle
>> 3);
1415 roce_set_field(cq_context
->byte_40_cqe_ba
, V2_CQC_BYTE_40_CQE_BA_M
,
1416 V2_CQC_BYTE_40_CQE_BA_S
, (dma_handle
>> (32 + 3)));
1419 static int hns_roce_v2_req_notify_cq(struct ib_cq
*ibcq
,
1420 enum ib_cq_notify_flags flags
)
1422 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
1423 u32 notification_flag
;
1429 notification_flag
= (flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
?
1430 V2_CQ_DB_REQ_NOT
: V2_CQ_DB_REQ_NOT_SOL
;
1432 * flags = 0; Notification Flag = 1, next
1433 * flags = 1; Notification Flag = 0, solocited
1435 roce_set_field(doorbell
[0], V2_CQ_DB_BYTE_4_TAG_M
, V2_DB_BYTE_4_TAG_S
,
1437 roce_set_field(doorbell
[0], V2_CQ_DB_BYTE_4_CMD_M
, V2_DB_BYTE_4_CMD_S
,
1438 HNS_ROCE_V2_CQ_DB_NTR
);
1439 roce_set_field(doorbell
[1], V2_CQ_DB_PARAMETER_CONS_IDX_M
,
1440 V2_CQ_DB_PARAMETER_CONS_IDX_S
,
1441 hr_cq
->cons_index
& ((hr_cq
->cq_depth
<< 1) - 1));
1442 roce_set_field(doorbell
[1], V2_CQ_DB_PARAMETER_CMD_SN_M
,
1443 V2_CQ_DB_PARAMETER_CMD_SN_S
, hr_cq
->arm_sn
& 0x3);
1444 roce_set_bit(doorbell
[1], V2_CQ_DB_PARAMETER_NOTIFY_S
,
1447 hns_roce_write64_k(doorbell
, hr_cq
->cq_db_l
);
1452 static int hns_roce_v2_poll_one(struct hns_roce_cq
*hr_cq
,
1453 struct hns_roce_qp
**cur_qp
, struct ib_wc
*wc
)
1455 struct hns_roce_dev
*hr_dev
;
1456 struct hns_roce_v2_cqe
*cqe
;
1457 struct hns_roce_qp
*hr_qp
;
1458 struct hns_roce_wq
*wq
;
1465 /* Find cqe according to consumer index */
1466 cqe
= next_cqe_sw_v2(hr_cq
);
1470 ++hr_cq
->cons_index
;
1471 /* Memory barrier */
1475 is_send
= !roce_get_bit(cqe
->byte_4
, V2_CQE_BYTE_4_S_R_S
);
1477 qpn
= roce_get_field(cqe
->byte_16
, V2_CQE_BYTE_16_LCL_QPN_M
,
1478 V2_CQE_BYTE_16_LCL_QPN_S
);
1480 if (!*cur_qp
|| (qpn
& HNS_ROCE_V2_CQE_QPN_MASK
) != (*cur_qp
)->qpn
) {
1481 hr_dev
= to_hr_dev(hr_cq
->ib_cq
.device
);
1482 hr_qp
= __hns_roce_qp_lookup(hr_dev
, qpn
);
1483 if (unlikely(!hr_qp
)) {
1484 dev_err(hr_dev
->dev
, "CQ %06lx with entry for unknown QPN %06x\n",
1485 hr_cq
->cqn
, (qpn
& HNS_ROCE_V2_CQE_QPN_MASK
));
1491 wc
->qp
= &(*cur_qp
)->ibqp
;
1494 status
= roce_get_field(cqe
->byte_4
, V2_CQE_BYTE_4_STATUS_M
,
1495 V2_CQE_BYTE_4_STATUS_S
);
1496 switch (status
& HNS_ROCE_V2_CQE_STATUS_MASK
) {
1497 case HNS_ROCE_CQE_V2_SUCCESS
:
1498 wc
->status
= IB_WC_SUCCESS
;
1500 case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR
:
1501 wc
->status
= IB_WC_LOC_LEN_ERR
;
1503 case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR
:
1504 wc
->status
= IB_WC_LOC_QP_OP_ERR
;
1506 case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR
:
1507 wc
->status
= IB_WC_LOC_PROT_ERR
;
1509 case HNS_ROCE_CQE_V2_WR_FLUSH_ERR
:
1510 wc
->status
= IB_WC_WR_FLUSH_ERR
;
1512 case HNS_ROCE_CQE_V2_MW_BIND_ERR
:
1513 wc
->status
= IB_WC_MW_BIND_ERR
;
1515 case HNS_ROCE_CQE_V2_BAD_RESP_ERR
:
1516 wc
->status
= IB_WC_BAD_RESP_ERR
;
1518 case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR
:
1519 wc
->status
= IB_WC_LOC_ACCESS_ERR
;
1521 case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR
:
1522 wc
->status
= IB_WC_REM_INV_REQ_ERR
;
1524 case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR
:
1525 wc
->status
= IB_WC_REM_ACCESS_ERR
;
1527 case HNS_ROCE_CQE_V2_REMOTE_OP_ERR
:
1528 wc
->status
= IB_WC_REM_OP_ERR
;
1530 case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR
:
1531 wc
->status
= IB_WC_RETRY_EXC_ERR
;
1533 case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR
:
1534 wc
->status
= IB_WC_RNR_RETRY_EXC_ERR
;
1536 case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR
:
1537 wc
->status
= IB_WC_REM_ABORT_ERR
;
1540 wc
->status
= IB_WC_GENERAL_ERR
;
1544 /* CQE status error, directly return */
1545 if (wc
->status
!= IB_WC_SUCCESS
)
1550 /* SQ corresponding to CQE */
1551 switch (roce_get_field(cqe
->byte_4
, V2_CQE_BYTE_4_OPCODE_M
,
1552 V2_CQE_BYTE_4_OPCODE_S
) & 0x1f) {
1553 case HNS_ROCE_SQ_OPCODE_SEND
:
1554 wc
->opcode
= IB_WC_SEND
;
1556 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV
:
1557 wc
->opcode
= IB_WC_SEND
;
1559 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM
:
1560 wc
->opcode
= IB_WC_SEND
;
1561 wc
->wc_flags
|= IB_WC_WITH_IMM
;
1563 case HNS_ROCE_SQ_OPCODE_RDMA_READ
:
1564 wc
->opcode
= IB_WC_RDMA_READ
;
1565 wc
->byte_len
= le32_to_cpu(cqe
->byte_cnt
);
1567 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE
:
1568 wc
->opcode
= IB_WC_RDMA_WRITE
;
1570 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM
:
1571 wc
->opcode
= IB_WC_RDMA_WRITE
;
1572 wc
->wc_flags
|= IB_WC_WITH_IMM
;
1574 case HNS_ROCE_SQ_OPCODE_LOCAL_INV
:
1575 wc
->opcode
= IB_WC_LOCAL_INV
;
1576 wc
->wc_flags
|= IB_WC_WITH_INVALIDATE
;
1578 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP
:
1579 wc
->opcode
= IB_WC_COMP_SWAP
;
1582 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD
:
1583 wc
->opcode
= IB_WC_FETCH_ADD
;
1586 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP
:
1587 wc
->opcode
= IB_WC_MASKED_COMP_SWAP
;
1590 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD
:
1591 wc
->opcode
= IB_WC_MASKED_FETCH_ADD
;
1594 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR
:
1595 wc
->opcode
= IB_WC_REG_MR
;
1597 case HNS_ROCE_SQ_OPCODE_BIND_MW
:
1598 wc
->opcode
= IB_WC_REG_MR
;
1601 wc
->status
= IB_WC_GENERAL_ERR
;
1605 wq
= &(*cur_qp
)->sq
;
1606 if ((*cur_qp
)->sq_signal_bits
) {
1608 * If sg_signal_bit is 1,
1609 * firstly tail pointer updated to wqe
1610 * which current cqe correspond to
1612 wqe_ctr
= (u16
)roce_get_field(cqe
->byte_4
,
1613 V2_CQE_BYTE_4_WQE_INDX_M
,
1614 V2_CQE_BYTE_4_WQE_INDX_S
);
1615 wq
->tail
+= (wqe_ctr
- (u16
)wq
->tail
) &
1619 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
1622 /* RQ correspond to CQE */
1623 wc
->byte_len
= le32_to_cpu(cqe
->byte_cnt
);
1625 opcode
= roce_get_field(cqe
->byte_4
, V2_CQE_BYTE_4_OPCODE_M
,
1626 V2_CQE_BYTE_4_OPCODE_S
);
1627 switch (opcode
& 0x1f) {
1628 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM
:
1629 wc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
1630 wc
->wc_flags
= IB_WC_WITH_IMM
;
1631 wc
->ex
.imm_data
= le32_to_cpu(cqe
->rkey_immtdata
);
1633 case HNS_ROCE_V2_OPCODE_SEND
:
1634 wc
->opcode
= IB_WC_RECV
;
1637 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM
:
1638 wc
->opcode
= IB_WC_RECV
;
1639 wc
->wc_flags
= IB_WC_WITH_IMM
;
1640 wc
->ex
.imm_data
= le32_to_cpu(cqe
->rkey_immtdata
);
1642 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV
:
1643 wc
->opcode
= IB_WC_RECV
;
1644 wc
->wc_flags
= IB_WC_WITH_INVALIDATE
;
1645 wc
->ex
.invalidate_rkey
= cqe
->rkey_immtdata
;
1648 wc
->status
= IB_WC_GENERAL_ERR
;
1652 /* Update tail pointer, record wr_id */
1653 wq
= &(*cur_qp
)->rq
;
1654 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
1657 wc
->sl
= (u8
)roce_get_field(cqe
->byte_32
, V2_CQE_BYTE_32_SL_M
,
1658 V2_CQE_BYTE_32_SL_S
);
1659 wc
->src_qp
= (u8
)roce_get_field(cqe
->byte_32
,
1660 V2_CQE_BYTE_32_RMT_QPN_M
,
1661 V2_CQE_BYTE_32_RMT_QPN_S
);
1662 wc
->wc_flags
|= (roce_get_bit(cqe
->byte_32
,
1663 V2_CQE_BYTE_32_GRH_S
) ?
1670 static int hns_roce_v2_poll_cq(struct ib_cq
*ibcq
, int num_entries
,
1673 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
1674 struct hns_roce_qp
*cur_qp
= NULL
;
1675 unsigned long flags
;
1678 spin_lock_irqsave(&hr_cq
->lock
, flags
);
1680 for (npolled
= 0; npolled
< num_entries
; ++npolled
) {
1681 if (hns_roce_v2_poll_one(hr_cq
, &cur_qp
, wc
+ npolled
))
1686 /* Memory barrier */
1688 hns_roce_v2_cq_set_ci(hr_cq
, hr_cq
->cons_index
);
1691 spin_unlock_irqrestore(&hr_cq
->lock
, flags
);
1696 static int hns_roce_v2_set_hem(struct hns_roce_dev
*hr_dev
,
1697 struct hns_roce_hem_table
*table
, int obj
,
1700 struct device
*dev
= hr_dev
->dev
;
1701 struct hns_roce_cmd_mailbox
*mailbox
;
1702 struct hns_roce_hem_iter iter
;
1703 struct hns_roce_hem_mhop mhop
;
1704 struct hns_roce_hem
*hem
;
1705 unsigned long mhop_obj
= obj
;
1715 if (!hns_roce_check_whether_mhop(hr_dev
, table
->type
))
1718 hns_roce_calc_hem_mhop(hr_dev
, table
, &mhop_obj
, &mhop
);
1722 hop_num
= mhop
.hop_num
;
1723 chunk_ba_num
= mhop
.bt_chunk_size
/ 8;
1726 hem_idx
= i
* chunk_ba_num
* chunk_ba_num
+ j
* chunk_ba_num
+
1728 l1_idx
= i
* chunk_ba_num
+ j
;
1729 } else if (hop_num
== 1) {
1730 hem_idx
= i
* chunk_ba_num
+ j
;
1731 } else if (hop_num
== HNS_ROCE_HOP_NUM_0
) {
1735 switch (table
->type
) {
1737 op
= HNS_ROCE_CMD_WRITE_QPC_BT0
;
1740 op
= HNS_ROCE_CMD_WRITE_MPT_BT0
;
1743 op
= HNS_ROCE_CMD_WRITE_CQC_BT0
;
1746 op
= HNS_ROCE_CMD_WRITE_SRQC_BT0
;
1749 dev_warn(dev
, "Table %d not to be written by mailbox!\n",
1755 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
1756 if (IS_ERR(mailbox
))
1757 return PTR_ERR(mailbox
);
1759 if (check_whether_last_step(hop_num
, step_idx
)) {
1760 hem
= table
->hem
[hem_idx
];
1761 for (hns_roce_hem_first(hem
, &iter
);
1762 !hns_roce_hem_last(&iter
); hns_roce_hem_next(&iter
)) {
1763 bt_ba
= hns_roce_hem_addr(&iter
);
1765 /* configure the ba, tag, and op */
1766 ret
= hns_roce_cmd_mbox(hr_dev
, bt_ba
, mailbox
->dma
,
1768 HNS_ROCE_CMD_TIMEOUT_MSECS
);
1772 bt_ba
= table
->bt_l0_dma_addr
[i
];
1773 else if (step_idx
== 1 && hop_num
== 2)
1774 bt_ba
= table
->bt_l1_dma_addr
[l1_idx
];
1776 /* configure the ba, tag, and op */
1777 ret
= hns_roce_cmd_mbox(hr_dev
, bt_ba
, mailbox
->dma
, obj
,
1778 0, op
, HNS_ROCE_CMD_TIMEOUT_MSECS
);
1781 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
1785 static int hns_roce_v2_clear_hem(struct hns_roce_dev
*hr_dev
,
1786 struct hns_roce_hem_table
*table
, int obj
,
1789 struct device
*dev
= hr_dev
->dev
;
1790 struct hns_roce_cmd_mailbox
*mailbox
;
1794 if (!hns_roce_check_whether_mhop(hr_dev
, table
->type
))
1797 switch (table
->type
) {
1799 op
= HNS_ROCE_CMD_DESTROY_QPC_BT0
;
1802 op
= HNS_ROCE_CMD_DESTROY_MPT_BT0
;
1805 op
= HNS_ROCE_CMD_DESTROY_CQC_BT0
;
1808 op
= HNS_ROCE_CMD_DESTROY_SRQC_BT0
;
1811 dev_warn(dev
, "Table %d not to be destroyed by mailbox!\n",
1817 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
1818 if (IS_ERR(mailbox
))
1819 return PTR_ERR(mailbox
);
1821 /* configure the tag and op */
1822 ret
= hns_roce_cmd_mbox(hr_dev
, 0, mailbox
->dma
, obj
, 0, op
,
1823 HNS_ROCE_CMD_TIMEOUT_MSECS
);
1825 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
1829 static int hns_roce_v2_qp_modify(struct hns_roce_dev
*hr_dev
,
1830 struct hns_roce_mtt
*mtt
,
1831 enum ib_qp_state cur_state
,
1832 enum ib_qp_state new_state
,
1833 struct hns_roce_v2_qp_context
*context
,
1834 struct hns_roce_qp
*hr_qp
)
1836 struct hns_roce_cmd_mailbox
*mailbox
;
1839 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
1840 if (IS_ERR(mailbox
))
1841 return PTR_ERR(mailbox
);
1843 memcpy(mailbox
->buf
, context
, sizeof(*context
) * 2);
1845 ret
= hns_roce_cmd_mbox(hr_dev
, mailbox
->dma
, 0, hr_qp
->qpn
, 0,
1846 HNS_ROCE_CMD_MODIFY_QPC
,
1847 HNS_ROCE_CMD_TIMEOUT_MSECS
);
1849 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
1854 static void modify_qp_reset_to_init(struct ib_qp
*ibqp
,
1855 const struct ib_qp_attr
*attr
,
1856 struct hns_roce_v2_qp_context
*context
,
1857 struct hns_roce_v2_qp_context
*qpc_mask
)
1859 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
1862 * In v2 engine, software pass context and context mask to hardware
1863 * when modifying qp. If software need modify some fields in context,
1864 * we should set all bits of the relevant fields in context mask to
1865 * 0 at the same time, else set them to 0x1.
1867 roce_set_field(context
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_TST_M
,
1868 V2_QPC_BYTE_4_TST_S
, to_hr_qp_type(hr_qp
->ibqp
.qp_type
));
1869 roce_set_field(qpc_mask
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_TST_M
,
1870 V2_QPC_BYTE_4_TST_S
, 0);
1872 roce_set_field(context
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_SGE_SHIFT_M
,
1873 V2_QPC_BYTE_4_SGE_SHIFT_S
, hr_qp
->sq
.max_gs
> 2 ?
1874 ilog2((unsigned int)hr_qp
->sge
.sge_cnt
) : 0);
1875 roce_set_field(qpc_mask
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_SGE_SHIFT_M
,
1876 V2_QPC_BYTE_4_SGE_SHIFT_S
, 0);
1878 roce_set_field(context
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_SQPN_M
,
1879 V2_QPC_BYTE_4_SQPN_S
, hr_qp
->qpn
);
1880 roce_set_field(qpc_mask
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_SQPN_M
,
1881 V2_QPC_BYTE_4_SQPN_S
, 0);
1883 roce_set_field(context
->byte_16_buf_ba_pg_sz
, V2_QPC_BYTE_16_PD_M
,
1884 V2_QPC_BYTE_16_PD_S
, to_hr_pd(ibqp
->pd
)->pdn
);
1885 roce_set_field(qpc_mask
->byte_16_buf_ba_pg_sz
, V2_QPC_BYTE_16_PD_M
,
1886 V2_QPC_BYTE_16_PD_S
, 0);
1888 roce_set_field(context
->byte_20_smac_sgid_idx
, V2_QPC_BYTE_20_RQWS_M
,
1889 V2_QPC_BYTE_20_RQWS_S
, ilog2(hr_qp
->rq
.max_gs
));
1890 roce_set_field(qpc_mask
->byte_20_smac_sgid_idx
, V2_QPC_BYTE_20_RQWS_M
,
1891 V2_QPC_BYTE_20_RQWS_S
, 0);
1893 roce_set_field(context
->byte_20_smac_sgid_idx
,
1894 V2_QPC_BYTE_20_SQ_SHIFT_M
, V2_QPC_BYTE_20_SQ_SHIFT_S
,
1895 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
1896 roce_set_field(qpc_mask
->byte_20_smac_sgid_idx
,
1897 V2_QPC_BYTE_20_SQ_SHIFT_M
, V2_QPC_BYTE_20_SQ_SHIFT_S
, 0);
1899 roce_set_field(context
->byte_20_smac_sgid_idx
,
1900 V2_QPC_BYTE_20_RQ_SHIFT_M
, V2_QPC_BYTE_20_RQ_SHIFT_S
,
1901 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
1902 roce_set_field(qpc_mask
->byte_20_smac_sgid_idx
,
1903 V2_QPC_BYTE_20_RQ_SHIFT_M
, V2_QPC_BYTE_20_RQ_SHIFT_S
, 0);
1905 /* No VLAN need to set 0xFFF */
1906 roce_set_field(context
->byte_24_mtu_tc
, V2_QPC_BYTE_24_VLAN_IDX_M
,
1907 V2_QPC_BYTE_24_VLAN_IDX_S
, 0xfff);
1908 roce_set_field(qpc_mask
->byte_24_mtu_tc
, V2_QPC_BYTE_24_VLAN_IDX_M
,
1909 V2_QPC_BYTE_24_VLAN_IDX_S
, 0);
1912 * Set some fields in context to zero, Because the default values
1913 * of all fields in context are zero, we need not set them to 0 again.
1914 * but we should set the relevant fields of context mask to 0.
1916 roce_set_bit(qpc_mask
->byte_56_dqpn_err
, V2_QPC_BYTE_56_SQ_TX_ERR_S
, 0);
1917 roce_set_bit(qpc_mask
->byte_56_dqpn_err
, V2_QPC_BYTE_56_SQ_RX_ERR_S
, 0);
1918 roce_set_bit(qpc_mask
->byte_56_dqpn_err
, V2_QPC_BYTE_56_RQ_TX_ERR_S
, 0);
1919 roce_set_bit(qpc_mask
->byte_56_dqpn_err
, V2_QPC_BYTE_56_RQ_RX_ERR_S
, 0);
1921 roce_set_field(qpc_mask
->byte_60_qpst_mapid
, V2_QPC_BYTE_60_MAPID_M
,
1922 V2_QPC_BYTE_60_MAPID_S
, 0);
1924 roce_set_bit(qpc_mask
->byte_60_qpst_mapid
,
1925 V2_QPC_BYTE_60_INNER_MAP_IND_S
, 0);
1926 roce_set_bit(qpc_mask
->byte_60_qpst_mapid
, V2_QPC_BYTE_60_SQ_MAP_IND_S
,
1928 roce_set_bit(qpc_mask
->byte_60_qpst_mapid
, V2_QPC_BYTE_60_RQ_MAP_IND_S
,
1930 roce_set_bit(qpc_mask
->byte_60_qpst_mapid
, V2_QPC_BYTE_60_EXT_MAP_IND_S
,
1932 roce_set_bit(qpc_mask
->byte_60_qpst_mapid
, V2_QPC_BYTE_60_SQ_RLS_IND_S
,
1934 roce_set_bit(qpc_mask
->byte_60_qpst_mapid
, V2_QPC_BYTE_60_SQ_EXT_IND_S
,
1936 roce_set_bit(qpc_mask
->byte_28_at_fl
, V2_QPC_BYTE_28_CNP_TX_FLAG_S
, 0);
1937 roce_set_bit(qpc_mask
->byte_28_at_fl
, V2_QPC_BYTE_28_CE_FLAG_S
, 0);
1939 roce_set_bit(context
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RRE_S
,
1940 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
));
1941 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RRE_S
, 0);
1943 roce_set_bit(context
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RWE_S
,
1944 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
));
1945 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RWE_S
, 0);
1947 roce_set_bit(context
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_ATE_S
,
1948 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_ATOMIC
));
1949 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_ATE_S
, 0);
1951 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RQIE_S
, 0);
1953 roce_set_field(context
->byte_80_rnr_rx_cqn
, V2_QPC_BYTE_80_RX_CQN_M
,
1954 V2_QPC_BYTE_80_RX_CQN_S
, to_hr_cq(ibqp
->recv_cq
)->cqn
);
1955 roce_set_field(qpc_mask
->byte_80_rnr_rx_cqn
, V2_QPC_BYTE_80_RX_CQN_M
,
1956 V2_QPC_BYTE_80_RX_CQN_S
, 0);
1958 roce_set_field(context
->byte_76_srqn_op_en
,
1959 V2_QPC_BYTE_76_SRQN_M
, V2_QPC_BYTE_76_SRQN_S
,
1960 to_hr_srq(ibqp
->srq
)->srqn
);
1961 roce_set_field(qpc_mask
->byte_76_srqn_op_en
,
1962 V2_QPC_BYTE_76_SRQN_M
, V2_QPC_BYTE_76_SRQN_S
, 0);
1963 roce_set_bit(context
->byte_76_srqn_op_en
,
1964 V2_QPC_BYTE_76_SRQ_EN_S
, 1);
1965 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
,
1966 V2_QPC_BYTE_76_SRQ_EN_S
, 0);
1969 roce_set_field(qpc_mask
->byte_84_rq_ci_pi
,
1970 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M
,
1971 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S
, 0);
1972 roce_set_field(qpc_mask
->byte_84_rq_ci_pi
,
1973 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M
,
1974 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S
, 0);
1976 roce_set_field(qpc_mask
->byte_92_srq_info
, V2_QPC_BYTE_92_SRQ_INFO_M
,
1977 V2_QPC_BYTE_92_SRQ_INFO_S
, 0);
1979 roce_set_field(qpc_mask
->byte_96_rx_reqmsn
, V2_QPC_BYTE_96_RX_REQ_MSN_M
,
1980 V2_QPC_BYTE_96_RX_REQ_MSN_S
, 0);
1982 roce_set_field(qpc_mask
->byte_104_rq_sge
,
1983 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M
,
1984 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S
, 0);
1986 roce_set_bit(qpc_mask
->byte_108_rx_reqepsn
,
1987 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S
, 0);
1988 roce_set_field(qpc_mask
->byte_108_rx_reqepsn
,
1989 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M
,
1990 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S
, 0);
1991 roce_set_bit(qpc_mask
->byte_108_rx_reqepsn
,
1992 V2_QPC_BYTE_108_RX_REQ_RNR_S
, 0);
1994 qpc_mask
->rq_rnr_timer
= 0;
1995 qpc_mask
->rx_msg_len
= 0;
1996 qpc_mask
->rx_rkey_pkt_info
= 0;
1997 qpc_mask
->rx_va
= 0;
1999 roce_set_field(qpc_mask
->byte_132_trrl
, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M
,
2000 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S
, 0);
2001 roce_set_field(qpc_mask
->byte_132_trrl
, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M
,
2002 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S
, 0);
2004 roce_set_bit(qpc_mask
->byte_140_raq
, V2_QPC_BYTE_140_RSVD_RAQ_MAP_S
, 0);
2005 roce_set_field(qpc_mask
->byte_140_raq
, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M
,
2006 V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S
, 0);
2007 roce_set_field(qpc_mask
->byte_140_raq
, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M
,
2008 V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S
, 0);
2010 roce_set_field(qpc_mask
->byte_144_raq
,
2011 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M
,
2012 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S
, 0);
2013 roce_set_bit(qpc_mask
->byte_144_raq
, V2_QPC_BYTE_144_RAQ_RTY_INI_IND_S
,
2015 roce_set_field(qpc_mask
->byte_144_raq
, V2_QPC_BYTE_144_RAQ_CREDIT_M
,
2016 V2_QPC_BYTE_144_RAQ_CREDIT_S
, 0);
2017 roce_set_bit(qpc_mask
->byte_144_raq
, V2_QPC_BYTE_144_RESP_RTY_FLG_S
, 0);
2019 roce_set_field(qpc_mask
->byte_148_raq
, V2_QPC_BYTE_148_RQ_MSN_M
,
2020 V2_QPC_BYTE_148_RQ_MSN_S
, 0);
2021 roce_set_field(qpc_mask
->byte_148_raq
, V2_QPC_BYTE_148_RAQ_SYNDROME_M
,
2022 V2_QPC_BYTE_148_RAQ_SYNDROME_S
, 0);
2024 roce_set_field(qpc_mask
->byte_152_raq
, V2_QPC_BYTE_152_RAQ_PSN_M
,
2025 V2_QPC_BYTE_152_RAQ_PSN_S
, 0);
2026 roce_set_field(qpc_mask
->byte_152_raq
,
2027 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M
,
2028 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S
, 0);
2030 roce_set_field(qpc_mask
->byte_156_raq
, V2_QPC_BYTE_156_RAQ_USE_PKTN_M
,
2031 V2_QPC_BYTE_156_RAQ_USE_PKTN_S
, 0);
2033 roce_set_field(qpc_mask
->byte_160_sq_ci_pi
,
2034 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M
,
2035 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S
, 0);
2036 roce_set_field(qpc_mask
->byte_160_sq_ci_pi
,
2037 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M
,
2038 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S
, 0);
2040 roce_set_field(context
->byte_168_irrl_idx
,
2041 V2_QPC_BYTE_168_SQ_SHIFT_BAK_M
,
2042 V2_QPC_BYTE_168_SQ_SHIFT_BAK_S
,
2043 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
2044 roce_set_field(qpc_mask
->byte_168_irrl_idx
,
2045 V2_QPC_BYTE_168_SQ_SHIFT_BAK_M
,
2046 V2_QPC_BYTE_168_SQ_SHIFT_BAK_S
, 0);
2048 roce_set_bit(qpc_mask
->byte_168_irrl_idx
,
2049 V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S
, 0);
2050 roce_set_bit(qpc_mask
->byte_168_irrl_idx
,
2051 V2_QPC_BYTE_168_SQ_INVLD_FLG_S
, 0);
2052 roce_set_field(qpc_mask
->byte_168_irrl_idx
,
2053 V2_QPC_BYTE_168_IRRL_IDX_LSB_M
,
2054 V2_QPC_BYTE_168_IRRL_IDX_LSB_S
, 0);
2056 roce_set_field(context
->byte_172_sq_psn
, V2_QPC_BYTE_172_ACK_REQ_FREQ_M
,
2057 V2_QPC_BYTE_172_ACK_REQ_FREQ_S
, 4);
2058 roce_set_field(qpc_mask
->byte_172_sq_psn
,
2059 V2_QPC_BYTE_172_ACK_REQ_FREQ_M
,
2060 V2_QPC_BYTE_172_ACK_REQ_FREQ_S
, 0);
2062 roce_set_bit(qpc_mask
->byte_172_sq_psn
, V2_QPC_BYTE_172_MSG_RNR_FLG_S
,
2065 roce_set_field(qpc_mask
->byte_176_msg_pktn
,
2066 V2_QPC_BYTE_176_MSG_USE_PKTN_M
,
2067 V2_QPC_BYTE_176_MSG_USE_PKTN_S
, 0);
2068 roce_set_field(qpc_mask
->byte_176_msg_pktn
,
2069 V2_QPC_BYTE_176_IRRL_HEAD_PRE_M
,
2070 V2_QPC_BYTE_176_IRRL_HEAD_PRE_S
, 0);
2072 roce_set_field(qpc_mask
->byte_184_irrl_idx
,
2073 V2_QPC_BYTE_184_IRRL_IDX_MSB_M
,
2074 V2_QPC_BYTE_184_IRRL_IDX_MSB_S
, 0);
2076 qpc_mask
->cur_sge_offset
= 0;
2078 roce_set_field(qpc_mask
->byte_192_ext_sge
,
2079 V2_QPC_BYTE_192_CUR_SGE_IDX_M
,
2080 V2_QPC_BYTE_192_CUR_SGE_IDX_S
, 0);
2081 roce_set_field(qpc_mask
->byte_192_ext_sge
,
2082 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M
,
2083 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S
, 0);
2085 roce_set_field(qpc_mask
->byte_196_sq_psn
, V2_QPC_BYTE_196_IRRL_HEAD_M
,
2086 V2_QPC_BYTE_196_IRRL_HEAD_S
, 0);
2088 roce_set_field(qpc_mask
->byte_200_sq_max
, V2_QPC_BYTE_200_SQ_MAX_IDX_M
,
2089 V2_QPC_BYTE_200_SQ_MAX_IDX_S
, 0);
2090 roce_set_field(qpc_mask
->byte_200_sq_max
,
2091 V2_QPC_BYTE_200_LCL_OPERATED_CNT_M
,
2092 V2_QPC_BYTE_200_LCL_OPERATED_CNT_S
, 0);
2094 roce_set_bit(qpc_mask
->byte_208_irrl
, V2_QPC_BYTE_208_PKT_RNR_FLG_S
, 0);
2095 roce_set_bit(qpc_mask
->byte_208_irrl
, V2_QPC_BYTE_208_PKT_RTY_FLG_S
, 0);
2097 roce_set_field(qpc_mask
->byte_212_lsn
, V2_QPC_BYTE_212_CHECK_FLG_M
,
2098 V2_QPC_BYTE_212_CHECK_FLG_S
, 0);
2100 qpc_mask
->sq_timer
= 0;
2102 roce_set_field(qpc_mask
->byte_220_retry_psn_msn
,
2103 V2_QPC_BYTE_220_RETRY_MSG_MSN_M
,
2104 V2_QPC_BYTE_220_RETRY_MSG_MSN_S
, 0);
2105 roce_set_field(qpc_mask
->byte_232_irrl_sge
,
2106 V2_QPC_BYTE_232_IRRL_SGE_IDX_M
,
2107 V2_QPC_BYTE_232_IRRL_SGE_IDX_S
, 0);
2109 qpc_mask
->irrl_cur_sge_offset
= 0;
2111 roce_set_field(qpc_mask
->byte_240_irrl_tail
,
2112 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M
,
2113 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S
, 0);
2114 roce_set_field(qpc_mask
->byte_240_irrl_tail
,
2115 V2_QPC_BYTE_240_IRRL_TAIL_RD_M
,
2116 V2_QPC_BYTE_240_IRRL_TAIL_RD_S
, 0);
2117 roce_set_field(qpc_mask
->byte_240_irrl_tail
,
2118 V2_QPC_BYTE_240_RX_ACK_MSN_M
,
2119 V2_QPC_BYTE_240_RX_ACK_MSN_S
, 0);
2121 roce_set_field(qpc_mask
->byte_248_ack_psn
, V2_QPC_BYTE_248_IRRL_PSN_M
,
2122 V2_QPC_BYTE_248_IRRL_PSN_S
, 0);
2123 roce_set_bit(qpc_mask
->byte_248_ack_psn
, V2_QPC_BYTE_248_ACK_PSN_ERR_S
,
2125 roce_set_field(qpc_mask
->byte_248_ack_psn
,
2126 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M
,
2127 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S
, 0);
2128 roce_set_bit(qpc_mask
->byte_248_ack_psn
, V2_QPC_BYTE_248_IRRL_PSN_VLD_S
,
2130 roce_set_bit(qpc_mask
->byte_248_ack_psn
,
2131 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S
, 0);
2132 roce_set_bit(qpc_mask
->byte_248_ack_psn
, V2_QPC_BYTE_248_CQ_ERR_IND_S
,
2135 hr_qp
->access_flags
= attr
->qp_access_flags
;
2136 hr_qp
->pkey_index
= attr
->pkey_index
;
2137 roce_set_field(context
->byte_252_err_txcqn
, V2_QPC_BYTE_252_TX_CQN_M
,
2138 V2_QPC_BYTE_252_TX_CQN_S
, to_hr_cq(ibqp
->send_cq
)->cqn
);
2139 roce_set_field(qpc_mask
->byte_252_err_txcqn
, V2_QPC_BYTE_252_TX_CQN_M
,
2140 V2_QPC_BYTE_252_TX_CQN_S
, 0);
2142 roce_set_field(qpc_mask
->byte_252_err_txcqn
, V2_QPC_BYTE_252_ERR_TYPE_M
,
2143 V2_QPC_BYTE_252_ERR_TYPE_S
, 0);
2145 roce_set_field(qpc_mask
->byte_256_sqflush_rqcqe
,
2146 V2_QPC_BYTE_256_RQ_CQE_IDX_M
,
2147 V2_QPC_BYTE_256_RQ_CQE_IDX_S
, 0);
2148 roce_set_field(qpc_mask
->byte_256_sqflush_rqcqe
,
2149 V2_QPC_BYTE_256_SQ_FLUSH_IDX_M
,
2150 V2_QPC_BYTE_256_SQ_FLUSH_IDX_S
, 0);
2153 static void modify_qp_init_to_init(struct ib_qp
*ibqp
,
2154 const struct ib_qp_attr
*attr
, int attr_mask
,
2155 struct hns_roce_v2_qp_context
*context
,
2156 struct hns_roce_v2_qp_context
*qpc_mask
)
2158 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2161 * In v2 engine, software pass context and context mask to hardware
2162 * when modifying qp. If software need modify some fields in context,
2163 * we should set all bits of the relevant fields in context mask to
2164 * 0 at the same time, else set them to 0x1.
2166 roce_set_field(context
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_TST_M
,
2167 V2_QPC_BYTE_4_TST_S
, to_hr_qp_type(hr_qp
->ibqp
.qp_type
));
2168 roce_set_field(qpc_mask
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_TST_M
,
2169 V2_QPC_BYTE_4_TST_S
, 0);
2171 roce_set_field(context
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_SGE_SHIFT_M
,
2172 V2_QPC_BYTE_4_SGE_SHIFT_S
, hr_qp
->sq
.max_gs
> 2 ?
2173 ilog2((unsigned int)hr_qp
->sge
.sge_cnt
) : 0);
2174 roce_set_field(qpc_mask
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_SGE_SHIFT_M
,
2175 V2_QPC_BYTE_4_SGE_SHIFT_S
, 0);
2177 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
2178 roce_set_bit(context
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RRE_S
,
2179 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
));
2180 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RRE_S
,
2183 roce_set_bit(context
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RWE_S
,
2184 !!(attr
->qp_access_flags
&
2185 IB_ACCESS_REMOTE_WRITE
));
2186 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RWE_S
,
2189 roce_set_bit(context
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_ATE_S
,
2190 !!(attr
->qp_access_flags
&
2191 IB_ACCESS_REMOTE_ATOMIC
));
2192 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_ATE_S
,
2195 roce_set_bit(context
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RRE_S
,
2196 !!(hr_qp
->access_flags
& IB_ACCESS_REMOTE_READ
));
2197 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RRE_S
,
2200 roce_set_bit(context
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RWE_S
,
2201 !!(hr_qp
->access_flags
& IB_ACCESS_REMOTE_WRITE
));
2202 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_RWE_S
,
2205 roce_set_bit(context
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_ATE_S
,
2206 !!(hr_qp
->access_flags
& IB_ACCESS_REMOTE_ATOMIC
));
2207 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
, V2_QPC_BYTE_76_ATE_S
,
2211 roce_set_field(context
->byte_20_smac_sgid_idx
,
2212 V2_QPC_BYTE_20_SQ_SHIFT_M
, V2_QPC_BYTE_20_SQ_SHIFT_S
,
2213 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
2214 roce_set_field(qpc_mask
->byte_20_smac_sgid_idx
,
2215 V2_QPC_BYTE_20_SQ_SHIFT_M
, V2_QPC_BYTE_20_SQ_SHIFT_S
, 0);
2217 roce_set_field(context
->byte_20_smac_sgid_idx
,
2218 V2_QPC_BYTE_20_RQ_SHIFT_M
, V2_QPC_BYTE_20_RQ_SHIFT_S
,
2219 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
2220 roce_set_field(qpc_mask
->byte_20_smac_sgid_idx
,
2221 V2_QPC_BYTE_20_RQ_SHIFT_M
, V2_QPC_BYTE_20_RQ_SHIFT_S
, 0);
2223 roce_set_field(context
->byte_16_buf_ba_pg_sz
, V2_QPC_BYTE_16_PD_M
,
2224 V2_QPC_BYTE_16_PD_S
, to_hr_pd(ibqp
->pd
)->pdn
);
2225 roce_set_field(qpc_mask
->byte_16_buf_ba_pg_sz
, V2_QPC_BYTE_16_PD_M
,
2226 V2_QPC_BYTE_16_PD_S
, 0);
2228 roce_set_field(context
->byte_80_rnr_rx_cqn
, V2_QPC_BYTE_80_RX_CQN_M
,
2229 V2_QPC_BYTE_80_RX_CQN_S
, to_hr_cq(ibqp
->recv_cq
)->cqn
);
2230 roce_set_field(qpc_mask
->byte_80_rnr_rx_cqn
, V2_QPC_BYTE_80_RX_CQN_M
,
2231 V2_QPC_BYTE_80_RX_CQN_S
, 0);
2233 roce_set_field(context
->byte_252_err_txcqn
, V2_QPC_BYTE_252_TX_CQN_M
,
2234 V2_QPC_BYTE_252_TX_CQN_S
, to_hr_cq(ibqp
->recv_cq
)->cqn
);
2235 roce_set_field(qpc_mask
->byte_252_err_txcqn
, V2_QPC_BYTE_252_TX_CQN_M
,
2236 V2_QPC_BYTE_252_TX_CQN_S
, 0);
2239 roce_set_bit(context
->byte_76_srqn_op_en
,
2240 V2_QPC_BYTE_76_SRQ_EN_S
, 1);
2241 roce_set_bit(qpc_mask
->byte_76_srqn_op_en
,
2242 V2_QPC_BYTE_76_SRQ_EN_S
, 0);
2243 roce_set_field(context
->byte_76_srqn_op_en
,
2244 V2_QPC_BYTE_76_SRQN_M
, V2_QPC_BYTE_76_SRQN_S
,
2245 to_hr_srq(ibqp
->srq
)->srqn
);
2246 roce_set_field(qpc_mask
->byte_76_srqn_op_en
,
2247 V2_QPC_BYTE_76_SRQN_M
, V2_QPC_BYTE_76_SRQN_S
, 0);
2250 if (attr_mask
& IB_QP_PKEY_INDEX
)
2251 context
->qkey_xrcd
= attr
->pkey_index
;
2253 context
->qkey_xrcd
= hr_qp
->pkey_index
;
2255 roce_set_field(context
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_SQPN_M
,
2256 V2_QPC_BYTE_4_SQPN_S
, hr_qp
->qpn
);
2257 roce_set_field(qpc_mask
->byte_4_sqpn_tst
, V2_QPC_BYTE_4_SQPN_M
,
2258 V2_QPC_BYTE_4_SQPN_S
, 0);
2260 roce_set_field(context
->byte_56_dqpn_err
, V2_QPC_BYTE_56_DQPN_M
,
2261 V2_QPC_BYTE_56_DQPN_S
, hr_qp
->qpn
);
2262 roce_set_field(qpc_mask
->byte_56_dqpn_err
, V2_QPC_BYTE_56_DQPN_M
,
2263 V2_QPC_BYTE_56_DQPN_S
, 0);
2264 roce_set_field(context
->byte_168_irrl_idx
,
2265 V2_QPC_BYTE_168_SQ_SHIFT_BAK_M
,
2266 V2_QPC_BYTE_168_SQ_SHIFT_BAK_S
,
2267 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
2268 roce_set_field(qpc_mask
->byte_168_irrl_idx
,
2269 V2_QPC_BYTE_168_SQ_SHIFT_BAK_M
,
2270 V2_QPC_BYTE_168_SQ_SHIFT_BAK_S
, 0);
2273 static int modify_qp_init_to_rtr(struct ib_qp
*ibqp
,
2274 const struct ib_qp_attr
*attr
, int attr_mask
,
2275 struct hns_roce_v2_qp_context
*context
,
2276 struct hns_roce_v2_qp_context
*qpc_mask
)
2278 const struct ib_global_route
*grh
= rdma_ah_read_grh(&attr
->ah_attr
);
2279 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2280 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2281 struct device
*dev
= hr_dev
->dev
;
2282 dma_addr_t dma_handle_3
;
2283 dma_addr_t dma_handle_2
;
2284 dma_addr_t dma_handle
;
2294 /* Search qp buf's mtts */
2295 mtts
= hns_roce_table_find(hr_dev
, &hr_dev
->mr_table
.mtt_table
,
2296 hr_qp
->mtt
.first_seg
, &dma_handle
);
2298 dev_err(dev
, "qp buf pa find failed\n");
2302 /* Search IRRL's mtts */
2303 mtts_2
= hns_roce_table_find(hr_dev
, &hr_dev
->qp_table
.irrl_table
,
2304 hr_qp
->qpn
, &dma_handle_2
);
2306 dev_err(dev
, "qp irrl_table find failed\n");
2310 /* Search TRRL's mtts */
2311 mtts_3
= hns_roce_table_find(hr_dev
, &hr_dev
->qp_table
.trrl_table
,
2312 hr_qp
->qpn
, &dma_handle_3
);
2314 dev_err(dev
, "qp trrl_table find failed\n");
2318 if ((attr_mask
& IB_QP_ALT_PATH
) || (attr_mask
& IB_QP_ACCESS_FLAGS
) ||
2319 (attr_mask
& IB_QP_PKEY_INDEX
) || (attr_mask
& IB_QP_QKEY
)) {
2320 dev_err(dev
, "INIT2RTR attr_mask (0x%x) error\n", attr_mask
);
2324 dmac
= (u8
*)attr
->ah_attr
.roce
.dmac
;
2325 context
->wqe_sge_ba
= (u32
)(dma_handle
>> 3);
2326 qpc_mask
->wqe_sge_ba
= 0;
2329 * In v2 engine, software pass context and context mask to hardware
2330 * when modifying qp. If software need modify some fields in context,
2331 * we should set all bits of the relevant fields in context mask to
2332 * 0 at the same time, else set them to 0x1.
2334 roce_set_field(context
->byte_12_sq_hop
, V2_QPC_BYTE_12_WQE_SGE_BA_M
,
2335 V2_QPC_BYTE_12_WQE_SGE_BA_S
, dma_handle
>> (32 + 3));
2336 roce_set_field(qpc_mask
->byte_12_sq_hop
, V2_QPC_BYTE_12_WQE_SGE_BA_M
,
2337 V2_QPC_BYTE_12_WQE_SGE_BA_S
, 0);
2339 roce_set_field(context
->byte_12_sq_hop
, V2_QPC_BYTE_12_SQ_HOP_NUM_M
,
2340 V2_QPC_BYTE_12_SQ_HOP_NUM_S
,
2341 hr_dev
->caps
.mtt_hop_num
== HNS_ROCE_HOP_NUM_0
?
2342 0 : hr_dev
->caps
.mtt_hop_num
);
2343 roce_set_field(qpc_mask
->byte_12_sq_hop
, V2_QPC_BYTE_12_SQ_HOP_NUM_M
,
2344 V2_QPC_BYTE_12_SQ_HOP_NUM_S
, 0);
2346 roce_set_field(context
->byte_20_smac_sgid_idx
,
2347 V2_QPC_BYTE_20_SGE_HOP_NUM_M
,
2348 V2_QPC_BYTE_20_SGE_HOP_NUM_S
,
2349 hr_qp
->sq
.max_gs
> 2 ? hr_dev
->caps
.mtt_hop_num
: 0);
2350 roce_set_field(qpc_mask
->byte_20_smac_sgid_idx
,
2351 V2_QPC_BYTE_20_SGE_HOP_NUM_M
,
2352 V2_QPC_BYTE_20_SGE_HOP_NUM_S
, 0);
2354 roce_set_field(context
->byte_20_smac_sgid_idx
,
2355 V2_QPC_BYTE_20_RQ_HOP_NUM_M
,
2356 V2_QPC_BYTE_20_RQ_HOP_NUM_S
,
2357 hr_dev
->caps
.mtt_hop_num
== HNS_ROCE_HOP_NUM_0
?
2358 0 : hr_dev
->caps
.mtt_hop_num
);
2359 roce_set_field(qpc_mask
->byte_20_smac_sgid_idx
,
2360 V2_QPC_BYTE_20_RQ_HOP_NUM_M
,
2361 V2_QPC_BYTE_20_RQ_HOP_NUM_S
, 0);
2363 roce_set_field(context
->byte_16_buf_ba_pg_sz
,
2364 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M
,
2365 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S
,
2366 hr_dev
->caps
.mtt_ba_pg_sz
);
2367 roce_set_field(qpc_mask
->byte_16_buf_ba_pg_sz
,
2368 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M
,
2369 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S
, 0);
2371 roce_set_field(context
->byte_16_buf_ba_pg_sz
,
2372 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M
,
2373 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S
,
2374 hr_dev
->caps
.mtt_buf_pg_sz
);
2375 roce_set_field(qpc_mask
->byte_16_buf_ba_pg_sz
,
2376 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M
,
2377 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S
, 0);
2379 roce_set_field(context
->byte_80_rnr_rx_cqn
,
2380 V2_QPC_BYTE_80_MIN_RNR_TIME_M
,
2381 V2_QPC_BYTE_80_MIN_RNR_TIME_S
, attr
->min_rnr_timer
);
2382 roce_set_field(qpc_mask
->byte_80_rnr_rx_cqn
,
2383 V2_QPC_BYTE_80_MIN_RNR_TIME_M
,
2384 V2_QPC_BYTE_80_MIN_RNR_TIME_S
, 0);
2386 page_size
= 1 << (hr_dev
->caps
.mtt_buf_pg_sz
+ PAGE_SHIFT
);
2387 context
->rq_cur_blk_addr
= (u32
)(mtts
[hr_qp
->rq
.offset
/ page_size
]
2388 >> PAGE_ADDR_SHIFT
);
2389 qpc_mask
->rq_cur_blk_addr
= 0;
2391 roce_set_field(context
->byte_92_srq_info
,
2392 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M
,
2393 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S
,
2394 mtts
[hr_qp
->rq
.offset
/ page_size
]
2395 >> (32 + PAGE_ADDR_SHIFT
));
2396 roce_set_field(qpc_mask
->byte_92_srq_info
,
2397 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M
,
2398 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S
, 0);
2400 context
->rq_nxt_blk_addr
= (u32
)(mtts
[hr_qp
->rq
.offset
/ page_size
+ 1]
2401 >> PAGE_ADDR_SHIFT
);
2402 qpc_mask
->rq_nxt_blk_addr
= 0;
2404 roce_set_field(context
->byte_104_rq_sge
,
2405 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M
,
2406 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S
,
2407 mtts
[hr_qp
->rq
.offset
/ page_size
+ 1]
2408 >> (32 + PAGE_ADDR_SHIFT
));
2409 roce_set_field(qpc_mask
->byte_104_rq_sge
,
2410 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M
,
2411 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S
, 0);
2413 roce_set_field(context
->byte_108_rx_reqepsn
,
2414 V2_QPC_BYTE_108_RX_REQ_EPSN_M
,
2415 V2_QPC_BYTE_108_RX_REQ_EPSN_S
, attr
->rq_psn
);
2416 roce_set_field(qpc_mask
->byte_108_rx_reqepsn
,
2417 V2_QPC_BYTE_108_RX_REQ_EPSN_M
,
2418 V2_QPC_BYTE_108_RX_REQ_EPSN_S
, 0);
2420 roce_set_field(context
->byte_132_trrl
, V2_QPC_BYTE_132_TRRL_BA_M
,
2421 V2_QPC_BYTE_132_TRRL_BA_S
, dma_handle_3
>> 4);
2422 roce_set_field(qpc_mask
->byte_132_trrl
, V2_QPC_BYTE_132_TRRL_BA_M
,
2423 V2_QPC_BYTE_132_TRRL_BA_S
, 0);
2424 context
->trrl_ba
= (u32
)(dma_handle_3
>> (16 + 4));
2425 qpc_mask
->trrl_ba
= 0;
2426 roce_set_field(context
->byte_140_raq
, V2_QPC_BYTE_140_TRRL_BA_M
,
2427 V2_QPC_BYTE_140_TRRL_BA_S
,
2428 (u32
)(dma_handle_3
>> (32 + 16 + 4)));
2429 roce_set_field(qpc_mask
->byte_140_raq
, V2_QPC_BYTE_140_TRRL_BA_M
,
2430 V2_QPC_BYTE_140_TRRL_BA_S
, 0);
2432 context
->irrl_ba
= (u32
)(dma_handle_2
>> 6);
2433 qpc_mask
->irrl_ba
= 0;
2434 roce_set_field(context
->byte_208_irrl
, V2_QPC_BYTE_208_IRRL_BA_M
,
2435 V2_QPC_BYTE_208_IRRL_BA_S
,
2436 dma_handle_2
>> (32 + 6));
2437 roce_set_field(qpc_mask
->byte_208_irrl
, V2_QPC_BYTE_208_IRRL_BA_M
,
2438 V2_QPC_BYTE_208_IRRL_BA_S
, 0);
2440 roce_set_bit(context
->byte_208_irrl
, V2_QPC_BYTE_208_RMT_E2E_S
, 1);
2441 roce_set_bit(qpc_mask
->byte_208_irrl
, V2_QPC_BYTE_208_RMT_E2E_S
, 0);
2443 roce_set_bit(context
->byte_252_err_txcqn
, V2_QPC_BYTE_252_SIG_TYPE_S
,
2444 hr_qp
->sq_signal_bits
);
2445 roce_set_bit(qpc_mask
->byte_252_err_txcqn
, V2_QPC_BYTE_252_SIG_TYPE_S
,
2448 port
= (attr_mask
& IB_QP_PORT
) ? (attr
->port_num
- 1) : hr_qp
->port
;
2450 smac
= (u8
*)hr_dev
->dev_addr
[port
];
2451 /* when dmac equals smac or loop_idc is 1, it should loopback */
2452 if (ether_addr_equal_unaligned(dmac
, smac
) ||
2453 hr_dev
->loop_idc
== 0x1) {
2454 roce_set_bit(context
->byte_28_at_fl
, V2_QPC_BYTE_28_LBI_S
, 1);
2455 roce_set_bit(qpc_mask
->byte_28_at_fl
, V2_QPC_BYTE_28_LBI_S
, 0);
2458 roce_set_field(context
->byte_140_raq
, V2_QPC_BYTE_140_RR_MAX_M
,
2459 V2_QPC_BYTE_140_RR_MAX_S
,
2460 ilog2((unsigned int)attr
->max_dest_rd_atomic
));
2461 roce_set_field(qpc_mask
->byte_140_raq
, V2_QPC_BYTE_140_RR_MAX_M
,
2462 V2_QPC_BYTE_140_RR_MAX_S
, 0);
2464 roce_set_field(context
->byte_56_dqpn_err
, V2_QPC_BYTE_56_DQPN_M
,
2465 V2_QPC_BYTE_56_DQPN_S
, attr
->dest_qp_num
);
2466 roce_set_field(qpc_mask
->byte_56_dqpn_err
, V2_QPC_BYTE_56_DQPN_M
,
2467 V2_QPC_BYTE_56_DQPN_S
, 0);
2469 /* Configure GID index */
2470 port_num
= rdma_ah_get_port_num(&attr
->ah_attr
);
2471 roce_set_field(context
->byte_20_smac_sgid_idx
,
2472 V2_QPC_BYTE_20_SGID_IDX_M
,
2473 V2_QPC_BYTE_20_SGID_IDX_S
,
2474 hns_get_gid_index(hr_dev
, port_num
- 1,
2476 roce_set_field(qpc_mask
->byte_20_smac_sgid_idx
,
2477 V2_QPC_BYTE_20_SGID_IDX_M
,
2478 V2_QPC_BYTE_20_SGID_IDX_S
, 0);
2479 memcpy(&(context
->dmac
), dmac
, 4);
2480 roce_set_field(context
->byte_52_udpspn_dmac
, V2_QPC_BYTE_52_DMAC_M
,
2481 V2_QPC_BYTE_52_DMAC_S
, *((u16
*)(&dmac
[4])));
2483 roce_set_field(qpc_mask
->byte_52_udpspn_dmac
, V2_QPC_BYTE_52_DMAC_M
,
2484 V2_QPC_BYTE_52_DMAC_S
, 0);
2486 roce_set_field(context
->byte_56_dqpn_err
, V2_QPC_BYTE_56_LP_PKTN_INI_M
,
2487 V2_QPC_BYTE_56_LP_PKTN_INI_S
, 4);
2488 roce_set_field(qpc_mask
->byte_56_dqpn_err
, V2_QPC_BYTE_56_LP_PKTN_INI_M
,
2489 V2_QPC_BYTE_56_LP_PKTN_INI_S
, 0);
2491 roce_set_field(context
->byte_24_mtu_tc
, V2_QPC_BYTE_24_HOP_LIMIT_M
,
2492 V2_QPC_BYTE_24_HOP_LIMIT_S
, grh
->hop_limit
);
2493 roce_set_field(qpc_mask
->byte_24_mtu_tc
, V2_QPC_BYTE_24_HOP_LIMIT_M
,
2494 V2_QPC_BYTE_24_HOP_LIMIT_S
, 0);
2496 roce_set_field(context
->byte_28_at_fl
, V2_QPC_BYTE_28_FL_M
,
2497 V2_QPC_BYTE_28_FL_S
, grh
->flow_label
);
2498 roce_set_field(qpc_mask
->byte_28_at_fl
, V2_QPC_BYTE_28_FL_M
,
2499 V2_QPC_BYTE_28_FL_S
, 0);
2501 roce_set_field(context
->byte_24_mtu_tc
, V2_QPC_BYTE_24_TC_M
,
2502 V2_QPC_BYTE_24_TC_S
, grh
->traffic_class
);
2503 roce_set_field(qpc_mask
->byte_24_mtu_tc
, V2_QPC_BYTE_24_TC_M
,
2504 V2_QPC_BYTE_24_TC_S
, 0);
2506 roce_set_field(context
->byte_24_mtu_tc
, V2_QPC_BYTE_24_MTU_M
,
2507 V2_QPC_BYTE_24_MTU_S
, attr
->path_mtu
);
2508 roce_set_field(qpc_mask
->byte_24_mtu_tc
, V2_QPC_BYTE_24_MTU_M
,
2509 V2_QPC_BYTE_24_MTU_S
, 0);
2511 memcpy(context
->dgid
, grh
->dgid
.raw
, sizeof(grh
->dgid
.raw
));
2512 memset(qpc_mask
->dgid
, 0, sizeof(grh
->dgid
.raw
));
2514 roce_set_field(context
->byte_84_rq_ci_pi
,
2515 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M
,
2516 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S
, hr_qp
->rq
.head
);
2517 roce_set_field(qpc_mask
->byte_84_rq_ci_pi
,
2518 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M
,
2519 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S
, 0);
2521 roce_set_field(qpc_mask
->byte_84_rq_ci_pi
,
2522 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M
,
2523 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S
, 0);
2524 roce_set_bit(qpc_mask
->byte_108_rx_reqepsn
,
2525 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S
, 0);
2526 roce_set_field(qpc_mask
->byte_96_rx_reqmsn
, V2_QPC_BYTE_96_RX_REQ_MSN_M
,
2527 V2_QPC_BYTE_96_RX_REQ_MSN_S
, 0);
2528 roce_set_field(qpc_mask
->byte_108_rx_reqepsn
,
2529 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M
,
2530 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S
, 0);
2532 context
->rq_rnr_timer
= 0;
2533 qpc_mask
->rq_rnr_timer
= 0;
2535 roce_set_field(context
->byte_152_raq
, V2_QPC_BYTE_152_RAQ_PSN_M
,
2536 V2_QPC_BYTE_152_RAQ_PSN_S
, attr
->rq_psn
- 1);
2537 roce_set_field(qpc_mask
->byte_152_raq
, V2_QPC_BYTE_152_RAQ_PSN_M
,
2538 V2_QPC_BYTE_152_RAQ_PSN_S
, 0);
2540 roce_set_field(qpc_mask
->byte_132_trrl
, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M
,
2541 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S
, 0);
2542 roce_set_field(qpc_mask
->byte_132_trrl
, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M
,
2543 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S
, 0);
2545 roce_set_field(context
->byte_168_irrl_idx
,
2546 V2_QPC_BYTE_168_LP_SGEN_INI_M
,
2547 V2_QPC_BYTE_168_LP_SGEN_INI_S
, 3);
2548 roce_set_field(qpc_mask
->byte_168_irrl_idx
,
2549 V2_QPC_BYTE_168_LP_SGEN_INI_M
,
2550 V2_QPC_BYTE_168_LP_SGEN_INI_S
, 0);
2552 roce_set_field(context
->byte_208_irrl
, V2_QPC_BYTE_208_SR_MAX_M
,
2553 V2_QPC_BYTE_208_SR_MAX_S
,
2554 ilog2((unsigned int)attr
->max_rd_atomic
));
2555 roce_set_field(qpc_mask
->byte_208_irrl
, V2_QPC_BYTE_208_SR_MAX_M
,
2556 V2_QPC_BYTE_208_SR_MAX_S
, 0);
2558 roce_set_field(context
->byte_28_at_fl
, V2_QPC_BYTE_28_SL_M
,
2559 V2_QPC_BYTE_28_SL_S
, rdma_ah_get_sl(&attr
->ah_attr
));
2560 roce_set_field(qpc_mask
->byte_28_at_fl
, V2_QPC_BYTE_28_SL_M
,
2561 V2_QPC_BYTE_28_SL_S
, 0);
2562 hr_qp
->sl
= rdma_ah_get_sl(&attr
->ah_attr
);
2567 static int modify_qp_rtr_to_rts(struct ib_qp
*ibqp
,
2568 const struct ib_qp_attr
*attr
, int attr_mask
,
2569 struct hns_roce_v2_qp_context
*context
,
2570 struct hns_roce_v2_qp_context
*qpc_mask
)
2572 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2573 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2574 struct device
*dev
= hr_dev
->dev
;
2575 dma_addr_t dma_handle
;
2579 /* Search qp buf's mtts */
2580 mtts
= hns_roce_table_find(hr_dev
, &hr_dev
->mr_table
.mtt_table
,
2581 hr_qp
->mtt
.first_seg
, &dma_handle
);
2583 dev_err(dev
, "qp buf pa find failed\n");
2587 /* If exist optional param, return error */
2588 if ((attr_mask
& IB_QP_ALT_PATH
) || (attr_mask
& IB_QP_ACCESS_FLAGS
) ||
2589 (attr_mask
& IB_QP_QKEY
) || (attr_mask
& IB_QP_PATH_MIG_STATE
) ||
2590 (attr_mask
& IB_QP_CUR_STATE
) ||
2591 (attr_mask
& IB_QP_MIN_RNR_TIMER
)) {
2592 dev_err(dev
, "RTR2RTS attr_mask (0x%x)error\n", attr_mask
);
2597 * In v2 engine, software pass context and context mask to hardware
2598 * when modifying qp. If software need modify some fields in context,
2599 * we should set all bits of the relevant fields in context mask to
2600 * 0 at the same time, else set them to 0x1.
2602 roce_set_field(context
->byte_60_qpst_mapid
,
2603 V2_QPC_BYTE_60_RTY_NUM_INI_BAK_M
,
2604 V2_QPC_BYTE_60_RTY_NUM_INI_BAK_S
, attr
->retry_cnt
);
2605 roce_set_field(qpc_mask
->byte_60_qpst_mapid
,
2606 V2_QPC_BYTE_60_RTY_NUM_INI_BAK_M
,
2607 V2_QPC_BYTE_60_RTY_NUM_INI_BAK_S
, 0);
2609 context
->sq_cur_blk_addr
= (u32
)(mtts
[0] >> PAGE_ADDR_SHIFT
);
2610 roce_set_field(context
->byte_168_irrl_idx
,
2611 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M
,
2612 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S
,
2613 mtts
[0] >> (32 + PAGE_ADDR_SHIFT
));
2614 qpc_mask
->sq_cur_blk_addr
= 0;
2615 roce_set_field(qpc_mask
->byte_168_irrl_idx
,
2616 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M
,
2617 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S
, 0);
2619 page_size
= 1 << (hr_dev
->caps
.mtt_buf_pg_sz
+ PAGE_SHIFT
);
2620 context
->sq_cur_sge_blk_addr
= hr_qp
->sq
.max_gs
> 2 ?
2621 ((u32
)(mtts
[hr_qp
->sge
.offset
/ page_size
]
2622 >> PAGE_ADDR_SHIFT
)) : 0;
2623 roce_set_field(context
->byte_184_irrl_idx
,
2624 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M
,
2625 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S
,
2626 hr_qp
->sq
.max_gs
> 2 ?
2627 (mtts
[hr_qp
->sge
.offset
/ page_size
] >>
2628 (32 + PAGE_ADDR_SHIFT
)) : 0);
2629 qpc_mask
->sq_cur_sge_blk_addr
= 0;
2630 roce_set_field(qpc_mask
->byte_184_irrl_idx
,
2631 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M
,
2632 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S
, 0);
2634 context
->rx_sq_cur_blk_addr
= (u32
)(mtts
[0] >> PAGE_ADDR_SHIFT
);
2635 roce_set_field(context
->byte_232_irrl_sge
,
2636 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M
,
2637 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S
,
2638 mtts
[0] >> (32 + PAGE_ADDR_SHIFT
));
2639 qpc_mask
->rx_sq_cur_blk_addr
= 0;
2640 roce_set_field(qpc_mask
->byte_232_irrl_sge
,
2641 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M
,
2642 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S
, 0);
2645 * Set some fields in context to zero, Because the default values
2646 * of all fields in context are zero, we need not set them to 0 again.
2647 * but we should set the relevant fields of context mask to 0.
2649 roce_set_field(qpc_mask
->byte_232_irrl_sge
,
2650 V2_QPC_BYTE_232_IRRL_SGE_IDX_M
,
2651 V2_QPC_BYTE_232_IRRL_SGE_IDX_S
, 0);
2653 roce_set_field(qpc_mask
->byte_240_irrl_tail
,
2654 V2_QPC_BYTE_240_RX_ACK_MSN_M
,
2655 V2_QPC_BYTE_240_RX_ACK_MSN_S
, 0);
2657 roce_set_field(context
->byte_244_rnr_rxack
,
2658 V2_QPC_BYTE_244_RX_ACK_EPSN_M
,
2659 V2_QPC_BYTE_244_RX_ACK_EPSN_S
, attr
->sq_psn
);
2660 roce_set_field(qpc_mask
->byte_244_rnr_rxack
,
2661 V2_QPC_BYTE_244_RX_ACK_EPSN_M
,
2662 V2_QPC_BYTE_244_RX_ACK_EPSN_S
, 0);
2664 roce_set_field(qpc_mask
->byte_248_ack_psn
,
2665 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M
,
2666 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S
, 0);
2667 roce_set_bit(qpc_mask
->byte_248_ack_psn
,
2668 V2_QPC_BYTE_248_IRRL_PSN_VLD_S
, 0);
2669 roce_set_field(qpc_mask
->byte_248_ack_psn
,
2670 V2_QPC_BYTE_248_IRRL_PSN_M
,
2671 V2_QPC_BYTE_248_IRRL_PSN_S
, 0);
2673 roce_set_field(qpc_mask
->byte_240_irrl_tail
,
2674 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M
,
2675 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S
, 0);
2677 roce_set_field(context
->byte_220_retry_psn_msn
,
2678 V2_QPC_BYTE_220_RETRY_MSG_PSN_M
,
2679 V2_QPC_BYTE_220_RETRY_MSG_PSN_S
, attr
->sq_psn
);
2680 roce_set_field(qpc_mask
->byte_220_retry_psn_msn
,
2681 V2_QPC_BYTE_220_RETRY_MSG_PSN_M
,
2682 V2_QPC_BYTE_220_RETRY_MSG_PSN_S
, 0);
2684 roce_set_field(context
->byte_224_retry_msg
,
2685 V2_QPC_BYTE_224_RETRY_MSG_PSN_M
,
2686 V2_QPC_BYTE_224_RETRY_MSG_PSN_S
, attr
->sq_psn
>> 16);
2687 roce_set_field(qpc_mask
->byte_224_retry_msg
,
2688 V2_QPC_BYTE_224_RETRY_MSG_PSN_M
,
2689 V2_QPC_BYTE_224_RETRY_MSG_PSN_S
, 0);
2691 roce_set_field(context
->byte_224_retry_msg
,
2692 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M
,
2693 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S
, attr
->sq_psn
);
2694 roce_set_field(qpc_mask
->byte_224_retry_msg
,
2695 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M
,
2696 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S
, 0);
2698 roce_set_field(qpc_mask
->byte_220_retry_psn_msn
,
2699 V2_QPC_BYTE_220_RETRY_MSG_MSN_M
,
2700 V2_QPC_BYTE_220_RETRY_MSG_MSN_S
, 0);
2702 roce_set_bit(qpc_mask
->byte_248_ack_psn
,
2703 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S
, 0);
2705 roce_set_field(qpc_mask
->byte_212_lsn
, V2_QPC_BYTE_212_CHECK_FLG_M
,
2706 V2_QPC_BYTE_212_CHECK_FLG_S
, 0);
2708 roce_set_field(context
->byte_212_lsn
, V2_QPC_BYTE_212_RETRY_CNT_M
,
2709 V2_QPC_BYTE_212_RETRY_CNT_S
, attr
->retry_cnt
);
2710 roce_set_field(qpc_mask
->byte_212_lsn
, V2_QPC_BYTE_212_RETRY_CNT_M
,
2711 V2_QPC_BYTE_212_RETRY_CNT_S
, 0);
2713 roce_set_field(context
->byte_212_lsn
, V2_QPC_BYTE_212_RETRY_NUM_INIT_M
,
2714 V2_QPC_BYTE_212_RETRY_NUM_INIT_S
, attr
->retry_cnt
);
2715 roce_set_field(qpc_mask
->byte_212_lsn
, V2_QPC_BYTE_212_RETRY_NUM_INIT_M
,
2716 V2_QPC_BYTE_212_RETRY_NUM_INIT_S
, 0);
2718 roce_set_field(context
->byte_244_rnr_rxack
,
2719 V2_QPC_BYTE_244_RNR_NUM_INIT_M
,
2720 V2_QPC_BYTE_244_RNR_NUM_INIT_S
, attr
->rnr_retry
);
2721 roce_set_field(qpc_mask
->byte_244_rnr_rxack
,
2722 V2_QPC_BYTE_244_RNR_NUM_INIT_M
,
2723 V2_QPC_BYTE_244_RNR_NUM_INIT_S
, 0);
2725 roce_set_field(context
->byte_244_rnr_rxack
, V2_QPC_BYTE_244_RNR_CNT_M
,
2726 V2_QPC_BYTE_244_RNR_CNT_S
, attr
->rnr_retry
);
2727 roce_set_field(qpc_mask
->byte_244_rnr_rxack
, V2_QPC_BYTE_244_RNR_CNT_M
,
2728 V2_QPC_BYTE_244_RNR_CNT_S
, 0);
2730 roce_set_field(context
->byte_212_lsn
, V2_QPC_BYTE_212_LSN_M
,
2731 V2_QPC_BYTE_212_LSN_S
, 0x100);
2732 roce_set_field(qpc_mask
->byte_212_lsn
, V2_QPC_BYTE_212_LSN_M
,
2733 V2_QPC_BYTE_212_LSN_S
, 0);
2735 if (attr_mask
& IB_QP_TIMEOUT
) {
2736 roce_set_field(context
->byte_28_at_fl
, V2_QPC_BYTE_28_AT_M
,
2737 V2_QPC_BYTE_28_AT_S
, attr
->timeout
);
2738 roce_set_field(qpc_mask
->byte_28_at_fl
, V2_QPC_BYTE_28_AT_M
,
2739 V2_QPC_BYTE_28_AT_S
, 0);
2742 roce_set_field(context
->byte_28_at_fl
, V2_QPC_BYTE_28_SL_M
,
2743 V2_QPC_BYTE_28_SL_S
,
2744 rdma_ah_get_sl(&attr
->ah_attr
));
2745 roce_set_field(qpc_mask
->byte_28_at_fl
, V2_QPC_BYTE_28_SL_M
,
2746 V2_QPC_BYTE_28_SL_S
, 0);
2747 hr_qp
->sl
= rdma_ah_get_sl(&attr
->ah_attr
);
2749 roce_set_field(context
->byte_172_sq_psn
, V2_QPC_BYTE_172_SQ_CUR_PSN_M
,
2750 V2_QPC_BYTE_172_SQ_CUR_PSN_S
, attr
->sq_psn
);
2751 roce_set_field(qpc_mask
->byte_172_sq_psn
, V2_QPC_BYTE_172_SQ_CUR_PSN_M
,
2752 V2_QPC_BYTE_172_SQ_CUR_PSN_S
, 0);
2754 roce_set_field(qpc_mask
->byte_196_sq_psn
, V2_QPC_BYTE_196_IRRL_HEAD_M
,
2755 V2_QPC_BYTE_196_IRRL_HEAD_S
, 0);
2756 roce_set_field(context
->byte_196_sq_psn
, V2_QPC_BYTE_196_SQ_MAX_PSN_M
,
2757 V2_QPC_BYTE_196_SQ_MAX_PSN_S
, attr
->sq_psn
);
2758 roce_set_field(qpc_mask
->byte_196_sq_psn
, V2_QPC_BYTE_196_SQ_MAX_PSN_M
,
2759 V2_QPC_BYTE_196_SQ_MAX_PSN_S
, 0);
2764 static int hns_roce_v2_modify_qp(struct ib_qp
*ibqp
,
2765 const struct ib_qp_attr
*attr
,
2766 int attr_mask
, enum ib_qp_state cur_state
,
2767 enum ib_qp_state new_state
)
2769 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2770 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2771 struct hns_roce_v2_qp_context
*context
;
2772 struct hns_roce_v2_qp_context
*qpc_mask
;
2773 struct device
*dev
= hr_dev
->dev
;
2776 context
= kzalloc(2 * sizeof(*context
), GFP_KERNEL
);
2780 qpc_mask
= context
+ 1;
2782 * In v2 engine, software pass context and context mask to hardware
2783 * when modifying qp. If software need modify some fields in context,
2784 * we should set all bits of the relevant fields in context mask to
2785 * 0 at the same time, else set them to 0x1.
2787 memset(qpc_mask
, 0xff, sizeof(*qpc_mask
));
2788 if (cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
) {
2789 modify_qp_reset_to_init(ibqp
, attr
, context
, qpc_mask
);
2790 } else if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_INIT
) {
2791 modify_qp_init_to_init(ibqp
, attr
, attr_mask
, context
,
2793 } else if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_RTR
) {
2794 ret
= modify_qp_init_to_rtr(ibqp
, attr
, attr_mask
, context
,
2798 } else if (cur_state
== IB_QPS_RTR
&& new_state
== IB_QPS_RTS
) {
2799 ret
= modify_qp_rtr_to_rts(ibqp
, attr
, attr_mask
, context
,
2803 } else if ((cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_RTS
) ||
2804 (cur_state
== IB_QPS_SQE
&& new_state
== IB_QPS_RTS
) ||
2805 (cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_SQD
) ||
2806 (cur_state
== IB_QPS_SQD
&& new_state
== IB_QPS_SQD
) ||
2807 (cur_state
== IB_QPS_SQD
&& new_state
== IB_QPS_RTS
) ||
2808 (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_RESET
) ||
2809 (cur_state
== IB_QPS_RTR
&& new_state
== IB_QPS_RESET
) ||
2810 (cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_RESET
) ||
2811 (cur_state
== IB_QPS_ERR
&& new_state
== IB_QPS_RESET
) ||
2812 (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_ERR
) ||
2813 (cur_state
== IB_QPS_RTR
&& new_state
== IB_QPS_ERR
) ||
2814 (cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_ERR
) ||
2815 (cur_state
== IB_QPS_SQD
&& new_state
== IB_QPS_ERR
) ||
2816 (cur_state
== IB_QPS_SQE
&& new_state
== IB_QPS_ERR
)) {
2820 dev_err(dev
, "Illegal state for QP!\n");
2824 /* Every status migrate must change state */
2825 roce_set_field(context
->byte_60_qpst_mapid
, V2_QPC_BYTE_60_QP_ST_M
,
2826 V2_QPC_BYTE_60_QP_ST_S
, new_state
);
2827 roce_set_field(qpc_mask
->byte_60_qpst_mapid
, V2_QPC_BYTE_60_QP_ST_M
,
2828 V2_QPC_BYTE_60_QP_ST_S
, 0);
2830 /* SW pass context to HW */
2831 ret
= hns_roce_v2_qp_modify(hr_dev
, &hr_qp
->mtt
, cur_state
, new_state
,
2834 dev_err(dev
, "hns_roce_qp_modify failed(%d)\n", ret
);
2838 hr_qp
->state
= new_state
;
2840 if (attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
)
2841 hr_qp
->resp_depth
= attr
->max_dest_rd_atomic
;
2842 if (attr_mask
& IB_QP_PORT
) {
2843 hr_qp
->port
= attr
->port_num
- 1;
2844 hr_qp
->phy_port
= hr_dev
->iboe
.phy_port
[hr_qp
->port
];
2847 if (new_state
== IB_QPS_RESET
&& !ibqp
->uobject
) {
2848 hns_roce_v2_cq_clean(to_hr_cq(ibqp
->recv_cq
), hr_qp
->qpn
,
2849 ibqp
->srq
? to_hr_srq(ibqp
->srq
) : NULL
);
2850 if (ibqp
->send_cq
!= ibqp
->recv_cq
)
2851 hns_roce_v2_cq_clean(to_hr_cq(ibqp
->send_cq
),
2858 hr_qp
->sq_next_wqe
= 0;
2859 hr_qp
->next_sge
= 0;
2867 static inline enum ib_qp_state
to_ib_qp_st(enum hns_roce_v2_qp_state state
)
2870 case HNS_ROCE_QP_ST_RST
: return IB_QPS_RESET
;
2871 case HNS_ROCE_QP_ST_INIT
: return IB_QPS_INIT
;
2872 case HNS_ROCE_QP_ST_RTR
: return IB_QPS_RTR
;
2873 case HNS_ROCE_QP_ST_RTS
: return IB_QPS_RTS
;
2874 case HNS_ROCE_QP_ST_SQ_DRAINING
:
2875 case HNS_ROCE_QP_ST_SQD
: return IB_QPS_SQD
;
2876 case HNS_ROCE_QP_ST_SQER
: return IB_QPS_SQE
;
2877 case HNS_ROCE_QP_ST_ERR
: return IB_QPS_ERR
;
2882 static int hns_roce_v2_query_qpc(struct hns_roce_dev
*hr_dev
,
2883 struct hns_roce_qp
*hr_qp
,
2884 struct hns_roce_v2_qp_context
*hr_context
)
2886 struct hns_roce_cmd_mailbox
*mailbox
;
2889 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
2890 if (IS_ERR(mailbox
))
2891 return PTR_ERR(mailbox
);
2893 ret
= hns_roce_cmd_mbox(hr_dev
, 0, mailbox
->dma
, hr_qp
->qpn
, 0,
2894 HNS_ROCE_CMD_QUERY_QPC
,
2895 HNS_ROCE_CMD_TIMEOUT_MSECS
);
2897 dev_err(hr_dev
->dev
, "QUERY QP cmd process error\n");
2901 memcpy(hr_context
, mailbox
->buf
, sizeof(*hr_context
));
2904 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
2908 static int hns_roce_v2_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
,
2910 struct ib_qp_init_attr
*qp_init_attr
)
2912 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2913 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2914 struct hns_roce_v2_qp_context
*context
;
2915 struct device
*dev
= hr_dev
->dev
;
2920 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
2924 memset(qp_attr
, 0, sizeof(*qp_attr
));
2925 memset(qp_init_attr
, 0, sizeof(*qp_init_attr
));
2927 mutex_lock(&hr_qp
->mutex
);
2929 if (hr_qp
->state
== IB_QPS_RESET
) {
2930 qp_attr
->qp_state
= IB_QPS_RESET
;
2935 ret
= hns_roce_v2_query_qpc(hr_dev
, hr_qp
, context
);
2937 dev_err(dev
, "query qpc error\n");
2942 state
= roce_get_field(context
->byte_60_qpst_mapid
,
2943 V2_QPC_BYTE_60_QP_ST_M
, V2_QPC_BYTE_60_QP_ST_S
);
2944 tmp_qp_state
= to_ib_qp_st((enum hns_roce_v2_qp_state
)state
);
2945 if (tmp_qp_state
== -1) {
2946 dev_err(dev
, "Illegal ib_qp_state\n");
2950 hr_qp
->state
= (u8
)tmp_qp_state
;
2951 qp_attr
->qp_state
= (enum ib_qp_state
)hr_qp
->state
;
2952 qp_attr
->path_mtu
= (enum ib_mtu
)roce_get_field(context
->byte_24_mtu_tc
,
2953 V2_QPC_BYTE_24_MTU_M
,
2954 V2_QPC_BYTE_24_MTU_S
);
2955 qp_attr
->path_mig_state
= IB_MIG_ARMED
;
2956 qp_attr
->ah_attr
.type
= RDMA_AH_ATTR_TYPE_ROCE
;
2957 if (hr_qp
->ibqp
.qp_type
== IB_QPT_UD
)
2958 qp_attr
->qkey
= V2_QKEY_VAL
;
2960 qp_attr
->rq_psn
= roce_get_field(context
->byte_108_rx_reqepsn
,
2961 V2_QPC_BYTE_108_RX_REQ_EPSN_M
,
2962 V2_QPC_BYTE_108_RX_REQ_EPSN_S
);
2963 qp_attr
->sq_psn
= (u32
)roce_get_field(context
->byte_172_sq_psn
,
2964 V2_QPC_BYTE_172_SQ_CUR_PSN_M
,
2965 V2_QPC_BYTE_172_SQ_CUR_PSN_S
);
2966 qp_attr
->dest_qp_num
= (u8
)roce_get_field(context
->byte_56_dqpn_err
,
2967 V2_QPC_BYTE_56_DQPN_M
,
2968 V2_QPC_BYTE_56_DQPN_S
);
2969 qp_attr
->qp_access_flags
= ((roce_get_bit(context
->byte_76_srqn_op_en
,
2970 V2_QPC_BYTE_76_RRE_S
)) << 2) |
2971 ((roce_get_bit(context
->byte_76_srqn_op_en
,
2972 V2_QPC_BYTE_76_RWE_S
)) << 1) |
2973 ((roce_get_bit(context
->byte_76_srqn_op_en
,
2974 V2_QPC_BYTE_76_ATE_S
)) << 3);
2975 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
||
2976 hr_qp
->ibqp
.qp_type
== IB_QPT_UC
) {
2977 struct ib_global_route
*grh
=
2978 rdma_ah_retrieve_grh(&qp_attr
->ah_attr
);
2980 rdma_ah_set_sl(&qp_attr
->ah_attr
,
2981 roce_get_field(context
->byte_28_at_fl
,
2982 V2_QPC_BYTE_28_SL_M
,
2983 V2_QPC_BYTE_28_SL_S
));
2984 grh
->flow_label
= roce_get_field(context
->byte_28_at_fl
,
2985 V2_QPC_BYTE_28_FL_M
,
2986 V2_QPC_BYTE_28_FL_S
);
2987 grh
->sgid_index
= roce_get_field(context
->byte_20_smac_sgid_idx
,
2988 V2_QPC_BYTE_20_SGID_IDX_M
,
2989 V2_QPC_BYTE_20_SGID_IDX_S
);
2990 grh
->hop_limit
= roce_get_field(context
->byte_24_mtu_tc
,
2991 V2_QPC_BYTE_24_HOP_LIMIT_M
,
2992 V2_QPC_BYTE_24_HOP_LIMIT_S
);
2993 grh
->traffic_class
= roce_get_field(context
->byte_24_mtu_tc
,
2994 V2_QPC_BYTE_24_TC_M
,
2995 V2_QPC_BYTE_24_TC_S
);
2997 memcpy(grh
->dgid
.raw
, context
->dgid
, sizeof(grh
->dgid
.raw
));
3000 qp_attr
->port_num
= hr_qp
->port
+ 1;
3001 qp_attr
->sq_draining
= 0;
3002 qp_attr
->max_rd_atomic
= 1 << roce_get_field(context
->byte_208_irrl
,
3003 V2_QPC_BYTE_208_SR_MAX_M
,
3004 V2_QPC_BYTE_208_SR_MAX_S
);
3005 qp_attr
->max_dest_rd_atomic
= 1 << roce_get_field(context
->byte_140_raq
,
3006 V2_QPC_BYTE_140_RR_MAX_M
,
3007 V2_QPC_BYTE_140_RR_MAX_S
);
3008 qp_attr
->min_rnr_timer
= (u8
)roce_get_field(context
->byte_80_rnr_rx_cqn
,
3009 V2_QPC_BYTE_80_MIN_RNR_TIME_M
,
3010 V2_QPC_BYTE_80_MIN_RNR_TIME_S
);
3011 qp_attr
->timeout
= (u8
)roce_get_field(context
->byte_28_at_fl
,
3012 V2_QPC_BYTE_28_AT_M
,
3013 V2_QPC_BYTE_28_AT_S
);
3014 qp_attr
->retry_cnt
= roce_get_field(context
->byte_212_lsn
,
3015 V2_QPC_BYTE_212_RETRY_CNT_M
,
3016 V2_QPC_BYTE_212_RETRY_CNT_S
);
3017 qp_attr
->rnr_retry
= context
->rq_rnr_timer
;
3020 qp_attr
->cur_qp_state
= qp_attr
->qp_state
;
3021 qp_attr
->cap
.max_recv_wr
= hr_qp
->rq
.wqe_cnt
;
3022 qp_attr
->cap
.max_recv_sge
= hr_qp
->rq
.max_gs
;
3024 if (!ibqp
->uobject
) {
3025 qp_attr
->cap
.max_send_wr
= hr_qp
->sq
.wqe_cnt
;
3026 qp_attr
->cap
.max_send_sge
= hr_qp
->sq
.max_gs
;
3028 qp_attr
->cap
.max_send_wr
= 0;
3029 qp_attr
->cap
.max_send_sge
= 0;
3032 qp_init_attr
->cap
= qp_attr
->cap
;
3035 mutex_unlock(&hr_qp
->mutex
);
3040 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev
*hr_dev
,
3041 struct hns_roce_qp
*hr_qp
,
3044 struct hns_roce_cq
*send_cq
, *recv_cq
;
3045 struct device
*dev
= hr_dev
->dev
;
3048 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
&& hr_qp
->state
!= IB_QPS_RESET
) {
3049 /* Modify qp to reset before destroying qp */
3050 ret
= hns_roce_v2_modify_qp(&hr_qp
->ibqp
, NULL
, 0,
3051 hr_qp
->state
, IB_QPS_RESET
);
3053 dev_err(dev
, "modify QP %06lx to ERR failed.\n",
3059 send_cq
= to_hr_cq(hr_qp
->ibqp
.send_cq
);
3060 recv_cq
= to_hr_cq(hr_qp
->ibqp
.recv_cq
);
3062 hns_roce_lock_cqs(send_cq
, recv_cq
);
3065 __hns_roce_v2_cq_clean(recv_cq
, hr_qp
->qpn
, hr_qp
->ibqp
.srq
?
3066 to_hr_srq(hr_qp
->ibqp
.srq
) : NULL
);
3067 if (send_cq
!= recv_cq
)
3068 __hns_roce_v2_cq_clean(send_cq
, hr_qp
->qpn
, NULL
);
3071 hns_roce_qp_remove(hr_dev
, hr_qp
);
3073 hns_roce_unlock_cqs(send_cq
, recv_cq
);
3075 hns_roce_qp_free(hr_dev
, hr_qp
);
3077 /* Not special_QP, free their QPN */
3078 if ((hr_qp
->ibqp
.qp_type
== IB_QPT_RC
) ||
3079 (hr_qp
->ibqp
.qp_type
== IB_QPT_UC
) ||
3080 (hr_qp
->ibqp
.qp_type
== IB_QPT_UD
))
3081 hns_roce_release_range_qp(hr_dev
, hr_qp
->qpn
, 1);
3083 hns_roce_mtt_cleanup(hr_dev
, &hr_qp
->mtt
);
3086 ib_umem_release(hr_qp
->umem
);
3088 kfree(hr_qp
->sq
.wrid
);
3089 kfree(hr_qp
->rq
.wrid
);
3090 hns_roce_buf_free(hr_dev
, hr_qp
->buff_size
, &hr_qp
->hr_buf
);
3096 static int hns_roce_v2_destroy_qp(struct ib_qp
*ibqp
)
3098 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
3099 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
3102 ret
= hns_roce_v2_destroy_qp_common(hr_dev
, hr_qp
, !!ibqp
->pd
->uobject
);
3104 dev_err(hr_dev
->dev
, "Destroy qp failed(%d)\n", ret
);
3108 if (hr_qp
->ibqp
.qp_type
== IB_QPT_GSI
)
3109 kfree(hr_to_hr_sqp(hr_qp
));
3116 static int hns_roce_v2_modify_cq(struct ib_cq
*cq
, u16 cq_count
, u16 cq_period
)
3118 struct hns_roce_dev
*hr_dev
= to_hr_dev(cq
->device
);
3119 struct hns_roce_v2_cq_context
*cq_context
;
3120 struct hns_roce_cq
*hr_cq
= to_hr_cq(cq
);
3121 struct hns_roce_v2_cq_context
*cqc_mask
;
3122 struct hns_roce_cmd_mailbox
*mailbox
;
3125 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
3126 if (IS_ERR(mailbox
))
3127 return PTR_ERR(mailbox
);
3129 cq_context
= mailbox
->buf
;
3130 cqc_mask
= (struct hns_roce_v2_cq_context
*)mailbox
->buf
+ 1;
3132 memset(cqc_mask
, 0xff, sizeof(*cqc_mask
));
3134 roce_set_field(cq_context
->byte_56_cqe_period_maxcnt
,
3135 V2_CQC_BYTE_56_CQ_MAX_CNT_M
, V2_CQC_BYTE_56_CQ_MAX_CNT_S
,
3137 roce_set_field(cqc_mask
->byte_56_cqe_period_maxcnt
,
3138 V2_CQC_BYTE_56_CQ_MAX_CNT_M
, V2_CQC_BYTE_56_CQ_MAX_CNT_S
,
3140 roce_set_field(cq_context
->byte_56_cqe_period_maxcnt
,
3141 V2_CQC_BYTE_56_CQ_PERIOD_M
, V2_CQC_BYTE_56_CQ_PERIOD_S
,
3143 roce_set_field(cqc_mask
->byte_56_cqe_period_maxcnt
,
3144 V2_CQC_BYTE_56_CQ_PERIOD_M
, V2_CQC_BYTE_56_CQ_PERIOD_S
,
3147 ret
= hns_roce_cmd_mbox(hr_dev
, mailbox
->dma
, 0, hr_cq
->cqn
, 1,
3148 HNS_ROCE_CMD_MODIFY_CQC
,
3149 HNS_ROCE_CMD_TIMEOUT_MSECS
);
3150 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
3152 dev_err(hr_dev
->dev
, "MODIFY CQ Failed to cmd mailbox.\n");
3157 static const struct hns_roce_hw hns_roce_hw_v2
= {
3158 .cmq_init
= hns_roce_v2_cmq_init
,
3159 .cmq_exit
= hns_roce_v2_cmq_exit
,
3160 .hw_profile
= hns_roce_v2_profile
,
3161 .post_mbox
= hns_roce_v2_post_mbox
,
3162 .chk_mbox
= hns_roce_v2_chk_mbox
,
3163 .set_gid
= hns_roce_v2_set_gid
,
3164 .set_mac
= hns_roce_v2_set_mac
,
3165 .write_mtpt
= hns_roce_v2_write_mtpt
,
3166 .rereg_write_mtpt
= hns_roce_v2_rereg_write_mtpt
,
3167 .write_cqc
= hns_roce_v2_write_cqc
,
3168 .set_hem
= hns_roce_v2_set_hem
,
3169 .clear_hem
= hns_roce_v2_clear_hem
,
3170 .modify_qp
= hns_roce_v2_modify_qp
,
3171 .query_qp
= hns_roce_v2_query_qp
,
3172 .destroy_qp
= hns_roce_v2_destroy_qp
,
3173 .modify_cq
= hns_roce_v2_modify_cq
,
3174 .post_send
= hns_roce_v2_post_send
,
3175 .post_recv
= hns_roce_v2_post_recv
,
3176 .req_notify_cq
= hns_roce_v2_req_notify_cq
,
3177 .poll_cq
= hns_roce_v2_poll_cq
,
3180 static const struct pci_device_id hns_roce_hw_v2_pci_tbl
[] = {
3181 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_25GE_RDMA
), 0},
3182 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_25GE_RDMA_MACSEC
), 0},
3183 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_100G_RDMA_MACSEC
), 0},
3184 /* required last entry */
3188 static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev
*hr_dev
,
3189 struct hnae3_handle
*handle
)
3191 const struct pci_device_id
*id
;
3193 id
= pci_match_id(hns_roce_hw_v2_pci_tbl
, hr_dev
->pci_dev
);
3195 dev_err(hr_dev
->dev
, "device is not compatible!\n");
3199 hr_dev
->hw
= &hns_roce_hw_v2
;
3200 hr_dev
->sdb_offset
= ROCEE_DB_SQ_L_0_REG
;
3201 hr_dev
->odb_offset
= hr_dev
->sdb_offset
;
3203 /* Get info from NIC driver. */
3204 hr_dev
->reg_base
= handle
->rinfo
.roce_io_base
;
3205 hr_dev
->caps
.num_ports
= 1;
3206 hr_dev
->iboe
.netdevs
[0] = handle
->rinfo
.netdev
;
3207 hr_dev
->iboe
.phy_port
[0] = 0;
3209 /* cmd issue mode: 0 is poll, 1 is event */
3210 hr_dev
->cmd_mod
= 0;
3211 hr_dev
->loop_idc
= 0;
3216 static int hns_roce_hw_v2_init_instance(struct hnae3_handle
*handle
)
3218 struct hns_roce_dev
*hr_dev
;
3221 hr_dev
= (struct hns_roce_dev
*)ib_alloc_device(sizeof(*hr_dev
));
3225 hr_dev
->priv
= kzalloc(sizeof(struct hns_roce_v2_priv
), GFP_KERNEL
);
3226 if (!hr_dev
->priv
) {
3228 goto error_failed_kzalloc
;
3231 hr_dev
->pci_dev
= handle
->pdev
;
3232 hr_dev
->dev
= &handle
->pdev
->dev
;
3233 handle
->priv
= hr_dev
;
3235 ret
= hns_roce_hw_v2_get_cfg(hr_dev
, handle
);
3237 dev_err(hr_dev
->dev
, "Get Configuration failed!\n");
3238 goto error_failed_get_cfg
;
3241 ret
= hns_roce_init(hr_dev
);
3243 dev_err(hr_dev
->dev
, "RoCE Engine init failed!\n");
3244 goto error_failed_get_cfg
;
3249 error_failed_get_cfg
:
3250 kfree(hr_dev
->priv
);
3252 error_failed_kzalloc
:
3253 ib_dealloc_device(&hr_dev
->ib_dev
);
3258 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle
*handle
,
3261 struct hns_roce_dev
*hr_dev
= (struct hns_roce_dev
*)handle
->priv
;
3263 hns_roce_exit(hr_dev
);
3264 kfree(hr_dev
->priv
);
3265 ib_dealloc_device(&hr_dev
->ib_dev
);
3268 static const struct hnae3_client_ops hns_roce_hw_v2_ops
= {
3269 .init_instance
= hns_roce_hw_v2_init_instance
,
3270 .uninit_instance
= hns_roce_hw_v2_uninit_instance
,
3273 static struct hnae3_client hns_roce_hw_v2_client
= {
3274 .name
= "hns_roce_hw_v2",
3275 .type
= HNAE3_CLIENT_ROCE
,
3276 .ops
= &hns_roce_hw_v2_ops
,
3279 static int __init
hns_roce_hw_v2_init(void)
3281 return hnae3_register_client(&hns_roce_hw_v2_client
);
3284 static void __exit
hns_roce_hw_v2_exit(void)
3286 hnae3_unregister_client(&hns_roce_hw_v2_client
);
3289 module_init(hns_roce_hw_v2_init
);
3290 module_exit(hns_roce_hw_v2_exit
);
3292 MODULE_LICENSE("Dual BSD/GPL");
3293 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
3294 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
3295 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
3296 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");