2 * Copyright (c) 2016 Hisilicon Limited.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/platform_device.h>
34 #include <linux/acpi.h>
35 #include <linux/etherdevice.h>
37 #include <rdma/ib_umem.h>
38 #include "hns_roce_common.h"
39 #include "hns_roce_device.h"
40 #include "hns_roce_cmd.h"
41 #include "hns_roce_hem.h"
42 #include "hns_roce_hw_v1.h"
44 static void set_data_seg(struct hns_roce_wqe_data_seg
*dseg
, struct ib_sge
*sg
)
46 dseg
->lkey
= cpu_to_le32(sg
->lkey
);
47 dseg
->addr
= cpu_to_le64(sg
->addr
);
48 dseg
->len
= cpu_to_le32(sg
->length
);
51 static void set_raddr_seg(struct hns_roce_wqe_raddr_seg
*rseg
, u64 remote_addr
,
54 rseg
->raddr
= cpu_to_le64(remote_addr
);
55 rseg
->rkey
= cpu_to_le32(rkey
);
59 int hns_roce_v1_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
60 struct ib_send_wr
**bad_wr
)
62 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
63 struct hns_roce_ah
*ah
= to_hr_ah(ud_wr(wr
)->ah
);
64 struct hns_roce_ud_send_wqe
*ud_sq_wqe
= NULL
;
65 struct hns_roce_wqe_ctrl_seg
*ctrl
= NULL
;
66 struct hns_roce_wqe_data_seg
*dseg
= NULL
;
67 struct hns_roce_qp
*qp
= to_hr_qp(ibqp
);
68 struct device
*dev
= &hr_dev
->pdev
->dev
;
69 struct hns_roce_sq_db sq_db
;
70 int ps_opcode
= 0, i
= 0;
71 unsigned long flags
= 0;
80 if (unlikely(ibqp
->qp_type
!= IB_QPT_GSI
&&
81 ibqp
->qp_type
!= IB_QPT_RC
)) {
82 dev_err(dev
, "un-supported QP type\n");
87 spin_lock_irqsave(&qp
->sq
.lock
, flags
);
88 ind
= qp
->sq_next_wqe
;
89 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
90 if (hns_roce_wq_overflow(&qp
->sq
, nreq
, qp
->ibqp
.send_cq
)) {
96 if (unlikely(wr
->num_sge
> qp
->sq
.max_gs
)) {
97 dev_err(dev
, "num_sge=%d > qp->sq.max_gs=%d\n",
98 wr
->num_sge
, qp
->sq
.max_gs
);
104 wqe
= get_send_wqe(qp
, ind
& (qp
->sq
.wqe_cnt
- 1));
105 qp
->sq
.wrid
[(qp
->sq
.head
+ nreq
) & (qp
->sq
.wqe_cnt
- 1)] =
108 /* Corresponding to the RC and RD type wqe process separately */
109 if (ibqp
->qp_type
== IB_QPT_GSI
) {
111 roce_set_field(ud_sq_wqe
->dmac_h
,
112 UD_SEND_WQE_U32_4_DMAC_0_M
,
113 UD_SEND_WQE_U32_4_DMAC_0_S
,
115 roce_set_field(ud_sq_wqe
->dmac_h
,
116 UD_SEND_WQE_U32_4_DMAC_1_M
,
117 UD_SEND_WQE_U32_4_DMAC_1_S
,
119 roce_set_field(ud_sq_wqe
->dmac_h
,
120 UD_SEND_WQE_U32_4_DMAC_2_M
,
121 UD_SEND_WQE_U32_4_DMAC_2_S
,
123 roce_set_field(ud_sq_wqe
->dmac_h
,
124 UD_SEND_WQE_U32_4_DMAC_3_M
,
125 UD_SEND_WQE_U32_4_DMAC_3_S
,
128 roce_set_field(ud_sq_wqe
->u32_8
,
129 UD_SEND_WQE_U32_8_DMAC_4_M
,
130 UD_SEND_WQE_U32_8_DMAC_4_S
,
132 roce_set_field(ud_sq_wqe
->u32_8
,
133 UD_SEND_WQE_U32_8_DMAC_5_M
,
134 UD_SEND_WQE_U32_8_DMAC_5_S
,
137 smac
= (u8
*)hr_dev
->dev_addr
[qp
->port
];
138 loopback
= ether_addr_equal_unaligned(ah
->av
.mac
,
140 roce_set_bit(ud_sq_wqe
->u32_8
,
141 UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S
,
144 roce_set_field(ud_sq_wqe
->u32_8
,
145 UD_SEND_WQE_U32_8_OPERATION_TYPE_M
,
146 UD_SEND_WQE_U32_8_OPERATION_TYPE_S
,
147 HNS_ROCE_WQE_OPCODE_SEND
);
148 roce_set_field(ud_sq_wqe
->u32_8
,
149 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M
,
150 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S
,
152 roce_set_bit(ud_sq_wqe
->u32_8
,
153 UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S
,
156 ud_sq_wqe
->u32_8
|= (wr
->send_flags
& IB_SEND_SIGNALED
?
157 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY
) : 0) |
158 (wr
->send_flags
& IB_SEND_SOLICITED
?
159 cpu_to_le32(HNS_ROCE_WQE_SE
) : 0) |
160 ((wr
->opcode
== IB_WR_SEND_WITH_IMM
) ?
161 cpu_to_le32(HNS_ROCE_WQE_IMM
) : 0);
163 roce_set_field(ud_sq_wqe
->u32_16
,
164 UD_SEND_WQE_U32_16_DEST_QP_M
,
165 UD_SEND_WQE_U32_16_DEST_QP_S
,
166 ud_wr(wr
)->remote_qpn
);
167 roce_set_field(ud_sq_wqe
->u32_16
,
168 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M
,
169 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S
,
172 roce_set_field(ud_sq_wqe
->u32_36
,
173 UD_SEND_WQE_U32_36_FLOW_LABEL_M
,
174 UD_SEND_WQE_U32_36_FLOW_LABEL_S
, 0);
175 roce_set_field(ud_sq_wqe
->u32_36
,
176 UD_SEND_WQE_U32_36_PRIORITY_M
,
177 UD_SEND_WQE_U32_36_PRIORITY_S
,
178 ah
->av
.sl_tclass_flowlabel
>>
180 roce_set_field(ud_sq_wqe
->u32_36
,
181 UD_SEND_WQE_U32_36_SGID_INDEX_M
,
182 UD_SEND_WQE_U32_36_SGID_INDEX_S
,
183 hns_get_gid_index(hr_dev
, qp
->phy_port
,
186 roce_set_field(ud_sq_wqe
->u32_40
,
187 UD_SEND_WQE_U32_40_HOP_LIMIT_M
,
188 UD_SEND_WQE_U32_40_HOP_LIMIT_S
,
190 roce_set_field(ud_sq_wqe
->u32_40
,
191 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M
,
192 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S
, 0);
194 memcpy(&ud_sq_wqe
->dgid
[0], &ah
->av
.dgid
[0], GID_LEN
);
196 ud_sq_wqe
->va0_l
= (u32
)wr
->sg_list
[0].addr
;
197 ud_sq_wqe
->va0_h
= (wr
->sg_list
[0].addr
) >> 32;
198 ud_sq_wqe
->l_key0
= wr
->sg_list
[0].lkey
;
200 ud_sq_wqe
->va1_l
= (u32
)wr
->sg_list
[1].addr
;
201 ud_sq_wqe
->va1_h
= (wr
->sg_list
[1].addr
) >> 32;
202 ud_sq_wqe
->l_key1
= wr
->sg_list
[1].lkey
;
204 } else if (ibqp
->qp_type
== IB_QPT_RC
) {
206 memset(ctrl
, 0, sizeof(struct hns_roce_wqe_ctrl_seg
));
207 for (i
= 0; i
< wr
->num_sge
; i
++)
208 ctrl
->msg_length
+= wr
->sg_list
[i
].length
;
212 ctrl
->imm_data
= send_ieth(wr
);
214 /*Ctrl field, ctrl set type: sig, solic, imm, fence */
215 /* SO wait for conforming application scenarios */
216 ctrl
->flag
|= (wr
->send_flags
& IB_SEND_SIGNALED
?
217 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY
) : 0) |
218 (wr
->send_flags
& IB_SEND_SOLICITED
?
219 cpu_to_le32(HNS_ROCE_WQE_SE
) : 0) |
220 ((wr
->opcode
== IB_WR_SEND_WITH_IMM
||
221 wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
) ?
222 cpu_to_le32(HNS_ROCE_WQE_IMM
) : 0) |
223 (wr
->send_flags
& IB_SEND_FENCE
?
224 (cpu_to_le32(HNS_ROCE_WQE_FENCE
)) : 0);
226 wqe
+= sizeof(struct hns_roce_wqe_ctrl_seg
);
228 switch (wr
->opcode
) {
229 case IB_WR_RDMA_READ
:
230 ps_opcode
= HNS_ROCE_WQE_OPCODE_RDMA_READ
;
231 set_raddr_seg(wqe
, rdma_wr(wr
)->remote_addr
,
234 case IB_WR_RDMA_WRITE
:
235 case IB_WR_RDMA_WRITE_WITH_IMM
:
236 ps_opcode
= HNS_ROCE_WQE_OPCODE_RDMA_WRITE
;
237 set_raddr_seg(wqe
, rdma_wr(wr
)->remote_addr
,
241 case IB_WR_SEND_WITH_INV
:
242 case IB_WR_SEND_WITH_IMM
:
243 ps_opcode
= HNS_ROCE_WQE_OPCODE_SEND
;
245 case IB_WR_LOCAL_INV
:
247 case IB_WR_ATOMIC_CMP_AND_SWP
:
248 case IB_WR_ATOMIC_FETCH_AND_ADD
:
251 ps_opcode
= HNS_ROCE_WQE_OPCODE_MASK
;
254 ctrl
->flag
|= cpu_to_le32(ps_opcode
);
255 wqe
+= sizeof(struct hns_roce_wqe_raddr_seg
);
258 if (wr
->send_flags
& IB_SEND_INLINE
&& wr
->num_sge
) {
259 if (ctrl
->msg_length
>
260 hr_dev
->caps
.max_sq_inline
) {
263 dev_err(dev
, "inline len(1-%d)=%d, illegal",
265 hr_dev
->caps
.max_sq_inline
);
268 for (i
= 0; i
< wr
->num_sge
; i
++) {
269 memcpy(wqe
, ((void *) (uintptr_t)
270 wr
->sg_list
[i
].addr
),
271 wr
->sg_list
[i
].length
);
272 wqe
+= wr
->sg_list
[i
].length
;
274 ctrl
->flag
|= HNS_ROCE_WQE_INLINE
;
277 for (i
= 0; i
< wr
->num_sge
; i
++)
278 set_data_seg(dseg
+ i
, wr
->sg_list
+ i
);
280 ctrl
->flag
|= cpu_to_le32(wr
->num_sge
<<
281 HNS_ROCE_WQE_SGE_NUM_BIT
);
296 roce_set_field(sq_db
.u32_4
, SQ_DOORBELL_U32_4_SQ_HEAD_M
,
297 SQ_DOORBELL_U32_4_SQ_HEAD_S
,
298 (qp
->sq
.head
& ((qp
->sq
.wqe_cnt
<< 1) - 1)));
299 roce_set_field(sq_db
.u32_4
, SQ_DOORBELL_U32_4_SL_M
,
300 SQ_DOORBELL_U32_4_SL_S
, qp
->sl
);
301 roce_set_field(sq_db
.u32_4
, SQ_DOORBELL_U32_4_PORT_M
,
302 SQ_DOORBELL_U32_4_PORT_S
, qp
->phy_port
);
303 roce_set_field(sq_db
.u32_8
, SQ_DOORBELL_U32_8_QPN_M
,
304 SQ_DOORBELL_U32_8_QPN_S
, qp
->doorbell_qpn
);
305 roce_set_bit(sq_db
.u32_8
, SQ_DOORBELL_HW_SYNC_S
, 1);
307 doorbell
[0] = sq_db
.u32_4
;
308 doorbell
[1] = sq_db
.u32_8
;
310 hns_roce_write64_k(doorbell
, qp
->sq
.db_reg_l
);
311 qp
->sq_next_wqe
= ind
;
314 spin_unlock_irqrestore(&qp
->sq
.lock
, flags
);
319 int hns_roce_v1_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
320 struct ib_recv_wr
**bad_wr
)
327 unsigned long flags
= 0;
328 struct hns_roce_rq_wqe_ctrl
*ctrl
= NULL
;
329 struct hns_roce_wqe_data_seg
*scat
= NULL
;
330 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
331 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
332 struct device
*dev
= &hr_dev
->pdev
->dev
;
333 struct hns_roce_rq_db rq_db
;
334 uint32_t doorbell
[2] = {0};
336 spin_lock_irqsave(&hr_qp
->rq
.lock
, flags
);
337 ind
= hr_qp
->rq
.head
& (hr_qp
->rq
.wqe_cnt
- 1);
339 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
340 if (hns_roce_wq_overflow(&hr_qp
->rq
, nreq
,
341 hr_qp
->ibqp
.recv_cq
)) {
347 if (unlikely(wr
->num_sge
> hr_qp
->rq
.max_gs
)) {
348 dev_err(dev
, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
349 wr
->num_sge
, hr_qp
->rq
.max_gs
);
355 ctrl
= get_recv_wqe(hr_qp
, ind
);
357 roce_set_field(ctrl
->rwqe_byte_12
,
358 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M
,
359 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S
,
362 scat
= (struct hns_roce_wqe_data_seg
*)(ctrl
+ 1);
364 for (i
= 0; i
< wr
->num_sge
; i
++)
365 set_data_seg(scat
+ i
, wr
->sg_list
+ i
);
367 hr_qp
->rq
.wrid
[ind
] = wr
->wr_id
;
369 ind
= (ind
+ 1) & (hr_qp
->rq
.wqe_cnt
- 1);
374 hr_qp
->rq
.head
+= nreq
;
378 if (ibqp
->qp_type
== IB_QPT_GSI
) {
379 /* SW update GSI rq header */
380 reg_val
= roce_read(to_hr_dev(ibqp
->device
),
381 ROCEE_QP1C_CFG3_0_REG
+
382 QP1C_CFGN_OFFSET
* hr_qp
->phy_port
);
383 roce_set_field(reg_val
,
384 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M
,
385 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S
,
387 roce_write(to_hr_dev(ibqp
->device
),
388 ROCEE_QP1C_CFG3_0_REG
+
389 QP1C_CFGN_OFFSET
* hr_qp
->phy_port
, reg_val
);
394 roce_set_field(rq_db
.u32_4
, RQ_DOORBELL_U32_4_RQ_HEAD_M
,
395 RQ_DOORBELL_U32_4_RQ_HEAD_S
,
397 roce_set_field(rq_db
.u32_8
, RQ_DOORBELL_U32_8_QPN_M
,
398 RQ_DOORBELL_U32_8_QPN_S
, hr_qp
->qpn
);
399 roce_set_field(rq_db
.u32_8
, RQ_DOORBELL_U32_8_CMD_M
,
400 RQ_DOORBELL_U32_8_CMD_S
, 1);
401 roce_set_bit(rq_db
.u32_8
, RQ_DOORBELL_U32_8_HW_SYNC_S
,
404 doorbell
[0] = rq_db
.u32_4
;
405 doorbell
[1] = rq_db
.u32_8
;
407 hns_roce_write64_k(doorbell
, hr_qp
->rq
.db_reg_l
);
410 spin_unlock_irqrestore(&hr_qp
->rq
.lock
, flags
);
415 static void hns_roce_set_db_event_mode(struct hns_roce_dev
*hr_dev
,
416 int sdb_mode
, int odb_mode
)
420 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
421 roce_set_bit(val
, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S
, sdb_mode
);
422 roce_set_bit(val
, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S
, odb_mode
);
423 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
426 static void hns_roce_set_db_ext_mode(struct hns_roce_dev
*hr_dev
, u32 sdb_mode
,
431 /* Configure SDB/ODB extend mode */
432 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
433 roce_set_bit(val
, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S
, sdb_mode
);
434 roce_set_bit(val
, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S
, odb_mode
);
435 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
438 static void hns_roce_set_sdb(struct hns_roce_dev
*hr_dev
, u32 sdb_alept
,
444 val
= roce_read(hr_dev
, ROCEE_DB_SQ_WL_REG
);
445 roce_set_field(val
, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M
,
446 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S
, sdb_alful
);
447 roce_set_field(val
, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M
,
448 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S
, sdb_alept
);
449 roce_write(hr_dev
, ROCEE_DB_SQ_WL_REG
, val
);
452 static void hns_roce_set_odb(struct hns_roce_dev
*hr_dev
, u32 odb_alept
,
458 val
= roce_read(hr_dev
, ROCEE_DB_OTHERS_WL_REG
);
459 roce_set_field(val
, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M
,
460 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S
, odb_alful
);
461 roce_set_field(val
, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M
,
462 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S
, odb_alept
);
463 roce_write(hr_dev
, ROCEE_DB_OTHERS_WL_REG
, val
);
466 static void hns_roce_set_sdb_ext(struct hns_roce_dev
*hr_dev
, u32 ext_sdb_alept
,
469 struct device
*dev
= &hr_dev
->pdev
->dev
;
470 struct hns_roce_v1_priv
*priv
;
471 struct hns_roce_db_table
*db
;
472 dma_addr_t sdb_dma_addr
;
475 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
476 db
= &priv
->db_table
;
478 /* Configure extend SDB threshold */
479 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_WL_EMPTY_REG
, ext_sdb_alept
);
480 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_WL_REG
, ext_sdb_alful
);
482 /* Configure extend SDB base addr */
483 sdb_dma_addr
= db
->ext_db
->sdb_buf_list
->map
;
484 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_REG
, (u32
)(sdb_dma_addr
>> 12));
486 /* Configure extend SDB depth */
487 val
= roce_read(hr_dev
, ROCEE_EXT_DB_SQ_H_REG
);
488 roce_set_field(val
, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M
,
489 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S
,
490 db
->ext_db
->esdb_dep
);
492 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
493 * using 4K page, and shift more 32 because of
494 * caculating the high 32 bit value evaluated to hardware.
496 roce_set_field(val
, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M
,
497 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S
, sdb_dma_addr
>> 44);
498 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_H_REG
, val
);
500 dev_dbg(dev
, "ext SDB depth: 0x%x\n", db
->ext_db
->esdb_dep
);
501 dev_dbg(dev
, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
502 ext_sdb_alept
, ext_sdb_alful
);
505 static void hns_roce_set_odb_ext(struct hns_roce_dev
*hr_dev
, u32 ext_odb_alept
,
508 struct device
*dev
= &hr_dev
->pdev
->dev
;
509 struct hns_roce_v1_priv
*priv
;
510 struct hns_roce_db_table
*db
;
511 dma_addr_t odb_dma_addr
;
514 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
515 db
= &priv
->db_table
;
517 /* Configure extend ODB threshold */
518 roce_write(hr_dev
, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG
, ext_odb_alept
);
519 roce_write(hr_dev
, ROCEE_EXT_DB_OTHERS_WL_REG
, ext_odb_alful
);
521 /* Configure extend ODB base addr */
522 odb_dma_addr
= db
->ext_db
->odb_buf_list
->map
;
523 roce_write(hr_dev
, ROCEE_EXT_DB_OTH_REG
, (u32
)(odb_dma_addr
>> 12));
525 /* Configure extend ODB depth */
526 val
= roce_read(hr_dev
, ROCEE_EXT_DB_OTH_H_REG
);
527 roce_set_field(val
, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M
,
528 ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S
,
529 db
->ext_db
->eodb_dep
);
530 roce_set_field(val
, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M
,
531 ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S
,
532 db
->ext_db
->eodb_dep
);
533 roce_write(hr_dev
, ROCEE_EXT_DB_OTH_H_REG
, val
);
535 dev_dbg(dev
, "ext ODB depth: 0x%x\n", db
->ext_db
->eodb_dep
);
536 dev_dbg(dev
, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
537 ext_odb_alept
, ext_odb_alful
);
540 static int hns_roce_db_ext_init(struct hns_roce_dev
*hr_dev
, u32 sdb_ext_mod
,
543 struct device
*dev
= &hr_dev
->pdev
->dev
;
544 struct hns_roce_v1_priv
*priv
;
545 struct hns_roce_db_table
*db
;
546 dma_addr_t sdb_dma_addr
;
547 dma_addr_t odb_dma_addr
;
550 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
551 db
= &priv
->db_table
;
553 db
->ext_db
= kmalloc(sizeof(*db
->ext_db
), GFP_KERNEL
);
558 db
->ext_db
->sdb_buf_list
= kmalloc(
559 sizeof(*db
->ext_db
->sdb_buf_list
), GFP_KERNEL
);
560 if (!db
->ext_db
->sdb_buf_list
) {
562 goto ext_sdb_buf_fail_out
;
565 db
->ext_db
->sdb_buf_list
->buf
= dma_alloc_coherent(dev
,
566 HNS_ROCE_V1_EXT_SDB_SIZE
,
567 &sdb_dma_addr
, GFP_KERNEL
);
568 if (!db
->ext_db
->sdb_buf_list
->buf
) {
570 goto alloc_sq_db_buf_fail
;
572 db
->ext_db
->sdb_buf_list
->map
= sdb_dma_addr
;
574 db
->ext_db
->esdb_dep
= ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH
);
575 hns_roce_set_sdb_ext(hr_dev
, HNS_ROCE_V1_EXT_SDB_ALEPT
,
576 HNS_ROCE_V1_EXT_SDB_ALFUL
);
578 hns_roce_set_sdb(hr_dev
, HNS_ROCE_V1_SDB_ALEPT
,
579 HNS_ROCE_V1_SDB_ALFUL
);
582 db
->ext_db
->odb_buf_list
= kmalloc(
583 sizeof(*db
->ext_db
->odb_buf_list
), GFP_KERNEL
);
584 if (!db
->ext_db
->odb_buf_list
) {
586 goto ext_odb_buf_fail_out
;
589 db
->ext_db
->odb_buf_list
->buf
= dma_alloc_coherent(dev
,
590 HNS_ROCE_V1_EXT_ODB_SIZE
,
591 &odb_dma_addr
, GFP_KERNEL
);
592 if (!db
->ext_db
->odb_buf_list
->buf
) {
594 goto alloc_otr_db_buf_fail
;
596 db
->ext_db
->odb_buf_list
->map
= odb_dma_addr
;
598 db
->ext_db
->eodb_dep
= ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH
);
599 hns_roce_set_odb_ext(hr_dev
, HNS_ROCE_V1_EXT_ODB_ALEPT
,
600 HNS_ROCE_V1_EXT_ODB_ALFUL
);
602 hns_roce_set_odb(hr_dev
, HNS_ROCE_V1_ODB_ALEPT
,
603 HNS_ROCE_V1_ODB_ALFUL
);
605 hns_roce_set_db_ext_mode(hr_dev
, sdb_ext_mod
, odb_ext_mod
);
609 alloc_otr_db_buf_fail
:
610 kfree(db
->ext_db
->odb_buf_list
);
612 ext_odb_buf_fail_out
:
614 dma_free_coherent(dev
, HNS_ROCE_V1_EXT_SDB_SIZE
,
615 db
->ext_db
->sdb_buf_list
->buf
,
616 db
->ext_db
->sdb_buf_list
->map
);
619 alloc_sq_db_buf_fail
:
621 kfree(db
->ext_db
->sdb_buf_list
);
623 ext_sdb_buf_fail_out
:
628 static struct hns_roce_qp
*hns_roce_v1_create_lp_qp(struct hns_roce_dev
*hr_dev
,
631 struct device
*dev
= &hr_dev
->pdev
->dev
;
632 struct ib_qp_init_attr init_attr
;
635 memset(&init_attr
, 0, sizeof(struct ib_qp_init_attr
));
636 init_attr
.qp_type
= IB_QPT_RC
;
637 init_attr
.sq_sig_type
= IB_SIGNAL_ALL_WR
;
638 init_attr
.cap
.max_recv_wr
= HNS_ROCE_MIN_WQE_NUM
;
639 init_attr
.cap
.max_send_wr
= HNS_ROCE_MIN_WQE_NUM
;
641 qp
= hns_roce_create_qp(pd
, &init_attr
, NULL
);
643 dev_err(dev
, "Create loop qp for mr free failed!");
650 static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev
*hr_dev
)
652 struct hns_roce_caps
*caps
= &hr_dev
->caps
;
653 struct device
*dev
= &hr_dev
->pdev
->dev
;
654 struct ib_cq_init_attr cq_init_attr
;
655 struct hns_roce_free_mr
*free_mr
;
656 struct ib_qp_attr attr
= { 0 };
657 struct hns_roce_v1_priv
*priv
;
658 struct hns_roce_qp
*hr_qp
;
666 u8 queue_en
[HNS_ROCE_V1_RESV_QP
] = { 0 };
671 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
672 free_mr
= &priv
->free_mr
;
674 /* Reserved cq for loop qp */
675 cq_init_attr
.cqe
= HNS_ROCE_MIN_WQE_NUM
* 2;
676 cq_init_attr
.comp_vector
= 0;
677 cq
= hns_roce_ib_create_cq(&hr_dev
->ib_dev
, &cq_init_attr
, NULL
, NULL
);
679 dev_err(dev
, "Create cq for reseved loop qp failed!");
682 free_mr
->mr_free_cq
= to_hr_cq(cq
);
683 free_mr
->mr_free_cq
->ib_cq
.device
= &hr_dev
->ib_dev
;
684 free_mr
->mr_free_cq
->ib_cq
.uobject
= NULL
;
685 free_mr
->mr_free_cq
->ib_cq
.comp_handler
= NULL
;
686 free_mr
->mr_free_cq
->ib_cq
.event_handler
= NULL
;
687 free_mr
->mr_free_cq
->ib_cq
.cq_context
= NULL
;
688 atomic_set(&free_mr
->mr_free_cq
->ib_cq
.usecnt
, 0);
690 pd
= hns_roce_alloc_pd(&hr_dev
->ib_dev
, NULL
, NULL
);
692 dev_err(dev
, "Create pd for reseved loop qp failed!");
694 goto alloc_pd_failed
;
696 free_mr
->mr_free_pd
= to_hr_pd(pd
);
697 free_mr
->mr_free_pd
->ibpd
.device
= &hr_dev
->ib_dev
;
698 free_mr
->mr_free_pd
->ibpd
.uobject
= NULL
;
699 atomic_set(&free_mr
->mr_free_pd
->ibpd
.usecnt
, 0);
701 attr
.qp_access_flags
= IB_ACCESS_REMOTE_WRITE
;
703 attr
.min_rnr_timer
= 0;
704 /* Disable read ability */
705 attr
.max_dest_rd_atomic
= 0;
706 attr
.max_rd_atomic
= 0;
707 /* Use arbitrary values as rq_psn and sq_psn */
708 attr
.rq_psn
= 0x0808;
709 attr
.sq_psn
= 0x0808;
713 attr
.path_mtu
= IB_MTU_256
;
714 attr
.ah_attr
.type
= RDMA_AH_ATTR_TYPE_ROCE
;
715 rdma_ah_set_grh(&attr
.ah_attr
, NULL
, 0, 0, 1, 0);
716 rdma_ah_set_static_rate(&attr
.ah_attr
, 3);
718 subnet_prefix
= cpu_to_be64(0xfe80000000000000LL
);
719 for (i
= 0; i
< HNS_ROCE_V1_RESV_QP
; i
++) {
720 phy_port
= (i
>= HNS_ROCE_MAX_PORTS
) ? (i
- 2) :
721 (i
% HNS_ROCE_MAX_PORTS
);
722 sl
= i
/ HNS_ROCE_MAX_PORTS
;
724 for (j
= 0; j
< caps
->num_ports
; j
++) {
725 if (hr_dev
->iboe
.phy_port
[j
] == phy_port
) {
735 free_mr
->mr_free_qp
[i
] = hns_roce_v1_create_lp_qp(hr_dev
, pd
);
736 if (!free_mr
->mr_free_qp
[i
]) {
737 dev_err(dev
, "Create loop qp failed!\n");
738 goto create_lp_qp_failed
;
740 hr_qp
= free_mr
->mr_free_qp
[i
];
743 hr_qp
->phy_port
= phy_port
;
744 hr_qp
->ibqp
.qp_type
= IB_QPT_RC
;
745 hr_qp
->ibqp
.device
= &hr_dev
->ib_dev
;
746 hr_qp
->ibqp
.uobject
= NULL
;
747 atomic_set(&hr_qp
->ibqp
.usecnt
, 0);
749 hr_qp
->ibqp
.recv_cq
= cq
;
750 hr_qp
->ibqp
.send_cq
= cq
;
752 rdma_ah_set_port_num(&attr
.ah_attr
, port
+ 1);
753 rdma_ah_set_sl(&attr
.ah_attr
, sl
);
754 attr
.port_num
= port
+ 1;
756 attr
.dest_qp_num
= hr_qp
->qpn
;
757 memcpy(rdma_ah_retrieve_dmac(&attr
.ah_attr
),
758 hr_dev
->dev_addr
[port
],
761 memcpy(&dgid
.raw
, &subnet_prefix
, sizeof(u64
));
762 memcpy(&dgid
.raw
[8], hr_dev
->dev_addr
[port
], 3);
763 memcpy(&dgid
.raw
[13], hr_dev
->dev_addr
[port
] + 3, 3);
767 rdma_ah_set_dgid_raw(&attr
.ah_attr
, dgid
.raw
);
769 ret
= hr_dev
->hw
->modify_qp(&hr_qp
->ibqp
, &attr
, attr_mask
,
770 IB_QPS_RESET
, IB_QPS_INIT
);
772 dev_err(dev
, "modify qp failed(%d)!\n", ret
);
773 goto create_lp_qp_failed
;
776 ret
= hr_dev
->hw
->modify_qp(&hr_qp
->ibqp
, &attr
, attr_mask
,
777 IB_QPS_INIT
, IB_QPS_RTR
);
779 dev_err(dev
, "modify qp failed(%d)!\n", ret
);
780 goto create_lp_qp_failed
;
783 ret
= hr_dev
->hw
->modify_qp(&hr_qp
->ibqp
, &attr
, attr_mask
,
784 IB_QPS_RTR
, IB_QPS_RTS
);
786 dev_err(dev
, "modify qp failed(%d)!\n", ret
);
787 goto create_lp_qp_failed
;
794 for (i
-= 1; i
>= 0; i
--) {
795 hr_qp
= free_mr
->mr_free_qp
[i
];
796 if (hns_roce_v1_destroy_qp(&hr_qp
->ibqp
))
797 dev_err(dev
, "Destroy qp %d for mr free failed!\n", i
);
800 if (hns_roce_dealloc_pd(pd
))
801 dev_err(dev
, "Destroy pd for create_lp_qp failed!\n");
804 if (hns_roce_ib_destroy_cq(cq
))
805 dev_err(dev
, "Destroy cq for create_lp_qp failed!\n");
810 static void hns_roce_v1_release_lp_qp(struct hns_roce_dev
*hr_dev
)
812 struct device
*dev
= &hr_dev
->pdev
->dev
;
813 struct hns_roce_free_mr
*free_mr
;
814 struct hns_roce_v1_priv
*priv
;
815 struct hns_roce_qp
*hr_qp
;
819 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
820 free_mr
= &priv
->free_mr
;
822 for (i
= 0; i
< HNS_ROCE_V1_RESV_QP
; i
++) {
823 hr_qp
= free_mr
->mr_free_qp
[i
];
827 ret
= hns_roce_v1_destroy_qp(&hr_qp
->ibqp
);
829 dev_err(dev
, "Destroy qp %d for mr free failed(%d)!\n",
833 ret
= hns_roce_ib_destroy_cq(&free_mr
->mr_free_cq
->ib_cq
);
835 dev_err(dev
, "Destroy cq for mr_free failed(%d)!\n", ret
);
837 ret
= hns_roce_dealloc_pd(&free_mr
->mr_free_pd
->ibpd
);
839 dev_err(dev
, "Destroy pd for mr_free failed(%d)!\n", ret
);
842 static int hns_roce_db_init(struct hns_roce_dev
*hr_dev
)
844 struct device
*dev
= &hr_dev
->pdev
->dev
;
845 struct hns_roce_v1_priv
*priv
;
846 struct hns_roce_db_table
*db
;
853 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
854 db
= &priv
->db_table
;
856 memset(db
, 0, sizeof(*db
));
858 /* Default DB mode */
859 sdb_ext_mod
= HNS_ROCE_SDB_EXTEND_MODE
;
860 odb_ext_mod
= HNS_ROCE_ODB_EXTEND_MODE
;
861 sdb_evt_mod
= HNS_ROCE_SDB_NORMAL_MODE
;
862 odb_evt_mod
= HNS_ROCE_ODB_POLL_MODE
;
864 db
->sdb_ext_mod
= sdb_ext_mod
;
865 db
->odb_ext_mod
= odb_ext_mod
;
868 ret
= hns_roce_db_ext_init(hr_dev
, sdb_ext_mod
, odb_ext_mod
);
870 dev_err(dev
, "Failed in extend DB configuration.\n");
874 hns_roce_set_db_event_mode(hr_dev
, sdb_evt_mod
, odb_evt_mod
);
879 void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct
*work
)
881 struct hns_roce_recreate_lp_qp_work
*lp_qp_work
;
882 struct hns_roce_dev
*hr_dev
;
884 lp_qp_work
= container_of(work
, struct hns_roce_recreate_lp_qp_work
,
886 hr_dev
= to_hr_dev(lp_qp_work
->ib_dev
);
888 hns_roce_v1_release_lp_qp(hr_dev
);
890 if (hns_roce_v1_rsv_lp_qp(hr_dev
))
891 dev_err(&hr_dev
->pdev
->dev
, "create reserver qp failed\n");
893 if (lp_qp_work
->comp_flag
)
894 complete(lp_qp_work
->comp
);
899 static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev
*hr_dev
)
901 struct device
*dev
= &hr_dev
->pdev
->dev
;
902 struct hns_roce_recreate_lp_qp_work
*lp_qp_work
;
903 struct hns_roce_free_mr
*free_mr
;
904 struct hns_roce_v1_priv
*priv
;
905 struct completion comp
;
907 msecs_to_jiffies(HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS
) + jiffies
;
909 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
910 free_mr
= &priv
->free_mr
;
912 lp_qp_work
= kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work
),
915 INIT_WORK(&(lp_qp_work
->work
), hns_roce_v1_recreate_lp_qp_work_fn
);
917 lp_qp_work
->ib_dev
= &(hr_dev
->ib_dev
);
918 lp_qp_work
->comp
= &comp
;
919 lp_qp_work
->comp_flag
= 1;
921 init_completion(lp_qp_work
->comp
);
923 queue_work(free_mr
->free_mr_wq
, &(lp_qp_work
->work
));
925 while (time_before_eq(jiffies
, end
)) {
926 if (try_wait_for_completion(&comp
))
928 msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE
);
931 lp_qp_work
->comp_flag
= 0;
932 if (try_wait_for_completion(&comp
))
935 dev_warn(dev
, "recreate lp qp failed 20s timeout and return failed!\n");
939 static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp
*hr_qp
)
941 struct hns_roce_dev
*hr_dev
= to_hr_dev(hr_qp
->ibqp
.device
);
942 struct device
*dev
= &hr_dev
->pdev
->dev
;
943 struct ib_send_wr send_wr
, *bad_wr
;
946 memset(&send_wr
, 0, sizeof(send_wr
));
949 send_wr
.send_flags
= 0;
950 send_wr
.sg_list
= NULL
;
951 send_wr
.wr_id
= (unsigned long long)&send_wr
;
952 send_wr
.opcode
= IB_WR_RDMA_WRITE
;
954 ret
= hns_roce_v1_post_send(&hr_qp
->ibqp
, &send_wr
, &bad_wr
);
956 dev_err(dev
, "Post write wqe for mr free failed(%d)!", ret
);
963 static void hns_roce_v1_mr_free_work_fn(struct work_struct
*work
)
965 struct hns_roce_mr_free_work
*mr_work
;
966 struct ib_wc wc
[HNS_ROCE_V1_RESV_QP
];
967 struct hns_roce_free_mr
*free_mr
;
968 struct hns_roce_cq
*mr_free_cq
;
969 struct hns_roce_v1_priv
*priv
;
970 struct hns_roce_dev
*hr_dev
;
971 struct hns_roce_mr
*hr_mr
;
972 struct hns_roce_qp
*hr_qp
;
975 msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS
) + jiffies
;
980 mr_work
= container_of(work
, struct hns_roce_mr_free_work
, work
);
981 hr_mr
= (struct hns_roce_mr
*)mr_work
->mr
;
982 hr_dev
= to_hr_dev(mr_work
->ib_dev
);
983 dev
= &hr_dev
->pdev
->dev
;
985 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
986 free_mr
= &priv
->free_mr
;
987 mr_free_cq
= free_mr
->mr_free_cq
;
989 for (i
= 0; i
< HNS_ROCE_V1_RESV_QP
; i
++) {
990 hr_qp
= free_mr
->mr_free_qp
[i
];
995 ret
= hns_roce_v1_send_lp_wqe(hr_qp
);
998 "Send wqe (qp:0x%lx) for mr free failed(%d)!\n",
1005 ret
= hns_roce_v1_poll_cq(&mr_free_cq
->ib_cq
, ne
, wc
);
1008 "(qp:0x%lx) starts, Poll cqe failed(%d) for mr 0x%x free! Remain %d cqe\n",
1009 hr_qp
->qpn
, ret
, hr_mr
->key
, ne
);
1013 usleep_range(HNS_ROCE_V1_FREE_MR_WAIT_VALUE
* 1000,
1014 (1 + HNS_ROCE_V1_FREE_MR_WAIT_VALUE
) * 1000);
1015 } while (ne
&& time_before_eq(jiffies
, end
));
1019 "Poll cqe for mr 0x%x free timeout! Remain %d cqe\n",
1023 if (mr_work
->comp_flag
)
1024 complete(mr_work
->comp
);
1028 int hns_roce_v1_dereg_mr(struct hns_roce_dev
*hr_dev
, struct hns_roce_mr
*mr
)
1030 struct device
*dev
= &hr_dev
->pdev
->dev
;
1031 struct hns_roce_mr_free_work
*mr_work
;
1032 struct hns_roce_free_mr
*free_mr
;
1033 struct hns_roce_v1_priv
*priv
;
1034 struct completion comp
;
1036 msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS
) + jiffies
;
1037 unsigned long start
= jiffies
;
1041 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1042 free_mr
= &priv
->free_mr
;
1045 if (hns_roce_hw2sw_mpt(hr_dev
, NULL
, key_to_hw_index(mr
->key
)
1046 & (hr_dev
->caps
.num_mtpts
- 1)))
1047 dev_warn(dev
, "HW2SW_MPT failed!\n");
1050 mr_work
= kzalloc(sizeof(*mr_work
), GFP_KERNEL
);
1056 INIT_WORK(&(mr_work
->work
), hns_roce_v1_mr_free_work_fn
);
1058 mr_work
->ib_dev
= &(hr_dev
->ib_dev
);
1059 mr_work
->comp
= &comp
;
1060 mr_work
->comp_flag
= 1;
1061 mr_work
->mr
= (void *)mr
;
1062 init_completion(mr_work
->comp
);
1064 queue_work(free_mr
->free_mr_wq
, &(mr_work
->work
));
1066 while (time_before_eq(jiffies
, end
)) {
1067 if (try_wait_for_completion(&comp
))
1069 msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE
);
1072 mr_work
->comp_flag
= 0;
1073 if (try_wait_for_completion(&comp
))
1076 dev_warn(dev
, "Free mr work 0x%x over 50s and failed!\n", mr
->key
);
1080 dev_dbg(dev
, "Free mr 0x%x use 0x%x us.\n",
1081 mr
->key
, jiffies_to_usecs(jiffies
) - jiffies_to_usecs(start
));
1083 if (mr
->size
!= ~0ULL) {
1084 npages
= ib_umem_page_count(mr
->umem
);
1085 dma_free_coherent(dev
, npages
* 8, mr
->pbl_buf
,
1089 hns_roce_bitmap_free(&hr_dev
->mr_table
.mtpt_bitmap
,
1090 key_to_hw_index(mr
->key
), 0);
1093 ib_umem_release(mr
->umem
);
1100 static void hns_roce_db_free(struct hns_roce_dev
*hr_dev
)
1102 struct device
*dev
= &hr_dev
->pdev
->dev
;
1103 struct hns_roce_v1_priv
*priv
;
1104 struct hns_roce_db_table
*db
;
1106 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1107 db
= &priv
->db_table
;
1109 if (db
->sdb_ext_mod
) {
1110 dma_free_coherent(dev
, HNS_ROCE_V1_EXT_SDB_SIZE
,
1111 db
->ext_db
->sdb_buf_list
->buf
,
1112 db
->ext_db
->sdb_buf_list
->map
);
1113 kfree(db
->ext_db
->sdb_buf_list
);
1116 if (db
->odb_ext_mod
) {
1117 dma_free_coherent(dev
, HNS_ROCE_V1_EXT_ODB_SIZE
,
1118 db
->ext_db
->odb_buf_list
->buf
,
1119 db
->ext_db
->odb_buf_list
->map
);
1120 kfree(db
->ext_db
->odb_buf_list
);
1126 static int hns_roce_raq_init(struct hns_roce_dev
*hr_dev
)
1132 struct hns_roce_v1_priv
*priv
;
1133 struct hns_roce_raq_table
*raq
;
1134 struct device
*dev
= &hr_dev
->pdev
->dev
;
1136 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1137 raq
= &priv
->raq_table
;
1139 raq
->e_raq_buf
= kzalloc(sizeof(*(raq
->e_raq_buf
)), GFP_KERNEL
);
1140 if (!raq
->e_raq_buf
)
1143 raq
->e_raq_buf
->buf
= dma_alloc_coherent(dev
, HNS_ROCE_V1_RAQ_SIZE
,
1145 if (!raq
->e_raq_buf
->buf
) {
1147 goto err_dma_alloc_raq
;
1149 raq
->e_raq_buf
->map
= addr
;
1151 /* Configure raq extended address. 48bit 4K align*/
1152 roce_write(hr_dev
, ROCEE_EXT_RAQ_REG
, raq
->e_raq_buf
->map
>> 12);
1154 /* Configure raq_shift */
1155 raq_shift
= ilog2(HNS_ROCE_V1_RAQ_SIZE
/ HNS_ROCE_V1_RAQ_ENTRY
);
1156 val
= roce_read(hr_dev
, ROCEE_EXT_RAQ_H_REG
);
1157 roce_set_field(val
, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M
,
1158 ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S
, raq_shift
);
1160 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1161 * using 4K page, and shift more 32 because of
1162 * caculating the high 32 bit value evaluated to hardware.
1164 roce_set_field(val
, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M
,
1165 ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S
,
1166 raq
->e_raq_buf
->map
>> 44);
1167 roce_write(hr_dev
, ROCEE_EXT_RAQ_H_REG
, val
);
1168 dev_dbg(dev
, "Configure raq_shift 0x%x.\n", val
);
1170 /* Configure raq threshold */
1171 val
= roce_read(hr_dev
, ROCEE_RAQ_WL_REG
);
1172 roce_set_field(val
, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M
,
1173 ROCEE_RAQ_WL_ROCEE_RAQ_WL_S
,
1174 HNS_ROCE_V1_EXT_RAQ_WF
);
1175 roce_write(hr_dev
, ROCEE_RAQ_WL_REG
, val
);
1176 dev_dbg(dev
, "Configure raq_wl 0x%x.\n", val
);
1178 /* Enable extend raq */
1179 val
= roce_read(hr_dev
, ROCEE_WRMS_POL_TIME_INTERVAL_REG
);
1181 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M
,
1182 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S
,
1183 POL_TIME_INTERVAL_VAL
);
1184 roce_set_bit(val
, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE
, 1);
1186 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M
,
1187 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S
,
1190 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S
, 1);
1191 roce_write(hr_dev
, ROCEE_WRMS_POL_TIME_INTERVAL_REG
, val
);
1192 dev_dbg(dev
, "Configure WrmsPolTimeInterval 0x%x.\n", val
);
1194 /* Enable raq drop */
1195 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
1196 roce_set_bit(val
, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S
, 1);
1197 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
1198 dev_dbg(dev
, "Configure GlbCfg = 0x%x.\n", val
);
1203 kfree(raq
->e_raq_buf
);
1207 static void hns_roce_raq_free(struct hns_roce_dev
*hr_dev
)
1209 struct device
*dev
= &hr_dev
->pdev
->dev
;
1210 struct hns_roce_v1_priv
*priv
;
1211 struct hns_roce_raq_table
*raq
;
1213 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1214 raq
= &priv
->raq_table
;
1216 dma_free_coherent(dev
, HNS_ROCE_V1_RAQ_SIZE
, raq
->e_raq_buf
->buf
,
1217 raq
->e_raq_buf
->map
);
1218 kfree(raq
->e_raq_buf
);
1221 static void hns_roce_port_enable(struct hns_roce_dev
*hr_dev
, int enable_flag
)
1226 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
1227 /* Open all ports */
1228 roce_set_field(val
, ROCEE_GLB_CFG_ROCEE_PORT_ST_M
,
1229 ROCEE_GLB_CFG_ROCEE_PORT_ST_S
,
1231 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
1233 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
1234 /* Close all ports */
1235 roce_set_field(val
, ROCEE_GLB_CFG_ROCEE_PORT_ST_M
,
1236 ROCEE_GLB_CFG_ROCEE_PORT_ST_S
, 0x0);
1237 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
1241 static int hns_roce_bt_init(struct hns_roce_dev
*hr_dev
)
1243 struct device
*dev
= &hr_dev
->pdev
->dev
;
1244 struct hns_roce_v1_priv
*priv
;
1247 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1249 priv
->bt_table
.qpc_buf
.buf
= dma_alloc_coherent(dev
,
1250 HNS_ROCE_BT_RSV_BUF_SIZE
, &priv
->bt_table
.qpc_buf
.map
,
1252 if (!priv
->bt_table
.qpc_buf
.buf
)
1255 priv
->bt_table
.mtpt_buf
.buf
= dma_alloc_coherent(dev
,
1256 HNS_ROCE_BT_RSV_BUF_SIZE
, &priv
->bt_table
.mtpt_buf
.map
,
1258 if (!priv
->bt_table
.mtpt_buf
.buf
) {
1260 goto err_failed_alloc_mtpt_buf
;
1263 priv
->bt_table
.cqc_buf
.buf
= dma_alloc_coherent(dev
,
1264 HNS_ROCE_BT_RSV_BUF_SIZE
, &priv
->bt_table
.cqc_buf
.map
,
1266 if (!priv
->bt_table
.cqc_buf
.buf
) {
1268 goto err_failed_alloc_cqc_buf
;
1273 err_failed_alloc_cqc_buf
:
1274 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1275 priv
->bt_table
.mtpt_buf
.buf
, priv
->bt_table
.mtpt_buf
.map
);
1277 err_failed_alloc_mtpt_buf
:
1278 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1279 priv
->bt_table
.qpc_buf
.buf
, priv
->bt_table
.qpc_buf
.map
);
1284 static void hns_roce_bt_free(struct hns_roce_dev
*hr_dev
)
1286 struct device
*dev
= &hr_dev
->pdev
->dev
;
1287 struct hns_roce_v1_priv
*priv
;
1289 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1291 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1292 priv
->bt_table
.cqc_buf
.buf
, priv
->bt_table
.cqc_buf
.map
);
1294 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1295 priv
->bt_table
.mtpt_buf
.buf
, priv
->bt_table
.mtpt_buf
.map
);
1297 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1298 priv
->bt_table
.qpc_buf
.buf
, priv
->bt_table
.qpc_buf
.map
);
1301 static int hns_roce_tptr_init(struct hns_roce_dev
*hr_dev
)
1303 struct device
*dev
= &hr_dev
->pdev
->dev
;
1304 struct hns_roce_buf_list
*tptr_buf
;
1305 struct hns_roce_v1_priv
*priv
;
1307 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1308 tptr_buf
= &priv
->tptr_table
.tptr_buf
;
1311 * This buffer will be used for CQ's tptr(tail pointer), also
1312 * named ci(customer index). Every CQ will use 2 bytes to save
1313 * cqe ci in hip06. Hardware will read this area to get new ci
1314 * when the queue is almost full.
1316 tptr_buf
->buf
= dma_alloc_coherent(dev
, HNS_ROCE_V1_TPTR_BUF_SIZE
,
1317 &tptr_buf
->map
, GFP_KERNEL
);
1321 hr_dev
->tptr_dma_addr
= tptr_buf
->map
;
1322 hr_dev
->tptr_size
= HNS_ROCE_V1_TPTR_BUF_SIZE
;
1327 static void hns_roce_tptr_free(struct hns_roce_dev
*hr_dev
)
1329 struct device
*dev
= &hr_dev
->pdev
->dev
;
1330 struct hns_roce_buf_list
*tptr_buf
;
1331 struct hns_roce_v1_priv
*priv
;
1333 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1334 tptr_buf
= &priv
->tptr_table
.tptr_buf
;
1336 dma_free_coherent(dev
, HNS_ROCE_V1_TPTR_BUF_SIZE
,
1337 tptr_buf
->buf
, tptr_buf
->map
);
1340 static int hns_roce_free_mr_init(struct hns_roce_dev
*hr_dev
)
1342 struct device
*dev
= &hr_dev
->pdev
->dev
;
1343 struct hns_roce_free_mr
*free_mr
;
1344 struct hns_roce_v1_priv
*priv
;
1347 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1348 free_mr
= &priv
->free_mr
;
1350 free_mr
->free_mr_wq
= create_singlethread_workqueue("hns_roce_free_mr");
1351 if (!free_mr
->free_mr_wq
) {
1352 dev_err(dev
, "Create free mr workqueue failed!\n");
1356 ret
= hns_roce_v1_rsv_lp_qp(hr_dev
);
1358 dev_err(dev
, "Reserved loop qp failed(%d)!\n", ret
);
1359 flush_workqueue(free_mr
->free_mr_wq
);
1360 destroy_workqueue(free_mr
->free_mr_wq
);
1366 static void hns_roce_free_mr_free(struct hns_roce_dev
*hr_dev
)
1368 struct hns_roce_free_mr
*free_mr
;
1369 struct hns_roce_v1_priv
*priv
;
1371 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1372 free_mr
= &priv
->free_mr
;
1374 flush_workqueue(free_mr
->free_mr_wq
);
1375 destroy_workqueue(free_mr
->free_mr_wq
);
1377 hns_roce_v1_release_lp_qp(hr_dev
);
1381 * hns_roce_v1_reset - reset RoCE
1382 * @hr_dev: RoCE device struct pointer
1383 * @enable: true -- drop reset, false -- reset
1384 * return 0 - success , negative --fail
1386 int hns_roce_v1_reset(struct hns_roce_dev
*hr_dev
, bool dereset
)
1388 struct device_node
*dsaf_node
;
1389 struct device
*dev
= &hr_dev
->pdev
->dev
;
1390 struct device_node
*np
= dev
->of_node
;
1391 struct fwnode_handle
*fwnode
;
1394 /* check if this is DT/ACPI case */
1395 if (dev_of_node(dev
)) {
1396 dsaf_node
= of_parse_phandle(np
, "dsaf-handle", 0);
1398 dev_err(dev
, "could not find dsaf-handle\n");
1401 fwnode
= &dsaf_node
->fwnode
;
1402 } else if (is_acpi_device_node(dev
->fwnode
)) {
1403 struct acpi_reference_args args
;
1405 ret
= acpi_node_get_property_reference(dev
->fwnode
,
1406 "dsaf-handle", 0, &args
);
1408 dev_err(dev
, "could not find dsaf-handle\n");
1411 fwnode
= acpi_fwnode_handle(args
.adev
);
1413 dev_err(dev
, "cannot read data from DT or ACPI\n");
1417 ret
= hns_dsaf_roce_reset(fwnode
, false);
1422 msleep(SLEEP_TIME_INTERVAL
);
1423 ret
= hns_dsaf_roce_reset(fwnode
, true);
1429 static int hns_roce_des_qp_init(struct hns_roce_dev
*hr_dev
)
1431 struct device
*dev
= &hr_dev
->pdev
->dev
;
1432 struct hns_roce_v1_priv
*priv
;
1433 struct hns_roce_des_qp
*des_qp
;
1435 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1436 des_qp
= &priv
->des_qp
;
1438 des_qp
->requeue_flag
= 1;
1439 des_qp
->qp_wq
= create_singlethread_workqueue("hns_roce_destroy_qp");
1440 if (!des_qp
->qp_wq
) {
1441 dev_err(dev
, "Create destroy qp workqueue failed!\n");
1448 static void hns_roce_des_qp_free(struct hns_roce_dev
*hr_dev
)
1450 struct hns_roce_v1_priv
*priv
;
1451 struct hns_roce_des_qp
*des_qp
;
1453 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1454 des_qp
= &priv
->des_qp
;
1456 des_qp
->requeue_flag
= 0;
1457 flush_workqueue(des_qp
->qp_wq
);
1458 destroy_workqueue(des_qp
->qp_wq
);
1461 void hns_roce_v1_profile(struct hns_roce_dev
*hr_dev
)
1464 struct hns_roce_caps
*caps
= &hr_dev
->caps
;
1466 hr_dev
->vendor_id
= le32_to_cpu(roce_read(hr_dev
, ROCEE_VENDOR_ID_REG
));
1467 hr_dev
->vendor_part_id
= le32_to_cpu(roce_read(hr_dev
,
1468 ROCEE_VENDOR_PART_ID_REG
));
1469 hr_dev
->sys_image_guid
= le32_to_cpu(roce_read(hr_dev
,
1470 ROCEE_SYS_IMAGE_GUID_L_REG
)) |
1471 ((u64
)le32_to_cpu(roce_read(hr_dev
,
1472 ROCEE_SYS_IMAGE_GUID_H_REG
)) << 32);
1473 hr_dev
->hw_rev
= HNS_ROCE_HW_VER1
;
1475 caps
->num_qps
= HNS_ROCE_V1_MAX_QP_NUM
;
1476 caps
->max_wqes
= HNS_ROCE_V1_MAX_WQE_NUM
;
1477 caps
->num_cqs
= HNS_ROCE_V1_MAX_CQ_NUM
;
1478 caps
->max_cqes
= HNS_ROCE_V1_MAX_CQE_NUM
;
1479 caps
->max_sq_sg
= HNS_ROCE_V1_SG_NUM
;
1480 caps
->max_rq_sg
= HNS_ROCE_V1_SG_NUM
;
1481 caps
->max_sq_inline
= HNS_ROCE_V1_INLINE_SIZE
;
1482 caps
->num_uars
= HNS_ROCE_V1_UAR_NUM
;
1483 caps
->phy_num_uars
= HNS_ROCE_V1_PHY_UAR_NUM
;
1484 caps
->num_aeq_vectors
= HNS_ROCE_AEQE_VEC_NUM
;
1485 caps
->num_comp_vectors
= HNS_ROCE_COMP_VEC_NUM
;
1486 caps
->num_other_vectors
= HNS_ROCE_AEQE_OF_VEC_NUM
;
1487 caps
->num_mtpts
= HNS_ROCE_V1_MAX_MTPT_NUM
;
1488 caps
->num_mtt_segs
= HNS_ROCE_V1_MAX_MTT_SEGS
;
1489 caps
->num_pds
= HNS_ROCE_V1_MAX_PD_NUM
;
1490 caps
->max_qp_init_rdma
= HNS_ROCE_V1_MAX_QP_INIT_RDMA
;
1491 caps
->max_qp_dest_rdma
= HNS_ROCE_V1_MAX_QP_DEST_RDMA
;
1492 caps
->max_sq_desc_sz
= HNS_ROCE_V1_MAX_SQ_DESC_SZ
;
1493 caps
->max_rq_desc_sz
= HNS_ROCE_V1_MAX_RQ_DESC_SZ
;
1494 caps
->qpc_entry_sz
= HNS_ROCE_V1_QPC_ENTRY_SIZE
;
1495 caps
->irrl_entry_sz
= HNS_ROCE_V1_IRRL_ENTRY_SIZE
;
1496 caps
->cqc_entry_sz
= HNS_ROCE_V1_CQC_ENTRY_SIZE
;
1497 caps
->mtpt_entry_sz
= HNS_ROCE_V1_MTPT_ENTRY_SIZE
;
1498 caps
->mtt_entry_sz
= HNS_ROCE_V1_MTT_ENTRY_SIZE
;
1499 caps
->cq_entry_sz
= HNS_ROCE_V1_CQE_ENTRY_SIZE
;
1500 caps
->page_size_cap
= HNS_ROCE_V1_PAGE_SIZE_SUPPORT
;
1501 caps
->reserved_lkey
= 0;
1502 caps
->reserved_pds
= 0;
1503 caps
->reserved_mrws
= 1;
1504 caps
->reserved_uars
= 0;
1505 caps
->reserved_cqs
= 0;
1507 for (i
= 0; i
< caps
->num_ports
; i
++)
1508 caps
->pkey_table_len
[i
] = 1;
1510 for (i
= 0; i
< caps
->num_ports
; i
++) {
1511 /* Six ports shared 16 GID in v1 engine */
1512 if (i
>= (HNS_ROCE_V1_GID_NUM
% caps
->num_ports
))
1513 caps
->gid_table_len
[i
] = HNS_ROCE_V1_GID_NUM
/
1516 caps
->gid_table_len
[i
] = HNS_ROCE_V1_GID_NUM
/
1517 caps
->num_ports
+ 1;
1520 for (i
= 0; i
< caps
->num_comp_vectors
; i
++)
1521 caps
->ceqe_depth
[i
] = HNS_ROCE_V1_NUM_COMP_EQE
;
1523 caps
->aeqe_depth
= HNS_ROCE_V1_NUM_ASYNC_EQE
;
1524 caps
->local_ca_ack_delay
= le32_to_cpu(roce_read(hr_dev
,
1525 ROCEE_ACK_DELAY_REG
));
1526 caps
->max_mtu
= IB_MTU_2048
;
1529 int hns_roce_v1_init(struct hns_roce_dev
*hr_dev
)
1533 struct device
*dev
= &hr_dev
->pdev
->dev
;
1535 /* DMAE user config */
1536 val
= roce_read(hr_dev
, ROCEE_DMAE_USER_CFG1_REG
);
1537 roce_set_field(val
, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M
,
1538 ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S
, 0xf);
1539 roce_set_field(val
, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M
,
1540 ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S
,
1541 1 << PAGES_SHIFT_16
);
1542 roce_write(hr_dev
, ROCEE_DMAE_USER_CFG1_REG
, val
);
1544 val
= roce_read(hr_dev
, ROCEE_DMAE_USER_CFG2_REG
);
1545 roce_set_field(val
, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M
,
1546 ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S
, 0xf);
1547 roce_set_field(val
, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M
,
1548 ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S
,
1549 1 << PAGES_SHIFT_16
);
1551 ret
= hns_roce_db_init(hr_dev
);
1553 dev_err(dev
, "doorbell init failed!\n");
1557 ret
= hns_roce_raq_init(hr_dev
);
1559 dev_err(dev
, "raq init failed!\n");
1560 goto error_failed_raq_init
;
1563 ret
= hns_roce_bt_init(hr_dev
);
1565 dev_err(dev
, "bt init failed!\n");
1566 goto error_failed_bt_init
;
1569 ret
= hns_roce_tptr_init(hr_dev
);
1571 dev_err(dev
, "tptr init failed!\n");
1572 goto error_failed_tptr_init
;
1575 ret
= hns_roce_des_qp_init(hr_dev
);
1577 dev_err(dev
, "des qp init failed!\n");
1578 goto error_failed_des_qp_init
;
1581 ret
= hns_roce_free_mr_init(hr_dev
);
1583 dev_err(dev
, "free mr init failed!\n");
1584 goto error_failed_free_mr_init
;
1587 hns_roce_port_enable(hr_dev
, HNS_ROCE_PORT_UP
);
1591 error_failed_free_mr_init
:
1592 hns_roce_des_qp_free(hr_dev
);
1594 error_failed_des_qp_init
:
1595 hns_roce_tptr_free(hr_dev
);
1597 error_failed_tptr_init
:
1598 hns_roce_bt_free(hr_dev
);
1600 error_failed_bt_init
:
1601 hns_roce_raq_free(hr_dev
);
1603 error_failed_raq_init
:
1604 hns_roce_db_free(hr_dev
);
1608 void hns_roce_v1_exit(struct hns_roce_dev
*hr_dev
)
1610 hns_roce_port_enable(hr_dev
, HNS_ROCE_PORT_DOWN
);
1611 hns_roce_free_mr_free(hr_dev
);
1612 hns_roce_des_qp_free(hr_dev
);
1613 hns_roce_tptr_free(hr_dev
);
1614 hns_roce_bt_free(hr_dev
);
1615 hns_roce_raq_free(hr_dev
);
1616 hns_roce_db_free(hr_dev
);
1619 void hns_roce_v1_set_gid(struct hns_roce_dev
*hr_dev
, u8 port
, int gid_index
,
1625 gid_idx
= hns_get_gid_index(hr_dev
, port
, gid_index
);
1627 p
= (u32
*)&gid
->raw
[0];
1628 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_L_0_REG
+
1629 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
1631 p
= (u32
*)&gid
->raw
[4];
1632 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_ML_0_REG
+
1633 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
1635 p
= (u32
*)&gid
->raw
[8];
1636 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_MH_0_REG
+
1637 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
1639 p
= (u32
*)&gid
->raw
[0xc];
1640 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_H_0_REG
+
1641 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
1644 void hns_roce_v1_set_mac(struct hns_roce_dev
*hr_dev
, u8 phy_port
, u8
*addr
)
1653 * When mac changed, loopback may fail
1654 * because of smac not equal to dmac.
1655 * We Need to release and create reserved qp again.
1657 if (hr_dev
->hw
->dereg_mr
&& hns_roce_v1_recreate_lp_qp(hr_dev
))
1658 dev_warn(&hr_dev
->pdev
->dev
, "recreate lp qp timeout!\n");
1660 p
= (u32
*)(&addr
[0]);
1662 roce_raw_write(reg_smac_l
, hr_dev
->reg_base
+ ROCEE_SMAC_L_0_REG
+
1663 PHY_PORT_OFFSET
* phy_port
);
1665 val
= roce_read(hr_dev
,
1666 ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
);
1667 p_h
= (u16
*)(&addr
[4]);
1669 roce_set_field(val
, ROCEE_SMAC_H_ROCEE_SMAC_H_M
,
1670 ROCEE_SMAC_H_ROCEE_SMAC_H_S
, reg_smac_h
);
1671 roce_write(hr_dev
, ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
,
1675 void hns_roce_v1_set_mtu(struct hns_roce_dev
*hr_dev
, u8 phy_port
,
1680 val
= roce_read(hr_dev
,
1681 ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
);
1682 roce_set_field(val
, ROCEE_SMAC_H_ROCEE_PORT_MTU_M
,
1683 ROCEE_SMAC_H_ROCEE_PORT_MTU_S
, mtu
);
1684 roce_write(hr_dev
, ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
,
1688 int hns_roce_v1_write_mtpt(void *mb_buf
, struct hns_roce_mr
*mr
,
1689 unsigned long mtpt_idx
)
1691 struct hns_roce_v1_mpt_entry
*mpt_entry
;
1692 struct scatterlist
*sg
;
1697 /* MPT filled into mailbox buf */
1698 mpt_entry
= (struct hns_roce_v1_mpt_entry
*)mb_buf
;
1699 memset(mpt_entry
, 0, sizeof(*mpt_entry
));
1701 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_KEY_STATE_M
,
1702 MPT_BYTE_4_KEY_STATE_S
, KEY_VALID
);
1703 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_KEY_M
,
1704 MPT_BYTE_4_KEY_S
, mr
->key
);
1705 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_PAGE_SIZE_M
,
1706 MPT_BYTE_4_PAGE_SIZE_S
, MR_SIZE_4K
);
1707 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_MW_TYPE_S
, 0);
1708 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_MW_BIND_ENABLE_S
,
1709 (mr
->access
& IB_ACCESS_MW_BIND
? 1 : 0));
1710 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_OWN_S
, 0);
1711 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M
,
1712 MPT_BYTE_4_MEMORY_LOCATION_TYPE_S
, mr
->type
);
1713 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_ATOMIC_S
, 0);
1714 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_LOCAL_WRITE_S
,
1715 (mr
->access
& IB_ACCESS_LOCAL_WRITE
? 1 : 0));
1716 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_WRITE_S
,
1717 (mr
->access
& IB_ACCESS_REMOTE_WRITE
? 1 : 0));
1718 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_READ_S
,
1719 (mr
->access
& IB_ACCESS_REMOTE_READ
? 1 : 0));
1720 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S
,
1722 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_ADDRESS_TYPE_S
, 0);
1724 roce_set_field(mpt_entry
->mpt_byte_12
, MPT_BYTE_12_PBL_ADDR_H_M
,
1725 MPT_BYTE_12_PBL_ADDR_H_S
, 0);
1726 roce_set_field(mpt_entry
->mpt_byte_12
, MPT_BYTE_12_MW_BIND_COUNTER_M
,
1727 MPT_BYTE_12_MW_BIND_COUNTER_S
, 0);
1729 mpt_entry
->virt_addr_l
= (u32
)mr
->iova
;
1730 mpt_entry
->virt_addr_h
= (u32
)(mr
->iova
>> 32);
1731 mpt_entry
->length
= (u32
)mr
->size
;
1733 roce_set_field(mpt_entry
->mpt_byte_28
, MPT_BYTE_28_PD_M
,
1734 MPT_BYTE_28_PD_S
, mr
->pd
);
1735 roce_set_field(mpt_entry
->mpt_byte_28
, MPT_BYTE_28_L_KEY_IDX_L_M
,
1736 MPT_BYTE_28_L_KEY_IDX_L_S
, mtpt_idx
);
1737 roce_set_field(mpt_entry
->mpt_byte_64
, MPT_BYTE_64_L_KEY_IDX_H_M
,
1738 MPT_BYTE_64_L_KEY_IDX_H_S
, mtpt_idx
>> MTPT_IDX_SHIFT
);
1740 /* DMA memory register */
1741 if (mr
->type
== MR_TYPE_DMA
)
1744 pages
= (u64
*) __get_free_page(GFP_KERNEL
);
1749 for_each_sg(mr
->umem
->sg_head
.sgl
, sg
, mr
->umem
->nmap
, entry
) {
1750 pages
[i
] = ((u64
)sg_dma_address(sg
)) >> 12;
1752 /* Directly record to MTPT table firstly 7 entry */
1753 if (i
>= HNS_ROCE_MAX_INNER_MTPT_NUM
)
1758 /* Register user mr */
1759 for (i
= 0; i
< HNS_ROCE_MAX_INNER_MTPT_NUM
; i
++) {
1762 mpt_entry
->pa0_l
= cpu_to_le32((u32
)(pages
[i
]));
1763 roce_set_field(mpt_entry
->mpt_byte_36
,
1764 MPT_BYTE_36_PA0_H_M
,
1765 MPT_BYTE_36_PA0_H_S
,
1766 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_32
)));
1769 roce_set_field(mpt_entry
->mpt_byte_36
,
1770 MPT_BYTE_36_PA1_L_M
,
1771 MPT_BYTE_36_PA1_L_S
,
1772 cpu_to_le32((u32
)(pages
[i
])));
1773 roce_set_field(mpt_entry
->mpt_byte_40
,
1774 MPT_BYTE_40_PA1_H_M
,
1775 MPT_BYTE_40_PA1_H_S
,
1776 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_24
)));
1779 roce_set_field(mpt_entry
->mpt_byte_40
,
1780 MPT_BYTE_40_PA2_L_M
,
1781 MPT_BYTE_40_PA2_L_S
,
1782 cpu_to_le32((u32
)(pages
[i
])));
1783 roce_set_field(mpt_entry
->mpt_byte_44
,
1784 MPT_BYTE_44_PA2_H_M
,
1785 MPT_BYTE_44_PA2_H_S
,
1786 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_16
)));
1789 roce_set_field(mpt_entry
->mpt_byte_44
,
1790 MPT_BYTE_44_PA3_L_M
,
1791 MPT_BYTE_44_PA3_L_S
,
1792 cpu_to_le32((u32
)(pages
[i
])));
1793 roce_set_field(mpt_entry
->mpt_byte_48
,
1794 MPT_BYTE_48_PA3_H_M
,
1795 MPT_BYTE_48_PA3_H_S
,
1796 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_8
)));
1799 mpt_entry
->pa4_l
= cpu_to_le32((u32
)(pages
[i
]));
1800 roce_set_field(mpt_entry
->mpt_byte_56
,
1801 MPT_BYTE_56_PA4_H_M
,
1802 MPT_BYTE_56_PA4_H_S
,
1803 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_32
)));
1806 roce_set_field(mpt_entry
->mpt_byte_56
,
1807 MPT_BYTE_56_PA5_L_M
,
1808 MPT_BYTE_56_PA5_L_S
,
1809 cpu_to_le32((u32
)(pages
[i
])));
1810 roce_set_field(mpt_entry
->mpt_byte_60
,
1811 MPT_BYTE_60_PA5_H_M
,
1812 MPT_BYTE_60_PA5_H_S
,
1813 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_24
)));
1816 roce_set_field(mpt_entry
->mpt_byte_60
,
1817 MPT_BYTE_60_PA6_L_M
,
1818 MPT_BYTE_60_PA6_L_S
,
1819 cpu_to_le32((u32
)(pages
[i
])));
1820 roce_set_field(mpt_entry
->mpt_byte_64
,
1821 MPT_BYTE_64_PA6_H_M
,
1822 MPT_BYTE_64_PA6_H_S
,
1823 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_16
)));
1830 free_page((unsigned long) pages
);
1832 mpt_entry
->pbl_addr_l
= (u32
)(mr
->pbl_dma_addr
);
1834 roce_set_field(mpt_entry
->mpt_byte_12
, MPT_BYTE_12_PBL_ADDR_H_M
,
1835 MPT_BYTE_12_PBL_ADDR_H_S
,
1836 ((u32
)(mr
->pbl_dma_addr
>> 32)));
1841 static void *get_cqe(struct hns_roce_cq
*hr_cq
, int n
)
1843 return hns_roce_buf_offset(&hr_cq
->hr_buf
.hr_buf
,
1844 n
* HNS_ROCE_V1_CQE_ENTRY_SIZE
);
1847 static void *get_sw_cqe(struct hns_roce_cq
*hr_cq
, int n
)
1849 struct hns_roce_cqe
*hr_cqe
= get_cqe(hr_cq
, n
& hr_cq
->ib_cq
.cqe
);
1851 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1852 return (roce_get_bit(hr_cqe
->cqe_byte_4
, CQE_BYTE_4_OWNER_S
) ^
1853 !!(n
& (hr_cq
->ib_cq
.cqe
+ 1))) ? hr_cqe
: NULL
;
1856 static struct hns_roce_cqe
*next_cqe_sw(struct hns_roce_cq
*hr_cq
)
1858 return get_sw_cqe(hr_cq
, hr_cq
->cons_index
);
1861 void hns_roce_v1_cq_set_ci(struct hns_roce_cq
*hr_cq
, u32 cons_index
)
1865 doorbell
[0] = cons_index
& ((hr_cq
->cq_depth
<< 1) - 1);
1867 roce_set_bit(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S
, 1);
1868 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M
,
1869 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S
, 3);
1870 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M
,
1871 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S
, 0);
1872 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M
,
1873 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S
, hr_cq
->cqn
);
1875 hns_roce_write64_k(doorbell
, hr_cq
->cq_db_l
);
1878 static void __hns_roce_v1_cq_clean(struct hns_roce_cq
*hr_cq
, u32 qpn
,
1879 struct hns_roce_srq
*srq
)
1881 struct hns_roce_cqe
*cqe
, *dest
;
1886 for (prod_index
= hr_cq
->cons_index
; get_sw_cqe(hr_cq
, prod_index
);
1888 if (prod_index
== hr_cq
->cons_index
+ hr_cq
->ib_cq
.cqe
)
1893 * Now backwards through the CQ, removing CQ entries
1894 * that match our QP by overwriting them with next entries.
1896 while ((int) --prod_index
- (int) hr_cq
->cons_index
>= 0) {
1897 cqe
= get_cqe(hr_cq
, prod_index
& hr_cq
->ib_cq
.cqe
);
1898 if ((roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
1899 CQE_BYTE_16_LOCAL_QPN_S
) &
1900 HNS_ROCE_CQE_QPN_MASK
) == qpn
) {
1901 /* In v1 engine, not support SRQ */
1903 } else if (nfreed
) {
1904 dest
= get_cqe(hr_cq
, (prod_index
+ nfreed
) &
1906 owner_bit
= roce_get_bit(dest
->cqe_byte_4
,
1907 CQE_BYTE_4_OWNER_S
);
1908 memcpy(dest
, cqe
, sizeof(*cqe
));
1909 roce_set_bit(dest
->cqe_byte_4
, CQE_BYTE_4_OWNER_S
,
1915 hr_cq
->cons_index
+= nfreed
;
1917 * Make sure update of buffer contents is done before
1918 * updating consumer index.
1922 hns_roce_v1_cq_set_ci(hr_cq
, hr_cq
->cons_index
);
1926 static void hns_roce_v1_cq_clean(struct hns_roce_cq
*hr_cq
, u32 qpn
,
1927 struct hns_roce_srq
*srq
)
1929 spin_lock_irq(&hr_cq
->lock
);
1930 __hns_roce_v1_cq_clean(hr_cq
, qpn
, srq
);
1931 spin_unlock_irq(&hr_cq
->lock
);
1934 void hns_roce_v1_write_cqc(struct hns_roce_dev
*hr_dev
,
1935 struct hns_roce_cq
*hr_cq
, void *mb_buf
, u64
*mtts
,
1936 dma_addr_t dma_handle
, int nent
, u32 vector
)
1938 struct hns_roce_cq_context
*cq_context
= NULL
;
1939 struct hns_roce_buf_list
*tptr_buf
;
1940 struct hns_roce_v1_priv
*priv
;
1941 dma_addr_t tptr_dma_addr
;
1944 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1945 tptr_buf
= &priv
->tptr_table
.tptr_buf
;
1947 cq_context
= mb_buf
;
1948 memset(cq_context
, 0, sizeof(*cq_context
));
1950 /* Get the tptr for this CQ. */
1951 offset
= hr_cq
->cqn
* HNS_ROCE_V1_TPTR_ENTRY_SIZE
;
1952 tptr_dma_addr
= tptr_buf
->map
+ offset
;
1953 hr_cq
->tptr_addr
= (u16
*)(tptr_buf
->buf
+ offset
);
1955 /* Register cq_context members */
1956 roce_set_field(cq_context
->cqc_byte_4
,
1957 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M
,
1958 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S
, CQ_STATE_VALID
);
1959 roce_set_field(cq_context
->cqc_byte_4
, CQ_CONTEXT_CQC_BYTE_4_CQN_M
,
1960 CQ_CONTEXT_CQC_BYTE_4_CQN_S
, hr_cq
->cqn
);
1961 cq_context
->cqc_byte_4
= cpu_to_le32(cq_context
->cqc_byte_4
);
1963 cq_context
->cq_bt_l
= (u32
)dma_handle
;
1964 cq_context
->cq_bt_l
= cpu_to_le32(cq_context
->cq_bt_l
);
1966 roce_set_field(cq_context
->cqc_byte_12
,
1967 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M
,
1968 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S
,
1969 ((u64
)dma_handle
>> 32));
1970 roce_set_field(cq_context
->cqc_byte_12
,
1971 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M
,
1972 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S
,
1973 ilog2((unsigned int)nent
));
1974 roce_set_field(cq_context
->cqc_byte_12
, CQ_CONTEXT_CQC_BYTE_12_CEQN_M
,
1975 CQ_CONTEXT_CQC_BYTE_12_CEQN_S
, vector
);
1976 cq_context
->cqc_byte_12
= cpu_to_le32(cq_context
->cqc_byte_12
);
1978 cq_context
->cur_cqe_ba0_l
= (u32
)(mtts
[0]);
1979 cq_context
->cur_cqe_ba0_l
= cpu_to_le32(cq_context
->cur_cqe_ba0_l
);
1981 roce_set_field(cq_context
->cqc_byte_20
,
1982 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M
,
1983 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S
,
1984 cpu_to_le32((mtts
[0]) >> 32));
1985 /* Dedicated hardware, directly set 0 */
1986 roce_set_field(cq_context
->cqc_byte_20
,
1987 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M
,
1988 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S
, 0);
1990 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1991 * using 4K page, and shift more 32 because of
1992 * caculating the high 32 bit value evaluated to hardware.
1994 roce_set_field(cq_context
->cqc_byte_20
,
1995 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M
,
1996 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S
,
1997 tptr_dma_addr
>> 44);
1998 cq_context
->cqc_byte_20
= cpu_to_le32(cq_context
->cqc_byte_20
);
2000 cq_context
->cqe_tptr_addr_l
= (u32
)(tptr_dma_addr
>> 12);
2002 roce_set_field(cq_context
->cqc_byte_32
,
2003 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M
,
2004 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S
, 0);
2005 roce_set_bit(cq_context
->cqc_byte_32
,
2006 CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S
, 0);
2007 roce_set_bit(cq_context
->cqc_byte_32
,
2008 CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S
, 0);
2009 roce_set_bit(cq_context
->cqc_byte_32
,
2010 CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S
, 0);
2011 roce_set_bit(cq_context
->cqc_byte_32
,
2012 CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S
,
2014 /* The initial value of cq's ci is 0 */
2015 roce_set_field(cq_context
->cqc_byte_32
,
2016 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M
,
2017 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S
, 0);
2018 cq_context
->cqc_byte_32
= cpu_to_le32(cq_context
->cqc_byte_32
);
2021 int hns_roce_v1_req_notify_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
2023 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
2024 u32 notification_flag
;
2028 notification_flag
= (flags
& IB_CQ_SOLICITED_MASK
) ==
2029 IB_CQ_SOLICITED
? CQ_DB_REQ_NOT
: CQ_DB_REQ_NOT_SOL
;
2031 * flags = 0; Notification Flag = 1, next
2032 * flags = 1; Notification Flag = 0, solocited
2034 doorbell
[0] = hr_cq
->cons_index
& ((hr_cq
->cq_depth
<< 1) - 1);
2035 roce_set_bit(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S
, 1);
2036 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M
,
2037 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S
, 3);
2038 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M
,
2039 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S
, 1);
2040 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M
,
2041 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S
,
2042 hr_cq
->cqn
| notification_flag
);
2044 hns_roce_write64_k(doorbell
, hr_cq
->cq_db_l
);
2049 static int hns_roce_v1_poll_one(struct hns_roce_cq
*hr_cq
,
2050 struct hns_roce_qp
**cur_qp
, struct ib_wc
*wc
)
2057 struct hns_roce_cqe
*cqe
;
2058 struct hns_roce_qp
*hr_qp
;
2059 struct hns_roce_wq
*wq
;
2060 struct hns_roce_wqe_ctrl_seg
*sq_wqe
;
2061 struct hns_roce_dev
*hr_dev
= to_hr_dev(hr_cq
->ib_cq
.device
);
2062 struct device
*dev
= &hr_dev
->pdev
->dev
;
2064 /* Find cqe according consumer index */
2065 cqe
= next_cqe_sw(hr_cq
);
2069 ++hr_cq
->cons_index
;
2070 /* Memory barrier */
2073 is_send
= !(roce_get_bit(cqe
->cqe_byte_4
, CQE_BYTE_4_SQ_RQ_FLAG_S
));
2075 /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
2076 if (roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
2077 CQE_BYTE_16_LOCAL_QPN_S
) <= 1) {
2078 qpn
= roce_get_field(cqe
->cqe_byte_20
, CQE_BYTE_20_PORT_NUM_M
,
2079 CQE_BYTE_20_PORT_NUM_S
) +
2080 roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
2081 CQE_BYTE_16_LOCAL_QPN_S
) *
2084 qpn
= roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
2085 CQE_BYTE_16_LOCAL_QPN_S
);
2088 if (!*cur_qp
|| (qpn
& HNS_ROCE_CQE_QPN_MASK
) != (*cur_qp
)->qpn
) {
2089 hr_qp
= __hns_roce_qp_lookup(hr_dev
, qpn
);
2090 if (unlikely(!hr_qp
)) {
2091 dev_err(dev
, "CQ %06lx with entry for unknown QPN %06x\n",
2092 hr_cq
->cqn
, (qpn
& HNS_ROCE_CQE_QPN_MASK
));
2099 wc
->qp
= &(*cur_qp
)->ibqp
;
2102 status
= roce_get_field(cqe
->cqe_byte_4
,
2103 CQE_BYTE_4_STATUS_OF_THE_OPERATION_M
,
2104 CQE_BYTE_4_STATUS_OF_THE_OPERATION_S
) &
2105 HNS_ROCE_CQE_STATUS_MASK
;
2107 case HNS_ROCE_CQE_SUCCESS
:
2108 wc
->status
= IB_WC_SUCCESS
;
2110 case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR
:
2111 wc
->status
= IB_WC_LOC_LEN_ERR
;
2113 case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR
:
2114 wc
->status
= IB_WC_LOC_QP_OP_ERR
;
2116 case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR
:
2117 wc
->status
= IB_WC_LOC_PROT_ERR
;
2119 case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR
:
2120 wc
->status
= IB_WC_WR_FLUSH_ERR
;
2122 case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR
:
2123 wc
->status
= IB_WC_MW_BIND_ERR
;
2125 case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR
:
2126 wc
->status
= IB_WC_BAD_RESP_ERR
;
2128 case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR
:
2129 wc
->status
= IB_WC_LOC_ACCESS_ERR
;
2131 case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR
:
2132 wc
->status
= IB_WC_REM_INV_REQ_ERR
;
2134 case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR
:
2135 wc
->status
= IB_WC_REM_ACCESS_ERR
;
2137 case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR
:
2138 wc
->status
= IB_WC_REM_OP_ERR
;
2140 case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR
:
2141 wc
->status
= IB_WC_RETRY_EXC_ERR
;
2143 case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR
:
2144 wc
->status
= IB_WC_RNR_RETRY_EXC_ERR
;
2147 wc
->status
= IB_WC_GENERAL_ERR
;
2151 /* CQE status error, directly return */
2152 if (wc
->status
!= IB_WC_SUCCESS
)
2156 /* SQ conrespond to CQE */
2157 sq_wqe
= get_send_wqe(*cur_qp
, roce_get_field(cqe
->cqe_byte_4
,
2158 CQE_BYTE_4_WQE_INDEX_M
,
2159 CQE_BYTE_4_WQE_INDEX_S
)&
2160 ((*cur_qp
)->sq
.wqe_cnt
-1));
2161 switch (sq_wqe
->flag
& HNS_ROCE_WQE_OPCODE_MASK
) {
2162 case HNS_ROCE_WQE_OPCODE_SEND
:
2163 wc
->opcode
= IB_WC_SEND
;
2165 case HNS_ROCE_WQE_OPCODE_RDMA_READ
:
2166 wc
->opcode
= IB_WC_RDMA_READ
;
2167 wc
->byte_len
= le32_to_cpu(cqe
->byte_cnt
);
2169 case HNS_ROCE_WQE_OPCODE_RDMA_WRITE
:
2170 wc
->opcode
= IB_WC_RDMA_WRITE
;
2172 case HNS_ROCE_WQE_OPCODE_LOCAL_INV
:
2173 wc
->opcode
= IB_WC_LOCAL_INV
;
2175 case HNS_ROCE_WQE_OPCODE_UD_SEND
:
2176 wc
->opcode
= IB_WC_SEND
;
2179 wc
->status
= IB_WC_GENERAL_ERR
;
2182 wc
->wc_flags
= (sq_wqe
->flag
& HNS_ROCE_WQE_IMM
?
2183 IB_WC_WITH_IMM
: 0);
2185 wq
= &(*cur_qp
)->sq
;
2186 if ((*cur_qp
)->sq_signal_bits
) {
2188 * If sg_signal_bit is 1,
2189 * firstly tail pointer updated to wqe
2190 * which current cqe correspond to
2192 wqe_ctr
= (u16
)roce_get_field(cqe
->cqe_byte_4
,
2193 CQE_BYTE_4_WQE_INDEX_M
,
2194 CQE_BYTE_4_WQE_INDEX_S
);
2195 wq
->tail
+= (wqe_ctr
- (u16
)wq
->tail
) &
2198 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
2201 /* RQ conrespond to CQE */
2202 wc
->byte_len
= le32_to_cpu(cqe
->byte_cnt
);
2203 opcode
= roce_get_field(cqe
->cqe_byte_4
,
2204 CQE_BYTE_4_OPERATION_TYPE_M
,
2205 CQE_BYTE_4_OPERATION_TYPE_S
) &
2206 HNS_ROCE_CQE_OPCODE_MASK
;
2208 case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE
:
2209 wc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
2210 wc
->wc_flags
= IB_WC_WITH_IMM
;
2211 wc
->ex
.imm_data
= le32_to_cpu(cqe
->immediate_data
);
2213 case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE
:
2214 if (roce_get_bit(cqe
->cqe_byte_4
,
2215 CQE_BYTE_4_IMM_INDICATOR_S
)) {
2216 wc
->opcode
= IB_WC_RECV
;
2217 wc
->wc_flags
= IB_WC_WITH_IMM
;
2218 wc
->ex
.imm_data
= le32_to_cpu(
2219 cqe
->immediate_data
);
2221 wc
->opcode
= IB_WC_RECV
;
2226 wc
->status
= IB_WC_GENERAL_ERR
;
2230 /* Update tail pointer, record wr_id */
2231 wq
= &(*cur_qp
)->rq
;
2232 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
2234 wc
->sl
= (u8
)roce_get_field(cqe
->cqe_byte_20
, CQE_BYTE_20_SL_M
,
2236 wc
->src_qp
= (u8
)roce_get_field(cqe
->cqe_byte_20
,
2237 CQE_BYTE_20_REMOTE_QPN_M
,
2238 CQE_BYTE_20_REMOTE_QPN_S
);
2239 wc
->wc_flags
|= (roce_get_bit(cqe
->cqe_byte_20
,
2240 CQE_BYTE_20_GRH_PRESENT_S
) ?
2242 wc
->pkey_index
= (u16
)roce_get_field(cqe
->cqe_byte_28
,
2243 CQE_BYTE_28_P_KEY_IDX_M
,
2244 CQE_BYTE_28_P_KEY_IDX_S
);
2250 int hns_roce_v1_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
)
2252 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
2253 struct hns_roce_qp
*cur_qp
= NULL
;
2254 unsigned long flags
;
2258 spin_lock_irqsave(&hr_cq
->lock
, flags
);
2260 for (npolled
= 0; npolled
< num_entries
; ++npolled
) {
2261 ret
= hns_roce_v1_poll_one(hr_cq
, &cur_qp
, wc
+ npolled
);
2267 *hr_cq
->tptr_addr
= hr_cq
->cons_index
&
2268 ((hr_cq
->cq_depth
<< 1) - 1);
2270 /* Memroy barrier */
2272 hns_roce_v1_cq_set_ci(hr_cq
, hr_cq
->cons_index
);
2275 spin_unlock_irqrestore(&hr_cq
->lock
, flags
);
2277 if (ret
== 0 || ret
== -EAGAIN
)
2283 int hns_roce_v1_clear_hem(struct hns_roce_dev
*hr_dev
,
2284 struct hns_roce_hem_table
*table
, int obj
)
2286 struct device
*dev
= &hr_dev
->pdev
->dev
;
2287 struct hns_roce_v1_priv
*priv
;
2288 unsigned long end
= 0, flags
= 0;
2289 uint32_t bt_cmd_val
[2] = {0};
2290 void __iomem
*bt_cmd
;
2293 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
2295 switch (table
->type
) {
2297 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
2298 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_QPC
);
2299 bt_ba
= priv
->bt_table
.qpc_buf
.map
>> 12;
2302 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
2303 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_MTPT
);
2304 bt_ba
= priv
->bt_table
.mtpt_buf
.map
>> 12;
2307 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
2308 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_CQC
);
2309 bt_ba
= priv
->bt_table
.cqc_buf
.map
>> 12;
2312 dev_dbg(dev
, "HEM_TYPE_SRQC not support.\n");
2317 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M
,
2318 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S
, obj
);
2319 roce_set_bit(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S
, 0);
2320 roce_set_bit(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S
, 1);
2322 spin_lock_irqsave(&hr_dev
->bt_cmd_lock
, flags
);
2324 bt_cmd
= hr_dev
->reg_base
+ ROCEE_BT_CMD_H_REG
;
2326 end
= msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS
) + jiffies
;
2328 if (readl(bt_cmd
) >> BT_CMD_SYNC_SHIFT
) {
2329 if (!(time_before(jiffies
, end
))) {
2330 dev_err(dev
, "Write bt_cmd err,hw_sync is not zero.\n");
2331 spin_unlock_irqrestore(&hr_dev
->bt_cmd_lock
,
2338 msleep(HW_SYNC_SLEEP_TIME_INTERVAL
);
2341 bt_cmd_val
[0] = (uint32_t)bt_ba
;
2342 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M
,
2343 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S
, bt_ba
>> 32);
2344 hns_roce_write64_k(bt_cmd_val
, hr_dev
->reg_base
+ ROCEE_BT_CMD_L_REG
);
2346 spin_unlock_irqrestore(&hr_dev
->bt_cmd_lock
, flags
);
2351 static int hns_roce_v1_qp_modify(struct hns_roce_dev
*hr_dev
,
2352 struct hns_roce_mtt
*mtt
,
2353 enum hns_roce_qp_state cur_state
,
2354 enum hns_roce_qp_state new_state
,
2355 struct hns_roce_qp_context
*context
,
2356 struct hns_roce_qp
*hr_qp
)
2359 op
[HNS_ROCE_QP_NUM_STATE
][HNS_ROCE_QP_NUM_STATE
] = {
2360 [HNS_ROCE_QP_STATE_RST
] = {
2361 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2362 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2363 [HNS_ROCE_QP_STATE_INIT
] = HNS_ROCE_CMD_RST2INIT_QP
,
2365 [HNS_ROCE_QP_STATE_INIT
] = {
2366 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2367 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2368 /* Note: In v1 engine, HW doesn't support RST2INIT.
2369 * We use RST2INIT cmd instead of INIT2INIT.
2371 [HNS_ROCE_QP_STATE_INIT
] = HNS_ROCE_CMD_RST2INIT_QP
,
2372 [HNS_ROCE_QP_STATE_RTR
] = HNS_ROCE_CMD_INIT2RTR_QP
,
2374 [HNS_ROCE_QP_STATE_RTR
] = {
2375 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2376 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2377 [HNS_ROCE_QP_STATE_RTS
] = HNS_ROCE_CMD_RTR2RTS_QP
,
2379 [HNS_ROCE_QP_STATE_RTS
] = {
2380 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2381 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2382 [HNS_ROCE_QP_STATE_RTS
] = HNS_ROCE_CMD_RTS2RTS_QP
,
2383 [HNS_ROCE_QP_STATE_SQD
] = HNS_ROCE_CMD_RTS2SQD_QP
,
2385 [HNS_ROCE_QP_STATE_SQD
] = {
2386 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2387 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2388 [HNS_ROCE_QP_STATE_RTS
] = HNS_ROCE_CMD_SQD2RTS_QP
,
2389 [HNS_ROCE_QP_STATE_SQD
] = HNS_ROCE_CMD_SQD2SQD_QP
,
2391 [HNS_ROCE_QP_STATE_ERR
] = {
2392 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2393 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2397 struct hns_roce_cmd_mailbox
*mailbox
;
2398 struct device
*dev
= &hr_dev
->pdev
->dev
;
2401 if (cur_state
>= HNS_ROCE_QP_NUM_STATE
||
2402 new_state
>= HNS_ROCE_QP_NUM_STATE
||
2403 !op
[cur_state
][new_state
]) {
2404 dev_err(dev
, "[modify_qp]not support state %d to %d\n",
2405 cur_state
, new_state
);
2409 if (op
[cur_state
][new_state
] == HNS_ROCE_CMD_2RST_QP
)
2410 return hns_roce_cmd_mbox(hr_dev
, 0, 0, hr_qp
->qpn
, 2,
2411 HNS_ROCE_CMD_2RST_QP
,
2412 HNS_ROCE_CMD_TIMEOUT_MSECS
);
2414 if (op
[cur_state
][new_state
] == HNS_ROCE_CMD_2ERR_QP
)
2415 return hns_roce_cmd_mbox(hr_dev
, 0, 0, hr_qp
->qpn
, 2,
2416 HNS_ROCE_CMD_2ERR_QP
,
2417 HNS_ROCE_CMD_TIMEOUT_MSECS
);
2419 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
2420 if (IS_ERR(mailbox
))
2421 return PTR_ERR(mailbox
);
2423 memcpy(mailbox
->buf
, context
, sizeof(*context
));
2425 ret
= hns_roce_cmd_mbox(hr_dev
, mailbox
->dma
, 0, hr_qp
->qpn
, 0,
2426 op
[cur_state
][new_state
],
2427 HNS_ROCE_CMD_TIMEOUT_MSECS
);
2429 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
2433 static int hns_roce_v1_m_sqp(struct ib_qp
*ibqp
, const struct ib_qp_attr
*attr
,
2434 int attr_mask
, enum ib_qp_state cur_state
,
2435 enum ib_qp_state new_state
)
2437 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2438 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2439 struct hns_roce_sqp_context
*context
;
2440 struct device
*dev
= &hr_dev
->pdev
->dev
;
2441 dma_addr_t dma_handle
= 0;
2447 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
2451 /* Search QP buf's MTTs */
2452 mtts
= hns_roce_table_find(&hr_dev
->mr_table
.mtt_table
,
2453 hr_qp
->mtt
.first_seg
, &dma_handle
);
2455 dev_err(dev
, "qp buf pa find failed\n");
2459 if (cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
) {
2460 roce_set_field(context
->qp1c_bytes_4
,
2461 QP1C_BYTES_4_SQ_WQE_SHIFT_M
,
2462 QP1C_BYTES_4_SQ_WQE_SHIFT_S
,
2463 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
2464 roce_set_field(context
->qp1c_bytes_4
,
2465 QP1C_BYTES_4_RQ_WQE_SHIFT_M
,
2466 QP1C_BYTES_4_RQ_WQE_SHIFT_S
,
2467 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
2468 roce_set_field(context
->qp1c_bytes_4
, QP1C_BYTES_4_PD_M
,
2469 QP1C_BYTES_4_PD_S
, to_hr_pd(ibqp
->pd
)->pdn
);
2471 context
->sq_rq_bt_l
= (u32
)(dma_handle
);
2472 roce_set_field(context
->qp1c_bytes_12
,
2473 QP1C_BYTES_12_SQ_RQ_BT_H_M
,
2474 QP1C_BYTES_12_SQ_RQ_BT_H_S
,
2475 ((u32
)(dma_handle
>> 32)));
2477 roce_set_field(context
->qp1c_bytes_16
, QP1C_BYTES_16_RQ_HEAD_M
,
2478 QP1C_BYTES_16_RQ_HEAD_S
, hr_qp
->rq
.head
);
2479 roce_set_field(context
->qp1c_bytes_16
, QP1C_BYTES_16_PORT_NUM_M
,
2480 QP1C_BYTES_16_PORT_NUM_S
, hr_qp
->phy_port
);
2481 roce_set_bit(context
->qp1c_bytes_16
,
2482 QP1C_BYTES_16_SIGNALING_TYPE_S
,
2483 hr_qp
->sq_signal_bits
);
2484 roce_set_bit(context
->qp1c_bytes_16
, QP1C_BYTES_16_RQ_BA_FLG_S
,
2486 roce_set_bit(context
->qp1c_bytes_16
, QP1C_BYTES_16_SQ_BA_FLG_S
,
2488 roce_set_bit(context
->qp1c_bytes_16
, QP1C_BYTES_16_QP1_ERR_S
,
2491 roce_set_field(context
->qp1c_bytes_20
, QP1C_BYTES_20_SQ_HEAD_M
,
2492 QP1C_BYTES_20_SQ_HEAD_S
, hr_qp
->sq
.head
);
2493 roce_set_field(context
->qp1c_bytes_20
, QP1C_BYTES_20_PKEY_IDX_M
,
2494 QP1C_BYTES_20_PKEY_IDX_S
, attr
->pkey_index
);
2496 rq_pa_start
= (u32
)hr_qp
->rq
.offset
/ PAGE_SIZE
;
2497 context
->cur_rq_wqe_ba_l
= (u32
)(mtts
[rq_pa_start
]);
2499 roce_set_field(context
->qp1c_bytes_28
,
2500 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M
,
2501 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S
,
2502 (mtts
[rq_pa_start
]) >> 32);
2503 roce_set_field(context
->qp1c_bytes_28
,
2504 QP1C_BYTES_28_RQ_CUR_IDX_M
,
2505 QP1C_BYTES_28_RQ_CUR_IDX_S
, 0);
2507 roce_set_field(context
->qp1c_bytes_32
,
2508 QP1C_BYTES_32_RX_CQ_NUM_M
,
2509 QP1C_BYTES_32_RX_CQ_NUM_S
,
2510 to_hr_cq(ibqp
->recv_cq
)->cqn
);
2511 roce_set_field(context
->qp1c_bytes_32
,
2512 QP1C_BYTES_32_TX_CQ_NUM_M
,
2513 QP1C_BYTES_32_TX_CQ_NUM_S
,
2514 to_hr_cq(ibqp
->send_cq
)->cqn
);
2516 context
->cur_sq_wqe_ba_l
= (u32
)mtts
[0];
2518 roce_set_field(context
->qp1c_bytes_40
,
2519 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M
,
2520 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S
,
2522 roce_set_field(context
->qp1c_bytes_40
,
2523 QP1C_BYTES_40_SQ_CUR_IDX_M
,
2524 QP1C_BYTES_40_SQ_CUR_IDX_S
, 0);
2526 /* Copy context to QP1C register */
2527 addr
= (u32
*)(hr_dev
->reg_base
+ ROCEE_QP1C_CFG0_0_REG
+
2528 hr_qp
->phy_port
* sizeof(*context
));
2530 writel(context
->qp1c_bytes_4
, addr
);
2531 writel(context
->sq_rq_bt_l
, addr
+ 1);
2532 writel(context
->qp1c_bytes_12
, addr
+ 2);
2533 writel(context
->qp1c_bytes_16
, addr
+ 3);
2534 writel(context
->qp1c_bytes_20
, addr
+ 4);
2535 writel(context
->cur_rq_wqe_ba_l
, addr
+ 5);
2536 writel(context
->qp1c_bytes_28
, addr
+ 6);
2537 writel(context
->qp1c_bytes_32
, addr
+ 7);
2538 writel(context
->cur_sq_wqe_ba_l
, addr
+ 8);
2539 writel(context
->qp1c_bytes_40
, addr
+ 9);
2542 /* Modify QP1C status */
2543 reg_val
= roce_read(hr_dev
, ROCEE_QP1C_CFG0_0_REG
+
2544 hr_qp
->phy_port
* sizeof(*context
));
2545 roce_set_field(reg_val
, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M
,
2546 ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S
, new_state
);
2547 roce_write(hr_dev
, ROCEE_QP1C_CFG0_0_REG
+
2548 hr_qp
->phy_port
* sizeof(*context
), reg_val
);
2550 hr_qp
->state
= new_state
;
2551 if (new_state
== IB_QPS_RESET
) {
2552 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->recv_cq
), hr_qp
->qpn
,
2553 ibqp
->srq
? to_hr_srq(ibqp
->srq
) : NULL
);
2554 if (ibqp
->send_cq
!= ibqp
->recv_cq
)
2555 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->send_cq
),
2562 hr_qp
->sq_next_wqe
= 0;
2573 static int hns_roce_v1_m_qp(struct ib_qp
*ibqp
, const struct ib_qp_attr
*attr
,
2574 int attr_mask
, enum ib_qp_state cur_state
,
2575 enum ib_qp_state new_state
)
2577 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2578 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2579 struct device
*dev
= &hr_dev
->pdev
->dev
;
2580 struct hns_roce_qp_context
*context
;
2581 const struct ib_global_route
*grh
= rdma_ah_read_grh(&attr
->ah_attr
);
2582 dma_addr_t dma_handle_2
= 0;
2583 dma_addr_t dma_handle
= 0;
2584 uint32_t doorbell
[2] = {0};
2585 int rq_pa_start
= 0;
2594 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
2598 /* Search qp buf's mtts */
2599 mtts
= hns_roce_table_find(&hr_dev
->mr_table
.mtt_table
,
2600 hr_qp
->mtt
.first_seg
, &dma_handle
);
2602 dev_err(dev
, "qp buf pa find failed\n");
2606 /* Search IRRL's mtts */
2607 mtts_2
= hns_roce_table_find(&hr_dev
->qp_table
.irrl_table
, hr_qp
->qpn
,
2609 if (mtts_2
== NULL
) {
2610 dev_err(dev
, "qp irrl_table find failed\n");
2617 * IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
2618 * Optional param: NA
2620 if (cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
) {
2621 roce_set_field(context
->qpc_bytes_4
,
2622 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M
,
2623 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S
,
2624 to_hr_qp_type(hr_qp
->ibqp
.qp_type
));
2626 roce_set_bit(context
->qpc_bytes_4
,
2627 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S
, 0);
2628 roce_set_bit(context
->qpc_bytes_4
,
2629 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
,
2630 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
));
2631 roce_set_bit(context
->qpc_bytes_4
,
2632 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
,
2633 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
)
2635 roce_set_bit(context
->qpc_bytes_4
,
2636 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S
,
2637 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_ATOMIC
)
2639 roce_set_bit(context
->qpc_bytes_4
,
2640 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S
, 1);
2641 roce_set_field(context
->qpc_bytes_4
,
2642 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M
,
2643 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S
,
2644 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
2645 roce_set_field(context
->qpc_bytes_4
,
2646 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M
,
2647 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S
,
2648 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
2649 roce_set_field(context
->qpc_bytes_4
,
2650 QP_CONTEXT_QPC_BYTES_4_PD_M
,
2651 QP_CONTEXT_QPC_BYTES_4_PD_S
,
2652 to_hr_pd(ibqp
->pd
)->pdn
);
2653 hr_qp
->access_flags
= attr
->qp_access_flags
;
2654 roce_set_field(context
->qpc_bytes_8
,
2655 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M
,
2656 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S
,
2657 to_hr_cq(ibqp
->send_cq
)->cqn
);
2658 roce_set_field(context
->qpc_bytes_8
,
2659 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M
,
2660 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S
,
2661 to_hr_cq(ibqp
->recv_cq
)->cqn
);
2664 roce_set_field(context
->qpc_bytes_12
,
2665 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M
,
2666 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S
,
2667 to_hr_srq(ibqp
->srq
)->srqn
);
2669 roce_set_field(context
->qpc_bytes_12
,
2670 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
2671 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
,
2673 hr_qp
->pkey_index
= attr
->pkey_index
;
2674 roce_set_field(context
->qpc_bytes_16
,
2675 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M
,
2676 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S
, hr_qp
->qpn
);
2678 } else if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_INIT
) {
2679 roce_set_field(context
->qpc_bytes_4
,
2680 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M
,
2681 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S
,
2682 to_hr_qp_type(hr_qp
->ibqp
.qp_type
));
2683 roce_set_bit(context
->qpc_bytes_4
,
2684 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S
, 0);
2685 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
2686 roce_set_bit(context
->qpc_bytes_4
,
2687 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
,
2688 !!(attr
->qp_access_flags
&
2689 IB_ACCESS_REMOTE_READ
));
2690 roce_set_bit(context
->qpc_bytes_4
,
2691 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
,
2692 !!(attr
->qp_access_flags
&
2693 IB_ACCESS_REMOTE_WRITE
));
2695 roce_set_bit(context
->qpc_bytes_4
,
2696 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
,
2697 !!(hr_qp
->access_flags
&
2698 IB_ACCESS_REMOTE_READ
));
2699 roce_set_bit(context
->qpc_bytes_4
,
2700 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
,
2701 !!(hr_qp
->access_flags
&
2702 IB_ACCESS_REMOTE_WRITE
));
2705 roce_set_bit(context
->qpc_bytes_4
,
2706 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S
, 1);
2707 roce_set_field(context
->qpc_bytes_4
,
2708 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M
,
2709 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S
,
2710 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
2711 roce_set_field(context
->qpc_bytes_4
,
2712 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M
,
2713 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S
,
2714 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
2715 roce_set_field(context
->qpc_bytes_4
,
2716 QP_CONTEXT_QPC_BYTES_4_PD_M
,
2717 QP_CONTEXT_QPC_BYTES_4_PD_S
,
2718 to_hr_pd(ibqp
->pd
)->pdn
);
2720 roce_set_field(context
->qpc_bytes_8
,
2721 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M
,
2722 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S
,
2723 to_hr_cq(ibqp
->send_cq
)->cqn
);
2724 roce_set_field(context
->qpc_bytes_8
,
2725 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M
,
2726 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S
,
2727 to_hr_cq(ibqp
->recv_cq
)->cqn
);
2730 roce_set_field(context
->qpc_bytes_12
,
2731 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M
,
2732 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S
,
2733 to_hr_srq(ibqp
->srq
)->srqn
);
2734 if (attr_mask
& IB_QP_PKEY_INDEX
)
2735 roce_set_field(context
->qpc_bytes_12
,
2736 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
2737 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
,
2740 roce_set_field(context
->qpc_bytes_12
,
2741 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
2742 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
,
2745 roce_set_field(context
->qpc_bytes_16
,
2746 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M
,
2747 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S
, hr_qp
->qpn
);
2748 } else if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_RTR
) {
2749 if ((attr_mask
& IB_QP_ALT_PATH
) ||
2750 (attr_mask
& IB_QP_ACCESS_FLAGS
) ||
2751 (attr_mask
& IB_QP_PKEY_INDEX
) ||
2752 (attr_mask
& IB_QP_QKEY
)) {
2753 dev_err(dev
, "INIT2RTR attr_mask error\n");
2757 dmac
= (u8
*)attr
->ah_attr
.roce
.dmac
;
2759 context
->sq_rq_bt_l
= (u32
)(dma_handle
);
2760 roce_set_field(context
->qpc_bytes_24
,
2761 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M
,
2762 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S
,
2763 ((u32
)(dma_handle
>> 32)));
2764 roce_set_bit(context
->qpc_bytes_24
,
2765 QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S
,
2767 roce_set_field(context
->qpc_bytes_24
,
2768 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M
,
2769 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S
,
2770 attr
->min_rnr_timer
);
2771 context
->irrl_ba_l
= (u32
)(dma_handle_2
);
2772 roce_set_field(context
->qpc_bytes_32
,
2773 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M
,
2774 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S
,
2775 ((u32
)(dma_handle_2
>> 32)) &
2776 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M
);
2777 roce_set_field(context
->qpc_bytes_32
,
2778 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M
,
2779 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S
, 0);
2780 roce_set_bit(context
->qpc_bytes_32
,
2781 QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S
,
2783 roce_set_bit(context
->qpc_bytes_32
,
2784 QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S
,
2785 hr_qp
->sq_signal_bits
);
2787 port
= (attr_mask
& IB_QP_PORT
) ? (attr
->port_num
- 1) :
2789 smac
= (u8
*)hr_dev
->dev_addr
[port
];
2790 /* when dmac equals smac or loop_idc is 1, it should loopback */
2791 if (ether_addr_equal_unaligned(dmac
, smac
) ||
2792 hr_dev
->loop_idc
== 0x1)
2793 roce_set_bit(context
->qpc_bytes_32
,
2794 QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S
, 1);
2796 roce_set_bit(context
->qpc_bytes_32
,
2797 QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S
,
2798 rdma_ah_get_ah_flags(&attr
->ah_attr
));
2799 roce_set_field(context
->qpc_bytes_32
,
2800 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M
,
2801 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S
,
2802 ilog2((unsigned int)attr
->max_dest_rd_atomic
));
2804 roce_set_field(context
->qpc_bytes_36
,
2805 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M
,
2806 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S
,
2809 /* Configure GID index */
2810 port_num
= rdma_ah_get_port_num(&attr
->ah_attr
);
2811 roce_set_field(context
->qpc_bytes_36
,
2812 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M
,
2813 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S
,
2814 hns_get_gid_index(hr_dev
,
2818 memcpy(&(context
->dmac_l
), dmac
, 4);
2820 roce_set_field(context
->qpc_bytes_44
,
2821 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M
,
2822 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S
,
2823 *((u16
*)(&dmac
[4])));
2824 roce_set_field(context
->qpc_bytes_44
,
2825 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M
,
2826 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S
,
2827 rdma_ah_get_static_rate(&attr
->ah_attr
));
2828 roce_set_field(context
->qpc_bytes_44
,
2829 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M
,
2830 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S
,
2833 roce_set_field(context
->qpc_bytes_48
,
2834 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M
,
2835 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S
,
2837 roce_set_field(context
->qpc_bytes_48
,
2838 QP_CONTEXT_QPC_BYTES_48_TCLASS_M
,
2839 QP_CONTEXT_QPC_BYTES_48_TCLASS_S
,
2840 grh
->traffic_class
);
2841 roce_set_field(context
->qpc_bytes_48
,
2842 QP_CONTEXT_QPC_BYTES_48_MTU_M
,
2843 QP_CONTEXT_QPC_BYTES_48_MTU_S
, attr
->path_mtu
);
2845 memcpy(context
->dgid
, grh
->dgid
.raw
,
2846 sizeof(grh
->dgid
.raw
));
2848 dev_dbg(dev
, "dmac:%x :%lx\n", context
->dmac_l
,
2849 roce_get_field(context
->qpc_bytes_44
,
2850 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M
,
2851 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S
));
2853 roce_set_field(context
->qpc_bytes_68
,
2854 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M
,
2855 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S
,
2857 roce_set_field(context
->qpc_bytes_68
,
2858 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M
,
2859 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S
, 0);
2861 rq_pa_start
= (u32
)hr_qp
->rq
.offset
/ PAGE_SIZE
;
2862 context
->cur_rq_wqe_ba_l
= (u32
)(mtts
[rq_pa_start
]);
2864 roce_set_field(context
->qpc_bytes_76
,
2865 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M
,
2866 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S
,
2867 mtts
[rq_pa_start
] >> 32);
2868 roce_set_field(context
->qpc_bytes_76
,
2869 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M
,
2870 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S
, 0);
2872 context
->rx_rnr_time
= 0;
2874 roce_set_field(context
->qpc_bytes_84
,
2875 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M
,
2876 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S
,
2878 roce_set_field(context
->qpc_bytes_84
,
2879 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M
,
2880 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S
, 0);
2882 roce_set_field(context
->qpc_bytes_88
,
2883 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M
,
2884 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S
,
2886 roce_set_bit(context
->qpc_bytes_88
,
2887 QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S
, 0);
2888 roce_set_bit(context
->qpc_bytes_88
,
2889 QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S
, 0);
2890 roce_set_field(context
->qpc_bytes_88
,
2891 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M
,
2892 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S
,
2894 roce_set_field(context
->qpc_bytes_88
,
2895 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M
,
2896 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S
,
2899 context
->dma_length
= 0;
2904 roce_set_field(context
->qpc_bytes_108
,
2905 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M
,
2906 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S
, 0);
2907 roce_set_bit(context
->qpc_bytes_108
,
2908 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S
, 0);
2909 roce_set_bit(context
->qpc_bytes_108
,
2910 QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S
, 0);
2912 roce_set_field(context
->qpc_bytes_112
,
2913 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M
,
2914 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S
, 0);
2915 roce_set_field(context
->qpc_bytes_112
,
2916 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M
,
2917 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S
, 0);
2919 /* For chip resp ack */
2920 roce_set_field(context
->qpc_bytes_156
,
2921 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M
,
2922 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S
,
2924 roce_set_field(context
->qpc_bytes_156
,
2925 QP_CONTEXT_QPC_BYTES_156_SL_M
,
2926 QP_CONTEXT_QPC_BYTES_156_SL_S
,
2927 rdma_ah_get_sl(&attr
->ah_attr
));
2928 hr_qp
->sl
= rdma_ah_get_sl(&attr
->ah_attr
);
2929 } else if (cur_state
== IB_QPS_RTR
&&
2930 new_state
== IB_QPS_RTS
) {
2931 /* If exist optional param, return error */
2932 if ((attr_mask
& IB_QP_ALT_PATH
) ||
2933 (attr_mask
& IB_QP_ACCESS_FLAGS
) ||
2934 (attr_mask
& IB_QP_QKEY
) ||
2935 (attr_mask
& IB_QP_PATH_MIG_STATE
) ||
2936 (attr_mask
& IB_QP_CUR_STATE
) ||
2937 (attr_mask
& IB_QP_MIN_RNR_TIMER
)) {
2938 dev_err(dev
, "RTR2RTS attr_mask error\n");
2942 context
->rx_cur_sq_wqe_ba_l
= (u32
)(mtts
[0]);
2944 roce_set_field(context
->qpc_bytes_120
,
2945 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M
,
2946 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S
,
2949 roce_set_field(context
->qpc_bytes_124
,
2950 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M
,
2951 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S
, 0);
2952 roce_set_field(context
->qpc_bytes_124
,
2953 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M
,
2954 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S
, 0);
2956 roce_set_field(context
->qpc_bytes_128
,
2957 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M
,
2958 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S
,
2960 roce_set_bit(context
->qpc_bytes_128
,
2961 QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S
, 0);
2962 roce_set_field(context
->qpc_bytes_128
,
2963 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M
,
2964 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S
,
2966 roce_set_bit(context
->qpc_bytes_128
,
2967 QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S
, 0);
2969 roce_set_field(context
->qpc_bytes_132
,
2970 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M
,
2971 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S
, 0);
2972 roce_set_field(context
->qpc_bytes_132
,
2973 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M
,
2974 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S
, 0);
2976 roce_set_field(context
->qpc_bytes_136
,
2977 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M
,
2978 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S
,
2980 roce_set_field(context
->qpc_bytes_136
,
2981 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M
,
2982 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S
,
2985 roce_set_field(context
->qpc_bytes_140
,
2986 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M
,
2987 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S
,
2988 (attr
->sq_psn
>> SQ_PSN_SHIFT
));
2989 roce_set_field(context
->qpc_bytes_140
,
2990 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M
,
2991 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S
, 0);
2992 roce_set_bit(context
->qpc_bytes_140
,
2993 QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S
, 0);
2995 roce_set_field(context
->qpc_bytes_148
,
2996 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M
,
2997 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S
, 0);
2998 roce_set_field(context
->qpc_bytes_148
,
2999 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M
,
3000 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S
,
3002 roce_set_field(context
->qpc_bytes_148
,
3003 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M
,
3004 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S
,
3006 roce_set_field(context
->qpc_bytes_148
,
3007 QP_CONTEXT_QPC_BYTES_148_LSN_M
,
3008 QP_CONTEXT_QPC_BYTES_148_LSN_S
, 0x100);
3010 context
->rnr_retry
= 0;
3012 roce_set_field(context
->qpc_bytes_156
,
3013 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M
,
3014 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S
,
3016 if (attr
->timeout
< 0x12) {
3017 dev_info(dev
, "ack timeout value(0x%x) must bigger than 0x12.\n",
3019 roce_set_field(context
->qpc_bytes_156
,
3020 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M
,
3021 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S
,
3024 roce_set_field(context
->qpc_bytes_156
,
3025 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M
,
3026 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S
,
3029 roce_set_field(context
->qpc_bytes_156
,
3030 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M
,
3031 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S
,
3033 roce_set_field(context
->qpc_bytes_156
,
3034 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M
,
3035 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S
,
3037 roce_set_field(context
->qpc_bytes_156
,
3038 QP_CONTEXT_QPC_BYTES_156_SL_M
,
3039 QP_CONTEXT_QPC_BYTES_156_SL_S
,
3040 rdma_ah_get_sl(&attr
->ah_attr
));
3041 hr_qp
->sl
= rdma_ah_get_sl(&attr
->ah_attr
);
3042 roce_set_field(context
->qpc_bytes_156
,
3043 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M
,
3044 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S
,
3045 ilog2((unsigned int)attr
->max_rd_atomic
));
3046 roce_set_field(context
->qpc_bytes_156
,
3047 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M
,
3048 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S
, 0);
3049 context
->pkt_use_len
= 0;
3051 roce_set_field(context
->qpc_bytes_164
,
3052 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M
,
3053 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S
, attr
->sq_psn
);
3054 roce_set_field(context
->qpc_bytes_164
,
3055 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M
,
3056 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S
, 0);
3058 roce_set_field(context
->qpc_bytes_168
,
3059 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M
,
3060 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S
,
3062 roce_set_field(context
->qpc_bytes_168
,
3063 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M
,
3064 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S
, 0);
3065 roce_set_field(context
->qpc_bytes_168
,
3066 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M
,
3067 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S
, 0);
3068 roce_set_bit(context
->qpc_bytes_168
,
3069 QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S
, 0);
3070 roce_set_bit(context
->qpc_bytes_168
,
3071 QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S
, 0);
3072 roce_set_bit(context
->qpc_bytes_168
,
3073 QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S
, 0);
3074 context
->sge_use_len
= 0;
3076 roce_set_field(context
->qpc_bytes_176
,
3077 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M
,
3078 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S
, 0);
3079 roce_set_field(context
->qpc_bytes_176
,
3080 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M
,
3081 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S
,
3083 roce_set_field(context
->qpc_bytes_180
,
3084 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M
,
3085 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S
, 0);
3086 roce_set_field(context
->qpc_bytes_180
,
3087 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M
,
3088 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S
, 0);
3090 context
->tx_cur_sq_wqe_ba_l
= (u32
)(mtts
[0]);
3092 roce_set_field(context
->qpc_bytes_188
,
3093 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M
,
3094 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S
,
3096 roce_set_bit(context
->qpc_bytes_188
,
3097 QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S
, 0);
3098 roce_set_field(context
->qpc_bytes_188
,
3099 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M
,
3100 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S
,
3102 } else if (!((cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_RESET
) ||
3103 (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_ERR
) ||
3104 (cur_state
== IB_QPS_RTR
&& new_state
== IB_QPS_RESET
) ||
3105 (cur_state
== IB_QPS_RTR
&& new_state
== IB_QPS_ERR
) ||
3106 (cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_RESET
) ||
3107 (cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_ERR
) ||
3108 (cur_state
== IB_QPS_ERR
&& new_state
== IB_QPS_RESET
) ||
3109 (cur_state
== IB_QPS_ERR
&& new_state
== IB_QPS_ERR
))) {
3110 dev_err(dev
, "not support this status migration\n");
3114 /* Every status migrate must change state */
3115 roce_set_field(context
->qpc_bytes_144
,
3116 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M
,
3117 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S
, new_state
);
3119 /* SW pass context to HW */
3120 ret
= hns_roce_v1_qp_modify(hr_dev
, &hr_qp
->mtt
,
3121 to_hns_roce_state(cur_state
),
3122 to_hns_roce_state(new_state
), context
,
3125 dev_err(dev
, "hns_roce_qp_modify failed\n");
3130 * Use rst2init to instead of init2init with drv,
3131 * need to hw to flash RQ HEAD by DB again
3133 if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_INIT
) {
3134 /* Memory barrier */
3137 roce_set_field(doorbell
[0], RQ_DOORBELL_U32_4_RQ_HEAD_M
,
3138 RQ_DOORBELL_U32_4_RQ_HEAD_S
, hr_qp
->rq
.head
);
3139 roce_set_field(doorbell
[1], RQ_DOORBELL_U32_8_QPN_M
,
3140 RQ_DOORBELL_U32_8_QPN_S
, hr_qp
->qpn
);
3141 roce_set_field(doorbell
[1], RQ_DOORBELL_U32_8_CMD_M
,
3142 RQ_DOORBELL_U32_8_CMD_S
, 1);
3143 roce_set_bit(doorbell
[1], RQ_DOORBELL_U32_8_HW_SYNC_S
, 1);
3145 if (ibqp
->uobject
) {
3146 hr_qp
->rq
.db_reg_l
= hr_dev
->reg_base
+
3147 ROCEE_DB_OTHERS_L_0_REG
+
3148 DB_REG_OFFSET
* hr_dev
->priv_uar
.index
;
3151 hns_roce_write64_k(doorbell
, hr_qp
->rq
.db_reg_l
);
3154 hr_qp
->state
= new_state
;
3156 if (attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
)
3157 hr_qp
->resp_depth
= attr
->max_dest_rd_atomic
;
3158 if (attr_mask
& IB_QP_PORT
) {
3159 hr_qp
->port
= attr
->port_num
- 1;
3160 hr_qp
->phy_port
= hr_dev
->iboe
.phy_port
[hr_qp
->port
];
3163 if (new_state
== IB_QPS_RESET
&& !ibqp
->uobject
) {
3164 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->recv_cq
), hr_qp
->qpn
,
3165 ibqp
->srq
? to_hr_srq(ibqp
->srq
) : NULL
);
3166 if (ibqp
->send_cq
!= ibqp
->recv_cq
)
3167 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->send_cq
),
3174 hr_qp
->sq_next_wqe
= 0;
3181 int hns_roce_v1_modify_qp(struct ib_qp
*ibqp
, const struct ib_qp_attr
*attr
,
3182 int attr_mask
, enum ib_qp_state cur_state
,
3183 enum ib_qp_state new_state
)
3186 if (ibqp
->qp_type
== IB_QPT_GSI
|| ibqp
->qp_type
== IB_QPT_SMI
)
3187 return hns_roce_v1_m_sqp(ibqp
, attr
, attr_mask
, cur_state
,
3190 return hns_roce_v1_m_qp(ibqp
, attr
, attr_mask
, cur_state
,
3194 static enum ib_qp_state
to_ib_qp_state(enum hns_roce_qp_state state
)
3197 case HNS_ROCE_QP_STATE_RST
:
3198 return IB_QPS_RESET
;
3199 case HNS_ROCE_QP_STATE_INIT
:
3201 case HNS_ROCE_QP_STATE_RTR
:
3203 case HNS_ROCE_QP_STATE_RTS
:
3205 case HNS_ROCE_QP_STATE_SQD
:
3207 case HNS_ROCE_QP_STATE_ERR
:
3214 static int hns_roce_v1_query_qpc(struct hns_roce_dev
*hr_dev
,
3215 struct hns_roce_qp
*hr_qp
,
3216 struct hns_roce_qp_context
*hr_context
)
3218 struct hns_roce_cmd_mailbox
*mailbox
;
3221 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
3222 if (IS_ERR(mailbox
))
3223 return PTR_ERR(mailbox
);
3225 ret
= hns_roce_cmd_mbox(hr_dev
, 0, mailbox
->dma
, hr_qp
->qpn
, 0,
3226 HNS_ROCE_CMD_QUERY_QP
,
3227 HNS_ROCE_CMD_TIMEOUT_MSECS
);
3229 memcpy(hr_context
, mailbox
->buf
, sizeof(*hr_context
));
3231 dev_err(&hr_dev
->pdev
->dev
, "QUERY QP cmd process error\n");
3233 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
3238 static int hns_roce_v1_q_sqp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
,
3240 struct ib_qp_init_attr
*qp_init_attr
)
3242 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
3243 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
3244 struct hns_roce_sqp_context context
;
3247 mutex_lock(&hr_qp
->mutex
);
3249 if (hr_qp
->state
== IB_QPS_RESET
) {
3250 qp_attr
->qp_state
= IB_QPS_RESET
;
3254 addr
= ROCEE_QP1C_CFG0_0_REG
+
3255 hr_qp
->port
* sizeof(struct hns_roce_sqp_context
);
3256 context
.qp1c_bytes_4
= roce_read(hr_dev
, addr
);
3257 context
.sq_rq_bt_l
= roce_read(hr_dev
, addr
+ 1);
3258 context
.qp1c_bytes_12
= roce_read(hr_dev
, addr
+ 2);
3259 context
.qp1c_bytes_16
= roce_read(hr_dev
, addr
+ 3);
3260 context
.qp1c_bytes_20
= roce_read(hr_dev
, addr
+ 4);
3261 context
.cur_rq_wqe_ba_l
= roce_read(hr_dev
, addr
+ 5);
3262 context
.qp1c_bytes_28
= roce_read(hr_dev
, addr
+ 6);
3263 context
.qp1c_bytes_32
= roce_read(hr_dev
, addr
+ 7);
3264 context
.cur_sq_wqe_ba_l
= roce_read(hr_dev
, addr
+ 8);
3265 context
.qp1c_bytes_40
= roce_read(hr_dev
, addr
+ 9);
3267 hr_qp
->state
= roce_get_field(context
.qp1c_bytes_4
,
3268 QP1C_BYTES_4_QP_STATE_M
,
3269 QP1C_BYTES_4_QP_STATE_S
);
3270 qp_attr
->qp_state
= hr_qp
->state
;
3271 qp_attr
->path_mtu
= IB_MTU_256
;
3272 qp_attr
->path_mig_state
= IB_MIG_ARMED
;
3273 qp_attr
->qkey
= QKEY_VAL
;
3274 qp_attr
->rq_psn
= 0;
3275 qp_attr
->sq_psn
= 0;
3276 qp_attr
->dest_qp_num
= 1;
3277 qp_attr
->qp_access_flags
= 6;
3279 qp_attr
->pkey_index
= roce_get_field(context
.qp1c_bytes_20
,
3280 QP1C_BYTES_20_PKEY_IDX_M
,
3281 QP1C_BYTES_20_PKEY_IDX_S
);
3282 qp_attr
->port_num
= hr_qp
->port
+ 1;
3283 qp_attr
->sq_draining
= 0;
3284 qp_attr
->max_rd_atomic
= 0;
3285 qp_attr
->max_dest_rd_atomic
= 0;
3286 qp_attr
->min_rnr_timer
= 0;
3287 qp_attr
->timeout
= 0;
3288 qp_attr
->retry_cnt
= 0;
3289 qp_attr
->rnr_retry
= 0;
3290 qp_attr
->alt_timeout
= 0;
3293 qp_attr
->cur_qp_state
= qp_attr
->qp_state
;
3294 qp_attr
->cap
.max_recv_wr
= hr_qp
->rq
.wqe_cnt
;
3295 qp_attr
->cap
.max_recv_sge
= hr_qp
->rq
.max_gs
;
3296 qp_attr
->cap
.max_send_wr
= hr_qp
->sq
.wqe_cnt
;
3297 qp_attr
->cap
.max_send_sge
= hr_qp
->sq
.max_gs
;
3298 qp_attr
->cap
.max_inline_data
= 0;
3299 qp_init_attr
->cap
= qp_attr
->cap
;
3300 qp_init_attr
->create_flags
= 0;
3302 mutex_unlock(&hr_qp
->mutex
);
3307 static int hns_roce_v1_q_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
,
3309 struct ib_qp_init_attr
*qp_init_attr
)
3311 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
3312 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
3313 struct device
*dev
= &hr_dev
->pdev
->dev
;
3314 struct hns_roce_qp_context
*context
;
3315 int tmp_qp_state
= 0;
3319 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
3323 memset(qp_attr
, 0, sizeof(*qp_attr
));
3324 memset(qp_init_attr
, 0, sizeof(*qp_init_attr
));
3326 mutex_lock(&hr_qp
->mutex
);
3328 if (hr_qp
->state
== IB_QPS_RESET
) {
3329 qp_attr
->qp_state
= IB_QPS_RESET
;
3333 ret
= hns_roce_v1_query_qpc(hr_dev
, hr_qp
, context
);
3335 dev_err(dev
, "query qpc error\n");
3340 state
= roce_get_field(context
->qpc_bytes_144
,
3341 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M
,
3342 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S
);
3343 tmp_qp_state
= (int)to_ib_qp_state((enum hns_roce_qp_state
)state
);
3344 if (tmp_qp_state
== -1) {
3345 dev_err(dev
, "to_ib_qp_state error\n");
3349 hr_qp
->state
= (u8
)tmp_qp_state
;
3350 qp_attr
->qp_state
= (enum ib_qp_state
)hr_qp
->state
;
3351 qp_attr
->path_mtu
= (enum ib_mtu
)roce_get_field(context
->qpc_bytes_48
,
3352 QP_CONTEXT_QPC_BYTES_48_MTU_M
,
3353 QP_CONTEXT_QPC_BYTES_48_MTU_S
);
3354 qp_attr
->path_mig_state
= IB_MIG_ARMED
;
3355 if (hr_qp
->ibqp
.qp_type
== IB_QPT_UD
)
3356 qp_attr
->qkey
= QKEY_VAL
;
3358 qp_attr
->rq_psn
= roce_get_field(context
->qpc_bytes_88
,
3359 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M
,
3360 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S
);
3361 qp_attr
->sq_psn
= (u32
)roce_get_field(context
->qpc_bytes_164
,
3362 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M
,
3363 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S
);
3364 qp_attr
->dest_qp_num
= (u8
)roce_get_field(context
->qpc_bytes_36
,
3365 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M
,
3366 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S
);
3367 qp_attr
->qp_access_flags
= ((roce_get_bit(context
->qpc_bytes_4
,
3368 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
)) << 2) |
3369 ((roce_get_bit(context
->qpc_bytes_4
,
3370 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
)) << 1) |
3371 ((roce_get_bit(context
->qpc_bytes_4
,
3372 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S
)) << 3);
3374 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
||
3375 hr_qp
->ibqp
.qp_type
== IB_QPT_UC
) {
3376 struct ib_global_route
*grh
=
3377 rdma_ah_retrieve_grh(&qp_attr
->ah_attr
);
3379 rdma_ah_set_sl(&qp_attr
->ah_attr
,
3380 roce_get_field(context
->qpc_bytes_156
,
3381 QP_CONTEXT_QPC_BYTES_156_SL_M
,
3382 QP_CONTEXT_QPC_BYTES_156_SL_S
));
3383 rdma_ah_set_ah_flags(&qp_attr
->ah_attr
, IB_AH_GRH
);
3385 roce_get_field(context
->qpc_bytes_48
,
3386 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M
,
3387 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S
);
3389 roce_get_field(context
->qpc_bytes_36
,
3390 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M
,
3391 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S
);
3393 roce_get_field(context
->qpc_bytes_44
,
3394 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M
,
3395 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S
);
3396 grh
->traffic_class
=
3397 roce_get_field(context
->qpc_bytes_48
,
3398 QP_CONTEXT_QPC_BYTES_48_TCLASS_M
,
3399 QP_CONTEXT_QPC_BYTES_48_TCLASS_S
);
3401 memcpy(grh
->dgid
.raw
, context
->dgid
,
3402 sizeof(grh
->dgid
.raw
));
3405 qp_attr
->pkey_index
= roce_get_field(context
->qpc_bytes_12
,
3406 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
3407 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
);
3408 qp_attr
->port_num
= hr_qp
->port
+ 1;
3409 qp_attr
->sq_draining
= 0;
3410 qp_attr
->max_rd_atomic
= roce_get_field(context
->qpc_bytes_156
,
3411 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M
,
3412 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S
);
3413 qp_attr
->max_dest_rd_atomic
= roce_get_field(context
->qpc_bytes_32
,
3414 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M
,
3415 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S
);
3416 qp_attr
->min_rnr_timer
= (u8
)(roce_get_field(context
->qpc_bytes_24
,
3417 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M
,
3418 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S
));
3419 qp_attr
->timeout
= (u8
)(roce_get_field(context
->qpc_bytes_156
,
3420 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M
,
3421 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S
));
3422 qp_attr
->retry_cnt
= roce_get_field(context
->qpc_bytes_148
,
3423 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M
,
3424 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S
);
3425 qp_attr
->rnr_retry
= context
->rnr_retry
;
3428 qp_attr
->cur_qp_state
= qp_attr
->qp_state
;
3429 qp_attr
->cap
.max_recv_wr
= hr_qp
->rq
.wqe_cnt
;
3430 qp_attr
->cap
.max_recv_sge
= hr_qp
->rq
.max_gs
;
3432 if (!ibqp
->uobject
) {
3433 qp_attr
->cap
.max_send_wr
= hr_qp
->sq
.wqe_cnt
;
3434 qp_attr
->cap
.max_send_sge
= hr_qp
->sq
.max_gs
;
3436 qp_attr
->cap
.max_send_wr
= 0;
3437 qp_attr
->cap
.max_send_sge
= 0;
3440 qp_init_attr
->cap
= qp_attr
->cap
;
3443 mutex_unlock(&hr_qp
->mutex
);
3448 int hns_roce_v1_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
,
3449 int qp_attr_mask
, struct ib_qp_init_attr
*qp_init_attr
)
3451 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
3453 return hr_qp
->doorbell_qpn
<= 1 ?
3454 hns_roce_v1_q_sqp(ibqp
, qp_attr
, qp_attr_mask
, qp_init_attr
) :
3455 hns_roce_v1_q_qp(ibqp
, qp_attr
, qp_attr_mask
, qp_init_attr
);
3458 static int check_qp_db_process_status(struct hns_roce_dev
*hr_dev
,
3459 struct hns_roce_qp
*hr_qp
,
3464 struct device
*dev
= &hr_dev
->pdev
->dev
;
3465 u32 sdb_retry_cnt
, old_retry
;
3466 u32 sdb_send_ptr
, old_send
;
3467 u32 success_flags
= 0;
3468 u32 cur_cnt
, old_cnt
;
3474 if (*wait_stage
> HNS_ROCE_V1_DB_STAGE2
||
3475 *wait_stage
< HNS_ROCE_V1_DB_STAGE1
) {
3476 dev_err(dev
, "QP(0x%lx) db status wait stage(%d) error!\n",
3477 hr_qp
->qpn
, *wait_stage
);
3481 /* Calculate the total timeout for the entire verification process */
3482 end
= msecs_to_jiffies(HNS_ROCE_V1_CHECK_DB_TIMEOUT_MSECS
) + jiffies
;
3484 if (*wait_stage
== HNS_ROCE_V1_DB_STAGE1
) {
3485 /* Query db process status, until hw process completely */
3486 sdb_send_ptr
= roce_read(hr_dev
, ROCEE_SDB_SEND_PTR_REG
);
3487 while (roce_hw_index_cmp_lt(sdb_send_ptr
, sdb_issue_ptr
,
3488 ROCEE_SDB_PTR_CMP_BITS
)) {
3489 if (!time_before(jiffies
, end
)) {
3490 dev_dbg(dev
, "QP(0x%lx) db process stage1 timeout. issue 0x%x send 0x%x.\n",
3491 hr_qp
->qpn
, sdb_issue_ptr
,
3496 msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS
);
3497 sdb_send_ptr
= roce_read(hr_dev
,
3498 ROCEE_SDB_SEND_PTR_REG
);
3501 if (roce_get_field(sdb_issue_ptr
,
3502 ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M
,
3503 ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S
) ==
3504 roce_get_field(sdb_send_ptr
,
3505 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3506 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
)) {
3507 old_send
= roce_read(hr_dev
, ROCEE_SDB_SEND_PTR_REG
);
3508 old_retry
= roce_read(hr_dev
, ROCEE_SDB_RETRY_CNT_REG
);
3511 tsp_st
= roce_read(hr_dev
, ROCEE_TSP_BP_ST_REG
);
3512 if (roce_get_bit(tsp_st
,
3513 ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S
) == 1) {
3514 *wait_stage
= HNS_ROCE_V1_DB_WAIT_OK
;
3518 if (!time_before(jiffies
, end
)) {
3519 dev_dbg(dev
, "QP(0x%lx) db process stage1 timeout when send ptr equals issue ptr.\n"
3520 "issue 0x%x send 0x%x.\n",
3521 hr_qp
->qpn
, sdb_issue_ptr
,
3526 msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS
);
3528 sdb_send_ptr
= roce_read(hr_dev
,
3529 ROCEE_SDB_SEND_PTR_REG
);
3530 sdb_retry_cnt
= roce_read(hr_dev
,
3531 ROCEE_SDB_RETRY_CNT_REG
);
3532 cur_cnt
= roce_get_field(sdb_send_ptr
,
3533 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3534 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
) +
3535 roce_get_field(sdb_retry_cnt
,
3536 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M
,
3537 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S
);
3538 if (!roce_get_bit(tsp_st
,
3539 ROCEE_CNT_CLR_CE_CNT_CLR_CE_S
)) {
3540 old_cnt
= roce_get_field(old_send
,
3541 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3542 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
) +
3543 roce_get_field(old_retry
,
3544 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M
,
3545 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S
);
3546 if (cur_cnt
- old_cnt
> SDB_ST_CMP_VAL
)
3549 old_cnt
= roce_get_field(old_send
,
3550 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3551 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
);
3552 if (cur_cnt
- old_cnt
>
3557 roce_get_field(old_send
,
3558 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3559 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
) +
3560 roce_get_field(sdb_retry_cnt
,
3561 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M
,
3562 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S
);
3563 roce_set_field(old_send
,
3564 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3565 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
,
3569 } while (!success_flags
);
3572 *wait_stage
= HNS_ROCE_V1_DB_STAGE2
;
3574 /* Get list pointer */
3575 *sdb_inv_cnt
= roce_read(hr_dev
, ROCEE_SDB_INV_CNT_REG
);
3576 dev_dbg(dev
, "QP(0x%lx) db process stage2. inv cnt = 0x%x.\n",
3577 hr_qp
->qpn
, *sdb_inv_cnt
);
3580 if (*wait_stage
== HNS_ROCE_V1_DB_STAGE2
) {
3581 /* Query db's list status, until hw reversal */
3582 inv_cnt
= roce_read(hr_dev
, ROCEE_SDB_INV_CNT_REG
);
3583 while (roce_hw_index_cmp_lt(inv_cnt
,
3584 *sdb_inv_cnt
+ SDB_INV_CNT_OFFSET
,
3585 ROCEE_SDB_CNT_CMP_BITS
)) {
3586 if (!time_before(jiffies
, end
)) {
3587 dev_dbg(dev
, "QP(0x%lx) db process stage2 timeout. inv cnt 0x%x.\n",
3588 hr_qp
->qpn
, inv_cnt
);
3592 msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS
);
3593 inv_cnt
= roce_read(hr_dev
, ROCEE_SDB_INV_CNT_REG
);
3596 *wait_stage
= HNS_ROCE_V1_DB_WAIT_OK
;
3602 static int check_qp_reset_state(struct hns_roce_dev
*hr_dev
,
3603 struct hns_roce_qp
*hr_qp
,
3604 struct hns_roce_qp_work
*qp_work_entry
,
3607 struct device
*dev
= &hr_dev
->pdev
->dev
;
3611 if (hr_qp
->state
!= IB_QPS_RESET
) {
3612 /* Set qp to ERR, waiting for hw complete processing all dbs */
3613 ret
= hns_roce_v1_modify_qp(&hr_qp
->ibqp
, NULL
, 0, hr_qp
->state
,
3616 dev_err(dev
, "Modify QP(0x%lx) to ERR failed!\n",
3621 /* Record issued doorbell */
3622 sdb_issue_ptr
= roce_read(hr_dev
, ROCEE_SDB_ISSUE_PTR_REG
);
3623 qp_work_entry
->sdb_issue_ptr
= sdb_issue_ptr
;
3624 qp_work_entry
->db_wait_stage
= HNS_ROCE_V1_DB_STAGE1
;
3626 /* Query db process status, until hw process completely */
3627 ret
= check_qp_db_process_status(hr_dev
, hr_qp
, sdb_issue_ptr
,
3628 &qp_work_entry
->sdb_inv_cnt
,
3629 &qp_work_entry
->db_wait_stage
);
3631 dev_err(dev
, "Check QP(0x%lx) db process status failed!\n",
3636 if (qp_work_entry
->db_wait_stage
!= HNS_ROCE_V1_DB_WAIT_OK
) {
3637 qp_work_entry
->sche_cnt
= 0;
3642 /* Modify qp to reset before destroying qp */
3643 ret
= hns_roce_v1_modify_qp(&hr_qp
->ibqp
, NULL
, 0, hr_qp
->state
,
3646 dev_err(dev
, "Modify QP(0x%lx) to RST failed!\n",
3655 static void hns_roce_v1_destroy_qp_work_fn(struct work_struct
*work
)
3657 struct hns_roce_qp_work
*qp_work_entry
;
3658 struct hns_roce_v1_priv
*priv
;
3659 struct hns_roce_dev
*hr_dev
;
3660 struct hns_roce_qp
*hr_qp
;
3665 qp_work_entry
= container_of(work
, struct hns_roce_qp_work
, work
);
3666 hr_dev
= to_hr_dev(qp_work_entry
->ib_dev
);
3667 dev
= &hr_dev
->pdev
->dev
;
3668 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
3669 hr_qp
= qp_work_entry
->qp
;
3672 dev_dbg(dev
, "Schedule destroy QP(0x%lx) work.\n", qpn
);
3674 qp_work_entry
->sche_cnt
++;
3676 /* Query db process status, until hw process completely */
3677 ret
= check_qp_db_process_status(hr_dev
, hr_qp
,
3678 qp_work_entry
->sdb_issue_ptr
,
3679 &qp_work_entry
->sdb_inv_cnt
,
3680 &qp_work_entry
->db_wait_stage
);
3682 dev_err(dev
, "Check QP(0x%lx) db process status failed!\n",
3687 if (qp_work_entry
->db_wait_stage
!= HNS_ROCE_V1_DB_WAIT_OK
&&
3688 priv
->des_qp
.requeue_flag
) {
3689 queue_work(priv
->des_qp
.qp_wq
, work
);
3693 /* Modify qp to reset before destroying qp */
3694 ret
= hns_roce_v1_modify_qp(&hr_qp
->ibqp
, NULL
, 0, hr_qp
->state
,
3697 dev_err(dev
, "Modify QP(0x%lx) to RST failed!\n", qpn
);
3701 hns_roce_qp_remove(hr_dev
, hr_qp
);
3702 hns_roce_qp_free(hr_dev
, hr_qp
);
3704 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
) {
3705 /* RC QP, release QPN */
3706 hns_roce_release_range_qp(hr_dev
, qpn
, 1);
3709 kfree(hr_to_hr_sqp(hr_qp
));
3711 kfree(qp_work_entry
);
3713 dev_dbg(dev
, "Accomplished destroy QP(0x%lx) work.\n", qpn
);
3716 int hns_roce_v1_destroy_qp(struct ib_qp
*ibqp
)
3718 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
3719 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
3720 struct device
*dev
= &hr_dev
->pdev
->dev
;
3721 struct hns_roce_qp_work qp_work_entry
;
3722 struct hns_roce_qp_work
*qp_work
;
3723 struct hns_roce_v1_priv
*priv
;
3724 struct hns_roce_cq
*send_cq
, *recv_cq
;
3725 int is_user
= !!ibqp
->pd
->uobject
;
3729 ret
= check_qp_reset_state(hr_dev
, hr_qp
, &qp_work_entry
, &is_timeout
);
3731 dev_err(dev
, "QP reset state check failed(%d)!\n", ret
);
3735 send_cq
= to_hr_cq(hr_qp
->ibqp
.send_cq
);
3736 recv_cq
= to_hr_cq(hr_qp
->ibqp
.recv_cq
);
3738 hns_roce_lock_cqs(send_cq
, recv_cq
);
3740 __hns_roce_v1_cq_clean(recv_cq
, hr_qp
->qpn
, hr_qp
->ibqp
.srq
?
3741 to_hr_srq(hr_qp
->ibqp
.srq
) : NULL
);
3742 if (send_cq
!= recv_cq
)
3743 __hns_roce_v1_cq_clean(send_cq
, hr_qp
->qpn
, NULL
);
3745 hns_roce_unlock_cqs(send_cq
, recv_cq
);
3748 hns_roce_qp_remove(hr_dev
, hr_qp
);
3749 hns_roce_qp_free(hr_dev
, hr_qp
);
3751 /* RC QP, release QPN */
3752 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
)
3753 hns_roce_release_range_qp(hr_dev
, hr_qp
->qpn
, 1);
3756 hns_roce_mtt_cleanup(hr_dev
, &hr_qp
->mtt
);
3759 ib_umem_release(hr_qp
->umem
);
3761 kfree(hr_qp
->sq
.wrid
);
3762 kfree(hr_qp
->rq
.wrid
);
3764 hns_roce_buf_free(hr_dev
, hr_qp
->buff_size
, &hr_qp
->hr_buf
);
3768 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
)
3771 kfree(hr_to_hr_sqp(hr_qp
));
3773 qp_work
= kzalloc(sizeof(*qp_work
), GFP_KERNEL
);
3777 INIT_WORK(&qp_work
->work
, hns_roce_v1_destroy_qp_work_fn
);
3778 qp_work
->ib_dev
= &hr_dev
->ib_dev
;
3779 qp_work
->qp
= hr_qp
;
3780 qp_work
->db_wait_stage
= qp_work_entry
.db_wait_stage
;
3781 qp_work
->sdb_issue_ptr
= qp_work_entry
.sdb_issue_ptr
;
3782 qp_work
->sdb_inv_cnt
= qp_work_entry
.sdb_inv_cnt
;
3783 qp_work
->sche_cnt
= qp_work_entry
.sche_cnt
;
3785 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
3786 queue_work(priv
->des_qp
.qp_wq
, &qp_work
->work
);
3787 dev_dbg(dev
, "Begin destroy QP(0x%lx) work.\n", hr_qp
->qpn
);
3793 int hns_roce_v1_destroy_cq(struct ib_cq
*ibcq
)
3795 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibcq
->device
);
3796 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
3797 struct device
*dev
= &hr_dev
->pdev
->dev
;
3804 hns_roce_free_cq(hr_dev
, hr_cq
);
3807 * Before freeing cq buffer, we need to ensure that the outstanding CQE
3808 * have been written by checking the CQE counter.
3810 cqe_cnt_ori
= roce_read(hr_dev
, ROCEE_SCAEP_WR_CQE_CNT
);
3812 if (roce_read(hr_dev
, ROCEE_CAEP_CQE_WCMD_EMPTY
) &
3813 HNS_ROCE_CQE_WCMD_EMPTY_BIT
)
3816 cqe_cnt_cur
= roce_read(hr_dev
, ROCEE_SCAEP_WR_CQE_CNT
);
3817 if ((cqe_cnt_cur
- cqe_cnt_ori
) >= HNS_ROCE_MIN_CQE_CNT
)
3820 msleep(HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS
);
3821 if (wait_time
> HNS_ROCE_MAX_FREE_CQ_WAIT_CNT
) {
3822 dev_warn(dev
, "Destroy cq 0x%lx timeout!\n",
3830 hns_roce_mtt_cleanup(hr_dev
, &hr_cq
->hr_buf
.hr_mtt
);
3833 ib_umem_release(hr_cq
->umem
);
3835 /* Free the buff of stored cq */
3836 cq_buf_size
= (ibcq
->cqe
+ 1) * hr_dev
->caps
.cq_entry_sz
;
3837 hns_roce_buf_free(hr_dev
, cq_buf_size
, &hr_cq
->hr_buf
.hr_buf
);
3845 struct hns_roce_v1_priv hr_v1_priv
;
3847 struct hns_roce_hw hns_roce_hw_v1
= {
3848 .reset
= hns_roce_v1_reset
,
3849 .hw_profile
= hns_roce_v1_profile
,
3850 .hw_init
= hns_roce_v1_init
,
3851 .hw_exit
= hns_roce_v1_exit
,
3852 .set_gid
= hns_roce_v1_set_gid
,
3853 .set_mac
= hns_roce_v1_set_mac
,
3854 .set_mtu
= hns_roce_v1_set_mtu
,
3855 .write_mtpt
= hns_roce_v1_write_mtpt
,
3856 .write_cqc
= hns_roce_v1_write_cqc
,
3857 .clear_hem
= hns_roce_v1_clear_hem
,
3858 .modify_qp
= hns_roce_v1_modify_qp
,
3859 .query_qp
= hns_roce_v1_query_qp
,
3860 .destroy_qp
= hns_roce_v1_destroy_qp
,
3861 .post_send
= hns_roce_v1_post_send
,
3862 .post_recv
= hns_roce_v1_post_recv
,
3863 .req_notify_cq
= hns_roce_v1_req_notify_cq
,
3864 .poll_cq
= hns_roce_v1_poll_cq
,
3865 .dereg_mr
= hns_roce_v1_dereg_mr
,
3866 .destroy_cq
= hns_roce_v1_destroy_cq
,
3867 .priv
= &hr_v1_priv
,