2 * Copyright (c) 2016 Hisilicon Limited.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/platform_device.h>
34 #include <linux/acpi.h>
35 #include <linux/etherdevice.h>
37 #include <rdma/ib_umem.h>
38 #include "hns_roce_common.h"
39 #include "hns_roce_device.h"
40 #include "hns_roce_cmd.h"
41 #include "hns_roce_hem.h"
42 #include "hns_roce_hw_v1.h"
44 static void set_data_seg(struct hns_roce_wqe_data_seg
*dseg
, struct ib_sge
*sg
)
46 dseg
->lkey
= cpu_to_le32(sg
->lkey
);
47 dseg
->addr
= cpu_to_le64(sg
->addr
);
48 dseg
->len
= cpu_to_le32(sg
->length
);
51 static void set_raddr_seg(struct hns_roce_wqe_raddr_seg
*rseg
, u64 remote_addr
,
54 rseg
->raddr
= cpu_to_le64(remote_addr
);
55 rseg
->rkey
= cpu_to_le32(rkey
);
59 int hns_roce_v1_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
60 struct ib_send_wr
**bad_wr
)
62 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
63 struct hns_roce_ah
*ah
= to_hr_ah(ud_wr(wr
)->ah
);
64 struct hns_roce_ud_send_wqe
*ud_sq_wqe
= NULL
;
65 struct hns_roce_wqe_ctrl_seg
*ctrl
= NULL
;
66 struct hns_roce_wqe_data_seg
*dseg
= NULL
;
67 struct hns_roce_qp
*qp
= to_hr_qp(ibqp
);
68 struct device
*dev
= &hr_dev
->pdev
->dev
;
69 struct hns_roce_sq_db sq_db
;
70 int ps_opcode
= 0, i
= 0;
71 unsigned long flags
= 0;
80 if (unlikely(ibqp
->qp_type
!= IB_QPT_GSI
&&
81 ibqp
->qp_type
!= IB_QPT_RC
)) {
82 dev_err(dev
, "un-supported QP type\n");
87 spin_lock_irqsave(&qp
->sq
.lock
, flags
);
88 ind
= qp
->sq_next_wqe
;
89 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
90 if (hns_roce_wq_overflow(&qp
->sq
, nreq
, qp
->ibqp
.send_cq
)) {
96 if (unlikely(wr
->num_sge
> qp
->sq
.max_gs
)) {
97 dev_err(dev
, "num_sge=%d > qp->sq.max_gs=%d\n",
98 wr
->num_sge
, qp
->sq
.max_gs
);
104 wqe
= get_send_wqe(qp
, ind
& (qp
->sq
.wqe_cnt
- 1));
105 qp
->sq
.wrid
[(qp
->sq
.head
+ nreq
) & (qp
->sq
.wqe_cnt
- 1)] =
108 /* Corresponding to the RC and RD type wqe process separately */
109 if (ibqp
->qp_type
== IB_QPT_GSI
) {
111 roce_set_field(ud_sq_wqe
->dmac_h
,
112 UD_SEND_WQE_U32_4_DMAC_0_M
,
113 UD_SEND_WQE_U32_4_DMAC_0_S
,
115 roce_set_field(ud_sq_wqe
->dmac_h
,
116 UD_SEND_WQE_U32_4_DMAC_1_M
,
117 UD_SEND_WQE_U32_4_DMAC_1_S
,
119 roce_set_field(ud_sq_wqe
->dmac_h
,
120 UD_SEND_WQE_U32_4_DMAC_2_M
,
121 UD_SEND_WQE_U32_4_DMAC_2_S
,
123 roce_set_field(ud_sq_wqe
->dmac_h
,
124 UD_SEND_WQE_U32_4_DMAC_3_M
,
125 UD_SEND_WQE_U32_4_DMAC_3_S
,
128 roce_set_field(ud_sq_wqe
->u32_8
,
129 UD_SEND_WQE_U32_8_DMAC_4_M
,
130 UD_SEND_WQE_U32_8_DMAC_4_S
,
132 roce_set_field(ud_sq_wqe
->u32_8
,
133 UD_SEND_WQE_U32_8_DMAC_5_M
,
134 UD_SEND_WQE_U32_8_DMAC_5_S
,
137 smac
= (u8
*)hr_dev
->dev_addr
[qp
->port
];
138 loopback
= ether_addr_equal_unaligned(ah
->av
.mac
,
140 roce_set_bit(ud_sq_wqe
->u32_8
,
141 UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S
,
144 roce_set_field(ud_sq_wqe
->u32_8
,
145 UD_SEND_WQE_U32_8_OPERATION_TYPE_M
,
146 UD_SEND_WQE_U32_8_OPERATION_TYPE_S
,
147 HNS_ROCE_WQE_OPCODE_SEND
);
148 roce_set_field(ud_sq_wqe
->u32_8
,
149 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M
,
150 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S
,
152 roce_set_bit(ud_sq_wqe
->u32_8
,
153 UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S
,
156 ud_sq_wqe
->u32_8
|= (wr
->send_flags
& IB_SEND_SIGNALED
?
157 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY
) : 0) |
158 (wr
->send_flags
& IB_SEND_SOLICITED
?
159 cpu_to_le32(HNS_ROCE_WQE_SE
) : 0) |
160 ((wr
->opcode
== IB_WR_SEND_WITH_IMM
) ?
161 cpu_to_le32(HNS_ROCE_WQE_IMM
) : 0);
163 roce_set_field(ud_sq_wqe
->u32_16
,
164 UD_SEND_WQE_U32_16_DEST_QP_M
,
165 UD_SEND_WQE_U32_16_DEST_QP_S
,
166 ud_wr(wr
)->remote_qpn
);
167 roce_set_field(ud_sq_wqe
->u32_16
,
168 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M
,
169 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S
,
172 roce_set_field(ud_sq_wqe
->u32_36
,
173 UD_SEND_WQE_U32_36_FLOW_LABEL_M
,
174 UD_SEND_WQE_U32_36_FLOW_LABEL_S
, 0);
175 roce_set_field(ud_sq_wqe
->u32_36
,
176 UD_SEND_WQE_U32_36_PRIORITY_M
,
177 UD_SEND_WQE_U32_36_PRIORITY_S
,
178 ah
->av
.sl_tclass_flowlabel
>>
180 roce_set_field(ud_sq_wqe
->u32_36
,
181 UD_SEND_WQE_U32_36_SGID_INDEX_M
,
182 UD_SEND_WQE_U32_36_SGID_INDEX_S
,
183 hns_get_gid_index(hr_dev
, qp
->phy_port
,
186 roce_set_field(ud_sq_wqe
->u32_40
,
187 UD_SEND_WQE_U32_40_HOP_LIMIT_M
,
188 UD_SEND_WQE_U32_40_HOP_LIMIT_S
,
190 roce_set_field(ud_sq_wqe
->u32_40
,
191 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M
,
192 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S
, 0);
194 memcpy(&ud_sq_wqe
->dgid
[0], &ah
->av
.dgid
[0], GID_LEN
);
196 ud_sq_wqe
->va0_l
= (u32
)wr
->sg_list
[0].addr
;
197 ud_sq_wqe
->va0_h
= (wr
->sg_list
[0].addr
) >> 32;
198 ud_sq_wqe
->l_key0
= wr
->sg_list
[0].lkey
;
200 ud_sq_wqe
->va1_l
= (u32
)wr
->sg_list
[1].addr
;
201 ud_sq_wqe
->va1_h
= (wr
->sg_list
[1].addr
) >> 32;
202 ud_sq_wqe
->l_key1
= wr
->sg_list
[1].lkey
;
204 } else if (ibqp
->qp_type
== IB_QPT_RC
) {
206 memset(ctrl
, 0, sizeof(struct hns_roce_wqe_ctrl_seg
));
207 for (i
= 0; i
< wr
->num_sge
; i
++)
208 ctrl
->msg_length
+= wr
->sg_list
[i
].length
;
212 ctrl
->imm_data
= send_ieth(wr
);
214 /*Ctrl field, ctrl set type: sig, solic, imm, fence */
215 /* SO wait for conforming application scenarios */
216 ctrl
->flag
|= (wr
->send_flags
& IB_SEND_SIGNALED
?
217 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY
) : 0) |
218 (wr
->send_flags
& IB_SEND_SOLICITED
?
219 cpu_to_le32(HNS_ROCE_WQE_SE
) : 0) |
220 ((wr
->opcode
== IB_WR_SEND_WITH_IMM
||
221 wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
) ?
222 cpu_to_le32(HNS_ROCE_WQE_IMM
) : 0) |
223 (wr
->send_flags
& IB_SEND_FENCE
?
224 (cpu_to_le32(HNS_ROCE_WQE_FENCE
)) : 0);
226 wqe
+= sizeof(struct hns_roce_wqe_ctrl_seg
);
228 switch (wr
->opcode
) {
229 case IB_WR_RDMA_READ
:
230 ps_opcode
= HNS_ROCE_WQE_OPCODE_RDMA_READ
;
231 set_raddr_seg(wqe
, atomic_wr(wr
)->remote_addr
,
232 atomic_wr(wr
)->rkey
);
234 case IB_WR_RDMA_WRITE
:
235 case IB_WR_RDMA_WRITE_WITH_IMM
:
236 ps_opcode
= HNS_ROCE_WQE_OPCODE_RDMA_WRITE
;
237 set_raddr_seg(wqe
, atomic_wr(wr
)->remote_addr
,
238 atomic_wr(wr
)->rkey
);
241 case IB_WR_SEND_WITH_INV
:
242 case IB_WR_SEND_WITH_IMM
:
243 ps_opcode
= HNS_ROCE_WQE_OPCODE_SEND
;
245 case IB_WR_LOCAL_INV
:
247 case IB_WR_ATOMIC_CMP_AND_SWP
:
248 case IB_WR_ATOMIC_FETCH_AND_ADD
:
251 ps_opcode
= HNS_ROCE_WQE_OPCODE_MASK
;
254 ctrl
->flag
|= cpu_to_le32(ps_opcode
);
255 wqe
+= sizeof(struct hns_roce_wqe_raddr_seg
);
258 if (wr
->send_flags
& IB_SEND_INLINE
&& wr
->num_sge
) {
259 if (ctrl
->msg_length
>
260 hr_dev
->caps
.max_sq_inline
) {
263 dev_err(dev
, "inline len(1-%d)=%d, illegal",
265 hr_dev
->caps
.max_sq_inline
);
268 for (i
= 0; i
< wr
->num_sge
; i
++) {
269 memcpy(wqe
, ((void *) (uintptr_t)
270 wr
->sg_list
[i
].addr
),
271 wr
->sg_list
[i
].length
);
272 wqe
+= wr
->sg_list
[i
].length
;
274 ctrl
->flag
|= HNS_ROCE_WQE_INLINE
;
277 for (i
= 0; i
< wr
->num_sge
; i
++)
278 set_data_seg(dseg
+ i
, wr
->sg_list
+ i
);
280 ctrl
->flag
|= cpu_to_le32(wr
->num_sge
<<
281 HNS_ROCE_WQE_SGE_NUM_BIT
);
296 roce_set_field(sq_db
.u32_4
, SQ_DOORBELL_U32_4_SQ_HEAD_M
,
297 SQ_DOORBELL_U32_4_SQ_HEAD_S
,
298 (qp
->sq
.head
& ((qp
->sq
.wqe_cnt
<< 1) - 1)));
299 roce_set_field(sq_db
.u32_4
, SQ_DOORBELL_U32_4_SL_M
,
300 SQ_DOORBELL_U32_4_SL_S
, qp
->sl
);
301 roce_set_field(sq_db
.u32_4
, SQ_DOORBELL_U32_4_PORT_M
,
302 SQ_DOORBELL_U32_4_PORT_S
, qp
->phy_port
);
303 roce_set_field(sq_db
.u32_8
, SQ_DOORBELL_U32_8_QPN_M
,
304 SQ_DOORBELL_U32_8_QPN_S
, qp
->doorbell_qpn
);
305 roce_set_bit(sq_db
.u32_8
, SQ_DOORBELL_HW_SYNC_S
, 1);
307 doorbell
[0] = sq_db
.u32_4
;
308 doorbell
[1] = sq_db
.u32_8
;
310 hns_roce_write64_k(doorbell
, qp
->sq
.db_reg_l
);
311 qp
->sq_next_wqe
= ind
;
314 spin_unlock_irqrestore(&qp
->sq
.lock
, flags
);
319 int hns_roce_v1_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
320 struct ib_recv_wr
**bad_wr
)
327 unsigned long flags
= 0;
328 struct hns_roce_rq_wqe_ctrl
*ctrl
= NULL
;
329 struct hns_roce_wqe_data_seg
*scat
= NULL
;
330 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
331 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
332 struct device
*dev
= &hr_dev
->pdev
->dev
;
333 struct hns_roce_rq_db rq_db
;
334 uint32_t doorbell
[2] = {0};
336 spin_lock_irqsave(&hr_qp
->rq
.lock
, flags
);
337 ind
= hr_qp
->rq
.head
& (hr_qp
->rq
.wqe_cnt
- 1);
339 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
340 if (hns_roce_wq_overflow(&hr_qp
->rq
, nreq
,
341 hr_qp
->ibqp
.recv_cq
)) {
347 if (unlikely(wr
->num_sge
> hr_qp
->rq
.max_gs
)) {
348 dev_err(dev
, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
349 wr
->num_sge
, hr_qp
->rq
.max_gs
);
355 ctrl
= get_recv_wqe(hr_qp
, ind
);
357 roce_set_field(ctrl
->rwqe_byte_12
,
358 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M
,
359 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S
,
362 scat
= (struct hns_roce_wqe_data_seg
*)(ctrl
+ 1);
364 for (i
= 0; i
< wr
->num_sge
; i
++)
365 set_data_seg(scat
+ i
, wr
->sg_list
+ i
);
367 hr_qp
->rq
.wrid
[ind
] = wr
->wr_id
;
369 ind
= (ind
+ 1) & (hr_qp
->rq
.wqe_cnt
- 1);
374 hr_qp
->rq
.head
+= nreq
;
378 if (ibqp
->qp_type
== IB_QPT_GSI
) {
379 /* SW update GSI rq header */
380 reg_val
= roce_read(to_hr_dev(ibqp
->device
),
381 ROCEE_QP1C_CFG3_0_REG
+
382 QP1C_CFGN_OFFSET
* hr_qp
->phy_port
);
383 roce_set_field(reg_val
,
384 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M
,
385 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S
,
387 roce_write(to_hr_dev(ibqp
->device
),
388 ROCEE_QP1C_CFG3_0_REG
+
389 QP1C_CFGN_OFFSET
* hr_qp
->phy_port
, reg_val
);
394 roce_set_field(rq_db
.u32_4
, RQ_DOORBELL_U32_4_RQ_HEAD_M
,
395 RQ_DOORBELL_U32_4_RQ_HEAD_S
,
397 roce_set_field(rq_db
.u32_8
, RQ_DOORBELL_U32_8_QPN_M
,
398 RQ_DOORBELL_U32_8_QPN_S
, hr_qp
->qpn
);
399 roce_set_field(rq_db
.u32_8
, RQ_DOORBELL_U32_8_CMD_M
,
400 RQ_DOORBELL_U32_8_CMD_S
, 1);
401 roce_set_bit(rq_db
.u32_8
, RQ_DOORBELL_U32_8_HW_SYNC_S
,
404 doorbell
[0] = rq_db
.u32_4
;
405 doorbell
[1] = rq_db
.u32_8
;
407 hns_roce_write64_k(doorbell
, hr_qp
->rq
.db_reg_l
);
410 spin_unlock_irqrestore(&hr_qp
->rq
.lock
, flags
);
415 static void hns_roce_set_db_event_mode(struct hns_roce_dev
*hr_dev
,
416 int sdb_mode
, int odb_mode
)
420 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
421 roce_set_bit(val
, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S
, sdb_mode
);
422 roce_set_bit(val
, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S
, odb_mode
);
423 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
426 static void hns_roce_set_db_ext_mode(struct hns_roce_dev
*hr_dev
, u32 sdb_mode
,
431 /* Configure SDB/ODB extend mode */
432 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
433 roce_set_bit(val
, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S
, sdb_mode
);
434 roce_set_bit(val
, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S
, odb_mode
);
435 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
438 static void hns_roce_set_sdb(struct hns_roce_dev
*hr_dev
, u32 sdb_alept
,
444 val
= roce_read(hr_dev
, ROCEE_DB_SQ_WL_REG
);
445 roce_set_field(val
, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M
,
446 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S
, sdb_alful
);
447 roce_set_field(val
, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M
,
448 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S
, sdb_alept
);
449 roce_write(hr_dev
, ROCEE_DB_SQ_WL_REG
, val
);
452 static void hns_roce_set_odb(struct hns_roce_dev
*hr_dev
, u32 odb_alept
,
458 val
= roce_read(hr_dev
, ROCEE_DB_OTHERS_WL_REG
);
459 roce_set_field(val
, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M
,
460 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S
, odb_alful
);
461 roce_set_field(val
, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M
,
462 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S
, odb_alept
);
463 roce_write(hr_dev
, ROCEE_DB_OTHERS_WL_REG
, val
);
466 static void hns_roce_set_sdb_ext(struct hns_roce_dev
*hr_dev
, u32 ext_sdb_alept
,
469 struct device
*dev
= &hr_dev
->pdev
->dev
;
470 struct hns_roce_v1_priv
*priv
;
471 struct hns_roce_db_table
*db
;
472 dma_addr_t sdb_dma_addr
;
475 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
476 db
= &priv
->db_table
;
478 /* Configure extend SDB threshold */
479 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_WL_EMPTY_REG
, ext_sdb_alept
);
480 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_WL_REG
, ext_sdb_alful
);
482 /* Configure extend SDB base addr */
483 sdb_dma_addr
= db
->ext_db
->sdb_buf_list
->map
;
484 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_REG
, (u32
)(sdb_dma_addr
>> 12));
486 /* Configure extend SDB depth */
487 val
= roce_read(hr_dev
, ROCEE_EXT_DB_SQ_H_REG
);
488 roce_set_field(val
, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M
,
489 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S
,
490 db
->ext_db
->esdb_dep
);
492 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
493 * using 4K page, and shift more 32 because of
494 * caculating the high 32 bit value evaluated to hardware.
496 roce_set_field(val
, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M
,
497 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S
, sdb_dma_addr
>> 44);
498 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_H_REG
, val
);
500 dev_dbg(dev
, "ext SDB depth: 0x%x\n", db
->ext_db
->esdb_dep
);
501 dev_dbg(dev
, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
502 ext_sdb_alept
, ext_sdb_alful
);
505 static void hns_roce_set_odb_ext(struct hns_roce_dev
*hr_dev
, u32 ext_odb_alept
,
508 struct device
*dev
= &hr_dev
->pdev
->dev
;
509 struct hns_roce_v1_priv
*priv
;
510 struct hns_roce_db_table
*db
;
511 dma_addr_t odb_dma_addr
;
514 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
515 db
= &priv
->db_table
;
517 /* Configure extend ODB threshold */
518 roce_write(hr_dev
, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG
, ext_odb_alept
);
519 roce_write(hr_dev
, ROCEE_EXT_DB_OTHERS_WL_REG
, ext_odb_alful
);
521 /* Configure extend ODB base addr */
522 odb_dma_addr
= db
->ext_db
->odb_buf_list
->map
;
523 roce_write(hr_dev
, ROCEE_EXT_DB_OTH_REG
, (u32
)(odb_dma_addr
>> 12));
525 /* Configure extend ODB depth */
526 val
= roce_read(hr_dev
, ROCEE_EXT_DB_OTH_H_REG
);
527 roce_set_field(val
, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M
,
528 ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S
,
529 db
->ext_db
->eodb_dep
);
530 roce_set_field(val
, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M
,
531 ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S
,
532 db
->ext_db
->eodb_dep
);
533 roce_write(hr_dev
, ROCEE_EXT_DB_OTH_H_REG
, val
);
535 dev_dbg(dev
, "ext ODB depth: 0x%x\n", db
->ext_db
->eodb_dep
);
536 dev_dbg(dev
, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
537 ext_odb_alept
, ext_odb_alful
);
540 static int hns_roce_db_ext_init(struct hns_roce_dev
*hr_dev
, u32 sdb_ext_mod
,
543 struct device
*dev
= &hr_dev
->pdev
->dev
;
544 struct hns_roce_v1_priv
*priv
;
545 struct hns_roce_db_table
*db
;
546 dma_addr_t sdb_dma_addr
;
547 dma_addr_t odb_dma_addr
;
550 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
551 db
= &priv
->db_table
;
553 db
->ext_db
= kmalloc(sizeof(*db
->ext_db
), GFP_KERNEL
);
558 db
->ext_db
->sdb_buf_list
= kmalloc(
559 sizeof(*db
->ext_db
->sdb_buf_list
), GFP_KERNEL
);
560 if (!db
->ext_db
->sdb_buf_list
) {
562 goto ext_sdb_buf_fail_out
;
565 db
->ext_db
->sdb_buf_list
->buf
= dma_alloc_coherent(dev
,
566 HNS_ROCE_V1_EXT_SDB_SIZE
,
567 &sdb_dma_addr
, GFP_KERNEL
);
568 if (!db
->ext_db
->sdb_buf_list
->buf
) {
570 goto alloc_sq_db_buf_fail
;
572 db
->ext_db
->sdb_buf_list
->map
= sdb_dma_addr
;
574 db
->ext_db
->esdb_dep
= ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH
);
575 hns_roce_set_sdb_ext(hr_dev
, HNS_ROCE_V1_EXT_SDB_ALEPT
,
576 HNS_ROCE_V1_EXT_SDB_ALFUL
);
578 hns_roce_set_sdb(hr_dev
, HNS_ROCE_V1_SDB_ALEPT
,
579 HNS_ROCE_V1_SDB_ALFUL
);
582 db
->ext_db
->odb_buf_list
= kmalloc(
583 sizeof(*db
->ext_db
->odb_buf_list
), GFP_KERNEL
);
584 if (!db
->ext_db
->odb_buf_list
) {
586 goto ext_odb_buf_fail_out
;
589 db
->ext_db
->odb_buf_list
->buf
= dma_alloc_coherent(dev
,
590 HNS_ROCE_V1_EXT_ODB_SIZE
,
591 &odb_dma_addr
, GFP_KERNEL
);
592 if (!db
->ext_db
->odb_buf_list
->buf
) {
594 goto alloc_otr_db_buf_fail
;
596 db
->ext_db
->odb_buf_list
->map
= odb_dma_addr
;
598 db
->ext_db
->eodb_dep
= ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH
);
599 hns_roce_set_odb_ext(hr_dev
, HNS_ROCE_V1_EXT_ODB_ALEPT
,
600 HNS_ROCE_V1_EXT_ODB_ALFUL
);
602 hns_roce_set_odb(hr_dev
, HNS_ROCE_V1_ODB_ALEPT
,
603 HNS_ROCE_V1_ODB_ALFUL
);
605 hns_roce_set_db_ext_mode(hr_dev
, sdb_ext_mod
, odb_ext_mod
);
609 alloc_otr_db_buf_fail
:
610 kfree(db
->ext_db
->odb_buf_list
);
612 ext_odb_buf_fail_out
:
614 dma_free_coherent(dev
, HNS_ROCE_V1_EXT_SDB_SIZE
,
615 db
->ext_db
->sdb_buf_list
->buf
,
616 db
->ext_db
->sdb_buf_list
->map
);
619 alloc_sq_db_buf_fail
:
621 kfree(db
->ext_db
->sdb_buf_list
);
623 ext_sdb_buf_fail_out
:
628 static struct hns_roce_qp
*hns_roce_v1_create_lp_qp(struct hns_roce_dev
*hr_dev
,
631 struct device
*dev
= &hr_dev
->pdev
->dev
;
632 struct ib_qp_init_attr init_attr
;
635 memset(&init_attr
, 0, sizeof(struct ib_qp_init_attr
));
636 init_attr
.qp_type
= IB_QPT_RC
;
637 init_attr
.sq_sig_type
= IB_SIGNAL_ALL_WR
;
638 init_attr
.cap
.max_recv_wr
= HNS_ROCE_MIN_WQE_NUM
;
639 init_attr
.cap
.max_send_wr
= HNS_ROCE_MIN_WQE_NUM
;
641 qp
= hns_roce_create_qp(pd
, &init_attr
, NULL
);
643 dev_err(dev
, "Create loop qp for mr free failed!");
650 static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev
*hr_dev
)
652 struct hns_roce_caps
*caps
= &hr_dev
->caps
;
653 struct device
*dev
= &hr_dev
->pdev
->dev
;
654 struct ib_cq_init_attr cq_init_attr
;
655 struct hns_roce_free_mr
*free_mr
;
656 struct ib_qp_attr attr
= { 0 };
657 struct hns_roce_v1_priv
*priv
;
658 struct hns_roce_qp
*hr_qp
;
669 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
670 free_mr
= &priv
->free_mr
;
672 /* Reserved cq for loop qp */
673 cq_init_attr
.cqe
= HNS_ROCE_MIN_WQE_NUM
* 2;
674 cq_init_attr
.comp_vector
= 0;
675 cq
= hns_roce_ib_create_cq(&hr_dev
->ib_dev
, &cq_init_attr
, NULL
, NULL
);
677 dev_err(dev
, "Create cq for reseved loop qp failed!");
680 free_mr
->mr_free_cq
= to_hr_cq(cq
);
681 free_mr
->mr_free_cq
->ib_cq
.device
= &hr_dev
->ib_dev
;
682 free_mr
->mr_free_cq
->ib_cq
.uobject
= NULL
;
683 free_mr
->mr_free_cq
->ib_cq
.comp_handler
= NULL
;
684 free_mr
->mr_free_cq
->ib_cq
.event_handler
= NULL
;
685 free_mr
->mr_free_cq
->ib_cq
.cq_context
= NULL
;
686 atomic_set(&free_mr
->mr_free_cq
->ib_cq
.usecnt
, 0);
688 pd
= hns_roce_alloc_pd(&hr_dev
->ib_dev
, NULL
, NULL
);
690 dev_err(dev
, "Create pd for reseved loop qp failed!");
692 goto alloc_pd_failed
;
694 free_mr
->mr_free_pd
= to_hr_pd(pd
);
695 free_mr
->mr_free_pd
->ibpd
.device
= &hr_dev
->ib_dev
;
696 free_mr
->mr_free_pd
->ibpd
.uobject
= NULL
;
697 atomic_set(&free_mr
->mr_free_pd
->ibpd
.usecnt
, 0);
699 attr
.qp_access_flags
= IB_ACCESS_REMOTE_WRITE
;
701 attr
.min_rnr_timer
= 0;
702 /* Disable read ability */
703 attr
.max_dest_rd_atomic
= 0;
704 attr
.max_rd_atomic
= 0;
705 /* Use arbitrary values as rq_psn and sq_psn */
706 attr
.rq_psn
= 0x0808;
707 attr
.sq_psn
= 0x0808;
711 attr
.path_mtu
= IB_MTU_256
;
712 rdma_ah_set_grh(&attr
.ah_attr
, NULL
, 0, 0, 1, 0);
713 rdma_ah_set_static_rate(&attr
.ah_attr
, 3);
715 subnet_prefix
= cpu_to_be64(0xfe80000000000000LL
);
716 for (i
= 0; i
< HNS_ROCE_V1_RESV_QP
; i
++) {
717 free_mr
->mr_free_qp
[i
] = hns_roce_v1_create_lp_qp(hr_dev
, pd
);
718 if (IS_ERR(free_mr
->mr_free_qp
[i
])) {
719 dev_err(dev
, "Create loop qp failed!\n");
720 goto create_lp_qp_failed
;
722 hr_qp
= free_mr
->mr_free_qp
[i
];
724 sl
= i
/ caps
->num_ports
;
726 if (caps
->num_ports
== HNS_ROCE_MAX_PORTS
)
727 phy_port
= (i
>= HNS_ROCE_MAX_PORTS
) ? (i
- 2) :
728 (i
% caps
->num_ports
);
730 phy_port
= i
% caps
->num_ports
;
732 hr_qp
->port
= phy_port
+ 1;
733 hr_qp
->phy_port
= phy_port
;
734 hr_qp
->ibqp
.qp_type
= IB_QPT_RC
;
735 hr_qp
->ibqp
.device
= &hr_dev
->ib_dev
;
736 hr_qp
->ibqp
.uobject
= NULL
;
737 atomic_set(&hr_qp
->ibqp
.usecnt
, 0);
739 hr_qp
->ibqp
.recv_cq
= cq
;
740 hr_qp
->ibqp
.send_cq
= cq
;
742 rdma_ah_set_port_num(&attr
.ah_attr
, phy_port
+ 1);
743 rdma_ah_set_sl(&attr
.ah_attr
, phy_port
+ 1);
744 attr
.port_num
= phy_port
+ 1;
746 attr
.dest_qp_num
= hr_qp
->qpn
;
747 memcpy(rdma_ah_retrieve_dmac(&attr
.ah_attr
),
748 hr_dev
->dev_addr
[phy_port
],
751 memcpy(&dgid
.raw
, &subnet_prefix
, sizeof(u64
));
752 memcpy(&dgid
.raw
[8], hr_dev
->dev_addr
[phy_port
], 3);
753 memcpy(&dgid
.raw
[13], hr_dev
->dev_addr
[phy_port
] + 3, 3);
757 rdma_ah_set_dgid_raw(&attr
.ah_attr
, dgid
.raw
);
758 attr_mask
|= IB_QP_PORT
;
760 ret
= hr_dev
->hw
->modify_qp(&hr_qp
->ibqp
, &attr
, attr_mask
,
761 IB_QPS_RESET
, IB_QPS_INIT
);
763 dev_err(dev
, "modify qp failed(%d)!\n", ret
);
764 goto create_lp_qp_failed
;
767 ret
= hr_dev
->hw
->modify_qp(&hr_qp
->ibqp
, &attr
, attr_mask
,
768 IB_QPS_INIT
, IB_QPS_RTR
);
770 dev_err(dev
, "modify qp failed(%d)!\n", ret
);
771 goto create_lp_qp_failed
;
774 ret
= hr_dev
->hw
->modify_qp(&hr_qp
->ibqp
, &attr
, attr_mask
,
775 IB_QPS_RTR
, IB_QPS_RTS
);
777 dev_err(dev
, "modify qp failed(%d)!\n", ret
);
778 goto create_lp_qp_failed
;
785 for (i
-= 1; i
>= 0; i
--) {
786 hr_qp
= free_mr
->mr_free_qp
[i
];
787 if (hns_roce_v1_destroy_qp(&hr_qp
->ibqp
))
788 dev_err(dev
, "Destroy qp %d for mr free failed!\n", i
);
791 if (hns_roce_dealloc_pd(pd
))
792 dev_err(dev
, "Destroy pd for create_lp_qp failed!\n");
795 if (hns_roce_ib_destroy_cq(cq
))
796 dev_err(dev
, "Destroy cq for create_lp_qp failed!\n");
801 static void hns_roce_v1_release_lp_qp(struct hns_roce_dev
*hr_dev
)
803 struct device
*dev
= &hr_dev
->pdev
->dev
;
804 struct hns_roce_free_mr
*free_mr
;
805 struct hns_roce_v1_priv
*priv
;
806 struct hns_roce_qp
*hr_qp
;
810 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
811 free_mr
= &priv
->free_mr
;
813 for (i
= 0; i
< HNS_ROCE_V1_RESV_QP
; i
++) {
814 hr_qp
= free_mr
->mr_free_qp
[i
];
815 ret
= hns_roce_v1_destroy_qp(&hr_qp
->ibqp
);
817 dev_err(dev
, "Destroy qp %d for mr free failed(%d)!\n",
821 ret
= hns_roce_ib_destroy_cq(&free_mr
->mr_free_cq
->ib_cq
);
823 dev_err(dev
, "Destroy cq for mr_free failed(%d)!\n", ret
);
825 ret
= hns_roce_dealloc_pd(&free_mr
->mr_free_pd
->ibpd
);
827 dev_err(dev
, "Destroy pd for mr_free failed(%d)!\n", ret
);
830 static int hns_roce_db_init(struct hns_roce_dev
*hr_dev
)
832 struct device
*dev
= &hr_dev
->pdev
->dev
;
833 struct hns_roce_v1_priv
*priv
;
834 struct hns_roce_db_table
*db
;
841 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
842 db
= &priv
->db_table
;
844 memset(db
, 0, sizeof(*db
));
846 /* Default DB mode */
847 sdb_ext_mod
= HNS_ROCE_SDB_EXTEND_MODE
;
848 odb_ext_mod
= HNS_ROCE_ODB_EXTEND_MODE
;
849 sdb_evt_mod
= HNS_ROCE_SDB_NORMAL_MODE
;
850 odb_evt_mod
= HNS_ROCE_ODB_POLL_MODE
;
852 db
->sdb_ext_mod
= sdb_ext_mod
;
853 db
->odb_ext_mod
= odb_ext_mod
;
856 ret
= hns_roce_db_ext_init(hr_dev
, sdb_ext_mod
, odb_ext_mod
);
858 dev_err(dev
, "Failed in extend DB configuration.\n");
862 hns_roce_set_db_event_mode(hr_dev
, sdb_evt_mod
, odb_evt_mod
);
867 void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct
*work
)
869 struct hns_roce_recreate_lp_qp_work
*lp_qp_work
;
870 struct hns_roce_dev
*hr_dev
;
872 lp_qp_work
= container_of(work
, struct hns_roce_recreate_lp_qp_work
,
874 hr_dev
= to_hr_dev(lp_qp_work
->ib_dev
);
876 hns_roce_v1_release_lp_qp(hr_dev
);
878 if (hns_roce_v1_rsv_lp_qp(hr_dev
))
879 dev_err(&hr_dev
->pdev
->dev
, "create reserver qp failed\n");
881 if (lp_qp_work
->comp_flag
)
882 complete(lp_qp_work
->comp
);
887 static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev
*hr_dev
)
889 struct device
*dev
= &hr_dev
->pdev
->dev
;
890 struct hns_roce_recreate_lp_qp_work
*lp_qp_work
;
891 struct hns_roce_free_mr
*free_mr
;
892 struct hns_roce_v1_priv
*priv
;
893 struct completion comp
;
895 msecs_to_jiffies(HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS
) + jiffies
;
897 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
898 free_mr
= &priv
->free_mr
;
900 lp_qp_work
= kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work
),
903 INIT_WORK(&(lp_qp_work
->work
), hns_roce_v1_recreate_lp_qp_work_fn
);
905 lp_qp_work
->ib_dev
= &(hr_dev
->ib_dev
);
906 lp_qp_work
->comp
= &comp
;
907 lp_qp_work
->comp_flag
= 1;
909 init_completion(lp_qp_work
->comp
);
911 queue_work(free_mr
->free_mr_wq
, &(lp_qp_work
->work
));
913 while (time_before_eq(jiffies
, end
)) {
914 if (try_wait_for_completion(&comp
))
916 msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE
);
919 lp_qp_work
->comp_flag
= 0;
920 if (try_wait_for_completion(&comp
))
923 dev_warn(dev
, "recreate lp qp failed 20s timeout and return failed!\n");
927 static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp
*hr_qp
)
929 struct hns_roce_dev
*hr_dev
= to_hr_dev(hr_qp
->ibqp
.device
);
930 struct device
*dev
= &hr_dev
->pdev
->dev
;
931 struct ib_send_wr send_wr
, *bad_wr
;
934 memset(&send_wr
, 0, sizeof(send_wr
));
937 send_wr
.send_flags
= 0;
938 send_wr
.sg_list
= NULL
;
939 send_wr
.wr_id
= (unsigned long long)&send_wr
;
940 send_wr
.opcode
= IB_WR_RDMA_WRITE
;
942 ret
= hns_roce_v1_post_send(&hr_qp
->ibqp
, &send_wr
, &bad_wr
);
944 dev_err(dev
, "Post write wqe for mr free failed(%d)!", ret
);
951 static void hns_roce_v1_mr_free_work_fn(struct work_struct
*work
)
953 struct hns_roce_mr_free_work
*mr_work
;
954 struct ib_wc wc
[HNS_ROCE_V1_RESV_QP
];
955 struct hns_roce_free_mr
*free_mr
;
956 struct hns_roce_cq
*mr_free_cq
;
957 struct hns_roce_v1_priv
*priv
;
958 struct hns_roce_dev
*hr_dev
;
959 struct hns_roce_mr
*hr_mr
;
960 struct hns_roce_qp
*hr_qp
;
963 msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS
) + jiffies
;
968 mr_work
= container_of(work
, struct hns_roce_mr_free_work
, work
);
969 hr_mr
= (struct hns_roce_mr
*)mr_work
->mr
;
970 hr_dev
= to_hr_dev(mr_work
->ib_dev
);
971 dev
= &hr_dev
->pdev
->dev
;
973 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
974 free_mr
= &priv
->free_mr
;
975 mr_free_cq
= free_mr
->mr_free_cq
;
977 for (i
= 0; i
< HNS_ROCE_V1_RESV_QP
; i
++) {
978 hr_qp
= free_mr
->mr_free_qp
[i
];
979 ret
= hns_roce_v1_send_lp_wqe(hr_qp
);
982 "Send wqe (qp:0x%lx) for mr free failed(%d)!\n",
988 ne
= HNS_ROCE_V1_RESV_QP
;
990 ret
= hns_roce_v1_poll_cq(&mr_free_cq
->ib_cq
, ne
, wc
);
993 "(qp:0x%lx) starts, Poll cqe failed(%d) for mr 0x%x free! Remain %d cqe\n",
994 hr_qp
->qpn
, ret
, hr_mr
->key
, ne
);
998 msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE
);
999 } while (ne
&& time_before_eq(jiffies
, end
));
1003 "Poll cqe for mr 0x%x free timeout! Remain %d cqe\n",
1007 if (mr_work
->comp_flag
)
1008 complete(mr_work
->comp
);
1012 int hns_roce_v1_dereg_mr(struct hns_roce_dev
*hr_dev
, struct hns_roce_mr
*mr
)
1014 struct device
*dev
= &hr_dev
->pdev
->dev
;
1015 struct hns_roce_mr_free_work
*mr_work
;
1016 struct hns_roce_free_mr
*free_mr
;
1017 struct hns_roce_v1_priv
*priv
;
1018 struct completion comp
;
1020 msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS
) + jiffies
;
1021 unsigned long start
= jiffies
;
1025 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1026 free_mr
= &priv
->free_mr
;
1029 if (hns_roce_hw2sw_mpt(hr_dev
, NULL
, key_to_hw_index(mr
->key
)
1030 & (hr_dev
->caps
.num_mtpts
- 1)))
1031 dev_warn(dev
, "HW2SW_MPT failed!\n");
1034 mr_work
= kzalloc(sizeof(*mr_work
), GFP_KERNEL
);
1040 INIT_WORK(&(mr_work
->work
), hns_roce_v1_mr_free_work_fn
);
1042 mr_work
->ib_dev
= &(hr_dev
->ib_dev
);
1043 mr_work
->comp
= &comp
;
1044 mr_work
->comp_flag
= 1;
1045 mr_work
->mr
= (void *)mr
;
1046 init_completion(mr_work
->comp
);
1048 queue_work(free_mr
->free_mr_wq
, &(mr_work
->work
));
1050 while (time_before_eq(jiffies
, end
)) {
1051 if (try_wait_for_completion(&comp
))
1053 msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE
);
1056 mr_work
->comp_flag
= 0;
1057 if (try_wait_for_completion(&comp
))
1060 dev_warn(dev
, "Free mr work 0x%x over 50s and failed!\n", mr
->key
);
1064 dev_dbg(dev
, "Free mr 0x%x use 0x%x us.\n",
1065 mr
->key
, jiffies_to_usecs(jiffies
) - jiffies_to_usecs(start
));
1067 if (mr
->size
!= ~0ULL) {
1068 npages
= ib_umem_page_count(mr
->umem
);
1069 dma_free_coherent(dev
, npages
* 8, mr
->pbl_buf
,
1073 hns_roce_bitmap_free(&hr_dev
->mr_table
.mtpt_bitmap
,
1074 key_to_hw_index(mr
->key
), 0);
1077 ib_umem_release(mr
->umem
);
1084 static void hns_roce_db_free(struct hns_roce_dev
*hr_dev
)
1086 struct device
*dev
= &hr_dev
->pdev
->dev
;
1087 struct hns_roce_v1_priv
*priv
;
1088 struct hns_roce_db_table
*db
;
1090 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1091 db
= &priv
->db_table
;
1093 if (db
->sdb_ext_mod
) {
1094 dma_free_coherent(dev
, HNS_ROCE_V1_EXT_SDB_SIZE
,
1095 db
->ext_db
->sdb_buf_list
->buf
,
1096 db
->ext_db
->sdb_buf_list
->map
);
1097 kfree(db
->ext_db
->sdb_buf_list
);
1100 if (db
->odb_ext_mod
) {
1101 dma_free_coherent(dev
, HNS_ROCE_V1_EXT_ODB_SIZE
,
1102 db
->ext_db
->odb_buf_list
->buf
,
1103 db
->ext_db
->odb_buf_list
->map
);
1104 kfree(db
->ext_db
->odb_buf_list
);
1110 static int hns_roce_raq_init(struct hns_roce_dev
*hr_dev
)
1116 struct hns_roce_v1_priv
*priv
;
1117 struct hns_roce_raq_table
*raq
;
1118 struct device
*dev
= &hr_dev
->pdev
->dev
;
1120 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1121 raq
= &priv
->raq_table
;
1123 raq
->e_raq_buf
= kzalloc(sizeof(*(raq
->e_raq_buf
)), GFP_KERNEL
);
1124 if (!raq
->e_raq_buf
)
1127 raq
->e_raq_buf
->buf
= dma_alloc_coherent(dev
, HNS_ROCE_V1_RAQ_SIZE
,
1129 if (!raq
->e_raq_buf
->buf
) {
1131 goto err_dma_alloc_raq
;
1133 raq
->e_raq_buf
->map
= addr
;
1135 /* Configure raq extended address. 48bit 4K align*/
1136 roce_write(hr_dev
, ROCEE_EXT_RAQ_REG
, raq
->e_raq_buf
->map
>> 12);
1138 /* Configure raq_shift */
1139 raq_shift
= ilog2(HNS_ROCE_V1_RAQ_SIZE
/ HNS_ROCE_V1_RAQ_ENTRY
);
1140 val
= roce_read(hr_dev
, ROCEE_EXT_RAQ_H_REG
);
1141 roce_set_field(val
, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M
,
1142 ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S
, raq_shift
);
1144 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1145 * using 4K page, and shift more 32 because of
1146 * caculating the high 32 bit value evaluated to hardware.
1148 roce_set_field(val
, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M
,
1149 ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S
,
1150 raq
->e_raq_buf
->map
>> 44);
1151 roce_write(hr_dev
, ROCEE_EXT_RAQ_H_REG
, val
);
1152 dev_dbg(dev
, "Configure raq_shift 0x%x.\n", val
);
1154 /* Configure raq threshold */
1155 val
= roce_read(hr_dev
, ROCEE_RAQ_WL_REG
);
1156 roce_set_field(val
, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M
,
1157 ROCEE_RAQ_WL_ROCEE_RAQ_WL_S
,
1158 HNS_ROCE_V1_EXT_RAQ_WF
);
1159 roce_write(hr_dev
, ROCEE_RAQ_WL_REG
, val
);
1160 dev_dbg(dev
, "Configure raq_wl 0x%x.\n", val
);
1162 /* Enable extend raq */
1163 val
= roce_read(hr_dev
, ROCEE_WRMS_POL_TIME_INTERVAL_REG
);
1165 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M
,
1166 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S
,
1167 POL_TIME_INTERVAL_VAL
);
1168 roce_set_bit(val
, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE
, 1);
1170 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M
,
1171 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S
,
1174 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S
, 1);
1175 roce_write(hr_dev
, ROCEE_WRMS_POL_TIME_INTERVAL_REG
, val
);
1176 dev_dbg(dev
, "Configure WrmsPolTimeInterval 0x%x.\n", val
);
1178 /* Enable raq drop */
1179 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
1180 roce_set_bit(val
, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S
, 1);
1181 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
1182 dev_dbg(dev
, "Configure GlbCfg = 0x%x.\n", val
);
1187 kfree(raq
->e_raq_buf
);
1191 static void hns_roce_raq_free(struct hns_roce_dev
*hr_dev
)
1193 struct device
*dev
= &hr_dev
->pdev
->dev
;
1194 struct hns_roce_v1_priv
*priv
;
1195 struct hns_roce_raq_table
*raq
;
1197 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1198 raq
= &priv
->raq_table
;
1200 dma_free_coherent(dev
, HNS_ROCE_V1_RAQ_SIZE
, raq
->e_raq_buf
->buf
,
1201 raq
->e_raq_buf
->map
);
1202 kfree(raq
->e_raq_buf
);
1205 static void hns_roce_port_enable(struct hns_roce_dev
*hr_dev
, int enable_flag
)
1210 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
1211 /* Open all ports */
1212 roce_set_field(val
, ROCEE_GLB_CFG_ROCEE_PORT_ST_M
,
1213 ROCEE_GLB_CFG_ROCEE_PORT_ST_S
,
1215 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
1217 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
1218 /* Close all ports */
1219 roce_set_field(val
, ROCEE_GLB_CFG_ROCEE_PORT_ST_M
,
1220 ROCEE_GLB_CFG_ROCEE_PORT_ST_S
, 0x0);
1221 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
1225 static int hns_roce_bt_init(struct hns_roce_dev
*hr_dev
)
1227 struct device
*dev
= &hr_dev
->pdev
->dev
;
1228 struct hns_roce_v1_priv
*priv
;
1231 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1233 priv
->bt_table
.qpc_buf
.buf
= dma_alloc_coherent(dev
,
1234 HNS_ROCE_BT_RSV_BUF_SIZE
, &priv
->bt_table
.qpc_buf
.map
,
1236 if (!priv
->bt_table
.qpc_buf
.buf
)
1239 priv
->bt_table
.mtpt_buf
.buf
= dma_alloc_coherent(dev
,
1240 HNS_ROCE_BT_RSV_BUF_SIZE
, &priv
->bt_table
.mtpt_buf
.map
,
1242 if (!priv
->bt_table
.mtpt_buf
.buf
) {
1244 goto err_failed_alloc_mtpt_buf
;
1247 priv
->bt_table
.cqc_buf
.buf
= dma_alloc_coherent(dev
,
1248 HNS_ROCE_BT_RSV_BUF_SIZE
, &priv
->bt_table
.cqc_buf
.map
,
1250 if (!priv
->bt_table
.cqc_buf
.buf
) {
1252 goto err_failed_alloc_cqc_buf
;
1257 err_failed_alloc_cqc_buf
:
1258 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1259 priv
->bt_table
.mtpt_buf
.buf
, priv
->bt_table
.mtpt_buf
.map
);
1261 err_failed_alloc_mtpt_buf
:
1262 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1263 priv
->bt_table
.qpc_buf
.buf
, priv
->bt_table
.qpc_buf
.map
);
1268 static void hns_roce_bt_free(struct hns_roce_dev
*hr_dev
)
1270 struct device
*dev
= &hr_dev
->pdev
->dev
;
1271 struct hns_roce_v1_priv
*priv
;
1273 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1275 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1276 priv
->bt_table
.cqc_buf
.buf
, priv
->bt_table
.cqc_buf
.map
);
1278 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1279 priv
->bt_table
.mtpt_buf
.buf
, priv
->bt_table
.mtpt_buf
.map
);
1281 dma_free_coherent(dev
, HNS_ROCE_BT_RSV_BUF_SIZE
,
1282 priv
->bt_table
.qpc_buf
.buf
, priv
->bt_table
.qpc_buf
.map
);
1285 static int hns_roce_tptr_init(struct hns_roce_dev
*hr_dev
)
1287 struct device
*dev
= &hr_dev
->pdev
->dev
;
1288 struct hns_roce_buf_list
*tptr_buf
;
1289 struct hns_roce_v1_priv
*priv
;
1291 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1292 tptr_buf
= &priv
->tptr_table
.tptr_buf
;
1295 * This buffer will be used for CQ's tptr(tail pointer), also
1296 * named ci(customer index). Every CQ will use 2 bytes to save
1297 * cqe ci in hip06. Hardware will read this area to get new ci
1298 * when the queue is almost full.
1300 tptr_buf
->buf
= dma_alloc_coherent(dev
, HNS_ROCE_V1_TPTR_BUF_SIZE
,
1301 &tptr_buf
->map
, GFP_KERNEL
);
1305 hr_dev
->tptr_dma_addr
= tptr_buf
->map
;
1306 hr_dev
->tptr_size
= HNS_ROCE_V1_TPTR_BUF_SIZE
;
1311 static void hns_roce_tptr_free(struct hns_roce_dev
*hr_dev
)
1313 struct device
*dev
= &hr_dev
->pdev
->dev
;
1314 struct hns_roce_buf_list
*tptr_buf
;
1315 struct hns_roce_v1_priv
*priv
;
1317 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1318 tptr_buf
= &priv
->tptr_table
.tptr_buf
;
1320 dma_free_coherent(dev
, HNS_ROCE_V1_TPTR_BUF_SIZE
,
1321 tptr_buf
->buf
, tptr_buf
->map
);
1324 static int hns_roce_free_mr_init(struct hns_roce_dev
*hr_dev
)
1326 struct device
*dev
= &hr_dev
->pdev
->dev
;
1327 struct hns_roce_free_mr
*free_mr
;
1328 struct hns_roce_v1_priv
*priv
;
1331 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1332 free_mr
= &priv
->free_mr
;
1334 free_mr
->free_mr_wq
= create_singlethread_workqueue("hns_roce_free_mr");
1335 if (!free_mr
->free_mr_wq
) {
1336 dev_err(dev
, "Create free mr workqueue failed!\n");
1340 ret
= hns_roce_v1_rsv_lp_qp(hr_dev
);
1342 dev_err(dev
, "Reserved loop qp failed(%d)!\n", ret
);
1343 flush_workqueue(free_mr
->free_mr_wq
);
1344 destroy_workqueue(free_mr
->free_mr_wq
);
1350 static void hns_roce_free_mr_free(struct hns_roce_dev
*hr_dev
)
1352 struct hns_roce_free_mr
*free_mr
;
1353 struct hns_roce_v1_priv
*priv
;
1355 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1356 free_mr
= &priv
->free_mr
;
1358 flush_workqueue(free_mr
->free_mr_wq
);
1359 destroy_workqueue(free_mr
->free_mr_wq
);
1361 hns_roce_v1_release_lp_qp(hr_dev
);
1365 * hns_roce_v1_reset - reset RoCE
1366 * @hr_dev: RoCE device struct pointer
1367 * @enable: true -- drop reset, false -- reset
1368 * return 0 - success , negative --fail
1370 int hns_roce_v1_reset(struct hns_roce_dev
*hr_dev
, bool dereset
)
1372 struct device_node
*dsaf_node
;
1373 struct device
*dev
= &hr_dev
->pdev
->dev
;
1374 struct device_node
*np
= dev
->of_node
;
1375 struct fwnode_handle
*fwnode
;
1378 /* check if this is DT/ACPI case */
1379 if (dev_of_node(dev
)) {
1380 dsaf_node
= of_parse_phandle(np
, "dsaf-handle", 0);
1382 dev_err(dev
, "could not find dsaf-handle\n");
1385 fwnode
= &dsaf_node
->fwnode
;
1386 } else if (is_acpi_device_node(dev
->fwnode
)) {
1387 struct acpi_reference_args args
;
1389 ret
= acpi_node_get_property_reference(dev
->fwnode
,
1390 "dsaf-handle", 0, &args
);
1392 dev_err(dev
, "could not find dsaf-handle\n");
1395 fwnode
= acpi_fwnode_handle(args
.adev
);
1397 dev_err(dev
, "cannot read data from DT or ACPI\n");
1401 ret
= hns_dsaf_roce_reset(fwnode
, false);
1406 msleep(SLEEP_TIME_INTERVAL
);
1407 ret
= hns_dsaf_roce_reset(fwnode
, true);
1413 static int hns_roce_des_qp_init(struct hns_roce_dev
*hr_dev
)
1415 struct device
*dev
= &hr_dev
->pdev
->dev
;
1416 struct hns_roce_v1_priv
*priv
;
1417 struct hns_roce_des_qp
*des_qp
;
1419 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1420 des_qp
= &priv
->des_qp
;
1422 des_qp
->requeue_flag
= 1;
1423 des_qp
->qp_wq
= create_singlethread_workqueue("hns_roce_destroy_qp");
1424 if (!des_qp
->qp_wq
) {
1425 dev_err(dev
, "Create destroy qp workqueue failed!\n");
1432 static void hns_roce_des_qp_free(struct hns_roce_dev
*hr_dev
)
1434 struct hns_roce_v1_priv
*priv
;
1435 struct hns_roce_des_qp
*des_qp
;
1437 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1438 des_qp
= &priv
->des_qp
;
1440 des_qp
->requeue_flag
= 0;
1441 flush_workqueue(des_qp
->qp_wq
);
1442 destroy_workqueue(des_qp
->qp_wq
);
1445 void hns_roce_v1_profile(struct hns_roce_dev
*hr_dev
)
1448 struct hns_roce_caps
*caps
= &hr_dev
->caps
;
1450 hr_dev
->vendor_id
= le32_to_cpu(roce_read(hr_dev
, ROCEE_VENDOR_ID_REG
));
1451 hr_dev
->vendor_part_id
= le32_to_cpu(roce_read(hr_dev
,
1452 ROCEE_VENDOR_PART_ID_REG
));
1453 hr_dev
->sys_image_guid
= le32_to_cpu(roce_read(hr_dev
,
1454 ROCEE_SYS_IMAGE_GUID_L_REG
)) |
1455 ((u64
)le32_to_cpu(roce_read(hr_dev
,
1456 ROCEE_SYS_IMAGE_GUID_H_REG
)) << 32);
1457 hr_dev
->hw_rev
= HNS_ROCE_HW_VER1
;
1459 caps
->num_qps
= HNS_ROCE_V1_MAX_QP_NUM
;
1460 caps
->max_wqes
= HNS_ROCE_V1_MAX_WQE_NUM
;
1461 caps
->num_cqs
= HNS_ROCE_V1_MAX_CQ_NUM
;
1462 caps
->max_cqes
= HNS_ROCE_V1_MAX_CQE_NUM
;
1463 caps
->max_sq_sg
= HNS_ROCE_V1_SG_NUM
;
1464 caps
->max_rq_sg
= HNS_ROCE_V1_SG_NUM
;
1465 caps
->max_sq_inline
= HNS_ROCE_V1_INLINE_SIZE
;
1466 caps
->num_uars
= HNS_ROCE_V1_UAR_NUM
;
1467 caps
->phy_num_uars
= HNS_ROCE_V1_PHY_UAR_NUM
;
1468 caps
->num_aeq_vectors
= HNS_ROCE_AEQE_VEC_NUM
;
1469 caps
->num_comp_vectors
= HNS_ROCE_COMP_VEC_NUM
;
1470 caps
->num_other_vectors
= HNS_ROCE_AEQE_OF_VEC_NUM
;
1471 caps
->num_mtpts
= HNS_ROCE_V1_MAX_MTPT_NUM
;
1472 caps
->num_mtt_segs
= HNS_ROCE_V1_MAX_MTT_SEGS
;
1473 caps
->num_pds
= HNS_ROCE_V1_MAX_PD_NUM
;
1474 caps
->max_qp_init_rdma
= HNS_ROCE_V1_MAX_QP_INIT_RDMA
;
1475 caps
->max_qp_dest_rdma
= HNS_ROCE_V1_MAX_QP_DEST_RDMA
;
1476 caps
->max_sq_desc_sz
= HNS_ROCE_V1_MAX_SQ_DESC_SZ
;
1477 caps
->max_rq_desc_sz
= HNS_ROCE_V1_MAX_RQ_DESC_SZ
;
1478 caps
->qpc_entry_sz
= HNS_ROCE_V1_QPC_ENTRY_SIZE
;
1479 caps
->irrl_entry_sz
= HNS_ROCE_V1_IRRL_ENTRY_SIZE
;
1480 caps
->cqc_entry_sz
= HNS_ROCE_V1_CQC_ENTRY_SIZE
;
1481 caps
->mtpt_entry_sz
= HNS_ROCE_V1_MTPT_ENTRY_SIZE
;
1482 caps
->mtt_entry_sz
= HNS_ROCE_V1_MTT_ENTRY_SIZE
;
1483 caps
->cq_entry_sz
= HNS_ROCE_V1_CQE_ENTRY_SIZE
;
1484 caps
->page_size_cap
= HNS_ROCE_V1_PAGE_SIZE_SUPPORT
;
1485 caps
->reserved_lkey
= 0;
1486 caps
->reserved_pds
= 0;
1487 caps
->reserved_mrws
= 1;
1488 caps
->reserved_uars
= 0;
1489 caps
->reserved_cqs
= 0;
1491 for (i
= 0; i
< caps
->num_ports
; i
++)
1492 caps
->pkey_table_len
[i
] = 1;
1494 for (i
= 0; i
< caps
->num_ports
; i
++) {
1495 /* Six ports shared 16 GID in v1 engine */
1496 if (i
>= (HNS_ROCE_V1_GID_NUM
% caps
->num_ports
))
1497 caps
->gid_table_len
[i
] = HNS_ROCE_V1_GID_NUM
/
1500 caps
->gid_table_len
[i
] = HNS_ROCE_V1_GID_NUM
/
1501 caps
->num_ports
+ 1;
1504 for (i
= 0; i
< caps
->num_comp_vectors
; i
++)
1505 caps
->ceqe_depth
[i
] = HNS_ROCE_V1_NUM_COMP_EQE
;
1507 caps
->aeqe_depth
= HNS_ROCE_V1_NUM_ASYNC_EQE
;
1508 caps
->local_ca_ack_delay
= le32_to_cpu(roce_read(hr_dev
,
1509 ROCEE_ACK_DELAY_REG
));
1510 caps
->max_mtu
= IB_MTU_2048
;
1513 int hns_roce_v1_init(struct hns_roce_dev
*hr_dev
)
1517 struct device
*dev
= &hr_dev
->pdev
->dev
;
1519 /* DMAE user config */
1520 val
= roce_read(hr_dev
, ROCEE_DMAE_USER_CFG1_REG
);
1521 roce_set_field(val
, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M
,
1522 ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S
, 0xf);
1523 roce_set_field(val
, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M
,
1524 ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S
,
1525 1 << PAGES_SHIFT_16
);
1526 roce_write(hr_dev
, ROCEE_DMAE_USER_CFG1_REG
, val
);
1528 val
= roce_read(hr_dev
, ROCEE_DMAE_USER_CFG2_REG
);
1529 roce_set_field(val
, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M
,
1530 ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S
, 0xf);
1531 roce_set_field(val
, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M
,
1532 ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S
,
1533 1 << PAGES_SHIFT_16
);
1535 ret
= hns_roce_db_init(hr_dev
);
1537 dev_err(dev
, "doorbell init failed!\n");
1541 ret
= hns_roce_raq_init(hr_dev
);
1543 dev_err(dev
, "raq init failed!\n");
1544 goto error_failed_raq_init
;
1547 ret
= hns_roce_bt_init(hr_dev
);
1549 dev_err(dev
, "bt init failed!\n");
1550 goto error_failed_bt_init
;
1553 ret
= hns_roce_tptr_init(hr_dev
);
1555 dev_err(dev
, "tptr init failed!\n");
1556 goto error_failed_tptr_init
;
1559 ret
= hns_roce_des_qp_init(hr_dev
);
1561 dev_err(dev
, "des qp init failed!\n");
1562 goto error_failed_des_qp_init
;
1565 ret
= hns_roce_free_mr_init(hr_dev
);
1567 dev_err(dev
, "free mr init failed!\n");
1568 goto error_failed_free_mr_init
;
1571 hns_roce_port_enable(hr_dev
, HNS_ROCE_PORT_UP
);
1575 error_failed_free_mr_init
:
1576 hns_roce_des_qp_free(hr_dev
);
1578 error_failed_des_qp_init
:
1579 hns_roce_tptr_free(hr_dev
);
1581 error_failed_tptr_init
:
1582 hns_roce_bt_free(hr_dev
);
1584 error_failed_bt_init
:
1585 hns_roce_raq_free(hr_dev
);
1587 error_failed_raq_init
:
1588 hns_roce_db_free(hr_dev
);
1592 void hns_roce_v1_exit(struct hns_roce_dev
*hr_dev
)
1594 hns_roce_port_enable(hr_dev
, HNS_ROCE_PORT_DOWN
);
1595 hns_roce_free_mr_free(hr_dev
);
1596 hns_roce_des_qp_free(hr_dev
);
1597 hns_roce_tptr_free(hr_dev
);
1598 hns_roce_bt_free(hr_dev
);
1599 hns_roce_raq_free(hr_dev
);
1600 hns_roce_db_free(hr_dev
);
1603 void hns_roce_v1_set_gid(struct hns_roce_dev
*hr_dev
, u8 port
, int gid_index
,
1609 gid_idx
= hns_get_gid_index(hr_dev
, port
, gid_index
);
1611 p
= (u32
*)&gid
->raw
[0];
1612 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_L_0_REG
+
1613 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
1615 p
= (u32
*)&gid
->raw
[4];
1616 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_ML_0_REG
+
1617 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
1619 p
= (u32
*)&gid
->raw
[8];
1620 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_MH_0_REG
+
1621 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
1623 p
= (u32
*)&gid
->raw
[0xc];
1624 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_H_0_REG
+
1625 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
1628 void hns_roce_v1_set_mac(struct hns_roce_dev
*hr_dev
, u8 phy_port
, u8
*addr
)
1637 * When mac changed, loopback may fail
1638 * because of smac not equal to dmac.
1639 * We Need to release and create reserved qp again.
1641 if (hr_dev
->hw
->dereg_mr
&& hns_roce_v1_recreate_lp_qp(hr_dev
))
1642 dev_warn(&hr_dev
->pdev
->dev
, "recreate lp qp timeout!\n");
1644 p
= (u32
*)(&addr
[0]);
1646 roce_raw_write(reg_smac_l
, hr_dev
->reg_base
+ ROCEE_SMAC_L_0_REG
+
1647 PHY_PORT_OFFSET
* phy_port
);
1649 val
= roce_read(hr_dev
,
1650 ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
);
1651 p_h
= (u16
*)(&addr
[4]);
1653 roce_set_field(val
, ROCEE_SMAC_H_ROCEE_SMAC_H_M
,
1654 ROCEE_SMAC_H_ROCEE_SMAC_H_S
, reg_smac_h
);
1655 roce_write(hr_dev
, ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
,
1659 void hns_roce_v1_set_mtu(struct hns_roce_dev
*hr_dev
, u8 phy_port
,
1664 val
= roce_read(hr_dev
,
1665 ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
);
1666 roce_set_field(val
, ROCEE_SMAC_H_ROCEE_PORT_MTU_M
,
1667 ROCEE_SMAC_H_ROCEE_PORT_MTU_S
, mtu
);
1668 roce_write(hr_dev
, ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
,
1672 int hns_roce_v1_write_mtpt(void *mb_buf
, struct hns_roce_mr
*mr
,
1673 unsigned long mtpt_idx
)
1675 struct hns_roce_v1_mpt_entry
*mpt_entry
;
1676 struct scatterlist
*sg
;
1681 /* MPT filled into mailbox buf */
1682 mpt_entry
= (struct hns_roce_v1_mpt_entry
*)mb_buf
;
1683 memset(mpt_entry
, 0, sizeof(*mpt_entry
));
1685 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_KEY_STATE_M
,
1686 MPT_BYTE_4_KEY_STATE_S
, KEY_VALID
);
1687 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_KEY_M
,
1688 MPT_BYTE_4_KEY_S
, mr
->key
);
1689 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_PAGE_SIZE_M
,
1690 MPT_BYTE_4_PAGE_SIZE_S
, MR_SIZE_4K
);
1691 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_MW_TYPE_S
, 0);
1692 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_MW_BIND_ENABLE_S
,
1693 (mr
->access
& IB_ACCESS_MW_BIND
? 1 : 0));
1694 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_OWN_S
, 0);
1695 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M
,
1696 MPT_BYTE_4_MEMORY_LOCATION_TYPE_S
, mr
->type
);
1697 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_ATOMIC_S
, 0);
1698 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_LOCAL_WRITE_S
,
1699 (mr
->access
& IB_ACCESS_LOCAL_WRITE
? 1 : 0));
1700 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_WRITE_S
,
1701 (mr
->access
& IB_ACCESS_REMOTE_WRITE
? 1 : 0));
1702 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_READ_S
,
1703 (mr
->access
& IB_ACCESS_REMOTE_READ
? 1 : 0));
1704 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S
,
1706 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_ADDRESS_TYPE_S
, 0);
1708 roce_set_field(mpt_entry
->mpt_byte_12
, MPT_BYTE_12_PBL_ADDR_H_M
,
1709 MPT_BYTE_12_PBL_ADDR_H_S
, 0);
1710 roce_set_field(mpt_entry
->mpt_byte_12
, MPT_BYTE_12_MW_BIND_COUNTER_M
,
1711 MPT_BYTE_12_MW_BIND_COUNTER_S
, 0);
1713 mpt_entry
->virt_addr_l
= (u32
)mr
->iova
;
1714 mpt_entry
->virt_addr_h
= (u32
)(mr
->iova
>> 32);
1715 mpt_entry
->length
= (u32
)mr
->size
;
1717 roce_set_field(mpt_entry
->mpt_byte_28
, MPT_BYTE_28_PD_M
,
1718 MPT_BYTE_28_PD_S
, mr
->pd
);
1719 roce_set_field(mpt_entry
->mpt_byte_28
, MPT_BYTE_28_L_KEY_IDX_L_M
,
1720 MPT_BYTE_28_L_KEY_IDX_L_S
, mtpt_idx
);
1721 roce_set_field(mpt_entry
->mpt_byte_64
, MPT_BYTE_64_L_KEY_IDX_H_M
,
1722 MPT_BYTE_64_L_KEY_IDX_H_S
, mtpt_idx
>> MTPT_IDX_SHIFT
);
1724 /* DMA memory register */
1725 if (mr
->type
== MR_TYPE_DMA
)
1728 pages
= (u64
*) __get_free_page(GFP_KERNEL
);
1733 for_each_sg(mr
->umem
->sg_head
.sgl
, sg
, mr
->umem
->nmap
, entry
) {
1734 pages
[i
] = ((u64
)sg_dma_address(sg
)) >> 12;
1736 /* Directly record to MTPT table firstly 7 entry */
1737 if (i
>= HNS_ROCE_MAX_INNER_MTPT_NUM
)
1742 /* Register user mr */
1743 for (i
= 0; i
< HNS_ROCE_MAX_INNER_MTPT_NUM
; i
++) {
1746 mpt_entry
->pa0_l
= cpu_to_le32((u32
)(pages
[i
]));
1747 roce_set_field(mpt_entry
->mpt_byte_36
,
1748 MPT_BYTE_36_PA0_H_M
,
1749 MPT_BYTE_36_PA0_H_S
,
1750 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_32
)));
1753 roce_set_field(mpt_entry
->mpt_byte_36
,
1754 MPT_BYTE_36_PA1_L_M
,
1755 MPT_BYTE_36_PA1_L_S
,
1756 cpu_to_le32((u32
)(pages
[i
])));
1757 roce_set_field(mpt_entry
->mpt_byte_40
,
1758 MPT_BYTE_40_PA1_H_M
,
1759 MPT_BYTE_40_PA1_H_S
,
1760 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_24
)));
1763 roce_set_field(mpt_entry
->mpt_byte_40
,
1764 MPT_BYTE_40_PA2_L_M
,
1765 MPT_BYTE_40_PA2_L_S
,
1766 cpu_to_le32((u32
)(pages
[i
])));
1767 roce_set_field(mpt_entry
->mpt_byte_44
,
1768 MPT_BYTE_44_PA2_H_M
,
1769 MPT_BYTE_44_PA2_H_S
,
1770 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_16
)));
1773 roce_set_field(mpt_entry
->mpt_byte_44
,
1774 MPT_BYTE_44_PA3_L_M
,
1775 MPT_BYTE_44_PA3_L_S
,
1776 cpu_to_le32((u32
)(pages
[i
])));
1777 roce_set_field(mpt_entry
->mpt_byte_48
,
1778 MPT_BYTE_48_PA3_H_M
,
1779 MPT_BYTE_48_PA3_H_S
,
1780 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_8
)));
1783 mpt_entry
->pa4_l
= cpu_to_le32((u32
)(pages
[i
]));
1784 roce_set_field(mpt_entry
->mpt_byte_56
,
1785 MPT_BYTE_56_PA4_H_M
,
1786 MPT_BYTE_56_PA4_H_S
,
1787 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_32
)));
1790 roce_set_field(mpt_entry
->mpt_byte_56
,
1791 MPT_BYTE_56_PA5_L_M
,
1792 MPT_BYTE_56_PA5_L_S
,
1793 cpu_to_le32((u32
)(pages
[i
])));
1794 roce_set_field(mpt_entry
->mpt_byte_60
,
1795 MPT_BYTE_60_PA5_H_M
,
1796 MPT_BYTE_60_PA5_H_S
,
1797 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_24
)));
1800 roce_set_field(mpt_entry
->mpt_byte_60
,
1801 MPT_BYTE_60_PA6_L_M
,
1802 MPT_BYTE_60_PA6_L_S
,
1803 cpu_to_le32((u32
)(pages
[i
])));
1804 roce_set_field(mpt_entry
->mpt_byte_64
,
1805 MPT_BYTE_64_PA6_H_M
,
1806 MPT_BYTE_64_PA6_H_S
,
1807 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_16
)));
1814 free_page((unsigned long) pages
);
1816 mpt_entry
->pbl_addr_l
= (u32
)(mr
->pbl_dma_addr
);
1818 roce_set_field(mpt_entry
->mpt_byte_12
, MPT_BYTE_12_PBL_ADDR_H_M
,
1819 MPT_BYTE_12_PBL_ADDR_H_S
,
1820 ((u32
)(mr
->pbl_dma_addr
>> 32)));
1825 static void *get_cqe(struct hns_roce_cq
*hr_cq
, int n
)
1827 return hns_roce_buf_offset(&hr_cq
->hr_buf
.hr_buf
,
1828 n
* HNS_ROCE_V1_CQE_ENTRY_SIZE
);
1831 static void *get_sw_cqe(struct hns_roce_cq
*hr_cq
, int n
)
1833 struct hns_roce_cqe
*hr_cqe
= get_cqe(hr_cq
, n
& hr_cq
->ib_cq
.cqe
);
1835 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1836 return (roce_get_bit(hr_cqe
->cqe_byte_4
, CQE_BYTE_4_OWNER_S
) ^
1837 !!(n
& (hr_cq
->ib_cq
.cqe
+ 1))) ? hr_cqe
: NULL
;
1840 static struct hns_roce_cqe
*next_cqe_sw(struct hns_roce_cq
*hr_cq
)
1842 return get_sw_cqe(hr_cq
, hr_cq
->cons_index
);
1845 void hns_roce_v1_cq_set_ci(struct hns_roce_cq
*hr_cq
, u32 cons_index
)
1849 doorbell
[0] = cons_index
& ((hr_cq
->cq_depth
<< 1) - 1);
1851 roce_set_bit(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S
, 1);
1852 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M
,
1853 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S
, 3);
1854 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M
,
1855 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S
, 0);
1856 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M
,
1857 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S
, hr_cq
->cqn
);
1859 hns_roce_write64_k(doorbell
, hr_cq
->cq_db_l
);
1862 static void __hns_roce_v1_cq_clean(struct hns_roce_cq
*hr_cq
, u32 qpn
,
1863 struct hns_roce_srq
*srq
)
1865 struct hns_roce_cqe
*cqe
, *dest
;
1870 for (prod_index
= hr_cq
->cons_index
; get_sw_cqe(hr_cq
, prod_index
);
1872 if (prod_index
== hr_cq
->cons_index
+ hr_cq
->ib_cq
.cqe
)
1877 * Now backwards through the CQ, removing CQ entries
1878 * that match our QP by overwriting them with next entries.
1880 while ((int) --prod_index
- (int) hr_cq
->cons_index
>= 0) {
1881 cqe
= get_cqe(hr_cq
, prod_index
& hr_cq
->ib_cq
.cqe
);
1882 if ((roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
1883 CQE_BYTE_16_LOCAL_QPN_S
) &
1884 HNS_ROCE_CQE_QPN_MASK
) == qpn
) {
1885 /* In v1 engine, not support SRQ */
1887 } else if (nfreed
) {
1888 dest
= get_cqe(hr_cq
, (prod_index
+ nfreed
) &
1890 owner_bit
= roce_get_bit(dest
->cqe_byte_4
,
1891 CQE_BYTE_4_OWNER_S
);
1892 memcpy(dest
, cqe
, sizeof(*cqe
));
1893 roce_set_bit(dest
->cqe_byte_4
, CQE_BYTE_4_OWNER_S
,
1899 hr_cq
->cons_index
+= nfreed
;
1901 * Make sure update of buffer contents is done before
1902 * updating consumer index.
1906 hns_roce_v1_cq_set_ci(hr_cq
, hr_cq
->cons_index
);
1910 static void hns_roce_v1_cq_clean(struct hns_roce_cq
*hr_cq
, u32 qpn
,
1911 struct hns_roce_srq
*srq
)
1913 spin_lock_irq(&hr_cq
->lock
);
1914 __hns_roce_v1_cq_clean(hr_cq
, qpn
, srq
);
1915 spin_unlock_irq(&hr_cq
->lock
);
1918 void hns_roce_v1_write_cqc(struct hns_roce_dev
*hr_dev
,
1919 struct hns_roce_cq
*hr_cq
, void *mb_buf
, u64
*mtts
,
1920 dma_addr_t dma_handle
, int nent
, u32 vector
)
1922 struct hns_roce_cq_context
*cq_context
= NULL
;
1923 struct hns_roce_buf_list
*tptr_buf
;
1924 struct hns_roce_v1_priv
*priv
;
1925 dma_addr_t tptr_dma_addr
;
1928 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
1929 tptr_buf
= &priv
->tptr_table
.tptr_buf
;
1931 cq_context
= mb_buf
;
1932 memset(cq_context
, 0, sizeof(*cq_context
));
1934 /* Get the tptr for this CQ. */
1935 offset
= hr_cq
->cqn
* HNS_ROCE_V1_TPTR_ENTRY_SIZE
;
1936 tptr_dma_addr
= tptr_buf
->map
+ offset
;
1937 hr_cq
->tptr_addr
= (u16
*)(tptr_buf
->buf
+ offset
);
1939 /* Register cq_context members */
1940 roce_set_field(cq_context
->cqc_byte_4
,
1941 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M
,
1942 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S
, CQ_STATE_VALID
);
1943 roce_set_field(cq_context
->cqc_byte_4
, CQ_CONTEXT_CQC_BYTE_4_CQN_M
,
1944 CQ_CONTEXT_CQC_BYTE_4_CQN_S
, hr_cq
->cqn
);
1945 cq_context
->cqc_byte_4
= cpu_to_le32(cq_context
->cqc_byte_4
);
1947 cq_context
->cq_bt_l
= (u32
)dma_handle
;
1948 cq_context
->cq_bt_l
= cpu_to_le32(cq_context
->cq_bt_l
);
1950 roce_set_field(cq_context
->cqc_byte_12
,
1951 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M
,
1952 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S
,
1953 ((u64
)dma_handle
>> 32));
1954 roce_set_field(cq_context
->cqc_byte_12
,
1955 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M
,
1956 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S
,
1957 ilog2((unsigned int)nent
));
1958 roce_set_field(cq_context
->cqc_byte_12
, CQ_CONTEXT_CQC_BYTE_12_CEQN_M
,
1959 CQ_CONTEXT_CQC_BYTE_12_CEQN_S
, vector
);
1960 cq_context
->cqc_byte_12
= cpu_to_le32(cq_context
->cqc_byte_12
);
1962 cq_context
->cur_cqe_ba0_l
= (u32
)(mtts
[0]);
1963 cq_context
->cur_cqe_ba0_l
= cpu_to_le32(cq_context
->cur_cqe_ba0_l
);
1965 roce_set_field(cq_context
->cqc_byte_20
,
1966 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M
,
1967 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S
,
1968 cpu_to_le32((mtts
[0]) >> 32));
1969 /* Dedicated hardware, directly set 0 */
1970 roce_set_field(cq_context
->cqc_byte_20
,
1971 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M
,
1972 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S
, 0);
1974 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1975 * using 4K page, and shift more 32 because of
1976 * caculating the high 32 bit value evaluated to hardware.
1978 roce_set_field(cq_context
->cqc_byte_20
,
1979 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M
,
1980 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S
,
1981 tptr_dma_addr
>> 44);
1982 cq_context
->cqc_byte_20
= cpu_to_le32(cq_context
->cqc_byte_20
);
1984 cq_context
->cqe_tptr_addr_l
= (u32
)(tptr_dma_addr
>> 12);
1986 roce_set_field(cq_context
->cqc_byte_32
,
1987 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M
,
1988 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S
, 0);
1989 roce_set_bit(cq_context
->cqc_byte_32
,
1990 CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S
, 0);
1991 roce_set_bit(cq_context
->cqc_byte_32
,
1992 CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S
, 0);
1993 roce_set_bit(cq_context
->cqc_byte_32
,
1994 CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S
, 0);
1995 roce_set_bit(cq_context
->cqc_byte_32
,
1996 CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S
,
1998 /* The initial value of cq's ci is 0 */
1999 roce_set_field(cq_context
->cqc_byte_32
,
2000 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M
,
2001 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S
, 0);
2002 cq_context
->cqc_byte_32
= cpu_to_le32(cq_context
->cqc_byte_32
);
2005 int hns_roce_v1_req_notify_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
2007 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
2008 u32 notification_flag
;
2012 notification_flag
= (flags
& IB_CQ_SOLICITED_MASK
) ==
2013 IB_CQ_SOLICITED
? CQ_DB_REQ_NOT
: CQ_DB_REQ_NOT_SOL
;
2015 * flags = 0; Notification Flag = 1, next
2016 * flags = 1; Notification Flag = 0, solocited
2018 doorbell
[0] = hr_cq
->cons_index
& ((hr_cq
->cq_depth
<< 1) - 1);
2019 roce_set_bit(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S
, 1);
2020 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M
,
2021 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S
, 3);
2022 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M
,
2023 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S
, 1);
2024 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M
,
2025 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S
,
2026 hr_cq
->cqn
| notification_flag
);
2028 hns_roce_write64_k(doorbell
, hr_cq
->cq_db_l
);
2033 static int hns_roce_v1_poll_one(struct hns_roce_cq
*hr_cq
,
2034 struct hns_roce_qp
**cur_qp
, struct ib_wc
*wc
)
2041 struct hns_roce_cqe
*cqe
;
2042 struct hns_roce_qp
*hr_qp
;
2043 struct hns_roce_wq
*wq
;
2044 struct hns_roce_wqe_ctrl_seg
*sq_wqe
;
2045 struct hns_roce_dev
*hr_dev
= to_hr_dev(hr_cq
->ib_cq
.device
);
2046 struct device
*dev
= &hr_dev
->pdev
->dev
;
2048 /* Find cqe according consumer index */
2049 cqe
= next_cqe_sw(hr_cq
);
2053 ++hr_cq
->cons_index
;
2054 /* Memory barrier */
2057 is_send
= !(roce_get_bit(cqe
->cqe_byte_4
, CQE_BYTE_4_SQ_RQ_FLAG_S
));
2059 /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
2060 if (roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
2061 CQE_BYTE_16_LOCAL_QPN_S
) <= 1) {
2062 qpn
= roce_get_field(cqe
->cqe_byte_20
, CQE_BYTE_20_PORT_NUM_M
,
2063 CQE_BYTE_20_PORT_NUM_S
) +
2064 roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
2065 CQE_BYTE_16_LOCAL_QPN_S
) *
2068 qpn
= roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
2069 CQE_BYTE_16_LOCAL_QPN_S
);
2072 if (!*cur_qp
|| (qpn
& HNS_ROCE_CQE_QPN_MASK
) != (*cur_qp
)->qpn
) {
2073 hr_qp
= __hns_roce_qp_lookup(hr_dev
, qpn
);
2074 if (unlikely(!hr_qp
)) {
2075 dev_err(dev
, "CQ %06lx with entry for unknown QPN %06x\n",
2076 hr_cq
->cqn
, (qpn
& HNS_ROCE_CQE_QPN_MASK
));
2083 wc
->qp
= &(*cur_qp
)->ibqp
;
2086 status
= roce_get_field(cqe
->cqe_byte_4
,
2087 CQE_BYTE_4_STATUS_OF_THE_OPERATION_M
,
2088 CQE_BYTE_4_STATUS_OF_THE_OPERATION_S
) &
2089 HNS_ROCE_CQE_STATUS_MASK
;
2091 case HNS_ROCE_CQE_SUCCESS
:
2092 wc
->status
= IB_WC_SUCCESS
;
2094 case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR
:
2095 wc
->status
= IB_WC_LOC_LEN_ERR
;
2097 case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR
:
2098 wc
->status
= IB_WC_LOC_QP_OP_ERR
;
2100 case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR
:
2101 wc
->status
= IB_WC_LOC_PROT_ERR
;
2103 case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR
:
2104 wc
->status
= IB_WC_WR_FLUSH_ERR
;
2106 case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR
:
2107 wc
->status
= IB_WC_MW_BIND_ERR
;
2109 case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR
:
2110 wc
->status
= IB_WC_BAD_RESP_ERR
;
2112 case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR
:
2113 wc
->status
= IB_WC_LOC_ACCESS_ERR
;
2115 case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR
:
2116 wc
->status
= IB_WC_REM_INV_REQ_ERR
;
2118 case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR
:
2119 wc
->status
= IB_WC_REM_ACCESS_ERR
;
2121 case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR
:
2122 wc
->status
= IB_WC_REM_OP_ERR
;
2124 case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR
:
2125 wc
->status
= IB_WC_RETRY_EXC_ERR
;
2127 case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR
:
2128 wc
->status
= IB_WC_RNR_RETRY_EXC_ERR
;
2131 wc
->status
= IB_WC_GENERAL_ERR
;
2135 /* CQE status error, directly return */
2136 if (wc
->status
!= IB_WC_SUCCESS
)
2140 /* SQ conrespond to CQE */
2141 sq_wqe
= get_send_wqe(*cur_qp
, roce_get_field(cqe
->cqe_byte_4
,
2142 CQE_BYTE_4_WQE_INDEX_M
,
2143 CQE_BYTE_4_WQE_INDEX_S
)&
2144 ((*cur_qp
)->sq
.wqe_cnt
-1));
2145 switch (sq_wqe
->flag
& HNS_ROCE_WQE_OPCODE_MASK
) {
2146 case HNS_ROCE_WQE_OPCODE_SEND
:
2147 wc
->opcode
= IB_WC_SEND
;
2149 case HNS_ROCE_WQE_OPCODE_RDMA_READ
:
2150 wc
->opcode
= IB_WC_RDMA_READ
;
2151 wc
->byte_len
= le32_to_cpu(cqe
->byte_cnt
);
2153 case HNS_ROCE_WQE_OPCODE_RDMA_WRITE
:
2154 wc
->opcode
= IB_WC_RDMA_WRITE
;
2156 case HNS_ROCE_WQE_OPCODE_LOCAL_INV
:
2157 wc
->opcode
= IB_WC_LOCAL_INV
;
2159 case HNS_ROCE_WQE_OPCODE_UD_SEND
:
2160 wc
->opcode
= IB_WC_SEND
;
2163 wc
->status
= IB_WC_GENERAL_ERR
;
2166 wc
->wc_flags
= (sq_wqe
->flag
& HNS_ROCE_WQE_IMM
?
2167 IB_WC_WITH_IMM
: 0);
2169 wq
= &(*cur_qp
)->sq
;
2170 if ((*cur_qp
)->sq_signal_bits
) {
2172 * If sg_signal_bit is 1,
2173 * firstly tail pointer updated to wqe
2174 * which current cqe correspond to
2176 wqe_ctr
= (u16
)roce_get_field(cqe
->cqe_byte_4
,
2177 CQE_BYTE_4_WQE_INDEX_M
,
2178 CQE_BYTE_4_WQE_INDEX_S
);
2179 wq
->tail
+= (wqe_ctr
- (u16
)wq
->tail
) &
2182 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
2185 /* RQ conrespond to CQE */
2186 wc
->byte_len
= le32_to_cpu(cqe
->byte_cnt
);
2187 opcode
= roce_get_field(cqe
->cqe_byte_4
,
2188 CQE_BYTE_4_OPERATION_TYPE_M
,
2189 CQE_BYTE_4_OPERATION_TYPE_S
) &
2190 HNS_ROCE_CQE_OPCODE_MASK
;
2192 case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE
:
2193 wc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
2194 wc
->wc_flags
= IB_WC_WITH_IMM
;
2195 wc
->ex
.imm_data
= le32_to_cpu(cqe
->immediate_data
);
2197 case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE
:
2198 if (roce_get_bit(cqe
->cqe_byte_4
,
2199 CQE_BYTE_4_IMM_INDICATOR_S
)) {
2200 wc
->opcode
= IB_WC_RECV
;
2201 wc
->wc_flags
= IB_WC_WITH_IMM
;
2202 wc
->ex
.imm_data
= le32_to_cpu(
2203 cqe
->immediate_data
);
2205 wc
->opcode
= IB_WC_RECV
;
2210 wc
->status
= IB_WC_GENERAL_ERR
;
2214 /* Update tail pointer, record wr_id */
2215 wq
= &(*cur_qp
)->rq
;
2216 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
2218 wc
->sl
= (u8
)roce_get_field(cqe
->cqe_byte_20
, CQE_BYTE_20_SL_M
,
2220 wc
->src_qp
= (u8
)roce_get_field(cqe
->cqe_byte_20
,
2221 CQE_BYTE_20_REMOTE_QPN_M
,
2222 CQE_BYTE_20_REMOTE_QPN_S
);
2223 wc
->wc_flags
|= (roce_get_bit(cqe
->cqe_byte_20
,
2224 CQE_BYTE_20_GRH_PRESENT_S
) ?
2226 wc
->pkey_index
= (u16
)roce_get_field(cqe
->cqe_byte_28
,
2227 CQE_BYTE_28_P_KEY_IDX_M
,
2228 CQE_BYTE_28_P_KEY_IDX_S
);
2234 int hns_roce_v1_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
)
2236 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
2237 struct hns_roce_qp
*cur_qp
= NULL
;
2238 unsigned long flags
;
2242 spin_lock_irqsave(&hr_cq
->lock
, flags
);
2244 for (npolled
= 0; npolled
< num_entries
; ++npolled
) {
2245 ret
= hns_roce_v1_poll_one(hr_cq
, &cur_qp
, wc
+ npolled
);
2251 *hr_cq
->tptr_addr
= hr_cq
->cons_index
&
2252 ((hr_cq
->cq_depth
<< 1) - 1);
2254 /* Memroy barrier */
2256 hns_roce_v1_cq_set_ci(hr_cq
, hr_cq
->cons_index
);
2259 spin_unlock_irqrestore(&hr_cq
->lock
, flags
);
2261 if (ret
== 0 || ret
== -EAGAIN
)
2267 int hns_roce_v1_clear_hem(struct hns_roce_dev
*hr_dev
,
2268 struct hns_roce_hem_table
*table
, int obj
)
2270 struct device
*dev
= &hr_dev
->pdev
->dev
;
2271 struct hns_roce_v1_priv
*priv
;
2272 unsigned long end
= 0, flags
= 0;
2273 uint32_t bt_cmd_val
[2] = {0};
2274 void __iomem
*bt_cmd
;
2277 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
2279 switch (table
->type
) {
2281 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
2282 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_QPC
);
2283 bt_ba
= priv
->bt_table
.qpc_buf
.map
>> 12;
2286 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
2287 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_MTPT
);
2288 bt_ba
= priv
->bt_table
.mtpt_buf
.map
>> 12;
2291 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
2292 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_CQC
);
2293 bt_ba
= priv
->bt_table
.cqc_buf
.map
>> 12;
2296 dev_dbg(dev
, "HEM_TYPE_SRQC not support.\n");
2301 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M
,
2302 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S
, obj
);
2303 roce_set_bit(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S
, 0);
2304 roce_set_bit(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S
, 1);
2306 spin_lock_irqsave(&hr_dev
->bt_cmd_lock
, flags
);
2308 bt_cmd
= hr_dev
->reg_base
+ ROCEE_BT_CMD_H_REG
;
2310 end
= msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS
) + jiffies
;
2312 if (readl(bt_cmd
) >> BT_CMD_SYNC_SHIFT
) {
2313 if (!(time_before(jiffies
, end
))) {
2314 dev_err(dev
, "Write bt_cmd err,hw_sync is not zero.\n");
2315 spin_unlock_irqrestore(&hr_dev
->bt_cmd_lock
,
2322 msleep(HW_SYNC_SLEEP_TIME_INTERVAL
);
2325 bt_cmd_val
[0] = (uint32_t)bt_ba
;
2326 roce_set_field(bt_cmd_val
[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M
,
2327 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S
, bt_ba
>> 32);
2328 hns_roce_write64_k(bt_cmd_val
, hr_dev
->reg_base
+ ROCEE_BT_CMD_L_REG
);
2330 spin_unlock_irqrestore(&hr_dev
->bt_cmd_lock
, flags
);
2335 static int hns_roce_v1_qp_modify(struct hns_roce_dev
*hr_dev
,
2336 struct hns_roce_mtt
*mtt
,
2337 enum hns_roce_qp_state cur_state
,
2338 enum hns_roce_qp_state new_state
,
2339 struct hns_roce_qp_context
*context
,
2340 struct hns_roce_qp
*hr_qp
)
2343 op
[HNS_ROCE_QP_NUM_STATE
][HNS_ROCE_QP_NUM_STATE
] = {
2344 [HNS_ROCE_QP_STATE_RST
] = {
2345 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2346 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2347 [HNS_ROCE_QP_STATE_INIT
] = HNS_ROCE_CMD_RST2INIT_QP
,
2349 [HNS_ROCE_QP_STATE_INIT
] = {
2350 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2351 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2352 /* Note: In v1 engine, HW doesn't support RST2INIT.
2353 * We use RST2INIT cmd instead of INIT2INIT.
2355 [HNS_ROCE_QP_STATE_INIT
] = HNS_ROCE_CMD_RST2INIT_QP
,
2356 [HNS_ROCE_QP_STATE_RTR
] = HNS_ROCE_CMD_INIT2RTR_QP
,
2358 [HNS_ROCE_QP_STATE_RTR
] = {
2359 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2360 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2361 [HNS_ROCE_QP_STATE_RTS
] = HNS_ROCE_CMD_RTR2RTS_QP
,
2363 [HNS_ROCE_QP_STATE_RTS
] = {
2364 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2365 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2366 [HNS_ROCE_QP_STATE_RTS
] = HNS_ROCE_CMD_RTS2RTS_QP
,
2367 [HNS_ROCE_QP_STATE_SQD
] = HNS_ROCE_CMD_RTS2SQD_QP
,
2369 [HNS_ROCE_QP_STATE_SQD
] = {
2370 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2371 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2372 [HNS_ROCE_QP_STATE_RTS
] = HNS_ROCE_CMD_SQD2RTS_QP
,
2373 [HNS_ROCE_QP_STATE_SQD
] = HNS_ROCE_CMD_SQD2SQD_QP
,
2375 [HNS_ROCE_QP_STATE_ERR
] = {
2376 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
2377 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
2381 struct hns_roce_cmd_mailbox
*mailbox
;
2382 struct device
*dev
= &hr_dev
->pdev
->dev
;
2385 if (cur_state
>= HNS_ROCE_QP_NUM_STATE
||
2386 new_state
>= HNS_ROCE_QP_NUM_STATE
||
2387 !op
[cur_state
][new_state
]) {
2388 dev_err(dev
, "[modify_qp]not support state %d to %d\n",
2389 cur_state
, new_state
);
2393 if (op
[cur_state
][new_state
] == HNS_ROCE_CMD_2RST_QP
)
2394 return hns_roce_cmd_mbox(hr_dev
, 0, 0, hr_qp
->qpn
, 2,
2395 HNS_ROCE_CMD_2RST_QP
,
2396 HNS_ROCE_CMD_TIMEOUT_MSECS
);
2398 if (op
[cur_state
][new_state
] == HNS_ROCE_CMD_2ERR_QP
)
2399 return hns_roce_cmd_mbox(hr_dev
, 0, 0, hr_qp
->qpn
, 2,
2400 HNS_ROCE_CMD_2ERR_QP
,
2401 HNS_ROCE_CMD_TIMEOUT_MSECS
);
2403 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
2404 if (IS_ERR(mailbox
))
2405 return PTR_ERR(mailbox
);
2407 memcpy(mailbox
->buf
, context
, sizeof(*context
));
2409 ret
= hns_roce_cmd_mbox(hr_dev
, mailbox
->dma
, 0, hr_qp
->qpn
, 0,
2410 op
[cur_state
][new_state
],
2411 HNS_ROCE_CMD_TIMEOUT_MSECS
);
2413 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
2417 static int hns_roce_v1_m_sqp(struct ib_qp
*ibqp
, const struct ib_qp_attr
*attr
,
2418 int attr_mask
, enum ib_qp_state cur_state
,
2419 enum ib_qp_state new_state
)
2421 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2422 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2423 struct hns_roce_sqp_context
*context
;
2424 struct device
*dev
= &hr_dev
->pdev
->dev
;
2425 dma_addr_t dma_handle
= 0;
2431 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
2435 /* Search QP buf's MTTs */
2436 mtts
= hns_roce_table_find(&hr_dev
->mr_table
.mtt_table
,
2437 hr_qp
->mtt
.first_seg
, &dma_handle
);
2439 dev_err(dev
, "qp buf pa find failed\n");
2443 if (cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
) {
2444 roce_set_field(context
->qp1c_bytes_4
,
2445 QP1C_BYTES_4_SQ_WQE_SHIFT_M
,
2446 QP1C_BYTES_4_SQ_WQE_SHIFT_S
,
2447 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
2448 roce_set_field(context
->qp1c_bytes_4
,
2449 QP1C_BYTES_4_RQ_WQE_SHIFT_M
,
2450 QP1C_BYTES_4_RQ_WQE_SHIFT_S
,
2451 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
2452 roce_set_field(context
->qp1c_bytes_4
, QP1C_BYTES_4_PD_M
,
2453 QP1C_BYTES_4_PD_S
, to_hr_pd(ibqp
->pd
)->pdn
);
2455 context
->sq_rq_bt_l
= (u32
)(dma_handle
);
2456 roce_set_field(context
->qp1c_bytes_12
,
2457 QP1C_BYTES_12_SQ_RQ_BT_H_M
,
2458 QP1C_BYTES_12_SQ_RQ_BT_H_S
,
2459 ((u32
)(dma_handle
>> 32)));
2461 roce_set_field(context
->qp1c_bytes_16
, QP1C_BYTES_16_RQ_HEAD_M
,
2462 QP1C_BYTES_16_RQ_HEAD_S
, hr_qp
->rq
.head
);
2463 roce_set_field(context
->qp1c_bytes_16
, QP1C_BYTES_16_PORT_NUM_M
,
2464 QP1C_BYTES_16_PORT_NUM_S
, hr_qp
->phy_port
);
2465 roce_set_bit(context
->qp1c_bytes_16
,
2466 QP1C_BYTES_16_SIGNALING_TYPE_S
,
2467 hr_qp
->sq_signal_bits
);
2468 roce_set_bit(context
->qp1c_bytes_16
, QP1C_BYTES_16_RQ_BA_FLG_S
,
2470 roce_set_bit(context
->qp1c_bytes_16
, QP1C_BYTES_16_SQ_BA_FLG_S
,
2472 roce_set_bit(context
->qp1c_bytes_16
, QP1C_BYTES_16_QP1_ERR_S
,
2475 roce_set_field(context
->qp1c_bytes_20
, QP1C_BYTES_20_SQ_HEAD_M
,
2476 QP1C_BYTES_20_SQ_HEAD_S
, hr_qp
->sq
.head
);
2477 roce_set_field(context
->qp1c_bytes_20
, QP1C_BYTES_20_PKEY_IDX_M
,
2478 QP1C_BYTES_20_PKEY_IDX_S
, attr
->pkey_index
);
2480 rq_pa_start
= (u32
)hr_qp
->rq
.offset
/ PAGE_SIZE
;
2481 context
->cur_rq_wqe_ba_l
= (u32
)(mtts
[rq_pa_start
]);
2483 roce_set_field(context
->qp1c_bytes_28
,
2484 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M
,
2485 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S
,
2486 (mtts
[rq_pa_start
]) >> 32);
2487 roce_set_field(context
->qp1c_bytes_28
,
2488 QP1C_BYTES_28_RQ_CUR_IDX_M
,
2489 QP1C_BYTES_28_RQ_CUR_IDX_S
, 0);
2491 roce_set_field(context
->qp1c_bytes_32
,
2492 QP1C_BYTES_32_RX_CQ_NUM_M
,
2493 QP1C_BYTES_32_RX_CQ_NUM_S
,
2494 to_hr_cq(ibqp
->recv_cq
)->cqn
);
2495 roce_set_field(context
->qp1c_bytes_32
,
2496 QP1C_BYTES_32_TX_CQ_NUM_M
,
2497 QP1C_BYTES_32_TX_CQ_NUM_S
,
2498 to_hr_cq(ibqp
->send_cq
)->cqn
);
2500 context
->cur_sq_wqe_ba_l
= (u32
)mtts
[0];
2502 roce_set_field(context
->qp1c_bytes_40
,
2503 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M
,
2504 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S
,
2506 roce_set_field(context
->qp1c_bytes_40
,
2507 QP1C_BYTES_40_SQ_CUR_IDX_M
,
2508 QP1C_BYTES_40_SQ_CUR_IDX_S
, 0);
2510 /* Copy context to QP1C register */
2511 addr
= (u32
*)(hr_dev
->reg_base
+ ROCEE_QP1C_CFG0_0_REG
+
2512 hr_qp
->phy_port
* sizeof(*context
));
2514 writel(context
->qp1c_bytes_4
, addr
);
2515 writel(context
->sq_rq_bt_l
, addr
+ 1);
2516 writel(context
->qp1c_bytes_12
, addr
+ 2);
2517 writel(context
->qp1c_bytes_16
, addr
+ 3);
2518 writel(context
->qp1c_bytes_20
, addr
+ 4);
2519 writel(context
->cur_rq_wqe_ba_l
, addr
+ 5);
2520 writel(context
->qp1c_bytes_28
, addr
+ 6);
2521 writel(context
->qp1c_bytes_32
, addr
+ 7);
2522 writel(context
->cur_sq_wqe_ba_l
, addr
+ 8);
2523 writel(context
->qp1c_bytes_40
, addr
+ 9);
2526 /* Modify QP1C status */
2527 reg_val
= roce_read(hr_dev
, ROCEE_QP1C_CFG0_0_REG
+
2528 hr_qp
->phy_port
* sizeof(*context
));
2529 roce_set_field(reg_val
, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M
,
2530 ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S
, new_state
);
2531 roce_write(hr_dev
, ROCEE_QP1C_CFG0_0_REG
+
2532 hr_qp
->phy_port
* sizeof(*context
), reg_val
);
2534 hr_qp
->state
= new_state
;
2535 if (new_state
== IB_QPS_RESET
) {
2536 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->recv_cq
), hr_qp
->qpn
,
2537 ibqp
->srq
? to_hr_srq(ibqp
->srq
) : NULL
);
2538 if (ibqp
->send_cq
!= ibqp
->recv_cq
)
2539 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->send_cq
),
2546 hr_qp
->sq_next_wqe
= 0;
2557 static int hns_roce_v1_m_qp(struct ib_qp
*ibqp
, const struct ib_qp_attr
*attr
,
2558 int attr_mask
, enum ib_qp_state cur_state
,
2559 enum ib_qp_state new_state
)
2561 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2562 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2563 struct device
*dev
= &hr_dev
->pdev
->dev
;
2564 struct hns_roce_qp_context
*context
;
2565 const struct ib_global_route
*grh
= rdma_ah_read_grh(&attr
->ah_attr
);
2566 dma_addr_t dma_handle_2
= 0;
2567 dma_addr_t dma_handle
= 0;
2568 uint32_t doorbell
[2] = {0};
2569 int rq_pa_start
= 0;
2578 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
2582 /* Search qp buf's mtts */
2583 mtts
= hns_roce_table_find(&hr_dev
->mr_table
.mtt_table
,
2584 hr_qp
->mtt
.first_seg
, &dma_handle
);
2586 dev_err(dev
, "qp buf pa find failed\n");
2590 /* Search IRRL's mtts */
2591 mtts_2
= hns_roce_table_find(&hr_dev
->qp_table
.irrl_table
, hr_qp
->qpn
,
2593 if (mtts_2
== NULL
) {
2594 dev_err(dev
, "qp irrl_table find failed\n");
2601 * IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
2602 * Optional param: NA
2604 if (cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
) {
2605 roce_set_field(context
->qpc_bytes_4
,
2606 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M
,
2607 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S
,
2608 to_hr_qp_type(hr_qp
->ibqp
.qp_type
));
2610 roce_set_bit(context
->qpc_bytes_4
,
2611 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S
, 0);
2612 roce_set_bit(context
->qpc_bytes_4
,
2613 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
,
2614 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
));
2615 roce_set_bit(context
->qpc_bytes_4
,
2616 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
,
2617 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
)
2619 roce_set_bit(context
->qpc_bytes_4
,
2620 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S
,
2621 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_ATOMIC
)
2623 roce_set_bit(context
->qpc_bytes_4
,
2624 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S
, 1);
2625 roce_set_field(context
->qpc_bytes_4
,
2626 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M
,
2627 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S
,
2628 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
2629 roce_set_field(context
->qpc_bytes_4
,
2630 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M
,
2631 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S
,
2632 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
2633 roce_set_field(context
->qpc_bytes_4
,
2634 QP_CONTEXT_QPC_BYTES_4_PD_M
,
2635 QP_CONTEXT_QPC_BYTES_4_PD_S
,
2636 to_hr_pd(ibqp
->pd
)->pdn
);
2637 hr_qp
->access_flags
= attr
->qp_access_flags
;
2638 roce_set_field(context
->qpc_bytes_8
,
2639 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M
,
2640 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S
,
2641 to_hr_cq(ibqp
->send_cq
)->cqn
);
2642 roce_set_field(context
->qpc_bytes_8
,
2643 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M
,
2644 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S
,
2645 to_hr_cq(ibqp
->recv_cq
)->cqn
);
2648 roce_set_field(context
->qpc_bytes_12
,
2649 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M
,
2650 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S
,
2651 to_hr_srq(ibqp
->srq
)->srqn
);
2653 roce_set_field(context
->qpc_bytes_12
,
2654 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
2655 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
,
2657 hr_qp
->pkey_index
= attr
->pkey_index
;
2658 roce_set_field(context
->qpc_bytes_16
,
2659 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M
,
2660 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S
, hr_qp
->qpn
);
2662 } else if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_INIT
) {
2663 roce_set_field(context
->qpc_bytes_4
,
2664 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M
,
2665 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S
,
2666 to_hr_qp_type(hr_qp
->ibqp
.qp_type
));
2667 roce_set_bit(context
->qpc_bytes_4
,
2668 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S
, 0);
2669 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
2670 roce_set_bit(context
->qpc_bytes_4
,
2671 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
,
2672 !!(attr
->qp_access_flags
&
2673 IB_ACCESS_REMOTE_READ
));
2674 roce_set_bit(context
->qpc_bytes_4
,
2675 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
,
2676 !!(attr
->qp_access_flags
&
2677 IB_ACCESS_REMOTE_WRITE
));
2679 roce_set_bit(context
->qpc_bytes_4
,
2680 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
,
2681 !!(hr_qp
->access_flags
&
2682 IB_ACCESS_REMOTE_READ
));
2683 roce_set_bit(context
->qpc_bytes_4
,
2684 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
,
2685 !!(hr_qp
->access_flags
&
2686 IB_ACCESS_REMOTE_WRITE
));
2689 roce_set_bit(context
->qpc_bytes_4
,
2690 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S
, 1);
2691 roce_set_field(context
->qpc_bytes_4
,
2692 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M
,
2693 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S
,
2694 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
2695 roce_set_field(context
->qpc_bytes_4
,
2696 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M
,
2697 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S
,
2698 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
2699 roce_set_field(context
->qpc_bytes_4
,
2700 QP_CONTEXT_QPC_BYTES_4_PD_M
,
2701 QP_CONTEXT_QPC_BYTES_4_PD_S
,
2702 to_hr_pd(ibqp
->pd
)->pdn
);
2704 roce_set_field(context
->qpc_bytes_8
,
2705 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M
,
2706 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S
,
2707 to_hr_cq(ibqp
->send_cq
)->cqn
);
2708 roce_set_field(context
->qpc_bytes_8
,
2709 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M
,
2710 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S
,
2711 to_hr_cq(ibqp
->recv_cq
)->cqn
);
2714 roce_set_field(context
->qpc_bytes_12
,
2715 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M
,
2716 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S
,
2717 to_hr_srq(ibqp
->srq
)->srqn
);
2718 if (attr_mask
& IB_QP_PKEY_INDEX
)
2719 roce_set_field(context
->qpc_bytes_12
,
2720 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
2721 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
,
2724 roce_set_field(context
->qpc_bytes_12
,
2725 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
2726 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
,
2729 roce_set_field(context
->qpc_bytes_16
,
2730 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M
,
2731 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S
, hr_qp
->qpn
);
2732 } else if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_RTR
) {
2733 if ((attr_mask
& IB_QP_ALT_PATH
) ||
2734 (attr_mask
& IB_QP_ACCESS_FLAGS
) ||
2735 (attr_mask
& IB_QP_PKEY_INDEX
) ||
2736 (attr_mask
& IB_QP_QKEY
)) {
2737 dev_err(dev
, "INIT2RTR attr_mask error\n");
2741 dmac
= (u8
*)attr
->ah_attr
.roce
.dmac
;
2743 context
->sq_rq_bt_l
= (u32
)(dma_handle
);
2744 roce_set_field(context
->qpc_bytes_24
,
2745 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M
,
2746 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S
,
2747 ((u32
)(dma_handle
>> 32)));
2748 roce_set_bit(context
->qpc_bytes_24
,
2749 QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S
,
2751 roce_set_field(context
->qpc_bytes_24
,
2752 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M
,
2753 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S
,
2754 attr
->min_rnr_timer
);
2755 context
->irrl_ba_l
= (u32
)(dma_handle_2
);
2756 roce_set_field(context
->qpc_bytes_32
,
2757 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M
,
2758 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S
,
2759 ((u32
)(dma_handle_2
>> 32)) &
2760 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M
);
2761 roce_set_field(context
->qpc_bytes_32
,
2762 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M
,
2763 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S
, 0);
2764 roce_set_bit(context
->qpc_bytes_32
,
2765 QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S
,
2767 roce_set_bit(context
->qpc_bytes_32
,
2768 QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S
,
2769 hr_qp
->sq_signal_bits
);
2771 port
= (attr_mask
& IB_QP_PORT
) ? (attr
->port_num
- 1) :
2773 smac
= (u8
*)hr_dev
->dev_addr
[port
];
2774 /* when dmac equals smac or loop_idc is 1, it should loopback */
2775 if (ether_addr_equal_unaligned(dmac
, smac
) ||
2776 hr_dev
->loop_idc
== 0x1)
2777 roce_set_bit(context
->qpc_bytes_32
,
2778 QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S
, 1);
2780 roce_set_bit(context
->qpc_bytes_32
,
2781 QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S
,
2782 rdma_ah_get_ah_flags(&attr
->ah_attr
));
2783 roce_set_field(context
->qpc_bytes_32
,
2784 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M
,
2785 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S
,
2786 ilog2((unsigned int)attr
->max_dest_rd_atomic
));
2788 roce_set_field(context
->qpc_bytes_36
,
2789 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M
,
2790 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S
,
2793 /* Configure GID index */
2794 port_num
= rdma_ah_get_port_num(&attr
->ah_attr
);
2795 roce_set_field(context
->qpc_bytes_36
,
2796 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M
,
2797 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S
,
2798 hns_get_gid_index(hr_dev
,
2802 memcpy(&(context
->dmac_l
), dmac
, 4);
2804 roce_set_field(context
->qpc_bytes_44
,
2805 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M
,
2806 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S
,
2807 *((u16
*)(&dmac
[4])));
2808 roce_set_field(context
->qpc_bytes_44
,
2809 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M
,
2810 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S
,
2811 rdma_ah_get_static_rate(&attr
->ah_attr
));
2812 roce_set_field(context
->qpc_bytes_44
,
2813 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M
,
2814 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S
,
2817 roce_set_field(context
->qpc_bytes_48
,
2818 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M
,
2819 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S
,
2821 roce_set_field(context
->qpc_bytes_48
,
2822 QP_CONTEXT_QPC_BYTES_48_TCLASS_M
,
2823 QP_CONTEXT_QPC_BYTES_48_TCLASS_S
,
2824 grh
->traffic_class
);
2825 roce_set_field(context
->qpc_bytes_48
,
2826 QP_CONTEXT_QPC_BYTES_48_MTU_M
,
2827 QP_CONTEXT_QPC_BYTES_48_MTU_S
, attr
->path_mtu
);
2829 memcpy(context
->dgid
, grh
->dgid
.raw
,
2830 sizeof(grh
->dgid
.raw
));
2832 dev_dbg(dev
, "dmac:%x :%lx\n", context
->dmac_l
,
2833 roce_get_field(context
->qpc_bytes_44
,
2834 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M
,
2835 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S
));
2837 roce_set_field(context
->qpc_bytes_68
,
2838 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M
,
2839 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S
,
2841 roce_set_field(context
->qpc_bytes_68
,
2842 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M
,
2843 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S
, 0);
2845 rq_pa_start
= (u32
)hr_qp
->rq
.offset
/ PAGE_SIZE
;
2846 context
->cur_rq_wqe_ba_l
= (u32
)(mtts
[rq_pa_start
]);
2848 roce_set_field(context
->qpc_bytes_76
,
2849 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M
,
2850 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S
,
2851 mtts
[rq_pa_start
] >> 32);
2852 roce_set_field(context
->qpc_bytes_76
,
2853 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M
,
2854 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S
, 0);
2856 context
->rx_rnr_time
= 0;
2858 roce_set_field(context
->qpc_bytes_84
,
2859 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M
,
2860 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S
,
2862 roce_set_field(context
->qpc_bytes_84
,
2863 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M
,
2864 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S
, 0);
2866 roce_set_field(context
->qpc_bytes_88
,
2867 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M
,
2868 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S
,
2870 roce_set_bit(context
->qpc_bytes_88
,
2871 QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S
, 0);
2872 roce_set_bit(context
->qpc_bytes_88
,
2873 QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S
, 0);
2874 roce_set_field(context
->qpc_bytes_88
,
2875 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M
,
2876 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S
,
2878 roce_set_field(context
->qpc_bytes_88
,
2879 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M
,
2880 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S
,
2883 context
->dma_length
= 0;
2888 roce_set_field(context
->qpc_bytes_108
,
2889 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M
,
2890 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S
, 0);
2891 roce_set_bit(context
->qpc_bytes_108
,
2892 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S
, 0);
2893 roce_set_bit(context
->qpc_bytes_108
,
2894 QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S
, 0);
2896 roce_set_field(context
->qpc_bytes_112
,
2897 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M
,
2898 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S
, 0);
2899 roce_set_field(context
->qpc_bytes_112
,
2900 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M
,
2901 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S
, 0);
2903 /* For chip resp ack */
2904 roce_set_field(context
->qpc_bytes_156
,
2905 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M
,
2906 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S
,
2908 roce_set_field(context
->qpc_bytes_156
,
2909 QP_CONTEXT_QPC_BYTES_156_SL_M
,
2910 QP_CONTEXT_QPC_BYTES_156_SL_S
,
2911 rdma_ah_get_sl(&attr
->ah_attr
));
2912 hr_qp
->sl
= rdma_ah_get_sl(&attr
->ah_attr
);
2913 } else if (cur_state
== IB_QPS_RTR
&&
2914 new_state
== IB_QPS_RTS
) {
2915 /* If exist optional param, return error */
2916 if ((attr_mask
& IB_QP_ALT_PATH
) ||
2917 (attr_mask
& IB_QP_ACCESS_FLAGS
) ||
2918 (attr_mask
& IB_QP_QKEY
) ||
2919 (attr_mask
& IB_QP_PATH_MIG_STATE
) ||
2920 (attr_mask
& IB_QP_CUR_STATE
) ||
2921 (attr_mask
& IB_QP_MIN_RNR_TIMER
)) {
2922 dev_err(dev
, "RTR2RTS attr_mask error\n");
2926 context
->rx_cur_sq_wqe_ba_l
= (u32
)(mtts
[0]);
2928 roce_set_field(context
->qpc_bytes_120
,
2929 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M
,
2930 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S
,
2933 roce_set_field(context
->qpc_bytes_124
,
2934 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M
,
2935 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S
, 0);
2936 roce_set_field(context
->qpc_bytes_124
,
2937 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M
,
2938 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S
, 0);
2940 roce_set_field(context
->qpc_bytes_128
,
2941 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M
,
2942 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S
,
2944 roce_set_bit(context
->qpc_bytes_128
,
2945 QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S
, 0);
2946 roce_set_field(context
->qpc_bytes_128
,
2947 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M
,
2948 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S
,
2950 roce_set_bit(context
->qpc_bytes_128
,
2951 QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S
, 0);
2953 roce_set_field(context
->qpc_bytes_132
,
2954 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M
,
2955 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S
, 0);
2956 roce_set_field(context
->qpc_bytes_132
,
2957 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M
,
2958 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S
, 0);
2960 roce_set_field(context
->qpc_bytes_136
,
2961 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M
,
2962 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S
,
2964 roce_set_field(context
->qpc_bytes_136
,
2965 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M
,
2966 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S
,
2969 roce_set_field(context
->qpc_bytes_140
,
2970 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M
,
2971 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S
,
2972 (attr
->sq_psn
>> SQ_PSN_SHIFT
));
2973 roce_set_field(context
->qpc_bytes_140
,
2974 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M
,
2975 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S
, 0);
2976 roce_set_bit(context
->qpc_bytes_140
,
2977 QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S
, 0);
2979 roce_set_field(context
->qpc_bytes_148
,
2980 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M
,
2981 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S
, 0);
2982 roce_set_field(context
->qpc_bytes_148
,
2983 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M
,
2984 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S
,
2986 roce_set_field(context
->qpc_bytes_148
,
2987 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M
,
2988 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S
,
2990 roce_set_field(context
->qpc_bytes_148
,
2991 QP_CONTEXT_QPC_BYTES_148_LSN_M
,
2992 QP_CONTEXT_QPC_BYTES_148_LSN_S
, 0x100);
2994 context
->rnr_retry
= 0;
2996 roce_set_field(context
->qpc_bytes_156
,
2997 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M
,
2998 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S
,
3000 if (attr
->timeout
< 0x12) {
3001 dev_info(dev
, "ack timeout value(0x%x) must bigger than 0x12.\n",
3003 roce_set_field(context
->qpc_bytes_156
,
3004 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M
,
3005 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S
,
3008 roce_set_field(context
->qpc_bytes_156
,
3009 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M
,
3010 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S
,
3013 roce_set_field(context
->qpc_bytes_156
,
3014 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M
,
3015 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S
,
3017 roce_set_field(context
->qpc_bytes_156
,
3018 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M
,
3019 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S
,
3021 roce_set_field(context
->qpc_bytes_156
,
3022 QP_CONTEXT_QPC_BYTES_156_SL_M
,
3023 QP_CONTEXT_QPC_BYTES_156_SL_S
,
3024 rdma_ah_get_sl(&attr
->ah_attr
));
3025 hr_qp
->sl
= rdma_ah_get_sl(&attr
->ah_attr
);
3026 roce_set_field(context
->qpc_bytes_156
,
3027 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M
,
3028 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S
,
3029 ilog2((unsigned int)attr
->max_rd_atomic
));
3030 roce_set_field(context
->qpc_bytes_156
,
3031 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M
,
3032 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S
, 0);
3033 context
->pkt_use_len
= 0;
3035 roce_set_field(context
->qpc_bytes_164
,
3036 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M
,
3037 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S
, attr
->sq_psn
);
3038 roce_set_field(context
->qpc_bytes_164
,
3039 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M
,
3040 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S
, 0);
3042 roce_set_field(context
->qpc_bytes_168
,
3043 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M
,
3044 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S
,
3046 roce_set_field(context
->qpc_bytes_168
,
3047 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M
,
3048 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S
, 0);
3049 roce_set_field(context
->qpc_bytes_168
,
3050 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M
,
3051 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S
, 0);
3052 roce_set_bit(context
->qpc_bytes_168
,
3053 QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S
, 0);
3054 roce_set_bit(context
->qpc_bytes_168
,
3055 QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S
, 0);
3056 roce_set_bit(context
->qpc_bytes_168
,
3057 QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S
, 0);
3058 context
->sge_use_len
= 0;
3060 roce_set_field(context
->qpc_bytes_176
,
3061 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M
,
3062 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S
, 0);
3063 roce_set_field(context
->qpc_bytes_176
,
3064 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M
,
3065 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S
,
3067 roce_set_field(context
->qpc_bytes_180
,
3068 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M
,
3069 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S
, 0);
3070 roce_set_field(context
->qpc_bytes_180
,
3071 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M
,
3072 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S
, 0);
3074 context
->tx_cur_sq_wqe_ba_l
= (u32
)(mtts
[0]);
3076 roce_set_field(context
->qpc_bytes_188
,
3077 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M
,
3078 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S
,
3080 roce_set_bit(context
->qpc_bytes_188
,
3081 QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S
, 0);
3082 roce_set_field(context
->qpc_bytes_188
,
3083 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M
,
3084 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S
,
3086 } else if (!((cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_RESET
) ||
3087 (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_ERR
) ||
3088 (cur_state
== IB_QPS_RTR
&& new_state
== IB_QPS_RESET
) ||
3089 (cur_state
== IB_QPS_RTR
&& new_state
== IB_QPS_ERR
) ||
3090 (cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_RESET
) ||
3091 (cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_ERR
) ||
3092 (cur_state
== IB_QPS_ERR
&& new_state
== IB_QPS_RESET
) ||
3093 (cur_state
== IB_QPS_ERR
&& new_state
== IB_QPS_ERR
))) {
3094 dev_err(dev
, "not support this status migration\n");
3098 /* Every status migrate must change state */
3099 roce_set_field(context
->qpc_bytes_144
,
3100 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M
,
3101 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S
, new_state
);
3103 /* SW pass context to HW */
3104 ret
= hns_roce_v1_qp_modify(hr_dev
, &hr_qp
->mtt
,
3105 to_hns_roce_state(cur_state
),
3106 to_hns_roce_state(new_state
), context
,
3109 dev_err(dev
, "hns_roce_qp_modify failed\n");
3114 * Use rst2init to instead of init2init with drv,
3115 * need to hw to flash RQ HEAD by DB again
3117 if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_INIT
) {
3118 /* Memory barrier */
3121 roce_set_field(doorbell
[0], RQ_DOORBELL_U32_4_RQ_HEAD_M
,
3122 RQ_DOORBELL_U32_4_RQ_HEAD_S
, hr_qp
->rq
.head
);
3123 roce_set_field(doorbell
[1], RQ_DOORBELL_U32_8_QPN_M
,
3124 RQ_DOORBELL_U32_8_QPN_S
, hr_qp
->qpn
);
3125 roce_set_field(doorbell
[1], RQ_DOORBELL_U32_8_CMD_M
,
3126 RQ_DOORBELL_U32_8_CMD_S
, 1);
3127 roce_set_bit(doorbell
[1], RQ_DOORBELL_U32_8_HW_SYNC_S
, 1);
3129 if (ibqp
->uobject
) {
3130 hr_qp
->rq
.db_reg_l
= hr_dev
->reg_base
+
3131 ROCEE_DB_OTHERS_L_0_REG
+
3132 DB_REG_OFFSET
* hr_dev
->priv_uar
.index
;
3135 hns_roce_write64_k(doorbell
, hr_qp
->rq
.db_reg_l
);
3138 hr_qp
->state
= new_state
;
3140 if (attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
)
3141 hr_qp
->resp_depth
= attr
->max_dest_rd_atomic
;
3142 if (attr_mask
& IB_QP_PORT
) {
3143 hr_qp
->port
= attr
->port_num
- 1;
3144 hr_qp
->phy_port
= hr_dev
->iboe
.phy_port
[hr_qp
->port
];
3147 if (new_state
== IB_QPS_RESET
&& !ibqp
->uobject
) {
3148 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->recv_cq
), hr_qp
->qpn
,
3149 ibqp
->srq
? to_hr_srq(ibqp
->srq
) : NULL
);
3150 if (ibqp
->send_cq
!= ibqp
->recv_cq
)
3151 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->send_cq
),
3158 hr_qp
->sq_next_wqe
= 0;
3165 int hns_roce_v1_modify_qp(struct ib_qp
*ibqp
, const struct ib_qp_attr
*attr
,
3166 int attr_mask
, enum ib_qp_state cur_state
,
3167 enum ib_qp_state new_state
)
3170 if (ibqp
->qp_type
== IB_QPT_GSI
|| ibqp
->qp_type
== IB_QPT_SMI
)
3171 return hns_roce_v1_m_sqp(ibqp
, attr
, attr_mask
, cur_state
,
3174 return hns_roce_v1_m_qp(ibqp
, attr
, attr_mask
, cur_state
,
3178 static enum ib_qp_state
to_ib_qp_state(enum hns_roce_qp_state state
)
3181 case HNS_ROCE_QP_STATE_RST
:
3182 return IB_QPS_RESET
;
3183 case HNS_ROCE_QP_STATE_INIT
:
3185 case HNS_ROCE_QP_STATE_RTR
:
3187 case HNS_ROCE_QP_STATE_RTS
:
3189 case HNS_ROCE_QP_STATE_SQD
:
3191 case HNS_ROCE_QP_STATE_ERR
:
3198 static int hns_roce_v1_query_qpc(struct hns_roce_dev
*hr_dev
,
3199 struct hns_roce_qp
*hr_qp
,
3200 struct hns_roce_qp_context
*hr_context
)
3202 struct hns_roce_cmd_mailbox
*mailbox
;
3205 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
3206 if (IS_ERR(mailbox
))
3207 return PTR_ERR(mailbox
);
3209 ret
= hns_roce_cmd_mbox(hr_dev
, 0, mailbox
->dma
, hr_qp
->qpn
, 0,
3210 HNS_ROCE_CMD_QUERY_QP
,
3211 HNS_ROCE_CMD_TIMEOUT_MSECS
);
3213 memcpy(hr_context
, mailbox
->buf
, sizeof(*hr_context
));
3215 dev_err(&hr_dev
->pdev
->dev
, "QUERY QP cmd process error\n");
3217 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
3222 static int hns_roce_v1_q_sqp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
,
3224 struct ib_qp_init_attr
*qp_init_attr
)
3226 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
3227 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
3228 struct hns_roce_sqp_context context
;
3231 mutex_lock(&hr_qp
->mutex
);
3233 if (hr_qp
->state
== IB_QPS_RESET
) {
3234 qp_attr
->qp_state
= IB_QPS_RESET
;
3238 addr
= ROCEE_QP1C_CFG0_0_REG
+
3239 hr_qp
->port
* sizeof(struct hns_roce_sqp_context
);
3240 context
.qp1c_bytes_4
= roce_read(hr_dev
, addr
);
3241 context
.sq_rq_bt_l
= roce_read(hr_dev
, addr
+ 1);
3242 context
.qp1c_bytes_12
= roce_read(hr_dev
, addr
+ 2);
3243 context
.qp1c_bytes_16
= roce_read(hr_dev
, addr
+ 3);
3244 context
.qp1c_bytes_20
= roce_read(hr_dev
, addr
+ 4);
3245 context
.cur_rq_wqe_ba_l
= roce_read(hr_dev
, addr
+ 5);
3246 context
.qp1c_bytes_28
= roce_read(hr_dev
, addr
+ 6);
3247 context
.qp1c_bytes_32
= roce_read(hr_dev
, addr
+ 7);
3248 context
.cur_sq_wqe_ba_l
= roce_read(hr_dev
, addr
+ 8);
3249 context
.qp1c_bytes_40
= roce_read(hr_dev
, addr
+ 9);
3251 hr_qp
->state
= roce_get_field(context
.qp1c_bytes_4
,
3252 QP1C_BYTES_4_QP_STATE_M
,
3253 QP1C_BYTES_4_QP_STATE_S
);
3254 qp_attr
->qp_state
= hr_qp
->state
;
3255 qp_attr
->path_mtu
= IB_MTU_256
;
3256 qp_attr
->path_mig_state
= IB_MIG_ARMED
;
3257 qp_attr
->qkey
= QKEY_VAL
;
3258 qp_attr
->rq_psn
= 0;
3259 qp_attr
->sq_psn
= 0;
3260 qp_attr
->dest_qp_num
= 1;
3261 qp_attr
->qp_access_flags
= 6;
3263 qp_attr
->pkey_index
= roce_get_field(context
.qp1c_bytes_20
,
3264 QP1C_BYTES_20_PKEY_IDX_M
,
3265 QP1C_BYTES_20_PKEY_IDX_S
);
3266 qp_attr
->port_num
= hr_qp
->port
+ 1;
3267 qp_attr
->sq_draining
= 0;
3268 qp_attr
->max_rd_atomic
= 0;
3269 qp_attr
->max_dest_rd_atomic
= 0;
3270 qp_attr
->min_rnr_timer
= 0;
3271 qp_attr
->timeout
= 0;
3272 qp_attr
->retry_cnt
= 0;
3273 qp_attr
->rnr_retry
= 0;
3274 qp_attr
->alt_timeout
= 0;
3277 qp_attr
->cur_qp_state
= qp_attr
->qp_state
;
3278 qp_attr
->cap
.max_recv_wr
= hr_qp
->rq
.wqe_cnt
;
3279 qp_attr
->cap
.max_recv_sge
= hr_qp
->rq
.max_gs
;
3280 qp_attr
->cap
.max_send_wr
= hr_qp
->sq
.wqe_cnt
;
3281 qp_attr
->cap
.max_send_sge
= hr_qp
->sq
.max_gs
;
3282 qp_attr
->cap
.max_inline_data
= 0;
3283 qp_init_attr
->cap
= qp_attr
->cap
;
3284 qp_init_attr
->create_flags
= 0;
3286 mutex_unlock(&hr_qp
->mutex
);
3291 static int hns_roce_v1_q_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
,
3293 struct ib_qp_init_attr
*qp_init_attr
)
3295 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
3296 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
3297 struct device
*dev
= &hr_dev
->pdev
->dev
;
3298 struct hns_roce_qp_context
*context
;
3299 int tmp_qp_state
= 0;
3303 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
3307 memset(qp_attr
, 0, sizeof(*qp_attr
));
3308 memset(qp_init_attr
, 0, sizeof(*qp_init_attr
));
3310 mutex_lock(&hr_qp
->mutex
);
3312 if (hr_qp
->state
== IB_QPS_RESET
) {
3313 qp_attr
->qp_state
= IB_QPS_RESET
;
3317 ret
= hns_roce_v1_query_qpc(hr_dev
, hr_qp
, context
);
3319 dev_err(dev
, "query qpc error\n");
3324 state
= roce_get_field(context
->qpc_bytes_144
,
3325 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M
,
3326 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S
);
3327 tmp_qp_state
= (int)to_ib_qp_state((enum hns_roce_qp_state
)state
);
3328 if (tmp_qp_state
== -1) {
3329 dev_err(dev
, "to_ib_qp_state error\n");
3333 hr_qp
->state
= (u8
)tmp_qp_state
;
3334 qp_attr
->qp_state
= (enum ib_qp_state
)hr_qp
->state
;
3335 qp_attr
->path_mtu
= (enum ib_mtu
)roce_get_field(context
->qpc_bytes_48
,
3336 QP_CONTEXT_QPC_BYTES_48_MTU_M
,
3337 QP_CONTEXT_QPC_BYTES_48_MTU_S
);
3338 qp_attr
->path_mig_state
= IB_MIG_ARMED
;
3339 if (hr_qp
->ibqp
.qp_type
== IB_QPT_UD
)
3340 qp_attr
->qkey
= QKEY_VAL
;
3342 qp_attr
->rq_psn
= roce_get_field(context
->qpc_bytes_88
,
3343 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M
,
3344 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S
);
3345 qp_attr
->sq_psn
= (u32
)roce_get_field(context
->qpc_bytes_164
,
3346 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M
,
3347 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S
);
3348 qp_attr
->dest_qp_num
= (u8
)roce_get_field(context
->qpc_bytes_36
,
3349 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M
,
3350 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S
);
3351 qp_attr
->qp_access_flags
= ((roce_get_bit(context
->qpc_bytes_4
,
3352 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
)) << 2) |
3353 ((roce_get_bit(context
->qpc_bytes_4
,
3354 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
)) << 1) |
3355 ((roce_get_bit(context
->qpc_bytes_4
,
3356 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S
)) << 3);
3358 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
||
3359 hr_qp
->ibqp
.qp_type
== IB_QPT_UC
) {
3360 struct ib_global_route
*grh
=
3361 rdma_ah_retrieve_grh(&qp_attr
->ah_attr
);
3363 rdma_ah_set_sl(&qp_attr
->ah_attr
,
3364 roce_get_field(context
->qpc_bytes_156
,
3365 QP_CONTEXT_QPC_BYTES_156_SL_M
,
3366 QP_CONTEXT_QPC_BYTES_156_SL_S
));
3367 rdma_ah_set_ah_flags(&qp_attr
->ah_attr
, IB_AH_GRH
);
3369 roce_get_field(context
->qpc_bytes_48
,
3370 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M
,
3371 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S
);
3373 roce_get_field(context
->qpc_bytes_36
,
3374 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M
,
3375 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S
);
3377 roce_get_field(context
->qpc_bytes_44
,
3378 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M
,
3379 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S
);
3380 grh
->traffic_class
=
3381 roce_get_field(context
->qpc_bytes_48
,
3382 QP_CONTEXT_QPC_BYTES_48_TCLASS_M
,
3383 QP_CONTEXT_QPC_BYTES_48_TCLASS_S
);
3385 memcpy(grh
->dgid
.raw
, context
->dgid
,
3386 sizeof(grh
->dgid
.raw
));
3389 qp_attr
->pkey_index
= roce_get_field(context
->qpc_bytes_12
,
3390 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
3391 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
);
3392 qp_attr
->port_num
= hr_qp
->port
+ 1;
3393 qp_attr
->sq_draining
= 0;
3394 qp_attr
->max_rd_atomic
= roce_get_field(context
->qpc_bytes_156
,
3395 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M
,
3396 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S
);
3397 qp_attr
->max_dest_rd_atomic
= roce_get_field(context
->qpc_bytes_32
,
3398 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M
,
3399 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S
);
3400 qp_attr
->min_rnr_timer
= (u8
)(roce_get_field(context
->qpc_bytes_24
,
3401 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M
,
3402 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S
));
3403 qp_attr
->timeout
= (u8
)(roce_get_field(context
->qpc_bytes_156
,
3404 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M
,
3405 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S
));
3406 qp_attr
->retry_cnt
= roce_get_field(context
->qpc_bytes_148
,
3407 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M
,
3408 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S
);
3409 qp_attr
->rnr_retry
= context
->rnr_retry
;
3412 qp_attr
->cur_qp_state
= qp_attr
->qp_state
;
3413 qp_attr
->cap
.max_recv_wr
= hr_qp
->rq
.wqe_cnt
;
3414 qp_attr
->cap
.max_recv_sge
= hr_qp
->rq
.max_gs
;
3416 if (!ibqp
->uobject
) {
3417 qp_attr
->cap
.max_send_wr
= hr_qp
->sq
.wqe_cnt
;
3418 qp_attr
->cap
.max_send_sge
= hr_qp
->sq
.max_gs
;
3420 qp_attr
->cap
.max_send_wr
= 0;
3421 qp_attr
->cap
.max_send_sge
= 0;
3424 qp_init_attr
->cap
= qp_attr
->cap
;
3427 mutex_unlock(&hr_qp
->mutex
);
3432 int hns_roce_v1_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
,
3433 int qp_attr_mask
, struct ib_qp_init_attr
*qp_init_attr
)
3435 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
3437 return hr_qp
->doorbell_qpn
<= 1 ?
3438 hns_roce_v1_q_sqp(ibqp
, qp_attr
, qp_attr_mask
, qp_init_attr
) :
3439 hns_roce_v1_q_qp(ibqp
, qp_attr
, qp_attr_mask
, qp_init_attr
);
3442 static int check_qp_db_process_status(struct hns_roce_dev
*hr_dev
,
3443 struct hns_roce_qp
*hr_qp
,
3448 struct device
*dev
= &hr_dev
->pdev
->dev
;
3449 u32 sdb_retry_cnt
, old_retry
;
3450 u32 sdb_send_ptr
, old_send
;
3451 u32 success_flags
= 0;
3452 u32 cur_cnt
, old_cnt
;
3458 if (*wait_stage
> HNS_ROCE_V1_DB_STAGE2
||
3459 *wait_stage
< HNS_ROCE_V1_DB_STAGE1
) {
3460 dev_err(dev
, "QP(0x%lx) db status wait stage(%d) error!\n",
3461 hr_qp
->qpn
, *wait_stage
);
3465 /* Calculate the total timeout for the entire verification process */
3466 end
= msecs_to_jiffies(HNS_ROCE_V1_CHECK_DB_TIMEOUT_MSECS
) + jiffies
;
3468 if (*wait_stage
== HNS_ROCE_V1_DB_STAGE1
) {
3469 /* Query db process status, until hw process completely */
3470 sdb_send_ptr
= roce_read(hr_dev
, ROCEE_SDB_SEND_PTR_REG
);
3471 while (roce_hw_index_cmp_lt(sdb_send_ptr
, sdb_issue_ptr
,
3472 ROCEE_SDB_PTR_CMP_BITS
)) {
3473 if (!time_before(jiffies
, end
)) {
3474 dev_dbg(dev
, "QP(0x%lx) db process stage1 timeout. issue 0x%x send 0x%x.\n",
3475 hr_qp
->qpn
, sdb_issue_ptr
,
3480 msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS
);
3481 sdb_send_ptr
= roce_read(hr_dev
,
3482 ROCEE_SDB_SEND_PTR_REG
);
3485 if (roce_get_field(sdb_issue_ptr
,
3486 ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M
,
3487 ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S
) ==
3488 roce_get_field(sdb_send_ptr
,
3489 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3490 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
)) {
3491 old_send
= roce_read(hr_dev
, ROCEE_SDB_SEND_PTR_REG
);
3492 old_retry
= roce_read(hr_dev
, ROCEE_SDB_RETRY_CNT_REG
);
3495 tsp_st
= roce_read(hr_dev
, ROCEE_TSP_BP_ST_REG
);
3496 if (roce_get_bit(tsp_st
,
3497 ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S
) == 1) {
3498 *wait_stage
= HNS_ROCE_V1_DB_WAIT_OK
;
3502 if (!time_before(jiffies
, end
)) {
3503 dev_dbg(dev
, "QP(0x%lx) db process stage1 timeout when send ptr equals issue ptr.\n"
3504 "issue 0x%x send 0x%x.\n",
3505 hr_qp
->qpn
, sdb_issue_ptr
,
3510 msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS
);
3512 sdb_send_ptr
= roce_read(hr_dev
,
3513 ROCEE_SDB_SEND_PTR_REG
);
3514 sdb_retry_cnt
= roce_read(hr_dev
,
3515 ROCEE_SDB_RETRY_CNT_REG
);
3516 cur_cnt
= roce_get_field(sdb_send_ptr
,
3517 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3518 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
) +
3519 roce_get_field(sdb_retry_cnt
,
3520 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M
,
3521 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S
);
3522 if (!roce_get_bit(tsp_st
,
3523 ROCEE_CNT_CLR_CE_CNT_CLR_CE_S
)) {
3524 old_cnt
= roce_get_field(old_send
,
3525 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3526 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
) +
3527 roce_get_field(old_retry
,
3528 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M
,
3529 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S
);
3530 if (cur_cnt
- old_cnt
> SDB_ST_CMP_VAL
)
3533 old_cnt
= roce_get_field(old_send
,
3534 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3535 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
);
3536 if (cur_cnt
- old_cnt
> SDB_ST_CMP_VAL
)
3539 send_ptr
= roce_get_field(old_send
,
3540 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3541 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
) +
3542 roce_get_field(sdb_retry_cnt
,
3543 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M
,
3544 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S
);
3545 roce_set_field(old_send
,
3546 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
3547 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
,
3551 } while (!success_flags
);
3554 *wait_stage
= HNS_ROCE_V1_DB_STAGE2
;
3556 /* Get list pointer */
3557 *sdb_inv_cnt
= roce_read(hr_dev
, ROCEE_SDB_INV_CNT_REG
);
3558 dev_dbg(dev
, "QP(0x%lx) db process stage2. inv cnt = 0x%x.\n",
3559 hr_qp
->qpn
, *sdb_inv_cnt
);
3562 if (*wait_stage
== HNS_ROCE_V1_DB_STAGE2
) {
3563 /* Query db's list status, until hw reversal */
3564 inv_cnt
= roce_read(hr_dev
, ROCEE_SDB_INV_CNT_REG
);
3565 while (roce_hw_index_cmp_lt(inv_cnt
,
3566 *sdb_inv_cnt
+ SDB_INV_CNT_OFFSET
,
3567 ROCEE_SDB_CNT_CMP_BITS
)) {
3568 if (!time_before(jiffies
, end
)) {
3569 dev_dbg(dev
, "QP(0x%lx) db process stage2 timeout. inv cnt 0x%x.\n",
3570 hr_qp
->qpn
, inv_cnt
);
3574 msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS
);
3575 inv_cnt
= roce_read(hr_dev
, ROCEE_SDB_INV_CNT_REG
);
3578 *wait_stage
= HNS_ROCE_V1_DB_WAIT_OK
;
3584 static int check_qp_reset_state(struct hns_roce_dev
*hr_dev
,
3585 struct hns_roce_qp
*hr_qp
,
3586 struct hns_roce_qp_work
*qp_work_entry
,
3589 struct device
*dev
= &hr_dev
->pdev
->dev
;
3593 if (hr_qp
->state
!= IB_QPS_RESET
) {
3594 /* Set qp to ERR, waiting for hw complete processing all dbs */
3595 ret
= hns_roce_v1_modify_qp(&hr_qp
->ibqp
, NULL
, 0, hr_qp
->state
,
3598 dev_err(dev
, "Modify QP(0x%lx) to ERR failed!\n",
3603 /* Record issued doorbell */
3604 sdb_issue_ptr
= roce_read(hr_dev
, ROCEE_SDB_ISSUE_PTR_REG
);
3605 qp_work_entry
->sdb_issue_ptr
= sdb_issue_ptr
;
3606 qp_work_entry
->db_wait_stage
= HNS_ROCE_V1_DB_STAGE1
;
3608 /* Query db process status, until hw process completely */
3609 ret
= check_qp_db_process_status(hr_dev
, hr_qp
, sdb_issue_ptr
,
3610 &qp_work_entry
->sdb_inv_cnt
,
3611 &qp_work_entry
->db_wait_stage
);
3613 dev_err(dev
, "Check QP(0x%lx) db process status failed!\n",
3618 if (qp_work_entry
->db_wait_stage
!= HNS_ROCE_V1_DB_WAIT_OK
) {
3619 qp_work_entry
->sche_cnt
= 0;
3624 /* Modify qp to reset before destroying qp */
3625 ret
= hns_roce_v1_modify_qp(&hr_qp
->ibqp
, NULL
, 0, hr_qp
->state
,
3628 dev_err(dev
, "Modify QP(0x%lx) to RST failed!\n",
3637 static void hns_roce_v1_destroy_qp_work_fn(struct work_struct
*work
)
3639 struct hns_roce_qp_work
*qp_work_entry
;
3640 struct hns_roce_v1_priv
*priv
;
3641 struct hns_roce_dev
*hr_dev
;
3642 struct hns_roce_qp
*hr_qp
;
3646 qp_work_entry
= container_of(work
, struct hns_roce_qp_work
, work
);
3647 hr_dev
= to_hr_dev(qp_work_entry
->ib_dev
);
3648 dev
= &hr_dev
->pdev
->dev
;
3649 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
3650 hr_qp
= qp_work_entry
->qp
;
3652 dev_dbg(dev
, "Schedule destroy QP(0x%lx) work.\n", hr_qp
->qpn
);
3654 qp_work_entry
->sche_cnt
++;
3656 /* Query db process status, until hw process completely */
3657 ret
= check_qp_db_process_status(hr_dev
, hr_qp
,
3658 qp_work_entry
->sdb_issue_ptr
,
3659 &qp_work_entry
->sdb_inv_cnt
,
3660 &qp_work_entry
->db_wait_stage
);
3662 dev_err(dev
, "Check QP(0x%lx) db process status failed!\n",
3667 if (qp_work_entry
->db_wait_stage
!= HNS_ROCE_V1_DB_WAIT_OK
&&
3668 priv
->des_qp
.requeue_flag
) {
3669 queue_work(priv
->des_qp
.qp_wq
, work
);
3673 /* Modify qp to reset before destroying qp */
3674 ret
= hns_roce_v1_modify_qp(&hr_qp
->ibqp
, NULL
, 0, hr_qp
->state
,
3677 dev_err(dev
, "Modify QP(0x%lx) to RST failed!\n", hr_qp
->qpn
);
3681 hns_roce_qp_remove(hr_dev
, hr_qp
);
3682 hns_roce_qp_free(hr_dev
, hr_qp
);
3684 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
) {
3685 /* RC QP, release QPN */
3686 hns_roce_release_range_qp(hr_dev
, hr_qp
->qpn
, 1);
3689 kfree(hr_to_hr_sqp(hr_qp
));
3691 kfree(qp_work_entry
);
3693 dev_dbg(dev
, "Accomplished destroy QP(0x%lx) work.\n", hr_qp
->qpn
);
3696 int hns_roce_v1_destroy_qp(struct ib_qp
*ibqp
)
3698 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
3699 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
3700 struct device
*dev
= &hr_dev
->pdev
->dev
;
3701 struct hns_roce_qp_work qp_work_entry
;
3702 struct hns_roce_qp_work
*qp_work
;
3703 struct hns_roce_v1_priv
*priv
;
3704 struct hns_roce_cq
*send_cq
, *recv_cq
;
3705 int is_user
= !!ibqp
->pd
->uobject
;
3709 ret
= check_qp_reset_state(hr_dev
, hr_qp
, &qp_work_entry
, &is_timeout
);
3711 dev_err(dev
, "QP reset state check failed(%d)!\n", ret
);
3715 send_cq
= to_hr_cq(hr_qp
->ibqp
.send_cq
);
3716 recv_cq
= to_hr_cq(hr_qp
->ibqp
.recv_cq
);
3718 hns_roce_lock_cqs(send_cq
, recv_cq
);
3720 __hns_roce_v1_cq_clean(recv_cq
, hr_qp
->qpn
, hr_qp
->ibqp
.srq
?
3721 to_hr_srq(hr_qp
->ibqp
.srq
) : NULL
);
3722 if (send_cq
!= recv_cq
)
3723 __hns_roce_v1_cq_clean(send_cq
, hr_qp
->qpn
, NULL
);
3725 hns_roce_unlock_cqs(send_cq
, recv_cq
);
3728 hns_roce_qp_remove(hr_dev
, hr_qp
);
3729 hns_roce_qp_free(hr_dev
, hr_qp
);
3731 /* RC QP, release QPN */
3732 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
)
3733 hns_roce_release_range_qp(hr_dev
, hr_qp
->qpn
, 1);
3736 hns_roce_mtt_cleanup(hr_dev
, &hr_qp
->mtt
);
3739 ib_umem_release(hr_qp
->umem
);
3741 kfree(hr_qp
->sq
.wrid
);
3742 kfree(hr_qp
->rq
.wrid
);
3744 hns_roce_buf_free(hr_dev
, hr_qp
->buff_size
, &hr_qp
->hr_buf
);
3748 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
)
3751 kfree(hr_to_hr_sqp(hr_qp
));
3753 qp_work
= kzalloc(sizeof(*qp_work
), GFP_KERNEL
);
3757 INIT_WORK(&qp_work
->work
, hns_roce_v1_destroy_qp_work_fn
);
3758 qp_work
->ib_dev
= &hr_dev
->ib_dev
;
3759 qp_work
->qp
= hr_qp
;
3760 qp_work
->db_wait_stage
= qp_work_entry
.db_wait_stage
;
3761 qp_work
->sdb_issue_ptr
= qp_work_entry
.sdb_issue_ptr
;
3762 qp_work
->sdb_inv_cnt
= qp_work_entry
.sdb_inv_cnt
;
3763 qp_work
->sche_cnt
= qp_work_entry
.sche_cnt
;
3765 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
3766 queue_work(priv
->des_qp
.qp_wq
, &qp_work
->work
);
3767 dev_dbg(dev
, "Begin destroy QP(0x%lx) work.\n", hr_qp
->qpn
);
3773 int hns_roce_v1_destroy_cq(struct ib_cq
*ibcq
)
3775 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibcq
->device
);
3776 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
3777 struct device
*dev
= &hr_dev
->pdev
->dev
;
3784 hns_roce_free_cq(hr_dev
, hr_cq
);
3787 * Before freeing cq buffer, we need to ensure that the outstanding CQE
3788 * have been written by checking the CQE counter.
3790 cqe_cnt_ori
= roce_read(hr_dev
, ROCEE_SCAEP_WR_CQE_CNT
);
3792 if (roce_read(hr_dev
, ROCEE_CAEP_CQE_WCMD_EMPTY
) &
3793 HNS_ROCE_CQE_WCMD_EMPTY_BIT
)
3796 cqe_cnt_cur
= roce_read(hr_dev
, ROCEE_SCAEP_WR_CQE_CNT
);
3797 if ((cqe_cnt_cur
- cqe_cnt_ori
) >= HNS_ROCE_MIN_CQE_CNT
)
3800 msleep(HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS
);
3801 if (wait_time
> HNS_ROCE_MAX_FREE_CQ_WAIT_CNT
) {
3802 dev_warn(dev
, "Destroy cq 0x%lx timeout!\n",
3810 hns_roce_mtt_cleanup(hr_dev
, &hr_cq
->hr_buf
.hr_mtt
);
3813 ib_umem_release(hr_cq
->umem
);
3815 /* Free the buff of stored cq */
3816 cq_buf_size
= (ibcq
->cqe
+ 1) * hr_dev
->caps
.cq_entry_sz
;
3817 hns_roce_buf_free(hr_dev
, cq_buf_size
, &hr_cq
->hr_buf
.hr_buf
);
3825 struct hns_roce_v1_priv hr_v1_priv
;
3827 struct hns_roce_hw hns_roce_hw_v1
= {
3828 .reset
= hns_roce_v1_reset
,
3829 .hw_profile
= hns_roce_v1_profile
,
3830 .hw_init
= hns_roce_v1_init
,
3831 .hw_exit
= hns_roce_v1_exit
,
3832 .set_gid
= hns_roce_v1_set_gid
,
3833 .set_mac
= hns_roce_v1_set_mac
,
3834 .set_mtu
= hns_roce_v1_set_mtu
,
3835 .write_mtpt
= hns_roce_v1_write_mtpt
,
3836 .write_cqc
= hns_roce_v1_write_cqc
,
3837 .clear_hem
= hns_roce_v1_clear_hem
,
3838 .modify_qp
= hns_roce_v1_modify_qp
,
3839 .query_qp
= hns_roce_v1_query_qp
,
3840 .destroy_qp
= hns_roce_v1_destroy_qp
,
3841 .post_send
= hns_roce_v1_post_send
,
3842 .post_recv
= hns_roce_v1_post_recv
,
3843 .req_notify_cq
= hns_roce_v1_req_notify_cq
,
3844 .poll_cq
= hns_roce_v1_poll_cq
,
3845 .dereg_mr
= hns_roce_v1_dereg_mr
,
3846 .destroy_cq
= hns_roce_v1_destroy_cq
,
3847 .priv
= &hr_v1_priv
,