1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright (c) 2018 Hisilicon Limited.
6 #include <rdma/ib_umem.h>
7 #include <rdma/hns-abi.h>
8 #include "hns_roce_device.h"
9 #include "hns_roce_cmd.h"
10 #include "hns_roce_hem.h"
12 void hns_roce_srq_event(struct hns_roce_dev
*hr_dev
, u32 srqn
, int event_type
)
14 struct hns_roce_srq_table
*srq_table
= &hr_dev
->srq_table
;
15 struct hns_roce_srq
*srq
;
17 xa_lock(&srq_table
->xa
);
18 srq
= xa_load(&srq_table
->xa
, srqn
& (hr_dev
->caps
.num_srqs
- 1));
20 atomic_inc(&srq
->refcount
);
21 xa_unlock(&srq_table
->xa
);
24 dev_warn(hr_dev
->dev
, "Async event for bogus SRQ %08x\n", srqn
);
28 srq
->event(srq
, event_type
);
30 if (atomic_dec_and_test(&srq
->refcount
))
34 static void hns_roce_ib_srq_event(struct hns_roce_srq
*srq
,
35 enum hns_roce_event event_type
)
37 struct hns_roce_dev
*hr_dev
= to_hr_dev(srq
->ibsrq
.device
);
38 struct ib_srq
*ibsrq
= &srq
->ibsrq
;
39 struct ib_event event
;
41 if (ibsrq
->event_handler
) {
42 event
.device
= ibsrq
->device
;
43 event
.element
.srq
= ibsrq
;
45 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH
:
46 event
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
48 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR
:
49 event
.event
= IB_EVENT_SRQ_ERR
;
53 "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n",
54 event_type
, srq
->srqn
);
58 ibsrq
->event_handler(&event
, ibsrq
->srq_context
);
62 static int hns_roce_hw_create_srq(struct hns_roce_dev
*dev
,
63 struct hns_roce_cmd_mailbox
*mailbox
,
64 unsigned long srq_num
)
66 return hns_roce_cmd_mbox(dev
, mailbox
->dma
, 0, srq_num
, 0,
67 HNS_ROCE_CMD_CREATE_SRQ
,
68 HNS_ROCE_CMD_TIMEOUT_MSECS
);
71 static int hns_roce_hw_destroy_srq(struct hns_roce_dev
*dev
,
72 struct hns_roce_cmd_mailbox
*mailbox
,
73 unsigned long srq_num
)
75 return hns_roce_cmd_mbox(dev
, 0, mailbox
? mailbox
->dma
: 0, srq_num
,
76 mailbox
? 0 : 1, HNS_ROCE_CMD_DESTROY_SRQ
,
77 HNS_ROCE_CMD_TIMEOUT_MSECS
);
80 static int alloc_srqc(struct hns_roce_dev
*hr_dev
, struct hns_roce_srq
*srq
,
81 u32 pdn
, u32 cqn
, u16 xrcd
, u64 db_rec_addr
)
83 struct hns_roce_srq_table
*srq_table
= &hr_dev
->srq_table
;
84 struct ib_device
*ibdev
= &hr_dev
->ib_dev
;
85 struct hns_roce_cmd_mailbox
*mailbox
;
86 u64 mtts_wqe
[MTT_MIN_COUNT
] = { 0 };
87 u64 mtts_idx
[MTT_MIN_COUNT
] = { 0 };
88 dma_addr_t dma_handle_wqe
= 0;
89 dma_addr_t dma_handle_idx
= 0;
92 /* Get the physical address of srq buf */
93 ret
= hns_roce_mtr_find(hr_dev
, &srq
->buf_mtr
, 0, mtts_wqe
,
94 ARRAY_SIZE(mtts_wqe
), &dma_handle_wqe
);
96 ibdev_err(ibdev
, "Failed to find mtr for SRQ WQE\n");
100 /* Get physical address of idx que buf */
101 ret
= hns_roce_mtr_find(hr_dev
, &srq
->idx_que
.mtr
, 0, mtts_idx
,
102 ARRAY_SIZE(mtts_idx
), &dma_handle_idx
);
104 ibdev_err(ibdev
, "Failed to find mtr for SRQ idx\n");
108 ret
= hns_roce_bitmap_alloc(&srq_table
->bitmap
, &srq
->srqn
);
110 ibdev_err(ibdev
, "Failed to alloc SRQ number, err %d\n", ret
);
114 ret
= hns_roce_table_get(hr_dev
, &srq_table
->table
, srq
->srqn
);
116 ibdev_err(ibdev
, "Failed to get SRQC table, err %d\n", ret
);
120 ret
= xa_err(xa_store(&srq_table
->xa
, srq
->srqn
, srq
, GFP_KERNEL
));
122 ibdev_err(ibdev
, "Failed to store SRQC, err %d\n", ret
);
126 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
127 if (IS_ERR_OR_NULL(mailbox
)) {
129 ibdev_err(ibdev
, "Failed to alloc mailbox for SRQC\n");
133 hr_dev
->hw
->write_srqc(hr_dev
, srq
, pdn
, xrcd
, cqn
, mailbox
->buf
,
134 mtts_wqe
, mtts_idx
, dma_handle_wqe
,
137 ret
= hns_roce_hw_create_srq(hr_dev
, mailbox
, srq
->srqn
);
138 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
140 ibdev_err(ibdev
, "Failed to config SRQC, err %d\n", ret
);
144 atomic_set(&srq
->refcount
, 1);
145 init_completion(&srq
->free
);
149 xa_erase(&srq_table
->xa
, srq
->srqn
);
152 hns_roce_table_put(hr_dev
, &srq_table
->table
, srq
->srqn
);
155 hns_roce_bitmap_free(&srq_table
->bitmap
, srq
->srqn
, BITMAP_NO_RR
);
159 static void free_srqc(struct hns_roce_dev
*hr_dev
, struct hns_roce_srq
*srq
)
161 struct hns_roce_srq_table
*srq_table
= &hr_dev
->srq_table
;
164 ret
= hns_roce_hw_destroy_srq(hr_dev
, NULL
, srq
->srqn
);
166 dev_err(hr_dev
->dev
, "DESTROY_SRQ failed (%d) for SRQN %06lx\n",
169 xa_erase(&srq_table
->xa
, srq
->srqn
);
171 if (atomic_dec_and_test(&srq
->refcount
))
172 complete(&srq
->free
);
173 wait_for_completion(&srq
->free
);
175 hns_roce_table_put(hr_dev
, &srq_table
->table
, srq
->srqn
);
176 hns_roce_bitmap_free(&srq_table
->bitmap
, srq
->srqn
, BITMAP_NO_RR
);
179 static int alloc_srq_buf(struct hns_roce_dev
*hr_dev
, struct hns_roce_srq
*srq
,
180 struct ib_udata
*udata
, unsigned long addr
)
182 struct ib_device
*ibdev
= &hr_dev
->ib_dev
;
183 struct hns_roce_buf_attr buf_attr
= {};
186 srq
->wqe_shift
= ilog2(roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE
,
190 buf_attr
.page_shift
= hr_dev
->caps
.srqwqe_buf_pg_sz
+ HNS_HW_PAGE_SHIFT
;
191 buf_attr
.region
[0].size
= to_hr_hem_entries_size(srq
->wqe_cnt
,
193 buf_attr
.region
[0].hopnum
= hr_dev
->caps
.srqwqe_hop_num
;
194 buf_attr
.region_count
= 1;
195 buf_attr
.fixed_page
= true;
197 err
= hns_roce_mtr_create(hr_dev
, &srq
->buf_mtr
, &buf_attr
,
198 hr_dev
->caps
.srqwqe_ba_pg_sz
+
199 HNS_HW_PAGE_SHIFT
, udata
, addr
);
201 ibdev_err(ibdev
, "Failed to alloc SRQ buf mtr, err %d\n", err
);
206 static void free_srq_buf(struct hns_roce_dev
*hr_dev
, struct hns_roce_srq
*srq
)
208 hns_roce_mtr_destroy(hr_dev
, &srq
->buf_mtr
);
211 static int alloc_srq_idx(struct hns_roce_dev
*hr_dev
, struct hns_roce_srq
*srq
,
212 struct ib_udata
*udata
, unsigned long addr
)
214 struct hns_roce_idx_que
*idx_que
= &srq
->idx_que
;
215 struct ib_device
*ibdev
= &hr_dev
->ib_dev
;
216 struct hns_roce_buf_attr buf_attr
= {};
219 srq
->idx_que
.entry_shift
= ilog2(HNS_ROCE_IDX_QUE_ENTRY_SZ
);
221 buf_attr
.page_shift
= hr_dev
->caps
.idx_buf_pg_sz
+ HNS_HW_PAGE_SHIFT
;
222 buf_attr
.region
[0].size
= to_hr_hem_entries_size(srq
->wqe_cnt
,
223 srq
->idx_que
.entry_shift
);
224 buf_attr
.region
[0].hopnum
= hr_dev
->caps
.idx_hop_num
;
225 buf_attr
.region_count
= 1;
226 buf_attr
.fixed_page
= true;
228 err
= hns_roce_mtr_create(hr_dev
, &idx_que
->mtr
, &buf_attr
,
229 hr_dev
->caps
.idx_ba_pg_sz
+ HNS_HW_PAGE_SHIFT
,
232 ibdev_err(ibdev
, "Failed to alloc SRQ idx mtr, err %d\n", err
);
237 idx_que
->bitmap
= bitmap_zalloc(srq
->wqe_cnt
, GFP_KERNEL
);
238 if (!idx_que
->bitmap
) {
239 ibdev_err(ibdev
, "Failed to alloc SRQ idx bitmap\n");
248 hns_roce_mtr_destroy(hr_dev
, &idx_que
->mtr
);
253 static void free_srq_idx(struct hns_roce_dev
*hr_dev
, struct hns_roce_srq
*srq
)
255 struct hns_roce_idx_que
*idx_que
= &srq
->idx_que
;
257 bitmap_free(idx_que
->bitmap
);
258 idx_que
->bitmap
= NULL
;
259 hns_roce_mtr_destroy(hr_dev
, &idx_que
->mtr
);
262 static int alloc_srq_wrid(struct hns_roce_dev
*hr_dev
, struct hns_roce_srq
*srq
)
265 srq
->tail
= srq
->wqe_cnt
- 1;
266 srq
->wrid
= kvmalloc_array(srq
->wqe_cnt
, sizeof(u64
), GFP_KERNEL
);
273 static void free_srq_wrid(struct hns_roce_srq
*srq
)
279 int hns_roce_create_srq(struct ib_srq
*ib_srq
,
280 struct ib_srq_init_attr
*init_attr
,
281 struct ib_udata
*udata
)
283 struct hns_roce_dev
*hr_dev
= to_hr_dev(ib_srq
->device
);
284 struct hns_roce_ib_create_srq_resp resp
= {};
285 struct hns_roce_srq
*srq
= to_hr_srq(ib_srq
);
286 struct ib_device
*ibdev
= &hr_dev
->ib_dev
;
287 struct hns_roce_ib_create_srq ucmd
= {};
291 /* Check the actual SRQ wqe and SRQ sge num */
292 if (init_attr
->attr
.max_wr
>= hr_dev
->caps
.max_srq_wrs
||
293 init_attr
->attr
.max_sge
> hr_dev
->caps
.max_srq_sges
)
296 mutex_init(&srq
->mutex
);
297 spin_lock_init(&srq
->lock
);
299 srq
->wqe_cnt
= roundup_pow_of_two(init_attr
->attr
.max_wr
+ 1);
300 srq
->max_gs
= init_attr
->attr
.max_sge
+ HNS_ROCE_RESERVED_SGE
;
303 ret
= ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
));
305 ibdev_err(ibdev
, "Failed to copy SRQ udata, err %d\n",
311 ret
= alloc_srq_buf(hr_dev
, srq
, udata
, ucmd
.buf_addr
);
313 ibdev_err(ibdev
, "Failed to alloc SRQ buffer, err %d\n", ret
);
317 ret
= alloc_srq_idx(hr_dev
, srq
, udata
, ucmd
.que_addr
);
319 ibdev_err(ibdev
, "Failed to alloc SRQ idx, err %d\n", ret
);
324 ret
= alloc_srq_wrid(hr_dev
, srq
);
326 ibdev_err(ibdev
, "Failed to alloc SRQ wrid, err %d\n",
332 cqn
= ib_srq_has_cq(init_attr
->srq_type
) ?
333 to_hr_cq(init_attr
->ext
.cq
)->cqn
: 0;
334 srq
->db_reg_l
= hr_dev
->reg_base
+ SRQ_DB_REG
;
336 ret
= alloc_srqc(hr_dev
, srq
, to_hr_pd(ib_srq
->pd
)->pdn
, cqn
, 0, 0);
338 ibdev_err(ibdev
, "Failed to alloc SRQ context, err %d\n", ret
);
342 srq
->event
= hns_roce_ib_srq_event
;
343 resp
.srqn
= srq
->srqn
;
346 if (ib_copy_to_udata(udata
, &resp
,
347 min(udata
->outlen
, sizeof(resp
)))) {
356 free_srqc(hr_dev
, srq
);
360 free_srq_idx(hr_dev
, srq
);
362 free_srq_buf(hr_dev
, srq
);
366 void hns_roce_destroy_srq(struct ib_srq
*ibsrq
, struct ib_udata
*udata
)
368 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibsrq
->device
);
369 struct hns_roce_srq
*srq
= to_hr_srq(ibsrq
);
371 free_srqc(hr_dev
, srq
);
372 free_srq_idx(hr_dev
, srq
);
374 free_srq_buf(hr_dev
, srq
);
377 int hns_roce_init_srq_table(struct hns_roce_dev
*hr_dev
)
379 struct hns_roce_srq_table
*srq_table
= &hr_dev
->srq_table
;
381 xa_init(&srq_table
->xa
);
383 return hns_roce_bitmap_init(&srq_table
->bitmap
, hr_dev
->caps
.num_srqs
,
384 hr_dev
->caps
.num_srqs
- 1,
385 hr_dev
->caps
.reserved_srqs
, 0);
388 void hns_roce_cleanup_srq_table(struct hns_roce_dev
*hr_dev
)
390 hns_roce_bitmap_cleanup(&hr_dev
->srq_table
.bitmap
);