2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/platform_device.h>
35 #include "hns_roce_device.h"
36 #include "hns_roce_hem.h"
37 #include "hns_roce_common.h"
39 #define DMA_ADDR_T_SHIFT 12
40 #define BT_BA_SHIFT 32
42 bool hns_roce_check_whether_mhop(struct hns_roce_dev
*hr_dev
, u32 type
)
44 if ((hr_dev
->caps
.qpc_hop_num
&& type
== HEM_TYPE_QPC
) ||
45 (hr_dev
->caps
.mpt_hop_num
&& type
== HEM_TYPE_MTPT
) ||
46 (hr_dev
->caps
.cqc_hop_num
&& type
== HEM_TYPE_CQC
) ||
47 (hr_dev
->caps
.srqc_hop_num
&& type
== HEM_TYPE_SRQC
) ||
48 (hr_dev
->caps
.cqe_hop_num
&& type
== HEM_TYPE_CQE
) ||
49 (hr_dev
->caps
.mtt_hop_num
&& type
== HEM_TYPE_MTT
))
54 EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop
);
56 static bool hns_roce_check_hem_null(struct hns_roce_hem
**hem
, u64 start_idx
,
61 for (i
= 0; i
< bt_chunk_num
; i
++)
62 if (hem
[start_idx
+ i
])
68 static bool hns_roce_check_bt_null(u64
**bt
, u64 start_idx
, u32 bt_chunk_num
)
72 for (i
= 0; i
< bt_chunk_num
; i
++)
73 if (bt
[start_idx
+ i
])
79 static int hns_roce_get_bt_num(u32 table_type
, u32 hop_num
)
81 if (check_whether_bt_num_3(table_type
, hop_num
))
83 else if (check_whether_bt_num_2(table_type
, hop_num
))
85 else if (check_whether_bt_num_1(table_type
, hop_num
))
91 int hns_roce_calc_hem_mhop(struct hns_roce_dev
*hr_dev
,
92 struct hns_roce_hem_table
*table
, unsigned long *obj
,
93 struct hns_roce_hem_mhop
*mhop
)
95 struct device
*dev
= hr_dev
->dev
;
101 switch (table
->type
) {
103 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.qpc_buf_pg_sz
105 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.qpc_ba_pg_sz
107 mhop
->ba_l0_num
= hr_dev
->caps
.qpc_bt_num
;
108 mhop
->hop_num
= hr_dev
->caps
.qpc_hop_num
;
111 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.mpt_buf_pg_sz
113 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.mpt_ba_pg_sz
115 mhop
->ba_l0_num
= hr_dev
->caps
.mpt_bt_num
;
116 mhop
->hop_num
= hr_dev
->caps
.mpt_hop_num
;
119 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.cqc_buf_pg_sz
121 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.cqc_ba_pg_sz
123 mhop
->ba_l0_num
= hr_dev
->caps
.cqc_bt_num
;
124 mhop
->hop_num
= hr_dev
->caps
.cqc_hop_num
;
127 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.srqc_buf_pg_sz
129 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.srqc_ba_pg_sz
131 mhop
->ba_l0_num
= hr_dev
->caps
.srqc_bt_num
;
132 mhop
->hop_num
= hr_dev
->caps
.srqc_hop_num
;
135 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.mtt_buf_pg_sz
137 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.mtt_ba_pg_sz
139 mhop
->ba_l0_num
= mhop
->bt_chunk_size
/ 8;
140 mhop
->hop_num
= hr_dev
->caps
.mtt_hop_num
;
143 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.cqe_buf_pg_sz
145 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.cqe_ba_pg_sz
147 mhop
->ba_l0_num
= mhop
->bt_chunk_size
/ 8;
148 mhop
->hop_num
= hr_dev
->caps
.cqe_hop_num
;
151 dev_err(dev
, "Table %d not support multi-hop addressing!\n",
160 * QPC/MTPT/CQC/SRQC alloc hem for buffer pages.
161 * MTT/CQE alloc hem for bt pages.
163 bt_num
= hns_roce_get_bt_num(table
->type
, mhop
->hop_num
);
164 chunk_ba_num
= mhop
->bt_chunk_size
/ 8;
165 chunk_size
= table
->type
< HEM_TYPE_MTT
? mhop
->buf_chunk_size
:
167 table_idx
= (*obj
& (table
->num_obj
- 1)) /
168 (chunk_size
/ table
->obj_size
);
171 mhop
->l2_idx
= table_idx
& (chunk_ba_num
- 1);
172 mhop
->l1_idx
= table_idx
/ chunk_ba_num
& (chunk_ba_num
- 1);
173 mhop
->l0_idx
= table_idx
/ chunk_ba_num
/ chunk_ba_num
;
176 mhop
->l1_idx
= table_idx
& (chunk_ba_num
- 1);
177 mhop
->l0_idx
= table_idx
/ chunk_ba_num
;
180 mhop
->l0_idx
= table_idx
;
183 dev_err(dev
, "Table %d not support hop_num = %d!\n",
184 table
->type
, mhop
->hop_num
);
187 if (mhop
->l0_idx
>= mhop
->ba_l0_num
)
188 mhop
->l0_idx
%= mhop
->ba_l0_num
;
192 EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop
);
194 static struct hns_roce_hem
*hns_roce_alloc_hem(struct hns_roce_dev
*hr_dev
,
196 unsigned long hem_alloc_size
,
199 struct hns_roce_hem_chunk
*chunk
= NULL
;
200 struct hns_roce_hem
*hem
;
201 struct scatterlist
*mem
;
205 WARN_ON(gfp_mask
& __GFP_HIGHMEM
);
207 hem
= kmalloc(sizeof(*hem
),
208 gfp_mask
& ~(__GFP_HIGHMEM
| __GFP_NOWARN
));
213 INIT_LIST_HEAD(&hem
->chunk_list
);
215 order
= get_order(hem_alloc_size
);
219 chunk
= kmalloc(sizeof(*chunk
),
220 gfp_mask
& ~(__GFP_HIGHMEM
| __GFP_NOWARN
));
224 sg_init_table(chunk
->mem
, HNS_ROCE_HEM_CHUNK_LEN
);
227 memset(chunk
->buf
, 0, sizeof(chunk
->buf
));
228 list_add_tail(&chunk
->list
, &hem
->chunk_list
);
231 while (1 << order
> npages
)
235 * Alloc memory one time. If failed, don't alloc small block
236 * memory, directly return fail.
238 mem
= &chunk
->mem
[chunk
->npages
];
239 buf
= dma_alloc_coherent(hr_dev
->dev
, PAGE_SIZE
<< order
,
240 &sg_dma_address(mem
), gfp_mask
);
244 chunk
->buf
[chunk
->npages
] = buf
;
245 sg_dma_len(mem
) = PAGE_SIZE
<< order
;
249 npages
-= 1 << order
;
255 hns_roce_free_hem(hr_dev
, hem
);
259 void hns_roce_free_hem(struct hns_roce_dev
*hr_dev
, struct hns_roce_hem
*hem
)
261 struct hns_roce_hem_chunk
*chunk
, *tmp
;
267 list_for_each_entry_safe(chunk
, tmp
, &hem
->chunk_list
, list
) {
268 for (i
= 0; i
< chunk
->npages
; ++i
)
269 dma_free_coherent(hr_dev
->dev
,
270 sg_dma_len(&chunk
->mem
[i
]),
272 sg_dma_address(&chunk
->mem
[i
]));
279 static int hns_roce_set_hem(struct hns_roce_dev
*hr_dev
,
280 struct hns_roce_hem_table
*table
, unsigned long obj
)
282 spinlock_t
*lock
= &hr_dev
->bt_cmd_lock
;
283 struct device
*dev
= hr_dev
->dev
;
284 unsigned long end
= 0;
286 struct hns_roce_hem_iter iter
;
287 void __iomem
*bt_cmd
;
288 u32 bt_cmd_h_val
= 0;
294 /* Find the HEM(Hardware Entry Memory) entry */
295 unsigned long i
= (obj
& (table
->num_obj
- 1)) /
296 (table
->table_chunk_size
/ table
->obj_size
);
298 switch (table
->type
) {
300 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
301 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_QPC
);
304 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
305 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
,
309 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
310 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_CQC
);
313 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
314 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
,
320 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M
,
321 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S
, obj
);
322 roce_set_bit(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S
, 0);
323 roce_set_bit(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S
, 1);
325 /* Currently iter only a chunk */
326 for (hns_roce_hem_first(table
->hem
[i
], &iter
);
327 !hns_roce_hem_last(&iter
); hns_roce_hem_next(&iter
)) {
328 bt_ba
= hns_roce_hem_addr(&iter
) >> DMA_ADDR_T_SHIFT
;
330 spin_lock_irqsave(lock
, flags
);
332 bt_cmd
= hr_dev
->reg_base
+ ROCEE_BT_CMD_H_REG
;
334 end
= msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS
) + jiffies
;
336 if (readl(bt_cmd
) >> BT_CMD_SYNC_SHIFT
) {
337 if (!(time_before(jiffies
, end
))) {
338 dev_err(dev
, "Write bt_cmd err,hw_sync is not zero.\n");
339 spin_unlock_irqrestore(lock
, flags
);
345 msleep(HW_SYNC_SLEEP_TIME_INTERVAL
);
348 bt_cmd_l
= (u32
)bt_ba
;
349 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M
,
350 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S
,
351 bt_ba
>> BT_BA_SHIFT
);
353 bt_cmd_val
[0] = bt_cmd_l
;
354 bt_cmd_val
[1] = bt_cmd_h_val
;
355 hns_roce_write64_k(bt_cmd_val
,
356 hr_dev
->reg_base
+ ROCEE_BT_CMD_L_REG
);
357 spin_unlock_irqrestore(lock
, flags
);
363 static int hns_roce_table_mhop_get(struct hns_roce_dev
*hr_dev
,
364 struct hns_roce_hem_table
*table
,
367 struct device
*dev
= hr_dev
->dev
;
368 struct hns_roce_hem_mhop mhop
;
369 struct hns_roce_hem_iter iter
;
380 unsigned long mhop_obj
= obj
;
381 int bt_l1_allocated
= 0;
382 int bt_l0_allocated
= 0;
386 ret
= hns_roce_calc_hem_mhop(hr_dev
, table
, &mhop_obj
, &mhop
);
390 buf_chunk_size
= mhop
.buf_chunk_size
;
391 bt_chunk_size
= mhop
.bt_chunk_size
;
392 hop_num
= mhop
.hop_num
;
393 chunk_ba_num
= bt_chunk_size
/ 8;
395 bt_num
= hns_roce_get_bt_num(table
->type
, hop_num
);
398 hem_idx
= mhop
.l0_idx
* chunk_ba_num
* chunk_ba_num
+
399 mhop
.l1_idx
* chunk_ba_num
+ mhop
.l2_idx
;
400 bt_l1_idx
= mhop
.l0_idx
* chunk_ba_num
+ mhop
.l1_idx
;
401 bt_l0_idx
= mhop
.l0_idx
;
404 hem_idx
= mhop
.l0_idx
* chunk_ba_num
+ mhop
.l1_idx
;
405 bt_l0_idx
= mhop
.l0_idx
;
408 hem_idx
= mhop
.l0_idx
;
411 dev_err(dev
, "Table %d not support hop_num = %d!\n",
412 table
->type
, hop_num
);
416 mutex_lock(&table
->mutex
);
418 if (table
->hem
[hem_idx
]) {
419 ++table
->hem
[hem_idx
]->refcount
;
423 /* alloc L1 BA's chunk */
424 if ((check_whether_bt_num_3(table
->type
, hop_num
) ||
425 check_whether_bt_num_2(table
->type
, hop_num
)) &&
426 !table
->bt_l0
[bt_l0_idx
]) {
427 table
->bt_l0
[bt_l0_idx
] = dma_alloc_coherent(dev
, bt_chunk_size
,
428 &(table
->bt_l0_dma_addr
[bt_l0_idx
]),
430 if (!table
->bt_l0
[bt_l0_idx
]) {
436 /* set base address to hardware */
437 if (table
->type
< HEM_TYPE_MTT
) {
439 if (hr_dev
->hw
->set_hem(hr_dev
, table
, obj
, step_idx
)) {
441 dev_err(dev
, "set HEM base address to HW failed!\n");
442 goto err_dma_alloc_l1
;
447 /* alloc L2 BA's chunk */
448 if (check_whether_bt_num_3(table
->type
, hop_num
) &&
449 !table
->bt_l1
[bt_l1_idx
]) {
450 table
->bt_l1
[bt_l1_idx
] = dma_alloc_coherent(dev
, bt_chunk_size
,
451 &(table
->bt_l1_dma_addr
[bt_l1_idx
]),
453 if (!table
->bt_l1
[bt_l1_idx
]) {
455 goto err_dma_alloc_l1
;
458 *(table
->bt_l0
[bt_l0_idx
] + mhop
.l1_idx
) =
459 table
->bt_l1_dma_addr
[bt_l1_idx
];
461 /* set base address to hardware */
463 if (hr_dev
->hw
->set_hem(hr_dev
, table
, obj
, step_idx
)) {
465 dev_err(dev
, "set HEM base address to HW failed!\n");
466 goto err_alloc_hem_buf
;
471 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC.
472 * alloc bt space chunk for MTT/CQE.
474 size
= table
->type
< HEM_TYPE_MTT
? buf_chunk_size
: bt_chunk_size
;
475 table
->hem
[hem_idx
] = hns_roce_alloc_hem(hr_dev
,
478 (table
->lowmem
? GFP_KERNEL
:
479 GFP_HIGHUSER
) | __GFP_NOWARN
);
480 if (!table
->hem
[hem_idx
]) {
482 goto err_alloc_hem_buf
;
485 hns_roce_hem_first(table
->hem
[hem_idx
], &iter
);
486 bt_ba
= hns_roce_hem_addr(&iter
);
488 if (table
->type
< HEM_TYPE_MTT
) {
490 *(table
->bt_l1
[bt_l1_idx
] + mhop
.l2_idx
) = bt_ba
;
492 } else if (hop_num
== 1) {
493 *(table
->bt_l0
[bt_l0_idx
] + mhop
.l1_idx
) = bt_ba
;
495 } else if (hop_num
== HNS_ROCE_HOP_NUM_0
) {
499 /* set HEM base address to hardware */
500 if (hr_dev
->hw
->set_hem(hr_dev
, table
, obj
, step_idx
)) {
502 dev_err(dev
, "set HEM base address to HW failed!\n");
503 goto err_alloc_hem_buf
;
505 } else if (hop_num
== 2) {
506 *(table
->bt_l0
[bt_l0_idx
] + mhop
.l1_idx
) = bt_ba
;
509 ++table
->hem
[hem_idx
]->refcount
;
513 if (bt_l1_allocated
) {
514 dma_free_coherent(dev
, bt_chunk_size
, table
->bt_l1
[bt_l1_idx
],
515 table
->bt_l1_dma_addr
[bt_l1_idx
]);
516 table
->bt_l1
[bt_l1_idx
] = NULL
;
520 if (bt_l0_allocated
) {
521 dma_free_coherent(dev
, bt_chunk_size
, table
->bt_l0
[bt_l0_idx
],
522 table
->bt_l0_dma_addr
[bt_l0_idx
]);
523 table
->bt_l0
[bt_l0_idx
] = NULL
;
527 mutex_unlock(&table
->mutex
);
531 int hns_roce_table_get(struct hns_roce_dev
*hr_dev
,
532 struct hns_roce_hem_table
*table
, unsigned long obj
)
534 struct device
*dev
= hr_dev
->dev
;
538 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
))
539 return hns_roce_table_mhop_get(hr_dev
, table
, obj
);
541 i
= (obj
& (table
->num_obj
- 1)) / (table
->table_chunk_size
/
544 mutex_lock(&table
->mutex
);
547 ++table
->hem
[i
]->refcount
;
551 table
->hem
[i
] = hns_roce_alloc_hem(hr_dev
,
552 table
->table_chunk_size
>> PAGE_SHIFT
,
553 table
->table_chunk_size
,
554 (table
->lowmem
? GFP_KERNEL
:
555 GFP_HIGHUSER
) | __GFP_NOWARN
);
556 if (!table
->hem
[i
]) {
561 /* Set HEM base address(128K/page, pa) to Hardware */
562 if (hns_roce_set_hem(hr_dev
, table
, obj
)) {
563 hns_roce_free_hem(hr_dev
, table
->hem
[i
]);
564 table
->hem
[i
] = NULL
;
566 dev_err(dev
, "set HEM base address to HW failed.\n");
570 ++table
->hem
[i
]->refcount
;
572 mutex_unlock(&table
->mutex
);
576 static void hns_roce_table_mhop_put(struct hns_roce_dev
*hr_dev
,
577 struct hns_roce_hem_table
*table
,
581 struct device
*dev
= hr_dev
->dev
;
582 struct hns_roce_hem_mhop mhop
;
583 unsigned long mhop_obj
= obj
;
593 ret
= hns_roce_calc_hem_mhop(hr_dev
, table
, &mhop_obj
, &mhop
);
597 bt_chunk_size
= mhop
.bt_chunk_size
;
598 hop_num
= mhop
.hop_num
;
599 chunk_ba_num
= bt_chunk_size
/ 8;
601 bt_num
= hns_roce_get_bt_num(table
->type
, hop_num
);
604 hem_idx
= mhop
.l0_idx
* chunk_ba_num
* chunk_ba_num
+
605 mhop
.l1_idx
* chunk_ba_num
+ mhop
.l2_idx
;
606 bt_l1_idx
= mhop
.l0_idx
* chunk_ba_num
+ mhop
.l1_idx
;
609 hem_idx
= mhop
.l0_idx
* chunk_ba_num
+ mhop
.l1_idx
;
612 hem_idx
= mhop
.l0_idx
;
615 dev_err(dev
, "Table %d not support hop_num = %d!\n",
616 table
->type
, hop_num
);
620 mutex_lock(&table
->mutex
);
622 if (check_refcount
&& (--table
->hem
[hem_idx
]->refcount
> 0)) {
623 mutex_unlock(&table
->mutex
);
627 if (table
->type
< HEM_TYPE_MTT
&& hop_num
== 1) {
628 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 1))
629 dev_warn(dev
, "Clear HEM base address failed.\n");
630 } else if (table
->type
< HEM_TYPE_MTT
&& hop_num
== 2) {
631 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 2))
632 dev_warn(dev
, "Clear HEM base address failed.\n");
633 } else if (table
->type
< HEM_TYPE_MTT
&&
634 hop_num
== HNS_ROCE_HOP_NUM_0
) {
635 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 0))
636 dev_warn(dev
, "Clear HEM base address failed.\n");
640 * free buffer space chunk for QPC/MTPT/CQC/SRQC.
641 * free bt space chunk for MTT/CQE.
643 hns_roce_free_hem(hr_dev
, table
->hem
[hem_idx
]);
644 table
->hem
[hem_idx
] = NULL
;
646 if (check_whether_bt_num_2(table
->type
, hop_num
)) {
647 start_idx
= mhop
.l0_idx
* chunk_ba_num
;
648 if (hns_roce_check_hem_null(table
->hem
, start_idx
,
650 if (table
->type
< HEM_TYPE_MTT
&&
651 hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 0))
652 dev_warn(dev
, "Clear HEM base address failed.\n");
654 dma_free_coherent(dev
, bt_chunk_size
,
655 table
->bt_l0
[mhop
.l0_idx
],
656 table
->bt_l0_dma_addr
[mhop
.l0_idx
]);
657 table
->bt_l0
[mhop
.l0_idx
] = NULL
;
659 } else if (check_whether_bt_num_3(table
->type
, hop_num
)) {
660 start_idx
= mhop
.l0_idx
* chunk_ba_num
* chunk_ba_num
+
661 mhop
.l1_idx
* chunk_ba_num
;
662 if (hns_roce_check_hem_null(table
->hem
, start_idx
,
664 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 1))
665 dev_warn(dev
, "Clear HEM base address failed.\n");
667 dma_free_coherent(dev
, bt_chunk_size
,
668 table
->bt_l1
[bt_l1_idx
],
669 table
->bt_l1_dma_addr
[bt_l1_idx
]);
670 table
->bt_l1
[bt_l1_idx
] = NULL
;
672 start_idx
= mhop
.l0_idx
* chunk_ba_num
;
673 if (hns_roce_check_bt_null(table
->bt_l1
, start_idx
,
675 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
,
677 dev_warn(dev
, "Clear HEM base address failed.\n");
679 dma_free_coherent(dev
, bt_chunk_size
,
680 table
->bt_l0
[mhop
.l0_idx
],
681 table
->bt_l0_dma_addr
[mhop
.l0_idx
]);
682 table
->bt_l0
[mhop
.l0_idx
] = NULL
;
687 mutex_unlock(&table
->mutex
);
690 void hns_roce_table_put(struct hns_roce_dev
*hr_dev
,
691 struct hns_roce_hem_table
*table
, unsigned long obj
)
693 struct device
*dev
= hr_dev
->dev
;
696 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
697 hns_roce_table_mhop_put(hr_dev
, table
, obj
, 1);
701 i
= (obj
& (table
->num_obj
- 1)) /
702 (table
->table_chunk_size
/ table
->obj_size
);
704 mutex_lock(&table
->mutex
);
706 if (--table
->hem
[i
]->refcount
== 0) {
707 /* Clear HEM base address */
708 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 0))
709 dev_warn(dev
, "Clear HEM base address failed.\n");
711 hns_roce_free_hem(hr_dev
, table
->hem
[i
]);
712 table
->hem
[i
] = NULL
;
715 mutex_unlock(&table
->mutex
);
718 void *hns_roce_table_find(struct hns_roce_dev
*hr_dev
,
719 struct hns_roce_hem_table
*table
,
720 unsigned long obj
, dma_addr_t
*dma_handle
)
722 struct hns_roce_hem_chunk
*chunk
;
723 struct hns_roce_hem_mhop mhop
;
724 struct hns_roce_hem
*hem
;
726 unsigned long mhop_obj
= obj
;
727 unsigned long obj_per_chunk
;
728 unsigned long idx_offset
;
729 int offset
, dma_offset
;
737 mutex_lock(&table
->mutex
);
739 if (!hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
740 obj_per_chunk
= table
->table_chunk_size
/ table
->obj_size
;
741 hem
= table
->hem
[(obj
& (table
->num_obj
- 1)) / obj_per_chunk
];
742 idx_offset
= (obj
& (table
->num_obj
- 1)) % obj_per_chunk
;
743 dma_offset
= offset
= idx_offset
* table
->obj_size
;
745 hns_roce_calc_hem_mhop(hr_dev
, table
, &mhop_obj
, &mhop
);
749 if (mhop
.hop_num
== 2)
750 hem_idx
= i
* (mhop
.bt_chunk_size
/ 8) + j
;
751 else if (mhop
.hop_num
== 1 ||
752 mhop
.hop_num
== HNS_ROCE_HOP_NUM_0
)
755 hem
= table
->hem
[hem_idx
];
756 dma_offset
= offset
= (obj
& (table
->num_obj
- 1)) *
757 table
->obj_size
% mhop
.bt_chunk_size
;
758 if (mhop
.hop_num
== 2)
759 dma_offset
= offset
= 0;
765 list_for_each_entry(chunk
, &hem
->chunk_list
, list
) {
766 for (i
= 0; i
< chunk
->npages
; ++i
) {
767 length
= sg_dma_len(&chunk
->mem
[i
]);
768 if (dma_handle
&& dma_offset
>= 0) {
769 if (length
> (u32
)dma_offset
)
770 *dma_handle
= sg_dma_address(
771 &chunk
->mem
[i
]) + dma_offset
;
772 dma_offset
-= length
;
775 if (length
> (u32
)offset
) {
776 addr
= chunk
->buf
[i
] + offset
;
784 mutex_unlock(&table
->mutex
);
787 EXPORT_SYMBOL_GPL(hns_roce_table_find
);
789 int hns_roce_table_get_range(struct hns_roce_dev
*hr_dev
,
790 struct hns_roce_hem_table
*table
,
791 unsigned long start
, unsigned long end
)
793 struct hns_roce_hem_mhop mhop
;
794 unsigned long inc
= table
->table_chunk_size
/ table
->obj_size
;
798 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
799 hns_roce_calc_hem_mhop(hr_dev
, table
, NULL
, &mhop
);
800 inc
= mhop
.bt_chunk_size
/ table
->obj_size
;
803 /* Allocate MTT entry memory according to chunk(128K) */
804 for (i
= start
; i
<= end
; i
+= inc
) {
805 ret
= hns_roce_table_get(hr_dev
, table
, i
);
815 hns_roce_table_put(hr_dev
, table
, i
);
820 void hns_roce_table_put_range(struct hns_roce_dev
*hr_dev
,
821 struct hns_roce_hem_table
*table
,
822 unsigned long start
, unsigned long end
)
824 struct hns_roce_hem_mhop mhop
;
825 unsigned long inc
= table
->table_chunk_size
/ table
->obj_size
;
828 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
829 hns_roce_calc_hem_mhop(hr_dev
, table
, NULL
, &mhop
);
830 inc
= mhop
.bt_chunk_size
/ table
->obj_size
;
833 for (i
= start
; i
<= end
; i
+= inc
)
834 hns_roce_table_put(hr_dev
, table
, i
);
837 int hns_roce_init_hem_table(struct hns_roce_dev
*hr_dev
,
838 struct hns_roce_hem_table
*table
, u32 type
,
839 unsigned long obj_size
, unsigned long nobj
,
842 struct device
*dev
= hr_dev
->dev
;
843 unsigned long obj_per_chunk
;
844 unsigned long num_hem
;
846 if (!hns_roce_check_whether_mhop(hr_dev
, type
)) {
847 table
->table_chunk_size
= hr_dev
->caps
.chunk_sz
;
848 obj_per_chunk
= table
->table_chunk_size
/ obj_size
;
849 num_hem
= (nobj
+ obj_per_chunk
- 1) / obj_per_chunk
;
851 table
->hem
= kcalloc(num_hem
, sizeof(*table
->hem
), GFP_KERNEL
);
855 unsigned long buf_chunk_size
;
856 unsigned long bt_chunk_size
;
857 unsigned long bt_chunk_num
;
858 unsigned long num_bt_l0
= 0;
863 buf_chunk_size
= 1 << (hr_dev
->caps
.qpc_buf_pg_sz
865 bt_chunk_size
= 1 << (hr_dev
->caps
.qpc_ba_pg_sz
867 num_bt_l0
= hr_dev
->caps
.qpc_bt_num
;
868 hop_num
= hr_dev
->caps
.qpc_hop_num
;
871 buf_chunk_size
= 1 << (hr_dev
->caps
.mpt_buf_pg_sz
873 bt_chunk_size
= 1 << (hr_dev
->caps
.mpt_ba_pg_sz
875 num_bt_l0
= hr_dev
->caps
.mpt_bt_num
;
876 hop_num
= hr_dev
->caps
.mpt_hop_num
;
879 buf_chunk_size
= 1 << (hr_dev
->caps
.cqc_buf_pg_sz
881 bt_chunk_size
= 1 << (hr_dev
->caps
.cqc_ba_pg_sz
883 num_bt_l0
= hr_dev
->caps
.cqc_bt_num
;
884 hop_num
= hr_dev
->caps
.cqc_hop_num
;
887 buf_chunk_size
= 1 << (hr_dev
->caps
.srqc_buf_pg_sz
889 bt_chunk_size
= 1 << (hr_dev
->caps
.srqc_ba_pg_sz
891 num_bt_l0
= hr_dev
->caps
.srqc_bt_num
;
892 hop_num
= hr_dev
->caps
.srqc_hop_num
;
895 buf_chunk_size
= 1 << (hr_dev
->caps
.mtt_ba_pg_sz
897 bt_chunk_size
= buf_chunk_size
;
898 hop_num
= hr_dev
->caps
.mtt_hop_num
;
901 buf_chunk_size
= 1 << (hr_dev
->caps
.cqe_ba_pg_sz
903 bt_chunk_size
= buf_chunk_size
;
904 hop_num
= hr_dev
->caps
.cqe_hop_num
;
908 "Table %d not support to init hem table here!\n",
912 obj_per_chunk
= buf_chunk_size
/ obj_size
;
913 num_hem
= (nobj
+ obj_per_chunk
- 1) / obj_per_chunk
;
914 bt_chunk_num
= bt_chunk_size
/ 8;
915 if (table
->type
>= HEM_TYPE_MTT
)
916 num_bt_l0
= bt_chunk_num
;
918 table
->hem
= kcalloc(num_hem
, sizeof(*table
->hem
),
921 goto err_kcalloc_hem_buf
;
923 if (check_whether_bt_num_3(table
->type
, hop_num
)) {
924 unsigned long num_bt_l1
;
926 num_bt_l1
= (num_hem
+ bt_chunk_num
- 1) /
928 table
->bt_l1
= kcalloc(num_bt_l1
,
929 sizeof(*table
->bt_l1
),
932 goto err_kcalloc_bt_l1
;
934 table
->bt_l1_dma_addr
= kcalloc(num_bt_l1
,
935 sizeof(*table
->bt_l1_dma_addr
),
938 if (!table
->bt_l1_dma_addr
)
939 goto err_kcalloc_l1_dma
;
942 if (check_whether_bt_num_2(table
->type
, hop_num
) ||
943 check_whether_bt_num_3(table
->type
, hop_num
)) {
944 table
->bt_l0
= kcalloc(num_bt_l0
, sizeof(*table
->bt_l0
),
947 goto err_kcalloc_bt_l0
;
949 table
->bt_l0_dma_addr
= kcalloc(num_bt_l0
,
950 sizeof(*table
->bt_l0_dma_addr
),
952 if (!table
->bt_l0_dma_addr
)
953 goto err_kcalloc_l0_dma
;
958 table
->num_hem
= num_hem
;
959 table
->num_obj
= nobj
;
960 table
->obj_size
= obj_size
;
961 table
->lowmem
= use_lowmem
;
962 mutex_init(&table
->mutex
);
971 kfree(table
->bt_l1_dma_addr
);
972 table
->bt_l1_dma_addr
= NULL
;
986 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev
*hr_dev
,
987 struct hns_roce_hem_table
*table
)
989 struct hns_roce_hem_mhop mhop
;
994 hns_roce_calc_hem_mhop(hr_dev
, table
, NULL
, &mhop
);
995 buf_chunk_size
= table
->type
< HEM_TYPE_MTT
? mhop
.buf_chunk_size
:
998 for (i
= 0; i
< table
->num_hem
; ++i
) {
999 obj
= i
* buf_chunk_size
/ table
->obj_size
;
1001 hns_roce_table_mhop_put(hr_dev
, table
, obj
, 0);
1006 kfree(table
->bt_l1
);
1007 table
->bt_l1
= NULL
;
1008 kfree(table
->bt_l1_dma_addr
);
1009 table
->bt_l1_dma_addr
= NULL
;
1010 kfree(table
->bt_l0
);
1011 table
->bt_l0
= NULL
;
1012 kfree(table
->bt_l0_dma_addr
);
1013 table
->bt_l0_dma_addr
= NULL
;
1016 void hns_roce_cleanup_hem_table(struct hns_roce_dev
*hr_dev
,
1017 struct hns_roce_hem_table
*table
)
1019 struct device
*dev
= hr_dev
->dev
;
1022 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
1023 hns_roce_cleanup_mhop_hem_table(hr_dev
, table
);
1027 for (i
= 0; i
< table
->num_hem
; ++i
)
1028 if (table
->hem
[i
]) {
1029 if (hr_dev
->hw
->clear_hem(hr_dev
, table
,
1030 i
* table
->table_chunk_size
/ table
->obj_size
, 0))
1031 dev_err(dev
, "Clear HEM base address failed.\n");
1033 hns_roce_free_hem(hr_dev
, table
->hem
[i
]);
1039 void hns_roce_cleanup_hem(struct hns_roce_dev
*hr_dev
)
1041 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->cq_table
.table
);
1042 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->qp_table
.irrl_table
);
1043 if (hr_dev
->caps
.trrl_entry_sz
)
1044 hns_roce_cleanup_hem_table(hr_dev
,
1045 &hr_dev
->qp_table
.trrl_table
);
1046 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->qp_table
.qp_table
);
1047 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->mr_table
.mtpt_table
);
1048 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->mr_table
.mtt_table
);
1049 if (hns_roce_check_whether_mhop(hr_dev
, HEM_TYPE_CQE
))
1050 hns_roce_cleanup_hem_table(hr_dev
,
1051 &hr_dev
->mr_table
.mtt_cqe_table
);