2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/platform_device.h>
35 #include "hns_roce_device.h"
36 #include "hns_roce_hem.h"
37 #include "hns_roce_common.h"
39 #define DMA_ADDR_T_SHIFT 12
40 #define BT_BA_SHIFT 32
42 bool hns_roce_check_whether_mhop(struct hns_roce_dev
*hr_dev
, u32 type
)
44 if ((hr_dev
->caps
.qpc_hop_num
&& type
== HEM_TYPE_QPC
) ||
45 (hr_dev
->caps
.mpt_hop_num
&& type
== HEM_TYPE_MTPT
) ||
46 (hr_dev
->caps
.cqc_hop_num
&& type
== HEM_TYPE_CQC
) ||
47 (hr_dev
->caps
.srqc_hop_num
&& type
== HEM_TYPE_SRQC
) ||
48 (hr_dev
->caps
.cqe_hop_num
&& type
== HEM_TYPE_CQE
) ||
49 (hr_dev
->caps
.mtt_hop_num
&& type
== HEM_TYPE_MTT
))
54 EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop
);
56 static bool hns_roce_check_hem_null(struct hns_roce_hem
**hem
, u64 start_idx
,
61 for (i
= 0; i
< bt_chunk_num
; i
++)
62 if (hem
[start_idx
+ i
])
68 static bool hns_roce_check_bt_null(u64
**bt
, u64 start_idx
, u32 bt_chunk_num
)
72 for (i
= 0; i
< bt_chunk_num
; i
++)
73 if (bt
[start_idx
+ i
])
79 static int hns_roce_get_bt_num(u32 table_type
, u32 hop_num
)
81 if (check_whether_bt_num_3(table_type
, hop_num
))
83 else if (check_whether_bt_num_2(table_type
, hop_num
))
85 else if (check_whether_bt_num_1(table_type
, hop_num
))
91 int hns_roce_calc_hem_mhop(struct hns_roce_dev
*hr_dev
,
92 struct hns_roce_hem_table
*table
, unsigned long *obj
,
93 struct hns_roce_hem_mhop
*mhop
)
95 struct device
*dev
= hr_dev
->dev
;
101 switch (table
->type
) {
103 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.qpc_buf_pg_sz
105 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.qpc_ba_pg_sz
107 mhop
->ba_l0_num
= hr_dev
->caps
.qpc_bt_num
;
108 mhop
->hop_num
= hr_dev
->caps
.qpc_hop_num
;
111 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.mpt_buf_pg_sz
113 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.mpt_ba_pg_sz
115 mhop
->ba_l0_num
= hr_dev
->caps
.mpt_bt_num
;
116 mhop
->hop_num
= hr_dev
->caps
.mpt_hop_num
;
119 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.cqc_buf_pg_sz
121 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.cqc_ba_pg_sz
123 mhop
->ba_l0_num
= hr_dev
->caps
.cqc_bt_num
;
124 mhop
->hop_num
= hr_dev
->caps
.cqc_hop_num
;
127 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.srqc_buf_pg_sz
129 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.srqc_ba_pg_sz
131 mhop
->ba_l0_num
= hr_dev
->caps
.srqc_bt_num
;
132 mhop
->hop_num
= hr_dev
->caps
.srqc_hop_num
;
135 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.mtt_buf_pg_sz
137 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.mtt_ba_pg_sz
139 mhop
->ba_l0_num
= mhop
->bt_chunk_size
/ 8;
140 mhop
->hop_num
= hr_dev
->caps
.mtt_hop_num
;
143 mhop
->buf_chunk_size
= 1 << (hr_dev
->caps
.cqe_buf_pg_sz
145 mhop
->bt_chunk_size
= 1 << (hr_dev
->caps
.cqe_ba_pg_sz
147 mhop
->ba_l0_num
= mhop
->bt_chunk_size
/ 8;
148 mhop
->hop_num
= hr_dev
->caps
.cqe_hop_num
;
151 dev_err(dev
, "Table %d not support multi-hop addressing!\n",
160 * QPC/MTPT/CQC/SRQC alloc hem for buffer pages.
161 * MTT/CQE alloc hem for bt pages.
163 bt_num
= hns_roce_get_bt_num(table
->type
, mhop
->hop_num
);
164 chunk_ba_num
= mhop
->bt_chunk_size
/ 8;
165 chunk_size
= table
->type
< HEM_TYPE_MTT
? mhop
->buf_chunk_size
:
167 table_idx
= (*obj
& (table
->num_obj
- 1)) /
168 (chunk_size
/ table
->obj_size
);
171 mhop
->l2_idx
= table_idx
& (chunk_ba_num
- 1);
172 mhop
->l1_idx
= table_idx
/ chunk_ba_num
& (chunk_ba_num
- 1);
173 mhop
->l0_idx
= table_idx
/ chunk_ba_num
/ chunk_ba_num
;
176 mhop
->l1_idx
= table_idx
& (chunk_ba_num
- 1);
177 mhop
->l0_idx
= table_idx
/ chunk_ba_num
;
180 mhop
->l0_idx
= table_idx
;
183 dev_err(dev
, "Table %d not support hop_num = %d!\n",
184 table
->type
, mhop
->hop_num
);
187 if (mhop
->l0_idx
>= mhop
->ba_l0_num
)
188 mhop
->l0_idx
%= mhop
->ba_l0_num
;
192 EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop
);
194 static struct hns_roce_hem
*hns_roce_alloc_hem(struct hns_roce_dev
*hr_dev
,
196 unsigned long hem_alloc_size
,
199 struct hns_roce_hem_chunk
*chunk
= NULL
;
200 struct hns_roce_hem
*hem
;
201 struct scatterlist
*mem
;
205 WARN_ON(gfp_mask
& __GFP_HIGHMEM
);
207 hem
= kmalloc(sizeof(*hem
),
208 gfp_mask
& ~(__GFP_HIGHMEM
| __GFP_NOWARN
));
213 INIT_LIST_HEAD(&hem
->chunk_list
);
215 order
= get_order(hem_alloc_size
);
219 chunk
= kmalloc(sizeof(*chunk
),
220 gfp_mask
& ~(__GFP_HIGHMEM
| __GFP_NOWARN
));
224 sg_init_table(chunk
->mem
, HNS_ROCE_HEM_CHUNK_LEN
);
227 list_add_tail(&chunk
->list
, &hem
->chunk_list
);
230 while (1 << order
> npages
)
234 * Alloc memory one time. If failed, don't alloc small block
235 * memory, directly return fail.
237 mem
= &chunk
->mem
[chunk
->npages
];
238 buf
= dma_alloc_coherent(hr_dev
->dev
, PAGE_SIZE
<< order
,
239 &sg_dma_address(mem
), gfp_mask
);
243 sg_set_buf(mem
, buf
, PAGE_SIZE
<< order
);
244 WARN_ON(mem
->offset
);
245 sg_dma_len(mem
) = PAGE_SIZE
<< order
;
249 npages
-= 1 << order
;
255 hns_roce_free_hem(hr_dev
, hem
);
259 void hns_roce_free_hem(struct hns_roce_dev
*hr_dev
, struct hns_roce_hem
*hem
)
261 struct hns_roce_hem_chunk
*chunk
, *tmp
;
267 list_for_each_entry_safe(chunk
, tmp
, &hem
->chunk_list
, list
) {
268 for (i
= 0; i
< chunk
->npages
; ++i
)
269 dma_free_coherent(hr_dev
->dev
,
270 chunk
->mem
[i
].length
,
271 lowmem_page_address(sg_page(&chunk
->mem
[i
])),
272 sg_dma_address(&chunk
->mem
[i
]));
279 static int hns_roce_set_hem(struct hns_roce_dev
*hr_dev
,
280 struct hns_roce_hem_table
*table
, unsigned long obj
)
282 spinlock_t
*lock
= &hr_dev
->bt_cmd_lock
;
283 struct device
*dev
= hr_dev
->dev
;
284 unsigned long end
= 0;
286 struct hns_roce_hem_iter iter
;
287 void __iomem
*bt_cmd
;
288 u32 bt_cmd_h_val
= 0;
294 /* Find the HEM(Hardware Entry Memory) entry */
295 unsigned long i
= (obj
& (table
->num_obj
- 1)) /
296 (table
->table_chunk_size
/ table
->obj_size
);
298 switch (table
->type
) {
300 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
301 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_QPC
);
304 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
305 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
,
309 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
310 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
, HEM_TYPE_CQC
);
313 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M
,
314 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S
,
320 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M
,
321 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S
, obj
);
322 roce_set_bit(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S
, 0);
323 roce_set_bit(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S
, 1);
325 /* Currently iter only a chunk */
326 for (hns_roce_hem_first(table
->hem
[i
], &iter
);
327 !hns_roce_hem_last(&iter
); hns_roce_hem_next(&iter
)) {
328 bt_ba
= hns_roce_hem_addr(&iter
) >> DMA_ADDR_T_SHIFT
;
330 spin_lock_irqsave(lock
, flags
);
332 bt_cmd
= hr_dev
->reg_base
+ ROCEE_BT_CMD_H_REG
;
334 end
= msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS
) + jiffies
;
336 if (readl(bt_cmd
) >> BT_CMD_SYNC_SHIFT
) {
337 if (!(time_before(jiffies
, end
))) {
338 dev_err(dev
, "Write bt_cmd err,hw_sync is not zero.\n");
339 spin_unlock_irqrestore(lock
, flags
);
345 msleep(HW_SYNC_SLEEP_TIME_INTERVAL
);
348 bt_cmd_l
= (u32
)bt_ba
;
349 roce_set_field(bt_cmd_h_val
, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M
,
350 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S
,
351 bt_ba
>> BT_BA_SHIFT
);
353 bt_cmd_val
[0] = bt_cmd_l
;
354 bt_cmd_val
[1] = bt_cmd_h_val
;
355 hns_roce_write64_k(bt_cmd_val
,
356 hr_dev
->reg_base
+ ROCEE_BT_CMD_L_REG
);
357 spin_unlock_irqrestore(lock
, flags
);
363 static int hns_roce_table_mhop_get(struct hns_roce_dev
*hr_dev
,
364 struct hns_roce_hem_table
*table
,
367 struct device
*dev
= hr_dev
->dev
;
368 struct hns_roce_hem_mhop mhop
;
369 struct hns_roce_hem_iter iter
;
380 unsigned long mhop_obj
= obj
;
381 int bt_l1_allocated
= 0;
382 int bt_l0_allocated
= 0;
386 ret
= hns_roce_calc_hem_mhop(hr_dev
, table
, &mhop_obj
, &mhop
);
390 buf_chunk_size
= mhop
.buf_chunk_size
;
391 bt_chunk_size
= mhop
.bt_chunk_size
;
392 hop_num
= mhop
.hop_num
;
393 chunk_ba_num
= bt_chunk_size
/ 8;
395 bt_num
= hns_roce_get_bt_num(table
->type
, hop_num
);
398 hem_idx
= mhop
.l0_idx
* chunk_ba_num
* chunk_ba_num
+
399 mhop
.l1_idx
* chunk_ba_num
+ mhop
.l2_idx
;
400 bt_l1_idx
= mhop
.l0_idx
* chunk_ba_num
+ mhop
.l1_idx
;
401 bt_l0_idx
= mhop
.l0_idx
;
404 hem_idx
= mhop
.l0_idx
* chunk_ba_num
+ mhop
.l1_idx
;
405 bt_l0_idx
= mhop
.l0_idx
;
408 hem_idx
= mhop
.l0_idx
;
411 dev_err(dev
, "Table %d not support hop_num = %d!\n",
412 table
->type
, hop_num
);
416 mutex_lock(&table
->mutex
);
418 if (table
->hem
[hem_idx
]) {
419 ++table
->hem
[hem_idx
]->refcount
;
423 /* alloc L1 BA's chunk */
424 if ((check_whether_bt_num_3(table
->type
, hop_num
) ||
425 check_whether_bt_num_2(table
->type
, hop_num
)) &&
426 !table
->bt_l0
[bt_l0_idx
]) {
427 table
->bt_l0
[bt_l0_idx
] = dma_alloc_coherent(dev
, bt_chunk_size
,
428 &(table
->bt_l0_dma_addr
[bt_l0_idx
]),
430 if (!table
->bt_l0
[bt_l0_idx
]) {
436 /* set base address to hardware */
437 if (table
->type
< HEM_TYPE_MTT
) {
439 if (hr_dev
->hw
->set_hem(hr_dev
, table
, obj
, step_idx
)) {
441 dev_err(dev
, "set HEM base address to HW failed!\n");
442 goto err_dma_alloc_l1
;
447 /* alloc L2 BA's chunk */
448 if (check_whether_bt_num_3(table
->type
, hop_num
) &&
449 !table
->bt_l1
[bt_l1_idx
]) {
450 table
->bt_l1
[bt_l1_idx
] = dma_alloc_coherent(dev
, bt_chunk_size
,
451 &(table
->bt_l1_dma_addr
[bt_l1_idx
]),
453 if (!table
->bt_l1
[bt_l1_idx
]) {
455 goto err_dma_alloc_l1
;
458 *(table
->bt_l0
[bt_l0_idx
] + mhop
.l1_idx
) =
459 table
->bt_l1_dma_addr
[bt_l1_idx
];
461 /* set base address to hardware */
463 if (hr_dev
->hw
->set_hem(hr_dev
, table
, obj
, step_idx
)) {
465 dev_err(dev
, "set HEM base address to HW failed!\n");
466 goto err_alloc_hem_buf
;
471 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC.
472 * alloc bt space chunk for MTT/CQE.
474 size
= table
->type
< HEM_TYPE_MTT
? buf_chunk_size
: bt_chunk_size
;
475 table
->hem
[hem_idx
] = hns_roce_alloc_hem(hr_dev
,
478 (table
->lowmem
? GFP_KERNEL
:
479 GFP_HIGHUSER
) | __GFP_NOWARN
);
480 if (!table
->hem
[hem_idx
]) {
482 goto err_alloc_hem_buf
;
485 hns_roce_hem_first(table
->hem
[hem_idx
], &iter
);
486 bt_ba
= hns_roce_hem_addr(&iter
);
488 if (table
->type
< HEM_TYPE_MTT
) {
490 *(table
->bt_l1
[bt_l1_idx
] + mhop
.l2_idx
) = bt_ba
;
492 } else if (hop_num
== 1) {
493 *(table
->bt_l0
[bt_l0_idx
] + mhop
.l1_idx
) = bt_ba
;
495 } else if (hop_num
== HNS_ROCE_HOP_NUM_0
) {
499 /* set HEM base address to hardware */
500 if (hr_dev
->hw
->set_hem(hr_dev
, table
, obj
, step_idx
)) {
502 dev_err(dev
, "set HEM base address to HW failed!\n");
503 goto err_alloc_hem_buf
;
505 } else if (hop_num
== 2) {
506 *(table
->bt_l0
[bt_l0_idx
] + mhop
.l1_idx
) = bt_ba
;
509 ++table
->hem
[hem_idx
]->refcount
;
513 if (bt_l1_allocated
) {
514 dma_free_coherent(dev
, bt_chunk_size
, table
->bt_l1
[bt_l1_idx
],
515 table
->bt_l1_dma_addr
[bt_l1_idx
]);
516 table
->bt_l1
[bt_l1_idx
] = NULL
;
520 if (bt_l0_allocated
) {
521 dma_free_coherent(dev
, bt_chunk_size
, table
->bt_l0
[bt_l0_idx
],
522 table
->bt_l0_dma_addr
[bt_l0_idx
]);
523 table
->bt_l0
[bt_l0_idx
] = NULL
;
527 mutex_unlock(&table
->mutex
);
531 int hns_roce_table_get(struct hns_roce_dev
*hr_dev
,
532 struct hns_roce_hem_table
*table
, unsigned long obj
)
534 struct device
*dev
= hr_dev
->dev
;
538 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
))
539 return hns_roce_table_mhop_get(hr_dev
, table
, obj
);
541 i
= (obj
& (table
->num_obj
- 1)) / (table
->table_chunk_size
/
544 mutex_lock(&table
->mutex
);
547 ++table
->hem
[i
]->refcount
;
551 table
->hem
[i
] = hns_roce_alloc_hem(hr_dev
,
552 table
->table_chunk_size
>> PAGE_SHIFT
,
553 table
->table_chunk_size
,
554 (table
->lowmem
? GFP_KERNEL
:
555 GFP_HIGHUSER
) | __GFP_NOWARN
);
556 if (!table
->hem
[i
]) {
561 /* Set HEM base address(128K/page, pa) to Hardware */
562 if (hns_roce_set_hem(hr_dev
, table
, obj
)) {
563 hns_roce_free_hem(hr_dev
, table
->hem
[i
]);
564 table
->hem
[i
] = NULL
;
566 dev_err(dev
, "set HEM base address to HW failed.\n");
570 ++table
->hem
[i
]->refcount
;
572 mutex_unlock(&table
->mutex
);
576 static void hns_roce_table_mhop_put(struct hns_roce_dev
*hr_dev
,
577 struct hns_roce_hem_table
*table
,
581 struct device
*dev
= hr_dev
->dev
;
582 struct hns_roce_hem_mhop mhop
;
583 unsigned long mhop_obj
= obj
;
593 ret
= hns_roce_calc_hem_mhop(hr_dev
, table
, &mhop_obj
, &mhop
);
597 bt_chunk_size
= mhop
.bt_chunk_size
;
598 hop_num
= mhop
.hop_num
;
599 chunk_ba_num
= bt_chunk_size
/ 8;
601 bt_num
= hns_roce_get_bt_num(table
->type
, hop_num
);
604 hem_idx
= mhop
.l0_idx
* chunk_ba_num
* chunk_ba_num
+
605 mhop
.l1_idx
* chunk_ba_num
+ mhop
.l2_idx
;
606 bt_l1_idx
= mhop
.l0_idx
* chunk_ba_num
+ mhop
.l1_idx
;
609 hem_idx
= mhop
.l0_idx
* chunk_ba_num
+ mhop
.l1_idx
;
612 hem_idx
= mhop
.l0_idx
;
615 dev_err(dev
, "Table %d not support hop_num = %d!\n",
616 table
->type
, hop_num
);
620 mutex_lock(&table
->mutex
);
622 if (check_refcount
&& (--table
->hem
[hem_idx
]->refcount
> 0)) {
623 mutex_unlock(&table
->mutex
);
627 if (table
->type
< HEM_TYPE_MTT
&& hop_num
== 1) {
628 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 1))
629 dev_warn(dev
, "Clear HEM base address failed.\n");
630 } else if (table
->type
< HEM_TYPE_MTT
&& hop_num
== 2) {
631 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 2))
632 dev_warn(dev
, "Clear HEM base address failed.\n");
633 } else if (table
->type
< HEM_TYPE_MTT
&&
634 hop_num
== HNS_ROCE_HOP_NUM_0
) {
635 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 0))
636 dev_warn(dev
, "Clear HEM base address failed.\n");
640 * free buffer space chunk for QPC/MTPT/CQC/SRQC.
641 * free bt space chunk for MTT/CQE.
643 hns_roce_free_hem(hr_dev
, table
->hem
[hem_idx
]);
644 table
->hem
[hem_idx
] = NULL
;
646 if (check_whether_bt_num_2(table
->type
, hop_num
)) {
647 start_idx
= mhop
.l0_idx
* chunk_ba_num
;
648 if (hns_roce_check_hem_null(table
->hem
, start_idx
,
650 if (table
->type
< HEM_TYPE_MTT
&&
651 hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 0))
652 dev_warn(dev
, "Clear HEM base address failed.\n");
654 dma_free_coherent(dev
, bt_chunk_size
,
655 table
->bt_l0
[mhop
.l0_idx
],
656 table
->bt_l0_dma_addr
[mhop
.l0_idx
]);
657 table
->bt_l0
[mhop
.l0_idx
] = NULL
;
659 } else if (check_whether_bt_num_3(table
->type
, hop_num
)) {
660 start_idx
= mhop
.l0_idx
* chunk_ba_num
* chunk_ba_num
+
661 mhop
.l1_idx
* chunk_ba_num
;
662 if (hns_roce_check_hem_null(table
->hem
, start_idx
,
664 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 1))
665 dev_warn(dev
, "Clear HEM base address failed.\n");
667 dma_free_coherent(dev
, bt_chunk_size
,
668 table
->bt_l1
[bt_l1_idx
],
669 table
->bt_l1_dma_addr
[bt_l1_idx
]);
670 table
->bt_l1
[bt_l1_idx
] = NULL
;
672 start_idx
= mhop
.l0_idx
* chunk_ba_num
;
673 if (hns_roce_check_bt_null(table
->bt_l1
, start_idx
,
675 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
,
677 dev_warn(dev
, "Clear HEM base address failed.\n");
679 dma_free_coherent(dev
, bt_chunk_size
,
680 table
->bt_l0
[mhop
.l0_idx
],
681 table
->bt_l0_dma_addr
[mhop
.l0_idx
]);
682 table
->bt_l0
[mhop
.l0_idx
] = NULL
;
687 mutex_unlock(&table
->mutex
);
690 void hns_roce_table_put(struct hns_roce_dev
*hr_dev
,
691 struct hns_roce_hem_table
*table
, unsigned long obj
)
693 struct device
*dev
= hr_dev
->dev
;
696 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
697 hns_roce_table_mhop_put(hr_dev
, table
, obj
, 1);
701 i
= (obj
& (table
->num_obj
- 1)) /
702 (table
->table_chunk_size
/ table
->obj_size
);
704 mutex_lock(&table
->mutex
);
706 if (--table
->hem
[i
]->refcount
== 0) {
707 /* Clear HEM base address */
708 if (hr_dev
->hw
->clear_hem(hr_dev
, table
, obj
, 0))
709 dev_warn(dev
, "Clear HEM base address failed.\n");
711 hns_roce_free_hem(hr_dev
, table
->hem
[i
]);
712 table
->hem
[i
] = NULL
;
715 mutex_unlock(&table
->mutex
);
718 void *hns_roce_table_find(struct hns_roce_dev
*hr_dev
,
719 struct hns_roce_hem_table
*table
,
720 unsigned long obj
, dma_addr_t
*dma_handle
)
722 struct hns_roce_hem_chunk
*chunk
;
723 struct hns_roce_hem_mhop mhop
;
724 struct hns_roce_hem
*hem
;
725 struct page
*page
= NULL
;
726 unsigned long mhop_obj
= obj
;
727 unsigned long obj_per_chunk
;
728 unsigned long idx_offset
;
729 int offset
, dma_offset
;
736 mutex_lock(&table
->mutex
);
738 if (!hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
739 obj_per_chunk
= table
->table_chunk_size
/ table
->obj_size
;
740 hem
= table
->hem
[(obj
& (table
->num_obj
- 1)) / obj_per_chunk
];
741 idx_offset
= (obj
& (table
->num_obj
- 1)) % obj_per_chunk
;
742 dma_offset
= offset
= idx_offset
* table
->obj_size
;
744 hns_roce_calc_hem_mhop(hr_dev
, table
, &mhop_obj
, &mhop
);
748 if (mhop
.hop_num
== 2)
749 hem_idx
= i
* (mhop
.bt_chunk_size
/ 8) + j
;
750 else if (mhop
.hop_num
== 1 ||
751 mhop
.hop_num
== HNS_ROCE_HOP_NUM_0
)
754 hem
= table
->hem
[hem_idx
];
755 dma_offset
= offset
= (obj
& (table
->num_obj
- 1)) *
756 table
->obj_size
% mhop
.bt_chunk_size
;
757 if (mhop
.hop_num
== 2)
758 dma_offset
= offset
= 0;
764 list_for_each_entry(chunk
, &hem
->chunk_list
, list
) {
765 for (i
= 0; i
< chunk
->npages
; ++i
) {
766 if (dma_handle
&& dma_offset
>= 0) {
767 if (sg_dma_len(&chunk
->mem
[i
]) >
769 *dma_handle
= sg_dma_address(
770 &chunk
->mem
[i
]) + dma_offset
;
771 dma_offset
-= sg_dma_len(&chunk
->mem
[i
]);
774 if (chunk
->mem
[i
].length
> (u32
)offset
) {
775 page
= sg_page(&chunk
->mem
[i
]);
778 offset
-= chunk
->mem
[i
].length
;
783 mutex_unlock(&table
->mutex
);
784 return page
? lowmem_page_address(page
) + offset
: NULL
;
786 EXPORT_SYMBOL_GPL(hns_roce_table_find
);
788 int hns_roce_table_get_range(struct hns_roce_dev
*hr_dev
,
789 struct hns_roce_hem_table
*table
,
790 unsigned long start
, unsigned long end
)
792 struct hns_roce_hem_mhop mhop
;
793 unsigned long inc
= table
->table_chunk_size
/ table
->obj_size
;
797 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
798 hns_roce_calc_hem_mhop(hr_dev
, table
, NULL
, &mhop
);
799 inc
= mhop
.bt_chunk_size
/ table
->obj_size
;
802 /* Allocate MTT entry memory according to chunk(128K) */
803 for (i
= start
; i
<= end
; i
+= inc
) {
804 ret
= hns_roce_table_get(hr_dev
, table
, i
);
814 hns_roce_table_put(hr_dev
, table
, i
);
819 void hns_roce_table_put_range(struct hns_roce_dev
*hr_dev
,
820 struct hns_roce_hem_table
*table
,
821 unsigned long start
, unsigned long end
)
823 struct hns_roce_hem_mhop mhop
;
824 unsigned long inc
= table
->table_chunk_size
/ table
->obj_size
;
827 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
828 hns_roce_calc_hem_mhop(hr_dev
, table
, NULL
, &mhop
);
829 inc
= mhop
.bt_chunk_size
/ table
->obj_size
;
832 for (i
= start
; i
<= end
; i
+= inc
)
833 hns_roce_table_put(hr_dev
, table
, i
);
836 int hns_roce_init_hem_table(struct hns_roce_dev
*hr_dev
,
837 struct hns_roce_hem_table
*table
, u32 type
,
838 unsigned long obj_size
, unsigned long nobj
,
841 struct device
*dev
= hr_dev
->dev
;
842 unsigned long obj_per_chunk
;
843 unsigned long num_hem
;
845 if (!hns_roce_check_whether_mhop(hr_dev
, type
)) {
846 table
->table_chunk_size
= hr_dev
->caps
.chunk_sz
;
847 obj_per_chunk
= table
->table_chunk_size
/ obj_size
;
848 num_hem
= (nobj
+ obj_per_chunk
- 1) / obj_per_chunk
;
850 table
->hem
= kcalloc(num_hem
, sizeof(*table
->hem
), GFP_KERNEL
);
854 unsigned long buf_chunk_size
;
855 unsigned long bt_chunk_size
;
856 unsigned long bt_chunk_num
;
857 unsigned long num_bt_l0
= 0;
862 buf_chunk_size
= 1 << (hr_dev
->caps
.qpc_buf_pg_sz
864 bt_chunk_size
= 1 << (hr_dev
->caps
.qpc_ba_pg_sz
866 num_bt_l0
= hr_dev
->caps
.qpc_bt_num
;
867 hop_num
= hr_dev
->caps
.qpc_hop_num
;
870 buf_chunk_size
= 1 << (hr_dev
->caps
.mpt_buf_pg_sz
872 bt_chunk_size
= 1 << (hr_dev
->caps
.mpt_ba_pg_sz
874 num_bt_l0
= hr_dev
->caps
.mpt_bt_num
;
875 hop_num
= hr_dev
->caps
.mpt_hop_num
;
878 buf_chunk_size
= 1 << (hr_dev
->caps
.cqc_buf_pg_sz
880 bt_chunk_size
= 1 << (hr_dev
->caps
.cqc_ba_pg_sz
882 num_bt_l0
= hr_dev
->caps
.cqc_bt_num
;
883 hop_num
= hr_dev
->caps
.cqc_hop_num
;
886 buf_chunk_size
= 1 << (hr_dev
->caps
.srqc_buf_pg_sz
888 bt_chunk_size
= 1 << (hr_dev
->caps
.srqc_ba_pg_sz
890 num_bt_l0
= hr_dev
->caps
.srqc_bt_num
;
891 hop_num
= hr_dev
->caps
.srqc_hop_num
;
894 buf_chunk_size
= 1 << (hr_dev
->caps
.mtt_ba_pg_sz
896 bt_chunk_size
= buf_chunk_size
;
897 hop_num
= hr_dev
->caps
.mtt_hop_num
;
900 buf_chunk_size
= 1 << (hr_dev
->caps
.cqe_ba_pg_sz
902 bt_chunk_size
= buf_chunk_size
;
903 hop_num
= hr_dev
->caps
.cqe_hop_num
;
907 "Table %d not support to init hem table here!\n",
911 obj_per_chunk
= buf_chunk_size
/ obj_size
;
912 num_hem
= (nobj
+ obj_per_chunk
- 1) / obj_per_chunk
;
913 bt_chunk_num
= bt_chunk_size
/ 8;
914 if (table
->type
>= HEM_TYPE_MTT
)
915 num_bt_l0
= bt_chunk_num
;
917 table
->hem
= kcalloc(num_hem
, sizeof(*table
->hem
),
920 goto err_kcalloc_hem_buf
;
922 if (check_whether_bt_num_3(table
->type
, hop_num
)) {
923 unsigned long num_bt_l1
;
925 num_bt_l1
= (num_hem
+ bt_chunk_num
- 1) /
927 table
->bt_l1
= kcalloc(num_bt_l1
,
928 sizeof(*table
->bt_l1
),
931 goto err_kcalloc_bt_l1
;
933 table
->bt_l1_dma_addr
= kcalloc(num_bt_l1
,
934 sizeof(*table
->bt_l1_dma_addr
),
937 if (!table
->bt_l1_dma_addr
)
938 goto err_kcalloc_l1_dma
;
941 if (check_whether_bt_num_2(table
->type
, hop_num
) ||
942 check_whether_bt_num_3(table
->type
, hop_num
)) {
943 table
->bt_l0
= kcalloc(num_bt_l0
, sizeof(*table
->bt_l0
),
946 goto err_kcalloc_bt_l0
;
948 table
->bt_l0_dma_addr
= kcalloc(num_bt_l0
,
949 sizeof(*table
->bt_l0_dma_addr
),
951 if (!table
->bt_l0_dma_addr
)
952 goto err_kcalloc_l0_dma
;
957 table
->num_hem
= num_hem
;
958 table
->num_obj
= nobj
;
959 table
->obj_size
= obj_size
;
960 table
->lowmem
= use_lowmem
;
961 mutex_init(&table
->mutex
);
970 kfree(table
->bt_l1_dma_addr
);
971 table
->bt_l1_dma_addr
= NULL
;
985 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev
*hr_dev
,
986 struct hns_roce_hem_table
*table
)
988 struct hns_roce_hem_mhop mhop
;
993 hns_roce_calc_hem_mhop(hr_dev
, table
, NULL
, &mhop
);
994 buf_chunk_size
= table
->type
< HEM_TYPE_MTT
? mhop
.buf_chunk_size
:
997 for (i
= 0; i
< table
->num_hem
; ++i
) {
998 obj
= i
* buf_chunk_size
/ table
->obj_size
;
1000 hns_roce_table_mhop_put(hr_dev
, table
, obj
, 0);
1005 kfree(table
->bt_l1
);
1006 table
->bt_l1
= NULL
;
1007 kfree(table
->bt_l1_dma_addr
);
1008 table
->bt_l1_dma_addr
= NULL
;
1009 kfree(table
->bt_l0
);
1010 table
->bt_l0
= NULL
;
1011 kfree(table
->bt_l0_dma_addr
);
1012 table
->bt_l0_dma_addr
= NULL
;
1015 void hns_roce_cleanup_hem_table(struct hns_roce_dev
*hr_dev
,
1016 struct hns_roce_hem_table
*table
)
1018 struct device
*dev
= hr_dev
->dev
;
1021 if (hns_roce_check_whether_mhop(hr_dev
, table
->type
)) {
1022 hns_roce_cleanup_mhop_hem_table(hr_dev
, table
);
1026 for (i
= 0; i
< table
->num_hem
; ++i
)
1027 if (table
->hem
[i
]) {
1028 if (hr_dev
->hw
->clear_hem(hr_dev
, table
,
1029 i
* table
->table_chunk_size
/ table
->obj_size
, 0))
1030 dev_err(dev
, "Clear HEM base address failed.\n");
1032 hns_roce_free_hem(hr_dev
, table
->hem
[i
]);
1038 void hns_roce_cleanup_hem(struct hns_roce_dev
*hr_dev
)
1040 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->cq_table
.table
);
1041 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->qp_table
.irrl_table
);
1042 if (hr_dev
->caps
.trrl_entry_sz
)
1043 hns_roce_cleanup_hem_table(hr_dev
,
1044 &hr_dev
->qp_table
.trrl_table
);
1045 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->qp_table
.qp_table
);
1046 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->mr_table
.mtpt_table
);
1047 hns_roce_cleanup_hem_table(hr_dev
, &hr_dev
->mr_table
.mtt_table
);
1048 if (hns_roce_check_whether_mhop(hr_dev
, HEM_TYPE_CQE
))
1049 hns_roce_cleanup_hem_table(hr_dev
,
1050 &hr_dev
->mr_table
.mtt_cqe_table
);