]>
Commit | Line | Data |
---|---|---|
9a443537 | 1 | /* |
2 | * Copyright (c) 2016 Hisilicon Limited. | |
3 | * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. | |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | */ | |
33 | ||
34 | #include <linux/platform_device.h> | |
35 | #include "hns_roce_device.h" | |
36 | #include "hns_roce_hem.h" | |
37 | #include "hns_roce_common.h" | |
38 | ||
9a443537 | 39 | #define DMA_ADDR_T_SHIFT 12 |
9a443537 | 40 | #define BT_BA_SHIFT 32 |
41 | ||
a25d13cb SX |
42 | bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) |
43 | { | |
44 | if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) || | |
45 | (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) || | |
46 | (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) || | |
6a93c77a SX |
47 | (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) || |
48 | (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) || | |
49 | (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT)) | |
a25d13cb SX |
50 | return true; |
51 | ||
52 | return false; | |
53 | } | |
54 | EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop); | |
55 | ||
56 | static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx, | |
57 | u32 bt_chunk_num) | |
58 | { | |
59 | int i; | |
60 | ||
61 | for (i = 0; i < bt_chunk_num; i++) | |
62 | if (hem[start_idx + i]) | |
63 | return false; | |
64 | ||
65 | return true; | |
66 | } | |
67 | ||
68 | static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num) | |
69 | { | |
70 | int i; | |
71 | ||
72 | for (i = 0; i < bt_chunk_num; i++) | |
73 | if (bt[start_idx + i]) | |
74 | return false; | |
75 | ||
76 | return true; | |
77 | } | |
78 | ||
79 | static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) | |
80 | { | |
81 | if (check_whether_bt_num_3(table_type, hop_num)) | |
82 | return 3; | |
83 | else if (check_whether_bt_num_2(table_type, hop_num)) | |
84 | return 2; | |
85 | else if (check_whether_bt_num_1(table_type, hop_num)) | |
86 | return 1; | |
87 | else | |
88 | return 0; | |
89 | } | |
90 | ||
91 | int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, | |
92 | struct hns_roce_hem_table *table, unsigned long *obj, | |
93 | struct hns_roce_hem_mhop *mhop) | |
94 | { | |
95 | struct device *dev = hr_dev->dev; | |
96 | u32 chunk_ba_num; | |
97 | u32 table_idx; | |
98 | u32 bt_num; | |
99 | u32 chunk_size; | |
100 | ||
101 | switch (table->type) { | |
102 | case HEM_TYPE_QPC: | |
103 | mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz | |
104 | + PAGE_SHIFT); | |
105 | mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz | |
106 | + PAGE_SHIFT); | |
107 | mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; | |
108 | mhop->hop_num = hr_dev->caps.qpc_hop_num; | |
109 | break; | |
110 | case HEM_TYPE_MTPT: | |
111 | mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz | |
112 | + PAGE_SHIFT); | |
113 | mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz | |
114 | + PAGE_SHIFT); | |
115 | mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; | |
116 | mhop->hop_num = hr_dev->caps.mpt_hop_num; | |
117 | break; | |
118 | case HEM_TYPE_CQC: | |
119 | mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz | |
120 | + PAGE_SHIFT); | |
121 | mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz | |
122 | + PAGE_SHIFT); | |
123 | mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; | |
124 | mhop->hop_num = hr_dev->caps.cqc_hop_num; | |
125 | break; | |
126 | case HEM_TYPE_SRQC: | |
127 | mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz | |
128 | + PAGE_SHIFT); | |
129 | mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz | |
130 | + PAGE_SHIFT); | |
131 | mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; | |
132 | mhop->hop_num = hr_dev->caps.srqc_hop_num; | |
133 | break; | |
6a93c77a SX |
134 | case HEM_TYPE_MTT: |
135 | mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz | |
136 | + PAGE_SHIFT); | |
137 | mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz | |
138 | + PAGE_SHIFT); | |
139 | mhop->ba_l0_num = mhop->bt_chunk_size / 8; | |
140 | mhop->hop_num = hr_dev->caps.mtt_hop_num; | |
141 | break; | |
142 | case HEM_TYPE_CQE: | |
143 | mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz | |
144 | + PAGE_SHIFT); | |
145 | mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz | |
146 | + PAGE_SHIFT); | |
147 | mhop->ba_l0_num = mhop->bt_chunk_size / 8; | |
148 | mhop->hop_num = hr_dev->caps.cqe_hop_num; | |
149 | break; | |
a25d13cb SX |
150 | default: |
151 | dev_err(dev, "Table %d not support multi-hop addressing!\n", | |
152 | table->type); | |
153 | return -EINVAL; | |
154 | } | |
155 | ||
156 | if (!obj) | |
157 | return 0; | |
158 | ||
6a93c77a SX |
159 | /* |
160 | * QPC/MTPT/CQC/SRQC alloc hem for buffer pages. | |
161 | * MTT/CQE alloc hem for bt pages. | |
162 | */ | |
a25d13cb SX |
163 | bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); |
164 | chunk_ba_num = mhop->bt_chunk_size / 8; | |
6a93c77a SX |
165 | chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : |
166 | mhop->bt_chunk_size; | |
a25d13cb SX |
167 | table_idx = (*obj & (table->num_obj - 1)) / |
168 | (chunk_size / table->obj_size); | |
169 | switch (bt_num) { | |
170 | case 3: | |
171 | mhop->l2_idx = table_idx & (chunk_ba_num - 1); | |
172 | mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); | |
173 | mhop->l0_idx = table_idx / chunk_ba_num / chunk_ba_num; | |
174 | break; | |
175 | case 2: | |
176 | mhop->l1_idx = table_idx & (chunk_ba_num - 1); | |
177 | mhop->l0_idx = table_idx / chunk_ba_num; | |
178 | break; | |
179 | case 1: | |
180 | mhop->l0_idx = table_idx; | |
181 | break; | |
182 | default: | |
183 | dev_err(dev, "Table %d not support hop_num = %d!\n", | |
184 | table->type, mhop->hop_num); | |
185 | return -EINVAL; | |
186 | } | |
187 | if (mhop->l0_idx >= mhop->ba_l0_num) | |
188 | mhop->l0_idx %= mhop->ba_l0_num; | |
189 | ||
190 | return 0; | |
191 | } | |
192 | EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop); | |
193 | ||
194 | static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, | |
195 | int npages, | |
196 | unsigned long hem_alloc_size, | |
197 | gfp_t gfp_mask) | |
9a443537 | 198 | { |
199 | struct hns_roce_hem_chunk *chunk = NULL; | |
200 | struct hns_roce_hem *hem; | |
201 | struct scatterlist *mem; | |
202 | int order; | |
203 | void *buf; | |
204 | ||
205 | WARN_ON(gfp_mask & __GFP_HIGHMEM); | |
206 | ||
207 | hem = kmalloc(sizeof(*hem), | |
208 | gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); | |
209 | if (!hem) | |
210 | return NULL; | |
211 | ||
212 | hem->refcount = 0; | |
213 | INIT_LIST_HEAD(&hem->chunk_list); | |
214 | ||
a25d13cb | 215 | order = get_order(hem_alloc_size); |
9a443537 | 216 | |
217 | while (npages > 0) { | |
218 | if (!chunk) { | |
219 | chunk = kmalloc(sizeof(*chunk), | |
220 | gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); | |
221 | if (!chunk) | |
222 | goto fail; | |
223 | ||
224 | sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN); | |
225 | chunk->npages = 0; | |
226 | chunk->nsg = 0; | |
378efe79 | 227 | memset(chunk->buf, 0, sizeof(chunk->buf)); |
9a443537 | 228 | list_add_tail(&chunk->list, &hem->chunk_list); |
229 | } | |
230 | ||
231 | while (1 << order > npages) | |
232 | --order; | |
233 | ||
234 | /* | |
e84e40be S |
235 | * Alloc memory one time. If failed, don't alloc small block |
236 | * memory, directly return fail. | |
237 | */ | |
9a443537 | 238 | mem = &chunk->mem[chunk->npages]; |
13ca970e | 239 | buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, |
9a443537 | 240 | &sg_dma_address(mem), gfp_mask); |
241 | if (!buf) | |
242 | goto fail; | |
243 | ||
378efe79 | 244 | chunk->buf[chunk->npages] = buf; |
9a443537 | 245 | sg_dma_len(mem) = PAGE_SIZE << order; |
246 | ||
247 | ++chunk->npages; | |
248 | ++chunk->nsg; | |
249 | npages -= 1 << order; | |
250 | } | |
251 | ||
252 | return hem; | |
253 | ||
254 | fail: | |
255 | hns_roce_free_hem(hr_dev, hem); | |
256 | return NULL; | |
257 | } | |
258 | ||
259 | void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) | |
260 | { | |
261 | struct hns_roce_hem_chunk *chunk, *tmp; | |
262 | int i; | |
263 | ||
264 | if (!hem) | |
265 | return; | |
266 | ||
267 | list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { | |
268 | for (i = 0; i < chunk->npages; ++i) | |
13ca970e | 269 | dma_free_coherent(hr_dev->dev, |
378efe79 WHX |
270 | sg_dma_len(&chunk->mem[i]), |
271 | chunk->buf[i], | |
9a443537 | 272 | sg_dma_address(&chunk->mem[i])); |
273 | kfree(chunk); | |
274 | } | |
275 | ||
276 | kfree(hem); | |
277 | } | |
278 | ||
279 | static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, | |
280 | struct hns_roce_hem_table *table, unsigned long obj) | |
281 | { | |
9a443537 | 282 | spinlock_t *lock = &hr_dev->bt_cmd_lock; |
13ca970e | 283 | struct device *dev = hr_dev->dev; |
9a443537 | 284 | unsigned long end = 0; |
285 | unsigned long flags; | |
286 | struct hns_roce_hem_iter iter; | |
287 | void __iomem *bt_cmd; | |
288 | u32 bt_cmd_h_val = 0; | |
289 | u32 bt_cmd_val[2]; | |
290 | u32 bt_cmd_l = 0; | |
291 | u64 bt_ba = 0; | |
292 | int ret = 0; | |
293 | ||
294 | /* Find the HEM(Hardware Entry Memory) entry */ | |
295 | unsigned long i = (obj & (table->num_obj - 1)) / | |
29a1fe5d | 296 | (table->table_chunk_size / table->obj_size); |
9a443537 | 297 | |
298 | switch (table->type) { | |
299 | case HEM_TYPE_QPC: | |
300 | roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, | |
301 | ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC); | |
302 | break; | |
303 | case HEM_TYPE_MTPT: | |
304 | roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, | |
305 | ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, | |
306 | HEM_TYPE_MTPT); | |
307 | break; | |
308 | case HEM_TYPE_CQC: | |
309 | roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, | |
310 | ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC); | |
311 | break; | |
312 | case HEM_TYPE_SRQC: | |
313 | roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, | |
314 | ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, | |
315 | HEM_TYPE_SRQC); | |
316 | break; | |
317 | default: | |
318 | return ret; | |
319 | } | |
320 | roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, | |
321 | ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); | |
322 | roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); | |
323 | roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); | |
324 | ||
325 | /* Currently iter only a chunk */ | |
326 | for (hns_roce_hem_first(table->hem[i], &iter); | |
327 | !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { | |
328 | bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT; | |
329 | ||
330 | spin_lock_irqsave(lock, flags); | |
331 | ||
332 | bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; | |
333 | ||
334 | end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies; | |
335 | while (1) { | |
336 | if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { | |
337 | if (!(time_before(jiffies, end))) { | |
338 | dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); | |
339 | spin_unlock_irqrestore(lock, flags); | |
340 | return -EBUSY; | |
341 | } | |
342 | } else { | |
343 | break; | |
344 | } | |
345 | msleep(HW_SYNC_SLEEP_TIME_INTERVAL); | |
346 | } | |
347 | ||
348 | bt_cmd_l = (u32)bt_ba; | |
349 | roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, | |
350 | ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, | |
351 | bt_ba >> BT_BA_SHIFT); | |
352 | ||
353 | bt_cmd_val[0] = bt_cmd_l; | |
354 | bt_cmd_val[1] = bt_cmd_h_val; | |
355 | hns_roce_write64_k(bt_cmd_val, | |
356 | hr_dev->reg_base + ROCEE_BT_CMD_L_REG); | |
357 | spin_unlock_irqrestore(lock, flags); | |
358 | } | |
359 | ||
360 | return ret; | |
361 | } | |
362 | ||
281d0ccf CIK |
363 | static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, |
364 | struct hns_roce_hem_table *table, | |
365 | unsigned long obj) | |
a25d13cb SX |
366 | { |
367 | struct device *dev = hr_dev->dev; | |
368 | struct hns_roce_hem_mhop mhop; | |
369 | struct hns_roce_hem_iter iter; | |
370 | u32 buf_chunk_size; | |
371 | u32 bt_chunk_size; | |
372 | u32 chunk_ba_num; | |
373 | u32 hop_num; | |
374 | u32 size; | |
375 | u32 bt_num; | |
376 | u64 hem_idx; | |
377 | u64 bt_l1_idx = 0; | |
378 | u64 bt_l0_idx = 0; | |
379 | u64 bt_ba; | |
380 | unsigned long mhop_obj = obj; | |
381 | int bt_l1_allocated = 0; | |
382 | int bt_l0_allocated = 0; | |
383 | int step_idx; | |
384 | int ret; | |
385 | ||
386 | ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); | |
387 | if (ret) | |
388 | return ret; | |
389 | ||
390 | buf_chunk_size = mhop.buf_chunk_size; | |
391 | bt_chunk_size = mhop.bt_chunk_size; | |
392 | hop_num = mhop.hop_num; | |
393 | chunk_ba_num = bt_chunk_size / 8; | |
394 | ||
395 | bt_num = hns_roce_get_bt_num(table->type, hop_num); | |
396 | switch (bt_num) { | |
397 | case 3: | |
398 | hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + | |
399 | mhop.l1_idx * chunk_ba_num + mhop.l2_idx; | |
400 | bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; | |
401 | bt_l0_idx = mhop.l0_idx; | |
402 | break; | |
403 | case 2: | |
404 | hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; | |
405 | bt_l0_idx = mhop.l0_idx; | |
406 | break; | |
407 | case 1: | |
408 | hem_idx = mhop.l0_idx; | |
409 | break; | |
410 | default: | |
411 | dev_err(dev, "Table %d not support hop_num = %d!\n", | |
412 | table->type, hop_num); | |
413 | return -EINVAL; | |
414 | } | |
415 | ||
416 | mutex_lock(&table->mutex); | |
417 | ||
418 | if (table->hem[hem_idx]) { | |
419 | ++table->hem[hem_idx]->refcount; | |
420 | goto out; | |
421 | } | |
422 | ||
423 | /* alloc L1 BA's chunk */ | |
424 | if ((check_whether_bt_num_3(table->type, hop_num) || | |
425 | check_whether_bt_num_2(table->type, hop_num)) && | |
426 | !table->bt_l0[bt_l0_idx]) { | |
427 | table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size, | |
428 | &(table->bt_l0_dma_addr[bt_l0_idx]), | |
429 | GFP_KERNEL); | |
430 | if (!table->bt_l0[bt_l0_idx]) { | |
431 | ret = -ENOMEM; | |
432 | goto out; | |
433 | } | |
434 | bt_l0_allocated = 1; | |
435 | ||
436 | /* set base address to hardware */ | |
437 | if (table->type < HEM_TYPE_MTT) { | |
438 | step_idx = 0; | |
439 | if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { | |
440 | ret = -ENODEV; | |
441 | dev_err(dev, "set HEM base address to HW failed!\n"); | |
442 | goto err_dma_alloc_l1; | |
443 | } | |
444 | } | |
445 | } | |
446 | ||
447 | /* alloc L2 BA's chunk */ | |
448 | if (check_whether_bt_num_3(table->type, hop_num) && | |
449 | !table->bt_l1[bt_l1_idx]) { | |
450 | table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size, | |
451 | &(table->bt_l1_dma_addr[bt_l1_idx]), | |
452 | GFP_KERNEL); | |
453 | if (!table->bt_l1[bt_l1_idx]) { | |
454 | ret = -ENOMEM; | |
455 | goto err_dma_alloc_l1; | |
456 | } | |
457 | bt_l1_allocated = 1; | |
458 | *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = | |
459 | table->bt_l1_dma_addr[bt_l1_idx]; | |
460 | ||
461 | /* set base address to hardware */ | |
462 | step_idx = 1; | |
463 | if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { | |
464 | ret = -ENODEV; | |
465 | dev_err(dev, "set HEM base address to HW failed!\n"); | |
466 | goto err_alloc_hem_buf; | |
467 | } | |
468 | } | |
469 | ||
6a93c77a SX |
470 | /* |
471 | * alloc buffer space chunk for QPC/MTPT/CQC/SRQC. | |
472 | * alloc bt space chunk for MTT/CQE. | |
473 | */ | |
474 | size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size; | |
a25d13cb SX |
475 | table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev, |
476 | size >> PAGE_SHIFT, | |
477 | size, | |
478 | (table->lowmem ? GFP_KERNEL : | |
479 | GFP_HIGHUSER) | __GFP_NOWARN); | |
480 | if (!table->hem[hem_idx]) { | |
481 | ret = -ENOMEM; | |
482 | goto err_alloc_hem_buf; | |
483 | } | |
484 | ||
485 | hns_roce_hem_first(table->hem[hem_idx], &iter); | |
486 | bt_ba = hns_roce_hem_addr(&iter); | |
487 | ||
488 | if (table->type < HEM_TYPE_MTT) { | |
489 | if (hop_num == 2) { | |
490 | *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba; | |
491 | step_idx = 2; | |
492 | } else if (hop_num == 1) { | |
493 | *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; | |
494 | step_idx = 1; | |
495 | } else if (hop_num == HNS_ROCE_HOP_NUM_0) { | |
496 | step_idx = 0; | |
497 | } | |
498 | ||
499 | /* set HEM base address to hardware */ | |
500 | if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { | |
501 | ret = -ENODEV; | |
502 | dev_err(dev, "set HEM base address to HW failed!\n"); | |
503 | goto err_alloc_hem_buf; | |
504 | } | |
6a93c77a SX |
505 | } else if (hop_num == 2) { |
506 | *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; | |
a25d13cb SX |
507 | } |
508 | ||
509 | ++table->hem[hem_idx]->refcount; | |
510 | goto out; | |
511 | ||
512 | err_alloc_hem_buf: | |
513 | if (bt_l1_allocated) { | |
514 | dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx], | |
515 | table->bt_l1_dma_addr[bt_l1_idx]); | |
516 | table->bt_l1[bt_l1_idx] = NULL; | |
517 | } | |
518 | ||
519 | err_dma_alloc_l1: | |
520 | if (bt_l0_allocated) { | |
521 | dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx], | |
522 | table->bt_l0_dma_addr[bt_l0_idx]); | |
523 | table->bt_l0[bt_l0_idx] = NULL; | |
524 | } | |
525 | ||
526 | out: | |
527 | mutex_unlock(&table->mutex); | |
528 | return ret; | |
529 | } | |
530 | ||
9a443537 | 531 | int hns_roce_table_get(struct hns_roce_dev *hr_dev, |
532 | struct hns_roce_hem_table *table, unsigned long obj) | |
533 | { | |
13ca970e | 534 | struct device *dev = hr_dev->dev; |
9a443537 | 535 | int ret = 0; |
536 | unsigned long i; | |
537 | ||
a25d13cb SX |
538 | if (hns_roce_check_whether_mhop(hr_dev, table->type)) |
539 | return hns_roce_table_mhop_get(hr_dev, table, obj); | |
540 | ||
29a1fe5d | 541 | i = (obj & (table->num_obj - 1)) / (table->table_chunk_size / |
9a443537 | 542 | table->obj_size); |
543 | ||
544 | mutex_lock(&table->mutex); | |
545 | ||
546 | if (table->hem[i]) { | |
547 | ++table->hem[i]->refcount; | |
548 | goto out; | |
549 | } | |
550 | ||
551 | table->hem[i] = hns_roce_alloc_hem(hr_dev, | |
29a1fe5d WHX |
552 | table->table_chunk_size >> PAGE_SHIFT, |
553 | table->table_chunk_size, | |
9a443537 | 554 | (table->lowmem ? GFP_KERNEL : |
555 | GFP_HIGHUSER) | __GFP_NOWARN); | |
556 | if (!table->hem[i]) { | |
557 | ret = -ENOMEM; | |
558 | goto out; | |
559 | } | |
560 | ||
561 | /* Set HEM base address(128K/page, pa) to Hardware */ | |
562 | if (hns_roce_set_hem(hr_dev, table, obj)) { | |
08eb3018 WHX |
563 | hns_roce_free_hem(hr_dev, table->hem[i]); |
564 | table->hem[i] = NULL; | |
9a443537 | 565 | ret = -ENODEV; |
566 | dev_err(dev, "set HEM base address to HW failed.\n"); | |
567 | goto out; | |
568 | } | |
569 | ||
570 | ++table->hem[i]->refcount; | |
571 | out: | |
572 | mutex_unlock(&table->mutex); | |
573 | return ret; | |
574 | } | |
575 | ||
281d0ccf CIK |
576 | static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, |
577 | struct hns_roce_hem_table *table, | |
578 | unsigned long obj, | |
579 | int check_refcount) | |
a25d13cb SX |
580 | { |
581 | struct device *dev = hr_dev->dev; | |
582 | struct hns_roce_hem_mhop mhop; | |
583 | unsigned long mhop_obj = obj; | |
584 | u32 bt_chunk_size; | |
585 | u32 chunk_ba_num; | |
586 | u32 hop_num; | |
587 | u32 start_idx; | |
588 | u32 bt_num; | |
589 | u64 hem_idx; | |
590 | u64 bt_l1_idx = 0; | |
591 | int ret; | |
592 | ||
593 | ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); | |
594 | if (ret) | |
595 | return; | |
596 | ||
597 | bt_chunk_size = mhop.bt_chunk_size; | |
598 | hop_num = mhop.hop_num; | |
599 | chunk_ba_num = bt_chunk_size / 8; | |
600 | ||
601 | bt_num = hns_roce_get_bt_num(table->type, hop_num); | |
602 | switch (bt_num) { | |
603 | case 3: | |
604 | hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + | |
605 | mhop.l1_idx * chunk_ba_num + mhop.l2_idx; | |
606 | bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; | |
607 | break; | |
608 | case 2: | |
609 | hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; | |
610 | break; | |
611 | case 1: | |
612 | hem_idx = mhop.l0_idx; | |
613 | break; | |
614 | default: | |
615 | dev_err(dev, "Table %d not support hop_num = %d!\n", | |
616 | table->type, hop_num); | |
617 | return; | |
618 | } | |
619 | ||
620 | mutex_lock(&table->mutex); | |
621 | ||
622 | if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) { | |
623 | mutex_unlock(&table->mutex); | |
624 | return; | |
625 | } | |
626 | ||
627 | if (table->type < HEM_TYPE_MTT && hop_num == 1) { | |
628 | if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) | |
629 | dev_warn(dev, "Clear HEM base address failed.\n"); | |
630 | } else if (table->type < HEM_TYPE_MTT && hop_num == 2) { | |
631 | if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2)) | |
632 | dev_warn(dev, "Clear HEM base address failed.\n"); | |
633 | } else if (table->type < HEM_TYPE_MTT && | |
634 | hop_num == HNS_ROCE_HOP_NUM_0) { | |
635 | if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) | |
636 | dev_warn(dev, "Clear HEM base address failed.\n"); | |
637 | } | |
638 | ||
6a93c77a SX |
639 | /* |
640 | * free buffer space chunk for QPC/MTPT/CQC/SRQC. | |
641 | * free bt space chunk for MTT/CQE. | |
642 | */ | |
a25d13cb SX |
643 | hns_roce_free_hem(hr_dev, table->hem[hem_idx]); |
644 | table->hem[hem_idx] = NULL; | |
645 | ||
646 | if (check_whether_bt_num_2(table->type, hop_num)) { | |
647 | start_idx = mhop.l0_idx * chunk_ba_num; | |
648 | if (hns_roce_check_hem_null(table->hem, start_idx, | |
649 | chunk_ba_num)) { | |
650 | if (table->type < HEM_TYPE_MTT && | |
651 | hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) | |
652 | dev_warn(dev, "Clear HEM base address failed.\n"); | |
653 | ||
654 | dma_free_coherent(dev, bt_chunk_size, | |
655 | table->bt_l0[mhop.l0_idx], | |
656 | table->bt_l0_dma_addr[mhop.l0_idx]); | |
657 | table->bt_l0[mhop.l0_idx] = NULL; | |
658 | } | |
659 | } else if (check_whether_bt_num_3(table->type, hop_num)) { | |
660 | start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + | |
661 | mhop.l1_idx * chunk_ba_num; | |
662 | if (hns_roce_check_hem_null(table->hem, start_idx, | |
663 | chunk_ba_num)) { | |
664 | if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) | |
665 | dev_warn(dev, "Clear HEM base address failed.\n"); | |
666 | ||
667 | dma_free_coherent(dev, bt_chunk_size, | |
668 | table->bt_l1[bt_l1_idx], | |
669 | table->bt_l1_dma_addr[bt_l1_idx]); | |
670 | table->bt_l1[bt_l1_idx] = NULL; | |
671 | ||
672 | start_idx = mhop.l0_idx * chunk_ba_num; | |
673 | if (hns_roce_check_bt_null(table->bt_l1, start_idx, | |
674 | chunk_ba_num)) { | |
675 | if (hr_dev->hw->clear_hem(hr_dev, table, obj, | |
676 | 0)) | |
677 | dev_warn(dev, "Clear HEM base address failed.\n"); | |
678 | ||
679 | dma_free_coherent(dev, bt_chunk_size, | |
680 | table->bt_l0[mhop.l0_idx], | |
681 | table->bt_l0_dma_addr[mhop.l0_idx]); | |
682 | table->bt_l0[mhop.l0_idx] = NULL; | |
683 | } | |
684 | } | |
685 | } | |
686 | ||
687 | mutex_unlock(&table->mutex); | |
688 | } | |
689 | ||
9a443537 | 690 | void hns_roce_table_put(struct hns_roce_dev *hr_dev, |
691 | struct hns_roce_hem_table *table, unsigned long obj) | |
692 | { | |
13ca970e | 693 | struct device *dev = hr_dev->dev; |
9a443537 | 694 | unsigned long i; |
695 | ||
a25d13cb SX |
696 | if (hns_roce_check_whether_mhop(hr_dev, table->type)) { |
697 | hns_roce_table_mhop_put(hr_dev, table, obj, 1); | |
698 | return; | |
699 | } | |
700 | ||
9a443537 | 701 | i = (obj & (table->num_obj - 1)) / |
29a1fe5d | 702 | (table->table_chunk_size / table->obj_size); |
9a443537 | 703 | |
704 | mutex_lock(&table->mutex); | |
705 | ||
706 | if (--table->hem[i]->refcount == 0) { | |
707 | /* Clear HEM base address */ | |
a25d13cb | 708 | if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) |
9a443537 | 709 | dev_warn(dev, "Clear HEM base address failed.\n"); |
710 | ||
711 | hns_roce_free_hem(hr_dev, table->hem[i]); | |
712 | table->hem[i] = NULL; | |
713 | } | |
714 | ||
715 | mutex_unlock(&table->mutex); | |
716 | } | |
717 | ||
6a93c77a SX |
718 | void *hns_roce_table_find(struct hns_roce_dev *hr_dev, |
719 | struct hns_roce_hem_table *table, | |
720 | unsigned long obj, dma_addr_t *dma_handle) | |
9a443537 | 721 | { |
722 | struct hns_roce_hem_chunk *chunk; | |
6a93c77a | 723 | struct hns_roce_hem_mhop mhop; |
9a443537 | 724 | struct hns_roce_hem *hem; |
378efe79 | 725 | void *addr = NULL; |
6a93c77a | 726 | unsigned long mhop_obj = obj; |
0203b14c | 727 | unsigned long obj_per_chunk; |
728 | unsigned long idx_offset; | |
6a93c77a | 729 | int offset, dma_offset; |
378efe79 | 730 | int length; |
6a93c77a SX |
731 | int i, j; |
732 | u32 hem_idx = 0; | |
9a443537 | 733 | |
734 | if (!table->lowmem) | |
735 | return NULL; | |
736 | ||
737 | mutex_lock(&table->mutex); | |
6a93c77a SX |
738 | |
739 | if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { | |
0203b14c | 740 | obj_per_chunk = table->table_chunk_size / table->obj_size; |
741 | hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk]; | |
742 | idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk; | |
743 | dma_offset = offset = idx_offset * table->obj_size; | |
6a93c77a SX |
744 | } else { |
745 | hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); | |
746 | /* mtt mhop */ | |
747 | i = mhop.l0_idx; | |
748 | j = mhop.l1_idx; | |
749 | if (mhop.hop_num == 2) | |
750 | hem_idx = i * (mhop.bt_chunk_size / 8) + j; | |
751 | else if (mhop.hop_num == 1 || | |
752 | mhop.hop_num == HNS_ROCE_HOP_NUM_0) | |
753 | hem_idx = i; | |
754 | ||
755 | hem = table->hem[hem_idx]; | |
756 | dma_offset = offset = (obj & (table->num_obj - 1)) * | |
757 | table->obj_size % mhop.bt_chunk_size; | |
758 | if (mhop.hop_num == 2) | |
759 | dma_offset = offset = 0; | |
760 | } | |
9a443537 | 761 | |
762 | if (!hem) | |
763 | goto out; | |
764 | ||
765 | list_for_each_entry(chunk, &hem->chunk_list, list) { | |
766 | for (i = 0; i < chunk->npages; ++i) { | |
378efe79 | 767 | length = sg_dma_len(&chunk->mem[i]); |
9a443537 | 768 | if (dma_handle && dma_offset >= 0) { |
378efe79 | 769 | if (length > (u32)dma_offset) |
9a443537 | 770 | *dma_handle = sg_dma_address( |
771 | &chunk->mem[i]) + dma_offset; | |
378efe79 | 772 | dma_offset -= length; |
9a443537 | 773 | } |
774 | ||
378efe79 WHX |
775 | if (length > (u32)offset) { |
776 | addr = chunk->buf[i] + offset; | |
9a443537 | 777 | goto out; |
778 | } | |
378efe79 | 779 | offset -= length; |
9a443537 | 780 | } |
781 | } | |
782 | ||
783 | out: | |
784 | mutex_unlock(&table->mutex); | |
378efe79 | 785 | return addr; |
9a443537 | 786 | } |
08805fdb | 787 | EXPORT_SYMBOL_GPL(hns_roce_table_find); |
9a443537 | 788 | |
789 | int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, | |
790 | struct hns_roce_hem_table *table, | |
791 | unsigned long start, unsigned long end) | |
792 | { | |
6a93c77a | 793 | struct hns_roce_hem_mhop mhop; |
29a1fe5d | 794 | unsigned long inc = table->table_chunk_size / table->obj_size; |
6a93c77a SX |
795 | unsigned long i; |
796 | int ret; | |
797 | ||
798 | if (hns_roce_check_whether_mhop(hr_dev, table->type)) { | |
799 | hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); | |
800 | inc = mhop.bt_chunk_size / table->obj_size; | |
801 | } | |
9a443537 | 802 | |
803 | /* Allocate MTT entry memory according to chunk(128K) */ | |
804 | for (i = start; i <= end; i += inc) { | |
805 | ret = hns_roce_table_get(hr_dev, table, i); | |
806 | if (ret) | |
807 | goto fail; | |
808 | } | |
809 | ||
810 | return 0; | |
811 | ||
812 | fail: | |
813 | while (i > start) { | |
814 | i -= inc; | |
815 | hns_roce_table_put(hr_dev, table, i); | |
816 | } | |
817 | return ret; | |
818 | } | |
819 | ||
820 | void hns_roce_table_put_range(struct hns_roce_dev *hr_dev, | |
821 | struct hns_roce_hem_table *table, | |
822 | unsigned long start, unsigned long end) | |
823 | { | |
6a93c77a | 824 | struct hns_roce_hem_mhop mhop; |
29a1fe5d | 825 | unsigned long inc = table->table_chunk_size / table->obj_size; |
9a443537 | 826 | unsigned long i; |
827 | ||
6a93c77a SX |
828 | if (hns_roce_check_whether_mhop(hr_dev, table->type)) { |
829 | hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); | |
830 | inc = mhop.bt_chunk_size / table->obj_size; | |
831 | } | |
832 | ||
29a1fe5d | 833 | for (i = start; i <= end; i += inc) |
9a443537 | 834 | hns_roce_table_put(hr_dev, table, i); |
835 | } | |
836 | ||
837 | int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, | |
838 | struct hns_roce_hem_table *table, u32 type, | |
839 | unsigned long obj_size, unsigned long nobj, | |
840 | int use_lowmem) | |
841 | { | |
a25d13cb | 842 | struct device *dev = hr_dev->dev; |
9a443537 | 843 | unsigned long obj_per_chunk; |
844 | unsigned long num_hem; | |
845 | ||
a25d13cb | 846 | if (!hns_roce_check_whether_mhop(hr_dev, type)) { |
29a1fe5d WHX |
847 | table->table_chunk_size = hr_dev->caps.chunk_sz; |
848 | obj_per_chunk = table->table_chunk_size / obj_size; | |
a25d13cb SX |
849 | num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; |
850 | ||
851 | table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); | |
852 | if (!table->hem) | |
853 | return -ENOMEM; | |
854 | } else { | |
855 | unsigned long buf_chunk_size; | |
856 | unsigned long bt_chunk_size; | |
857 | unsigned long bt_chunk_num; | |
6a93c77a | 858 | unsigned long num_bt_l0 = 0; |
a25d13cb SX |
859 | u32 hop_num; |
860 | ||
861 | switch (type) { | |
862 | case HEM_TYPE_QPC: | |
863 | buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz | |
864 | + PAGE_SHIFT); | |
865 | bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz | |
866 | + PAGE_SHIFT); | |
867 | num_bt_l0 = hr_dev->caps.qpc_bt_num; | |
868 | hop_num = hr_dev->caps.qpc_hop_num; | |
869 | break; | |
870 | case HEM_TYPE_MTPT: | |
871 | buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz | |
872 | + PAGE_SHIFT); | |
873 | bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz | |
874 | + PAGE_SHIFT); | |
875 | num_bt_l0 = hr_dev->caps.mpt_bt_num; | |
876 | hop_num = hr_dev->caps.mpt_hop_num; | |
877 | break; | |
878 | case HEM_TYPE_CQC: | |
879 | buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz | |
880 | + PAGE_SHIFT); | |
881 | bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz | |
882 | + PAGE_SHIFT); | |
883 | num_bt_l0 = hr_dev->caps.cqc_bt_num; | |
884 | hop_num = hr_dev->caps.cqc_hop_num; | |
885 | break; | |
886 | case HEM_TYPE_SRQC: | |
887 | buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz | |
888 | + PAGE_SHIFT); | |
889 | bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz | |
890 | + PAGE_SHIFT); | |
891 | num_bt_l0 = hr_dev->caps.srqc_bt_num; | |
892 | hop_num = hr_dev->caps.srqc_hop_num; | |
893 | break; | |
6a93c77a SX |
894 | case HEM_TYPE_MTT: |
895 | buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz | |
896 | + PAGE_SHIFT); | |
897 | bt_chunk_size = buf_chunk_size; | |
898 | hop_num = hr_dev->caps.mtt_hop_num; | |
899 | break; | |
900 | case HEM_TYPE_CQE: | |
901 | buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz | |
902 | + PAGE_SHIFT); | |
903 | bt_chunk_size = buf_chunk_size; | |
904 | hop_num = hr_dev->caps.cqe_hop_num; | |
905 | break; | |
a25d13cb SX |
906 | default: |
907 | dev_err(dev, | |
908 | "Table %d not support to init hem table here!\n", | |
909 | type); | |
910 | return -EINVAL; | |
911 | } | |
912 | obj_per_chunk = buf_chunk_size / obj_size; | |
913 | num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; | |
914 | bt_chunk_num = bt_chunk_size / 8; | |
6a93c77a SX |
915 | if (table->type >= HEM_TYPE_MTT) |
916 | num_bt_l0 = bt_chunk_num; | |
a25d13cb SX |
917 | |
918 | table->hem = kcalloc(num_hem, sizeof(*table->hem), | |
919 | GFP_KERNEL); | |
920 | if (!table->hem) | |
921 | goto err_kcalloc_hem_buf; | |
922 | ||
923 | if (check_whether_bt_num_3(table->type, hop_num)) { | |
924 | unsigned long num_bt_l1; | |
925 | ||
926 | num_bt_l1 = (num_hem + bt_chunk_num - 1) / | |
927 | bt_chunk_num; | |
928 | table->bt_l1 = kcalloc(num_bt_l1, | |
929 | sizeof(*table->bt_l1), | |
930 | GFP_KERNEL); | |
931 | if (!table->bt_l1) | |
932 | goto err_kcalloc_bt_l1; | |
933 | ||
934 | table->bt_l1_dma_addr = kcalloc(num_bt_l1, | |
935 | sizeof(*table->bt_l1_dma_addr), | |
936 | GFP_KERNEL); | |
937 | ||
938 | if (!table->bt_l1_dma_addr) | |
939 | goto err_kcalloc_l1_dma; | |
940 | } | |
9a443537 | 941 | |
a25d13cb SX |
942 | if (check_whether_bt_num_2(table->type, hop_num) || |
943 | check_whether_bt_num_3(table->type, hop_num)) { | |
944 | table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), | |
945 | GFP_KERNEL); | |
946 | if (!table->bt_l0) | |
947 | goto err_kcalloc_bt_l0; | |
948 | ||
949 | table->bt_l0_dma_addr = kcalloc(num_bt_l0, | |
950 | sizeof(*table->bt_l0_dma_addr), | |
951 | GFP_KERNEL); | |
952 | if (!table->bt_l0_dma_addr) | |
953 | goto err_kcalloc_l0_dma; | |
954 | } | |
955 | } | |
9a443537 | 956 | |
957 | table->type = type; | |
958 | table->num_hem = num_hem; | |
959 | table->num_obj = nobj; | |
960 | table->obj_size = obj_size; | |
961 | table->lowmem = use_lowmem; | |
962 | mutex_init(&table->mutex); | |
963 | ||
964 | return 0; | |
a25d13cb SX |
965 | |
966 | err_kcalloc_l0_dma: | |
967 | kfree(table->bt_l0); | |
968 | table->bt_l0 = NULL; | |
969 | ||
970 | err_kcalloc_bt_l0: | |
971 | kfree(table->bt_l1_dma_addr); | |
972 | table->bt_l1_dma_addr = NULL; | |
973 | ||
974 | err_kcalloc_l1_dma: | |
975 | kfree(table->bt_l1); | |
976 | table->bt_l1 = NULL; | |
977 | ||
978 | err_kcalloc_bt_l1: | |
979 | kfree(table->hem); | |
980 | table->hem = NULL; | |
981 | ||
982 | err_kcalloc_hem_buf: | |
983 | return -ENOMEM; | |
984 | } | |
985 | ||
281d0ccf CIK |
986 | static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, |
987 | struct hns_roce_hem_table *table) | |
a25d13cb SX |
988 | { |
989 | struct hns_roce_hem_mhop mhop; | |
990 | u32 buf_chunk_size; | |
991 | int i; | |
992 | u64 obj; | |
993 | ||
994 | hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); | |
6a93c77a SX |
995 | buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : |
996 | mhop.bt_chunk_size; | |
a25d13cb SX |
997 | |
998 | for (i = 0; i < table->num_hem; ++i) { | |
999 | obj = i * buf_chunk_size / table->obj_size; | |
1000 | if (table->hem[i]) | |
1001 | hns_roce_table_mhop_put(hr_dev, table, obj, 0); | |
1002 | } | |
1003 | ||
1004 | kfree(table->hem); | |
1005 | table->hem = NULL; | |
1006 | kfree(table->bt_l1); | |
1007 | table->bt_l1 = NULL; | |
1008 | kfree(table->bt_l1_dma_addr); | |
1009 | table->bt_l1_dma_addr = NULL; | |
1010 | kfree(table->bt_l0); | |
1011 | table->bt_l0 = NULL; | |
1012 | kfree(table->bt_l0_dma_addr); | |
1013 | table->bt_l0_dma_addr = NULL; | |
9a443537 | 1014 | } |
1015 | ||
1016 | void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, | |
1017 | struct hns_roce_hem_table *table) | |
1018 | { | |
13ca970e | 1019 | struct device *dev = hr_dev->dev; |
9a443537 | 1020 | unsigned long i; |
1021 | ||
a25d13cb SX |
1022 | if (hns_roce_check_whether_mhop(hr_dev, table->type)) { |
1023 | hns_roce_cleanup_mhop_hem_table(hr_dev, table); | |
1024 | return; | |
1025 | } | |
1026 | ||
9a443537 | 1027 | for (i = 0; i < table->num_hem; ++i) |
1028 | if (table->hem[i]) { | |
97f0e39f | 1029 | if (hr_dev->hw->clear_hem(hr_dev, table, |
29a1fe5d | 1030 | i * table->table_chunk_size / table->obj_size, 0)) |
9a443537 | 1031 | dev_err(dev, "Clear HEM base address failed.\n"); |
1032 | ||
1033 | hns_roce_free_hem(hr_dev, table->hem[i]); | |
1034 | } | |
1035 | ||
1036 | kfree(table->hem); | |
1037 | } | |
1038 | ||
1039 | void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) | |
1040 | { | |
1041 | hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); | |
1042 | hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); | |
e92f2c18 | 1043 | if (hr_dev->caps.trrl_entry_sz) |
1044 | hns_roce_cleanup_hem_table(hr_dev, | |
1045 | &hr_dev->qp_table.trrl_table); | |
9a443537 | 1046 | hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); |
1047 | hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); | |
1048 | hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); | |
9766edc3 SX |
1049 | if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) |
1050 | hns_roce_cleanup_hem_table(hr_dev, | |
1051 | &hr_dev->mr_table.mtt_cqe_table); | |
9a443537 | 1052 | } |