]>
Commit | Line | Data |
---|---|---|
9f95a23c | 1 | /* SPDX-License-Identifier: BSD-3-Clause |
11fdf7f2 TL |
2 | * |
3 | * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. | |
9f95a23c | 4 | * Copyright 2016 NXP |
11fdf7f2 | 5 | * |
11fdf7f2 TL |
6 | */ |
7 | ||
8 | #include <unistd.h> | |
9 | #include <stdio.h> | |
10 | #include <sys/types.h> | |
11 | #include <string.h> | |
12 | #include <stdlib.h> | |
13 | #include <fcntl.h> | |
14 | #include <errno.h> | |
15 | ||
16 | #include <rte_mbuf.h> | |
9f95a23c | 17 | #include <rte_ethdev_driver.h> |
11fdf7f2 TL |
18 | #include <rte_malloc.h> |
19 | #include <rte_memcpy.h> | |
20 | #include <rte_string_fns.h> | |
21 | #include <rte_cycles.h> | |
22 | #include <rte_kvargs.h> | |
23 | #include <rte_dev.h> | |
9f95a23c | 24 | #include "rte_dpaa2_mempool.h" |
11fdf7f2 TL |
25 | |
26 | #include <fslmc_logs.h> | |
27 | #include <mc/fsl_dpbp.h> | |
28 | #include <portal/dpaa2_hw_pvt.h> | |
29 | #include <portal/dpaa2_hw_dpio.h> | |
30 | #include "dpaa2_hw_mempool.h" | |
9f95a23c TL |
31 | #include "dpaa2_hw_mempool_logs.h" |
32 | ||
33 | #include <dpaax_iova_table.h> | |
11fdf7f2 | 34 | |
9f95a23c | 35 | struct dpaa2_bp_info *rte_dpaa2_bpid_info; |
11fdf7f2 TL |
36 | static struct dpaa2_bp_list *h_bp_list; |
37 | ||
9f95a23c TL |
38 | /* Dynamic logging identified for mempool */ |
39 | int dpaa2_logtype_mempool; | |
40 | ||
11fdf7f2 TL |
41 | static int |
42 | rte_hw_mbuf_create_pool(struct rte_mempool *mp) | |
43 | { | |
44 | struct dpaa2_bp_list *bp_list; | |
45 | struct dpaa2_dpbp_dev *avail_dpbp; | |
9f95a23c | 46 | struct dpaa2_bp_info *bp_info; |
11fdf7f2 TL |
47 | struct dpbp_attr dpbp_attr; |
48 | uint32_t bpid; | |
9f95a23c | 49 | int ret; |
11fdf7f2 TL |
50 | |
51 | avail_dpbp = dpaa2_alloc_dpbp_dev(); | |
52 | ||
9f95a23c TL |
53 | if (rte_dpaa2_bpid_info == NULL) { |
54 | rte_dpaa2_bpid_info = (struct dpaa2_bp_info *)rte_malloc(NULL, | |
55 | sizeof(struct dpaa2_bp_info) * MAX_BPID, | |
56 | RTE_CACHE_LINE_SIZE); | |
57 | if (rte_dpaa2_bpid_info == NULL) | |
58 | return -ENOMEM; | |
59 | memset(rte_dpaa2_bpid_info, 0, | |
60 | sizeof(struct dpaa2_bp_info) * MAX_BPID); | |
61 | } | |
62 | ||
11fdf7f2 | 63 | if (!avail_dpbp) { |
9f95a23c | 64 | DPAA2_MEMPOOL_ERR("DPAA2 pool not available!"); |
11fdf7f2 TL |
65 | return -ENOENT; |
66 | } | |
67 | ||
68 | if (unlikely(!DPAA2_PER_LCORE_DPIO)) { | |
69 | ret = dpaa2_affine_qbman_swp(); | |
70 | if (ret) { | |
9f95a23c TL |
71 | DPAA2_MEMPOOL_ERR("Failure in affining portal"); |
72 | goto err1; | |
11fdf7f2 TL |
73 | } |
74 | } | |
75 | ||
76 | ret = dpbp_enable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token); | |
77 | if (ret != 0) { | |
9f95a23c TL |
78 | DPAA2_MEMPOOL_ERR("Resource enable failure with err code: %d", |
79 | ret); | |
80 | goto err1; | |
11fdf7f2 TL |
81 | } |
82 | ||
83 | ret = dpbp_get_attributes(&avail_dpbp->dpbp, CMD_PRI_LOW, | |
84 | avail_dpbp->token, &dpbp_attr); | |
85 | if (ret != 0) { | |
9f95a23c TL |
86 | DPAA2_MEMPOOL_ERR("Resource read failure with err code: %d", |
87 | ret); | |
88 | goto err2; | |
89 | } | |
90 | ||
91 | bp_info = rte_malloc(NULL, | |
92 | sizeof(struct dpaa2_bp_info), | |
93 | RTE_CACHE_LINE_SIZE); | |
94 | if (!bp_info) { | |
95 | DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory"); | |
96 | ret = -ENOMEM; | |
97 | goto err2; | |
11fdf7f2 TL |
98 | } |
99 | ||
100 | /* Allocate the bp_list which will be added into global_bp_list */ | |
101 | bp_list = rte_malloc(NULL, sizeof(struct dpaa2_bp_list), | |
102 | RTE_CACHE_LINE_SIZE); | |
103 | if (!bp_list) { | |
9f95a23c TL |
104 | DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory"); |
105 | ret = -ENOMEM; | |
106 | goto err3; | |
11fdf7f2 TL |
107 | } |
108 | ||
109 | /* Set parameters of buffer pool list */ | |
110 | bp_list->buf_pool.num_bufs = mp->size; | |
111 | bp_list->buf_pool.size = mp->elt_size | |
112 | - sizeof(struct rte_mbuf) - rte_pktmbuf_priv_size(mp); | |
113 | bp_list->buf_pool.bpid = dpbp_attr.bpid; | |
114 | bp_list->buf_pool.h_bpool_mem = NULL; | |
115 | bp_list->buf_pool.dpbp_node = avail_dpbp; | |
116 | /* Identification for our offloaded pool_data structure */ | |
117 | bp_list->dpaa2_ops_index = mp->ops_index; | |
118 | bp_list->next = h_bp_list; | |
119 | bp_list->mp = mp; | |
120 | ||
121 | bpid = dpbp_attr.bpid; | |
122 | ||
123 | rte_dpaa2_bpid_info[bpid].meta_data_size = sizeof(struct rte_mbuf) | |
124 | + rte_pktmbuf_priv_size(mp); | |
125 | rte_dpaa2_bpid_info[bpid].bp_list = bp_list; | |
126 | rte_dpaa2_bpid_info[bpid].bpid = bpid; | |
127 | ||
9f95a23c TL |
128 | rte_memcpy(bp_info, (void *)&rte_dpaa2_bpid_info[bpid], |
129 | sizeof(struct dpaa2_bp_info)); | |
130 | mp->pool_data = (void *)bp_info; | |
11fdf7f2 | 131 | |
9f95a23c | 132 | DPAA2_MEMPOOL_DEBUG("BP List created for bpid =%d", dpbp_attr.bpid); |
11fdf7f2 TL |
133 | |
134 | h_bp_list = bp_list; | |
135 | return 0; | |
9f95a23c TL |
136 | err3: |
137 | rte_free(bp_info); | |
138 | err2: | |
139 | dpbp_disable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token); | |
140 | err1: | |
141 | dpaa2_free_dpbp_dev(avail_dpbp); | |
142 | ||
143 | return ret; | |
11fdf7f2 TL |
144 | } |
145 | ||
146 | static void | |
147 | rte_hw_mbuf_free_pool(struct rte_mempool *mp) | |
148 | { | |
149 | struct dpaa2_bp_info *bpinfo; | |
150 | struct dpaa2_bp_list *bp; | |
151 | struct dpaa2_dpbp_dev *dpbp_node; | |
152 | ||
153 | if (!mp->pool_data) { | |
9f95a23c | 154 | DPAA2_MEMPOOL_ERR("Not a valid dpaa2 buffer pool"); |
11fdf7f2 TL |
155 | return; |
156 | } | |
157 | ||
158 | bpinfo = (struct dpaa2_bp_info *)mp->pool_data; | |
159 | bp = bpinfo->bp_list; | |
160 | dpbp_node = bp->buf_pool.dpbp_node; | |
161 | ||
162 | dpbp_disable(&(dpbp_node->dpbp), CMD_PRI_LOW, dpbp_node->token); | |
163 | ||
164 | if (h_bp_list == bp) { | |
165 | h_bp_list = h_bp_list->next; | |
166 | } else { /* if it is not the first node */ | |
167 | struct dpaa2_bp_list *prev = h_bp_list, *temp; | |
168 | temp = h_bp_list->next; | |
169 | while (temp) { | |
170 | if (temp == bp) { | |
171 | prev->next = temp->next; | |
9f95a23c | 172 | rte_free(bp); |
11fdf7f2 TL |
173 | break; |
174 | } | |
175 | prev = temp; | |
176 | temp = temp->next; | |
177 | } | |
178 | } | |
179 | ||
9f95a23c | 180 | rte_free(mp->pool_data); |
11fdf7f2 TL |
181 | dpaa2_free_dpbp_dev(dpbp_node); |
182 | } | |
183 | ||
184 | static void | |
185 | rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused, | |
186 | void * const *obj_table, | |
187 | uint32_t bpid, | |
188 | uint32_t meta_data_size, | |
189 | int count) | |
190 | { | |
191 | struct qbman_release_desc releasedesc; | |
192 | struct qbman_swp *swp; | |
193 | int ret; | |
194 | int i, n; | |
195 | uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; | |
196 | ||
197 | if (unlikely(!DPAA2_PER_LCORE_DPIO)) { | |
198 | ret = dpaa2_affine_qbman_swp(); | |
199 | if (ret != 0) { | |
9f95a23c | 200 | DPAA2_MEMPOOL_ERR("Failed to allocate IO portal"); |
11fdf7f2 TL |
201 | return; |
202 | } | |
203 | } | |
204 | swp = DPAA2_PER_LCORE_PORTAL; | |
205 | ||
206 | /* Create a release descriptor required for releasing | |
207 | * buffers into QBMAN | |
208 | */ | |
209 | qbman_release_desc_clear(&releasedesc); | |
210 | qbman_release_desc_set_bpid(&releasedesc, bpid); | |
211 | ||
212 | n = count % DPAA2_MBUF_MAX_ACQ_REL; | |
213 | if (unlikely(!n)) | |
214 | goto aligned; | |
215 | ||
216 | /* convert mbuf to buffers for the remainder */ | |
217 | for (i = 0; i < n ; i++) { | |
218 | #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA | |
9f95a23c | 219 | bufs[i] = (uint64_t)rte_mempool_virt2iova(obj_table[i]) |
11fdf7f2 TL |
220 | + meta_data_size; |
221 | #else | |
222 | bufs[i] = (uint64_t)obj_table[i] + meta_data_size; | |
223 | #endif | |
224 | } | |
225 | ||
226 | /* feed them to bman */ | |
227 | do { | |
228 | ret = qbman_swp_release(swp, &releasedesc, bufs, n); | |
229 | } while (ret == -EBUSY); | |
230 | ||
231 | aligned: | |
232 | /* if there are more buffers to free */ | |
233 | while (n < count) { | |
234 | /* convert mbuf to buffers */ | |
235 | for (i = 0; i < DPAA2_MBUF_MAX_ACQ_REL; i++) { | |
236 | #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA | |
237 | bufs[i] = (uint64_t) | |
9f95a23c | 238 | rte_mempool_virt2iova(obj_table[n + i]) |
11fdf7f2 TL |
239 | + meta_data_size; |
240 | #else | |
241 | bufs[i] = (uint64_t)obj_table[n + i] + meta_data_size; | |
242 | #endif | |
243 | } | |
244 | ||
245 | do { | |
246 | ret = qbman_swp_release(swp, &releasedesc, bufs, | |
247 | DPAA2_MBUF_MAX_ACQ_REL); | |
248 | } while (ret == -EBUSY); | |
249 | n += DPAA2_MBUF_MAX_ACQ_REL; | |
250 | } | |
251 | } | |
252 | ||
9f95a23c TL |
253 | uint16_t |
254 | rte_dpaa2_mbuf_pool_bpid(struct rte_mempool *mp) | |
255 | { | |
256 | struct dpaa2_bp_info *bp_info; | |
257 | ||
258 | bp_info = mempool_to_bpinfo(mp); | |
259 | if (!(bp_info->bp_list)) { | |
260 | RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); | |
261 | return -ENOMEM; | |
262 | } | |
263 | ||
264 | return bp_info->bpid; | |
265 | } | |
266 | ||
267 | struct rte_mbuf * | |
268 | rte_dpaa2_mbuf_from_buf_addr(struct rte_mempool *mp, void *buf_addr) | |
269 | { | |
270 | struct dpaa2_bp_info *bp_info; | |
271 | ||
272 | bp_info = mempool_to_bpinfo(mp); | |
273 | if (!(bp_info->bp_list)) { | |
274 | RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); | |
275 | return NULL; | |
276 | } | |
277 | ||
278 | return (struct rte_mbuf *)((uint8_t *)buf_addr - | |
279 | bp_info->meta_data_size); | |
280 | } | |
281 | ||
11fdf7f2 TL |
282 | int |
283 | rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool, | |
284 | void **obj_table, unsigned int count) | |
285 | { | |
286 | #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER | |
287 | static int alloc; | |
288 | #endif | |
289 | struct qbman_swp *swp; | |
290 | uint16_t bpid; | |
9f95a23c | 291 | size_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; |
11fdf7f2 TL |
292 | int i, ret; |
293 | unsigned int n = 0; | |
294 | struct dpaa2_bp_info *bp_info; | |
295 | ||
296 | bp_info = mempool_to_bpinfo(pool); | |
297 | ||
298 | if (!(bp_info->bp_list)) { | |
9f95a23c | 299 | DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured"); |
11fdf7f2 TL |
300 | return -ENOENT; |
301 | } | |
302 | ||
303 | bpid = bp_info->bpid; | |
304 | ||
305 | if (unlikely(!DPAA2_PER_LCORE_DPIO)) { | |
306 | ret = dpaa2_affine_qbman_swp(); | |
307 | if (ret != 0) { | |
9f95a23c | 308 | DPAA2_MEMPOOL_ERR("Failed to allocate IO portal"); |
11fdf7f2 TL |
309 | return ret; |
310 | } | |
311 | } | |
312 | swp = DPAA2_PER_LCORE_PORTAL; | |
313 | ||
314 | while (n < count) { | |
315 | /* Acquire is all-or-nothing, so we drain in 7s, | |
316 | * then the remainder. | |
317 | */ | |
318 | if ((count - n) > DPAA2_MBUF_MAX_ACQ_REL) { | |
9f95a23c | 319 | ret = qbman_swp_acquire(swp, bpid, (void *)bufs, |
11fdf7f2 TL |
320 | DPAA2_MBUF_MAX_ACQ_REL); |
321 | } else { | |
9f95a23c | 322 | ret = qbman_swp_acquire(swp, bpid, (void *)bufs, |
11fdf7f2 TL |
323 | count - n); |
324 | } | |
325 | /* In case of less than requested number of buffers available | |
326 | * in pool, qbman_swp_acquire returns 0 | |
327 | */ | |
328 | if (ret <= 0) { | |
9f95a23c TL |
329 | DPAA2_MEMPOOL_DP_DEBUG( |
330 | "Buffer acquire failed with err code: %d", ret); | |
11fdf7f2 TL |
331 | /* The API expect the exact number of requested bufs */ |
332 | /* Releasing all buffers allocated */ | |
333 | rte_dpaa2_mbuf_release(pool, obj_table, bpid, | |
334 | bp_info->meta_data_size, n); | |
9f95a23c | 335 | return -ENOBUFS; |
11fdf7f2 TL |
336 | } |
337 | /* assigning mbuf from the acquired objects */ | |
338 | for (i = 0; (i < ret) && bufs[i]; i++) { | |
9f95a23c | 339 | DPAA2_MODIFY_IOVA_TO_VADDR(bufs[i], size_t); |
11fdf7f2 TL |
340 | obj_table[n] = (struct rte_mbuf *) |
341 | (bufs[i] - bp_info->meta_data_size); | |
9f95a23c TL |
342 | DPAA2_MEMPOOL_DP_DEBUG( |
343 | "Acquired %p address %p from BMAN\n", | |
11fdf7f2 TL |
344 | (void *)bufs[i], (void *)obj_table[n]); |
345 | n++; | |
346 | } | |
347 | } | |
348 | ||
349 | #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER | |
350 | alloc += n; | |
9f95a23c TL |
351 | DPAA2_MEMPOOL_DP_DEBUG("Total = %d , req = %d done = %d\n", |
352 | alloc, count, n); | |
11fdf7f2 TL |
353 | #endif |
354 | return 0; | |
355 | } | |
356 | ||
357 | static int | |
358 | rte_hw_mbuf_free_bulk(struct rte_mempool *pool, | |
359 | void * const *obj_table, unsigned int n) | |
360 | { | |
361 | struct dpaa2_bp_info *bp_info; | |
362 | ||
363 | bp_info = mempool_to_bpinfo(pool); | |
364 | if (!(bp_info->bp_list)) { | |
9f95a23c | 365 | DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured"); |
11fdf7f2 TL |
366 | return -ENOENT; |
367 | } | |
368 | rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid, | |
369 | bp_info->meta_data_size, n); | |
370 | ||
371 | return 0; | |
372 | } | |
373 | ||
374 | static unsigned int | |
375 | rte_hw_mbuf_get_count(const struct rte_mempool *mp) | |
376 | { | |
377 | int ret; | |
378 | unsigned int num_of_bufs = 0; | |
379 | struct dpaa2_bp_info *bp_info; | |
380 | struct dpaa2_dpbp_dev *dpbp_node; | |
381 | ||
382 | if (!mp || !mp->pool_data) { | |
9f95a23c | 383 | DPAA2_MEMPOOL_ERR("Invalid mempool provided"); |
11fdf7f2 TL |
384 | return 0; |
385 | } | |
386 | ||
387 | bp_info = (struct dpaa2_bp_info *)mp->pool_data; | |
388 | dpbp_node = bp_info->bp_list->buf_pool.dpbp_node; | |
389 | ||
390 | ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW, | |
391 | dpbp_node->token, &num_of_bufs); | |
392 | if (ret) { | |
9f95a23c TL |
393 | DPAA2_MEMPOOL_ERR("Unable to obtain free buf count (err=%d)", |
394 | ret); | |
11fdf7f2 TL |
395 | return 0; |
396 | } | |
397 | ||
9f95a23c | 398 | DPAA2_MEMPOOL_DP_DEBUG("Free bufs = %u\n", num_of_bufs); |
11fdf7f2 TL |
399 | |
400 | return num_of_bufs; | |
401 | } | |
402 | ||
9f95a23c TL |
403 | static int |
404 | dpaa2_populate(struct rte_mempool *mp, unsigned int max_objs, | |
405 | void *vaddr, rte_iova_t paddr, size_t len, | |
406 | rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg) | |
407 | { | |
408 | /* Insert entry into the PA->VA Table */ | |
409 | dpaax_iova_table_update(paddr, vaddr, len); | |
410 | ||
411 | return rte_mempool_op_populate_default(mp, max_objs, vaddr, paddr, len, | |
412 | obj_cb, obj_cb_arg); | |
413 | } | |
414 | ||
415 | static const struct rte_mempool_ops dpaa2_mpool_ops = { | |
416 | .name = DPAA2_MEMPOOL_OPS_NAME, | |
11fdf7f2 TL |
417 | .alloc = rte_hw_mbuf_create_pool, |
418 | .free = rte_hw_mbuf_free_pool, | |
419 | .enqueue = rte_hw_mbuf_free_bulk, | |
420 | .dequeue = rte_dpaa2_mbuf_alloc_bulk, | |
421 | .get_count = rte_hw_mbuf_get_count, | |
9f95a23c | 422 | .populate = dpaa2_populate, |
11fdf7f2 TL |
423 | }; |
424 | ||
425 | MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops); | |
9f95a23c TL |
426 | |
427 | RTE_INIT(dpaa2_mempool_init_log) | |
428 | { | |
429 | dpaa2_logtype_mempool = rte_log_register("mempool.dpaa2"); | |
430 | if (dpaa2_logtype_mempool >= 0) | |
431 | rte_log_set_level(dpaa2_logtype_mempool, RTE_LOG_NOTICE); | |
432 | } |