4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include <sys/queue.h>
40 #include <rte_memcpy.h>
41 #include <rte_memory.h>
43 #include <rte_eal_memconfig.h>
44 #include <rte_branch_prediction.h>
45 #include <rte_debug.h>
46 #include <rte_launch.h>
47 #include <rte_per_lcore.h>
48 #include <rte_lcore.h>
49 #include <rte_common.h>
50 #include <rte_spinlock.h>
52 #include <rte_malloc.h>
53 #include "malloc_elem.h"
54 #include "malloc_heap.h"
57 /* Free the memory space back to heap */
58 void rte_free(void *addr
)
60 if (addr
== NULL
) return;
61 if (malloc_elem_free(malloc_elem_from_data(addr
)) < 0)
62 rte_panic("Fatal error: Invalid memory\n");
66 * Allocate memory on specified heap.
69 rte_malloc_socket(const char *type
, size_t size
, unsigned align
, int socket_arg
)
71 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
75 /* return NULL if size is 0 or alignment is not power-of-2 */
76 if (size
== 0 || (align
&& !rte_is_power_of_2(align
)))
79 if (!rte_eal_has_hugepages())
80 socket_arg
= SOCKET_ID_ANY
;
82 if (socket_arg
== SOCKET_ID_ANY
)
83 socket
= malloc_get_numa_socket();
87 /* Check socket parameter */
88 if (socket
>= RTE_MAX_NUMA_NODES
)
91 ret
= malloc_heap_alloc(&mcfg
->malloc_heaps
[socket
], type
,
92 size
, 0, align
== 0 ? 1 : align
, 0);
93 if (ret
!= NULL
|| socket_arg
!= SOCKET_ID_ANY
)
97 for (i
= 0; i
< RTE_MAX_NUMA_NODES
; i
++) {
98 /* we already tried this one */
102 ret
= malloc_heap_alloc(&mcfg
->malloc_heaps
[i
], type
,
103 size
, 0, align
== 0 ? 1 : align
, 0);
112 * Allocate memory on default heap.
115 rte_malloc(const char *type
, size_t size
, unsigned align
)
117 return rte_malloc_socket(type
, size
, align
, SOCKET_ID_ANY
);
121 * Allocate zero'd memory on specified heap.
124 rte_zmalloc_socket(const char *type
, size_t size
, unsigned align
, int socket
)
126 return rte_malloc_socket(type
, size
, align
, socket
);
130 * Allocate zero'd memory on default heap.
133 rte_zmalloc(const char *type
, size_t size
, unsigned align
)
135 return rte_zmalloc_socket(type
, size
, align
, SOCKET_ID_ANY
);
139 * Allocate zero'd memory on specified heap.
142 rte_calloc_socket(const char *type
, size_t num
, size_t size
, unsigned align
, int socket
)
144 return rte_zmalloc_socket(type
, num
* size
, align
, socket
);
148 * Allocate zero'd memory on default heap.
151 rte_calloc(const char *type
, size_t num
, size_t size
, unsigned align
)
153 return rte_zmalloc(type
, num
* size
, align
);
157 * Resize allocated memory.
160 rte_realloc(void *ptr
, size_t size
, unsigned align
)
163 return rte_malloc(NULL
, size
, align
);
165 struct malloc_elem
*elem
= malloc_elem_from_data(ptr
);
167 rte_panic("Fatal error: memory corruption detected\n");
169 size
= RTE_CACHE_LINE_ROUNDUP(size
), align
= RTE_CACHE_LINE_ROUNDUP(align
);
170 /* check alignment matches first, and if ok, see if we can resize block */
171 if (RTE_PTR_ALIGN(ptr
,align
) == ptr
&&
172 malloc_elem_resize(elem
, size
) == 0)
175 /* either alignment is off, or we have no room to expand,
177 void *new_ptr
= rte_malloc(NULL
, size
, align
);
180 const unsigned old_size
= elem
->size
- MALLOC_ELEM_OVERHEAD
;
181 rte_memcpy(new_ptr
, ptr
, old_size
< size
? old_size
: size
);
188 rte_malloc_validate(const void *ptr
, size_t *size
)
190 const struct malloc_elem
*elem
= malloc_elem_from_data(ptr
);
191 if (!malloc_elem_cookies_ok(elem
))
194 *size
= elem
->size
- elem
->pad
- MALLOC_ELEM_OVERHEAD
;
199 * Function to retrieve data for heap on given socket
202 rte_malloc_get_socket_stats(int socket
,
203 struct rte_malloc_socket_stats
*socket_stats
)
205 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
207 if (socket
>= RTE_MAX_NUMA_NODES
|| socket
< 0)
210 return malloc_heap_get_stats(&mcfg
->malloc_heaps
[socket
], socket_stats
);
214 * Print stats on memory type. If type is NULL, info on all types is printed
217 rte_malloc_dump_stats(FILE *f
, __rte_unused
const char *type
)
220 struct rte_malloc_socket_stats sock_stats
;
221 /* Iterate through all initialised heaps */
222 for (socket
=0; socket
< RTE_MAX_NUMA_NODES
; socket
++) {
223 if ((rte_malloc_get_socket_stats(socket
, &sock_stats
) < 0))
226 fprintf(f
, "Socket:%u\n", socket
);
227 fprintf(f
, "\tHeap_size:%zu,\n", sock_stats
.heap_totalsz_bytes
);
228 fprintf(f
, "\tFree_size:%zu,\n", sock_stats
.heap_freesz_bytes
);
229 fprintf(f
, "\tAlloc_size:%zu,\n", sock_stats
.heap_allocsz_bytes
);
230 fprintf(f
, "\tGreatest_free_size:%zu,\n",
231 sock_stats
.greatest_free_size
);
232 fprintf(f
, "\tAlloc_count:%u,\n",sock_stats
.alloc_count
);
233 fprintf(f
, "\tFree_count:%u,\n", sock_stats
.free_count
);
239 * TODO: Set limit to memory that can be allocated to memory type
242 rte_malloc_set_limit(__rte_unused
const char *type
,
243 __rte_unused
size_t max
)
249 * Return the physical address of a virtual address obtained through rte_malloc
252 rte_malloc_virt2phy(const void *addr
)
254 const struct malloc_elem
*elem
= malloc_elem_from_data(addr
);
257 return elem
->ms
->phys_addr
+ ((uintptr_t)addr
- (uintptr_t)elem
->ms
->addr
);