]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. | |
5 | * All rights reserved. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of Intel Corporation nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
34 | #include <stdint.h> | |
35 | #include <stddef.h> | |
36 | #include <stdio.h> | |
37 | #include <string.h> | |
38 | #include <sys/queue.h> | |
39 | ||
40 | #include <rte_memcpy.h> | |
41 | #include <rte_memory.h> | |
42 | #include <rte_eal.h> | |
43 | #include <rte_eal_memconfig.h> | |
44 | #include <rte_branch_prediction.h> | |
45 | #include <rte_debug.h> | |
46 | #include <rte_launch.h> | |
47 | #include <rte_per_lcore.h> | |
48 | #include <rte_lcore.h> | |
49 | #include <rte_common.h> | |
50 | #include <rte_spinlock.h> | |
51 | ||
52 | #include <rte_malloc.h> | |
53 | #include "malloc_elem.h" | |
54 | #include "malloc_heap.h" | |
55 | ||
56 | ||
57 | /* Free the memory space back to heap */ | |
58 | void rte_free(void *addr) | |
59 | { | |
60 | if (addr == NULL) return; | |
61 | if (malloc_elem_free(malloc_elem_from_data(addr)) < 0) | |
62 | rte_panic("Fatal error: Invalid memory\n"); | |
63 | } | |
64 | ||
65 | /* | |
66 | * Allocate memory on specified heap. | |
67 | */ | |
68 | void * | |
69 | rte_malloc_socket(const char *type, size_t size, unsigned align, int socket_arg) | |
70 | { | |
71 | struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; | |
72 | int socket, i; | |
73 | void *ret; | |
74 | ||
75 | /* return NULL if size is 0 or alignment is not power-of-2 */ | |
76 | if (size == 0 || (align && !rte_is_power_of_2(align))) | |
77 | return NULL; | |
78 | ||
79 | if (!rte_eal_has_hugepages()) | |
80 | socket_arg = SOCKET_ID_ANY; | |
81 | ||
82 | if (socket_arg == SOCKET_ID_ANY) | |
83 | socket = malloc_get_numa_socket(); | |
84 | else | |
85 | socket = socket_arg; | |
86 | ||
87 | /* Check socket parameter */ | |
88 | if (socket >= RTE_MAX_NUMA_NODES) | |
89 | return NULL; | |
90 | ||
91 | ret = malloc_heap_alloc(&mcfg->malloc_heaps[socket], type, | |
92 | size, 0, align == 0 ? 1 : align, 0); | |
93 | if (ret != NULL || socket_arg != SOCKET_ID_ANY) | |
94 | return ret; | |
95 | ||
96 | /* try other heaps */ | |
97 | for (i = 0; i < RTE_MAX_NUMA_NODES; i++) { | |
98 | /* we already tried this one */ | |
99 | if (i == socket) | |
100 | continue; | |
101 | ||
102 | ret = malloc_heap_alloc(&mcfg->malloc_heaps[i], type, | |
103 | size, 0, align == 0 ? 1 : align, 0); | |
104 | if (ret != NULL) | |
105 | return ret; | |
106 | } | |
107 | ||
108 | return NULL; | |
109 | } | |
110 | ||
111 | /* | |
112 | * Allocate memory on default heap. | |
113 | */ | |
114 | void * | |
115 | rte_malloc(const char *type, size_t size, unsigned align) | |
116 | { | |
117 | return rte_malloc_socket(type, size, align, SOCKET_ID_ANY); | |
118 | } | |
119 | ||
120 | /* | |
121 | * Allocate zero'd memory on specified heap. | |
122 | */ | |
123 | void * | |
124 | rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket) | |
125 | { | |
126 | return rte_malloc_socket(type, size, align, socket); | |
127 | } | |
128 | ||
129 | /* | |
130 | * Allocate zero'd memory on default heap. | |
131 | */ | |
132 | void * | |
133 | rte_zmalloc(const char *type, size_t size, unsigned align) | |
134 | { | |
135 | return rte_zmalloc_socket(type, size, align, SOCKET_ID_ANY); | |
136 | } | |
137 | ||
138 | /* | |
139 | * Allocate zero'd memory on specified heap. | |
140 | */ | |
141 | void * | |
142 | rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket) | |
143 | { | |
144 | return rte_zmalloc_socket(type, num * size, align, socket); | |
145 | } | |
146 | ||
147 | /* | |
148 | * Allocate zero'd memory on default heap. | |
149 | */ | |
150 | void * | |
151 | rte_calloc(const char *type, size_t num, size_t size, unsigned align) | |
152 | { | |
153 | return rte_zmalloc(type, num * size, align); | |
154 | } | |
155 | ||
156 | /* | |
157 | * Resize allocated memory. | |
158 | */ | |
159 | void * | |
160 | rte_realloc(void *ptr, size_t size, unsigned align) | |
161 | { | |
162 | if (ptr == NULL) | |
163 | return rte_malloc(NULL, size, align); | |
164 | ||
165 | struct malloc_elem *elem = malloc_elem_from_data(ptr); | |
166 | if (elem == NULL) | |
167 | rte_panic("Fatal error: memory corruption detected\n"); | |
168 | ||
169 | size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align); | |
170 | /* check alignment matches first, and if ok, see if we can resize block */ | |
171 | if (RTE_PTR_ALIGN(ptr,align) == ptr && | |
172 | malloc_elem_resize(elem, size) == 0) | |
173 | return ptr; | |
174 | ||
175 | /* either alignment is off, or we have no room to expand, | |
176 | * so move data. */ | |
177 | void *new_ptr = rte_malloc(NULL, size, align); | |
178 | if (new_ptr == NULL) | |
179 | return NULL; | |
180 | const unsigned old_size = elem->size - MALLOC_ELEM_OVERHEAD; | |
181 | rte_memcpy(new_ptr, ptr, old_size < size ? old_size : size); | |
182 | rte_free(ptr); | |
183 | ||
184 | return new_ptr; | |
185 | } | |
186 | ||
187 | int | |
188 | rte_malloc_validate(const void *ptr, size_t *size) | |
189 | { | |
190 | const struct malloc_elem *elem = malloc_elem_from_data(ptr); | |
191 | if (!malloc_elem_cookies_ok(elem)) | |
192 | return -1; | |
193 | if (size != NULL) | |
194 | *size = elem->size - elem->pad - MALLOC_ELEM_OVERHEAD; | |
195 | return 0; | |
196 | } | |
197 | ||
198 | /* | |
199 | * Function to retrieve data for heap on given socket | |
200 | */ | |
201 | int | |
202 | rte_malloc_get_socket_stats(int socket, | |
203 | struct rte_malloc_socket_stats *socket_stats) | |
204 | { | |
205 | struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; | |
206 | ||
207 | if (socket >= RTE_MAX_NUMA_NODES || socket < 0) | |
208 | return -1; | |
209 | ||
210 | return malloc_heap_get_stats(&mcfg->malloc_heaps[socket], socket_stats); | |
211 | } | |
212 | ||
213 | /* | |
214 | * Print stats on memory type. If type is NULL, info on all types is printed | |
215 | */ | |
216 | void | |
217 | rte_malloc_dump_stats(FILE *f, __rte_unused const char *type) | |
218 | { | |
219 | unsigned int socket; | |
220 | struct rte_malloc_socket_stats sock_stats; | |
221 | /* Iterate through all initialised heaps */ | |
222 | for (socket=0; socket< RTE_MAX_NUMA_NODES; socket++) { | |
223 | if ((rte_malloc_get_socket_stats(socket, &sock_stats) < 0)) | |
224 | continue; | |
225 | ||
226 | fprintf(f, "Socket:%u\n", socket); | |
227 | fprintf(f, "\tHeap_size:%zu,\n", sock_stats.heap_totalsz_bytes); | |
228 | fprintf(f, "\tFree_size:%zu,\n", sock_stats.heap_freesz_bytes); | |
229 | fprintf(f, "\tAlloc_size:%zu,\n", sock_stats.heap_allocsz_bytes); | |
230 | fprintf(f, "\tGreatest_free_size:%zu,\n", | |
231 | sock_stats.greatest_free_size); | |
232 | fprintf(f, "\tAlloc_count:%u,\n",sock_stats.alloc_count); | |
233 | fprintf(f, "\tFree_count:%u,\n", sock_stats.free_count); | |
234 | } | |
235 | return; | |
236 | } | |
237 | ||
238 | /* | |
239 | * TODO: Set limit to memory that can be allocated to memory type | |
240 | */ | |
241 | int | |
242 | rte_malloc_set_limit(__rte_unused const char *type, | |
243 | __rte_unused size_t max) | |
244 | { | |
245 | return 0; | |
246 | } | |
247 | ||
248 | /* | |
249 | * Return the physical address of a virtual address obtained through rte_malloc | |
250 | */ | |
251 | phys_addr_t | |
252 | rte_malloc_virt2phy(const void *addr) | |
253 | { | |
254 | const struct malloc_elem *elem = malloc_elem_from_data(addr); | |
255 | if (elem == NULL) | |
256 | return 0; | |
257 | return elem->ms->phys_addr + ((uintptr_t)addr - (uintptr_t)elem->ms->addr); | |
258 | } |