]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright (c) Intel Corporation. | |
5 | * All rights reserved. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of Intel Corporation nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
11fdf7f2 | 34 | #include "spdk/stdinc.h" |
7c673cae | 35 | |
9f95a23c | 36 | #include "env_internal.h" |
7c673cae FG |
37 | |
38 | #include <rte_config.h> | |
39 | #include <rte_cycles.h> | |
40 | #include <rte_malloc.h> | |
41 | #include <rte_mempool.h> | |
42 | #include <rte_memzone.h> | |
43 | #include <rte_version.h> | |
44 | ||
11fdf7f2 TL |
45 | static uint64_t |
46 | virt_to_phys(void *vaddr) | |
47 | { | |
48 | uint64_t ret; | |
49 | ||
11fdf7f2 TL |
50 | ret = rte_malloc_virt2iova(vaddr); |
51 | if (ret != RTE_BAD_IOVA) { | |
52 | return ret; | |
53 | } | |
11fdf7f2 | 54 | |
9f95a23c | 55 | return spdk_vtophys(vaddr, NULL); |
11fdf7f2 TL |
56 | } |
57 | ||
7c673cae | 58 | void * |
11fdf7f2 | 59 | spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) |
7c673cae | 60 | { |
11fdf7f2 TL |
61 | if (flags == 0) { |
62 | return NULL; | |
63 | } | |
64 | ||
65 | void *buf = rte_malloc_socket(NULL, size, align, socket_id); | |
7c673cae | 66 | if (buf && phys_addr) { |
9f95a23c TL |
67 | #ifdef DEBUG |
68 | fprintf(stderr, "phys_addr param in spdk_*malloc() is deprecated\n"); | |
69 | #endif | |
11fdf7f2 | 70 | *phys_addr = virt_to_phys(buf); |
7c673cae FG |
71 | } |
72 | return buf; | |
73 | } | |
74 | ||
75 | void * | |
11fdf7f2 | 76 | spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) |
7c673cae | 77 | { |
11fdf7f2 | 78 | void *buf = spdk_malloc(size, align, phys_addr, socket_id, flags); |
7c673cae FG |
79 | if (buf) { |
80 | memset(buf, 0, size); | |
81 | } | |
82 | return buf; | |
83 | } | |
84 | ||
9f95a23c TL |
85 | void * |
86 | spdk_realloc(void *buf, size_t size, size_t align) | |
87 | { | |
88 | return rte_realloc(buf, size, align); | |
89 | } | |
90 | ||
11fdf7f2 TL |
91 | void |
92 | spdk_free(void *buf) | |
93 | { | |
94 | rte_free(buf); | |
95 | } | |
96 | ||
97 | void * | |
98 | spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) | |
99 | { | |
100 | return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); | |
101 | } | |
102 | ||
103 | void * | |
104 | spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) | |
105 | { | |
106 | return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); | |
107 | } | |
108 | ||
109 | void * | |
110 | spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) | |
111 | { | |
112 | return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); | |
113 | } | |
114 | ||
115 | void * | |
116 | spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) | |
117 | { | |
118 | return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); | |
119 | } | |
120 | ||
7c673cae | 121 | void * |
11fdf7f2 | 122 | spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) |
7c673cae FG |
123 | { |
124 | void *new_buf = rte_realloc(buf, size, align); | |
125 | if (new_buf && phys_addr) { | |
11fdf7f2 | 126 | *phys_addr = virt_to_phys(new_buf); |
7c673cae FG |
127 | } |
128 | return new_buf; | |
129 | } | |
130 | ||
131 | void | |
11fdf7f2 | 132 | spdk_dma_free(void *buf) |
7c673cae | 133 | { |
11fdf7f2 | 134 | spdk_free(buf); |
7c673cae FG |
135 | } |
136 | ||
137 | void * | |
11fdf7f2 TL |
138 | spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, |
139 | unsigned flags, unsigned align) | |
7c673cae FG |
140 | { |
141 | const struct rte_memzone *mz; | |
11fdf7f2 TL |
142 | unsigned dpdk_flags = 0; |
143 | ||
144 | #if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0) | |
145 | /* Older DPDKs do not offer such flag since their | |
146 | * memzones are iova-contiguous by default. | |
147 | */ | |
148 | if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { | |
149 | dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; | |
150 | } | |
151 | #endif | |
7c673cae FG |
152 | |
153 | if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { | |
154 | socket_id = SOCKET_ID_ANY; | |
155 | } | |
156 | ||
11fdf7f2 | 157 | mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); |
7c673cae FG |
158 | |
159 | if (mz != NULL) { | |
160 | memset(mz->addr, 0, len); | |
161 | return mz->addr; | |
162 | } else { | |
163 | return NULL; | |
164 | } | |
165 | } | |
166 | ||
11fdf7f2 TL |
167 | void * |
168 | spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) | |
169 | { | |
170 | return spdk_memzone_reserve_aligned(name, len, socket_id, flags, | |
171 | RTE_CACHE_LINE_SIZE); | |
172 | } | |
173 | ||
7c673cae FG |
174 | void * |
175 | spdk_memzone_lookup(const char *name) | |
176 | { | |
177 | const struct rte_memzone *mz = rte_memzone_lookup(name); | |
178 | ||
179 | if (mz != NULL) { | |
180 | return mz->addr; | |
181 | } else { | |
182 | return NULL; | |
183 | } | |
184 | } | |
185 | ||
186 | int | |
187 | spdk_memzone_free(const char *name) | |
188 | { | |
189 | const struct rte_memzone *mz = rte_memzone_lookup(name); | |
190 | ||
191 | if (mz != NULL) { | |
192 | return rte_memzone_free(mz); | |
193 | } | |
194 | ||
195 | return -1; | |
196 | } | |
197 | ||
198 | void | |
199 | spdk_memzone_dump(FILE *f) | |
200 | { | |
201 | rte_memzone_dump(f); | |
202 | } | |
203 | ||
204 | struct spdk_mempool * | |
11fdf7f2 TL |
205 | spdk_mempool_create_ctor(const char *name, size_t count, |
206 | size_t ele_size, size_t cache_size, int socket_id, | |
207 | spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) | |
7c673cae FG |
208 | { |
209 | struct rte_mempool *mp; | |
210 | size_t tmp; | |
211 | ||
212 | if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { | |
213 | socket_id = SOCKET_ID_ANY; | |
214 | } | |
215 | ||
216 | /* No more than half of all elements can be in cache */ | |
217 | tmp = (count / 2) / rte_lcore_count(); | |
218 | if (cache_size > tmp) { | |
219 | cache_size = tmp; | |
220 | } | |
221 | ||
222 | if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { | |
223 | cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; | |
224 | } | |
225 | ||
226 | mp = rte_mempool_create(name, count, ele_size, cache_size, | |
11fdf7f2 TL |
227 | 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, |
228 | socket_id, MEMPOOL_F_NO_PHYS_CONTIG); | |
7c673cae FG |
229 | |
230 | return (struct spdk_mempool *)mp; | |
231 | } | |
232 | ||
11fdf7f2 TL |
233 | |
234 | struct spdk_mempool * | |
235 | spdk_mempool_create(const char *name, size_t count, | |
236 | size_t ele_size, size_t cache_size, int socket_id) | |
237 | { | |
238 | return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, | |
239 | NULL, NULL); | |
240 | } | |
241 | ||
242 | char * | |
243 | spdk_mempool_get_name(struct spdk_mempool *mp) | |
244 | { | |
245 | return ((struct rte_mempool *)mp)->name; | |
246 | } | |
247 | ||
7c673cae FG |
248 | void |
249 | spdk_mempool_free(struct spdk_mempool *mp) | |
250 | { | |
7c673cae | 251 | rte_mempool_free((struct rte_mempool *)mp); |
7c673cae FG |
252 | } |
253 | ||
254 | void * | |
255 | spdk_mempool_get(struct spdk_mempool *mp) | |
256 | { | |
257 | void *ele = NULL; | |
11fdf7f2 | 258 | int rc; |
7c673cae | 259 | |
11fdf7f2 TL |
260 | rc = rte_mempool_get((struct rte_mempool *)mp, &ele); |
261 | if (rc != 0) { | |
262 | return NULL; | |
263 | } | |
7c673cae FG |
264 | return ele; |
265 | } | |
266 | ||
11fdf7f2 TL |
267 | int |
268 | spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) | |
269 | { | |
270 | return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); | |
271 | } | |
272 | ||
7c673cae FG |
273 | void |
274 | spdk_mempool_put(struct spdk_mempool *mp, void *ele) | |
275 | { | |
276 | rte_mempool_put((struct rte_mempool *)mp, ele); | |
277 | } | |
278 | ||
279 | void | |
11fdf7f2 | 280 | spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) |
7c673cae FG |
281 | { |
282 | rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); | |
283 | } | |
284 | ||
11fdf7f2 TL |
285 | size_t |
286 | spdk_mempool_count(const struct spdk_mempool *pool) | |
287 | { | |
11fdf7f2 | 288 | return rte_mempool_avail_count((struct rte_mempool *)pool); |
9f95a23c TL |
289 | } |
290 | ||
291 | uint32_t | |
292 | spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, | |
293 | void *obj_cb_arg) | |
294 | { | |
295 | return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb, | |
296 | obj_cb_arg); | |
11fdf7f2 TL |
297 | } |
298 | ||
7c673cae FG |
299 | bool |
300 | spdk_process_is_primary(void) | |
301 | { | |
302 | return (rte_eal_process_type() == RTE_PROC_PRIMARY); | |
303 | } | |
304 | ||
305 | uint64_t spdk_get_ticks(void) | |
306 | { | |
307 | return rte_get_timer_cycles(); | |
308 | } | |
309 | ||
310 | uint64_t spdk_get_ticks_hz(void) | |
311 | { | |
312 | return rte_get_timer_hz(); | |
313 | } | |
314 | ||
315 | void spdk_delay_us(unsigned int us) | |
316 | { | |
317 | rte_delay_us(us); | |
318 | } | |
319 | ||
9f95a23c TL |
320 | void spdk_pause(void) |
321 | { | |
322 | rte_pause(); | |
323 | } | |
324 | ||
11fdf7f2 TL |
325 | void |
326 | spdk_unaffinitize_thread(void) | |
7c673cae | 327 | { |
11fdf7f2 | 328 | rte_cpuset_t new_cpuset; |
7c673cae FG |
329 | long num_cores, i; |
330 | ||
7c673cae FG |
331 | CPU_ZERO(&new_cpuset); |
332 | ||
333 | num_cores = sysconf(_SC_NPROCESSORS_CONF); | |
334 | ||
335 | /* Create a mask containing all CPUs */ | |
336 | for (i = 0; i < num_cores; i++) { | |
337 | CPU_SET(i, &new_cpuset); | |
338 | } | |
339 | ||
340 | rte_thread_set_affinity(&new_cpuset); | |
11fdf7f2 TL |
341 | } |
342 | ||
343 | void * | |
344 | spdk_call_unaffinitized(void *cb(void *arg), void *arg) | |
345 | { | |
346 | rte_cpuset_t orig_cpuset; | |
347 | void *ret; | |
348 | ||
349 | if (cb == NULL) { | |
350 | return NULL; | |
351 | } | |
352 | ||
353 | rte_thread_get_affinity(&orig_cpuset); | |
354 | ||
355 | spdk_unaffinitize_thread(); | |
7c673cae FG |
356 | |
357 | ret = cb(arg); | |
358 | ||
359 | rte_thread_set_affinity(&orig_cpuset); | |
360 | ||
361 | return ret; | |
362 | } | |
11fdf7f2 TL |
363 | |
364 | struct spdk_ring * | |
365 | spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) | |
366 | { | |
367 | char ring_name[64]; | |
368 | static uint32_t ring_num = 0; | |
9f95a23c | 369 | unsigned flags = RING_F_EXACT_SZ; |
11fdf7f2 TL |
370 | |
371 | switch (type) { | |
372 | case SPDK_RING_TYPE_SP_SC: | |
9f95a23c | 373 | flags |= RING_F_SP_ENQ | RING_F_SC_DEQ; |
11fdf7f2 TL |
374 | break; |
375 | case SPDK_RING_TYPE_MP_SC: | |
9f95a23c | 376 | flags |= RING_F_SC_DEQ; |
11fdf7f2 TL |
377 | break; |
378 | case SPDK_RING_TYPE_MP_MC: | |
9f95a23c | 379 | flags |= 0; |
11fdf7f2 TL |
380 | break; |
381 | default: | |
382 | return NULL; | |
383 | } | |
384 | ||
385 | snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", | |
386 | __sync_fetch_and_add(&ring_num, 1), getpid()); | |
387 | ||
388 | return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); | |
389 | } | |
390 | ||
391 | void | |
392 | spdk_ring_free(struct spdk_ring *ring) | |
393 | { | |
394 | rte_ring_free((struct rte_ring *)ring); | |
395 | } | |
396 | ||
397 | size_t | |
398 | spdk_ring_count(struct spdk_ring *ring) | |
399 | { | |
400 | return rte_ring_count((struct rte_ring *)ring); | |
401 | } | |
402 | ||
403 | size_t | |
9f95a23c TL |
404 | spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, |
405 | size_t *free_space) | |
11fdf7f2 | 406 | { |
9f95a23c TL |
407 | return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, |
408 | (unsigned int *)free_space); | |
11fdf7f2 TL |
409 | } |
410 | ||
411 | size_t | |
412 | spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) | |
413 | { | |
11fdf7f2 | 414 | return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); |
11fdf7f2 | 415 | } |