1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
5 #define _FILE_OFFSET_BITS 64
15 #include <sys/types.h>
17 #include <sys/queue.h>
22 #include <sys/ioctl.h>
26 #ifdef F_ADD_SEALS /* if file sealing is supported, so is memfd */
27 #include <linux/memfd.h>
28 #define MEMFD_SUPPORTED
30 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
34 #include <linux/falloc.h>
35 #include <linux/mman.h> /* for hugetlb-related mmap flags */
37 #include <rte_common.h>
39 #include <rte_eal_memconfig.h>
41 #include <rte_errno.h>
42 #include <rte_memory.h>
43 #include <rte_spinlock.h>
45 #include "eal_filesystem.h"
46 #include "eal_internal_cfg.h"
47 #include "eal_memalloc.h"
48 #include "eal_private.h"
50 const int anonymous_hugepages_supported
=
53 #define RTE_MAP_HUGE_SHIFT MAP_HUGE_SHIFT
56 #define RTE_MAP_HUGE_SHIFT 26
60 * we've already checked memfd support at compile-time, but we also need to
61 * check if we can create hugepage files with memfd.
63 * also, this is not a constant, because while we may be *compiled* with memfd
64 * hugetlbfs support, we might not be *running* on a system that supports memfd
65 * and/or memfd with hugetlbfs, so we need to be able to adjust this flag at
66 * runtime, and fall back to anonymous memory.
68 static int memfd_create_supported
=
71 #define RTE_MFD_HUGETLB MFD_HUGETLB
74 #define RTE_MFD_HUGETLB 4U
78 * not all kernel version support fallocate on hugetlbfs, so fall back to
79 * ftruncate and disallow deallocation if fallocate is not supported.
81 static int fallocate_supported
= -1; /* unknown */
84 * we have two modes - single file segments, and file-per-page mode.
86 * for single-file segments, we use memseg_list_fd to store the segment fd,
87 * while the fds[] will not be allocated, and len will be set to 0.
89 * for file-per-page mode, each page will have its own fd, so 'memseg_list_fd'
90 * will be invalid (set to -1), and we'll use 'fds' to keep track of page fd's.
92 * we cannot know how many pages a system will have in advance, but we do know
93 * that they come in lists, and we know lengths of these lists. so, simply store
94 * a malloc'd array of fd's indexed by list and segment index.
96 * they will be initialized at startup, and filled as we allocate/deallocate
100 int *fds
; /**< dynamically allocated array of segment lock fd's */
101 int memseg_list_fd
; /**< memseg list fd */
102 int len
; /**< total length of the array */
103 int count
; /**< entries used in an array */
104 } fd_list
[RTE_MAX_MEMSEG_LISTS
];
106 /** local copy of a memory map, used to synchronize memory hotplug in MP */
107 static struct rte_memseg_list local_memsegs
[RTE_MAX_MEMSEG_LISTS
];
109 static sigjmp_buf huge_jmpenv
;
111 static void __rte_unused
huge_sigbus_handler(int signo __rte_unused
)
113 siglongjmp(huge_jmpenv
, 1);
116 /* Put setjmp into a wrap method to avoid compiling error. Any non-volatile,
117 * non-static local variable in the stack frame calling sigsetjmp might be
118 * clobbered by a call to longjmp.
120 static int __rte_unused
huge_wrap_sigsetjmp(void)
122 return sigsetjmp(huge_jmpenv
, 1);
125 static struct sigaction huge_action_old
;
126 static int huge_need_recover
;
128 static void __rte_unused
129 huge_register_sigbus(void)
132 struct sigaction action
;
135 sigaddset(&mask
, SIGBUS
);
137 action
.sa_mask
= mask
;
138 action
.sa_handler
= huge_sigbus_handler
;
140 huge_need_recover
= !sigaction(SIGBUS
, &action
, &huge_action_old
);
143 static void __rte_unused
144 huge_recover_sigbus(void)
146 if (huge_need_recover
) {
147 sigaction(SIGBUS
, &huge_action_old
, NULL
);
148 huge_need_recover
= 0;
152 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
157 /* Check if kernel supports NUMA. */
158 if (numa_available() != 0) {
159 RTE_LOG(DEBUG
, EAL
, "NUMA is not supported.\n");
166 prepare_numa(int *oldpolicy
, struct bitmask
*oldmask
, int socket_id
)
168 RTE_LOG(DEBUG
, EAL
, "Trying to obtain current memory policy.\n");
169 if (get_mempolicy(oldpolicy
, oldmask
->maskp
,
170 oldmask
->size
+ 1, 0, 0) < 0) {
172 "Failed to get current mempolicy: %s. "
173 "Assuming MPOL_DEFAULT.\n", strerror(errno
));
174 *oldpolicy
= MPOL_DEFAULT
;
177 "Setting policy MPOL_PREFERRED for socket %d\n",
179 numa_set_preferred(socket_id
);
183 restore_numa(int *oldpolicy
, struct bitmask
*oldmask
)
186 "Restoring previous memory policy: %d\n", *oldpolicy
);
187 if (*oldpolicy
== MPOL_DEFAULT
) {
188 numa_set_localalloc();
189 } else if (set_mempolicy(*oldpolicy
, oldmask
->maskp
,
190 oldmask
->size
+ 1) < 0) {
191 RTE_LOG(ERR
, EAL
, "Failed to restore mempolicy: %s\n",
193 numa_set_localalloc();
195 numa_free_cpumask(oldmask
);
200 * uses fstat to report the size of a file on disk
203 get_file_size(int fd
)
206 if (fstat(fd
, &st
) < 0)
212 pagesz_flags(uint64_t page_sz
)
214 /* as per mmap() manpage, all page sizes are log2 of page size
215 * shifted by MAP_HUGE_SHIFT
217 int log2
= rte_log2_u64(page_sz
);
218 return log2
<< RTE_MAP_HUGE_SHIFT
;
221 /* returns 1 on successful lock, 0 on unsuccessful lock, -1 on error */
222 static int lock(int fd
, int type
)
226 /* flock may be interrupted */
228 ret
= flock(fd
, type
| LOCK_NB
);
229 } while (ret
&& errno
== EINTR
);
231 if (ret
&& errno
== EWOULDBLOCK
) {
235 RTE_LOG(ERR
, EAL
, "%s(): error calling flock(): %s\n",
236 __func__
, strerror(errno
));
239 /* lock was successful */
244 get_seg_memfd(struct hugepage_info
*hi __rte_unused
,
245 unsigned int list_idx __rte_unused
,
246 unsigned int seg_idx __rte_unused
)
248 #ifdef MEMFD_SUPPORTED
250 char segname
[250]; /* as per manpage, limit is 249 bytes plus null */
252 int flags
= RTE_MFD_HUGETLB
| pagesz_flags(hi
->hugepage_sz
);
254 if (internal_config
.single_file_segments
) {
255 fd
= fd_list
[list_idx
].memseg_list_fd
;
258 snprintf(segname
, sizeof(segname
), "seg_%i", list_idx
);
259 fd
= memfd_create(segname
, flags
);
261 RTE_LOG(DEBUG
, EAL
, "%s(): memfd create failed: %s\n",
262 __func__
, strerror(errno
));
265 fd_list
[list_idx
].memseg_list_fd
= fd
;
268 fd
= fd_list
[list_idx
].fds
[seg_idx
];
271 snprintf(segname
, sizeof(segname
), "seg_%i-%i",
273 fd
= memfd_create(segname
, flags
);
275 RTE_LOG(DEBUG
, EAL
, "%s(): memfd create failed: %s\n",
276 __func__
, strerror(errno
));
279 fd_list
[list_idx
].fds
[seg_idx
] = fd
;
288 get_seg_fd(char *path
, int buflen
, struct hugepage_info
*hi
,
289 unsigned int list_idx
, unsigned int seg_idx
)
293 /* for in-memory mode, we only make it here when we're sure we support
294 * memfd, and this is a special case.
296 if (internal_config
.in_memory
)
297 return get_seg_memfd(hi
, list_idx
, seg_idx
);
299 if (internal_config
.single_file_segments
) {
300 /* create a hugepage file path */
301 eal_get_hugefile_path(path
, buflen
, hi
->hugedir
, list_idx
);
303 fd
= fd_list
[list_idx
].memseg_list_fd
;
306 fd
= open(path
, O_CREAT
| O_RDWR
, 0600);
308 RTE_LOG(ERR
, EAL
, "%s(): open failed: %s\n",
309 __func__
, strerror(errno
));
312 /* take out a read lock and keep it indefinitely */
313 if (lock(fd
, LOCK_SH
) < 0) {
314 RTE_LOG(ERR
, EAL
, "%s(): lock failed: %s\n",
315 __func__
, strerror(errno
));
319 fd_list
[list_idx
].memseg_list_fd
= fd
;
322 /* create a hugepage file path */
323 eal_get_hugefile_path(path
, buflen
, hi
->hugedir
,
324 list_idx
* RTE_MAX_MEMSEG_PER_LIST
+ seg_idx
);
326 fd
= fd_list
[list_idx
].fds
[seg_idx
];
329 fd
= open(path
, O_CREAT
| O_RDWR
, 0600);
331 RTE_LOG(DEBUG
, EAL
, "%s(): open failed: %s\n",
332 __func__
, strerror(errno
));
335 /* take out a read lock */
336 if (lock(fd
, LOCK_SH
) < 0) {
337 RTE_LOG(ERR
, EAL
, "%s(): lock failed: %s\n",
338 __func__
, strerror(errno
));
342 fd_list
[list_idx
].fds
[seg_idx
] = fd
;
349 resize_hugefile_in_memory(int fd
, uint64_t fa_offset
,
350 uint64_t page_sz
, bool grow
)
352 int flags
= grow
? 0 : FALLOC_FL_PUNCH_HOLE
|
356 /* grow or shrink the file */
357 ret
= fallocate(fd
, flags
, fa_offset
, page_sz
);
360 RTE_LOG(DEBUG
, EAL
, "%s(): fallocate() failed: %s\n",
369 resize_hugefile_in_filesystem(int fd
, uint64_t fa_offset
, uint64_t page_sz
,
375 if (fallocate_supported
== 0) {
376 /* we cannot deallocate memory if fallocate() is not
377 * supported, and hugepage file is already locked at
378 * creation, so no further synchronization needed.
382 RTE_LOG(DEBUG
, EAL
, "%s(): fallocate not supported, not freeing page back to the system\n",
386 uint64_t new_size
= fa_offset
+ page_sz
;
387 uint64_t cur_size
= get_file_size(fd
);
389 /* fallocate isn't supported, fall back to ftruncate */
390 if (new_size
> cur_size
&&
391 ftruncate(fd
, new_size
) < 0) {
392 RTE_LOG(DEBUG
, EAL
, "%s(): ftruncate() failed: %s\n",
393 __func__
, strerror(errno
));
397 int flags
= grow
? 0 : FALLOC_FL_PUNCH_HOLE
|
402 * technically, it is perfectly safe for both primary
403 * and secondary to grow and shrink the page files:
404 * growing the file repeatedly has no effect because
405 * a page can only be allocated once, while mmap ensures
406 * that secondaries hold on to the page even after the
407 * page itself is removed from the filesystem.
409 * however, leaving growing/shrinking to the primary
410 * tends to expose bugs in fdlist page count handling,
411 * so leave this here just in case.
413 if (rte_eal_process_type() != RTE_PROC_PRIMARY
)
416 /* grow or shrink the file */
417 ret
= fallocate(fd
, flags
, fa_offset
, page_sz
);
420 if (fallocate_supported
== -1 &&
422 RTE_LOG(ERR
, EAL
, "%s(): fallocate() not supported, hugepage deallocation will be disabled\n",
425 fallocate_supported
= 0;
427 RTE_LOG(DEBUG
, EAL
, "%s(): fallocate() failed: %s\n",
433 fallocate_supported
= 1;
441 close_hugefile(int fd
, char *path
, int list_idx
)
444 * primary process must unlink the file, but only when not in in-memory
445 * mode (as in that case there is no file to unlink).
447 if (!internal_config
.in_memory
&&
448 rte_eal_process_type() == RTE_PROC_PRIMARY
&&
450 RTE_LOG(ERR
, EAL
, "%s(): unlinking '%s' failed: %s\n",
451 __func__
, path
, strerror(errno
));
454 fd_list
[list_idx
].memseg_list_fd
= -1;
458 resize_hugefile(int fd
, uint64_t fa_offset
, uint64_t page_sz
, bool grow
)
460 /* in-memory mode is a special case, because we can be sure that
461 * fallocate() is supported.
463 if (internal_config
.in_memory
)
464 return resize_hugefile_in_memory(fd
, fa_offset
,
467 return resize_hugefile_in_filesystem(fd
, fa_offset
, page_sz
,
472 alloc_seg(struct rte_memseg
*ms
, void *addr
, int socket_id
,
473 struct hugepage_info
*hi
, unsigned int list_idx
,
474 unsigned int seg_idx
)
476 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
477 int cur_socket_id
= 0;
489 alloc_sz
= hi
->hugepage_sz
;
491 /* these are checked at init, but code analyzers don't know that */
492 if (internal_config
.in_memory
&& !anonymous_hugepages_supported
) {
493 RTE_LOG(ERR
, EAL
, "Anonymous hugepages not supported, in-memory mode cannot allocate memory\n");
496 if (internal_config
.in_memory
&& !memfd_create_supported
&&
497 internal_config
.single_file_segments
) {
498 RTE_LOG(ERR
, EAL
, "Single-file segments are not supported without memfd support\n");
502 /* in-memory without memfd is a special case */
505 if (internal_config
.in_memory
&& !memfd_create_supported
) {
506 const int in_memory_flags
= MAP_HUGETLB
| MAP_FIXED
|
507 MAP_PRIVATE
| MAP_ANONYMOUS
;
510 pagesz_flag
= pagesz_flags(alloc_sz
);
512 mmap_flags
= in_memory_flags
| pagesz_flag
;
514 /* single-file segments codepath will never be active
515 * here because in-memory mode is incompatible with the
516 * fallback path, and it's stopped at EAL initialization
521 /* takes out a read lock on segment or segment list */
522 fd
= get_seg_fd(path
, sizeof(path
), hi
, list_idx
, seg_idx
);
524 RTE_LOG(ERR
, EAL
, "Couldn't get fd on hugepage file\n");
528 if (internal_config
.single_file_segments
) {
529 map_offset
= seg_idx
* alloc_sz
;
530 ret
= resize_hugefile(fd
, map_offset
, alloc_sz
, true);
534 fd_list
[list_idx
].count
++;
537 if (ftruncate(fd
, alloc_sz
) < 0) {
538 RTE_LOG(DEBUG
, EAL
, "%s(): ftruncate() failed: %s\n",
539 __func__
, strerror(errno
));
542 if (internal_config
.hugepage_unlink
&&
543 !internal_config
.in_memory
) {
545 RTE_LOG(DEBUG
, EAL
, "%s(): unlink() failed: %s\n",
546 __func__
, strerror(errno
));
551 mmap_flags
= MAP_SHARED
| MAP_POPULATE
| MAP_FIXED
;
555 * map the segment, and populate page tables, the kernel fills
556 * this segment with zeros if it's a new page.
558 va
= mmap(addr
, alloc_sz
, PROT_READ
| PROT_WRITE
, mmap_flags
, fd
,
561 if (va
== MAP_FAILED
) {
562 RTE_LOG(DEBUG
, EAL
, "%s(): mmap() failed: %s\n", __func__
,
564 /* mmap failed, but the previous region might have been
565 * unmapped anyway. try to remap it
570 RTE_LOG(DEBUG
, EAL
, "%s(): wrong mmap() address\n", __func__
);
571 munmap(va
, alloc_sz
);
575 /* In linux, hugetlb limitations, like cgroup, are
576 * enforced at fault time instead of mmap(), even
577 * with the option of MAP_POPULATE. Kernel will send
578 * a SIGBUS signal. To avoid to be killed, save stack
579 * environment here, if SIGBUS happens, we can jump
582 if (huge_wrap_sigsetjmp()) {
583 RTE_LOG(DEBUG
, EAL
, "SIGBUS: Cannot mmap more hugepages of size %uMB\n",
584 (unsigned int)(alloc_sz
>> 20));
588 /* we need to trigger a write to the page to enforce page fault and
589 * ensure that page is accessible to us, but we can't overwrite value
590 * that is already there, so read the old value, and write itback.
591 * kernel populates the page with zeroes initially.
593 *(volatile int *)addr
= *(volatile int *)addr
;
595 iova
= rte_mem_virt2iova(addr
);
596 if (iova
== RTE_BAD_PHYS_ADDR
) {
597 RTE_LOG(DEBUG
, EAL
, "%s(): can't get IOVA addr\n",
602 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
603 move_pages(getpid(), 1, &addr
, NULL
, &cur_socket_id
, 0);
605 if (cur_socket_id
!= socket_id
) {
607 "%s(): allocation happened on wrong socket (wanted %d, got %d)\n",
608 __func__
, socket_id
, cur_socket_id
);
612 if (rte_socket_count() > 1)
613 RTE_LOG(DEBUG
, EAL
, "%s(): not checking hugepage NUMA node.\n",
618 ms
->hugepage_sz
= alloc_sz
;
620 ms
->nchannel
= rte_memory_get_nchannel();
621 ms
->nrank
= rte_memory_get_nrank();
623 ms
->socket_id
= socket_id
;
628 munmap(addr
, alloc_sz
);
631 new_addr
= eal_get_virtual_area(addr
, &alloc_sz
, alloc_sz
, 0, flags
);
632 if (new_addr
!= addr
) {
633 if (new_addr
!= NULL
)
634 munmap(new_addr
, alloc_sz
);
635 /* we're leaving a hole in our virtual address space. if
636 * somebody else maps this hole now, we could accidentally
637 * override it in the future.
639 RTE_LOG(CRIT
, EAL
, "Can't mmap holes in our virtual address space\n");
641 /* roll back the ref count */
642 if (internal_config
.single_file_segments
)
643 fd_list
[list_idx
].count
--;
645 /* some codepaths will return negative fd, so exit early */
649 if (internal_config
.single_file_segments
) {
650 resize_hugefile(fd
, map_offset
, alloc_sz
, false);
651 /* ignore failure, can't make it any worse */
653 /* if refcount is at zero, close the file */
654 if (fd_list
[list_idx
].count
== 0)
655 close_hugefile(fd
, path
, list_idx
);
657 /* only remove file if we can take out a write lock */
658 if (internal_config
.hugepage_unlink
== 0 &&
659 internal_config
.in_memory
== 0 &&
660 lock(fd
, LOCK_EX
) == 1)
663 fd_list
[list_idx
].fds
[seg_idx
] = -1;
669 free_seg(struct rte_memseg
*ms
, struct hugepage_info
*hi
,
670 unsigned int list_idx
, unsigned int seg_idx
)
677 /* erase page data */
678 memset(ms
->addr
, 0, ms
->len
);
680 if (mmap(ms
->addr
, ms
->len
, PROT_READ
,
681 MAP_PRIVATE
| MAP_ANONYMOUS
| MAP_FIXED
, -1, 0) ==
683 RTE_LOG(DEBUG
, EAL
, "couldn't unmap page\n");
689 /* if we're using anonymous hugepages, nothing to be done */
690 if (internal_config
.in_memory
&& !memfd_create_supported
)
693 /* if we've already unlinked the page, nothing needs to be done */
694 if (!internal_config
.in_memory
&& internal_config
.hugepage_unlink
)
698 memset(ms
, 0, sizeof(*ms
));
702 /* if we are not in single file segments mode, we're going to unmap the
703 * segment and thus drop the lock on original fd, but hugepage dir is
704 * now locked so we can take out another one without races.
706 fd
= get_seg_fd(path
, sizeof(path
), hi
, list_idx
, seg_idx
);
710 if (internal_config
.single_file_segments
) {
711 map_offset
= seg_idx
* ms
->len
;
712 if (resize_hugefile(fd
, map_offset
, ms
->len
, false))
715 if (--(fd_list
[list_idx
].count
) == 0)
716 close_hugefile(fd
, path
, list_idx
);
720 /* if we're able to take out a write lock, we're the last one
721 * holding onto this page.
723 if (!internal_config
.in_memory
) {
724 ret
= lock(fd
, LOCK_EX
);
726 /* no one else is using this page */
731 /* closing fd will drop the lock */
733 fd_list
[list_idx
].fds
[seg_idx
] = -1;
736 memset(ms
, 0, sizeof(*ms
));
738 return ret
< 0 ? -1 : 0;
741 struct alloc_walk_param
{
742 struct hugepage_info
*hi
;
743 struct rte_memseg
**ms
;
745 unsigned int segs_allocated
;
751 alloc_seg_walk(const struct rte_memseg_list
*msl
, void *arg
)
753 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
754 struct alloc_walk_param
*wa
= arg
;
755 struct rte_memseg_list
*cur_msl
;
757 int cur_idx
, start_idx
, j
, dir_fd
= -1;
758 unsigned int msl_idx
, need
, i
;
760 if (msl
->page_sz
!= wa
->page_sz
)
762 if (msl
->socket_id
!= wa
->socket
)
765 page_sz
= (size_t)msl
->page_sz
;
767 msl_idx
= msl
- mcfg
->memsegs
;
768 cur_msl
= &mcfg
->memsegs
[msl_idx
];
772 /* try finding space in memseg list */
774 /* if we require exact number of pages in a list, find them */
775 cur_idx
= rte_fbarray_find_next_n_free(&cur_msl
->memseg_arr
, 0,
783 /* we don't require exact number of pages, so we're going to go
784 * for best-effort allocation. that means finding the biggest
785 * unused block, and going with that.
787 cur_idx
= rte_fbarray_find_biggest_free(&cur_msl
->memseg_arr
,
792 /* adjust the size to possibly be smaller than original
793 * request, but do not allow it to be bigger.
795 cur_len
= rte_fbarray_find_contig_free(&cur_msl
->memseg_arr
,
797 need
= RTE_MIN(need
, (unsigned int)cur_len
);
800 /* do not allow any page allocations during the time we're allocating,
801 * because file creation and locking operations are not atomic,
802 * and we might be the first or the last ones to use a particular page,
803 * so we need to ensure atomicity of every operation.
805 * during init, we already hold a write lock, so don't try to take out
808 if (wa
->hi
->lock_descriptor
== -1 && !internal_config
.in_memory
) {
809 dir_fd
= open(wa
->hi
->hugedir
, O_RDONLY
);
811 RTE_LOG(ERR
, EAL
, "%s(): Cannot open '%s': %s\n",
812 __func__
, wa
->hi
->hugedir
, strerror(errno
));
815 /* blocking writelock */
816 if (flock(dir_fd
, LOCK_EX
)) {
817 RTE_LOG(ERR
, EAL
, "%s(): Cannot lock '%s': %s\n",
818 __func__
, wa
->hi
->hugedir
, strerror(errno
));
824 for (i
= 0; i
< need
; i
++, cur_idx
++) {
825 struct rte_memseg
*cur
;
828 cur
= rte_fbarray_get(&cur_msl
->memseg_arr
, cur_idx
);
829 map_addr
= RTE_PTR_ADD(cur_msl
->base_va
,
832 if (alloc_seg(cur
, map_addr
, wa
->socket
, wa
->hi
,
834 RTE_LOG(DEBUG
, EAL
, "attempted to allocate %i segments, but only %i were allocated\n",
837 /* if exact number wasn't requested, stop */
842 for (j
= start_idx
; j
< cur_idx
; j
++) {
843 struct rte_memseg
*tmp
;
844 struct rte_fbarray
*arr
=
845 &cur_msl
->memseg_arr
;
847 tmp
= rte_fbarray_get(arr
, j
);
848 rte_fbarray_set_free(arr
, j
);
850 /* free_seg may attempt to create a file, which
853 if (free_seg(tmp
, wa
->hi
, msl_idx
, j
))
854 RTE_LOG(DEBUG
, EAL
, "Cannot free page\n");
858 memset(wa
->ms
, 0, sizeof(*wa
->ms
) * wa
->n_segs
);
867 rte_fbarray_set_used(&cur_msl
->memseg_arr
, cur_idx
);
870 wa
->segs_allocated
= i
;
875 /* if we didn't allocate any segments, move on to the next list */
879 struct free_walk_param
{
880 struct hugepage_info
*hi
;
881 struct rte_memseg
*ms
;
884 free_seg_walk(const struct rte_memseg_list
*msl
, void *arg
)
886 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
887 struct rte_memseg_list
*found_msl
;
888 struct free_walk_param
*wa
= arg
;
889 uintptr_t start_addr
, end_addr
;
890 int msl_idx
, seg_idx
, ret
, dir_fd
= -1;
892 start_addr
= (uintptr_t) msl
->base_va
;
893 end_addr
= start_addr
+ msl
->len
;
895 if ((uintptr_t)wa
->ms
->addr
< start_addr
||
896 (uintptr_t)wa
->ms
->addr
>= end_addr
)
899 msl_idx
= msl
- mcfg
->memsegs
;
900 seg_idx
= RTE_PTR_DIFF(wa
->ms
->addr
, start_addr
) / msl
->page_sz
;
903 found_msl
= &mcfg
->memsegs
[msl_idx
];
905 /* do not allow any page allocations during the time we're freeing,
906 * because file creation and locking operations are not atomic,
907 * and we might be the first or the last ones to use a particular page,
908 * so we need to ensure atomicity of every operation.
910 * during init, we already hold a write lock, so don't try to take out
913 if (wa
->hi
->lock_descriptor
== -1 && !internal_config
.in_memory
) {
914 dir_fd
= open(wa
->hi
->hugedir
, O_RDONLY
);
916 RTE_LOG(ERR
, EAL
, "%s(): Cannot open '%s': %s\n",
917 __func__
, wa
->hi
->hugedir
, strerror(errno
));
920 /* blocking writelock */
921 if (flock(dir_fd
, LOCK_EX
)) {
922 RTE_LOG(ERR
, EAL
, "%s(): Cannot lock '%s': %s\n",
923 __func__
, wa
->hi
->hugedir
, strerror(errno
));
929 found_msl
->version
++;
931 rte_fbarray_set_free(&found_msl
->memseg_arr
, seg_idx
);
933 ret
= free_seg(wa
->ms
, wa
->hi
, msl_idx
, seg_idx
);
945 eal_memalloc_alloc_seg_bulk(struct rte_memseg
**ms
, int n_segs
, size_t page_sz
,
946 int socket
, bool exact
)
949 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
950 bool have_numa
= false;
952 struct bitmask
*oldmask
;
954 struct alloc_walk_param wa
;
955 struct hugepage_info
*hi
= NULL
;
957 memset(&wa
, 0, sizeof(wa
));
959 /* dynamic allocation not supported in legacy mode */
960 if (internal_config
.legacy_mem
)
963 for (i
= 0; i
< (int) RTE_DIM(internal_config
.hugepage_info
); i
++) {
965 internal_config
.hugepage_info
[i
].hugepage_sz
) {
966 hi
= &internal_config
.hugepage_info
[i
];
971 RTE_LOG(ERR
, EAL
, "%s(): can't find relevant hugepage_info entry\n",
976 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
978 oldmask
= numa_allocate_nodemask();
979 prepare_numa(&oldpolicy
, oldmask
, socket
);
988 wa
.page_sz
= page_sz
;
990 wa
.segs_allocated
= 0;
992 /* memalloc is locked, so it's safe to use thread-unsafe version */
993 ret
= rte_memseg_list_walk_thread_unsafe(alloc_seg_walk
, &wa
);
995 RTE_LOG(ERR
, EAL
, "%s(): couldn't find suitable memseg_list\n",
998 } else if (ret
> 0) {
999 ret
= (int)wa
.segs_allocated
;
1002 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
1004 restore_numa(&oldpolicy
, oldmask
);
1010 eal_memalloc_alloc_seg(size_t page_sz
, int socket
)
1012 struct rte_memseg
*ms
;
1013 if (eal_memalloc_alloc_seg_bulk(&ms
, 1, page_sz
, socket
, true) < 0)
1015 /* return pointer to newly allocated memseg */
1020 eal_memalloc_free_seg_bulk(struct rte_memseg
**ms
, int n_segs
)
1024 /* dynamic free not supported in legacy mode */
1025 if (internal_config
.legacy_mem
)
1028 for (seg
= 0; seg
< n_segs
; seg
++) {
1029 struct rte_memseg
*cur
= ms
[seg
];
1030 struct hugepage_info
*hi
= NULL
;
1031 struct free_walk_param wa
;
1034 /* if this page is marked as unfreeable, fail */
1035 if (cur
->flags
& RTE_MEMSEG_FLAG_DO_NOT_FREE
) {
1036 RTE_LOG(DEBUG
, EAL
, "Page is not allowed to be freed\n");
1041 memset(&wa
, 0, sizeof(wa
));
1043 for (i
= 0; i
< (int)RTE_DIM(internal_config
.hugepage_info
);
1045 hi
= &internal_config
.hugepage_info
[i
];
1046 if (cur
->hugepage_sz
== hi
->hugepage_sz
)
1049 if (i
== (int)RTE_DIM(internal_config
.hugepage_info
)) {
1050 RTE_LOG(ERR
, EAL
, "Can't find relevant hugepage_info entry\n");
1058 /* memalloc is locked, so it's safe to use thread-unsafe version
1060 walk_res
= rte_memseg_list_walk_thread_unsafe(free_seg_walk
,
1065 RTE_LOG(ERR
, EAL
, "Couldn't find memseg list\n");
1072 eal_memalloc_free_seg(struct rte_memseg
*ms
)
1074 /* dynamic free not supported in legacy mode */
1075 if (internal_config
.legacy_mem
)
1078 return eal_memalloc_free_seg_bulk(&ms
, 1);
1082 sync_chunk(struct rte_memseg_list
*primary_msl
,
1083 struct rte_memseg_list
*local_msl
, struct hugepage_info
*hi
,
1084 unsigned int msl_idx
, bool used
, int start
, int end
)
1086 struct rte_fbarray
*l_arr
, *p_arr
;
1087 int i
, ret
, chunk_len
, diff_len
;
1089 l_arr
= &local_msl
->memseg_arr
;
1090 p_arr
= &primary_msl
->memseg_arr
;
1092 /* we need to aggregate allocations/deallocations into bigger chunks,
1093 * as we don't want to spam the user with per-page callbacks.
1095 * to avoid any potential issues, we also want to trigger
1096 * deallocation callbacks *before* we actually deallocate
1097 * memory, so that the user application could wrap up its use
1098 * before it goes away.
1101 chunk_len
= end
- start
;
1103 /* find how many contiguous pages we can map/unmap for this chunk */
1105 rte_fbarray_find_contig_free(l_arr
, start
) :
1106 rte_fbarray_find_contig_used(l_arr
, start
);
1108 /* has to be at least one page */
1112 diff_len
= RTE_MIN(chunk_len
, diff_len
);
1114 /* if we are freeing memory, notify the application */
1116 struct rte_memseg
*ms
;
1118 size_t len
, page_sz
;
1120 ms
= rte_fbarray_get(l_arr
, start
);
1121 start_va
= ms
->addr
;
1122 page_sz
= (size_t)primary_msl
->page_sz
;
1123 len
= page_sz
* diff_len
;
1125 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE
,
1129 for (i
= 0; i
< diff_len
; i
++) {
1130 struct rte_memseg
*p_ms
, *l_ms
;
1131 int seg_idx
= start
+ i
;
1133 l_ms
= rte_fbarray_get(l_arr
, seg_idx
);
1134 p_ms
= rte_fbarray_get(p_arr
, seg_idx
);
1136 if (l_ms
== NULL
|| p_ms
== NULL
)
1140 ret
= alloc_seg(l_ms
, p_ms
->addr
,
1141 p_ms
->socket_id
, hi
,
1145 rte_fbarray_set_used(l_arr
, seg_idx
);
1147 ret
= free_seg(l_ms
, hi
, msl_idx
, seg_idx
);
1148 rte_fbarray_set_free(l_arr
, seg_idx
);
1154 /* if we just allocated memory, notify the application */
1156 struct rte_memseg
*ms
;
1158 size_t len
, page_sz
;
1160 ms
= rte_fbarray_get(l_arr
, start
);
1161 start_va
= ms
->addr
;
1162 page_sz
= (size_t)primary_msl
->page_sz
;
1163 len
= page_sz
* diff_len
;
1165 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_ALLOC
,
1169 /* calculate how much we can advance until next chunk */
1171 rte_fbarray_find_contig_used(l_arr
, start
) :
1172 rte_fbarray_find_contig_free(l_arr
, start
);
1173 ret
= RTE_MIN(chunk_len
, diff_len
);
1179 sync_status(struct rte_memseg_list
*primary_msl
,
1180 struct rte_memseg_list
*local_msl
, struct hugepage_info
*hi
,
1181 unsigned int msl_idx
, bool used
)
1183 struct rte_fbarray
*l_arr
, *p_arr
;
1184 int p_idx
, l_chunk_len
, p_chunk_len
, ret
;
1187 /* this is a little bit tricky, but the basic idea is - walk both lists
1188 * and spot any places where there are discrepancies. walking both lists
1189 * and noting discrepancies in a single go is a hard problem, so we do
1190 * it in two passes - first we spot any places where allocated segments
1191 * mismatch (i.e. ensure that everything that's allocated in the primary
1192 * is also allocated in the secondary), and then we do it by looking at
1193 * free segments instead.
1195 * we also need to aggregate changes into chunks, as we have to call
1196 * callbacks per allocation, not per page.
1198 l_arr
= &local_msl
->memseg_arr
;
1199 p_arr
= &primary_msl
->memseg_arr
;
1202 p_idx
= rte_fbarray_find_next_used(p_arr
, 0);
1204 p_idx
= rte_fbarray_find_next_free(p_arr
, 0);
1206 while (p_idx
>= 0) {
1207 int next_chunk_search_idx
;
1210 p_chunk_len
= rte_fbarray_find_contig_used(p_arr
,
1212 l_chunk_len
= rte_fbarray_find_contig_used(l_arr
,
1215 p_chunk_len
= rte_fbarray_find_contig_free(p_arr
,
1217 l_chunk_len
= rte_fbarray_find_contig_free(l_arr
,
1220 /* best case scenario - no differences (or bigger, which will be
1221 * fixed during next iteration), look for next chunk
1223 if (l_chunk_len
>= p_chunk_len
) {
1224 next_chunk_search_idx
= p_idx
+ p_chunk_len
;
1228 /* if both chunks start at the same point, skip parts we know
1229 * are identical, and sync the rest. each call to sync_chunk
1230 * will only sync contiguous segments, so we need to call this
1231 * until we are sure there are no more differences in this
1234 start
= p_idx
+ l_chunk_len
;
1235 end
= p_idx
+ p_chunk_len
;
1237 ret
= sync_chunk(primary_msl
, local_msl
, hi
, msl_idx
,
1240 } while (start
< end
&& ret
>= 0);
1241 /* if ret is negative, something went wrong */
1245 next_chunk_search_idx
= p_idx
+ p_chunk_len
;
1247 /* skip to end of this chunk */
1249 p_idx
= rte_fbarray_find_next_used(p_arr
,
1250 next_chunk_search_idx
);
1252 p_idx
= rte_fbarray_find_next_free(p_arr
,
1253 next_chunk_search_idx
);
1260 sync_existing(struct rte_memseg_list
*primary_msl
,
1261 struct rte_memseg_list
*local_msl
, struct hugepage_info
*hi
,
1262 unsigned int msl_idx
)
1266 /* do not allow any page allocations during the time we're allocating,
1267 * because file creation and locking operations are not atomic,
1268 * and we might be the first or the last ones to use a particular page,
1269 * so we need to ensure atomicity of every operation.
1271 dir_fd
= open(hi
->hugedir
, O_RDONLY
);
1273 RTE_LOG(ERR
, EAL
, "%s(): Cannot open '%s': %s\n", __func__
,
1274 hi
->hugedir
, strerror(errno
));
1277 /* blocking writelock */
1278 if (flock(dir_fd
, LOCK_EX
)) {
1279 RTE_LOG(ERR
, EAL
, "%s(): Cannot lock '%s': %s\n", __func__
,
1280 hi
->hugedir
, strerror(errno
));
1285 /* ensure all allocated space is the same in both lists */
1286 ret
= sync_status(primary_msl
, local_msl
, hi
, msl_idx
, true);
1290 /* ensure all unallocated space is the same in both lists */
1291 ret
= sync_status(primary_msl
, local_msl
, hi
, msl_idx
, false);
1295 /* update version number */
1296 local_msl
->version
= primary_msl
->version
;
1307 sync_walk(const struct rte_memseg_list
*msl
, void *arg __rte_unused
)
1309 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
1310 struct rte_memseg_list
*primary_msl
, *local_msl
;
1311 struct hugepage_info
*hi
= NULL
;
1318 msl_idx
= msl
- mcfg
->memsegs
;
1319 primary_msl
= &mcfg
->memsegs
[msl_idx
];
1320 local_msl
= &local_memsegs
[msl_idx
];
1322 for (i
= 0; i
< RTE_DIM(internal_config
.hugepage_info
); i
++) {
1324 internal_config
.hugepage_info
[i
].hugepage_sz
;
1325 uint64_t msl_sz
= primary_msl
->page_sz
;
1326 if (msl_sz
== cur_sz
) {
1327 hi
= &internal_config
.hugepage_info
[i
];
1332 RTE_LOG(ERR
, EAL
, "Can't find relevant hugepage_info entry\n");
1336 /* if versions don't match, synchronize everything */
1337 if (local_msl
->version
!= primary_msl
->version
&&
1338 sync_existing(primary_msl
, local_msl
, hi
, msl_idx
))
1345 eal_memalloc_sync_with_primary(void)
1347 /* nothing to be done in primary */
1348 if (rte_eal_process_type() == RTE_PROC_PRIMARY
)
1351 /* memalloc is locked, so it's safe to call thread-unsafe version */
1352 if (rte_memseg_list_walk_thread_unsafe(sync_walk
, NULL
))
1358 secondary_msl_create_walk(const struct rte_memseg_list
*msl
,
1359 void *arg __rte_unused
)
1361 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
1362 struct rte_memseg_list
*primary_msl
, *local_msl
;
1363 char name
[PATH_MAX
];
1369 msl_idx
= msl
- mcfg
->memsegs
;
1370 primary_msl
= &mcfg
->memsegs
[msl_idx
];
1371 local_msl
= &local_memsegs
[msl_idx
];
1373 /* create distinct fbarrays for each secondary */
1374 snprintf(name
, RTE_FBARRAY_NAME_LEN
, "%s_%i",
1375 primary_msl
->memseg_arr
.name
, getpid());
1377 ret
= rte_fbarray_init(&local_msl
->memseg_arr
, name
,
1378 primary_msl
->memseg_arr
.len
,
1379 primary_msl
->memseg_arr
.elt_sz
);
1381 RTE_LOG(ERR
, EAL
, "Cannot initialize local memory map\n");
1384 local_msl
->base_va
= primary_msl
->base_va
;
1385 local_msl
->len
= primary_msl
->len
;
1391 alloc_list(int list_idx
, int len
)
1396 /* single-file segments mode does not need fd list */
1397 if (!internal_config
.single_file_segments
) {
1398 /* ensure we have space to store fd per each possible segment */
1399 data
= malloc(sizeof(int) * len
);
1401 RTE_LOG(ERR
, EAL
, "Unable to allocate space for file descriptors\n");
1404 /* set all fd's as invalid */
1405 for (i
= 0; i
< len
; i
++)
1407 fd_list
[list_idx
].fds
= data
;
1408 fd_list
[list_idx
].len
= len
;
1410 fd_list
[list_idx
].fds
= NULL
;
1411 fd_list
[list_idx
].len
= 0;
1414 fd_list
[list_idx
].count
= 0;
1415 fd_list
[list_idx
].memseg_list_fd
= -1;
1421 fd_list_create_walk(const struct rte_memseg_list
*msl
,
1422 void *arg __rte_unused
)
1424 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
1431 msl_idx
= msl
- mcfg
->memsegs
;
1432 len
= msl
->memseg_arr
.len
;
1434 return alloc_list(msl_idx
, len
);
1438 eal_memalloc_set_seg_fd(int list_idx
, int seg_idx
, int fd
)
1440 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
1442 /* single file segments mode doesn't support individual segment fd's */
1443 if (internal_config
.single_file_segments
)
1446 /* if list is not allocated, allocate it */
1447 if (fd_list
[list_idx
].len
== 0) {
1448 int len
= mcfg
->memsegs
[list_idx
].memseg_arr
.len
;
1450 if (alloc_list(list_idx
, len
) < 0)
1453 fd_list
[list_idx
].fds
[seg_idx
] = fd
;
1459 eal_memalloc_set_seg_list_fd(int list_idx
, int fd
)
1461 /* non-single file segment mode doesn't support segment list fd's */
1462 if (!internal_config
.single_file_segments
)
1465 fd_list
[list_idx
].memseg_list_fd
= fd
;
1471 eal_memalloc_get_seg_fd(int list_idx
, int seg_idx
)
1475 if (internal_config
.in_memory
|| internal_config
.no_hugetlbfs
) {
1476 #ifndef MEMFD_SUPPORTED
1477 /* in in-memory or no-huge mode, we rely on memfd support */
1480 /* memfd supported, but hugetlbfs memfd may not be */
1481 if (!internal_config
.no_hugetlbfs
&& !memfd_create_supported
)
1485 if (internal_config
.single_file_segments
) {
1486 fd
= fd_list
[list_idx
].memseg_list_fd
;
1487 } else if (fd_list
[list_idx
].len
== 0) {
1488 /* list not initialized */
1491 fd
= fd_list
[list_idx
].fds
[seg_idx
];
1499 test_memfd_create(void)
1501 #ifdef MEMFD_SUPPORTED
1503 for (i
= 0; i
< internal_config
.num_hugepage_sizes
; i
++) {
1504 uint64_t pagesz
= internal_config
.hugepage_info
[i
].hugepage_sz
;
1505 int pagesz_flag
= pagesz_flags(pagesz
);
1508 flags
= pagesz_flag
| RTE_MFD_HUGETLB
;
1509 int fd
= memfd_create("test", flags
);
1511 /* we failed - let memalloc know this isn't working */
1512 if (errno
== EINVAL
) {
1513 memfd_create_supported
= 0;
1514 return 0; /* not supported */
1517 /* we got other error - something's wrong */
1518 return -1; /* error */
1521 return 1; /* supported */
1524 return 0; /* not supported */
1528 eal_memalloc_get_seg_fd_offset(int list_idx
, int seg_idx
, size_t *offset
)
1530 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
1532 if (internal_config
.in_memory
|| internal_config
.no_hugetlbfs
) {
1533 #ifndef MEMFD_SUPPORTED
1534 /* in in-memory or no-huge mode, we rely on memfd support */
1537 /* memfd supported, but hugetlbfs memfd may not be */
1538 if (!internal_config
.no_hugetlbfs
&& !memfd_create_supported
)
1542 if (internal_config
.single_file_segments
) {
1543 size_t pgsz
= mcfg
->memsegs
[list_idx
].page_sz
;
1545 /* segment not active? */
1546 if (fd_list
[list_idx
].memseg_list_fd
< 0)
1548 *offset
= pgsz
* seg_idx
;
1550 /* fd_list not initialized? */
1551 if (fd_list
[list_idx
].len
== 0)
1554 /* segment not active? */
1555 if (fd_list
[list_idx
].fds
[seg_idx
] < 0)
1563 eal_memalloc_init(void)
1565 if (rte_eal_process_type() == RTE_PROC_SECONDARY
)
1566 if (rte_memseg_list_walk(secondary_msl_create_walk
, NULL
) < 0)
1568 if (rte_eal_process_type() == RTE_PROC_PRIMARY
&&
1569 internal_config
.in_memory
) {
1570 int mfd_res
= test_memfd_create();
1573 RTE_LOG(ERR
, EAL
, "Unable to check if memfd is supported\n");
1577 RTE_LOG(DEBUG
, EAL
, "Using memfd for anonymous memory\n");
1579 RTE_LOG(INFO
, EAL
, "Using memfd is not supported, falling back to anonymous hugepages\n");
1581 /* we only support single-file segments mode with in-memory mode
1582 * if we support hugetlbfs with memfd_create. this code will
1585 if (internal_config
.single_file_segments
&&
1587 RTE_LOG(ERR
, EAL
, "Single-file segments mode cannot be used without memfd support\n");
1590 /* this cannot ever happen but better safe than sorry */
1591 if (!anonymous_hugepages_supported
) {
1592 RTE_LOG(ERR
, EAL
, "Using anonymous memory is not supported\n");
1597 /* initialize all of the fd lists */
1598 if (rte_memseg_list_walk(fd_list_create_walk
, NULL
))