1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation.
3 * Copyright(c) 2013 6WIND S.A.
6 #define _FILE_OFFSET_BITS 64
17 #include <sys/types.h>
19 #include <sys/queue.h>
21 #include <sys/resource.h>
24 #include <sys/ioctl.h>
28 #ifdef F_ADD_SEALS /* if file sealing is supported, so is memfd */
29 #include <linux/memfd.h>
30 #define MEMFD_SUPPORTED
32 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
37 #include <rte_errno.h>
39 #include <rte_memory.h>
40 #include <rte_launch.h>
42 #include <rte_eal_memconfig.h>
43 #include <rte_per_lcore.h>
44 #include <rte_lcore.h>
45 #include <rte_common.h>
46 #include <rte_string_fns.h>
48 #include "eal_private.h"
49 #include "eal_memalloc.h"
50 #include "eal_internal_cfg.h"
51 #include "eal_filesystem.h"
52 #include "eal_hugepages.h"
53 #include "eal_options.h"
55 #define PFN_MASK_SIZE 8
59 * Huge page mapping under linux
61 * To reserve a big contiguous amount of memory, we use the hugepage
62 * feature of linux. For that, we need to have hugetlbfs mounted. This
63 * code will create many files in this directory (one per page) and
64 * map them in virtual memory. For each page, we will retrieve its
65 * physical address and remap it in order to have a virtual contiguous
66 * zone as well as a physical contiguous zone.
69 static bool phys_addrs_available
= true;
71 #define RANDOMIZE_VA_SPACE_FILE "/proc/sys/kernel/randomize_va_space"
74 test_phys_addrs_available(void)
79 if (!rte_eal_has_hugepages()) {
81 "Started without hugepages support, physical addresses not available\n");
82 phys_addrs_available
= false;
86 physaddr
= rte_mem_virt2phy(&tmp
);
87 if (physaddr
== RTE_BAD_PHYS_ADDR
) {
88 if (rte_eal_iova_mode() == RTE_IOVA_PA
)
90 "Cannot obtain physical addresses: %s. "
91 "Only vfio will function.\n",
93 phys_addrs_available
= false;
98 * Get physical address of any mapped virtual address in the current process.
101 rte_mem_virt2phy(const void *virtaddr
)
104 uint64_t page
, physaddr
;
105 unsigned long virt_pfn
;
109 /* Cannot parse /proc/self/pagemap, no need to log errors everywhere */
110 if (!phys_addrs_available
)
113 /* standard page size */
114 page_size
= getpagesize();
116 fd
= open("/proc/self/pagemap", O_RDONLY
);
118 RTE_LOG(INFO
, EAL
, "%s(): cannot open /proc/self/pagemap: %s\n",
119 __func__
, strerror(errno
));
123 virt_pfn
= (unsigned long)virtaddr
/ page_size
;
124 offset
= sizeof(uint64_t) * virt_pfn
;
125 if (lseek(fd
, offset
, SEEK_SET
) == (off_t
) -1) {
126 RTE_LOG(INFO
, EAL
, "%s(): seek error in /proc/self/pagemap: %s\n",
127 __func__
, strerror(errno
));
132 retval
= read(fd
, &page
, PFN_MASK_SIZE
);
135 RTE_LOG(INFO
, EAL
, "%s(): cannot read /proc/self/pagemap: %s\n",
136 __func__
, strerror(errno
));
138 } else if (retval
!= PFN_MASK_SIZE
) {
139 RTE_LOG(INFO
, EAL
, "%s(): read %d bytes from /proc/self/pagemap "
140 "but expected %d:\n",
141 __func__
, retval
, PFN_MASK_SIZE
);
146 * the pfn (page frame number) are bits 0-54 (see
147 * pagemap.txt in linux Documentation)
149 if ((page
& 0x7fffffffffffffULL
) == 0)
152 physaddr
= ((page
& 0x7fffffffffffffULL
) * page_size
)
153 + ((unsigned long)virtaddr
% page_size
);
159 rte_mem_virt2iova(const void *virtaddr
)
161 if (rte_eal_iova_mode() == RTE_IOVA_VA
)
162 return (uintptr_t)virtaddr
;
163 return rte_mem_virt2phy(virtaddr
);
167 * For each hugepage in hugepg_tbl, fill the physaddr value. We find
168 * it by browsing the /proc/self/pagemap special file.
171 find_physaddrs(struct hugepage_file
*hugepg_tbl
, struct hugepage_info
*hpi
)
176 for (i
= 0; i
< hpi
->num_pages
[0]; i
++) {
177 addr
= rte_mem_virt2phy(hugepg_tbl
[i
].orig_va
);
178 if (addr
== RTE_BAD_PHYS_ADDR
)
180 hugepg_tbl
[i
].physaddr
= addr
;
186 * For each hugepage in hugepg_tbl, fill the physaddr value sequentially.
189 set_physaddrs(struct hugepage_file
*hugepg_tbl
, struct hugepage_info
*hpi
)
192 static phys_addr_t addr
;
194 for (i
= 0; i
< hpi
->num_pages
[0]; i
++) {
195 hugepg_tbl
[i
].physaddr
= addr
;
196 addr
+= hugepg_tbl
[i
].size
;
202 * Check whether address-space layout randomization is enabled in
203 * the kernel. This is important for multi-process as it can prevent
204 * two processes mapping data to the same virtual address
206 * 0 - address space randomization disabled
207 * 1/2 - address space randomization enabled
208 * negative error code on error
214 int retval
, fd
= open(RANDOMIZE_VA_SPACE_FILE
, O_RDONLY
);
217 retval
= read(fd
, &c
, 1);
227 default: return -EINVAL
;
231 static sigjmp_buf huge_jmpenv
;
233 static void huge_sigbus_handler(int signo __rte_unused
)
235 siglongjmp(huge_jmpenv
, 1);
238 /* Put setjmp into a wrap method to avoid compiling error. Any non-volatile,
239 * non-static local variable in the stack frame calling sigsetjmp might be
240 * clobbered by a call to longjmp.
242 static int huge_wrap_sigsetjmp(void)
244 return sigsetjmp(huge_jmpenv
, 1);
247 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
248 /* Callback for numa library. */
249 void numa_error(char *where
)
251 RTE_LOG(ERR
, EAL
, "%s failed: %s\n", where
, strerror(errno
));
256 * Mmap all hugepages of hugepage table: it first open a file in
257 * hugetlbfs, then mmap() hugepage_sz data in it. If orig is set, the
258 * virtual address is stored in hugepg_tbl[i].orig_va, else it is stored
259 * in hugepg_tbl[i].final_va. The second mapping (when orig is 0) tries to
260 * map contiguous physical blocks in contiguous virtual blocks.
263 map_all_hugepages(struct hugepage_file
*hugepg_tbl
, struct hugepage_info
*hpi
,
264 uint64_t *essential_memory __rte_unused
)
269 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
271 int essential_prev
= 0;
273 struct bitmask
*oldmask
= NULL
;
274 bool have_numa
= true;
275 unsigned long maxnode
= 0;
277 /* Check if kernel supports NUMA. */
278 if (numa_available() != 0) {
279 RTE_LOG(DEBUG
, EAL
, "NUMA is not supported.\n");
284 RTE_LOG(DEBUG
, EAL
, "Trying to obtain current memory policy.\n");
285 oldmask
= numa_allocate_nodemask();
286 if (get_mempolicy(&oldpolicy
, oldmask
->maskp
,
287 oldmask
->size
+ 1, 0, 0) < 0) {
289 "Failed to get current mempolicy: %s. "
290 "Assuming MPOL_DEFAULT.\n", strerror(errno
));
291 oldpolicy
= MPOL_DEFAULT
;
293 for (i
= 0; i
< RTE_MAX_NUMA_NODES
; i
++)
294 if (internal_config
.socket_mem
[i
])
299 for (i
= 0; i
< hpi
->num_pages
[0]; i
++) {
300 struct hugepage_file
*hf
= &hugepg_tbl
[i
];
301 uint64_t hugepage_sz
= hpi
->hugepage_sz
;
303 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
307 for (j
= 0; j
< maxnode
; j
++)
308 if (essential_memory
[j
])
312 node_id
= (node_id
+ 1) % maxnode
;
313 while (!internal_config
.socket_mem
[node_id
]) {
320 essential_prev
= essential_memory
[j
];
322 if (essential_memory
[j
] < hugepage_sz
)
323 essential_memory
[j
] = 0;
325 essential_memory
[j
] -= hugepage_sz
;
329 "Setting policy MPOL_PREFERRED for socket %d\n",
331 numa_set_preferred(node_id
);
336 hf
->size
= hugepage_sz
;
337 eal_get_hugefile_path(hf
->filepath
, sizeof(hf
->filepath
),
338 hpi
->hugedir
, hf
->file_id
);
339 hf
->filepath
[sizeof(hf
->filepath
) - 1] = '\0';
341 /* try to create hugepage file */
342 fd
= open(hf
->filepath
, O_CREAT
| O_RDWR
, 0600);
344 RTE_LOG(DEBUG
, EAL
, "%s(): open failed: %s\n", __func__
,
349 /* map the segment, and populate page tables,
350 * the kernel fills this segment with zeros. we don't care where
351 * this gets mapped - we already have contiguous memory areas
352 * ready for us to map into.
354 virtaddr
= mmap(NULL
, hugepage_sz
, PROT_READ
| PROT_WRITE
,
355 MAP_SHARED
| MAP_POPULATE
, fd
, 0);
356 if (virtaddr
== MAP_FAILED
) {
357 RTE_LOG(DEBUG
, EAL
, "%s(): mmap failed: %s\n", __func__
,
363 hf
->orig_va
= virtaddr
;
365 /* In linux, hugetlb limitations, like cgroup, are
366 * enforced at fault time instead of mmap(), even
367 * with the option of MAP_POPULATE. Kernel will send
368 * a SIGBUS signal. To avoid to be killed, save stack
369 * environment here, if SIGBUS happens, we can jump
372 if (huge_wrap_sigsetjmp()) {
373 RTE_LOG(DEBUG
, EAL
, "SIGBUS: Cannot mmap more "
374 "hugepages of size %u MB\n",
375 (unsigned int)(hugepage_sz
/ 0x100000));
376 munmap(virtaddr
, hugepage_sz
);
378 unlink(hugepg_tbl
[i
].filepath
);
379 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
381 essential_memory
[node_id
] =
386 *(int *)virtaddr
= 0;
388 /* set shared lock on the file. */
389 if (flock(fd
, LOCK_SH
) < 0) {
390 RTE_LOG(DEBUG
, EAL
, "%s(): Locking file failed:%s \n",
391 __func__
, strerror(errno
));
400 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
403 "Restoring previous memory policy: %d\n", oldpolicy
);
404 if (oldpolicy
== MPOL_DEFAULT
) {
405 numa_set_localalloc();
406 } else if (set_mempolicy(oldpolicy
, oldmask
->maskp
,
407 oldmask
->size
+ 1) < 0) {
408 RTE_LOG(ERR
, EAL
, "Failed to restore mempolicy: %s\n",
410 numa_set_localalloc();
414 numa_free_cpumask(oldmask
);
420 * Parse /proc/self/numa_maps to get the NUMA socket ID for each huge
424 find_numasocket(struct hugepage_file
*hugepg_tbl
, struct hugepage_info
*hpi
)
428 unsigned i
, hp_count
= 0;
431 char hugedir_str
[PATH_MAX
];
434 f
= fopen("/proc/self/numa_maps", "r");
436 RTE_LOG(NOTICE
, EAL
, "NUMA support not available"
437 " consider that all memory is in socket_id 0\n");
441 snprintf(hugedir_str
, sizeof(hugedir_str
),
442 "%s/%s", hpi
->hugedir
, eal_get_hugefile_prefix());
445 while (fgets(buf
, sizeof(buf
), f
) != NULL
) {
447 /* ignore non huge page */
448 if (strstr(buf
, " huge ") == NULL
&&
449 strstr(buf
, hugedir_str
) == NULL
)
453 virt_addr
= strtoull(buf
, &end
, 16);
454 if (virt_addr
== 0 || end
== buf
) {
455 RTE_LOG(ERR
, EAL
, "%s(): error in numa_maps parsing\n", __func__
);
459 /* get node id (socket id) */
460 nodestr
= strstr(buf
, " N");
461 if (nodestr
== NULL
) {
462 RTE_LOG(ERR
, EAL
, "%s(): error in numa_maps parsing\n", __func__
);
466 end
= strstr(nodestr
, "=");
468 RTE_LOG(ERR
, EAL
, "%s(): error in numa_maps parsing\n", __func__
);
474 socket_id
= strtoul(nodestr
, &end
, 0);
475 if ((nodestr
[0] == '\0') || (end
== NULL
) || (*end
!= '\0')) {
476 RTE_LOG(ERR
, EAL
, "%s(): error in numa_maps parsing\n", __func__
);
480 /* if we find this page in our mappings, set socket_id */
481 for (i
= 0; i
< hpi
->num_pages
[0]; i
++) {
482 void *va
= (void *)(unsigned long)virt_addr
;
483 if (hugepg_tbl
[i
].orig_va
== va
) {
484 hugepg_tbl
[i
].socket_id
= socket_id
;
486 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
488 "Hugepage %s is on socket %d\n",
489 hugepg_tbl
[i
].filepath
, socket_id
);
495 if (hp_count
< hpi
->num_pages
[0])
507 cmp_physaddr(const void *a
, const void *b
)
509 #ifndef RTE_ARCH_PPC_64
510 const struct hugepage_file
*p1
= a
;
511 const struct hugepage_file
*p2
= b
;
513 /* PowerPC needs memory sorted in reverse order from x86 */
514 const struct hugepage_file
*p1
= b
;
515 const struct hugepage_file
*p2
= a
;
517 if (p1
->physaddr
< p2
->physaddr
)
519 else if (p1
->physaddr
> p2
->physaddr
)
526 * Uses mmap to create a shared memory area for storage of data
527 * Used in this file to store the hugepage file map on disk
530 create_shared_memory(const char *filename
, const size_t mem_size
)
535 /* if no shared files mode is used, create anonymous memory instead */
536 if (internal_config
.no_shconf
) {
537 retval
= mmap(NULL
, mem_size
, PROT_READ
| PROT_WRITE
,
538 MAP_PRIVATE
| MAP_ANONYMOUS
, -1, 0);
539 if (retval
== MAP_FAILED
)
544 fd
= open(filename
, O_CREAT
| O_RDWR
, 0600);
547 if (ftruncate(fd
, mem_size
) < 0) {
551 retval
= mmap(NULL
, mem_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
, fd
, 0);
553 if (retval
== MAP_FAILED
)
559 * this copies *active* hugepages from one hugepage table to another.
560 * destination is typically the shared memory.
563 copy_hugepages_to_shared_mem(struct hugepage_file
* dst
, int dest_size
,
564 const struct hugepage_file
* src
, int src_size
)
566 int src_pos
, dst_pos
= 0;
568 for (src_pos
= 0; src_pos
< src_size
; src_pos
++) {
569 if (src
[src_pos
].orig_va
!= NULL
) {
570 /* error on overflow attempt */
571 if (dst_pos
== dest_size
)
573 memcpy(&dst
[dst_pos
], &src
[src_pos
], sizeof(struct hugepage_file
));
581 unlink_hugepage_files(struct hugepage_file
*hugepg_tbl
,
582 unsigned num_hp_info
)
584 unsigned socket
, size
;
585 int page
, nrpages
= 0;
587 /* get total number of hugepages */
588 for (size
= 0; size
< num_hp_info
; size
++)
589 for (socket
= 0; socket
< RTE_MAX_NUMA_NODES
; socket
++)
591 internal_config
.hugepage_info
[size
].num_pages
[socket
];
593 for (page
= 0; page
< nrpages
; page
++) {
594 struct hugepage_file
*hp
= &hugepg_tbl
[page
];
596 if (hp
->orig_va
!= NULL
&& unlink(hp
->filepath
)) {
597 RTE_LOG(WARNING
, EAL
, "%s(): Removing %s failed: %s\n",
598 __func__
, hp
->filepath
, strerror(errno
));
605 * unmaps hugepages that are not going to be used. since we originally allocate
606 * ALL hugepages (not just those we need), additional unmapping needs to be done.
609 unmap_unneeded_hugepages(struct hugepage_file
*hugepg_tbl
,
610 struct hugepage_info
*hpi
,
611 unsigned num_hp_info
)
613 unsigned socket
, size
;
614 int page
, nrpages
= 0;
616 /* get total number of hugepages */
617 for (size
= 0; size
< num_hp_info
; size
++)
618 for (socket
= 0; socket
< RTE_MAX_NUMA_NODES
; socket
++)
619 nrpages
+= internal_config
.hugepage_info
[size
].num_pages
[socket
];
621 for (size
= 0; size
< num_hp_info
; size
++) {
622 for (socket
= 0; socket
< RTE_MAX_NUMA_NODES
; socket
++) {
623 unsigned pages_found
= 0;
625 /* traverse until we have unmapped all the unused pages */
626 for (page
= 0; page
< nrpages
; page
++) {
627 struct hugepage_file
*hp
= &hugepg_tbl
[page
];
629 /* find a page that matches the criteria */
630 if ((hp
->size
== hpi
[size
].hugepage_sz
) &&
631 (hp
->socket_id
== (int) socket
)) {
633 /* if we skipped enough pages, unmap the rest */
634 if (pages_found
== hpi
[size
].num_pages
[socket
]) {
637 unmap_len
= hp
->size
;
639 /* get start addr and len of the remaining segment */
644 if (unlink(hp
->filepath
) == -1) {
645 RTE_LOG(ERR
, EAL
, "%s(): Removing %s failed: %s\n",
646 __func__
, hp
->filepath
, strerror(errno
));
650 /* lock the page and skip */
656 } /* foreach socket */
657 } /* foreach pagesize */
663 remap_segment(struct hugepage_file
*hugepages
, int seg_start
, int seg_end
)
665 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
666 struct rte_memseg_list
*msl
;
667 struct rte_fbarray
*arr
;
668 int cur_page
, seg_len
;
669 unsigned int msl_idx
;
675 page_sz
= hugepages
[seg_start
].size
;
676 socket_id
= hugepages
[seg_start
].socket_id
;
677 seg_len
= seg_end
- seg_start
;
679 RTE_LOG(DEBUG
, EAL
, "Attempting to map %" PRIu64
"M on socket %i\n",
680 (seg_len
* page_sz
) >> 20ULL, socket_id
);
682 /* find free space in memseg lists */
683 for (msl_idx
= 0; msl_idx
< RTE_MAX_MEMSEG_LISTS
; msl_idx
++) {
685 msl
= &mcfg
->memsegs
[msl_idx
];
686 arr
= &msl
->memseg_arr
;
688 if (msl
->page_sz
!= page_sz
)
690 if (msl
->socket_id
!= socket_id
)
693 /* leave space for a hole if array is not empty */
694 empty
= arr
->count
== 0;
695 ms_idx
= rte_fbarray_find_next_n_free(arr
, 0,
696 seg_len
+ (empty
? 0 : 1));
698 /* memseg list is full? */
702 /* leave some space between memsegs, they are not IOVA
703 * contiguous, so they shouldn't be VA contiguous either.
709 if (msl_idx
== RTE_MAX_MEMSEG_LISTS
) {
710 RTE_LOG(ERR
, EAL
, "Could not find space for memseg. Please increase %s and/or %s in configuration.\n",
711 RTE_STR(CONFIG_RTE_MAX_MEMSEG_PER_TYPE
),
712 RTE_STR(CONFIG_RTE_MAX_MEM_PER_TYPE
));
716 #ifdef RTE_ARCH_PPC64
717 /* for PPC64 we go through the list backwards */
718 for (cur_page
= seg_end
- 1; cur_page
>= seg_start
;
719 cur_page
--, ms_idx
++) {
721 for (cur_page
= seg_start
; cur_page
< seg_end
; cur_page
++, ms_idx
++) {
723 struct hugepage_file
*hfile
= &hugepages
[cur_page
];
724 struct rte_memseg
*ms
= rte_fbarray_get(arr
, ms_idx
);
728 fd
= open(hfile
->filepath
, O_RDWR
);
730 RTE_LOG(ERR
, EAL
, "Could not open '%s': %s\n",
731 hfile
->filepath
, strerror(errno
));
734 /* set shared lock on the file. */
735 if (flock(fd
, LOCK_SH
) < 0) {
736 RTE_LOG(DEBUG
, EAL
, "Could not lock '%s': %s\n",
737 hfile
->filepath
, strerror(errno
));
741 memseg_len
= (size_t)page_sz
;
742 addr
= RTE_PTR_ADD(msl
->base_va
, ms_idx
* memseg_len
);
744 /* we know this address is already mmapped by memseg list, so
745 * using MAP_FIXED here is safe
747 addr
= mmap(addr
, page_sz
, PROT_READ
| PROT_WRITE
,
748 MAP_SHARED
| MAP_POPULATE
| MAP_FIXED
, fd
, 0);
749 if (addr
== MAP_FAILED
) {
750 RTE_LOG(ERR
, EAL
, "Couldn't remap '%s': %s\n",
751 hfile
->filepath
, strerror(errno
));
756 /* we have a new address, so unmap previous one */
758 /* in 32-bit legacy mode, we have already unmapped the page */
759 if (!internal_config
.legacy_mem
)
760 munmap(hfile
->orig_va
, page_sz
);
762 munmap(hfile
->orig_va
, page_sz
);
765 hfile
->orig_va
= NULL
;
766 hfile
->final_va
= addr
;
768 /* rewrite physical addresses in IOVA as VA mode */
769 if (rte_eal_iova_mode() == RTE_IOVA_VA
)
770 hfile
->physaddr
= (uintptr_t)addr
;
772 /* set up memseg data */
774 ms
->hugepage_sz
= page_sz
;
775 ms
->len
= memseg_len
;
776 ms
->iova
= hfile
->physaddr
;
777 ms
->socket_id
= hfile
->socket_id
;
778 ms
->nchannel
= rte_memory_get_nchannel();
779 ms
->nrank
= rte_memory_get_nrank();
781 rte_fbarray_set_used(arr
, ms_idx
);
783 /* store segment fd internally */
784 if (eal_memalloc_set_seg_fd(msl_idx
, ms_idx
, fd
) < 0)
785 RTE_LOG(ERR
, EAL
, "Could not store segment fd: %s\n",
786 rte_strerror(rte_errno
));
788 RTE_LOG(DEBUG
, EAL
, "Allocated %" PRIu64
"M on socket %i\n",
789 (seg_len
* page_sz
) >> 20, socket_id
);
794 get_mem_amount(uint64_t page_sz
, uint64_t max_mem
)
796 uint64_t area_sz
, max_pages
;
798 /* limit to RTE_MAX_MEMSEG_PER_LIST pages or RTE_MAX_MEM_MB_PER_LIST */
799 max_pages
= RTE_MAX_MEMSEG_PER_LIST
;
800 max_mem
= RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_LIST
<< 20, max_mem
);
802 area_sz
= RTE_MIN(page_sz
* max_pages
, max_mem
);
804 /* make sure the list isn't smaller than the page size */
805 area_sz
= RTE_MAX(area_sz
, page_sz
);
807 return RTE_ALIGN(area_sz
, page_sz
);
811 free_memseg_list(struct rte_memseg_list
*msl
)
813 if (rte_fbarray_destroy(&msl
->memseg_arr
)) {
814 RTE_LOG(ERR
, EAL
, "Cannot destroy memseg list\n");
817 memset(msl
, 0, sizeof(*msl
));
821 #define MEMSEG_LIST_FMT "memseg-%" PRIu64 "k-%i-%i"
823 alloc_memseg_list(struct rte_memseg_list
*msl
, uint64_t page_sz
,
824 int n_segs
, int socket_id
, int type_msl_idx
)
826 char name
[RTE_FBARRAY_NAME_LEN
];
828 snprintf(name
, sizeof(name
), MEMSEG_LIST_FMT
, page_sz
>> 10, socket_id
,
830 if (rte_fbarray_init(&msl
->memseg_arr
, name
, n_segs
,
831 sizeof(struct rte_memseg
))) {
832 RTE_LOG(ERR
, EAL
, "Cannot allocate memseg list: %s\n",
833 rte_strerror(rte_errno
));
837 msl
->page_sz
= page_sz
;
838 msl
->socket_id
= socket_id
;
841 RTE_LOG(DEBUG
, EAL
, "Memseg list allocated: 0x%zxkB at socket %i\n",
842 (size_t)page_sz
>> 10, socket_id
);
848 alloc_va_space(struct rte_memseg_list
*msl
)
855 page_sz
= msl
->page_sz
;
856 mem_sz
= page_sz
* msl
->memseg_arr
.len
;
858 addr
= eal_get_virtual_area(msl
->base_va
, &mem_sz
, page_sz
, 0, flags
);
860 if (rte_errno
== EADDRNOTAVAIL
)
861 RTE_LOG(ERR
, EAL
, "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n",
862 (unsigned long long)mem_sz
, msl
->base_va
);
864 RTE_LOG(ERR
, EAL
, "Cannot reserve memory\n");
874 * Our VA space is not preallocated yet, so preallocate it here. We need to know
875 * how many segments there are in order to map all pages into one address space,
876 * and leave appropriate holes between segments so that rte_malloc does not
877 * concatenate them into one big segment.
879 * we also need to unmap original pages to free up address space.
881 static int __rte_unused
882 prealloc_segments(struct hugepage_file
*hugepages
, int n_pages
)
884 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
885 int cur_page
, seg_start_page
, end_seg
, new_memseg
;
886 unsigned int hpi_idx
, socket
, i
;
887 int n_contig_segs
, n_segs
;
890 /* before we preallocate segments, we need to free up our VA space.
891 * we're not removing files, and we already have information about
892 * PA-contiguousness, so it is safe to unmap everything.
894 for (cur_page
= 0; cur_page
< n_pages
; cur_page
++) {
895 struct hugepage_file
*hpi
= &hugepages
[cur_page
];
896 munmap(hpi
->orig_va
, hpi
->size
);
900 /* we cannot know how many page sizes and sockets we have discovered, so
901 * loop over all of them
903 for (hpi_idx
= 0; hpi_idx
< internal_config
.num_hugepage_sizes
;
906 internal_config
.hugepage_info
[hpi_idx
].hugepage_sz
;
908 for (i
= 0; i
< rte_socket_count(); i
++) {
909 struct rte_memseg_list
*msl
;
911 socket
= rte_socket_id_by_idx(i
);
916 for (cur_page
= 0; cur_page
< n_pages
; cur_page
++) {
917 struct hugepage_file
*prev
, *cur
;
918 int prev_seg_start_page
= -1;
920 cur
= &hugepages
[cur_page
];
921 prev
= cur_page
== 0 ? NULL
:
922 &hugepages
[cur_page
- 1];
929 else if (cur
->socket_id
!= (int) socket
)
931 else if (cur
->size
!= page_sz
)
933 else if (cur_page
== 0)
935 #ifdef RTE_ARCH_PPC_64
936 /* On PPC64 architecture, the mmap always start
937 * from higher address to lower address. Here,
938 * physical addresses are in descending order.
940 else if ((prev
->physaddr
- cur
->physaddr
) !=
944 else if ((cur
->physaddr
- prev
->physaddr
) !=
949 /* if we're already inside a segment,
950 * new segment means end of current one
952 if (seg_start_page
!= -1) {
954 prev_seg_start_page
=
957 seg_start_page
= cur_page
;
961 if (prev_seg_start_page
!= -1) {
962 /* we've found a new segment */
966 } else if (seg_start_page
!= -1) {
967 /* we didn't find new segment,
968 * but did end current one
976 /* we're skipping this page */
980 /* segment continues */
982 /* check if we missed last segment */
983 if (seg_start_page
!= -1) {
985 n_segs
+= cur_page
- seg_start_page
;
988 /* if no segments were found, do not preallocate */
992 /* we now have total number of pages that we will
993 * allocate for this segment list. add separator pages
994 * to the total count, and preallocate VA space.
996 n_segs
+= n_contig_segs
- 1;
998 /* now, preallocate VA space for these segments */
1000 /* first, find suitable memseg list for this */
1001 for (msl_idx
= 0; msl_idx
< RTE_MAX_MEMSEG_LISTS
;
1003 msl
= &mcfg
->memsegs
[msl_idx
];
1005 if (msl
->base_va
!= NULL
)
1009 if (msl_idx
== RTE_MAX_MEMSEG_LISTS
) {
1010 RTE_LOG(ERR
, EAL
, "Not enough space in memseg lists, please increase %s\n",
1011 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS
));
1015 /* now, allocate fbarray itself */
1016 if (alloc_memseg_list(msl
, page_sz
, n_segs
, socket
,
1020 /* finally, allocate VA space */
1021 if (alloc_va_space(msl
) < 0)
1029 * We cannot reallocate memseg lists on the fly because PPC64 stores pages
1030 * backwards, therefore we have to process the entire memseg first before
1031 * remapping it into memseg list VA space.
1034 remap_needed_hugepages(struct hugepage_file
*hugepages
, int n_pages
)
1036 int cur_page
, seg_start_page
, new_memseg
, ret
;
1039 for (cur_page
= 0; cur_page
< n_pages
; cur_page
++) {
1040 struct hugepage_file
*prev
, *cur
;
1044 cur
= &hugepages
[cur_page
];
1045 prev
= cur_page
== 0 ? NULL
: &hugepages
[cur_page
- 1];
1047 /* if size is zero, no more pages left */
1053 else if (cur
->socket_id
!= prev
->socket_id
)
1055 else if (cur
->size
!= prev
->size
)
1057 #ifdef RTE_ARCH_PPC_64
1058 /* On PPC64 architecture, the mmap always start from higher
1059 * address to lower address. Here, physical addresses are in
1062 else if ((prev
->physaddr
- cur
->physaddr
) != cur
->size
)
1065 else if ((cur
->physaddr
- prev
->physaddr
) != cur
->size
)
1070 /* if this isn't the first time, remap segment */
1071 if (cur_page
!= 0) {
1072 ret
= remap_segment(hugepages
, seg_start_page
,
1077 /* remember where we started */
1078 seg_start_page
= cur_page
;
1080 /* continuation of previous memseg */
1082 /* we were stopped, but we didn't remap the last segment, do it now */
1083 if (cur_page
!= 0) {
1084 ret
= remap_segment(hugepages
, seg_start_page
,
1092 static inline uint64_t
1093 get_socket_mem_size(int socket
)
1098 for (i
= 0; i
< internal_config
.num_hugepage_sizes
; i
++){
1099 struct hugepage_info
*hpi
= &internal_config
.hugepage_info
[i
];
1100 size
+= hpi
->hugepage_sz
* hpi
->num_pages
[socket
];
1107 * This function is a NUMA-aware equivalent of calc_num_pages.
1108 * It takes in the list of hugepage sizes and the
1109 * number of pages thereof, and calculates the best number of
1110 * pages of each size to fulfill the request for <memory> ram
1113 calc_num_pages_per_socket(uint64_t * memory
,
1114 struct hugepage_info
*hp_info
,
1115 struct hugepage_info
*hp_used
,
1116 unsigned num_hp_info
)
1118 unsigned socket
, j
, i
= 0;
1119 unsigned requested
, available
;
1120 int total_num_pages
= 0;
1121 uint64_t remaining_mem
, cur_mem
;
1122 uint64_t total_mem
= internal_config
.memory
;
1124 if (num_hp_info
== 0)
1127 /* if specific memory amounts per socket weren't requested */
1128 if (internal_config
.force_sockets
== 0) {
1131 int cpu_per_socket
[RTE_MAX_NUMA_NODES
];
1132 size_t default_size
;
1135 /* Compute number of cores per socket */
1136 memset(cpu_per_socket
, 0, sizeof(cpu_per_socket
));
1137 RTE_LCORE_FOREACH(lcore_id
) {
1138 cpu_per_socket
[rte_lcore_to_socket_id(lcore_id
)]++;
1142 * Automatically spread requested memory amongst detected sockets according
1143 * to number of cores from cpu mask present on each socket
1145 total_size
= internal_config
.memory
;
1146 for (socket
= 0; socket
< RTE_MAX_NUMA_NODES
&& total_size
!= 0; socket
++) {
1148 /* Set memory amount per socket */
1149 default_size
= (internal_config
.memory
* cpu_per_socket
[socket
])
1150 / rte_lcore_count();
1152 /* Limit to maximum available memory on socket */
1153 default_size
= RTE_MIN(default_size
, get_socket_mem_size(socket
));
1156 memory
[socket
] = default_size
;
1157 total_size
-= default_size
;
1161 * If some memory is remaining, try to allocate it by getting all
1162 * available memory from sockets, one after the other
1164 for (socket
= 0; socket
< RTE_MAX_NUMA_NODES
&& total_size
!= 0; socket
++) {
1165 /* take whatever is available */
1166 default_size
= RTE_MIN(get_socket_mem_size(socket
) - memory
[socket
],
1170 memory
[socket
] += default_size
;
1171 total_size
-= default_size
;
1174 /* in 32-bit mode, allocate all of the memory only on master
1177 total_size
= internal_config
.memory
;
1178 for (socket
= 0; socket
< RTE_MAX_NUMA_NODES
&& total_size
!= 0;
1180 struct rte_config
*cfg
= rte_eal_get_configuration();
1181 unsigned int master_lcore_socket
;
1183 master_lcore_socket
=
1184 rte_lcore_to_socket_id(cfg
->master_lcore
);
1186 if (master_lcore_socket
!= socket
)
1190 memory
[socket
] = total_size
;
1196 for (socket
= 0; socket
< RTE_MAX_NUMA_NODES
&& total_mem
!= 0; socket
++) {
1197 /* skips if the memory on specific socket wasn't requested */
1198 for (i
= 0; i
< num_hp_info
&& memory
[socket
] != 0; i
++){
1199 strlcpy(hp_used
[i
].hugedir
, hp_info
[i
].hugedir
,
1200 sizeof(hp_used
[i
].hugedir
));
1201 hp_used
[i
].num_pages
[socket
] = RTE_MIN(
1202 memory
[socket
] / hp_info
[i
].hugepage_sz
,
1203 hp_info
[i
].num_pages
[socket
]);
1205 cur_mem
= hp_used
[i
].num_pages
[socket
] *
1206 hp_used
[i
].hugepage_sz
;
1208 memory
[socket
] -= cur_mem
;
1209 total_mem
-= cur_mem
;
1211 total_num_pages
+= hp_used
[i
].num_pages
[socket
];
1213 /* check if we have met all memory requests */
1214 if (memory
[socket
] == 0)
1217 /* check if we have any more pages left at this size, if so
1218 * move on to next size */
1219 if (hp_used
[i
].num_pages
[socket
] == hp_info
[i
].num_pages
[socket
])
1221 /* At this point we know that there are more pages available that are
1222 * bigger than the memory we want, so lets see if we can get enough
1223 * from other page sizes.
1226 for (j
= i
+1; j
< num_hp_info
; j
++)
1227 remaining_mem
+= hp_info
[j
].hugepage_sz
*
1228 hp_info
[j
].num_pages
[socket
];
1230 /* is there enough other memory, if not allocate another page and quit */
1231 if (remaining_mem
< memory
[socket
]){
1232 cur_mem
= RTE_MIN(memory
[socket
],
1233 hp_info
[i
].hugepage_sz
);
1234 memory
[socket
] -= cur_mem
;
1235 total_mem
-= cur_mem
;
1236 hp_used
[i
].num_pages
[socket
]++;
1238 break; /* we are done with this socket*/
1241 /* if we didn't satisfy all memory requirements per socket */
1242 if (memory
[socket
] > 0 &&
1243 internal_config
.socket_mem
[socket
] != 0) {
1244 /* to prevent icc errors */
1245 requested
= (unsigned) (internal_config
.socket_mem
[socket
] /
1247 available
= requested
-
1248 ((unsigned) (memory
[socket
] / 0x100000));
1249 RTE_LOG(ERR
, EAL
, "Not enough memory available on socket %u! "
1250 "Requested: %uMB, available: %uMB\n", socket
,
1251 requested
, available
);
1256 /* if we didn't satisfy total memory requirements */
1257 if (total_mem
> 0) {
1258 requested
= (unsigned) (internal_config
.memory
/ 0x100000);
1259 available
= requested
- (unsigned) (total_mem
/ 0x100000);
1260 RTE_LOG(ERR
, EAL
, "Not enough memory available! Requested: %uMB,"
1261 " available: %uMB\n", requested
, available
);
1264 return total_num_pages
;
1267 static inline size_t
1268 eal_get_hugepage_mem_size(void)
1273 for (i
= 0; i
< internal_config
.num_hugepage_sizes
; i
++) {
1274 struct hugepage_info
*hpi
= &internal_config
.hugepage_info
[i
];
1275 if (strnlen(hpi
->hugedir
, sizeof(hpi
->hugedir
)) != 0) {
1276 for (j
= 0; j
< RTE_MAX_NUMA_NODES
; j
++) {
1277 size
+= hpi
->hugepage_sz
* hpi
->num_pages
[j
];
1282 return (size
< SIZE_MAX
) ? (size_t)(size
) : SIZE_MAX
;
1285 static struct sigaction huge_action_old
;
1286 static int huge_need_recover
;
1289 huge_register_sigbus(void)
1292 struct sigaction action
;
1295 sigaddset(&mask
, SIGBUS
);
1296 action
.sa_flags
= 0;
1297 action
.sa_mask
= mask
;
1298 action
.sa_handler
= huge_sigbus_handler
;
1300 huge_need_recover
= !sigaction(SIGBUS
, &action
, &huge_action_old
);
1304 huge_recover_sigbus(void)
1306 if (huge_need_recover
) {
1307 sigaction(SIGBUS
, &huge_action_old
, NULL
);
1308 huge_need_recover
= 0;
1313 * Prepare physical memory mapping: fill configuration structure with
1314 * these infos, return 0 on success.
1315 * 1. map N huge pages in separate files in hugetlbfs
1316 * 2. find associated physical addr
1317 * 3. find associated NUMA socket ID
1318 * 4. sort all huge pages by physical address
1319 * 5. remap these N huge pages in the correct order
1320 * 6. unmap the first mapping
1321 * 7. fill memsegs in configuration with contiguous zones
1324 eal_legacy_hugepage_init(void)
1326 struct rte_mem_config
*mcfg
;
1327 struct hugepage_file
*hugepage
= NULL
, *tmp_hp
= NULL
;
1328 struct hugepage_info used_hp
[MAX_HUGEPAGE_SIZES
];
1329 struct rte_fbarray
*arr
;
1330 struct rte_memseg
*ms
;
1332 uint64_t memory
[RTE_MAX_NUMA_NODES
];
1336 int nr_hugefiles
, nr_hugepages
= 0;
1339 test_phys_addrs_available();
1341 memset(used_hp
, 0, sizeof(used_hp
));
1343 /* get pointer to global configuration */
1344 mcfg
= rte_eal_get_configuration()->mem_config
;
1346 /* hugetlbfs can be disabled */
1347 if (internal_config
.no_hugetlbfs
) {
1348 struct rte_memseg_list
*msl
;
1349 int n_segs
, cur_seg
, fd
, flags
;
1350 #ifdef MEMFD_SUPPORTED
1355 /* nohuge mode is legacy mode */
1356 internal_config
.legacy_mem
= 1;
1358 /* nohuge mode is single-file segments mode */
1359 internal_config
.single_file_segments
= 1;
1361 /* create a memseg list */
1362 msl
= &mcfg
->memsegs
[0];
1364 page_sz
= RTE_PGSIZE_4K
;
1365 n_segs
= internal_config
.memory
/ page_sz
;
1367 if (rte_fbarray_init(&msl
->memseg_arr
, "nohugemem", n_segs
,
1368 sizeof(struct rte_memseg
))) {
1369 RTE_LOG(ERR
, EAL
, "Cannot allocate memseg list\n");
1373 /* set up parameters for anonymous mmap */
1375 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
1377 #ifdef MEMFD_SUPPORTED
1378 /* create a memfd and store it in the segment fd table */
1379 memfd
= memfd_create("nohuge", 0);
1381 RTE_LOG(DEBUG
, EAL
, "Cannot create memfd: %s\n",
1383 RTE_LOG(DEBUG
, EAL
, "Falling back to anonymous map\n");
1385 /* we got an fd - now resize it */
1386 if (ftruncate(memfd
, internal_config
.memory
) < 0) {
1387 RTE_LOG(ERR
, EAL
, "Cannot resize memfd: %s\n",
1389 RTE_LOG(ERR
, EAL
, "Falling back to anonymous map\n");
1392 /* creating memfd-backed file was successful.
1393 * we want changes to memfd to be visible to
1394 * other processes (such as vhost backend), so
1395 * map it as shared memory.
1397 RTE_LOG(DEBUG
, EAL
, "Using memfd for anonymous memory\n");
1403 addr
= mmap(NULL
, internal_config
.memory
, PROT_READ
| PROT_WRITE
,
1405 if (addr
== MAP_FAILED
) {
1406 RTE_LOG(ERR
, EAL
, "%s: mmap() failed: %s\n", __func__
,
1410 msl
->base_va
= addr
;
1411 msl
->page_sz
= page_sz
;
1413 msl
->len
= internal_config
.memory
;
1415 /* we're in single-file segments mode, so only the segment list
1416 * fd needs to be set up.
1419 if (eal_memalloc_set_seg_list_fd(0, fd
) < 0) {
1420 RTE_LOG(ERR
, EAL
, "Cannot set up segment list fd\n");
1421 /* not a serious error, proceed */
1425 /* populate memsegs. each memseg is one page long */
1426 for (cur_seg
= 0; cur_seg
< n_segs
; cur_seg
++) {
1427 arr
= &msl
->memseg_arr
;
1429 ms
= rte_fbarray_get(arr
, cur_seg
);
1430 if (rte_eal_iova_mode() == RTE_IOVA_VA
)
1431 ms
->iova
= (uintptr_t)addr
;
1433 ms
->iova
= RTE_BAD_IOVA
;
1435 ms
->hugepage_sz
= page_sz
;
1439 rte_fbarray_set_used(arr
, cur_seg
);
1441 addr
= RTE_PTR_ADD(addr
, (size_t)page_sz
);
1443 if (mcfg
->dma_maskbits
&&
1444 rte_mem_check_dma_mask_thread_unsafe(mcfg
->dma_maskbits
)) {
1446 "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n",
1448 if (rte_eal_iova_mode() == RTE_IOVA_VA
&&
1449 rte_eal_using_phys_addrs())
1451 "%s(): Please try initializing EAL with --iova-mode=pa parameter.\n",
1458 /* calculate total number of hugepages available. at this point we haven't
1459 * yet started sorting them so they all are on socket 0 */
1460 for (i
= 0; i
< (int) internal_config
.num_hugepage_sizes
; i
++) {
1461 /* meanwhile, also initialize used_hp hugepage sizes in used_hp */
1462 used_hp
[i
].hugepage_sz
= internal_config
.hugepage_info
[i
].hugepage_sz
;
1464 nr_hugepages
+= internal_config
.hugepage_info
[i
].num_pages
[0];
1468 * allocate a memory area for hugepage table.
1469 * this isn't shared memory yet. due to the fact that we need some
1470 * processing done on these pages, shared memory will be created
1473 tmp_hp
= malloc(nr_hugepages
* sizeof(struct hugepage_file
));
1477 memset(tmp_hp
, 0, nr_hugepages
* sizeof(struct hugepage_file
));
1479 hp_offset
= 0; /* where we start the current page size entries */
1481 huge_register_sigbus();
1483 /* make a copy of socket_mem, needed for balanced allocation. */
1484 for (i
= 0; i
< RTE_MAX_NUMA_NODES
; i
++)
1485 memory
[i
] = internal_config
.socket_mem
[i
];
1487 /* map all hugepages and sort them */
1488 for (i
= 0; i
< (int)internal_config
.num_hugepage_sizes
; i
++){
1489 unsigned pages_old
, pages_new
;
1490 struct hugepage_info
*hpi
;
1493 * we don't yet mark hugepages as used at this stage, so
1494 * we just map all hugepages available to the system
1495 * all hugepages are still located on socket 0
1497 hpi
= &internal_config
.hugepage_info
[i
];
1499 if (hpi
->num_pages
[0] == 0)
1502 /* map all hugepages available */
1503 pages_old
= hpi
->num_pages
[0];
1504 pages_new
= map_all_hugepages(&tmp_hp
[hp_offset
], hpi
, memory
);
1505 if (pages_new
< pages_old
) {
1507 "%d not %d hugepages of size %u MB allocated\n",
1508 pages_new
, pages_old
,
1509 (unsigned)(hpi
->hugepage_sz
/ 0x100000));
1511 int pages
= pages_old
- pages_new
;
1513 nr_hugepages
-= pages
;
1514 hpi
->num_pages
[0] = pages_new
;
1519 if (phys_addrs_available
&&
1520 rte_eal_iova_mode() != RTE_IOVA_VA
) {
1521 /* find physical addresses for each hugepage */
1522 if (find_physaddrs(&tmp_hp
[hp_offset
], hpi
) < 0) {
1523 RTE_LOG(DEBUG
, EAL
, "Failed to find phys addr "
1524 "for %u MB pages\n",
1525 (unsigned int)(hpi
->hugepage_sz
/ 0x100000));
1529 /* set physical addresses for each hugepage */
1530 if (set_physaddrs(&tmp_hp
[hp_offset
], hpi
) < 0) {
1531 RTE_LOG(DEBUG
, EAL
, "Failed to set phys addr "
1532 "for %u MB pages\n",
1533 (unsigned int)(hpi
->hugepage_sz
/ 0x100000));
1538 if (find_numasocket(&tmp_hp
[hp_offset
], hpi
) < 0){
1539 RTE_LOG(DEBUG
, EAL
, "Failed to find NUMA socket for %u MB pages\n",
1540 (unsigned)(hpi
->hugepage_sz
/ 0x100000));
1544 qsort(&tmp_hp
[hp_offset
], hpi
->num_pages
[0],
1545 sizeof(struct hugepage_file
), cmp_physaddr
);
1547 /* we have processed a num of hugepages of this size, so inc offset */
1548 hp_offset
+= hpi
->num_pages
[0];
1551 huge_recover_sigbus();
1553 if (internal_config
.memory
== 0 && internal_config
.force_sockets
== 0)
1554 internal_config
.memory
= eal_get_hugepage_mem_size();
1556 nr_hugefiles
= nr_hugepages
;
1559 /* clean out the numbers of pages */
1560 for (i
= 0; i
< (int) internal_config
.num_hugepage_sizes
; i
++)
1561 for (j
= 0; j
< RTE_MAX_NUMA_NODES
; j
++)
1562 internal_config
.hugepage_info
[i
].num_pages
[j
] = 0;
1564 /* get hugepages for each socket */
1565 for (i
= 0; i
< nr_hugefiles
; i
++) {
1566 int socket
= tmp_hp
[i
].socket_id
;
1568 /* find a hugepage info with right size and increment num_pages */
1569 const int nb_hpsizes
= RTE_MIN(MAX_HUGEPAGE_SIZES
,
1570 (int)internal_config
.num_hugepage_sizes
);
1571 for (j
= 0; j
< nb_hpsizes
; j
++) {
1572 if (tmp_hp
[i
].size
==
1573 internal_config
.hugepage_info
[j
].hugepage_sz
) {
1574 internal_config
.hugepage_info
[j
].num_pages
[socket
]++;
1579 /* make a copy of socket_mem, needed for number of pages calculation */
1580 for (i
= 0; i
< RTE_MAX_NUMA_NODES
; i
++)
1581 memory
[i
] = internal_config
.socket_mem
[i
];
1583 /* calculate final number of pages */
1584 nr_hugepages
= calc_num_pages_per_socket(memory
,
1585 internal_config
.hugepage_info
, used_hp
,
1586 internal_config
.num_hugepage_sizes
);
1588 /* error if not enough memory available */
1589 if (nr_hugepages
< 0)
1593 for (i
= 0; i
< (int) internal_config
.num_hugepage_sizes
; i
++) {
1594 for (j
= 0; j
< RTE_MAX_NUMA_NODES
; j
++) {
1595 if (used_hp
[i
].num_pages
[j
] > 0) {
1597 "Requesting %u pages of size %uMB"
1598 " from socket %i\n",
1599 used_hp
[i
].num_pages
[j
],
1601 (used_hp
[i
].hugepage_sz
/ 0x100000),
1607 /* create shared memory */
1608 hugepage
= create_shared_memory(eal_hugepage_data_path(),
1609 nr_hugefiles
* sizeof(struct hugepage_file
));
1611 if (hugepage
== NULL
) {
1612 RTE_LOG(ERR
, EAL
, "Failed to create shared memory!\n");
1615 memset(hugepage
, 0, nr_hugefiles
* sizeof(struct hugepage_file
));
1618 * unmap pages that we won't need (looks at used_hp).
1619 * also, sets final_va to NULL on pages that were unmapped.
1621 if (unmap_unneeded_hugepages(tmp_hp
, used_hp
,
1622 internal_config
.num_hugepage_sizes
) < 0) {
1623 RTE_LOG(ERR
, EAL
, "Unmapping and locking hugepages failed!\n");
1628 * copy stuff from malloc'd hugepage* to the actual shared memory.
1629 * this procedure only copies those hugepages that have orig_va
1630 * not NULL. has overflow protection.
1632 if (copy_hugepages_to_shared_mem(hugepage
, nr_hugefiles
,
1633 tmp_hp
, nr_hugefiles
) < 0) {
1634 RTE_LOG(ERR
, EAL
, "Copying tables to shared memory failed!\n");
1639 /* for legacy 32-bit mode, we did not preallocate VA space, so do it */
1640 if (internal_config
.legacy_mem
&&
1641 prealloc_segments(hugepage
, nr_hugefiles
)) {
1642 RTE_LOG(ERR
, EAL
, "Could not preallocate VA space for hugepages\n");
1647 /* remap all pages we do need into memseg list VA space, so that those
1648 * pages become first-class citizens in DPDK memory subsystem
1650 if (remap_needed_hugepages(hugepage
, nr_hugefiles
)) {
1651 RTE_LOG(ERR
, EAL
, "Couldn't remap hugepage files into memseg lists\n");
1655 /* free the hugepage backing files */
1656 if (internal_config
.hugepage_unlink
&&
1657 unlink_hugepage_files(tmp_hp
, internal_config
.num_hugepage_sizes
) < 0) {
1658 RTE_LOG(ERR
, EAL
, "Unlinking hugepage files failed!\n");
1662 /* free the temporary hugepage table */
1666 munmap(hugepage
, nr_hugefiles
* sizeof(struct hugepage_file
));
1669 /* we're not going to allocate more pages, so release VA space for
1670 * unused memseg lists
1672 for (i
= 0; i
< RTE_MAX_MEMSEG_LISTS
; i
++) {
1673 struct rte_memseg_list
*msl
= &mcfg
->memsegs
[i
];
1676 /* skip inactive lists */
1677 if (msl
->base_va
== NULL
)
1679 /* skip lists where there is at least one page allocated */
1680 if (msl
->memseg_arr
.count
> 0)
1682 /* this is an unused list, deallocate it */
1684 munmap(msl
->base_va
, mem_sz
);
1685 msl
->base_va
= NULL
;
1687 /* destroy backing fbarray */
1688 rte_fbarray_destroy(&msl
->memseg_arr
);
1691 if (mcfg
->dma_maskbits
&&
1692 rte_mem_check_dma_mask_thread_unsafe(mcfg
->dma_maskbits
)) {
1694 "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n",
1702 huge_recover_sigbus();
1704 if (hugepage
!= NULL
)
1705 munmap(hugepage
, nr_hugefiles
* sizeof(struct hugepage_file
));
1710 static int __rte_unused
1711 hugepage_count_walk(const struct rte_memseg_list
*msl
, void *arg
)
1713 struct hugepage_info
*hpi
= arg
;
1715 if (msl
->page_sz
!= hpi
->hugepage_sz
)
1718 hpi
->num_pages
[msl
->socket_id
] += msl
->memseg_arr
.len
;
1723 limits_callback(int socket_id
, size_t cur_limit
, size_t new_len
)
1725 RTE_SET_USED(socket_id
);
1726 RTE_SET_USED(cur_limit
);
1727 RTE_SET_USED(new_len
);
1732 eal_hugepage_init(void)
1734 struct hugepage_info used_hp
[MAX_HUGEPAGE_SIZES
];
1735 uint64_t memory
[RTE_MAX_NUMA_NODES
];
1736 int hp_sz_idx
, socket_id
;
1738 test_phys_addrs_available();
1740 memset(used_hp
, 0, sizeof(used_hp
));
1743 hp_sz_idx
< (int) internal_config
.num_hugepage_sizes
;
1746 struct hugepage_info dummy
;
1749 /* also initialize used_hp hugepage sizes in used_hp */
1750 struct hugepage_info
*hpi
;
1751 hpi
= &internal_config
.hugepage_info
[hp_sz_idx
];
1752 used_hp
[hp_sz_idx
].hugepage_sz
= hpi
->hugepage_sz
;
1755 /* for 32-bit, limit number of pages on socket to whatever we've
1756 * preallocated, as we cannot allocate more.
1758 memset(&dummy
, 0, sizeof(dummy
));
1759 dummy
.hugepage_sz
= hpi
->hugepage_sz
;
1760 if (rte_memseg_list_walk(hugepage_count_walk
, &dummy
) < 0)
1763 for (i
= 0; i
< RTE_DIM(dummy
.num_pages
); i
++) {
1764 hpi
->num_pages
[i
] = RTE_MIN(hpi
->num_pages
[i
],
1765 dummy
.num_pages
[i
]);
1770 /* make a copy of socket_mem, needed for balanced allocation. */
1771 for (hp_sz_idx
= 0; hp_sz_idx
< RTE_MAX_NUMA_NODES
; hp_sz_idx
++)
1772 memory
[hp_sz_idx
] = internal_config
.socket_mem
[hp_sz_idx
];
1774 /* calculate final number of pages */
1775 if (calc_num_pages_per_socket(memory
,
1776 internal_config
.hugepage_info
, used_hp
,
1777 internal_config
.num_hugepage_sizes
) < 0)
1781 hp_sz_idx
< (int)internal_config
.num_hugepage_sizes
;
1783 for (socket_id
= 0; socket_id
< RTE_MAX_NUMA_NODES
;
1785 struct rte_memseg
**pages
;
1786 struct hugepage_info
*hpi
= &used_hp
[hp_sz_idx
];
1787 unsigned int num_pages
= hpi
->num_pages
[socket_id
];
1788 unsigned int num_pages_alloc
;
1793 RTE_LOG(DEBUG
, EAL
, "Allocating %u pages of size %" PRIu64
"M on socket %i\n",
1794 num_pages
, hpi
->hugepage_sz
>> 20, socket_id
);
1796 /* we may not be able to allocate all pages in one go,
1797 * because we break up our memory map into multiple
1798 * memseg lists. therefore, try allocating multiple
1799 * times and see if we can get the desired number of
1800 * pages from multiple allocations.
1803 num_pages_alloc
= 0;
1805 int i
, cur_pages
, needed
;
1807 needed
= num_pages
- num_pages_alloc
;
1809 pages
= malloc(sizeof(*pages
) * needed
);
1811 /* do not request exact number of pages */
1812 cur_pages
= eal_memalloc_alloc_seg_bulk(pages
,
1813 needed
, hpi
->hugepage_sz
,
1815 if (cur_pages
<= 0) {
1820 /* mark preallocated pages as unfreeable */
1821 for (i
= 0; i
< cur_pages
; i
++) {
1822 struct rte_memseg
*ms
= pages
[i
];
1823 ms
->flags
|= RTE_MEMSEG_FLAG_DO_NOT_FREE
;
1827 num_pages_alloc
+= cur_pages
;
1828 } while (num_pages_alloc
!= num_pages
);
1831 /* if socket limits were specified, set them */
1832 if (internal_config
.force_socket_limits
) {
1834 for (i
= 0; i
< RTE_MAX_NUMA_NODES
; i
++) {
1835 uint64_t limit
= internal_config
.socket_limit
[i
];
1838 if (rte_mem_alloc_validator_register("socket-limit",
1839 limits_callback
, i
, limit
))
1840 RTE_LOG(ERR
, EAL
, "Failed to register socket limits validator callback\n");
1847 * uses fstat to report the size of a file on disk
1853 if (fstat(fd
, &st
) < 0)
1859 * This creates the memory mappings in the secondary process to match that of
1860 * the server process. It goes through each memory segment in the DPDK runtime
1861 * configuration and finds the hugepages which form that segment, mapping them
1862 * in order to form a contiguous block in the virtual memory space
1865 eal_legacy_hugepage_attach(void)
1867 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
1868 struct hugepage_file
*hp
= NULL
;
1869 unsigned int num_hp
= 0;
1871 unsigned int cur_seg
;
1873 int fd
, fd_hugepage
= -1;
1875 if (aslr_enabled() > 0) {
1876 RTE_LOG(WARNING
, EAL
, "WARNING: Address Space Layout Randomization "
1877 "(ASLR) is enabled in the kernel.\n");
1878 RTE_LOG(WARNING
, EAL
, " This may cause issues with mapping memory "
1879 "into secondary processes\n");
1882 test_phys_addrs_available();
1884 fd_hugepage
= open(eal_hugepage_data_path(), O_RDONLY
);
1885 if (fd_hugepage
< 0) {
1886 RTE_LOG(ERR
, EAL
, "Could not open %s\n",
1887 eal_hugepage_data_path());
1891 size
= getFileSize(fd_hugepage
);
1892 hp
= mmap(NULL
, size
, PROT_READ
, MAP_PRIVATE
, fd_hugepage
, 0);
1893 if (hp
== MAP_FAILED
) {
1894 RTE_LOG(ERR
, EAL
, "Could not mmap %s\n",
1895 eal_hugepage_data_path());
1899 num_hp
= size
/ sizeof(struct hugepage_file
);
1900 RTE_LOG(DEBUG
, EAL
, "Analysing %u files\n", num_hp
);
1902 /* map all segments into memory to make sure we get the addrs. the
1903 * segments themselves are already in memseg list (which is shared and
1904 * has its VA space already preallocated), so we just need to map
1905 * everything into correct addresses.
1907 for (i
= 0; i
< num_hp
; i
++) {
1908 struct hugepage_file
*hf
= &hp
[i
];
1909 size_t map_sz
= hf
->size
;
1910 void *map_addr
= hf
->final_va
;
1911 int msl_idx
, ms_idx
;
1912 struct rte_memseg_list
*msl
;
1913 struct rte_memseg
*ms
;
1915 /* if size is zero, no more pages left */
1919 fd
= open(hf
->filepath
, O_RDWR
);
1921 RTE_LOG(ERR
, EAL
, "Could not open %s: %s\n",
1922 hf
->filepath
, strerror(errno
));
1926 map_addr
= mmap(map_addr
, map_sz
, PROT_READ
| PROT_WRITE
,
1927 MAP_SHARED
| MAP_FIXED
, fd
, 0);
1928 if (map_addr
== MAP_FAILED
) {
1929 RTE_LOG(ERR
, EAL
, "Could not map %s: %s\n",
1930 hf
->filepath
, strerror(errno
));
1934 /* set shared lock on the file. */
1935 if (flock(fd
, LOCK_SH
) < 0) {
1936 RTE_LOG(DEBUG
, EAL
, "%s(): Locking file failed: %s\n",
1937 __func__
, strerror(errno
));
1941 /* find segment data */
1942 msl
= rte_mem_virt2memseg_list(map_addr
);
1944 RTE_LOG(DEBUG
, EAL
, "%s(): Cannot find memseg list\n",
1948 ms
= rte_mem_virt2memseg(map_addr
, msl
);
1950 RTE_LOG(DEBUG
, EAL
, "%s(): Cannot find memseg\n",
1955 msl_idx
= msl
- mcfg
->memsegs
;
1956 ms_idx
= rte_fbarray_find_idx(&msl
->memseg_arr
, ms
);
1958 RTE_LOG(DEBUG
, EAL
, "%s(): Cannot find memseg idx\n",
1963 /* store segment fd internally */
1964 if (eal_memalloc_set_seg_fd(msl_idx
, ms_idx
, fd
) < 0)
1965 RTE_LOG(ERR
, EAL
, "Could not store segment fd: %s\n",
1966 rte_strerror(rte_errno
));
1968 /* unmap the hugepage config file, since we are done using it */
1976 /* map all segments into memory to make sure we get the addrs */
1978 for (cur_seg
= 0; cur_seg
< i
; cur_seg
++) {
1979 struct hugepage_file
*hf
= &hp
[i
];
1980 size_t map_sz
= hf
->size
;
1981 void *map_addr
= hf
->final_va
;
1983 munmap(map_addr
, map_sz
);
1985 if (hp
!= NULL
&& hp
!= MAP_FAILED
)
1987 if (fd_hugepage
>= 0)
1993 eal_hugepage_attach(void)
1995 if (eal_memalloc_sync_with_primary()) {
1996 RTE_LOG(ERR
, EAL
, "Could not map memory from primary process\n");
1997 if (aslr_enabled() > 0)
1998 RTE_LOG(ERR
, EAL
, "It is recommended to disable ASLR in the kernel and retry running both primary and secondary processes\n");
2005 rte_eal_hugepage_init(void)
2007 return internal_config
.legacy_mem
?
2008 eal_legacy_hugepage_init() :
2009 eal_hugepage_init();
2013 rte_eal_hugepage_attach(void)
2015 return internal_config
.legacy_mem
?
2016 eal_legacy_hugepage_attach() :
2017 eal_hugepage_attach();
2021 rte_eal_using_phys_addrs(void)
2023 return phys_addrs_available
;
2026 static int __rte_unused
2027 memseg_primary_init_32(void)
2029 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
2030 int active_sockets
, hpi_idx
, msl_idx
= 0;
2031 unsigned int socket_id
, i
;
2032 struct rte_memseg_list
*msl
;
2033 uint64_t extra_mem_per_socket
, total_extra_mem
, total_requested_mem
;
2036 /* no-huge does not need this at all */
2037 if (internal_config
.no_hugetlbfs
)
2040 /* this is a giant hack, but desperate times call for desperate
2041 * measures. in legacy 32-bit mode, we cannot preallocate VA space,
2042 * because having upwards of 2 gigabytes of VA space already mapped will
2043 * interfere with our ability to map and sort hugepages.
2045 * therefore, in legacy 32-bit mode, we will be initializing memseg
2046 * lists much later - in eal_memory.c, right after we unmap all the
2047 * unneeded pages. this will not affect secondary processes, as those
2048 * should be able to mmap the space without (too many) problems.
2050 if (internal_config
.legacy_mem
)
2053 /* 32-bit mode is a very special case. we cannot know in advance where
2054 * the user will want to allocate their memory, so we have to do some
2058 total_requested_mem
= 0;
2059 if (internal_config
.force_sockets
)
2060 for (i
= 0; i
< rte_socket_count(); i
++) {
2063 socket_id
= rte_socket_id_by_idx(i
);
2064 mem
= internal_config
.socket_mem
[socket_id
];
2070 total_requested_mem
+= mem
;
2073 total_requested_mem
= internal_config
.memory
;
2075 max_mem
= (uint64_t)RTE_MAX_MEM_MB
<< 20;
2076 if (total_requested_mem
> max_mem
) {
2077 RTE_LOG(ERR
, EAL
, "Invalid parameters: 32-bit process can at most use %uM of memory\n",
2078 (unsigned int)(max_mem
>> 20));
2081 total_extra_mem
= max_mem
- total_requested_mem
;
2082 extra_mem_per_socket
= active_sockets
== 0 ? total_extra_mem
:
2083 total_extra_mem
/ active_sockets
;
2085 /* the allocation logic is a little bit convoluted, but here's how it
2086 * works, in a nutshell:
2087 * - if user hasn't specified on which sockets to allocate memory via
2088 * --socket-mem, we allocate all of our memory on master core socket.
2089 * - if user has specified sockets to allocate memory on, there may be
2090 * some "unused" memory left (e.g. if user has specified --socket-mem
2091 * such that not all memory adds up to 2 gigabytes), so add it to all
2092 * sockets that are in use equally.
2094 * page sizes are sorted by size in descending order, so we can safely
2095 * assume that we dispense with bigger page sizes first.
2098 /* create memseg lists */
2099 for (i
= 0; i
< rte_socket_count(); i
++) {
2100 int hp_sizes
= (int) internal_config
.num_hugepage_sizes
;
2101 uint64_t max_socket_mem
, cur_socket_mem
;
2102 unsigned int master_lcore_socket
;
2103 struct rte_config
*cfg
= rte_eal_get_configuration();
2106 socket_id
= rte_socket_id_by_idx(i
);
2108 #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
2109 /* we can still sort pages by socket in legacy mode */
2110 if (!internal_config
.legacy_mem
&& socket_id
> 0)
2114 /* if we didn't specifically request memory on this socket */
2115 skip
= active_sockets
!= 0 &&
2116 internal_config
.socket_mem
[socket_id
] == 0;
2117 /* ...or if we didn't specifically request memory on *any*
2118 * socket, and this is not master lcore
2120 master_lcore_socket
= rte_lcore_to_socket_id(cfg
->master_lcore
);
2121 skip
|= active_sockets
== 0 && socket_id
!= master_lcore_socket
;
2124 RTE_LOG(DEBUG
, EAL
, "Will not preallocate memory on socket %u\n",
2129 /* max amount of memory on this socket */
2130 max_socket_mem
= (active_sockets
!= 0 ?
2131 internal_config
.socket_mem
[socket_id
] :
2132 internal_config
.memory
) +
2133 extra_mem_per_socket
;
2136 for (hpi_idx
= 0; hpi_idx
< hp_sizes
; hpi_idx
++) {
2137 uint64_t max_pagesz_mem
, cur_pagesz_mem
= 0;
2138 uint64_t hugepage_sz
;
2139 struct hugepage_info
*hpi
;
2140 int type_msl_idx
, max_segs
, total_segs
= 0;
2142 hpi
= &internal_config
.hugepage_info
[hpi_idx
];
2143 hugepage_sz
= hpi
->hugepage_sz
;
2145 /* check if pages are actually available */
2146 if (hpi
->num_pages
[socket_id
] == 0)
2149 max_segs
= RTE_MAX_MEMSEG_PER_TYPE
;
2150 max_pagesz_mem
= max_socket_mem
- cur_socket_mem
;
2152 /* make it multiple of page size */
2153 max_pagesz_mem
= RTE_ALIGN_FLOOR(max_pagesz_mem
,
2156 RTE_LOG(DEBUG
, EAL
, "Attempting to preallocate "
2157 "%" PRIu64
"M on socket %i\n",
2158 max_pagesz_mem
>> 20, socket_id
);
2161 while (cur_pagesz_mem
< max_pagesz_mem
&&
2162 total_segs
< max_segs
) {
2164 unsigned int n_segs
;
2166 if (msl_idx
>= RTE_MAX_MEMSEG_LISTS
) {
2168 "No more space in memseg lists, please increase %s\n",
2169 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS
));
2173 msl
= &mcfg
->memsegs
[msl_idx
];
2175 cur_mem
= get_mem_amount(hugepage_sz
,
2177 n_segs
= cur_mem
/ hugepage_sz
;
2179 if (alloc_memseg_list(msl
, hugepage_sz
, n_segs
,
2180 socket_id
, type_msl_idx
)) {
2181 /* failing to allocate a memseg list is
2184 RTE_LOG(ERR
, EAL
, "Cannot allocate memseg list\n");
2188 if (alloc_va_space(msl
)) {
2189 /* if we couldn't allocate VA space, we
2190 * can try with smaller page sizes.
2192 RTE_LOG(ERR
, EAL
, "Cannot allocate VA space for memseg list, retrying with different page size\n");
2193 /* deallocate memseg list */
2194 if (free_memseg_list(msl
))
2199 total_segs
+= msl
->memseg_arr
.len
;
2200 cur_pagesz_mem
= total_segs
* hugepage_sz
;
2204 cur_socket_mem
+= cur_pagesz_mem
;
2206 if (cur_socket_mem
== 0) {
2207 RTE_LOG(ERR
, EAL
, "Cannot allocate VA space on socket %u\n",
2216 static int __rte_unused
2217 memseg_primary_init(void)
2219 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
2224 int i
, hpi_idx
, msl_idx
, ret
= -1; /* fail unless told to succeed */
2225 struct rte_memseg_list
*msl
;
2226 uint64_t max_mem
, max_mem_per_type
;
2227 unsigned int max_seglists_per_type
;
2228 unsigned int n_memtypes
, cur_type
;
2230 /* no-huge does not need this at all */
2231 if (internal_config
.no_hugetlbfs
)
2235 * figuring out amount of memory we're going to have is a long and very
2236 * involved process. the basic element we're operating with is a memory
2237 * type, defined as a combination of NUMA node ID and page size (so that
2238 * e.g. 2 sockets with 2 page sizes yield 4 memory types in total).
2240 * deciding amount of memory going towards each memory type is a
2241 * balancing act between maximum segments per type, maximum memory per
2242 * type, and number of detected NUMA nodes. the goal is to make sure
2243 * each memory type gets at least one memseg list.
2245 * the total amount of memory is limited by RTE_MAX_MEM_MB value.
2247 * the total amount of memory per type is limited by either
2248 * RTE_MAX_MEM_MB_PER_TYPE, or by RTE_MAX_MEM_MB divided by the number
2249 * of detected NUMA nodes. additionally, maximum number of segments per
2250 * type is also limited by RTE_MAX_MEMSEG_PER_TYPE. this is because for
2251 * smaller page sizes, it can take hundreds of thousands of segments to
2252 * reach the above specified per-type memory limits.
2254 * additionally, each type may have multiple memseg lists associated
2255 * with it, each limited by either RTE_MAX_MEM_MB_PER_LIST for bigger
2256 * page sizes, or RTE_MAX_MEMSEG_PER_LIST segments for smaller ones.
2258 * the number of memseg lists per type is decided based on the above
2259 * limits, and also taking number of detected NUMA nodes, to make sure
2260 * that we don't run out of memseg lists before we populate all NUMA
2261 * nodes with memory.
2263 * we do this in three stages. first, we collect the number of types.
2264 * then, we figure out memory constraints and populate the list of
2265 * would-be memseg lists. then, we go ahead and allocate the memseg
2269 /* create space for mem types */
2270 n_memtypes
= internal_config
.num_hugepage_sizes
* rte_socket_count();
2271 memtypes
= calloc(n_memtypes
, sizeof(*memtypes
));
2272 if (memtypes
== NULL
) {
2273 RTE_LOG(ERR
, EAL
, "Cannot allocate space for memory types\n");
2277 /* populate mem types */
2279 for (hpi_idx
= 0; hpi_idx
< (int) internal_config
.num_hugepage_sizes
;
2281 struct hugepage_info
*hpi
;
2282 uint64_t hugepage_sz
;
2284 hpi
= &internal_config
.hugepage_info
[hpi_idx
];
2285 hugepage_sz
= hpi
->hugepage_sz
;
2287 for (i
= 0; i
< (int) rte_socket_count(); i
++, cur_type
++) {
2288 int socket_id
= rte_socket_id_by_idx(i
);
2290 #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
2291 /* we can still sort pages by socket in legacy mode */
2292 if (!internal_config
.legacy_mem
&& socket_id
> 0)
2295 memtypes
[cur_type
].page_sz
= hugepage_sz
;
2296 memtypes
[cur_type
].socket_id
= socket_id
;
2298 RTE_LOG(DEBUG
, EAL
, "Detected memory type: "
2299 "socket_id:%u hugepage_sz:%" PRIu64
"\n",
2300 socket_id
, hugepage_sz
);
2303 /* number of memtypes could have been lower due to no NUMA support */
2304 n_memtypes
= cur_type
;
2306 /* set up limits for types */
2307 max_mem
= (uint64_t)RTE_MAX_MEM_MB
<< 20;
2308 max_mem_per_type
= RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_TYPE
<< 20,
2309 max_mem
/ n_memtypes
);
2311 * limit maximum number of segment lists per type to ensure there's
2312 * space for memseg lists for all NUMA nodes with all page sizes
2314 max_seglists_per_type
= RTE_MAX_MEMSEG_LISTS
/ n_memtypes
;
2316 if (max_seglists_per_type
== 0) {
2317 RTE_LOG(ERR
, EAL
, "Cannot accommodate all memory types, please increase %s\n",
2318 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS
));
2322 /* go through all mem types and create segment lists */
2324 for (cur_type
= 0; cur_type
< n_memtypes
; cur_type
++) {
2325 unsigned int cur_seglist
, n_seglists
, n_segs
;
2326 unsigned int max_segs_per_type
, max_segs_per_list
;
2327 struct memtype
*type
= &memtypes
[cur_type
];
2328 uint64_t max_mem_per_list
, pagesz
;
2331 pagesz
= type
->page_sz
;
2332 socket_id
= type
->socket_id
;
2335 * we need to create segment lists for this type. we must take
2336 * into account the following things:
2338 * 1. total amount of memory we can use for this memory type
2339 * 2. total amount of memory per memseg list allowed
2340 * 3. number of segments needed to fit the amount of memory
2341 * 4. number of segments allowed per type
2342 * 5. number of segments allowed per memseg list
2343 * 6. number of memseg lists we are allowed to take up
2346 /* calculate how much segments we will need in total */
2347 max_segs_per_type
= max_mem_per_type
/ pagesz
;
2348 /* limit number of segments to maximum allowed per type */
2349 max_segs_per_type
= RTE_MIN(max_segs_per_type
,
2350 (unsigned int)RTE_MAX_MEMSEG_PER_TYPE
);
2351 /* limit number of segments to maximum allowed per list */
2352 max_segs_per_list
= RTE_MIN(max_segs_per_type
,
2353 (unsigned int)RTE_MAX_MEMSEG_PER_LIST
);
2355 /* calculate how much memory we can have per segment list */
2356 max_mem_per_list
= RTE_MIN(max_segs_per_list
* pagesz
,
2357 (uint64_t)RTE_MAX_MEM_MB_PER_LIST
<< 20);
2359 /* calculate how many segments each segment list will have */
2360 n_segs
= RTE_MIN(max_segs_per_list
, max_mem_per_list
/ pagesz
);
2362 /* calculate how many segment lists we can have */
2363 n_seglists
= RTE_MIN(max_segs_per_type
/ n_segs
,
2364 max_mem_per_type
/ max_mem_per_list
);
2366 /* limit number of segment lists according to our maximum */
2367 n_seglists
= RTE_MIN(n_seglists
, max_seglists_per_type
);
2369 RTE_LOG(DEBUG
, EAL
, "Creating %i segment lists: "
2370 "n_segs:%i socket_id:%i hugepage_sz:%" PRIu64
"\n",
2371 n_seglists
, n_segs
, socket_id
, pagesz
);
2373 /* create all segment lists */
2374 for (cur_seglist
= 0; cur_seglist
< n_seglists
; cur_seglist
++) {
2375 if (msl_idx
>= RTE_MAX_MEMSEG_LISTS
) {
2377 "No more space in memseg lists, please increase %s\n",
2378 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS
));
2381 msl
= &mcfg
->memsegs
[msl_idx
++];
2383 if (alloc_memseg_list(msl
, pagesz
, n_segs
,
2384 socket_id
, cur_seglist
))
2387 if (alloc_va_space(msl
)) {
2388 RTE_LOG(ERR
, EAL
, "Cannot allocate VA space for memseg list\n");
2393 /* we're successful */
2401 memseg_secondary_init(void)
2403 struct rte_mem_config
*mcfg
= rte_eal_get_configuration()->mem_config
;
2405 struct rte_memseg_list
*msl
;
2407 for (msl_idx
= 0; msl_idx
< RTE_MAX_MEMSEG_LISTS
; msl_idx
++) {
2409 msl
= &mcfg
->memsegs
[msl_idx
];
2411 /* skip empty memseg lists */
2412 if (msl
->memseg_arr
.len
== 0)
2415 if (rte_fbarray_attach(&msl
->memseg_arr
)) {
2416 RTE_LOG(ERR
, EAL
, "Cannot attach to primary process memseg lists\n");
2420 /* preallocate VA space */
2421 if (alloc_va_space(msl
)) {
2422 RTE_LOG(ERR
, EAL
, "Cannot preallocate VA space for hugepage memory\n");
2431 rte_eal_memseg_init(void)
2433 /* increase rlimit to maximum */
2436 if (getrlimit(RLIMIT_NOFILE
, &lim
) == 0) {
2437 /* set limit to maximum */
2438 lim
.rlim_cur
= lim
.rlim_max
;
2440 if (setrlimit(RLIMIT_NOFILE
, &lim
) < 0) {
2441 RTE_LOG(DEBUG
, EAL
, "Setting maximum number of open files failed: %s\n",
2444 RTE_LOG(DEBUG
, EAL
, "Setting maximum number of open files to %"
2446 (uint64_t)lim
.rlim_cur
);
2449 RTE_LOG(ERR
, EAL
, "Cannot get current resource limits\n");
2451 #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
2452 if (!internal_config
.legacy_mem
&& rte_socket_count() > 1) {
2453 RTE_LOG(WARNING
, EAL
, "DPDK is running on a NUMA system, but is compiled without NUMA support.\n");
2454 RTE_LOG(WARNING
, EAL
, "This will have adverse consequences for performance and usability.\n");
2455 RTE_LOG(WARNING
, EAL
, "Please use --"OPT_LEGACY_MEM
" option, or recompile with NUMA support.\n");
2459 return rte_eal_process_type() == RTE_PROC_PRIMARY
?
2461 memseg_primary_init_32() :
2463 memseg_primary_init() :
2465 memseg_secondary_init();