1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
4 * Common eBPF ELF object loading operations.
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc.
9 * Copyright (C) 2017 Nicira, Inc.
10 * Copyright (C) 2019 Isovalent, Inc.
26 #include <asm/unistd.h>
27 #include <linux/err.h>
28 #include <linux/kernel.h>
29 #include <linux/bpf.h>
30 #include <linux/btf.h>
31 #include <linux/filter.h>
32 #include <linux/list.h>
33 #include <linux/limits.h>
34 #include <linux/perf_event.h>
35 #include <linux/ring_buffer.h>
36 #include <sys/epoll.h>
37 #include <sys/ioctl.h>
40 #include <sys/types.h>
42 #include <sys/utsname.h>
43 #include <tools/libc_compat.h>
50 #include "str_error.h"
51 #include "libbpf_internal.h"
59 #define BPF_FS_MAGIC 0xcafe4a11
62 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
63 * compilation if user enables corresponding warning. Disable it explicitly.
65 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
67 #define __printf(a, b) __attribute__((format(printf, a, b)))
69 static int __base_pr(enum libbpf_print_level level
, const char *format
,
72 if (level
== LIBBPF_DEBUG
)
75 return vfprintf(stderr
, format
, args
);
78 static libbpf_print_fn_t __libbpf_pr
= __base_pr
;
80 libbpf_print_fn_t
libbpf_set_print(libbpf_print_fn_t fn
)
82 libbpf_print_fn_t old_print_fn
= __libbpf_pr
;
89 void libbpf_print(enum libbpf_print_level level
, const char *format
, ...)
96 va_start(args
, format
);
97 __libbpf_pr(level
, format
, args
);
101 #define STRERR_BUFSIZE 128
103 #define CHECK_ERR(action, err, out) do { \
110 /* Copied from tools/perf/util/util.h */
112 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
116 # define zclose(fd) ({ \
119 ___err = close((fd)); \
124 #ifdef HAVE_LIBELF_MMAP_SUPPORT
125 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
127 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
130 static inline __u64
ptr_to_u64(const void *ptr
)
132 return (__u64
) (unsigned long) ptr
;
135 struct bpf_capabilities
{
136 /* v4.14: kernel support for program & map names. */
138 /* v5.2: kernel support for global data sections. */
140 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
142 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
147 * bpf_prog should be a better name but it has been used in
151 /* Index in elf obj file, for relocation use. */
156 /* section_name with / replaced by _; makes recursive pinning
157 * in bpf_object__pin_programs easier
160 struct bpf_insn
*insns
;
161 size_t insns_cnt
, main_prog_cnt
;
162 enum bpf_prog_type type
;
183 bpf_program_prep_t preprocessor
;
185 struct bpf_object
*obj
;
187 bpf_program_clear_priv_t clear_priv
;
189 enum bpf_attach_type expected_attach_type
;
191 __u32 func_info_rec_size
;
194 struct bpf_capabilities
*caps
;
197 __u32 line_info_rec_size
;
202 enum libbpf_map_type
{
209 static const char * const libbpf_type_to_btf_name
[] = {
210 [LIBBPF_MAP_DATA
] = ".data",
211 [LIBBPF_MAP_BSS
] = ".bss",
212 [LIBBPF_MAP_RODATA
] = ".rodata",
222 struct bpf_map_def def
;
223 __u32 btf_key_type_id
;
224 __u32 btf_value_type_id
;
226 bpf_map_clear_priv_t clear_priv
;
227 enum libbpf_map_type libbpf_type
;
235 static LIST_HEAD(bpf_objects_list
);
238 char name
[BPF_OBJ_NAME_LEN
];
242 struct bpf_program
*programs
;
244 struct bpf_map
*maps
;
247 struct bpf_secdata sections
;
250 bool has_pseudo_calls
;
253 * Information when doing elf related work. Only valid if fd
280 * All loaded bpf_object is linked in a list, which is
281 * hidden to caller. bpf_objects__<func> handlers deal with
284 struct list_head list
;
287 struct btf_ext
*btf_ext
;
290 bpf_object_clear_priv_t clear_priv
;
292 struct bpf_capabilities caps
;
296 #define obj_elf_valid(o) ((o)->efile.elf)
298 void bpf_program__unload(struct bpf_program
*prog
)
306 * If the object is opened but the program was never loaded,
307 * it is possible that prog->instances.nr == -1.
309 if (prog
->instances
.nr
> 0) {
310 for (i
= 0; i
< prog
->instances
.nr
; i
++)
311 zclose(prog
->instances
.fds
[i
]);
312 } else if (prog
->instances
.nr
!= -1) {
313 pr_warning("Internal error: instances.nr is %d\n",
317 prog
->instances
.nr
= -1;
318 zfree(&prog
->instances
.fds
);
320 zfree(&prog
->func_info
);
321 zfree(&prog
->line_info
);
324 static void bpf_program__exit(struct bpf_program
*prog
)
329 if (prog
->clear_priv
)
330 prog
->clear_priv(prog
, prog
->priv
);
333 prog
->clear_priv
= NULL
;
335 bpf_program__unload(prog
);
337 zfree(&prog
->section_name
);
338 zfree(&prog
->pin_name
);
340 zfree(&prog
->reloc_desc
);
347 static char *__bpf_program__pin_name(struct bpf_program
*prog
)
351 name
= p
= strdup(prog
->section_name
);
352 while ((p
= strchr(p
, '/')))
359 bpf_program__init(void *data
, size_t size
, char *section_name
, int idx
,
360 struct bpf_program
*prog
)
362 const size_t bpf_insn_sz
= sizeof(struct bpf_insn
);
364 if (size
== 0 || size
% bpf_insn_sz
) {
365 pr_warning("corrupted section '%s', size: %zu\n",
370 memset(prog
, 0, sizeof(*prog
));
372 prog
->section_name
= strdup(section_name
);
373 if (!prog
->section_name
) {
374 pr_warning("failed to alloc name for prog under section(%d) %s\n",
379 prog
->pin_name
= __bpf_program__pin_name(prog
);
380 if (!prog
->pin_name
) {
381 pr_warning("failed to alloc pin name for prog under section(%d) %s\n",
386 prog
->insns
= malloc(size
);
388 pr_warning("failed to alloc insns for prog under section %s\n",
392 prog
->insns_cnt
= size
/ bpf_insn_sz
;
393 memcpy(prog
->insns
, data
, size
);
395 prog
->instances
.fds
= NULL
;
396 prog
->instances
.nr
= -1;
397 prog
->type
= BPF_PROG_TYPE_UNSPEC
;
401 bpf_program__exit(prog
);
406 bpf_object__add_program(struct bpf_object
*obj
, void *data
, size_t size
,
407 char *section_name
, int idx
)
409 struct bpf_program prog
, *progs
;
412 err
= bpf_program__init(data
, size
, section_name
, idx
, &prog
);
416 prog
.caps
= &obj
->caps
;
417 progs
= obj
->programs
;
418 nr_progs
= obj
->nr_programs
;
420 progs
= reallocarray(progs
, nr_progs
+ 1, sizeof(progs
[0]));
423 * In this case the original obj->programs
424 * is still valid, so don't need special treat for
425 * bpf_close_object().
427 pr_warning("failed to alloc a new program under section '%s'\n",
429 bpf_program__exit(&prog
);
433 pr_debug("found program %s\n", prog
.section_name
);
434 obj
->programs
= progs
;
435 obj
->nr_programs
= nr_progs
+ 1;
437 progs
[nr_progs
] = prog
;
442 bpf_object__init_prog_names(struct bpf_object
*obj
)
444 Elf_Data
*symbols
= obj
->efile
.symbols
;
445 struct bpf_program
*prog
;
448 for (pi
= 0; pi
< obj
->nr_programs
; pi
++) {
449 const char *name
= NULL
;
451 prog
= &obj
->programs
[pi
];
453 for (si
= 0; si
< symbols
->d_size
/ sizeof(GElf_Sym
) && !name
;
457 if (!gelf_getsym(symbols
, si
, &sym
))
459 if (sym
.st_shndx
!= prog
->idx
)
461 if (GELF_ST_BIND(sym
.st_info
) != STB_GLOBAL
)
464 name
= elf_strptr(obj
->efile
.elf
,
465 obj
->efile
.strtabidx
,
468 pr_warning("failed to get sym name string for prog %s\n",
470 return -LIBBPF_ERRNO__LIBELF
;
474 if (!name
&& prog
->idx
== obj
->efile
.text_shndx
)
478 pr_warning("failed to find sym for prog %s\n",
483 prog
->name
= strdup(name
);
485 pr_warning("failed to allocate memory for prog sym %s\n",
494 static struct bpf_object
*bpf_object__new(const char *path
,
498 struct bpf_object
*obj
;
501 obj
= calloc(1, sizeof(struct bpf_object
) + strlen(path
) + 1);
503 pr_warning("alloc memory failed for %s\n", path
);
504 return ERR_PTR(-ENOMEM
);
507 strcpy(obj
->path
, path
);
508 /* Using basename() GNU version which doesn't modify arg. */
509 strncpy(obj
->name
, basename((void *)path
), sizeof(obj
->name
) - 1);
510 end
= strchr(obj
->name
, '.');
516 * Caller of this function should also call
517 * bpf_object__elf_finish() after data collection to return
518 * obj_buf to user. If not, we should duplicate the buffer to
519 * avoid user freeing them before elf finish.
521 obj
->efile
.obj_buf
= obj_buf
;
522 obj
->efile
.obj_buf_sz
= obj_buf_sz
;
523 obj
->efile
.maps_shndx
= -1;
524 obj
->efile
.btf_maps_shndx
= -1;
525 obj
->efile
.data_shndx
= -1;
526 obj
->efile
.rodata_shndx
= -1;
527 obj
->efile
.bss_shndx
= -1;
531 INIT_LIST_HEAD(&obj
->list
);
532 list_add(&obj
->list
, &bpf_objects_list
);
536 static void bpf_object__elf_finish(struct bpf_object
*obj
)
538 if (!obj_elf_valid(obj
))
541 if (obj
->efile
.elf
) {
542 elf_end(obj
->efile
.elf
);
543 obj
->efile
.elf
= NULL
;
545 obj
->efile
.symbols
= NULL
;
546 obj
->efile
.data
= NULL
;
547 obj
->efile
.rodata
= NULL
;
548 obj
->efile
.bss
= NULL
;
550 zfree(&obj
->efile
.reloc
);
551 obj
->efile
.nr_reloc
= 0;
552 zclose(obj
->efile
.fd
);
553 obj
->efile
.obj_buf
= NULL
;
554 obj
->efile
.obj_buf_sz
= 0;
557 static int bpf_object__elf_init(struct bpf_object
*obj
)
562 if (obj_elf_valid(obj
)) {
563 pr_warning("elf init: internal error\n");
564 return -LIBBPF_ERRNO__LIBELF
;
567 if (obj
->efile
.obj_buf_sz
> 0) {
569 * obj_buf should have been validated by
570 * bpf_object__open_buffer().
572 obj
->efile
.elf
= elf_memory(obj
->efile
.obj_buf
,
573 obj
->efile
.obj_buf_sz
);
575 obj
->efile
.fd
= open(obj
->path
, O_RDONLY
);
576 if (obj
->efile
.fd
< 0) {
577 char errmsg
[STRERR_BUFSIZE
], *cp
;
580 cp
= libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
));
581 pr_warning("failed to open %s: %s\n", obj
->path
, cp
);
585 obj
->efile
.elf
= elf_begin(obj
->efile
.fd
,
586 LIBBPF_ELF_C_READ_MMAP
, NULL
);
589 if (!obj
->efile
.elf
) {
590 pr_warning("failed to open %s as ELF file\n", obj
->path
);
591 err
= -LIBBPF_ERRNO__LIBELF
;
595 if (!gelf_getehdr(obj
->efile
.elf
, &obj
->efile
.ehdr
)) {
596 pr_warning("failed to get EHDR from %s\n", obj
->path
);
597 err
= -LIBBPF_ERRNO__FORMAT
;
600 ep
= &obj
->efile
.ehdr
;
602 /* Old LLVM set e_machine to EM_NONE */
603 if (ep
->e_type
!= ET_REL
||
604 (ep
->e_machine
&& ep
->e_machine
!= EM_BPF
)) {
605 pr_warning("%s is not an eBPF object file\n", obj
->path
);
606 err
= -LIBBPF_ERRNO__FORMAT
;
612 bpf_object__elf_finish(obj
);
616 static int bpf_object__check_endianness(struct bpf_object
*obj
)
618 #if __BYTE_ORDER == __LITTLE_ENDIAN
619 if (obj
->efile
.ehdr
.e_ident
[EI_DATA
] == ELFDATA2LSB
)
621 #elif __BYTE_ORDER == __BIG_ENDIAN
622 if (obj
->efile
.ehdr
.e_ident
[EI_DATA
] == ELFDATA2MSB
)
625 # error "Unrecognized __BYTE_ORDER__"
627 pr_warning("endianness mismatch.\n");
628 return -LIBBPF_ERRNO__ENDIAN
;
632 bpf_object__init_license(struct bpf_object
*obj
, void *data
, size_t size
)
634 memcpy(obj
->license
, data
, min(size
, sizeof(obj
->license
) - 1));
635 pr_debug("license of %s is %s\n", obj
->path
, obj
->license
);
640 bpf_object__init_kversion(struct bpf_object
*obj
, void *data
, size_t size
)
644 if (size
!= sizeof(kver
)) {
645 pr_warning("invalid kver section in %s\n", obj
->path
);
646 return -LIBBPF_ERRNO__FORMAT
;
648 memcpy(&kver
, data
, sizeof(kver
));
649 obj
->kern_version
= kver
;
650 pr_debug("kernel version of %s is %x\n", obj
->path
, obj
->kern_version
);
654 static int compare_bpf_map(const void *_a
, const void *_b
)
656 const struct bpf_map
*a
= _a
;
657 const struct bpf_map
*b
= _b
;
659 if (a
->sec_idx
!= b
->sec_idx
)
660 return a
->sec_idx
- b
->sec_idx
;
661 return a
->sec_offset
- b
->sec_offset
;
664 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type
)
666 if (type
== BPF_MAP_TYPE_ARRAY_OF_MAPS
||
667 type
== BPF_MAP_TYPE_HASH_OF_MAPS
)
672 static int bpf_object_search_section_size(const struct bpf_object
*obj
,
673 const char *name
, size_t *d_size
)
675 const GElf_Ehdr
*ep
= &obj
->efile
.ehdr
;
676 Elf
*elf
= obj
->efile
.elf
;
680 while ((scn
= elf_nextscn(elf
, scn
)) != NULL
) {
681 const char *sec_name
;
686 if (gelf_getshdr(scn
, &sh
) != &sh
) {
687 pr_warning("failed to get section(%d) header from %s\n",
692 sec_name
= elf_strptr(elf
, ep
->e_shstrndx
, sh
.sh_name
);
694 pr_warning("failed to get section(%d) name from %s\n",
699 if (strcmp(name
, sec_name
))
702 data
= elf_getdata(scn
, 0);
704 pr_warning("failed to get section(%d) data from %s(%s)\n",
705 idx
, name
, obj
->path
);
709 *d_size
= data
->d_size
;
716 int bpf_object__section_size(const struct bpf_object
*obj
, const char *name
,
725 } else if (!strcmp(name
, ".data")) {
727 *size
= obj
->efile
.data
->d_size
;
728 } else if (!strcmp(name
, ".bss")) {
730 *size
= obj
->efile
.bss
->d_size
;
731 } else if (!strcmp(name
, ".rodata")) {
732 if (obj
->efile
.rodata
)
733 *size
= obj
->efile
.rodata
->d_size
;
735 ret
= bpf_object_search_section_size(obj
, name
, &d_size
);
740 return *size
? 0 : ret
;
743 int bpf_object__variable_offset(const struct bpf_object
*obj
, const char *name
,
746 Elf_Data
*symbols
= obj
->efile
.symbols
;
753 for (si
= 0; si
< symbols
->d_size
/ sizeof(GElf_Sym
); si
++) {
756 if (!gelf_getsym(symbols
, si
, &sym
))
758 if (GELF_ST_BIND(sym
.st_info
) != STB_GLOBAL
||
759 GELF_ST_TYPE(sym
.st_info
) != STT_OBJECT
)
762 sname
= elf_strptr(obj
->efile
.elf
, obj
->efile
.strtabidx
,
765 pr_warning("failed to get sym name string for var %s\n",
769 if (strcmp(name
, sname
) == 0) {
778 static struct bpf_map
*bpf_object__add_map(struct bpf_object
*obj
)
780 struct bpf_map
*new_maps
;
784 if (obj
->nr_maps
< obj
->maps_cap
)
785 return &obj
->maps
[obj
->nr_maps
++];
787 new_cap
= max((size_t)4, obj
->maps_cap
* 3 / 2);
788 new_maps
= realloc(obj
->maps
, new_cap
* sizeof(*obj
->maps
));
790 pr_warning("alloc maps for object failed\n");
791 return ERR_PTR(-ENOMEM
);
794 obj
->maps_cap
= new_cap
;
795 obj
->maps
= new_maps
;
797 /* zero out new maps */
798 memset(obj
->maps
+ obj
->nr_maps
, 0,
799 (obj
->maps_cap
- obj
->nr_maps
) * sizeof(*obj
->maps
));
801 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
802 * when failure (zclose won't close negative fd)).
804 for (i
= obj
->nr_maps
; i
< obj
->maps_cap
; i
++) {
805 obj
->maps
[i
].fd
= -1;
806 obj
->maps
[i
].inner_map_fd
= -1;
809 return &obj
->maps
[obj
->nr_maps
++];
813 bpf_object__init_internal_map(struct bpf_object
*obj
, enum libbpf_map_type type
,
814 int sec_idx
, Elf_Data
*data
, void **data_buff
)
816 char map_name
[BPF_OBJ_NAME_LEN
];
817 struct bpf_map_def
*def
;
820 map
= bpf_object__add_map(obj
);
824 map
->libbpf_type
= type
;
825 map
->sec_idx
= sec_idx
;
827 snprintf(map_name
, sizeof(map_name
), "%.8s%.7s", obj
->name
,
828 libbpf_type_to_btf_name
[type
]);
829 map
->name
= strdup(map_name
);
831 pr_warning("failed to alloc map name\n");
834 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n",
835 map_name
, map
->sec_idx
, map
->sec_offset
);
838 def
->type
= BPF_MAP_TYPE_ARRAY
;
839 def
->key_size
= sizeof(int);
840 def
->value_size
= data
->d_size
;
841 def
->max_entries
= 1;
842 def
->map_flags
= type
== LIBBPF_MAP_RODATA
? BPF_F_RDONLY_PROG
: 0;
844 *data_buff
= malloc(data
->d_size
);
847 pr_warning("failed to alloc map content buffer\n");
850 memcpy(*data_buff
, data
->d_buf
, data
->d_size
);
853 pr_debug("map %td is \"%s\"\n", map
- obj
->maps
, map
->name
);
857 static int bpf_object__init_global_data_maps(struct bpf_object
*obj
)
861 if (!obj
->caps
.global_data
)
864 * Populate obj->maps with libbpf internal maps.
866 if (obj
->efile
.data_shndx
>= 0) {
867 err
= bpf_object__init_internal_map(obj
, LIBBPF_MAP_DATA
,
868 obj
->efile
.data_shndx
,
870 &obj
->sections
.data
);
874 if (obj
->efile
.rodata_shndx
>= 0) {
875 err
= bpf_object__init_internal_map(obj
, LIBBPF_MAP_RODATA
,
876 obj
->efile
.rodata_shndx
,
878 &obj
->sections
.rodata
);
882 if (obj
->efile
.bss_shndx
>= 0) {
883 err
= bpf_object__init_internal_map(obj
, LIBBPF_MAP_BSS
,
884 obj
->efile
.bss_shndx
,
885 obj
->efile
.bss
, NULL
);
892 static int bpf_object__init_user_maps(struct bpf_object
*obj
, bool strict
)
894 Elf_Data
*symbols
= obj
->efile
.symbols
;
895 int i
, map_def_sz
= 0, nr_maps
= 0, nr_syms
;
896 Elf_Data
*data
= NULL
;
899 if (obj
->efile
.maps_shndx
< 0)
905 scn
= elf_getscn(obj
->efile
.elf
, obj
->efile
.maps_shndx
);
907 data
= elf_getdata(scn
, NULL
);
909 pr_warning("failed to get Elf_Data from map section %d\n",
910 obj
->efile
.maps_shndx
);
915 * Count number of maps. Each map has a name.
916 * Array of maps is not supported: only the first element is
919 * TODO: Detect array of map and report error.
921 nr_syms
= symbols
->d_size
/ sizeof(GElf_Sym
);
922 for (i
= 0; i
< nr_syms
; i
++) {
925 if (!gelf_getsym(symbols
, i
, &sym
))
927 if (sym
.st_shndx
!= obj
->efile
.maps_shndx
)
931 /* Assume equally sized map definitions */
932 pr_debug("maps in %s: %d maps in %zd bytes\n",
933 obj
->path
, nr_maps
, data
->d_size
);
935 map_def_sz
= data
->d_size
/ nr_maps
;
936 if (!data
->d_size
|| (data
->d_size
% nr_maps
) != 0) {
937 pr_warning("unable to determine map definition size "
938 "section %s, %d maps in %zd bytes\n",
939 obj
->path
, nr_maps
, data
->d_size
);
943 /* Fill obj->maps using data in "maps" section. */
944 for (i
= 0; i
< nr_syms
; i
++) {
946 const char *map_name
;
947 struct bpf_map_def
*def
;
950 if (!gelf_getsym(symbols
, i
, &sym
))
952 if (sym
.st_shndx
!= obj
->efile
.maps_shndx
)
955 map
= bpf_object__add_map(obj
);
959 map_name
= elf_strptr(obj
->efile
.elf
, obj
->efile
.strtabidx
,
962 pr_warning("failed to get map #%d name sym string for obj %s\n",
964 return -LIBBPF_ERRNO__FORMAT
;
967 map
->libbpf_type
= LIBBPF_MAP_UNSPEC
;
968 map
->sec_idx
= sym
.st_shndx
;
969 map
->sec_offset
= sym
.st_value
;
970 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
971 map_name
, map
->sec_idx
, map
->sec_offset
);
972 if (sym
.st_value
+ map_def_sz
> data
->d_size
) {
973 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
974 obj
->path
, map_name
);
978 map
->name
= strdup(map_name
);
980 pr_warning("failed to alloc map name\n");
983 pr_debug("map %d is \"%s\"\n", i
, map
->name
);
984 def
= (struct bpf_map_def
*)(data
->d_buf
+ sym
.st_value
);
986 * If the definition of the map in the object file fits in
987 * bpf_map_def, copy it. Any extra fields in our version
988 * of bpf_map_def will default to zero as a result of the
991 if (map_def_sz
<= sizeof(struct bpf_map_def
)) {
992 memcpy(&map
->def
, def
, map_def_sz
);
995 * Here the map structure being read is bigger than what
996 * we expect, truncate if the excess bits are all zero.
997 * If they are not zero, reject this map as
1001 for (b
= ((char *)def
) + sizeof(struct bpf_map_def
);
1002 b
< ((char *)def
) + map_def_sz
; b
++) {
1004 pr_warning("maps section in %s: \"%s\" "
1005 "has unrecognized, non-zero "
1007 obj
->path
, map_name
);
1012 memcpy(&map
->def
, def
, sizeof(struct bpf_map_def
));
1018 static const struct btf_type
*
1019 skip_mods_and_typedefs(const struct btf
*btf
, __u32 id
, __u32
*res_id
)
1021 const struct btf_type
*t
= btf__type_by_id(btf
, id
);
1026 while (btf_is_mod(t
) || btf_is_typedef(t
)) {
1029 t
= btf__type_by_id(btf
, t
->type
);
1036 * Fetch integer attribute of BTF map definition. Such attributes are
1037 * represented using a pointer to an array, in which dimensionality of array
1038 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
1039 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
1040 * type definition, while using only sizeof(void *) space in ELF data section.
1042 static bool get_map_field_int(const char *map_name
, const struct btf
*btf
,
1043 const struct btf_type
*def
,
1044 const struct btf_member
*m
, __u32
*res
) {
1045 const struct btf_type
*t
= skip_mods_and_typedefs(btf
, m
->type
, NULL
);
1046 const char *name
= btf__name_by_offset(btf
, m
->name_off
);
1047 const struct btf_array
*arr_info
;
1048 const struct btf_type
*arr_t
;
1050 if (!btf_is_ptr(t
)) {
1051 pr_warning("map '%s': attr '%s': expected PTR, got %u.\n",
1052 map_name
, name
, btf_kind(t
));
1056 arr_t
= btf__type_by_id(btf
, t
->type
);
1058 pr_warning("map '%s': attr '%s': type [%u] not found.\n",
1059 map_name
, name
, t
->type
);
1062 if (!btf_is_array(arr_t
)) {
1063 pr_warning("map '%s': attr '%s': expected ARRAY, got %u.\n",
1064 map_name
, name
, btf_kind(arr_t
));
1067 arr_info
= btf_array(arr_t
);
1068 *res
= arr_info
->nelems
;
1072 static int bpf_object__init_user_btf_map(struct bpf_object
*obj
,
1073 const struct btf_type
*sec
,
1074 int var_idx
, int sec_idx
,
1075 const Elf_Data
*data
, bool strict
)
1077 const struct btf_type
*var
, *def
, *t
;
1078 const struct btf_var_secinfo
*vi
;
1079 const struct btf_var
*var_extra
;
1080 const struct btf_member
*m
;
1081 const char *map_name
;
1082 struct bpf_map
*map
;
1085 vi
= btf_var_secinfos(sec
) + var_idx
;
1086 var
= btf__type_by_id(obj
->btf
, vi
->type
);
1087 var_extra
= btf_var(var
);
1088 map_name
= btf__name_by_offset(obj
->btf
, var
->name_off
);
1089 vlen
= btf_vlen(var
);
1091 if (map_name
== NULL
|| map_name
[0] == '\0') {
1092 pr_warning("map #%d: empty name.\n", var_idx
);
1095 if ((__u64
)vi
->offset
+ vi
->size
> data
->d_size
) {
1096 pr_warning("map '%s' BTF data is corrupted.\n", map_name
);
1099 if (!btf_is_var(var
)) {
1100 pr_warning("map '%s': unexpected var kind %u.\n",
1101 map_name
, btf_kind(var
));
1104 if (var_extra
->linkage
!= BTF_VAR_GLOBAL_ALLOCATED
&&
1105 var_extra
->linkage
!= BTF_VAR_STATIC
) {
1106 pr_warning("map '%s': unsupported var linkage %u.\n",
1107 map_name
, var_extra
->linkage
);
1111 def
= skip_mods_and_typedefs(obj
->btf
, var
->type
, NULL
);
1112 if (!btf_is_struct(def
)) {
1113 pr_warning("map '%s': unexpected def kind %u.\n",
1114 map_name
, btf_kind(var
));
1117 if (def
->size
> vi
->size
) {
1118 pr_warning("map '%s': invalid def size.\n", map_name
);
1122 map
= bpf_object__add_map(obj
);
1124 return PTR_ERR(map
);
1125 map
->name
= strdup(map_name
);
1127 pr_warning("map '%s': failed to alloc map name.\n", map_name
);
1130 map
->libbpf_type
= LIBBPF_MAP_UNSPEC
;
1131 map
->def
.type
= BPF_MAP_TYPE_UNSPEC
;
1132 map
->sec_idx
= sec_idx
;
1133 map
->sec_offset
= vi
->offset
;
1134 pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
1135 map_name
, map
->sec_idx
, map
->sec_offset
);
1137 vlen
= btf_vlen(def
);
1138 m
= btf_members(def
);
1139 for (i
= 0; i
< vlen
; i
++, m
++) {
1140 const char *name
= btf__name_by_offset(obj
->btf
, m
->name_off
);
1143 pr_warning("map '%s': invalid field #%d.\n",
1147 if (strcmp(name
, "type") == 0) {
1148 if (!get_map_field_int(map_name
, obj
->btf
, def
, m
,
1151 pr_debug("map '%s': found type = %u.\n",
1152 map_name
, map
->def
.type
);
1153 } else if (strcmp(name
, "max_entries") == 0) {
1154 if (!get_map_field_int(map_name
, obj
->btf
, def
, m
,
1155 &map
->def
.max_entries
))
1157 pr_debug("map '%s': found max_entries = %u.\n",
1158 map_name
, map
->def
.max_entries
);
1159 } else if (strcmp(name
, "map_flags") == 0) {
1160 if (!get_map_field_int(map_name
, obj
->btf
, def
, m
,
1161 &map
->def
.map_flags
))
1163 pr_debug("map '%s': found map_flags = %u.\n",
1164 map_name
, map
->def
.map_flags
);
1165 } else if (strcmp(name
, "key_size") == 0) {
1168 if (!get_map_field_int(map_name
, obj
->btf
, def
, m
,
1171 pr_debug("map '%s': found key_size = %u.\n",
1173 if (map
->def
.key_size
&& map
->def
.key_size
!= sz
) {
1174 pr_warning("map '%s': conflicting key size %u != %u.\n",
1175 map_name
, map
->def
.key_size
, sz
);
1178 map
->def
.key_size
= sz
;
1179 } else if (strcmp(name
, "key") == 0) {
1182 t
= btf__type_by_id(obj
->btf
, m
->type
);
1184 pr_warning("map '%s': key type [%d] not found.\n",
1188 if (!btf_is_ptr(t
)) {
1189 pr_warning("map '%s': key spec is not PTR: %u.\n",
1190 map_name
, btf_kind(t
));
1193 sz
= btf__resolve_size(obj
->btf
, t
->type
);
1195 pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n",
1196 map_name
, t
->type
, sz
);
1199 pr_debug("map '%s': found key [%u], sz = %lld.\n",
1200 map_name
, t
->type
, sz
);
1201 if (map
->def
.key_size
&& map
->def
.key_size
!= sz
) {
1202 pr_warning("map '%s': conflicting key size %u != %lld.\n",
1203 map_name
, map
->def
.key_size
, sz
);
1206 map
->def
.key_size
= sz
;
1207 map
->btf_key_type_id
= t
->type
;
1208 } else if (strcmp(name
, "value_size") == 0) {
1211 if (!get_map_field_int(map_name
, obj
->btf
, def
, m
,
1214 pr_debug("map '%s': found value_size = %u.\n",
1216 if (map
->def
.value_size
&& map
->def
.value_size
!= sz
) {
1217 pr_warning("map '%s': conflicting value size %u != %u.\n",
1218 map_name
, map
->def
.value_size
, sz
);
1221 map
->def
.value_size
= sz
;
1222 } else if (strcmp(name
, "value") == 0) {
1225 t
= btf__type_by_id(obj
->btf
, m
->type
);
1227 pr_warning("map '%s': value type [%d] not found.\n",
1231 if (!btf_is_ptr(t
)) {
1232 pr_warning("map '%s': value spec is not PTR: %u.\n",
1233 map_name
, btf_kind(t
));
1236 sz
= btf__resolve_size(obj
->btf
, t
->type
);
1238 pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n",
1239 map_name
, t
->type
, sz
);
1242 pr_debug("map '%s': found value [%u], sz = %lld.\n",
1243 map_name
, t
->type
, sz
);
1244 if (map
->def
.value_size
&& map
->def
.value_size
!= sz
) {
1245 pr_warning("map '%s': conflicting value size %u != %lld.\n",
1246 map_name
, map
->def
.value_size
, sz
);
1249 map
->def
.value_size
= sz
;
1250 map
->btf_value_type_id
= t
->type
;
1253 pr_warning("map '%s': unknown field '%s'.\n",
1257 pr_debug("map '%s': ignoring unknown field '%s'.\n",
1262 if (map
->def
.type
== BPF_MAP_TYPE_UNSPEC
) {
1263 pr_warning("map '%s': map type isn't specified.\n", map_name
);
1270 static int bpf_object__init_user_btf_maps(struct bpf_object
*obj
, bool strict
)
1272 const struct btf_type
*sec
= NULL
;
1273 int nr_types
, i
, vlen
, err
;
1274 const struct btf_type
*t
;
1279 if (obj
->efile
.btf_maps_shndx
< 0)
1282 scn
= elf_getscn(obj
->efile
.elf
, obj
->efile
.btf_maps_shndx
);
1284 data
= elf_getdata(scn
, NULL
);
1285 if (!scn
|| !data
) {
1286 pr_warning("failed to get Elf_Data from map section %d (%s)\n",
1287 obj
->efile
.maps_shndx
, MAPS_ELF_SEC
);
1291 nr_types
= btf__get_nr_types(obj
->btf
);
1292 for (i
= 1; i
<= nr_types
; i
++) {
1293 t
= btf__type_by_id(obj
->btf
, i
);
1294 if (!btf_is_datasec(t
))
1296 name
= btf__name_by_offset(obj
->btf
, t
->name_off
);
1297 if (strcmp(name
, MAPS_ELF_SEC
) == 0) {
1304 pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC
);
1308 vlen
= btf_vlen(sec
);
1309 for (i
= 0; i
< vlen
; i
++) {
1310 err
= bpf_object__init_user_btf_map(obj
, sec
, i
,
1311 obj
->efile
.btf_maps_shndx
,
1320 static int bpf_object__init_maps(struct bpf_object
*obj
, int flags
)
1322 bool strict
= !(flags
& MAPS_RELAX_COMPAT
);
1325 err
= bpf_object__init_user_maps(obj
, strict
);
1329 err
= bpf_object__init_user_btf_maps(obj
, strict
);
1333 err
= bpf_object__init_global_data_maps(obj
);
1338 qsort(obj
->maps
, obj
->nr_maps
, sizeof(obj
->maps
[0]),
1344 static bool section_have_execinstr(struct bpf_object
*obj
, int idx
)
1349 scn
= elf_getscn(obj
->efile
.elf
, idx
);
1353 if (gelf_getshdr(scn
, &sh
) != &sh
)
1356 if (sh
.sh_flags
& SHF_EXECINSTR
)
1362 static void bpf_object__sanitize_btf(struct bpf_object
*obj
)
1364 bool has_datasec
= obj
->caps
.btf_datasec
;
1365 bool has_func
= obj
->caps
.btf_func
;
1366 struct btf
*btf
= obj
->btf
;
1370 if (!obj
->btf
|| (has_func
&& has_datasec
))
1373 for (i
= 1; i
<= btf__get_nr_types(btf
); i
++) {
1374 t
= (struct btf_type
*)btf__type_by_id(btf
, i
);
1376 if (!has_datasec
&& btf_is_var(t
)) {
1377 /* replace VAR with INT */
1378 t
->info
= BTF_INFO_ENC(BTF_KIND_INT
, 0, 0);
1380 * using size = 1 is the safest choice, 4 will be too
1381 * big and cause kernel BTF validation failure if
1382 * original variable took less than 4 bytes
1385 *(int *)(t
+ 1) = BTF_INT_ENC(0, 0, 8);
1386 } else if (!has_datasec
&& btf_is_datasec(t
)) {
1387 /* replace DATASEC with STRUCT */
1388 const struct btf_var_secinfo
*v
= btf_var_secinfos(t
);
1389 struct btf_member
*m
= btf_members(t
);
1390 struct btf_type
*vt
;
1393 name
= (char *)btf__name_by_offset(btf
, t
->name_off
);
1401 t
->info
= BTF_INFO_ENC(BTF_KIND_STRUCT
, 0, vlen
);
1402 for (j
= 0; j
< vlen
; j
++, v
++, m
++) {
1403 /* order of field assignments is important */
1404 m
->offset
= v
->offset
* 8;
1406 /* preserve variable name as member name */
1407 vt
= (void *)btf__type_by_id(btf
, v
->type
);
1408 m
->name_off
= vt
->name_off
;
1410 } else if (!has_func
&& btf_is_func_proto(t
)) {
1411 /* replace FUNC_PROTO with ENUM */
1413 t
->info
= BTF_INFO_ENC(BTF_KIND_ENUM
, 0, vlen
);
1414 t
->size
= sizeof(__u32
); /* kernel enforced */
1415 } else if (!has_func
&& btf_is_func(t
)) {
1416 /* replace FUNC with TYPEDEF */
1417 t
->info
= BTF_INFO_ENC(BTF_KIND_TYPEDEF
, 0, 0);
1422 static void bpf_object__sanitize_btf_ext(struct bpf_object
*obj
)
1427 if (!obj
->caps
.btf_func
) {
1428 btf_ext__free(obj
->btf_ext
);
1429 obj
->btf_ext
= NULL
;
1433 static bool bpf_object__is_btf_mandatory(const struct bpf_object
*obj
)
1435 return obj
->efile
.btf_maps_shndx
>= 0;
1438 static int bpf_object__init_btf(struct bpf_object
*obj
,
1440 Elf_Data
*btf_ext_data
)
1442 bool btf_required
= bpf_object__is_btf_mandatory(obj
);
1446 obj
->btf
= btf__new(btf_data
->d_buf
, btf_data
->d_size
);
1447 if (IS_ERR(obj
->btf
)) {
1448 pr_warning("Error loading ELF section %s: %d.\n",
1452 err
= btf__finalize_data(obj
, obj
->btf
);
1454 pr_warning("Error finalizing %s: %d.\n",
1461 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
1462 BTF_EXT_ELF_SEC
, BTF_ELF_SEC
);
1465 obj
->btf_ext
= btf_ext__new(btf_ext_data
->d_buf
,
1466 btf_ext_data
->d_size
);
1467 if (IS_ERR(obj
->btf_ext
)) {
1468 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
1469 BTF_EXT_ELF_SEC
, PTR_ERR(obj
->btf_ext
));
1470 obj
->btf_ext
= NULL
;
1475 if (err
|| IS_ERR(obj
->btf
)) {
1477 err
= err
? : PTR_ERR(obj
->btf
);
1480 if (!IS_ERR_OR_NULL(obj
->btf
))
1481 btf__free(obj
->btf
);
1484 if (btf_required
&& !obj
->btf
) {
1485 pr_warning("BTF is required, but is missing or corrupted.\n");
1486 return err
== 0 ? -ENOENT
: err
;
1491 static int bpf_object__sanitize_and_load_btf(struct bpf_object
*obj
)
1498 bpf_object__sanitize_btf(obj
);
1499 bpf_object__sanitize_btf_ext(obj
);
1501 err
= btf__load(obj
->btf
);
1503 pr_warning("Error loading %s into kernel: %d.\n",
1505 btf__free(obj
->btf
);
1507 /* btf_ext can't exist without btf, so free it as well */
1509 btf_ext__free(obj
->btf_ext
);
1510 obj
->btf_ext
= NULL
;
1513 if (bpf_object__is_btf_mandatory(obj
))
1519 static int bpf_object__elf_collect(struct bpf_object
*obj
, int flags
)
1521 Elf
*elf
= obj
->efile
.elf
;
1522 GElf_Ehdr
*ep
= &obj
->efile
.ehdr
;
1523 Elf_Data
*btf_ext_data
= NULL
;
1524 Elf_Data
*btf_data
= NULL
;
1525 Elf_Scn
*scn
= NULL
;
1526 int idx
= 0, err
= 0;
1528 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1529 if (!elf_rawdata(elf_getscn(elf
, ep
->e_shstrndx
), NULL
)) {
1530 pr_warning("failed to get e_shstrndx from %s\n", obj
->path
);
1531 return -LIBBPF_ERRNO__FORMAT
;
1534 while ((scn
= elf_nextscn(elf
, scn
)) != NULL
) {
1540 if (gelf_getshdr(scn
, &sh
) != &sh
) {
1541 pr_warning("failed to get section(%d) header from %s\n",
1543 return -LIBBPF_ERRNO__FORMAT
;
1546 name
= elf_strptr(elf
, ep
->e_shstrndx
, sh
.sh_name
);
1548 pr_warning("failed to get section(%d) name from %s\n",
1550 return -LIBBPF_ERRNO__FORMAT
;
1553 data
= elf_getdata(scn
, 0);
1555 pr_warning("failed to get section(%d) data from %s(%s)\n",
1556 idx
, name
, obj
->path
);
1557 return -LIBBPF_ERRNO__FORMAT
;
1559 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
1560 idx
, name
, (unsigned long)data
->d_size
,
1561 (int)sh
.sh_link
, (unsigned long)sh
.sh_flags
,
1564 if (strcmp(name
, "license") == 0) {
1565 err
= bpf_object__init_license(obj
,
1570 } else if (strcmp(name
, "version") == 0) {
1571 err
= bpf_object__init_kversion(obj
,
1576 } else if (strcmp(name
, "maps") == 0) {
1577 obj
->efile
.maps_shndx
= idx
;
1578 } else if (strcmp(name
, MAPS_ELF_SEC
) == 0) {
1579 obj
->efile
.btf_maps_shndx
= idx
;
1580 } else if (strcmp(name
, BTF_ELF_SEC
) == 0) {
1582 } else if (strcmp(name
, BTF_EXT_ELF_SEC
) == 0) {
1583 btf_ext_data
= data
;
1584 } else if (sh
.sh_type
== SHT_SYMTAB
) {
1585 if (obj
->efile
.symbols
) {
1586 pr_warning("bpf: multiple SYMTAB in %s\n",
1588 return -LIBBPF_ERRNO__FORMAT
;
1590 obj
->efile
.symbols
= data
;
1591 obj
->efile
.strtabidx
= sh
.sh_link
;
1592 } else if (sh
.sh_type
== SHT_PROGBITS
&& data
->d_size
> 0) {
1593 if (sh
.sh_flags
& SHF_EXECINSTR
) {
1594 if (strcmp(name
, ".text") == 0)
1595 obj
->efile
.text_shndx
= idx
;
1596 err
= bpf_object__add_program(obj
, data
->d_buf
,
1597 data
->d_size
, name
, idx
);
1599 char errmsg
[STRERR_BUFSIZE
];
1600 char *cp
= libbpf_strerror_r(-err
, errmsg
,
1603 pr_warning("failed to alloc program %s (%s): %s",
1604 name
, obj
->path
, cp
);
1607 } else if (strcmp(name
, ".data") == 0) {
1608 obj
->efile
.data
= data
;
1609 obj
->efile
.data_shndx
= idx
;
1610 } else if (strcmp(name
, ".rodata") == 0) {
1611 obj
->efile
.rodata
= data
;
1612 obj
->efile
.rodata_shndx
= idx
;
1614 pr_debug("skip section(%d) %s\n", idx
, name
);
1616 } else if (sh
.sh_type
== SHT_REL
) {
1617 int nr_reloc
= obj
->efile
.nr_reloc
;
1618 void *reloc
= obj
->efile
.reloc
;
1619 int sec
= sh
.sh_info
; /* points to other section */
1621 /* Only do relo for section with exec instructions */
1622 if (!section_have_execinstr(obj
, sec
)) {
1623 pr_debug("skip relo %s(%d) for section(%d)\n",
1628 reloc
= reallocarray(reloc
, nr_reloc
+ 1,
1629 sizeof(*obj
->efile
.reloc
));
1631 pr_warning("realloc failed\n");
1635 obj
->efile
.reloc
= reloc
;
1636 obj
->efile
.nr_reloc
++;
1638 obj
->efile
.reloc
[nr_reloc
].shdr
= sh
;
1639 obj
->efile
.reloc
[nr_reloc
].data
= data
;
1640 } else if (sh
.sh_type
== SHT_NOBITS
&& strcmp(name
, ".bss") == 0) {
1641 obj
->efile
.bss
= data
;
1642 obj
->efile
.bss_shndx
= idx
;
1644 pr_debug("skip section(%d) %s\n", idx
, name
);
1648 if (!obj
->efile
.strtabidx
|| obj
->efile
.strtabidx
>= idx
) {
1649 pr_warning("Corrupted ELF file: index of strtab invalid\n");
1650 return -LIBBPF_ERRNO__FORMAT
;
1652 err
= bpf_object__init_btf(obj
, btf_data
, btf_ext_data
);
1654 err
= bpf_object__init_maps(obj
, flags
);
1656 err
= bpf_object__sanitize_and_load_btf(obj
);
1658 err
= bpf_object__init_prog_names(obj
);
1662 static struct bpf_program
*
1663 bpf_object__find_prog_by_idx(struct bpf_object
*obj
, int idx
)
1665 struct bpf_program
*prog
;
1668 for (i
= 0; i
< obj
->nr_programs
; i
++) {
1669 prog
= &obj
->programs
[i
];
1670 if (prog
->idx
== idx
)
1676 struct bpf_program
*
1677 bpf_object__find_program_by_title(const struct bpf_object
*obj
,
1680 struct bpf_program
*pos
;
1682 bpf_object__for_each_program(pos
, obj
) {
1683 if (pos
->section_name
&& !strcmp(pos
->section_name
, title
))
1689 static bool bpf_object__shndx_is_data(const struct bpf_object
*obj
,
1692 return shndx
== obj
->efile
.data_shndx
||
1693 shndx
== obj
->efile
.bss_shndx
||
1694 shndx
== obj
->efile
.rodata_shndx
;
1697 static bool bpf_object__shndx_is_maps(const struct bpf_object
*obj
,
1700 return shndx
== obj
->efile
.maps_shndx
||
1701 shndx
== obj
->efile
.btf_maps_shndx
;
1704 static bool bpf_object__relo_in_known_section(const struct bpf_object
*obj
,
1707 return shndx
== obj
->efile
.text_shndx
||
1708 bpf_object__shndx_is_maps(obj
, shndx
) ||
1709 bpf_object__shndx_is_data(obj
, shndx
);
1712 static enum libbpf_map_type
1713 bpf_object__section_to_libbpf_map_type(const struct bpf_object
*obj
, int shndx
)
1715 if (shndx
== obj
->efile
.data_shndx
)
1716 return LIBBPF_MAP_DATA
;
1717 else if (shndx
== obj
->efile
.bss_shndx
)
1718 return LIBBPF_MAP_BSS
;
1719 else if (shndx
== obj
->efile
.rodata_shndx
)
1720 return LIBBPF_MAP_RODATA
;
1722 return LIBBPF_MAP_UNSPEC
;
1726 bpf_program__collect_reloc(struct bpf_program
*prog
, GElf_Shdr
*shdr
,
1727 Elf_Data
*data
, struct bpf_object
*obj
)
1729 Elf_Data
*symbols
= obj
->efile
.symbols
;
1730 struct bpf_map
*maps
= obj
->maps
;
1731 size_t nr_maps
= obj
->nr_maps
;
1734 pr_debug("collecting relocating info for: '%s'\n", prog
->section_name
);
1735 nrels
= shdr
->sh_size
/ shdr
->sh_entsize
;
1737 prog
->reloc_desc
= malloc(sizeof(*prog
->reloc_desc
) * nrels
);
1738 if (!prog
->reloc_desc
) {
1739 pr_warning("failed to alloc memory in relocation\n");
1742 prog
->nr_reloc
= nrels
;
1744 for (i
= 0; i
< nrels
; i
++) {
1745 struct bpf_insn
*insns
= prog
->insns
;
1746 enum libbpf_map_type type
;
1747 unsigned int insn_idx
;
1748 unsigned int shdr_idx
;
1754 if (!gelf_getrel(data
, i
, &rel
)) {
1755 pr_warning("relocation: failed to get %d reloc\n", i
);
1756 return -LIBBPF_ERRNO__FORMAT
;
1759 if (!gelf_getsym(symbols
, GELF_R_SYM(rel
.r_info
), &sym
)) {
1760 pr_warning("relocation: symbol %"PRIx64
" not found\n",
1761 GELF_R_SYM(rel
.r_info
));
1762 return -LIBBPF_ERRNO__FORMAT
;
1765 name
= elf_strptr(obj
->efile
.elf
, obj
->efile
.strtabidx
,
1766 sym
.st_name
) ? : "<?>";
1768 pr_debug("relo for %lld value %lld name %d (\'%s\')\n",
1769 (long long) (rel
.r_info
>> 32),
1770 (long long) sym
.st_value
, sym
.st_name
, name
);
1772 shdr_idx
= sym
.st_shndx
;
1773 insn_idx
= rel
.r_offset
/ sizeof(struct bpf_insn
);
1774 pr_debug("relocation: insn_idx=%u, shdr_idx=%u\n",
1775 insn_idx
, shdr_idx
);
1777 if (shdr_idx
>= SHN_LORESERVE
) {
1778 pr_warning("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n",
1779 name
, shdr_idx
, insn_idx
,
1780 insns
[insn_idx
].code
);
1781 return -LIBBPF_ERRNO__RELOC
;
1783 if (!bpf_object__relo_in_known_section(obj
, shdr_idx
)) {
1784 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n",
1785 prog
->section_name
, shdr_idx
);
1786 return -LIBBPF_ERRNO__RELOC
;
1789 if (insns
[insn_idx
].code
== (BPF_JMP
| BPF_CALL
)) {
1790 if (insns
[insn_idx
].src_reg
!= BPF_PSEUDO_CALL
) {
1791 pr_warning("incorrect bpf_call opcode\n");
1792 return -LIBBPF_ERRNO__RELOC
;
1794 prog
->reloc_desc
[i
].type
= RELO_CALL
;
1795 prog
->reloc_desc
[i
].insn_idx
= insn_idx
;
1796 prog
->reloc_desc
[i
].text_off
= sym
.st_value
;
1797 obj
->has_pseudo_calls
= true;
1801 if (insns
[insn_idx
].code
!= (BPF_LD
| BPF_IMM
| BPF_DW
)) {
1802 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
1803 insn_idx
, insns
[insn_idx
].code
);
1804 return -LIBBPF_ERRNO__RELOC
;
1807 if (bpf_object__shndx_is_maps(obj
, shdr_idx
) ||
1808 bpf_object__shndx_is_data(obj
, shdr_idx
)) {
1809 type
= bpf_object__section_to_libbpf_map_type(obj
, shdr_idx
);
1810 if (type
!= LIBBPF_MAP_UNSPEC
) {
1811 if (GELF_ST_BIND(sym
.st_info
) == STB_GLOBAL
) {
1812 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n",
1813 name
, insn_idx
, insns
[insn_idx
].code
);
1814 return -LIBBPF_ERRNO__RELOC
;
1816 if (!obj
->caps
.global_data
) {
1817 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n",
1819 return -LIBBPF_ERRNO__RELOC
;
1823 for (map_idx
= 0; map_idx
< nr_maps
; map_idx
++) {
1824 if (maps
[map_idx
].libbpf_type
!= type
)
1826 if (type
!= LIBBPF_MAP_UNSPEC
||
1827 (maps
[map_idx
].sec_idx
== sym
.st_shndx
&&
1828 maps
[map_idx
].sec_offset
== sym
.st_value
)) {
1829 pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n",
1830 map_idx
, maps
[map_idx
].name
,
1831 maps
[map_idx
].sec_idx
,
1832 maps
[map_idx
].sec_offset
,
1838 if (map_idx
>= nr_maps
) {
1839 pr_warning("bpf relocation: map_idx %d larger than %d\n",
1840 (int)map_idx
, (int)nr_maps
- 1);
1841 return -LIBBPF_ERRNO__RELOC
;
1844 prog
->reloc_desc
[i
].type
= type
!= LIBBPF_MAP_UNSPEC
?
1845 RELO_DATA
: RELO_LD64
;
1846 prog
->reloc_desc
[i
].insn_idx
= insn_idx
;
1847 prog
->reloc_desc
[i
].map_idx
= map_idx
;
1853 static int bpf_map_find_btf_info(struct bpf_object
*obj
, struct bpf_map
*map
)
1855 struct bpf_map_def
*def
= &map
->def
;
1856 __u32 key_type_id
= 0, value_type_id
= 0;
1859 /* if it's BTF-defined map, we don't need to search for type IDs */
1860 if (map
->sec_idx
== obj
->efile
.btf_maps_shndx
)
1863 if (!bpf_map__is_internal(map
)) {
1864 ret
= btf__get_map_kv_tids(obj
->btf
, map
->name
, def
->key_size
,
1865 def
->value_size
, &key_type_id
,
1869 * LLVM annotates global data differently in BTF, that is,
1870 * only as '.data', '.bss' or '.rodata'.
1872 ret
= btf__find_by_name(obj
->btf
,
1873 libbpf_type_to_btf_name
[map
->libbpf_type
]);
1878 map
->btf_key_type_id
= key_type_id
;
1879 map
->btf_value_type_id
= bpf_map__is_internal(map
) ?
1880 ret
: value_type_id
;
1884 int bpf_map__reuse_fd(struct bpf_map
*map
, int fd
)
1886 struct bpf_map_info info
= {};
1887 __u32 len
= sizeof(info
);
1891 err
= bpf_obj_get_info_by_fd(fd
, &info
, &len
);
1895 new_name
= strdup(info
.name
);
1899 new_fd
= open("/", O_RDONLY
| O_CLOEXEC
);
1901 goto err_free_new_name
;
1903 new_fd
= dup3(fd
, new_fd
, O_CLOEXEC
);
1905 goto err_close_new_fd
;
1907 err
= zclose(map
->fd
);
1909 goto err_close_new_fd
;
1913 map
->name
= new_name
;
1914 map
->def
.type
= info
.type
;
1915 map
->def
.key_size
= info
.key_size
;
1916 map
->def
.value_size
= info
.value_size
;
1917 map
->def
.max_entries
= info
.max_entries
;
1918 map
->def
.map_flags
= info
.map_flags
;
1919 map
->btf_key_type_id
= info
.btf_key_type_id
;
1920 map
->btf_value_type_id
= info
.btf_value_type_id
;
1931 int bpf_map__resize(struct bpf_map
*map
, __u32 max_entries
)
1933 if (!map
|| !max_entries
)
1936 /* If map already created, its attributes can't be changed. */
1940 map
->def
.max_entries
= max_entries
;
1946 bpf_object__probe_name(struct bpf_object
*obj
)
1948 struct bpf_load_program_attr attr
;
1949 char *cp
, errmsg
[STRERR_BUFSIZE
];
1950 struct bpf_insn insns
[] = {
1951 BPF_MOV64_IMM(BPF_REG_0
, 0),
1956 /* make sure basic loading works */
1958 memset(&attr
, 0, sizeof(attr
));
1959 attr
.prog_type
= BPF_PROG_TYPE_SOCKET_FILTER
;
1961 attr
.insns_cnt
= ARRAY_SIZE(insns
);
1962 attr
.license
= "GPL";
1964 ret
= bpf_load_program_xattr(&attr
, NULL
, 0);
1966 cp
= libbpf_strerror_r(errno
, errmsg
, sizeof(errmsg
));
1967 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n",
1968 __func__
, cp
, errno
);
1973 /* now try the same program, but with the name */
1976 ret
= bpf_load_program_xattr(&attr
, NULL
, 0);
1986 bpf_object__probe_global_data(struct bpf_object
*obj
)
1988 struct bpf_load_program_attr prg_attr
;
1989 struct bpf_create_map_attr map_attr
;
1990 char *cp
, errmsg
[STRERR_BUFSIZE
];
1991 struct bpf_insn insns
[] = {
1992 BPF_LD_MAP_VALUE(BPF_REG_1
, 0, 16),
1993 BPF_ST_MEM(BPF_DW
, BPF_REG_1
, 0, 42),
1994 BPF_MOV64_IMM(BPF_REG_0
, 0),
1999 memset(&map_attr
, 0, sizeof(map_attr
));
2000 map_attr
.map_type
= BPF_MAP_TYPE_ARRAY
;
2001 map_attr
.key_size
= sizeof(int);
2002 map_attr
.value_size
= 32;
2003 map_attr
.max_entries
= 1;
2005 map
= bpf_create_map_xattr(&map_attr
);
2007 cp
= libbpf_strerror_r(errno
, errmsg
, sizeof(errmsg
));
2008 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n",
2009 __func__
, cp
, errno
);
2015 memset(&prg_attr
, 0, sizeof(prg_attr
));
2016 prg_attr
.prog_type
= BPF_PROG_TYPE_SOCKET_FILTER
;
2017 prg_attr
.insns
= insns
;
2018 prg_attr
.insns_cnt
= ARRAY_SIZE(insns
);
2019 prg_attr
.license
= "GPL";
2021 ret
= bpf_load_program_xattr(&prg_attr
, NULL
, 0);
2023 obj
->caps
.global_data
= 1;
2031 static int bpf_object__probe_btf_func(struct bpf_object
*obj
)
2033 const char strs
[] = "\0int\0x\0a";
2034 /* void x(int a) {} */
2037 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED
, 0, 32, 4), /* [1] */
2038 /* FUNC_PROTO */ /* [2] */
2039 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO
, 0, 1), 0),
2040 BTF_PARAM_ENC(7, 1),
2041 /* FUNC x */ /* [3] */
2042 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC
, 0, 0), 2),
2046 btf_fd
= libbpf__load_raw_btf((char *)types
, sizeof(types
),
2047 strs
, sizeof(strs
));
2049 obj
->caps
.btf_func
= 1;
2057 static int bpf_object__probe_btf_datasec(struct bpf_object
*obj
)
2059 const char strs
[] = "\0x\0.data";
2063 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED
, 0, 32, 4), /* [1] */
2064 /* VAR x */ /* [2] */
2065 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR
, 0, 0), 1),
2067 /* DATASEC val */ /* [3] */
2068 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC
, 0, 1), 4),
2069 BTF_VAR_SECINFO_ENC(2, 0, 4),
2073 btf_fd
= libbpf__load_raw_btf((char *)types
, sizeof(types
),
2074 strs
, sizeof(strs
));
2076 obj
->caps
.btf_datasec
= 1;
2085 bpf_object__probe_caps(struct bpf_object
*obj
)
2087 int (*probe_fn
[])(struct bpf_object
*obj
) = {
2088 bpf_object__probe_name
,
2089 bpf_object__probe_global_data
,
2090 bpf_object__probe_btf_func
,
2091 bpf_object__probe_btf_datasec
,
2095 for (i
= 0; i
< ARRAY_SIZE(probe_fn
); i
++) {
2096 ret
= probe_fn
[i
](obj
);
2098 pr_debug("Probe #%d failed with %d.\n", i
, ret
);
2105 bpf_object__populate_internal_map(struct bpf_object
*obj
, struct bpf_map
*map
)
2107 char *cp
, errmsg
[STRERR_BUFSIZE
];
2111 /* Nothing to do here since kernel already zero-initializes .bss map. */
2112 if (map
->libbpf_type
== LIBBPF_MAP_BSS
)
2115 data
= map
->libbpf_type
== LIBBPF_MAP_DATA
?
2116 obj
->sections
.data
: obj
->sections
.rodata
;
2118 err
= bpf_map_update_elem(map
->fd
, &zero
, data
, 0);
2119 /* Freeze .rodata map as read-only from syscall side. */
2120 if (!err
&& map
->libbpf_type
== LIBBPF_MAP_RODATA
) {
2121 err
= bpf_map_freeze(map
->fd
);
2123 cp
= libbpf_strerror_r(errno
, errmsg
, sizeof(errmsg
));
2124 pr_warning("Error freezing map(%s) as read-only: %s\n",
2133 bpf_object__create_maps(struct bpf_object
*obj
)
2135 struct bpf_create_map_attr create_attr
= {};
2140 for (i
= 0; i
< obj
->nr_maps
; i
++) {
2141 struct bpf_map
*map
= &obj
->maps
[i
];
2142 struct bpf_map_def
*def
= &map
->def
;
2143 char *cp
, errmsg
[STRERR_BUFSIZE
];
2144 int *pfd
= &map
->fd
;
2147 pr_debug("skip map create (preset) %s: fd=%d\n",
2148 map
->name
, map
->fd
);
2153 create_attr
.name
= map
->name
;
2154 create_attr
.map_ifindex
= map
->map_ifindex
;
2155 create_attr
.map_type
= def
->type
;
2156 create_attr
.map_flags
= def
->map_flags
;
2157 create_attr
.key_size
= def
->key_size
;
2158 create_attr
.value_size
= def
->value_size
;
2159 if (def
->type
== BPF_MAP_TYPE_PERF_EVENT_ARRAY
&&
2160 !def
->max_entries
) {
2162 nr_cpus
= libbpf_num_possible_cpus();
2164 pr_warning("failed to determine number of system CPUs: %d\n",
2169 pr_debug("map '%s': setting size to %d\n",
2170 map
->name
, nr_cpus
);
2171 create_attr
.max_entries
= nr_cpus
;
2173 create_attr
.max_entries
= def
->max_entries
;
2175 create_attr
.btf_fd
= 0;
2176 create_attr
.btf_key_type_id
= 0;
2177 create_attr
.btf_value_type_id
= 0;
2178 if (bpf_map_type__is_map_in_map(def
->type
) &&
2179 map
->inner_map_fd
>= 0)
2180 create_attr
.inner_map_fd
= map
->inner_map_fd
;
2182 if (obj
->btf
&& !bpf_map_find_btf_info(obj
, map
)) {
2183 create_attr
.btf_fd
= btf__fd(obj
->btf
);
2184 create_attr
.btf_key_type_id
= map
->btf_key_type_id
;
2185 create_attr
.btf_value_type_id
= map
->btf_value_type_id
;
2188 *pfd
= bpf_create_map_xattr(&create_attr
);
2189 if (*pfd
< 0 && (create_attr
.btf_key_type_id
||
2190 create_attr
.btf_value_type_id
)) {
2192 cp
= libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
));
2193 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
2194 map
->name
, cp
, err
);
2195 create_attr
.btf_fd
= 0;
2196 create_attr
.btf_key_type_id
= 0;
2197 create_attr
.btf_value_type_id
= 0;
2198 map
->btf_key_type_id
= 0;
2199 map
->btf_value_type_id
= 0;
2200 *pfd
= bpf_create_map_xattr(&create_attr
);
2208 cp
= libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
));
2209 pr_warning("failed to create map (name: '%s'): %s(%d)\n",
2210 map
->name
, cp
, err
);
2211 for (j
= 0; j
< i
; j
++)
2212 zclose(obj
->maps
[j
].fd
);
2216 if (bpf_map__is_internal(map
)) {
2217 err
= bpf_object__populate_internal_map(obj
, map
);
2224 pr_debug("created map %s: fd=%d\n", map
->name
, *pfd
);
2231 check_btf_ext_reloc_err(struct bpf_program
*prog
, int err
,
2232 void *btf_prog_info
, const char *info_name
)
2234 if (err
!= -ENOENT
) {
2235 pr_warning("Error in loading %s for sec %s.\n",
2236 info_name
, prog
->section_name
);
2240 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */
2242 if (btf_prog_info
) {
2244 * Some info has already been found but has problem
2245 * in the last btf_ext reloc. Must have to error out.
2247 pr_warning("Error in relocating %s for sec %s.\n",
2248 info_name
, prog
->section_name
);
2252 /* Have problem loading the very first info. Ignore the rest. */
2253 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n",
2254 info_name
, prog
->section_name
, info_name
);
2259 bpf_program_reloc_btf_ext(struct bpf_program
*prog
, struct bpf_object
*obj
,
2260 const char *section_name
, __u32 insn_offset
)
2264 if (!insn_offset
|| prog
->func_info
) {
2266 * !insn_offset => main program
2268 * For sub prog, the main program's func_info has to
2269 * be loaded first (i.e. prog->func_info != NULL)
2271 err
= btf_ext__reloc_func_info(obj
->btf
, obj
->btf_ext
,
2272 section_name
, insn_offset
,
2274 &prog
->func_info_cnt
);
2276 return check_btf_ext_reloc_err(prog
, err
,
2280 prog
->func_info_rec_size
= btf_ext__func_info_rec_size(obj
->btf_ext
);
2283 if (!insn_offset
|| prog
->line_info
) {
2284 err
= btf_ext__reloc_line_info(obj
->btf
, obj
->btf_ext
,
2285 section_name
, insn_offset
,
2287 &prog
->line_info_cnt
);
2289 return check_btf_ext_reloc_err(prog
, err
,
2293 prog
->line_info_rec_size
= btf_ext__line_info_rec_size(obj
->btf_ext
);
2299 #define BPF_CORE_SPEC_MAX_LEN 64
2301 /* represents BPF CO-RE field or array element accessor */
2302 struct bpf_core_accessor
{
2303 __u32 type_id
; /* struct/union type or array element type */
2304 __u32 idx
; /* field index or array index */
2305 const char *name
; /* field name or NULL for array accessor */
2308 struct bpf_core_spec
{
2309 const struct btf
*btf
;
2310 /* high-level spec: named fields and array indices only */
2311 struct bpf_core_accessor spec
[BPF_CORE_SPEC_MAX_LEN
];
2312 /* high-level spec length */
2314 /* raw, low-level spec: 1-to-1 with accessor spec string */
2315 int raw_spec
[BPF_CORE_SPEC_MAX_LEN
];
2316 /* raw spec length */
2318 /* field byte offset represented by spec */
2322 static bool str_is_empty(const char *s
)
2328 * Turn bpf_offset_reloc into a low- and high-level spec representation,
2329 * validating correctness along the way, as well as calculating resulting
2330 * field offset (in bytes), specified by accessor string. Low-level spec
2331 * captures every single level of nestedness, including traversing anonymous
2332 * struct/union members. High-level one only captures semantically meaningful
2333 * "turning points": named fields and array indicies.
2334 * E.g., for this case:
2337 * int __unimportant;
2345 * struct sample *s = ...;
2347 * int x = &s->a[3]; // access string = '0:1:2:3'
2349 * Low-level spec has 1:1 mapping with each element of access string (it's
2350 * just a parsed access string representation): [0, 1, 2, 3].
2352 * High-level spec will capture only 3 points:
2353 * - intial zero-index access by pointer (&s->... is the same as &s[0]...);
2354 * - field 'a' access (corresponds to '2' in low-level spec);
2355 * - array element #3 access (corresponds to '3' in low-level spec).
2358 static int bpf_core_spec_parse(const struct btf
*btf
,
2360 const char *spec_str
,
2361 struct bpf_core_spec
*spec
)
2363 int access_idx
, parsed_len
, i
;
2364 const struct btf_type
*t
;
2369 if (str_is_empty(spec_str
) || *spec_str
== ':')
2372 memset(spec
, 0, sizeof(*spec
));
2375 /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
2377 if (*spec_str
== ':')
2379 if (sscanf(spec_str
, "%d%n", &access_idx
, &parsed_len
) != 1)
2381 if (spec
->raw_len
== BPF_CORE_SPEC_MAX_LEN
)
2383 spec_str
+= parsed_len
;
2384 spec
->raw_spec
[spec
->raw_len
++] = access_idx
;
2387 if (spec
->raw_len
== 0)
2390 /* first spec value is always reloc type array index */
2391 t
= skip_mods_and_typedefs(btf
, type_id
, &id
);
2395 access_idx
= spec
->raw_spec
[0];
2396 spec
->spec
[0].type_id
= id
;
2397 spec
->spec
[0].idx
= access_idx
;
2400 sz
= btf__resolve_size(btf
, id
);
2403 spec
->offset
= access_idx
* sz
;
2405 for (i
= 1; i
< spec
->raw_len
; i
++) {
2406 t
= skip_mods_and_typedefs(btf
, id
, &id
);
2410 access_idx
= spec
->raw_spec
[i
];
2412 if (btf_is_composite(t
)) {
2413 const struct btf_member
*m
;
2416 if (access_idx
>= btf_vlen(t
))
2418 if (btf_member_bitfield_size(t
, access_idx
))
2421 offset
= btf_member_bit_offset(t
, access_idx
);
2424 spec
->offset
+= offset
/ 8;
2426 m
= btf_members(t
) + access_idx
;
2428 name
= btf__name_by_offset(btf
, m
->name_off
);
2429 if (str_is_empty(name
))
2432 spec
->spec
[spec
->len
].type_id
= id
;
2433 spec
->spec
[spec
->len
].idx
= access_idx
;
2434 spec
->spec
[spec
->len
].name
= name
;
2439 } else if (btf_is_array(t
)) {
2440 const struct btf_array
*a
= btf_array(t
);
2442 t
= skip_mods_and_typedefs(btf
, a
->type
, &id
);
2443 if (!t
|| access_idx
>= a
->nelems
)
2446 spec
->spec
[spec
->len
].type_id
= id
;
2447 spec
->spec
[spec
->len
].idx
= access_idx
;
2450 sz
= btf__resolve_size(btf
, id
);
2453 spec
->offset
+= access_idx
* sz
;
2455 pr_warning("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n",
2456 type_id
, spec_str
, i
, id
, btf_kind(t
));
2464 static bool bpf_core_is_flavor_sep(const char *s
)
2466 /* check X___Y name pattern, where X and Y are not underscores */
2467 return s
[0] != '_' && /* X */
2468 s
[1] == '_' && s
[2] == '_' && s
[3] == '_' && /* ___ */
2469 s
[4] != '_'; /* Y */
2472 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
2473 * before last triple underscore. Struct name part after last triple
2474 * underscore is ignored by BPF CO-RE relocation during relocation matching.
2476 static size_t bpf_core_essential_name_len(const char *name
)
2478 size_t n
= strlen(name
);
2481 for (i
= n
- 5; i
>= 0; i
--) {
2482 if (bpf_core_is_flavor_sep(name
+ i
))
2488 /* dynamically sized list of type IDs */
2494 static void bpf_core_free_cands(struct ids_vec
*cand_ids
)
2496 free(cand_ids
->data
);
2500 static struct ids_vec
*bpf_core_find_cands(const struct btf
*local_btf
,
2501 __u32 local_type_id
,
2502 const struct btf
*targ_btf
)
2504 size_t local_essent_len
, targ_essent_len
;
2505 const char *local_name
, *targ_name
;
2506 const struct btf_type
*t
;
2507 struct ids_vec
*cand_ids
;
2511 t
= btf__type_by_id(local_btf
, local_type_id
);
2513 return ERR_PTR(-EINVAL
);
2515 local_name
= btf__name_by_offset(local_btf
, t
->name_off
);
2516 if (str_is_empty(local_name
))
2517 return ERR_PTR(-EINVAL
);
2518 local_essent_len
= bpf_core_essential_name_len(local_name
);
2520 cand_ids
= calloc(1, sizeof(*cand_ids
));
2522 return ERR_PTR(-ENOMEM
);
2524 n
= btf__get_nr_types(targ_btf
);
2525 for (i
= 1; i
<= n
; i
++) {
2526 t
= btf__type_by_id(targ_btf
, i
);
2527 targ_name
= btf__name_by_offset(targ_btf
, t
->name_off
);
2528 if (str_is_empty(targ_name
))
2531 targ_essent_len
= bpf_core_essential_name_len(targ_name
);
2532 if (targ_essent_len
!= local_essent_len
)
2535 if (strncmp(local_name
, targ_name
, local_essent_len
) == 0) {
2536 pr_debug("[%d] %s: found candidate [%d] %s\n",
2537 local_type_id
, local_name
, i
, targ_name
);
2538 new_ids
= realloc(cand_ids
->data
, cand_ids
->len
+ 1);
2543 cand_ids
->data
= new_ids
;
2544 cand_ids
->data
[cand_ids
->len
++] = i
;
2549 bpf_core_free_cands(cand_ids
);
2550 return ERR_PTR(err
);
2553 /* Check two types for compatibility, skipping const/volatile/restrict and
2554 * typedefs, to ensure we are relocating offset to the compatible entities:
2555 * - any two STRUCTs/UNIONs are compatible and can be mixed;
2556 * - any two FWDs are compatible;
2557 * - any two PTRs are always compatible;
2558 * - for ENUMs, check sizes, names are ignored;
2559 * - for INT, size and bitness should match, signedness is ignored;
2560 * - for ARRAY, dimensionality is ignored, element types are checked for
2561 * compatibility recursively;
2562 * - everything else shouldn't be ever a target of relocation.
2563 * These rules are not set in stone and probably will be adjusted as we get
2564 * more experience with using BPF CO-RE relocations.
2566 static int bpf_core_fields_are_compat(const struct btf
*local_btf
,
2568 const struct btf
*targ_btf
,
2571 const struct btf_type
*local_type
, *targ_type
;
2574 local_type
= skip_mods_and_typedefs(local_btf
, local_id
, &local_id
);
2575 targ_type
= skip_mods_and_typedefs(targ_btf
, targ_id
, &targ_id
);
2576 if (!local_type
|| !targ_type
)
2579 if (btf_is_composite(local_type
) && btf_is_composite(targ_type
))
2581 if (btf_kind(local_type
) != btf_kind(targ_type
))
2584 switch (btf_kind(local_type
)) {
2589 return local_type
->size
== targ_type
->size
;
2591 return btf_int_offset(local_type
) == 0 &&
2592 btf_int_offset(targ_type
) == 0 &&
2593 local_type
->size
== targ_type
->size
&&
2594 btf_int_bits(local_type
) == btf_int_bits(targ_type
);
2595 case BTF_KIND_ARRAY
:
2596 local_id
= btf_array(local_type
)->type
;
2597 targ_id
= btf_array(targ_type
)->type
;
2600 pr_warning("unexpected kind %d relocated, local [%d], target [%d]\n",
2601 btf_kind(local_type
), local_id
, targ_id
);
2607 * Given single high-level named field accessor in local type, find
2608 * corresponding high-level accessor for a target type. Along the way,
2609 * maintain low-level spec for target as well. Also keep updating target
2612 * Searching is performed through recursive exhaustive enumeration of all
2613 * fields of a struct/union. If there are any anonymous (embedded)
2614 * structs/unions, they are recursively searched as well. If field with
2615 * desired name is found, check compatibility between local and target types,
2616 * before returning result.
2618 * 1 is returned, if field is found.
2619 * 0 is returned if no compatible field is found.
2620 * <0 is returned on error.
2622 static int bpf_core_match_member(const struct btf
*local_btf
,
2623 const struct bpf_core_accessor
*local_acc
,
2624 const struct btf
*targ_btf
,
2626 struct bpf_core_spec
*spec
,
2627 __u32
*next_targ_id
)
2629 const struct btf_type
*local_type
, *targ_type
;
2630 const struct btf_member
*local_member
, *m
;
2631 const char *local_name
, *targ_name
;
2635 targ_type
= skip_mods_and_typedefs(targ_btf
, targ_id
, &targ_id
);
2638 if (!btf_is_composite(targ_type
))
2641 local_id
= local_acc
->type_id
;
2642 local_type
= btf__type_by_id(local_btf
, local_id
);
2643 local_member
= btf_members(local_type
) + local_acc
->idx
;
2644 local_name
= btf__name_by_offset(local_btf
, local_member
->name_off
);
2646 n
= btf_vlen(targ_type
);
2647 m
= btf_members(targ_type
);
2648 for (i
= 0; i
< n
; i
++, m
++) {
2651 /* bitfield relocations not supported */
2652 if (btf_member_bitfield_size(targ_type
, i
))
2654 offset
= btf_member_bit_offset(targ_type
, i
);
2658 /* too deep struct/union/array nesting */
2659 if (spec
->raw_len
== BPF_CORE_SPEC_MAX_LEN
)
2662 /* speculate this member will be the good one */
2663 spec
->offset
+= offset
/ 8;
2664 spec
->raw_spec
[spec
->raw_len
++] = i
;
2666 targ_name
= btf__name_by_offset(targ_btf
, m
->name_off
);
2667 if (str_is_empty(targ_name
)) {
2668 /* embedded struct/union, we need to go deeper */
2669 found
= bpf_core_match_member(local_btf
, local_acc
,
2671 spec
, next_targ_id
);
2672 if (found
) /* either found or error */
2674 } else if (strcmp(local_name
, targ_name
) == 0) {
2675 /* matching named field */
2676 struct bpf_core_accessor
*targ_acc
;
2678 targ_acc
= &spec
->spec
[spec
->len
++];
2679 targ_acc
->type_id
= targ_id
;
2681 targ_acc
->name
= targ_name
;
2683 *next_targ_id
= m
->type
;
2684 found
= bpf_core_fields_are_compat(local_btf
,
2688 spec
->len
--; /* pop accessor */
2691 /* member turned out not to be what we looked for */
2692 spec
->offset
-= offset
/ 8;
2700 * Try to match local spec to a target type and, if successful, produce full
2701 * target spec (high-level, low-level + offset).
2703 static int bpf_core_spec_match(struct bpf_core_spec
*local_spec
,
2704 const struct btf
*targ_btf
, __u32 targ_id
,
2705 struct bpf_core_spec
*targ_spec
)
2707 const struct btf_type
*targ_type
;
2708 const struct bpf_core_accessor
*local_acc
;
2709 struct bpf_core_accessor
*targ_acc
;
2712 memset(targ_spec
, 0, sizeof(*targ_spec
));
2713 targ_spec
->btf
= targ_btf
;
2715 local_acc
= &local_spec
->spec
[0];
2716 targ_acc
= &targ_spec
->spec
[0];
2718 for (i
= 0; i
< local_spec
->len
; i
++, local_acc
++, targ_acc
++) {
2719 targ_type
= skip_mods_and_typedefs(targ_spec
->btf
, targ_id
,
2724 if (local_acc
->name
) {
2725 matched
= bpf_core_match_member(local_spec
->btf
,
2728 targ_spec
, &targ_id
);
2732 /* for i=0, targ_id is already treated as array element
2733 * type (because it's the original struct), for others
2734 * we should find array element type first
2737 const struct btf_array
*a
;
2739 if (!btf_is_array(targ_type
))
2742 a
= btf_array(targ_type
);
2743 if (local_acc
->idx
>= a
->nelems
)
2745 if (!skip_mods_and_typedefs(targ_btf
, a
->type
,
2750 /* too deep struct/union/array nesting */
2751 if (targ_spec
->raw_len
== BPF_CORE_SPEC_MAX_LEN
)
2754 targ_acc
->type_id
= targ_id
;
2755 targ_acc
->idx
= local_acc
->idx
;
2756 targ_acc
->name
= NULL
;
2758 targ_spec
->raw_spec
[targ_spec
->raw_len
] = targ_acc
->idx
;
2759 targ_spec
->raw_len
++;
2761 sz
= btf__resolve_size(targ_btf
, targ_id
);
2764 targ_spec
->offset
+= local_acc
->idx
* sz
;
2772 * Patch relocatable BPF instruction.
2773 * Expected insn->imm value is provided for validation, as well as the new
2776 * Currently three kinds of BPF instructions are supported:
2777 * 1. rX = <imm> (assignment with immediate operand);
2778 * 2. rX += <imm> (arithmetic operations with immediate operand);
2779 * 3. *(rX) = <imm> (indirect memory assignment with immediate operand).
2781 * If actual insn->imm value is wrong, bail out.
2783 static int bpf_core_reloc_insn(struct bpf_program
*prog
, int insn_off
,
2784 __u32 orig_off
, __u32 new_off
)
2786 struct bpf_insn
*insn
;
2790 if (insn_off
% sizeof(struct bpf_insn
))
2792 insn_idx
= insn_off
/ sizeof(struct bpf_insn
);
2794 insn
= &prog
->insns
[insn_idx
];
2795 class = BPF_CLASS(insn
->code
);
2797 if (class == BPF_ALU
|| class == BPF_ALU64
) {
2798 if (BPF_SRC(insn
->code
) != BPF_K
)
2800 if (insn
->imm
!= orig_off
)
2802 insn
->imm
= new_off
;
2803 pr_debug("prog '%s': patched insn #%d (ALU/ALU64) imm %d -> %d\n",
2804 bpf_program__title(prog
, false),
2805 insn_idx
, orig_off
, new_off
);
2807 pr_warning("prog '%s': trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n",
2808 bpf_program__title(prog
, false),
2809 insn_idx
, insn
->code
, insn
->src_reg
, insn
->dst_reg
,
2810 insn
->off
, insn
->imm
);
2816 static struct btf
*btf_load_raw(const char *path
)
2824 if (stat(path
, &st
))
2825 return ERR_PTR(-errno
);
2827 data
= malloc(st
.st_size
);
2829 return ERR_PTR(-ENOMEM
);
2831 f
= fopen(path
, "rb");
2833 btf
= ERR_PTR(-errno
);
2837 read_cnt
= fread(data
, 1, st
.st_size
, f
);
2839 if (read_cnt
< st
.st_size
) {
2840 btf
= ERR_PTR(-EBADF
);
2844 btf
= btf__new(data
, read_cnt
);
2852 * Probe few well-known locations for vmlinux kernel image and try to load BTF
2853 * data out of it to use for target BTF.
2855 static struct btf
*bpf_core_find_kernel_btf(void)
2858 const char *path_fmt
;
2861 /* try canonical vmlinux BTF through sysfs first */
2862 { "/sys/kernel/btf/vmlinux", true /* raw BTF */ },
2863 /* fall back to trying to find vmlinux ELF on disk otherwise */
2864 { "/boot/vmlinux-%1$s" },
2865 { "/lib/modules/%1$s/vmlinux-%1$s" },
2866 { "/lib/modules/%1$s/build/vmlinux" },
2867 { "/usr/lib/modules/%1$s/kernel/vmlinux" },
2868 { "/usr/lib/debug/boot/vmlinux-%1$s" },
2869 { "/usr/lib/debug/boot/vmlinux-%1$s.debug" },
2870 { "/usr/lib/debug/lib/modules/%1$s/vmlinux" },
2872 char path
[PATH_MAX
+ 1];
2879 for (i
= 0; i
< ARRAY_SIZE(locations
); i
++) {
2880 snprintf(path
, PATH_MAX
, locations
[i
].path_fmt
, buf
.release
);
2882 if (access(path
, R_OK
))
2885 if (locations
[i
].raw_btf
)
2886 btf
= btf_load_raw(path
);
2888 btf
= btf__parse_elf(path
, NULL
);
2890 pr_debug("loading kernel BTF '%s': %ld\n",
2891 path
, IS_ERR(btf
) ? PTR_ERR(btf
) : 0);
2898 pr_warning("failed to find valid kernel BTF\n");
2899 return ERR_PTR(-ESRCH
);
2902 /* Output spec definition in the format:
2903 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
2904 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
2906 static void bpf_core_dump_spec(int level
, const struct bpf_core_spec
*spec
)
2908 const struct btf_type
*t
;
2913 type_id
= spec
->spec
[0].type_id
;
2914 t
= btf__type_by_id(spec
->btf
, type_id
);
2915 s
= btf__name_by_offset(spec
->btf
, t
->name_off
);
2916 libbpf_print(level
, "[%u] %s + ", type_id
, s
);
2918 for (i
= 0; i
< spec
->raw_len
; i
++)
2919 libbpf_print(level
, "%d%s", spec
->raw_spec
[i
],
2920 i
== spec
->raw_len
- 1 ? " => " : ":");
2922 libbpf_print(level
, "%u @ &x", spec
->offset
);
2924 for (i
= 0; i
< spec
->len
; i
++) {
2925 if (spec
->spec
[i
].name
)
2926 libbpf_print(level
, ".%s", spec
->spec
[i
].name
);
2928 libbpf_print(level
, "[%u]", spec
->spec
[i
].idx
);
2933 static size_t bpf_core_hash_fn(const void *key
, void *ctx
)
2938 static bool bpf_core_equal_fn(const void *k1
, const void *k2
, void *ctx
)
2943 static void *u32_as_hash_key(__u32 x
)
2945 return (void *)(uintptr_t)x
;
2949 * CO-RE relocate single instruction.
2951 * The outline and important points of the algorithm:
2952 * 1. For given local type, find corresponding candidate target types.
2953 * Candidate type is a type with the same "essential" name, ignoring
2954 * everything after last triple underscore (___). E.g., `sample`,
2955 * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
2956 * for each other. Names with triple underscore are referred to as
2957 * "flavors" and are useful, among other things, to allow to
2958 * specify/support incompatible variations of the same kernel struct, which
2959 * might differ between different kernel versions and/or build
2962 * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
2963 * converter, when deduplicated BTF of a kernel still contains more than
2964 * one different types with the same name. In that case, ___2, ___3, etc
2965 * are appended starting from second name conflict. But start flavors are
2966 * also useful to be defined "locally", in BPF program, to extract same
2967 * data from incompatible changes between different kernel
2968 * versions/configurations. For instance, to handle field renames between
2969 * kernel versions, one can use two flavors of the struct name with the
2970 * same common name and use conditional relocations to extract that field,
2971 * depending on target kernel version.
2972 * 2. For each candidate type, try to match local specification to this
2973 * candidate target type. Matching involves finding corresponding
2974 * high-level spec accessors, meaning that all named fields should match,
2975 * as well as all array accesses should be within the actual bounds. Also,
2976 * types should be compatible (see bpf_core_fields_are_compat for details).
2977 * 3. It is supported and expected that there might be multiple flavors
2978 * matching the spec. As long as all the specs resolve to the same set of
2979 * offsets across all candidates, there is not error. If there is any
2980 * ambiguity, CO-RE relocation will fail. This is necessary to accomodate
2981 * imprefection of BTF deduplication, which can cause slight duplication of
2982 * the same BTF type, if some directly or indirectly referenced (by
2983 * pointer) type gets resolved to different actual types in different
2984 * object files. If such situation occurs, deduplicated BTF will end up
2985 * with two (or more) structurally identical types, which differ only in
2986 * types they refer to through pointer. This should be OK in most cases and
2988 * 4. Candidate types search is performed by linearly scanning through all
2989 * types in target BTF. It is anticipated that this is overall more
2990 * efficient memory-wise and not significantly worse (if not better)
2991 * CPU-wise compared to prebuilding a map from all local type names to
2992 * a list of candidate type names. It's also sped up by caching resolved
2993 * list of matching candidates per each local "root" type ID, that has at
2994 * least one bpf_offset_reloc associated with it. This list is shared
2995 * between multiple relocations for the same type ID and is updated as some
2996 * of the candidates are pruned due to structural incompatibility.
2998 static int bpf_core_reloc_offset(struct bpf_program
*prog
,
2999 const struct bpf_offset_reloc
*relo
,
3001 const struct btf
*local_btf
,
3002 const struct btf
*targ_btf
,
3003 struct hashmap
*cand_cache
)
3005 const char *prog_name
= bpf_program__title(prog
, false);
3006 struct bpf_core_spec local_spec
, cand_spec
, targ_spec
;
3007 const void *type_key
= u32_as_hash_key(relo
->type_id
);
3008 const struct btf_type
*local_type
, *cand_type
;
3009 const char *local_name
, *cand_name
;
3010 struct ids_vec
*cand_ids
;
3011 __u32 local_id
, cand_id
;
3012 const char *spec_str
;
3015 local_id
= relo
->type_id
;
3016 local_type
= btf__type_by_id(local_btf
, local_id
);
3020 local_name
= btf__name_by_offset(local_btf
, local_type
->name_off
);
3021 if (str_is_empty(local_name
))
3024 spec_str
= btf__name_by_offset(local_btf
, relo
->access_str_off
);
3025 if (str_is_empty(spec_str
))
3028 err
= bpf_core_spec_parse(local_btf
, local_id
, spec_str
, &local_spec
);
3030 pr_warning("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n",
3031 prog_name
, relo_idx
, local_id
, local_name
, spec_str
,
3036 pr_debug("prog '%s': relo #%d: spec is ", prog_name
, relo_idx
);
3037 bpf_core_dump_spec(LIBBPF_DEBUG
, &local_spec
);
3038 libbpf_print(LIBBPF_DEBUG
, "\n");
3040 if (!hashmap__find(cand_cache
, type_key
, (void **)&cand_ids
)) {
3041 cand_ids
= bpf_core_find_cands(local_btf
, local_id
, targ_btf
);
3042 if (IS_ERR(cand_ids
)) {
3043 pr_warning("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld",
3044 prog_name
, relo_idx
, local_id
, local_name
,
3046 return PTR_ERR(cand_ids
);
3048 err
= hashmap__set(cand_cache
, type_key
, cand_ids
, NULL
, NULL
);
3050 bpf_core_free_cands(cand_ids
);
3055 for (i
= 0, j
= 0; i
< cand_ids
->len
; i
++) {
3056 cand_id
= cand_ids
->data
[i
];
3057 cand_type
= btf__type_by_id(targ_btf
, cand_id
);
3058 cand_name
= btf__name_by_offset(targ_btf
, cand_type
->name_off
);
3060 err
= bpf_core_spec_match(&local_spec
, targ_btf
,
3061 cand_id
, &cand_spec
);
3062 pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ",
3063 prog_name
, relo_idx
, i
, cand_name
);
3064 bpf_core_dump_spec(LIBBPF_DEBUG
, &cand_spec
);
3065 libbpf_print(LIBBPF_DEBUG
, ": %d\n", err
);
3067 pr_warning("prog '%s': relo #%d: matching error: %d\n",
3068 prog_name
, relo_idx
, err
);
3075 targ_spec
= cand_spec
;
3076 } else if (cand_spec
.offset
!= targ_spec
.offset
) {
3077 /* if there are many candidates, they should all
3078 * resolve to the same offset
3080 pr_warning("prog '%s': relo #%d: offset ambiguity: %u != %u\n",
3081 prog_name
, relo_idx
, cand_spec
.offset
,
3086 cand_ids
->data
[j
++] = cand_spec
.spec
[0].type_id
;
3090 if (cand_ids
->len
== 0) {
3091 pr_warning("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n",
3092 prog_name
, relo_idx
, local_id
, local_name
, spec_str
);
3096 err
= bpf_core_reloc_insn(prog
, relo
->insn_off
,
3097 local_spec
.offset
, targ_spec
.offset
);
3099 pr_warning("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
3100 prog_name
, relo_idx
, relo
->insn_off
, err
);
3108 bpf_core_reloc_offsets(struct bpf_object
*obj
, const char *targ_btf_path
)
3110 const struct btf_ext_info_sec
*sec
;
3111 const struct bpf_offset_reloc
*rec
;
3112 const struct btf_ext_info
*seg
;
3113 struct hashmap_entry
*entry
;
3114 struct hashmap
*cand_cache
= NULL
;
3115 struct bpf_program
*prog
;
3116 struct btf
*targ_btf
;
3117 const char *sec_name
;
3121 targ_btf
= btf__parse_elf(targ_btf_path
, NULL
);
3123 targ_btf
= bpf_core_find_kernel_btf();
3124 if (IS_ERR(targ_btf
)) {
3125 pr_warning("failed to get target BTF: %ld\n",
3127 return PTR_ERR(targ_btf
);
3130 cand_cache
= hashmap__new(bpf_core_hash_fn
, bpf_core_equal_fn
, NULL
);
3131 if (IS_ERR(cand_cache
)) {
3132 err
= PTR_ERR(cand_cache
);
3136 seg
= &obj
->btf_ext
->offset_reloc_info
;
3137 for_each_btf_ext_sec(seg
, sec
) {
3138 sec_name
= btf__name_by_offset(obj
->btf
, sec
->sec_name_off
);
3139 if (str_is_empty(sec_name
)) {
3143 prog
= bpf_object__find_program_by_title(obj
, sec_name
);
3145 pr_warning("failed to find program '%s' for CO-RE offset relocation\n",
3151 pr_debug("prog '%s': performing %d CO-RE offset relocs\n",
3152 sec_name
, sec
->num_info
);
3154 for_each_btf_ext_rec(seg
, sec
, i
, rec
) {
3155 err
= bpf_core_reloc_offset(prog
, rec
, i
, obj
->btf
,
3156 targ_btf
, cand_cache
);
3158 pr_warning("prog '%s': relo #%d: failed to relocate: %d\n",
3166 btf__free(targ_btf
);
3167 if (!IS_ERR_OR_NULL(cand_cache
)) {
3168 hashmap__for_each_entry(cand_cache
, entry
, i
) {
3169 bpf_core_free_cands(entry
->value
);
3171 hashmap__free(cand_cache
);
3177 bpf_object__relocate_core(struct bpf_object
*obj
, const char *targ_btf_path
)
3181 if (obj
->btf_ext
->offset_reloc_info
.len
)
3182 err
= bpf_core_reloc_offsets(obj
, targ_btf_path
);
3188 bpf_program__reloc_text(struct bpf_program
*prog
, struct bpf_object
*obj
,
3189 struct reloc_desc
*relo
)
3191 struct bpf_insn
*insn
, *new_insn
;
3192 struct bpf_program
*text
;
3196 if (relo
->type
!= RELO_CALL
)
3197 return -LIBBPF_ERRNO__RELOC
;
3199 if (prog
->idx
== obj
->efile
.text_shndx
) {
3200 pr_warning("relo in .text insn %d into off %d\n",
3201 relo
->insn_idx
, relo
->text_off
);
3202 return -LIBBPF_ERRNO__RELOC
;
3205 if (prog
->main_prog_cnt
== 0) {
3206 text
= bpf_object__find_prog_by_idx(obj
, obj
->efile
.text_shndx
);
3208 pr_warning("no .text section found yet relo into text exist\n");
3209 return -LIBBPF_ERRNO__RELOC
;
3211 new_cnt
= prog
->insns_cnt
+ text
->insns_cnt
;
3212 new_insn
= reallocarray(prog
->insns
, new_cnt
, sizeof(*insn
));
3214 pr_warning("oom in prog realloc\n");
3219 err
= bpf_program_reloc_btf_ext(prog
, obj
,
3226 memcpy(new_insn
+ prog
->insns_cnt
, text
->insns
,
3227 text
->insns_cnt
* sizeof(*insn
));
3228 prog
->insns
= new_insn
;
3229 prog
->main_prog_cnt
= prog
->insns_cnt
;
3230 prog
->insns_cnt
= new_cnt
;
3231 pr_debug("added %zd insn from %s to prog %s\n",
3232 text
->insns_cnt
, text
->section_name
,
3233 prog
->section_name
);
3235 insn
= &prog
->insns
[relo
->insn_idx
];
3236 insn
->imm
+= prog
->main_prog_cnt
- relo
->insn_idx
;
3241 bpf_program__relocate(struct bpf_program
*prog
, struct bpf_object
*obj
)
3249 err
= bpf_program_reloc_btf_ext(prog
, obj
,
3250 prog
->section_name
, 0);
3255 if (!prog
->reloc_desc
)
3258 for (i
= 0; i
< prog
->nr_reloc
; i
++) {
3259 if (prog
->reloc_desc
[i
].type
== RELO_LD64
||
3260 prog
->reloc_desc
[i
].type
== RELO_DATA
) {
3261 bool relo_data
= prog
->reloc_desc
[i
].type
== RELO_DATA
;
3262 struct bpf_insn
*insns
= prog
->insns
;
3263 int insn_idx
, map_idx
;
3265 insn_idx
= prog
->reloc_desc
[i
].insn_idx
;
3266 map_idx
= prog
->reloc_desc
[i
].map_idx
;
3268 if (insn_idx
+ 1 >= (int)prog
->insns_cnt
) {
3269 pr_warning("relocation out of range: '%s'\n",
3270 prog
->section_name
);
3271 return -LIBBPF_ERRNO__RELOC
;
3275 insns
[insn_idx
].src_reg
= BPF_PSEUDO_MAP_FD
;
3277 insns
[insn_idx
].src_reg
= BPF_PSEUDO_MAP_VALUE
;
3278 insns
[insn_idx
+ 1].imm
= insns
[insn_idx
].imm
;
3280 insns
[insn_idx
].imm
= obj
->maps
[map_idx
].fd
;
3281 } else if (prog
->reloc_desc
[i
].type
== RELO_CALL
) {
3282 err
= bpf_program__reloc_text(prog
, obj
,
3283 &prog
->reloc_desc
[i
]);
3289 zfree(&prog
->reloc_desc
);
3295 bpf_object__relocate(struct bpf_object
*obj
, const char *targ_btf_path
)
3297 struct bpf_program
*prog
;
3302 err
= bpf_object__relocate_core(obj
, targ_btf_path
);
3304 pr_warning("failed to perform CO-RE relocations: %d\n",
3309 for (i
= 0; i
< obj
->nr_programs
; i
++) {
3310 prog
= &obj
->programs
[i
];
3312 err
= bpf_program__relocate(prog
, obj
);
3314 pr_warning("failed to relocate '%s'\n",
3315 prog
->section_name
);
3322 static int bpf_object__collect_reloc(struct bpf_object
*obj
)
3326 if (!obj_elf_valid(obj
)) {
3327 pr_warning("Internal error: elf object is closed\n");
3328 return -LIBBPF_ERRNO__INTERNAL
;
3331 for (i
= 0; i
< obj
->efile
.nr_reloc
; i
++) {
3332 GElf_Shdr
*shdr
= &obj
->efile
.reloc
[i
].shdr
;
3333 Elf_Data
*data
= obj
->efile
.reloc
[i
].data
;
3334 int idx
= shdr
->sh_info
;
3335 struct bpf_program
*prog
;
3337 if (shdr
->sh_type
!= SHT_REL
) {
3338 pr_warning("internal error at %d\n", __LINE__
);
3339 return -LIBBPF_ERRNO__INTERNAL
;
3342 prog
= bpf_object__find_prog_by_idx(obj
, idx
);
3344 pr_warning("relocation failed: no section(%d)\n", idx
);
3345 return -LIBBPF_ERRNO__RELOC
;
3348 err
= bpf_program__collect_reloc(prog
, shdr
, data
, obj
);
3356 load_program(struct bpf_program
*prog
, struct bpf_insn
*insns
, int insns_cnt
,
3357 char *license
, __u32 kern_version
, int *pfd
)
3359 struct bpf_load_program_attr load_attr
;
3360 char *cp
, errmsg
[STRERR_BUFSIZE
];
3361 int log_buf_size
= BPF_LOG_BUF_SIZE
;
3365 if (!insns
|| !insns_cnt
)
3368 memset(&load_attr
, 0, sizeof(struct bpf_load_program_attr
));
3369 load_attr
.prog_type
= prog
->type
;
3370 load_attr
.expected_attach_type
= prog
->expected_attach_type
;
3371 if (prog
->caps
->name
)
3372 load_attr
.name
= prog
->name
;
3373 load_attr
.insns
= insns
;
3374 load_attr
.insns_cnt
= insns_cnt
;
3375 load_attr
.license
= license
;
3376 load_attr
.kern_version
= kern_version
;
3377 load_attr
.prog_ifindex
= prog
->prog_ifindex
;
3378 /* if .BTF.ext was loaded, kernel supports associated BTF for prog */
3379 if (prog
->obj
->btf_ext
)
3380 btf_fd
= bpf_object__btf_fd(prog
->obj
);
3383 load_attr
.prog_btf_fd
= btf_fd
>= 0 ? btf_fd
: 0;
3384 load_attr
.func_info
= prog
->func_info
;
3385 load_attr
.func_info_rec_size
= prog
->func_info_rec_size
;
3386 load_attr
.func_info_cnt
= prog
->func_info_cnt
;
3387 load_attr
.line_info
= prog
->line_info
;
3388 load_attr
.line_info_rec_size
= prog
->line_info_rec_size
;
3389 load_attr
.line_info_cnt
= prog
->line_info_cnt
;
3390 load_attr
.log_level
= prog
->log_level
;
3391 load_attr
.prog_flags
= prog
->prog_flags
;
3394 log_buf
= malloc(log_buf_size
);
3396 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
3398 ret
= bpf_load_program_xattr(&load_attr
, log_buf
, log_buf_size
);
3401 if (load_attr
.log_level
)
3402 pr_debug("verifier log:\n%s", log_buf
);
3408 if (errno
== ENOSPC
) {
3413 ret
= -LIBBPF_ERRNO__LOAD
;
3414 cp
= libbpf_strerror_r(errno
, errmsg
, sizeof(errmsg
));
3415 pr_warning("load bpf program failed: %s\n", cp
);
3417 if (log_buf
&& log_buf
[0] != '\0') {
3418 ret
= -LIBBPF_ERRNO__VERIFY
;
3419 pr_warning("-- BEGIN DUMP LOG ---\n");
3420 pr_warning("\n%s\n", log_buf
);
3421 pr_warning("-- END LOG --\n");
3422 } else if (load_attr
.insns_cnt
>= BPF_MAXINSNS
) {
3423 pr_warning("Program too large (%zu insns), at most %d insns\n",
3424 load_attr
.insns_cnt
, BPF_MAXINSNS
);
3425 ret
= -LIBBPF_ERRNO__PROG2BIG
;
3427 /* Wrong program type? */
3428 if (load_attr
.prog_type
!= BPF_PROG_TYPE_KPROBE
) {
3431 load_attr
.prog_type
= BPF_PROG_TYPE_KPROBE
;
3432 load_attr
.expected_attach_type
= 0;
3433 fd
= bpf_load_program_xattr(&load_attr
, NULL
, 0);
3436 ret
= -LIBBPF_ERRNO__PROGTYPE
;
3442 ret
= -LIBBPF_ERRNO__KVER
;
3451 bpf_program__load(struct bpf_program
*prog
,
3452 char *license
, __u32 kern_version
)
3456 if (prog
->instances
.nr
< 0 || !prog
->instances
.fds
) {
3457 if (prog
->preprocessor
) {
3458 pr_warning("Internal error: can't load program '%s'\n",
3459 prog
->section_name
);
3460 return -LIBBPF_ERRNO__INTERNAL
;
3463 prog
->instances
.fds
= malloc(sizeof(int));
3464 if (!prog
->instances
.fds
) {
3465 pr_warning("Not enough memory for BPF fds\n");
3468 prog
->instances
.nr
= 1;
3469 prog
->instances
.fds
[0] = -1;
3472 if (!prog
->preprocessor
) {
3473 if (prog
->instances
.nr
!= 1) {
3474 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
3475 prog
->section_name
, prog
->instances
.nr
);
3477 err
= load_program(prog
, prog
->insns
, prog
->insns_cnt
,
3478 license
, kern_version
, &fd
);
3480 prog
->instances
.fds
[0] = fd
;
3484 for (i
= 0; i
< prog
->instances
.nr
; i
++) {
3485 struct bpf_prog_prep_result result
;
3486 bpf_program_prep_t preprocessor
= prog
->preprocessor
;
3488 memset(&result
, 0, sizeof(result
));
3489 err
= preprocessor(prog
, i
, prog
->insns
,
3490 prog
->insns_cnt
, &result
);
3492 pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
3493 i
, prog
->section_name
);
3497 if (!result
.new_insn_ptr
|| !result
.new_insn_cnt
) {
3498 pr_debug("Skip loading the %dth instance of program '%s'\n",
3499 i
, prog
->section_name
);
3500 prog
->instances
.fds
[i
] = -1;
3506 err
= load_program(prog
, result
.new_insn_ptr
,
3507 result
.new_insn_cnt
,
3508 license
, kern_version
, &fd
);
3511 pr_warning("Loading the %dth instance of program '%s' failed\n",
3512 i
, prog
->section_name
);
3518 prog
->instances
.fds
[i
] = fd
;
3522 pr_warning("failed to load program '%s'\n",
3523 prog
->section_name
);
3524 zfree(&prog
->insns
);
3525 prog
->insns_cnt
= 0;
3529 static bool bpf_program__is_function_storage(const struct bpf_program
*prog
,
3530 const struct bpf_object
*obj
)
3532 return prog
->idx
== obj
->efile
.text_shndx
&& obj
->has_pseudo_calls
;
3536 bpf_object__load_progs(struct bpf_object
*obj
, int log_level
)
3541 for (i
= 0; i
< obj
->nr_programs
; i
++) {
3542 if (bpf_program__is_function_storage(&obj
->programs
[i
], obj
))
3544 obj
->programs
[i
].log_level
|= log_level
;
3545 err
= bpf_program__load(&obj
->programs
[i
],
3554 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type
)
3557 case BPF_PROG_TYPE_SOCKET_FILTER
:
3558 case BPF_PROG_TYPE_SCHED_CLS
:
3559 case BPF_PROG_TYPE_SCHED_ACT
:
3560 case BPF_PROG_TYPE_XDP
:
3561 case BPF_PROG_TYPE_CGROUP_SKB
:
3562 case BPF_PROG_TYPE_CGROUP_SOCK
:
3563 case BPF_PROG_TYPE_LWT_IN
:
3564 case BPF_PROG_TYPE_LWT_OUT
:
3565 case BPF_PROG_TYPE_LWT_XMIT
:
3566 case BPF_PROG_TYPE_LWT_SEG6LOCAL
:
3567 case BPF_PROG_TYPE_SOCK_OPS
:
3568 case BPF_PROG_TYPE_SK_SKB
:
3569 case BPF_PROG_TYPE_CGROUP_DEVICE
:
3570 case BPF_PROG_TYPE_SK_MSG
:
3571 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR
:
3572 case BPF_PROG_TYPE_LIRC_MODE2
:
3573 case BPF_PROG_TYPE_SK_REUSEPORT
:
3574 case BPF_PROG_TYPE_FLOW_DISSECTOR
:
3575 case BPF_PROG_TYPE_UNSPEC
:
3576 case BPF_PROG_TYPE_TRACEPOINT
:
3577 case BPF_PROG_TYPE_RAW_TRACEPOINT
:
3578 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE
:
3579 case BPF_PROG_TYPE_PERF_EVENT
:
3580 case BPF_PROG_TYPE_CGROUP_SYSCTL
:
3581 case BPF_PROG_TYPE_CGROUP_SOCKOPT
:
3583 case BPF_PROG_TYPE_KPROBE
:
3589 static int bpf_object__validate(struct bpf_object
*obj
, bool needs_kver
)
3591 if (needs_kver
&& obj
->kern_version
== 0) {
3592 pr_warning("%s doesn't provide kernel version\n",
3594 return -LIBBPF_ERRNO__KVERSION
;
3599 static struct bpf_object
*
3600 __bpf_object__open(const char *path
, void *obj_buf
, size_t obj_buf_sz
,
3601 bool needs_kver
, int flags
)
3603 struct bpf_object
*obj
;
3606 if (elf_version(EV_CURRENT
) == EV_NONE
) {
3607 pr_warning("failed to init libelf for %s\n", path
);
3608 return ERR_PTR(-LIBBPF_ERRNO__LIBELF
);
3611 obj
= bpf_object__new(path
, obj_buf
, obj_buf_sz
);
3615 CHECK_ERR(bpf_object__elf_init(obj
), err
, out
);
3616 CHECK_ERR(bpf_object__check_endianness(obj
), err
, out
);
3617 CHECK_ERR(bpf_object__probe_caps(obj
), err
, out
);
3618 CHECK_ERR(bpf_object__elf_collect(obj
, flags
), err
, out
);
3619 CHECK_ERR(bpf_object__collect_reloc(obj
), err
, out
);
3620 CHECK_ERR(bpf_object__validate(obj
, needs_kver
), err
, out
);
3622 bpf_object__elf_finish(obj
);
3625 bpf_object__close(obj
);
3626 return ERR_PTR(err
);
3629 struct bpf_object
*__bpf_object__open_xattr(struct bpf_object_open_attr
*attr
,
3632 /* param validation */
3636 pr_debug("loading %s\n", attr
->file
);
3638 return __bpf_object__open(attr
->file
, NULL
, 0,
3639 bpf_prog_type__needs_kver(attr
->prog_type
),
3643 struct bpf_object
*bpf_object__open_xattr(struct bpf_object_open_attr
*attr
)
3645 return __bpf_object__open_xattr(attr
, 0);
3648 struct bpf_object
*bpf_object__open(const char *path
)
3650 struct bpf_object_open_attr attr
= {
3652 .prog_type
= BPF_PROG_TYPE_UNSPEC
,
3655 return bpf_object__open_xattr(&attr
);
3658 struct bpf_object
*bpf_object__open_buffer(void *obj_buf
,
3664 /* param validation */
3665 if (!obj_buf
|| obj_buf_sz
<= 0)
3669 snprintf(tmp_name
, sizeof(tmp_name
), "%lx-%lx",
3670 (unsigned long)obj_buf
,
3671 (unsigned long)obj_buf_sz
);
3674 pr_debug("loading object '%s' from buffer\n", name
);
3676 return __bpf_object__open(name
, obj_buf
, obj_buf_sz
, true, true);
3679 int bpf_object__unload(struct bpf_object
*obj
)
3686 for (i
= 0; i
< obj
->nr_maps
; i
++)
3687 zclose(obj
->maps
[i
].fd
);
3689 for (i
= 0; i
< obj
->nr_programs
; i
++)
3690 bpf_program__unload(&obj
->programs
[i
]);
3695 int bpf_object__load_xattr(struct bpf_object_load_attr
*attr
)
3697 struct bpf_object
*obj
;
3707 pr_warning("object should not be loaded twice\n");
3713 CHECK_ERR(bpf_object__create_maps(obj
), err
, out
);
3714 CHECK_ERR(bpf_object__relocate(obj
, attr
->target_btf_path
), err
, out
);
3715 CHECK_ERR(bpf_object__load_progs(obj
, attr
->log_level
), err
, out
);
3719 bpf_object__unload(obj
);
3720 pr_warning("failed to load object '%s'\n", obj
->path
);
3724 int bpf_object__load(struct bpf_object
*obj
)
3726 struct bpf_object_load_attr attr
= {
3730 return bpf_object__load_xattr(&attr
);
3733 static int check_path(const char *path
)
3735 char *cp
, errmsg
[STRERR_BUFSIZE
];
3736 struct statfs st_fs
;
3743 dname
= strdup(path
);
3747 dir
= dirname(dname
);
3748 if (statfs(dir
, &st_fs
)) {
3749 cp
= libbpf_strerror_r(errno
, errmsg
, sizeof(errmsg
));
3750 pr_warning("failed to statfs %s: %s\n", dir
, cp
);
3755 if (!err
&& st_fs
.f_type
!= BPF_FS_MAGIC
) {
3756 pr_warning("specified path %s is not on BPF FS\n", path
);
3763 int bpf_program__pin_instance(struct bpf_program
*prog
, const char *path
,
3766 char *cp
, errmsg
[STRERR_BUFSIZE
];
3769 err
= check_path(path
);
3774 pr_warning("invalid program pointer\n");
3778 if (instance
< 0 || instance
>= prog
->instances
.nr
) {
3779 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
3780 instance
, prog
->section_name
, prog
->instances
.nr
);
3784 if (bpf_obj_pin(prog
->instances
.fds
[instance
], path
)) {
3785 cp
= libbpf_strerror_r(errno
, errmsg
, sizeof(errmsg
));
3786 pr_warning("failed to pin program: %s\n", cp
);
3789 pr_debug("pinned program '%s'\n", path
);
3794 int bpf_program__unpin_instance(struct bpf_program
*prog
, const char *path
,
3799 err
= check_path(path
);
3804 pr_warning("invalid program pointer\n");
3808 if (instance
< 0 || instance
>= prog
->instances
.nr
) {
3809 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
3810 instance
, prog
->section_name
, prog
->instances
.nr
);
3817 pr_debug("unpinned program '%s'\n", path
);
3822 static int make_dir(const char *path
)
3824 char *cp
, errmsg
[STRERR_BUFSIZE
];
3827 if (mkdir(path
, 0700) && errno
!= EEXIST
)
3831 cp
= libbpf_strerror_r(-err
, errmsg
, sizeof(errmsg
));
3832 pr_warning("failed to mkdir %s: %s\n", path
, cp
);
3837 int bpf_program__pin(struct bpf_program
*prog
, const char *path
)
3841 err
= check_path(path
);
3846 pr_warning("invalid program pointer\n");
3850 if (prog
->instances
.nr
<= 0) {
3851 pr_warning("no instances of prog %s to pin\n",
3852 prog
->section_name
);
3856 if (prog
->instances
.nr
== 1) {
3857 /* don't create subdirs when pinning single instance */
3858 return bpf_program__pin_instance(prog
, path
, 0);
3861 err
= make_dir(path
);
3865 for (i
= 0; i
< prog
->instances
.nr
; i
++) {
3869 len
= snprintf(buf
, PATH_MAX
, "%s/%d", path
, i
);
3873 } else if (len
>= PATH_MAX
) {
3874 err
= -ENAMETOOLONG
;
3878 err
= bpf_program__pin_instance(prog
, buf
, i
);
3886 for (i
= i
- 1; i
>= 0; i
--) {
3890 len
= snprintf(buf
, PATH_MAX
, "%s/%d", path
, i
);
3893 else if (len
>= PATH_MAX
)
3896 bpf_program__unpin_instance(prog
, buf
, i
);
3904 int bpf_program__unpin(struct bpf_program
*prog
, const char *path
)
3908 err
= check_path(path
);
3913 pr_warning("invalid program pointer\n");
3917 if (prog
->instances
.nr
<= 0) {
3918 pr_warning("no instances of prog %s to pin\n",
3919 prog
->section_name
);
3923 if (prog
->instances
.nr
== 1) {
3924 /* don't create subdirs when pinning single instance */
3925 return bpf_program__unpin_instance(prog
, path
, 0);
3928 for (i
= 0; i
< prog
->instances
.nr
; i
++) {
3932 len
= snprintf(buf
, PATH_MAX
, "%s/%d", path
, i
);
3935 else if (len
>= PATH_MAX
)
3936 return -ENAMETOOLONG
;
3938 err
= bpf_program__unpin_instance(prog
, buf
, i
);
3950 int bpf_map__pin(struct bpf_map
*map
, const char *path
)
3952 char *cp
, errmsg
[STRERR_BUFSIZE
];
3955 err
= check_path(path
);
3960 pr_warning("invalid map pointer\n");
3964 if (bpf_obj_pin(map
->fd
, path
)) {
3965 cp
= libbpf_strerror_r(errno
, errmsg
, sizeof(errmsg
));
3966 pr_warning("failed to pin map: %s\n", cp
);
3970 pr_debug("pinned map '%s'\n", path
);
3975 int bpf_map__unpin(struct bpf_map
*map
, const char *path
)
3979 err
= check_path(path
);
3984 pr_warning("invalid map pointer\n");
3991 pr_debug("unpinned map '%s'\n", path
);
3996 int bpf_object__pin_maps(struct bpf_object
*obj
, const char *path
)
3998 struct bpf_map
*map
;
4005 pr_warning("object not yet loaded; load it first\n");
4009 err
= make_dir(path
);
4013 bpf_object__for_each_map(map
, obj
) {
4017 len
= snprintf(buf
, PATH_MAX
, "%s/%s", path
,
4018 bpf_map__name(map
));
4021 goto err_unpin_maps
;
4022 } else if (len
>= PATH_MAX
) {
4023 err
= -ENAMETOOLONG
;
4024 goto err_unpin_maps
;
4027 err
= bpf_map__pin(map
, buf
);
4029 goto err_unpin_maps
;
4035 while ((map
= bpf_map__prev(map
, obj
))) {
4039 len
= snprintf(buf
, PATH_MAX
, "%s/%s", path
,
4040 bpf_map__name(map
));
4043 else if (len
>= PATH_MAX
)
4046 bpf_map__unpin(map
, buf
);
4052 int bpf_object__unpin_maps(struct bpf_object
*obj
, const char *path
)
4054 struct bpf_map
*map
;
4060 bpf_object__for_each_map(map
, obj
) {
4064 len
= snprintf(buf
, PATH_MAX
, "%s/%s", path
,
4065 bpf_map__name(map
));
4068 else if (len
>= PATH_MAX
)
4069 return -ENAMETOOLONG
;
4071 err
= bpf_map__unpin(map
, buf
);
4079 int bpf_object__pin_programs(struct bpf_object
*obj
, const char *path
)
4081 struct bpf_program
*prog
;
4088 pr_warning("object not yet loaded; load it first\n");
4092 err
= make_dir(path
);
4096 bpf_object__for_each_program(prog
, obj
) {
4100 len
= snprintf(buf
, PATH_MAX
, "%s/%s", path
,
4104 goto err_unpin_programs
;
4105 } else if (len
>= PATH_MAX
) {
4106 err
= -ENAMETOOLONG
;
4107 goto err_unpin_programs
;
4110 err
= bpf_program__pin(prog
, buf
);
4112 goto err_unpin_programs
;
4118 while ((prog
= bpf_program__prev(prog
, obj
))) {
4122 len
= snprintf(buf
, PATH_MAX
, "%s/%s", path
,
4126 else if (len
>= PATH_MAX
)
4129 bpf_program__unpin(prog
, buf
);
4135 int bpf_object__unpin_programs(struct bpf_object
*obj
, const char *path
)
4137 struct bpf_program
*prog
;
4143 bpf_object__for_each_program(prog
, obj
) {
4147 len
= snprintf(buf
, PATH_MAX
, "%s/%s", path
,
4151 else if (len
>= PATH_MAX
)
4152 return -ENAMETOOLONG
;
4154 err
= bpf_program__unpin(prog
, buf
);
4162 int bpf_object__pin(struct bpf_object
*obj
, const char *path
)
4166 err
= bpf_object__pin_maps(obj
, path
);
4170 err
= bpf_object__pin_programs(obj
, path
);
4172 bpf_object__unpin_maps(obj
, path
);
4179 void bpf_object__close(struct bpf_object
*obj
)
4186 if (obj
->clear_priv
)
4187 obj
->clear_priv(obj
, obj
->priv
);
4189 bpf_object__elf_finish(obj
);
4190 bpf_object__unload(obj
);
4191 btf__free(obj
->btf
);
4192 btf_ext__free(obj
->btf_ext
);
4194 for (i
= 0; i
< obj
->nr_maps
; i
++) {
4195 zfree(&obj
->maps
[i
].name
);
4196 if (obj
->maps
[i
].clear_priv
)
4197 obj
->maps
[i
].clear_priv(&obj
->maps
[i
],
4199 obj
->maps
[i
].priv
= NULL
;
4200 obj
->maps
[i
].clear_priv
= NULL
;
4203 zfree(&obj
->sections
.rodata
);
4204 zfree(&obj
->sections
.data
);
4208 if (obj
->programs
&& obj
->nr_programs
) {
4209 for (i
= 0; i
< obj
->nr_programs
; i
++)
4210 bpf_program__exit(&obj
->programs
[i
]);
4212 zfree(&obj
->programs
);
4214 list_del(&obj
->list
);
4219 bpf_object__next(struct bpf_object
*prev
)
4221 struct bpf_object
*next
;
4224 next
= list_first_entry(&bpf_objects_list
,
4228 next
= list_next_entry(prev
, list
);
4230 /* Empty list is noticed here so don't need checking on entry. */
4231 if (&next
->list
== &bpf_objects_list
)
4237 const char *bpf_object__name(const struct bpf_object
*obj
)
4239 return obj
? obj
->path
: ERR_PTR(-EINVAL
);
4242 unsigned int bpf_object__kversion(const struct bpf_object
*obj
)
4244 return obj
? obj
->kern_version
: 0;
4247 struct btf
*bpf_object__btf(const struct bpf_object
*obj
)
4249 return obj
? obj
->btf
: NULL
;
4252 int bpf_object__btf_fd(const struct bpf_object
*obj
)
4254 return obj
->btf
? btf__fd(obj
->btf
) : -1;
4257 int bpf_object__set_priv(struct bpf_object
*obj
, void *priv
,
4258 bpf_object_clear_priv_t clear_priv
)
4260 if (obj
->priv
&& obj
->clear_priv
)
4261 obj
->clear_priv(obj
, obj
->priv
);
4264 obj
->clear_priv
= clear_priv
;
4268 void *bpf_object__priv(const struct bpf_object
*obj
)
4270 return obj
? obj
->priv
: ERR_PTR(-EINVAL
);
4273 static struct bpf_program
*
4274 __bpf_program__iter(const struct bpf_program
*p
, const struct bpf_object
*obj
,
4277 size_t nr_programs
= obj
->nr_programs
;
4284 /* Iter from the beginning */
4285 return forward
? &obj
->programs
[0] :
4286 &obj
->programs
[nr_programs
- 1];
4288 if (p
->obj
!= obj
) {
4289 pr_warning("error: program handler doesn't match object\n");
4293 idx
= (p
- obj
->programs
) + (forward
? 1 : -1);
4294 if (idx
>= obj
->nr_programs
|| idx
< 0)
4296 return &obj
->programs
[idx
];
4299 struct bpf_program
*
4300 bpf_program__next(struct bpf_program
*prev
, const struct bpf_object
*obj
)
4302 struct bpf_program
*prog
= prev
;
4305 prog
= __bpf_program__iter(prog
, obj
, true);
4306 } while (prog
&& bpf_program__is_function_storage(prog
, obj
));
4311 struct bpf_program
*
4312 bpf_program__prev(struct bpf_program
*next
, const struct bpf_object
*obj
)
4314 struct bpf_program
*prog
= next
;
4317 prog
= __bpf_program__iter(prog
, obj
, false);
4318 } while (prog
&& bpf_program__is_function_storage(prog
, obj
));
4323 int bpf_program__set_priv(struct bpf_program
*prog
, void *priv
,
4324 bpf_program_clear_priv_t clear_priv
)
4326 if (prog
->priv
&& prog
->clear_priv
)
4327 prog
->clear_priv(prog
, prog
->priv
);
4330 prog
->clear_priv
= clear_priv
;
4334 void *bpf_program__priv(const struct bpf_program
*prog
)
4336 return prog
? prog
->priv
: ERR_PTR(-EINVAL
);
4339 void bpf_program__set_ifindex(struct bpf_program
*prog
, __u32 ifindex
)
4341 prog
->prog_ifindex
= ifindex
;
4344 const char *bpf_program__title(const struct bpf_program
*prog
, bool needs_copy
)
4348 title
= prog
->section_name
;
4350 title
= strdup(title
);
4352 pr_warning("failed to strdup program title\n");
4353 return ERR_PTR(-ENOMEM
);
4360 int bpf_program__fd(const struct bpf_program
*prog
)
4362 return bpf_program__nth_fd(prog
, 0);
4365 int bpf_program__set_prep(struct bpf_program
*prog
, int nr_instances
,
4366 bpf_program_prep_t prep
)
4370 if (nr_instances
<= 0 || !prep
)
4373 if (prog
->instances
.nr
> 0 || prog
->instances
.fds
) {
4374 pr_warning("Can't set pre-processor after loading\n");
4378 instances_fds
= malloc(sizeof(int) * nr_instances
);
4379 if (!instances_fds
) {
4380 pr_warning("alloc memory failed for fds\n");
4384 /* fill all fd with -1 */
4385 memset(instances_fds
, -1, sizeof(int) * nr_instances
);
4387 prog
->instances
.nr
= nr_instances
;
4388 prog
->instances
.fds
= instances_fds
;
4389 prog
->preprocessor
= prep
;
4393 int bpf_program__nth_fd(const struct bpf_program
*prog
, int n
)
4400 if (n
>= prog
->instances
.nr
|| n
< 0) {
4401 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
4402 n
, prog
->section_name
, prog
->instances
.nr
);
4406 fd
= prog
->instances
.fds
[n
];
4408 pr_warning("%dth instance of program '%s' is invalid\n",
4409 n
, prog
->section_name
);
4416 void bpf_program__set_type(struct bpf_program
*prog
, enum bpf_prog_type type
)
4421 static bool bpf_program__is_type(const struct bpf_program
*prog
,
4422 enum bpf_prog_type type
)
4424 return prog
? (prog
->type
== type
) : false;
4427 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \
4428 int bpf_program__set_##NAME(struct bpf_program *prog) \
4432 bpf_program__set_type(prog, TYPE); \
4436 bool bpf_program__is_##NAME(const struct bpf_program *prog) \
4438 return bpf_program__is_type(prog, TYPE); \
4441 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
4442 BPF_PROG_TYPE_FNS(kprobe
, BPF_PROG_TYPE_KPROBE
);
4443 BPF_PROG_TYPE_FNS(sched_cls
, BPF_PROG_TYPE_SCHED_CLS
);
4444 BPF_PROG_TYPE_FNS(sched_act
, BPF_PROG_TYPE_SCHED_ACT
);
4445 BPF_PROG_TYPE_FNS(tracepoint
, BPF_PROG_TYPE_TRACEPOINT
);
4446 BPF_PROG_TYPE_FNS(raw_tracepoint
, BPF_PROG_TYPE_RAW_TRACEPOINT
);
4447 BPF_PROG_TYPE_FNS(xdp
, BPF_PROG_TYPE_XDP
);
4448 BPF_PROG_TYPE_FNS(perf_event
, BPF_PROG_TYPE_PERF_EVENT
);
4450 void bpf_program__set_expected_attach_type(struct bpf_program
*prog
,
4451 enum bpf_attach_type type
)
4453 prog
->expected_attach_type
= type
;
4456 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \
4457 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype }
4459 /* Programs that can NOT be attached. */
4460 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0)
4462 /* Programs that can be attached. */
4463 #define BPF_APROG_SEC(string, ptype, atype) \
4464 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype)
4466 /* Programs that must specify expected attach type at load time. */
4467 #define BPF_EAPROG_SEC(string, ptype, eatype) \
4468 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype)
4470 /* Programs that can be attached but attach type can't be identified by section
4471 * name. Kept for backward compatibility.
4473 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
4475 static const struct {
4478 enum bpf_prog_type prog_type
;
4479 enum bpf_attach_type expected_attach_type
;
4481 enum bpf_attach_type attach_type
;
4482 } section_names
[] = {
4483 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER
),
4484 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE
),
4485 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE
),
4486 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS
),
4487 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT
),
4488 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT
),
4489 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT
),
4490 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP
),
4491 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT
),
4492 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN
),
4493 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT
),
4494 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT
),
4495 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL
),
4496 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB
,
4497 BPF_CGROUP_INET_INGRESS
),
4498 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB
,
4499 BPF_CGROUP_INET_EGRESS
),
4500 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB
),
4501 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK
,
4502 BPF_CGROUP_INET_SOCK_CREATE
),
4503 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK
,
4504 BPF_CGROUP_INET4_POST_BIND
),
4505 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK
,
4506 BPF_CGROUP_INET6_POST_BIND
),
4507 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE
,
4509 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS
,
4510 BPF_CGROUP_SOCK_OPS
),
4511 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB
,
4512 BPF_SK_SKB_STREAM_PARSER
),
4513 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB
,
4514 BPF_SK_SKB_STREAM_VERDICT
),
4515 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB
),
4516 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG
,
4517 BPF_SK_MSG_VERDICT
),
4518 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2
,
4520 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR
,
4521 BPF_FLOW_DISSECTOR
),
4522 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR
,
4523 BPF_CGROUP_INET4_BIND
),
4524 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR
,
4525 BPF_CGROUP_INET6_BIND
),
4526 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR
,
4527 BPF_CGROUP_INET4_CONNECT
),
4528 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR
,
4529 BPF_CGROUP_INET6_CONNECT
),
4530 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR
,
4531 BPF_CGROUP_UDP4_SENDMSG
),
4532 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR
,
4533 BPF_CGROUP_UDP6_SENDMSG
),
4534 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR
,
4535 BPF_CGROUP_UDP4_RECVMSG
),
4536 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR
,
4537 BPF_CGROUP_UDP6_RECVMSG
),
4538 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL
,
4540 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT
,
4541 BPF_CGROUP_GETSOCKOPT
),
4542 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT
,
4543 BPF_CGROUP_SETSOCKOPT
),
4546 #undef BPF_PROG_SEC_IMPL
4548 #undef BPF_APROG_SEC
4549 #undef BPF_EAPROG_SEC
4550 #undef BPF_APROG_COMPAT
4552 #define MAX_TYPE_NAME_SIZE 32
4554 static char *libbpf_get_type_names(bool attach_type
)
4556 int i
, len
= ARRAY_SIZE(section_names
) * MAX_TYPE_NAME_SIZE
;
4564 /* Forge string buf with all available names */
4565 for (i
= 0; i
< ARRAY_SIZE(section_names
); i
++) {
4566 if (attach_type
&& !section_names
[i
].is_attachable
)
4569 if (strlen(buf
) + strlen(section_names
[i
].sec
) + 2 > len
) {
4574 strcat(buf
, section_names
[i
].sec
);
4580 int libbpf_prog_type_by_name(const char *name
, enum bpf_prog_type
*prog_type
,
4581 enum bpf_attach_type
*expected_attach_type
)
4589 for (i
= 0; i
< ARRAY_SIZE(section_names
); i
++) {
4590 if (strncmp(name
, section_names
[i
].sec
, section_names
[i
].len
))
4592 *prog_type
= section_names
[i
].prog_type
;
4593 *expected_attach_type
= section_names
[i
].expected_attach_type
;
4596 pr_warning("failed to guess program type based on ELF section name '%s'\n", name
);
4597 type_names
= libbpf_get_type_names(false);
4598 if (type_names
!= NULL
) {
4599 pr_info("supported section(type) names are:%s\n", type_names
);
4606 int libbpf_attach_type_by_name(const char *name
,
4607 enum bpf_attach_type
*attach_type
)
4615 for (i
= 0; i
< ARRAY_SIZE(section_names
); i
++) {
4616 if (strncmp(name
, section_names
[i
].sec
, section_names
[i
].len
))
4618 if (!section_names
[i
].is_attachable
)
4620 *attach_type
= section_names
[i
].attach_type
;
4623 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name
);
4624 type_names
= libbpf_get_type_names(true);
4625 if (type_names
!= NULL
) {
4626 pr_info("attachable section(type) names are:%s\n", type_names
);
4634 bpf_program__identify_section(struct bpf_program
*prog
,
4635 enum bpf_prog_type
*prog_type
,
4636 enum bpf_attach_type
*expected_attach_type
)
4638 return libbpf_prog_type_by_name(prog
->section_name
, prog_type
,
4639 expected_attach_type
);
4642 int bpf_map__fd(const struct bpf_map
*map
)
4644 return map
? map
->fd
: -EINVAL
;
4647 const struct bpf_map_def
*bpf_map__def(const struct bpf_map
*map
)
4649 return map
? &map
->def
: ERR_PTR(-EINVAL
);
4652 const char *bpf_map__name(const struct bpf_map
*map
)
4654 return map
? map
->name
: NULL
;
4657 __u32
bpf_map__btf_key_type_id(const struct bpf_map
*map
)
4659 return map
? map
->btf_key_type_id
: 0;
4662 __u32
bpf_map__btf_value_type_id(const struct bpf_map
*map
)
4664 return map
? map
->btf_value_type_id
: 0;
4667 int bpf_map__set_priv(struct bpf_map
*map
, void *priv
,
4668 bpf_map_clear_priv_t clear_priv
)
4674 if (map
->clear_priv
)
4675 map
->clear_priv(map
, map
->priv
);
4679 map
->clear_priv
= clear_priv
;
4683 void *bpf_map__priv(const struct bpf_map
*map
)
4685 return map
? map
->priv
: ERR_PTR(-EINVAL
);
4688 bool bpf_map__is_offload_neutral(const struct bpf_map
*map
)
4690 return map
->def
.type
== BPF_MAP_TYPE_PERF_EVENT_ARRAY
;
4693 bool bpf_map__is_internal(const struct bpf_map
*map
)
4695 return map
->libbpf_type
!= LIBBPF_MAP_UNSPEC
;
4698 void bpf_map__set_ifindex(struct bpf_map
*map
, __u32 ifindex
)
4700 map
->map_ifindex
= ifindex
;
4703 int bpf_map__set_inner_map_fd(struct bpf_map
*map
, int fd
)
4705 if (!bpf_map_type__is_map_in_map(map
->def
.type
)) {
4706 pr_warning("error: unsupported map type\n");
4709 if (map
->inner_map_fd
!= -1) {
4710 pr_warning("error: inner_map_fd already specified\n");
4713 map
->inner_map_fd
= fd
;
4717 static struct bpf_map
*
4718 __bpf_map__iter(const struct bpf_map
*m
, const struct bpf_object
*obj
, int i
)
4721 struct bpf_map
*s
, *e
;
4723 if (!obj
|| !obj
->maps
)
4727 e
= obj
->maps
+ obj
->nr_maps
;
4729 if ((m
< s
) || (m
>= e
)) {
4730 pr_warning("error in %s: map handler doesn't belong to object\n",
4735 idx
= (m
- obj
->maps
) + i
;
4736 if (idx
>= obj
->nr_maps
|| idx
< 0)
4738 return &obj
->maps
[idx
];
4742 bpf_map__next(const struct bpf_map
*prev
, const struct bpf_object
*obj
)
4747 return __bpf_map__iter(prev
, obj
, 1);
4751 bpf_map__prev(const struct bpf_map
*next
, const struct bpf_object
*obj
)
4756 return obj
->maps
+ obj
->nr_maps
- 1;
4759 return __bpf_map__iter(next
, obj
, -1);
4763 bpf_object__find_map_by_name(const struct bpf_object
*obj
, const char *name
)
4765 struct bpf_map
*pos
;
4767 bpf_object__for_each_map(pos
, obj
) {
4768 if (pos
->name
&& !strcmp(pos
->name
, name
))
4775 bpf_object__find_map_fd_by_name(const struct bpf_object
*obj
, const char *name
)
4777 return bpf_map__fd(bpf_object__find_map_by_name(obj
, name
));
4781 bpf_object__find_map_by_offset(struct bpf_object
*obj
, size_t offset
)
4783 return ERR_PTR(-ENOTSUP
);
4786 long libbpf_get_error(const void *ptr
)
4788 return PTR_ERR_OR_ZERO(ptr
);
4791 int bpf_prog_load(const char *file
, enum bpf_prog_type type
,
4792 struct bpf_object
**pobj
, int *prog_fd
)
4794 struct bpf_prog_load_attr attr
;
4796 memset(&attr
, 0, sizeof(struct bpf_prog_load_attr
));
4798 attr
.prog_type
= type
;
4799 attr
.expected_attach_type
= 0;
4801 return bpf_prog_load_xattr(&attr
, pobj
, prog_fd
);
4804 int bpf_prog_load_xattr(const struct bpf_prog_load_attr
*attr
,
4805 struct bpf_object
**pobj
, int *prog_fd
)
4807 struct bpf_object_open_attr open_attr
= {};
4808 struct bpf_program
*prog
, *first_prog
= NULL
;
4809 enum bpf_attach_type expected_attach_type
;
4810 enum bpf_prog_type prog_type
;
4811 struct bpf_object
*obj
;
4812 struct bpf_map
*map
;
4820 open_attr
.file
= attr
->file
;
4821 open_attr
.prog_type
= attr
->prog_type
;
4823 obj
= bpf_object__open_xattr(&open_attr
);
4824 if (IS_ERR_OR_NULL(obj
))
4827 bpf_object__for_each_program(prog
, obj
) {
4829 * If type is not specified, try to guess it based on
4832 prog_type
= attr
->prog_type
;
4833 prog
->prog_ifindex
= attr
->ifindex
;
4834 expected_attach_type
= attr
->expected_attach_type
;
4835 if (prog_type
== BPF_PROG_TYPE_UNSPEC
) {
4836 err
= bpf_program__identify_section(prog
, &prog_type
,
4837 &expected_attach_type
);
4839 bpf_object__close(obj
);
4844 bpf_program__set_type(prog
, prog_type
);
4845 bpf_program__set_expected_attach_type(prog
,
4846 expected_attach_type
);
4848 prog
->log_level
= attr
->log_level
;
4849 prog
->prog_flags
= attr
->prog_flags
;
4854 bpf_object__for_each_map(map
, obj
) {
4855 if (!bpf_map__is_offload_neutral(map
))
4856 map
->map_ifindex
= attr
->ifindex
;
4860 pr_warning("object file doesn't contain bpf program\n");
4861 bpf_object__close(obj
);
4865 err
= bpf_object__load(obj
);
4867 bpf_object__close(obj
);
4872 *prog_fd
= bpf_program__fd(first_prog
);
4877 int (*destroy
)(struct bpf_link
*link
);
4880 int bpf_link__destroy(struct bpf_link
*link
)
4887 err
= link
->destroy(link
);
4893 struct bpf_link_fd
{
4894 struct bpf_link link
; /* has to be at the top of struct */
4895 int fd
; /* hook FD */
4898 static int bpf_link__destroy_perf_event(struct bpf_link
*link
)
4900 struct bpf_link_fd
*l
= (void *)link
;
4903 err
= ioctl(l
->fd
, PERF_EVENT_IOC_DISABLE
, 0);
4911 struct bpf_link
*bpf_program__attach_perf_event(struct bpf_program
*prog
,
4914 char errmsg
[STRERR_BUFSIZE
];
4915 struct bpf_link_fd
*link
;
4919 pr_warning("program '%s': invalid perf event FD %d\n",
4920 bpf_program__title(prog
, false), pfd
);
4921 return ERR_PTR(-EINVAL
);
4923 prog_fd
= bpf_program__fd(prog
);
4925 pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n",
4926 bpf_program__title(prog
, false));
4927 return ERR_PTR(-EINVAL
);
4930 link
= malloc(sizeof(*link
));
4932 return ERR_PTR(-ENOMEM
);
4933 link
->link
.destroy
= &bpf_link__destroy_perf_event
;
4936 if (ioctl(pfd
, PERF_EVENT_IOC_SET_BPF
, prog_fd
) < 0) {
4939 pr_warning("program '%s': failed to attach to pfd %d: %s\n",
4940 bpf_program__title(prog
, false), pfd
,
4941 libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
)));
4942 return ERR_PTR(err
);
4944 if (ioctl(pfd
, PERF_EVENT_IOC_ENABLE
, 0) < 0) {
4947 pr_warning("program '%s': failed to enable pfd %d: %s\n",
4948 bpf_program__title(prog
, false), pfd
,
4949 libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
)));
4950 return ERR_PTR(err
);
4952 return (struct bpf_link
*)link
;
4956 * this function is expected to parse integer in the range of [0, 2^31-1] from
4957 * given file using scanf format string fmt. If actual parsed value is
4958 * negative, the result might be indistinguishable from error
4960 static int parse_uint_from_file(const char *file
, const char *fmt
)
4962 char buf
[STRERR_BUFSIZE
];
4966 f
= fopen(file
, "r");
4969 pr_debug("failed to open '%s': %s\n", file
,
4970 libbpf_strerror_r(err
, buf
, sizeof(buf
)));
4973 err
= fscanf(f
, fmt
, &ret
);
4975 err
= err
== EOF
? -EIO
: -errno
;
4976 pr_debug("failed to parse '%s': %s\n", file
,
4977 libbpf_strerror_r(err
, buf
, sizeof(buf
)));
4985 static int determine_kprobe_perf_type(void)
4987 const char *file
= "/sys/bus/event_source/devices/kprobe/type";
4989 return parse_uint_from_file(file
, "%d\n");
4992 static int determine_uprobe_perf_type(void)
4994 const char *file
= "/sys/bus/event_source/devices/uprobe/type";
4996 return parse_uint_from_file(file
, "%d\n");
4999 static int determine_kprobe_retprobe_bit(void)
5001 const char *file
= "/sys/bus/event_source/devices/kprobe/format/retprobe";
5003 return parse_uint_from_file(file
, "config:%d\n");
5006 static int determine_uprobe_retprobe_bit(void)
5008 const char *file
= "/sys/bus/event_source/devices/uprobe/format/retprobe";
5010 return parse_uint_from_file(file
, "config:%d\n");
5013 static int perf_event_open_probe(bool uprobe
, bool retprobe
, const char *name
,
5014 uint64_t offset
, int pid
)
5016 struct perf_event_attr attr
= {};
5017 char errmsg
[STRERR_BUFSIZE
];
5020 type
= uprobe
? determine_uprobe_perf_type()
5021 : determine_kprobe_perf_type();
5023 pr_warning("failed to determine %s perf type: %s\n",
5024 uprobe
? "uprobe" : "kprobe",
5025 libbpf_strerror_r(type
, errmsg
, sizeof(errmsg
)));
5029 int bit
= uprobe
? determine_uprobe_retprobe_bit()
5030 : determine_kprobe_retprobe_bit();
5033 pr_warning("failed to determine %s retprobe bit: %s\n",
5034 uprobe
? "uprobe" : "kprobe",
5035 libbpf_strerror_r(bit
, errmsg
,
5039 attr
.config
|= 1 << bit
;
5041 attr
.size
= sizeof(attr
);
5043 attr
.config1
= ptr_to_u64(name
); /* kprobe_func or uprobe_path */
5044 attr
.config2
= offset
; /* kprobe_addr or probe_offset */
5046 /* pid filter is meaningful only for uprobes */
5047 pfd
= syscall(__NR_perf_event_open
, &attr
,
5048 pid
< 0 ? -1 : pid
/* pid */,
5049 pid
== -1 ? 0 : -1 /* cpu */,
5050 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC
);
5053 pr_warning("%s perf_event_open() failed: %s\n",
5054 uprobe
? "uprobe" : "kprobe",
5055 libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
)));
5061 struct bpf_link
*bpf_program__attach_kprobe(struct bpf_program
*prog
,
5063 const char *func_name
)
5065 char errmsg
[STRERR_BUFSIZE
];
5066 struct bpf_link
*link
;
5069 pfd
= perf_event_open_probe(false /* uprobe */, retprobe
, func_name
,
5070 0 /* offset */, -1 /* pid */);
5072 pr_warning("program '%s': failed to create %s '%s' perf event: %s\n",
5073 bpf_program__title(prog
, false),
5074 retprobe
? "kretprobe" : "kprobe", func_name
,
5075 libbpf_strerror_r(pfd
, errmsg
, sizeof(errmsg
)));
5076 return ERR_PTR(pfd
);
5078 link
= bpf_program__attach_perf_event(prog
, pfd
);
5081 err
= PTR_ERR(link
);
5082 pr_warning("program '%s': failed to attach to %s '%s': %s\n",
5083 bpf_program__title(prog
, false),
5084 retprobe
? "kretprobe" : "kprobe", func_name
,
5085 libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
)));
5091 struct bpf_link
*bpf_program__attach_uprobe(struct bpf_program
*prog
,
5092 bool retprobe
, pid_t pid
,
5093 const char *binary_path
,
5096 char errmsg
[STRERR_BUFSIZE
];
5097 struct bpf_link
*link
;
5100 pfd
= perf_event_open_probe(true /* uprobe */, retprobe
,
5101 binary_path
, func_offset
, pid
);
5103 pr_warning("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
5104 bpf_program__title(prog
, false),
5105 retprobe
? "uretprobe" : "uprobe",
5106 binary_path
, func_offset
,
5107 libbpf_strerror_r(pfd
, errmsg
, sizeof(errmsg
)));
5108 return ERR_PTR(pfd
);
5110 link
= bpf_program__attach_perf_event(prog
, pfd
);
5113 err
= PTR_ERR(link
);
5114 pr_warning("program '%s': failed to attach to %s '%s:0x%zx': %s\n",
5115 bpf_program__title(prog
, false),
5116 retprobe
? "uretprobe" : "uprobe",
5117 binary_path
, func_offset
,
5118 libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
)));
5124 static int determine_tracepoint_id(const char *tp_category
,
5125 const char *tp_name
)
5127 char file
[PATH_MAX
];
5130 ret
= snprintf(file
, sizeof(file
),
5131 "/sys/kernel/debug/tracing/events/%s/%s/id",
5132 tp_category
, tp_name
);
5135 if (ret
>= sizeof(file
)) {
5136 pr_debug("tracepoint %s/%s path is too long\n",
5137 tp_category
, tp_name
);
5140 return parse_uint_from_file(file
, "%d\n");
5143 static int perf_event_open_tracepoint(const char *tp_category
,
5144 const char *tp_name
)
5146 struct perf_event_attr attr
= {};
5147 char errmsg
[STRERR_BUFSIZE
];
5148 int tp_id
, pfd
, err
;
5150 tp_id
= determine_tracepoint_id(tp_category
, tp_name
);
5152 pr_warning("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
5153 tp_category
, tp_name
,
5154 libbpf_strerror_r(tp_id
, errmsg
, sizeof(errmsg
)));
5158 attr
.type
= PERF_TYPE_TRACEPOINT
;
5159 attr
.size
= sizeof(attr
);
5160 attr
.config
= tp_id
;
5162 pfd
= syscall(__NR_perf_event_open
, &attr
, -1 /* pid */, 0 /* cpu */,
5163 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC
);
5166 pr_warning("tracepoint '%s/%s' perf_event_open() failed: %s\n",
5167 tp_category
, tp_name
,
5168 libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
)));
5174 struct bpf_link
*bpf_program__attach_tracepoint(struct bpf_program
*prog
,
5175 const char *tp_category
,
5176 const char *tp_name
)
5178 char errmsg
[STRERR_BUFSIZE
];
5179 struct bpf_link
*link
;
5182 pfd
= perf_event_open_tracepoint(tp_category
, tp_name
);
5184 pr_warning("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
5185 bpf_program__title(prog
, false),
5186 tp_category
, tp_name
,
5187 libbpf_strerror_r(pfd
, errmsg
, sizeof(errmsg
)));
5188 return ERR_PTR(pfd
);
5190 link
= bpf_program__attach_perf_event(prog
, pfd
);
5193 err
= PTR_ERR(link
);
5194 pr_warning("program '%s': failed to attach to tracepoint '%s/%s': %s\n",
5195 bpf_program__title(prog
, false),
5196 tp_category
, tp_name
,
5197 libbpf_strerror_r(err
, errmsg
, sizeof(errmsg
)));
5203 static int bpf_link__destroy_fd(struct bpf_link
*link
)
5205 struct bpf_link_fd
*l
= (void *)link
;
5207 return close(l
->fd
);
5210 struct bpf_link
*bpf_program__attach_raw_tracepoint(struct bpf_program
*prog
,
5211 const char *tp_name
)
5213 char errmsg
[STRERR_BUFSIZE
];
5214 struct bpf_link_fd
*link
;
5217 prog_fd
= bpf_program__fd(prog
);
5219 pr_warning("program '%s': can't attach before loaded\n",
5220 bpf_program__title(prog
, false));
5221 return ERR_PTR(-EINVAL
);
5224 link
= malloc(sizeof(*link
));
5226 return ERR_PTR(-ENOMEM
);
5227 link
->link
.destroy
= &bpf_link__destroy_fd
;
5229 pfd
= bpf_raw_tracepoint_open(tp_name
, prog_fd
);
5233 pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n",
5234 bpf_program__title(prog
, false), tp_name
,
5235 libbpf_strerror_r(pfd
, errmsg
, sizeof(errmsg
)));
5236 return ERR_PTR(pfd
);
5239 return (struct bpf_link
*)link
;
5242 enum bpf_perf_event_ret
5243 bpf_perf_event_read_simple(void *mmap_mem
, size_t mmap_size
, size_t page_size
,
5244 void **copy_mem
, size_t *copy_size
,
5245 bpf_perf_event_print_t fn
, void *private_data
)
5247 struct perf_event_mmap_page
*header
= mmap_mem
;
5248 __u64 data_head
= ring_buffer_read_head(header
);
5249 __u64 data_tail
= header
->data_tail
;
5250 void *base
= ((__u8
*)header
) + page_size
;
5251 int ret
= LIBBPF_PERF_EVENT_CONT
;
5252 struct perf_event_header
*ehdr
;
5255 while (data_head
!= data_tail
) {
5256 ehdr
= base
+ (data_tail
& (mmap_size
- 1));
5257 ehdr_size
= ehdr
->size
;
5259 if (((void *)ehdr
) + ehdr_size
> base
+ mmap_size
) {
5260 void *copy_start
= ehdr
;
5261 size_t len_first
= base
+ mmap_size
- copy_start
;
5262 size_t len_secnd
= ehdr_size
- len_first
;
5264 if (*copy_size
< ehdr_size
) {
5266 *copy_mem
= malloc(ehdr_size
);
5269 ret
= LIBBPF_PERF_EVENT_ERROR
;
5272 *copy_size
= ehdr_size
;
5275 memcpy(*copy_mem
, copy_start
, len_first
);
5276 memcpy(*copy_mem
+ len_first
, base
, len_secnd
);
5280 ret
= fn(ehdr
, private_data
);
5281 data_tail
+= ehdr_size
;
5282 if (ret
!= LIBBPF_PERF_EVENT_CONT
)
5286 ring_buffer_write_tail(header
, data_tail
);
5292 struct perf_buffer_params
{
5293 struct perf_event_attr
*attr
;
5294 /* if event_cb is specified, it takes precendence */
5295 perf_buffer_event_fn event_cb
;
5296 /* sample_cb and lost_cb are higher-level common-case callbacks */
5297 perf_buffer_sample_fn sample_cb
;
5298 perf_buffer_lost_fn lost_cb
;
5305 struct perf_cpu_buf
{
5306 struct perf_buffer
*pb
;
5307 void *base
; /* mmap()'ed memory */
5308 void *buf
; /* for reconstructing segmented data */
5315 struct perf_buffer
{
5316 perf_buffer_event_fn event_cb
;
5317 perf_buffer_sample_fn sample_cb
;
5318 perf_buffer_lost_fn lost_cb
;
5319 void *ctx
; /* passed into callbacks */
5323 struct perf_cpu_buf
**cpu_bufs
;
5324 struct epoll_event
*events
;
5326 int epoll_fd
; /* perf event FD */
5327 int map_fd
; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
5330 static void perf_buffer__free_cpu_buf(struct perf_buffer
*pb
,
5331 struct perf_cpu_buf
*cpu_buf
)
5335 if (cpu_buf
->base
&&
5336 munmap(cpu_buf
->base
, pb
->mmap_size
+ pb
->page_size
))
5337 pr_warning("failed to munmap cpu_buf #%d\n", cpu_buf
->cpu
);
5338 if (cpu_buf
->fd
>= 0) {
5339 ioctl(cpu_buf
->fd
, PERF_EVENT_IOC_DISABLE
, 0);
5346 void perf_buffer__free(struct perf_buffer
*pb
)
5353 for (i
= 0; i
< pb
->cpu_cnt
&& pb
->cpu_bufs
[i
]; i
++) {
5354 struct perf_cpu_buf
*cpu_buf
= pb
->cpu_bufs
[i
];
5356 bpf_map_delete_elem(pb
->map_fd
, &cpu_buf
->map_key
);
5357 perf_buffer__free_cpu_buf(pb
, cpu_buf
);
5361 if (pb
->epoll_fd
>= 0)
5362 close(pb
->epoll_fd
);
5367 static struct perf_cpu_buf
*
5368 perf_buffer__open_cpu_buf(struct perf_buffer
*pb
, struct perf_event_attr
*attr
,
5369 int cpu
, int map_key
)
5371 struct perf_cpu_buf
*cpu_buf
;
5372 char msg
[STRERR_BUFSIZE
];
5375 cpu_buf
= calloc(1, sizeof(*cpu_buf
));
5377 return ERR_PTR(-ENOMEM
);
5381 cpu_buf
->map_key
= map_key
;
5383 cpu_buf
->fd
= syscall(__NR_perf_event_open
, attr
, -1 /* pid */, cpu
,
5384 -1, PERF_FLAG_FD_CLOEXEC
);
5385 if (cpu_buf
->fd
< 0) {
5387 pr_warning("failed to open perf buffer event on cpu #%d: %s\n",
5388 cpu
, libbpf_strerror_r(err
, msg
, sizeof(msg
)));
5392 cpu_buf
->base
= mmap(NULL
, pb
->mmap_size
+ pb
->page_size
,
5393 PROT_READ
| PROT_WRITE
, MAP_SHARED
,
5395 if (cpu_buf
->base
== MAP_FAILED
) {
5396 cpu_buf
->base
= NULL
;
5398 pr_warning("failed to mmap perf buffer on cpu #%d: %s\n",
5399 cpu
, libbpf_strerror_r(err
, msg
, sizeof(msg
)));
5403 if (ioctl(cpu_buf
->fd
, PERF_EVENT_IOC_ENABLE
, 0) < 0) {
5405 pr_warning("failed to enable perf buffer event on cpu #%d: %s\n",
5406 cpu
, libbpf_strerror_r(err
, msg
, sizeof(msg
)));
5413 perf_buffer__free_cpu_buf(pb
, cpu_buf
);
5414 return (struct perf_cpu_buf
*)ERR_PTR(err
);
5417 static struct perf_buffer
*__perf_buffer__new(int map_fd
, size_t page_cnt
,
5418 struct perf_buffer_params
*p
);
5420 struct perf_buffer
*perf_buffer__new(int map_fd
, size_t page_cnt
,
5421 const struct perf_buffer_opts
*opts
)
5423 struct perf_buffer_params p
= {};
5424 struct perf_event_attr attr
= { 0, };
5426 attr
.config
= PERF_COUNT_SW_BPF_OUTPUT
,
5427 attr
.type
= PERF_TYPE_SOFTWARE
;
5428 attr
.sample_type
= PERF_SAMPLE_RAW
;
5429 attr
.sample_period
= 1;
5430 attr
.wakeup_events
= 1;
5433 p
.sample_cb
= opts
? opts
->sample_cb
: NULL
;
5434 p
.lost_cb
= opts
? opts
->lost_cb
: NULL
;
5435 p
.ctx
= opts
? opts
->ctx
: NULL
;
5437 return __perf_buffer__new(map_fd
, page_cnt
, &p
);
5440 struct perf_buffer
*
5441 perf_buffer__new_raw(int map_fd
, size_t page_cnt
,
5442 const struct perf_buffer_raw_opts
*opts
)
5444 struct perf_buffer_params p
= {};
5446 p
.attr
= opts
->attr
;
5447 p
.event_cb
= opts
->event_cb
;
5449 p
.cpu_cnt
= opts
->cpu_cnt
;
5450 p
.cpus
= opts
->cpus
;
5451 p
.map_keys
= opts
->map_keys
;
5453 return __perf_buffer__new(map_fd
, page_cnt
, &p
);
5456 static struct perf_buffer
*__perf_buffer__new(int map_fd
, size_t page_cnt
,
5457 struct perf_buffer_params
*p
)
5459 struct bpf_map_info map
= {};
5460 char msg
[STRERR_BUFSIZE
];
5461 struct perf_buffer
*pb
;
5465 if (page_cnt
& (page_cnt
- 1)) {
5466 pr_warning("page count should be power of two, but is %zu\n",
5468 return ERR_PTR(-EINVAL
);
5471 map_info_len
= sizeof(map
);
5472 err
= bpf_obj_get_info_by_fd(map_fd
, &map
, &map_info_len
);
5475 pr_warning("failed to get map info for map FD %d: %s\n",
5476 map_fd
, libbpf_strerror_r(err
, msg
, sizeof(msg
)));
5477 return ERR_PTR(err
);
5480 if (map
.type
!= BPF_MAP_TYPE_PERF_EVENT_ARRAY
) {
5481 pr_warning("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
5483 return ERR_PTR(-EINVAL
);
5486 pb
= calloc(1, sizeof(*pb
));
5488 return ERR_PTR(-ENOMEM
);
5490 pb
->event_cb
= p
->event_cb
;
5491 pb
->sample_cb
= p
->sample_cb
;
5492 pb
->lost_cb
= p
->lost_cb
;
5495 pb
->page_size
= getpagesize();
5496 pb
->mmap_size
= pb
->page_size
* page_cnt
;
5497 pb
->map_fd
= map_fd
;
5499 pb
->epoll_fd
= epoll_create1(EPOLL_CLOEXEC
);
5500 if (pb
->epoll_fd
< 0) {
5502 pr_warning("failed to create epoll instance: %s\n",
5503 libbpf_strerror_r(err
, msg
, sizeof(msg
)));
5507 if (p
->cpu_cnt
> 0) {
5508 pb
->cpu_cnt
= p
->cpu_cnt
;
5510 pb
->cpu_cnt
= libbpf_num_possible_cpus();
5511 if (pb
->cpu_cnt
< 0) {
5515 if (map
.max_entries
< pb
->cpu_cnt
)
5516 pb
->cpu_cnt
= map
.max_entries
;
5519 pb
->events
= calloc(pb
->cpu_cnt
, sizeof(*pb
->events
));
5522 pr_warning("failed to allocate events: out of memory\n");
5525 pb
->cpu_bufs
= calloc(pb
->cpu_cnt
, sizeof(*pb
->cpu_bufs
));
5526 if (!pb
->cpu_bufs
) {
5528 pr_warning("failed to allocate buffers: out of memory\n");
5532 for (i
= 0; i
< pb
->cpu_cnt
; i
++) {
5533 struct perf_cpu_buf
*cpu_buf
;
5536 cpu
= p
->cpu_cnt
> 0 ? p
->cpus
[i
] : i
;
5537 map_key
= p
->cpu_cnt
> 0 ? p
->map_keys
[i
] : i
;
5539 cpu_buf
= perf_buffer__open_cpu_buf(pb
, p
->attr
, cpu
, map_key
);
5540 if (IS_ERR(cpu_buf
)) {
5541 err
= PTR_ERR(cpu_buf
);
5545 pb
->cpu_bufs
[i
] = cpu_buf
;
5547 err
= bpf_map_update_elem(pb
->map_fd
, &map_key
,
5551 pr_warning("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
5552 cpu
, map_key
, cpu_buf
->fd
,
5553 libbpf_strerror_r(err
, msg
, sizeof(msg
)));
5557 pb
->events
[i
].events
= EPOLLIN
;
5558 pb
->events
[i
].data
.ptr
= cpu_buf
;
5559 if (epoll_ctl(pb
->epoll_fd
, EPOLL_CTL_ADD
, cpu_buf
->fd
,
5560 &pb
->events
[i
]) < 0) {
5562 pr_warning("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
5564 libbpf_strerror_r(err
, msg
, sizeof(msg
)));
5573 perf_buffer__free(pb
);
5574 return ERR_PTR(err
);
5577 struct perf_sample_raw
{
5578 struct perf_event_header header
;
5583 struct perf_sample_lost
{
5584 struct perf_event_header header
;
5590 static enum bpf_perf_event_ret
5591 perf_buffer__process_record(struct perf_event_header
*e
, void *ctx
)
5593 struct perf_cpu_buf
*cpu_buf
= ctx
;
5594 struct perf_buffer
*pb
= cpu_buf
->pb
;
5597 /* user wants full control over parsing perf event */
5599 return pb
->event_cb(pb
->ctx
, cpu_buf
->cpu
, e
);
5602 case PERF_RECORD_SAMPLE
: {
5603 struct perf_sample_raw
*s
= data
;
5606 pb
->sample_cb(pb
->ctx
, cpu_buf
->cpu
, s
->data
, s
->size
);
5609 case PERF_RECORD_LOST
: {
5610 struct perf_sample_lost
*s
= data
;
5613 pb
->lost_cb(pb
->ctx
, cpu_buf
->cpu
, s
->lost
);
5617 pr_warning("unknown perf sample type %d\n", e
->type
);
5618 return LIBBPF_PERF_EVENT_ERROR
;
5620 return LIBBPF_PERF_EVENT_CONT
;
5623 static int perf_buffer__process_records(struct perf_buffer
*pb
,
5624 struct perf_cpu_buf
*cpu_buf
)
5626 enum bpf_perf_event_ret ret
;
5628 ret
= bpf_perf_event_read_simple(cpu_buf
->base
, pb
->mmap_size
,
5629 pb
->page_size
, &cpu_buf
->buf
,
5631 perf_buffer__process_record
, cpu_buf
);
5632 if (ret
!= LIBBPF_PERF_EVENT_CONT
)
5637 int perf_buffer__poll(struct perf_buffer
*pb
, int timeout_ms
)
5641 cnt
= epoll_wait(pb
->epoll_fd
, pb
->events
, pb
->cpu_cnt
, timeout_ms
);
5642 for (i
= 0; i
< cnt
; i
++) {
5643 struct perf_cpu_buf
*cpu_buf
= pb
->events
[i
].data
.ptr
;
5645 err
= perf_buffer__process_records(pb
, cpu_buf
);
5647 pr_warning("error while processing records: %d\n", err
);
5651 return cnt
< 0 ? -errno
: cnt
;
5654 struct bpf_prog_info_array_desc
{
5655 int array_offset
; /* e.g. offset of jited_prog_insns */
5656 int count_offset
; /* e.g. offset of jited_prog_len */
5657 int size_offset
; /* > 0: offset of rec size,
5658 * < 0: fix size of -size_offset
5662 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc
[] = {
5663 [BPF_PROG_INFO_JITED_INSNS
] = {
5664 offsetof(struct bpf_prog_info
, jited_prog_insns
),
5665 offsetof(struct bpf_prog_info
, jited_prog_len
),
5668 [BPF_PROG_INFO_XLATED_INSNS
] = {
5669 offsetof(struct bpf_prog_info
, xlated_prog_insns
),
5670 offsetof(struct bpf_prog_info
, xlated_prog_len
),
5673 [BPF_PROG_INFO_MAP_IDS
] = {
5674 offsetof(struct bpf_prog_info
, map_ids
),
5675 offsetof(struct bpf_prog_info
, nr_map_ids
),
5676 -(int)sizeof(__u32
),
5678 [BPF_PROG_INFO_JITED_KSYMS
] = {
5679 offsetof(struct bpf_prog_info
, jited_ksyms
),
5680 offsetof(struct bpf_prog_info
, nr_jited_ksyms
),
5681 -(int)sizeof(__u64
),
5683 [BPF_PROG_INFO_JITED_FUNC_LENS
] = {
5684 offsetof(struct bpf_prog_info
, jited_func_lens
),
5685 offsetof(struct bpf_prog_info
, nr_jited_func_lens
),
5686 -(int)sizeof(__u32
),
5688 [BPF_PROG_INFO_FUNC_INFO
] = {
5689 offsetof(struct bpf_prog_info
, func_info
),
5690 offsetof(struct bpf_prog_info
, nr_func_info
),
5691 offsetof(struct bpf_prog_info
, func_info_rec_size
),
5693 [BPF_PROG_INFO_LINE_INFO
] = {
5694 offsetof(struct bpf_prog_info
, line_info
),
5695 offsetof(struct bpf_prog_info
, nr_line_info
),
5696 offsetof(struct bpf_prog_info
, line_info_rec_size
),
5698 [BPF_PROG_INFO_JITED_LINE_INFO
] = {
5699 offsetof(struct bpf_prog_info
, jited_line_info
),
5700 offsetof(struct bpf_prog_info
, nr_jited_line_info
),
5701 offsetof(struct bpf_prog_info
, jited_line_info_rec_size
),
5703 [BPF_PROG_INFO_PROG_TAGS
] = {
5704 offsetof(struct bpf_prog_info
, prog_tags
),
5705 offsetof(struct bpf_prog_info
, nr_prog_tags
),
5706 -(int)sizeof(__u8
) * BPF_TAG_SIZE
,
5711 static __u32
bpf_prog_info_read_offset_u32(struct bpf_prog_info
*info
, int offset
)
5713 __u32
*array
= (__u32
*)info
;
5716 return array
[offset
/ sizeof(__u32
)];
5717 return -(int)offset
;
5720 static __u64
bpf_prog_info_read_offset_u64(struct bpf_prog_info
*info
, int offset
)
5722 __u64
*array
= (__u64
*)info
;
5725 return array
[offset
/ sizeof(__u64
)];
5726 return -(int)offset
;
5729 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info
*info
, int offset
,
5732 __u32
*array
= (__u32
*)info
;
5735 array
[offset
/ sizeof(__u32
)] = val
;
5738 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info
*info
, int offset
,
5741 __u64
*array
= (__u64
*)info
;
5744 array
[offset
/ sizeof(__u64
)] = val
;
5747 struct bpf_prog_info_linear
*
5748 bpf_program__get_prog_info_linear(int fd
, __u64 arrays
)
5750 struct bpf_prog_info_linear
*info_linear
;
5751 struct bpf_prog_info info
= {};
5752 __u32 info_len
= sizeof(info
);
5757 if (arrays
>> BPF_PROG_INFO_LAST_ARRAY
)
5758 return ERR_PTR(-EINVAL
);
5760 /* step 1: get array dimensions */
5761 err
= bpf_obj_get_info_by_fd(fd
, &info
, &info_len
);
5763 pr_debug("can't get prog info: %s", strerror(errno
));
5764 return ERR_PTR(-EFAULT
);
5767 /* step 2: calculate total size of all arrays */
5768 for (i
= BPF_PROG_INFO_FIRST_ARRAY
; i
< BPF_PROG_INFO_LAST_ARRAY
; ++i
) {
5769 bool include_array
= (arrays
& (1UL << i
)) > 0;
5770 struct bpf_prog_info_array_desc
*desc
;
5773 desc
= bpf_prog_info_array_desc
+ i
;
5775 /* kernel is too old to support this field */
5776 if (info_len
< desc
->array_offset
+ sizeof(__u32
) ||
5777 info_len
< desc
->count_offset
+ sizeof(__u32
) ||
5778 (desc
->size_offset
> 0 && info_len
< desc
->size_offset
))
5779 include_array
= false;
5781 if (!include_array
) {
5782 arrays
&= ~(1UL << i
); /* clear the bit */
5786 count
= bpf_prog_info_read_offset_u32(&info
, desc
->count_offset
);
5787 size
= bpf_prog_info_read_offset_u32(&info
, desc
->size_offset
);
5789 data_len
+= count
* size
;
5792 /* step 3: allocate continuous memory */
5793 data_len
= roundup(data_len
, sizeof(__u64
));
5794 info_linear
= malloc(sizeof(struct bpf_prog_info_linear
) + data_len
);
5796 return ERR_PTR(-ENOMEM
);
5798 /* step 4: fill data to info_linear->info */
5799 info_linear
->arrays
= arrays
;
5800 memset(&info_linear
->info
, 0, sizeof(info
));
5801 ptr
= info_linear
->data
;
5803 for (i
= BPF_PROG_INFO_FIRST_ARRAY
; i
< BPF_PROG_INFO_LAST_ARRAY
; ++i
) {
5804 struct bpf_prog_info_array_desc
*desc
;
5807 if ((arrays
& (1UL << i
)) == 0)
5810 desc
= bpf_prog_info_array_desc
+ i
;
5811 count
= bpf_prog_info_read_offset_u32(&info
, desc
->count_offset
);
5812 size
= bpf_prog_info_read_offset_u32(&info
, desc
->size_offset
);
5813 bpf_prog_info_set_offset_u32(&info_linear
->info
,
5814 desc
->count_offset
, count
);
5815 bpf_prog_info_set_offset_u32(&info_linear
->info
,
5816 desc
->size_offset
, size
);
5817 bpf_prog_info_set_offset_u64(&info_linear
->info
,
5820 ptr
+= count
* size
;
5823 /* step 5: call syscall again to get required arrays */
5824 err
= bpf_obj_get_info_by_fd(fd
, &info_linear
->info
, &info_len
);
5826 pr_debug("can't get prog info: %s", strerror(errno
));
5828 return ERR_PTR(-EFAULT
);
5831 /* step 6: verify the data */
5832 for (i
= BPF_PROG_INFO_FIRST_ARRAY
; i
< BPF_PROG_INFO_LAST_ARRAY
; ++i
) {
5833 struct bpf_prog_info_array_desc
*desc
;
5836 if ((arrays
& (1UL << i
)) == 0)
5839 desc
= bpf_prog_info_array_desc
+ i
;
5840 v1
= bpf_prog_info_read_offset_u32(&info
, desc
->count_offset
);
5841 v2
= bpf_prog_info_read_offset_u32(&info_linear
->info
,
5842 desc
->count_offset
);
5844 pr_warning("%s: mismatch in element count\n", __func__
);
5846 v1
= bpf_prog_info_read_offset_u32(&info
, desc
->size_offset
);
5847 v2
= bpf_prog_info_read_offset_u32(&info_linear
->info
,
5850 pr_warning("%s: mismatch in rec size\n", __func__
);
5853 /* step 7: update info_len and data_len */
5854 info_linear
->info_len
= sizeof(struct bpf_prog_info
);
5855 info_linear
->data_len
= data_len
;
5860 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear
*info_linear
)
5864 for (i
= BPF_PROG_INFO_FIRST_ARRAY
; i
< BPF_PROG_INFO_LAST_ARRAY
; ++i
) {
5865 struct bpf_prog_info_array_desc
*desc
;
5868 if ((info_linear
->arrays
& (1UL << i
)) == 0)
5871 desc
= bpf_prog_info_array_desc
+ i
;
5872 addr
= bpf_prog_info_read_offset_u64(&info_linear
->info
,
5873 desc
->array_offset
);
5874 offs
= addr
- ptr_to_u64(info_linear
->data
);
5875 bpf_prog_info_set_offset_u64(&info_linear
->info
,
5876 desc
->array_offset
, offs
);
5880 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear
*info_linear
)
5884 for (i
= BPF_PROG_INFO_FIRST_ARRAY
; i
< BPF_PROG_INFO_LAST_ARRAY
; ++i
) {
5885 struct bpf_prog_info_array_desc
*desc
;
5888 if ((info_linear
->arrays
& (1UL << i
)) == 0)
5891 desc
= bpf_prog_info_array_desc
+ i
;
5892 offs
= bpf_prog_info_read_offset_u64(&info_linear
->info
,
5893 desc
->array_offset
);
5894 addr
= offs
+ ptr_to_u64(info_linear
->data
);
5895 bpf_prog_info_set_offset_u64(&info_linear
->info
,
5896 desc
->array_offset
, addr
);
5900 int libbpf_num_possible_cpus(void)
5902 static const char *fcpu
= "/sys/devices/system/cpu/possible";
5903 int len
= 0, n
= 0, il
= 0, ir
= 0;
5904 unsigned int start
= 0, end
= 0;
5911 tmp_cpus
= READ_ONCE(cpus
);
5915 fd
= open(fcpu
, O_RDONLY
);
5918 pr_warning("Failed to open file %s: %s\n",
5919 fcpu
, strerror(error
));
5922 len
= read(fd
, buf
, sizeof(buf
));
5925 error
= len
? errno
: EINVAL
;
5926 pr_warning("Failed to read # of possible cpus from %s: %s\n",
5927 fcpu
, strerror(error
));
5930 if (len
== sizeof(buf
)) {
5931 pr_warning("File %s size overflow\n", fcpu
);
5936 for (ir
= 0, tmp_cpus
= 0; ir
<= len
; ir
++) {
5937 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
5938 if (buf
[ir
] == ',' || buf
[ir
] == '\0') {
5940 n
= sscanf(&buf
[il
], "%u-%u", &start
, &end
);
5942 pr_warning("Failed to get # CPUs from %s\n",
5945 } else if (n
== 1) {
5948 tmp_cpus
+= end
- start
+ 1;
5952 if (tmp_cpus
<= 0) {
5953 pr_warning("Invalid #CPUs %d from %s\n", tmp_cpus
, fcpu
);
5957 WRITE_ONCE(cpus
, tmp_cpus
);