2 * Common eBPF ELF object loading operations.
4 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
6 * Copyright (C) 2015 Huawei Inc.
7 * Copyright (C) 2017 Nicira, Inc.
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation;
12 * version 2.1 of the License (not later!)
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this program; if not, see <http://www.gnu.org/licenses>
32 #include <asm/unistd.h>
33 #include <linux/err.h>
34 #include <linux/kernel.h>
35 #include <linux/bpf.h>
36 #include <linux/list.h>
37 #include <linux/limits.h>
39 #include <sys/types.h>
52 #define BPF_FS_MAGIC 0xcafe4a11
55 #define __printf(a, b) __attribute__((format(printf, a, b)))
58 static int __base_pr(const char *format
, ...)
63 va_start(args
, format
);
64 err
= vfprintf(stderr
, format
, args
);
69 static __printf(1, 2) libbpf_print_fn_t __pr_warning
= __base_pr
;
70 static __printf(1, 2) libbpf_print_fn_t __pr_info
= __base_pr
;
71 static __printf(1, 2) libbpf_print_fn_t __pr_debug
;
73 #define __pr(func, fmt, ...) \
76 (func)("libbpf: " fmt, ##__VA_ARGS__); \
79 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__)
80 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__)
81 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__)
83 void libbpf_set_print(libbpf_print_fn_t warn
,
84 libbpf_print_fn_t info
,
85 libbpf_print_fn_t debug
)
92 #define STRERR_BUFSIZE 128
94 #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START)
95 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c)
96 #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START)
98 static const char *libbpf_strerror_table
[NR_ERRNO
] = {
99 [ERRCODE_OFFSET(LIBELF
)] = "Something wrong in libelf",
100 [ERRCODE_OFFSET(FORMAT
)] = "BPF object format invalid",
101 [ERRCODE_OFFSET(KVERSION
)] = "'version' section incorrect or lost",
102 [ERRCODE_OFFSET(ENDIAN
)] = "Endian mismatch",
103 [ERRCODE_OFFSET(INTERNAL
)] = "Internal error in libbpf",
104 [ERRCODE_OFFSET(RELOC
)] = "Relocation failed",
105 [ERRCODE_OFFSET(VERIFY
)] = "Kernel verifier blocks program loading",
106 [ERRCODE_OFFSET(PROG2BIG
)] = "Program too big",
107 [ERRCODE_OFFSET(KVER
)] = "Incorrect kernel version",
108 [ERRCODE_OFFSET(PROGTYPE
)] = "Kernel doesn't support this program type",
111 int libbpf_strerror(int err
, char *buf
, size_t size
)
116 err
= err
> 0 ? err
: -err
;
118 if (err
< __LIBBPF_ERRNO__START
) {
121 ret
= strerror_r(err
, buf
, size
);
122 buf
[size
- 1] = '\0';
126 if (err
< __LIBBPF_ERRNO__END
) {
129 msg
= libbpf_strerror_table
[ERRNO_OFFSET(err
)];
130 snprintf(buf
, size
, "%s", msg
);
131 buf
[size
- 1] = '\0';
135 snprintf(buf
, size
, "Unknown libbpf error %d", err
);
136 buf
[size
- 1] = '\0';
140 #define CHECK_ERR(action, err, out) do { \
147 /* Copied from tools/perf/util/util.h */
149 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
153 # define zclose(fd) ({ \
156 ___err = close((fd)); \
161 #ifdef HAVE_LIBELF_MMAP_SUPPORT
162 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
164 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
168 * bpf_prog should be a better name but it has been used in
172 /* Index in elf obj file, for relocation use. */
175 struct bpf_insn
*insns
;
177 enum bpf_prog_type type
;
189 bpf_program_prep_t preprocessor
;
191 struct bpf_object
*obj
;
193 bpf_program_clear_priv_t clear_priv
;
200 struct bpf_map_def def
;
202 bpf_map_clear_priv_t clear_priv
;
205 static LIST_HEAD(bpf_objects_list
);
211 struct bpf_program
*programs
;
213 struct bpf_map
*maps
;
219 * Information when doing elf related work. Only valid if fd
238 * All loaded bpf_object is linked in a list, which is
239 * hidden to caller. bpf_objects__<func> handlers deal with
242 struct list_head list
;
245 bpf_object_clear_priv_t clear_priv
;
249 #define obj_elf_valid(o) ((o)->efile.elf)
251 static void bpf_program__unload(struct bpf_program
*prog
)
259 * If the object is opened but the program was never loaded,
260 * it is possible that prog->instances.nr == -1.
262 if (prog
->instances
.nr
> 0) {
263 for (i
= 0; i
< prog
->instances
.nr
; i
++)
264 zclose(prog
->instances
.fds
[i
]);
265 } else if (prog
->instances
.nr
!= -1) {
266 pr_warning("Internal error: instances.nr is %d\n",
270 prog
->instances
.nr
= -1;
271 zfree(&prog
->instances
.fds
);
274 static void bpf_program__exit(struct bpf_program
*prog
)
279 if (prog
->clear_priv
)
280 prog
->clear_priv(prog
, prog
->priv
);
283 prog
->clear_priv
= NULL
;
285 bpf_program__unload(prog
);
286 zfree(&prog
->section_name
);
288 zfree(&prog
->reloc_desc
);
296 bpf_program__init(void *data
, size_t size
, char *name
, int idx
,
297 struct bpf_program
*prog
)
299 if (size
< sizeof(struct bpf_insn
)) {
300 pr_warning("corrupted section '%s'\n", name
);
304 bzero(prog
, sizeof(*prog
));
306 prog
->section_name
= strdup(name
);
307 if (!prog
->section_name
) {
308 pr_warning("failed to alloc name for prog %s\n",
313 prog
->insns
= malloc(size
);
315 pr_warning("failed to alloc insns for %s\n", name
);
318 prog
->insns_cnt
= size
/ sizeof(struct bpf_insn
);
319 memcpy(prog
->insns
, data
,
320 prog
->insns_cnt
* sizeof(struct bpf_insn
));
322 prog
->instances
.fds
= NULL
;
323 prog
->instances
.nr
= -1;
324 prog
->type
= BPF_PROG_TYPE_KPROBE
;
328 bpf_program__exit(prog
);
333 bpf_object__add_program(struct bpf_object
*obj
, void *data
, size_t size
,
336 struct bpf_program prog
, *progs
;
339 err
= bpf_program__init(data
, size
, name
, idx
, &prog
);
343 progs
= obj
->programs
;
344 nr_progs
= obj
->nr_programs
;
346 progs
= realloc(progs
, sizeof(progs
[0]) * (nr_progs
+ 1));
349 * In this case the original obj->programs
350 * is still valid, so don't need special treat for
351 * bpf_close_object().
353 pr_warning("failed to alloc a new program '%s'\n",
355 bpf_program__exit(&prog
);
359 pr_debug("found program %s\n", prog
.section_name
);
360 obj
->programs
= progs
;
361 obj
->nr_programs
= nr_progs
+ 1;
363 progs
[nr_progs
] = prog
;
367 static struct bpf_object
*bpf_object__new(const char *path
,
371 struct bpf_object
*obj
;
373 obj
= calloc(1, sizeof(struct bpf_object
) + strlen(path
) + 1);
375 pr_warning("alloc memory failed for %s\n", path
);
376 return ERR_PTR(-ENOMEM
);
379 strcpy(obj
->path
, path
);
383 * Caller of this function should also calls
384 * bpf_object__elf_finish() after data collection to return
385 * obj_buf to user. If not, we should duplicate the buffer to
386 * avoid user freeing them before elf finish.
388 obj
->efile
.obj_buf
= obj_buf
;
389 obj
->efile
.obj_buf_sz
= obj_buf_sz
;
390 obj
->efile
.maps_shndx
= -1;
394 INIT_LIST_HEAD(&obj
->list
);
395 list_add(&obj
->list
, &bpf_objects_list
);
399 static void bpf_object__elf_finish(struct bpf_object
*obj
)
401 if (!obj_elf_valid(obj
))
404 if (obj
->efile
.elf
) {
405 elf_end(obj
->efile
.elf
);
406 obj
->efile
.elf
= NULL
;
408 obj
->efile
.symbols
= NULL
;
410 zfree(&obj
->efile
.reloc
);
411 obj
->efile
.nr_reloc
= 0;
412 zclose(obj
->efile
.fd
);
413 obj
->efile
.obj_buf
= NULL
;
414 obj
->efile
.obj_buf_sz
= 0;
417 static int bpf_object__elf_init(struct bpf_object
*obj
)
422 if (obj_elf_valid(obj
)) {
423 pr_warning("elf init: internal error\n");
424 return -LIBBPF_ERRNO__LIBELF
;
427 if (obj
->efile
.obj_buf_sz
> 0) {
429 * obj_buf should have been validated by
430 * bpf_object__open_buffer().
432 obj
->efile
.elf
= elf_memory(obj
->efile
.obj_buf
,
433 obj
->efile
.obj_buf_sz
);
435 obj
->efile
.fd
= open(obj
->path
, O_RDONLY
);
436 if (obj
->efile
.fd
< 0) {
437 pr_warning("failed to open %s: %s\n", obj
->path
,
442 obj
->efile
.elf
= elf_begin(obj
->efile
.fd
,
443 LIBBPF_ELF_C_READ_MMAP
,
447 if (!obj
->efile
.elf
) {
448 pr_warning("failed to open %s as ELF file\n",
450 err
= -LIBBPF_ERRNO__LIBELF
;
454 if (!gelf_getehdr(obj
->efile
.elf
, &obj
->efile
.ehdr
)) {
455 pr_warning("failed to get EHDR from %s\n",
457 err
= -LIBBPF_ERRNO__FORMAT
;
460 ep
= &obj
->efile
.ehdr
;
462 /* Old LLVM set e_machine to EM_NONE */
463 if ((ep
->e_type
!= ET_REL
) || (ep
->e_machine
&& (ep
->e_machine
!= EM_BPF
))) {
464 pr_warning("%s is not an eBPF object file\n",
466 err
= -LIBBPF_ERRNO__FORMAT
;
472 bpf_object__elf_finish(obj
);
477 bpf_object__check_endianness(struct bpf_object
*obj
)
479 static unsigned int const endian
= 1;
481 switch (obj
->efile
.ehdr
.e_ident
[EI_DATA
]) {
483 /* We are big endian, BPF obj is little endian. */
484 if (*(unsigned char const *)&endian
!= 1)
489 /* We are little endian, BPF obj is big endian. */
490 if (*(unsigned char const *)&endian
!= 0)
494 return -LIBBPF_ERRNO__ENDIAN
;
500 pr_warning("Error: endianness mismatch.\n");
501 return -LIBBPF_ERRNO__ENDIAN
;
505 bpf_object__init_license(struct bpf_object
*obj
,
506 void *data
, size_t size
)
508 memcpy(obj
->license
, data
,
509 min(size
, sizeof(obj
->license
) - 1));
510 pr_debug("license of %s is %s\n", obj
->path
, obj
->license
);
515 bpf_object__init_kversion(struct bpf_object
*obj
,
516 void *data
, size_t size
)
520 if (size
!= sizeof(kver
)) {
521 pr_warning("invalid kver section in %s\n", obj
->path
);
522 return -LIBBPF_ERRNO__FORMAT
;
524 memcpy(&kver
, data
, sizeof(kver
));
525 obj
->kern_version
= kver
;
526 pr_debug("kernel version of %s is %x\n", obj
->path
,
532 bpf_object__validate_maps(struct bpf_object
*obj
)
537 * If there's only 1 map, the only error case should have been
538 * catched in bpf_object__init_maps().
540 if (!obj
->maps
|| !obj
->nr_maps
|| (obj
->nr_maps
== 1))
543 for (i
= 1; i
< obj
->nr_maps
; i
++) {
544 const struct bpf_map
*a
= &obj
->maps
[i
- 1];
545 const struct bpf_map
*b
= &obj
->maps
[i
];
547 if (b
->offset
- a
->offset
< sizeof(struct bpf_map_def
)) {
548 pr_warning("corrupted map section in %s: map \"%s\" too small\n",
556 static int compare_bpf_map(const void *_a
, const void *_b
)
558 const struct bpf_map
*a
= _a
;
559 const struct bpf_map
*b
= _b
;
561 return a
->offset
- b
->offset
;
565 bpf_object__init_maps(struct bpf_object
*obj
)
567 int i
, map_idx
, nr_maps
= 0;
570 Elf_Data
*symbols
= obj
->efile
.symbols
;
572 if (obj
->efile
.maps_shndx
< 0)
577 scn
= elf_getscn(obj
->efile
.elf
, obj
->efile
.maps_shndx
);
579 data
= elf_getdata(scn
, NULL
);
581 pr_warning("failed to get Elf_Data from map section %d\n",
582 obj
->efile
.maps_shndx
);
587 * Count number of maps. Each map has a name.
588 * Array of maps is not supported: only the first element is
591 * TODO: Detect array of map and report error.
593 for (i
= 0; i
< symbols
->d_size
/ sizeof(GElf_Sym
); i
++) {
596 if (!gelf_getsym(symbols
, i
, &sym
))
598 if (sym
.st_shndx
!= obj
->efile
.maps_shndx
)
603 /* Alloc obj->maps and fill nr_maps. */
604 pr_debug("maps in %s: %d maps in %zd bytes\n", obj
->path
,
605 nr_maps
, data
->d_size
);
610 obj
->maps
= calloc(nr_maps
, sizeof(obj
->maps
[0]));
612 pr_warning("alloc maps for object failed\n");
615 obj
->nr_maps
= nr_maps
;
618 * fill all fd with -1 so won't close incorrect
619 * fd (fd=0 is stdin) when failure (zclose won't close
622 for (i
= 0; i
< nr_maps
; i
++)
623 obj
->maps
[i
].fd
= -1;
626 * Fill obj->maps using data in "maps" section.
628 for (i
= 0, map_idx
= 0; i
< symbols
->d_size
/ sizeof(GElf_Sym
); i
++) {
630 const char *map_name
;
631 struct bpf_map_def
*def
;
633 if (!gelf_getsym(symbols
, i
, &sym
))
635 if (sym
.st_shndx
!= obj
->efile
.maps_shndx
)
638 map_name
= elf_strptr(obj
->efile
.elf
,
639 obj
->efile
.strtabidx
,
641 obj
->maps
[map_idx
].offset
= sym
.st_value
;
642 if (sym
.st_value
+ sizeof(struct bpf_map_def
) > data
->d_size
) {
643 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
644 obj
->path
, map_name
);
648 obj
->maps
[map_idx
].name
= strdup(map_name
);
649 if (!obj
->maps
[map_idx
].name
) {
650 pr_warning("failed to alloc map name\n");
653 pr_debug("map %d is \"%s\"\n", map_idx
,
654 obj
->maps
[map_idx
].name
);
655 def
= (struct bpf_map_def
*)(data
->d_buf
+ sym
.st_value
);
656 obj
->maps
[map_idx
].def
= *def
;
660 qsort(obj
->maps
, obj
->nr_maps
, sizeof(obj
->maps
[0]), compare_bpf_map
);
661 return bpf_object__validate_maps(obj
);
664 static int bpf_object__elf_collect(struct bpf_object
*obj
)
666 Elf
*elf
= obj
->efile
.elf
;
667 GElf_Ehdr
*ep
= &obj
->efile
.ehdr
;
669 int idx
= 0, err
= 0;
671 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
672 if (!elf_rawdata(elf_getscn(elf
, ep
->e_shstrndx
), NULL
)) {
673 pr_warning("failed to get e_shstrndx from %s\n",
675 return -LIBBPF_ERRNO__FORMAT
;
678 while ((scn
= elf_nextscn(elf
, scn
)) != NULL
) {
684 if (gelf_getshdr(scn
, &sh
) != &sh
) {
685 pr_warning("failed to get section header from %s\n",
687 err
= -LIBBPF_ERRNO__FORMAT
;
691 name
= elf_strptr(elf
, ep
->e_shstrndx
, sh
.sh_name
);
693 pr_warning("failed to get section name from %s\n",
695 err
= -LIBBPF_ERRNO__FORMAT
;
699 data
= elf_getdata(scn
, 0);
701 pr_warning("failed to get section data from %s(%s)\n",
703 err
= -LIBBPF_ERRNO__FORMAT
;
706 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n",
707 name
, (unsigned long)data
->d_size
,
708 (int)sh
.sh_link
, (unsigned long)sh
.sh_flags
,
711 if (strcmp(name
, "license") == 0)
712 err
= bpf_object__init_license(obj
,
715 else if (strcmp(name
, "version") == 0)
716 err
= bpf_object__init_kversion(obj
,
719 else if (strcmp(name
, "maps") == 0)
720 obj
->efile
.maps_shndx
= idx
;
721 else if (sh
.sh_type
== SHT_SYMTAB
) {
722 if (obj
->efile
.symbols
) {
723 pr_warning("bpf: multiple SYMTAB in %s\n",
725 err
= -LIBBPF_ERRNO__FORMAT
;
727 obj
->efile
.symbols
= data
;
728 obj
->efile
.strtabidx
= sh
.sh_link
;
730 } else if ((sh
.sh_type
== SHT_PROGBITS
) &&
731 (sh
.sh_flags
& SHF_EXECINSTR
) &&
732 (data
->d_size
> 0)) {
733 err
= bpf_object__add_program(obj
, data
->d_buf
,
734 data
->d_size
, name
, idx
);
736 char errmsg
[STRERR_BUFSIZE
];
738 strerror_r(-err
, errmsg
, sizeof(errmsg
));
739 pr_warning("failed to alloc program %s (%s): %s",
740 name
, obj
->path
, errmsg
);
742 } else if (sh
.sh_type
== SHT_REL
) {
743 void *reloc
= obj
->efile
.reloc
;
744 int nr_reloc
= obj
->efile
.nr_reloc
+ 1;
746 reloc
= realloc(reloc
,
747 sizeof(*obj
->efile
.reloc
) * nr_reloc
);
749 pr_warning("realloc failed\n");
752 int n
= nr_reloc
- 1;
754 obj
->efile
.reloc
= reloc
;
755 obj
->efile
.nr_reloc
= nr_reloc
;
757 obj
->efile
.reloc
[n
].shdr
= sh
;
758 obj
->efile
.reloc
[n
].data
= data
;
765 if (!obj
->efile
.strtabidx
|| obj
->efile
.strtabidx
>= idx
) {
766 pr_warning("Corrupted ELF file: index of strtab invalid\n");
767 return LIBBPF_ERRNO__FORMAT
;
769 if (obj
->efile
.maps_shndx
>= 0)
770 err
= bpf_object__init_maps(obj
);
775 static struct bpf_program
*
776 bpf_object__find_prog_by_idx(struct bpf_object
*obj
, int idx
)
778 struct bpf_program
*prog
;
781 for (i
= 0; i
< obj
->nr_programs
; i
++) {
782 prog
= &obj
->programs
[i
];
783 if (prog
->idx
== idx
)
790 bpf_program__collect_reloc(struct bpf_program
*prog
,
791 size_t nr_maps
, GElf_Shdr
*shdr
,
792 Elf_Data
*data
, Elf_Data
*symbols
,
793 int maps_shndx
, struct bpf_map
*maps
)
797 pr_debug("collecting relocating info for: '%s'\n",
799 nrels
= shdr
->sh_size
/ shdr
->sh_entsize
;
801 prog
->reloc_desc
= malloc(sizeof(*prog
->reloc_desc
) * nrels
);
802 if (!prog
->reloc_desc
) {
803 pr_warning("failed to alloc memory in relocation\n");
806 prog
->nr_reloc
= nrels
;
808 for (i
= 0; i
< nrels
; i
++) {
811 unsigned int insn_idx
;
812 struct bpf_insn
*insns
= prog
->insns
;
815 if (!gelf_getrel(data
, i
, &rel
)) {
816 pr_warning("relocation: failed to get %d reloc\n", i
);
817 return -LIBBPF_ERRNO__FORMAT
;
820 if (!gelf_getsym(symbols
,
821 GELF_R_SYM(rel
.r_info
),
823 pr_warning("relocation: symbol %"PRIx64
" not found\n",
824 GELF_R_SYM(rel
.r_info
));
825 return -LIBBPF_ERRNO__FORMAT
;
828 if (sym
.st_shndx
!= maps_shndx
) {
829 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
830 prog
->section_name
, sym
.st_shndx
);
831 return -LIBBPF_ERRNO__RELOC
;
834 insn_idx
= rel
.r_offset
/ sizeof(struct bpf_insn
);
835 pr_debug("relocation: insn_idx=%u\n", insn_idx
);
837 if (insns
[insn_idx
].code
!= (BPF_LD
| BPF_IMM
| BPF_DW
)) {
838 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
839 insn_idx
, insns
[insn_idx
].code
);
840 return -LIBBPF_ERRNO__RELOC
;
843 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
844 for (map_idx
= 0; map_idx
< nr_maps
; map_idx
++) {
845 if (maps
[map_idx
].offset
== sym
.st_value
) {
846 pr_debug("relocation: find map %zd (%s) for insn %u\n",
847 map_idx
, maps
[map_idx
].name
, insn_idx
);
852 if (map_idx
>= nr_maps
) {
853 pr_warning("bpf relocation: map_idx %d large than %d\n",
854 (int)map_idx
, (int)nr_maps
- 1);
855 return -LIBBPF_ERRNO__RELOC
;
858 prog
->reloc_desc
[i
].insn_idx
= insn_idx
;
859 prog
->reloc_desc
[i
].map_idx
= map_idx
;
865 bpf_object__create_maps(struct bpf_object
*obj
)
869 for (i
= 0; i
< obj
->nr_maps
; i
++) {
870 struct bpf_map_def
*def
= &obj
->maps
[i
].def
;
871 int *pfd
= &obj
->maps
[i
].fd
;
873 *pfd
= bpf_create_map(def
->type
,
882 pr_warning("failed to create map (name: '%s'): %s\n",
885 for (j
= 0; j
< i
; j
++)
886 zclose(obj
->maps
[j
].fd
);
889 pr_debug("create map %s: fd=%d\n", obj
->maps
[i
].name
, *pfd
);
896 bpf_program__relocate(struct bpf_program
*prog
, struct bpf_object
*obj
)
900 if (!prog
|| !prog
->reloc_desc
)
903 for (i
= 0; i
< prog
->nr_reloc
; i
++) {
904 int insn_idx
, map_idx
;
905 struct bpf_insn
*insns
= prog
->insns
;
907 insn_idx
= prog
->reloc_desc
[i
].insn_idx
;
908 map_idx
= prog
->reloc_desc
[i
].map_idx
;
910 if (insn_idx
>= (int)prog
->insns_cnt
) {
911 pr_warning("relocation out of range: '%s'\n",
913 return -LIBBPF_ERRNO__RELOC
;
915 insns
[insn_idx
].src_reg
= BPF_PSEUDO_MAP_FD
;
916 insns
[insn_idx
].imm
= obj
->maps
[map_idx
].fd
;
919 zfree(&prog
->reloc_desc
);
926 bpf_object__relocate(struct bpf_object
*obj
)
928 struct bpf_program
*prog
;
932 for (i
= 0; i
< obj
->nr_programs
; i
++) {
933 prog
= &obj
->programs
[i
];
935 err
= bpf_program__relocate(prog
, obj
);
937 pr_warning("failed to relocate '%s'\n",
945 static int bpf_object__collect_reloc(struct bpf_object
*obj
)
949 if (!obj_elf_valid(obj
)) {
950 pr_warning("Internal error: elf object is closed\n");
951 return -LIBBPF_ERRNO__INTERNAL
;
954 for (i
= 0; i
< obj
->efile
.nr_reloc
; i
++) {
955 GElf_Shdr
*shdr
= &obj
->efile
.reloc
[i
].shdr
;
956 Elf_Data
*data
= obj
->efile
.reloc
[i
].data
;
957 int idx
= shdr
->sh_info
;
958 struct bpf_program
*prog
;
959 size_t nr_maps
= obj
->nr_maps
;
961 if (shdr
->sh_type
!= SHT_REL
) {
962 pr_warning("internal error at %d\n", __LINE__
);
963 return -LIBBPF_ERRNO__INTERNAL
;
966 prog
= bpf_object__find_prog_by_idx(obj
, idx
);
968 pr_warning("relocation failed: no %d section\n",
970 return -LIBBPF_ERRNO__RELOC
;
973 err
= bpf_program__collect_reloc(prog
, nr_maps
,
976 obj
->efile
.maps_shndx
,
985 load_program(enum bpf_prog_type type
, struct bpf_insn
*insns
,
986 int insns_cnt
, char *license
, u32 kern_version
, int *pfd
)
991 if (!insns
|| !insns_cnt
)
994 log_buf
= malloc(BPF_LOG_BUF_SIZE
);
996 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
998 ret
= bpf_load_program(type
, insns
, insns_cnt
, license
,
999 kern_version
, log_buf
, BPF_LOG_BUF_SIZE
);
1007 ret
= -LIBBPF_ERRNO__LOAD
;
1008 pr_warning("load bpf program failed: %s\n", strerror(errno
));
1010 if (log_buf
&& log_buf
[0] != '\0') {
1011 ret
= -LIBBPF_ERRNO__VERIFY
;
1012 pr_warning("-- BEGIN DUMP LOG ---\n");
1013 pr_warning("\n%s\n", log_buf
);
1014 pr_warning("-- END LOG --\n");
1015 } else if (insns_cnt
>= BPF_MAXINSNS
) {
1016 pr_warning("Program too large (%d insns), at most %d insns\n",
1017 insns_cnt
, BPF_MAXINSNS
);
1018 ret
= -LIBBPF_ERRNO__PROG2BIG
;
1020 /* Wrong program type? */
1021 if (type
!= BPF_PROG_TYPE_KPROBE
) {
1024 fd
= bpf_load_program(BPF_PROG_TYPE_KPROBE
, insns
,
1025 insns_cnt
, license
, kern_version
,
1029 ret
= -LIBBPF_ERRNO__PROGTYPE
;
1035 ret
= -LIBBPF_ERRNO__KVER
;
1044 bpf_program__load(struct bpf_program
*prog
,
1045 char *license
, u32 kern_version
)
1049 if (prog
->instances
.nr
< 0 || !prog
->instances
.fds
) {
1050 if (prog
->preprocessor
) {
1051 pr_warning("Internal error: can't load program '%s'\n",
1052 prog
->section_name
);
1053 return -LIBBPF_ERRNO__INTERNAL
;
1056 prog
->instances
.fds
= malloc(sizeof(int));
1057 if (!prog
->instances
.fds
) {
1058 pr_warning("Not enough memory for BPF fds\n");
1061 prog
->instances
.nr
= 1;
1062 prog
->instances
.fds
[0] = -1;
1065 if (!prog
->preprocessor
) {
1066 if (prog
->instances
.nr
!= 1) {
1067 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
1068 prog
->section_name
, prog
->instances
.nr
);
1070 err
= load_program(prog
->type
, prog
->insns
, prog
->insns_cnt
,
1071 license
, kern_version
, &fd
);
1073 prog
->instances
.fds
[0] = fd
;
1077 for (i
= 0; i
< prog
->instances
.nr
; i
++) {
1078 struct bpf_prog_prep_result result
;
1079 bpf_program_prep_t preprocessor
= prog
->preprocessor
;
1081 bzero(&result
, sizeof(result
));
1082 err
= preprocessor(prog
, i
, prog
->insns
,
1083 prog
->insns_cnt
, &result
);
1085 pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1086 i
, prog
->section_name
);
1090 if (!result
.new_insn_ptr
|| !result
.new_insn_cnt
) {
1091 pr_debug("Skip loading the %dth instance of program '%s'\n",
1092 i
, prog
->section_name
);
1093 prog
->instances
.fds
[i
] = -1;
1099 err
= load_program(prog
->type
, result
.new_insn_ptr
,
1100 result
.new_insn_cnt
,
1101 license
, kern_version
, &fd
);
1104 pr_warning("Loading the %dth instance of program '%s' failed\n",
1105 i
, prog
->section_name
);
1111 prog
->instances
.fds
[i
] = fd
;
1115 pr_warning("failed to load program '%s'\n",
1116 prog
->section_name
);
1117 zfree(&prog
->insns
);
1118 prog
->insns_cnt
= 0;
1123 bpf_object__load_progs(struct bpf_object
*obj
)
1128 for (i
= 0; i
< obj
->nr_programs
; i
++) {
1129 err
= bpf_program__load(&obj
->programs
[i
],
1138 static int bpf_object__validate(struct bpf_object
*obj
)
1140 if (obj
->kern_version
== 0) {
1141 pr_warning("%s doesn't provide kernel version\n",
1143 return -LIBBPF_ERRNO__KVERSION
;
1148 static struct bpf_object
*
1149 __bpf_object__open(const char *path
, void *obj_buf
, size_t obj_buf_sz
)
1151 struct bpf_object
*obj
;
1154 if (elf_version(EV_CURRENT
) == EV_NONE
) {
1155 pr_warning("failed to init libelf for %s\n", path
);
1156 return ERR_PTR(-LIBBPF_ERRNO__LIBELF
);
1159 obj
= bpf_object__new(path
, obj_buf
, obj_buf_sz
);
1163 CHECK_ERR(bpf_object__elf_init(obj
), err
, out
);
1164 CHECK_ERR(bpf_object__check_endianness(obj
), err
, out
);
1165 CHECK_ERR(bpf_object__elf_collect(obj
), err
, out
);
1166 CHECK_ERR(bpf_object__collect_reloc(obj
), err
, out
);
1167 CHECK_ERR(bpf_object__validate(obj
), err
, out
);
1169 bpf_object__elf_finish(obj
);
1172 bpf_object__close(obj
);
1173 return ERR_PTR(err
);
1176 struct bpf_object
*bpf_object__open(const char *path
)
1178 /* param validation */
1182 pr_debug("loading %s\n", path
);
1184 return __bpf_object__open(path
, NULL
, 0);
1187 struct bpf_object
*bpf_object__open_buffer(void *obj_buf
,
1193 /* param validation */
1194 if (!obj_buf
|| obj_buf_sz
<= 0)
1198 snprintf(tmp_name
, sizeof(tmp_name
), "%lx-%lx",
1199 (unsigned long)obj_buf
,
1200 (unsigned long)obj_buf_sz
);
1201 tmp_name
[sizeof(tmp_name
) - 1] = '\0';
1204 pr_debug("loading object '%s' from buffer\n",
1207 return __bpf_object__open(name
, obj_buf
, obj_buf_sz
);
1210 int bpf_object__unload(struct bpf_object
*obj
)
1217 for (i
= 0; i
< obj
->nr_maps
; i
++)
1218 zclose(obj
->maps
[i
].fd
);
1220 for (i
= 0; i
< obj
->nr_programs
; i
++)
1221 bpf_program__unload(&obj
->programs
[i
]);
1226 int bpf_object__load(struct bpf_object
*obj
)
1234 pr_warning("object should not be loaded twice\n");
1240 CHECK_ERR(bpf_object__create_maps(obj
), err
, out
);
1241 CHECK_ERR(bpf_object__relocate(obj
), err
, out
);
1242 CHECK_ERR(bpf_object__load_progs(obj
), err
, out
);
1246 bpf_object__unload(obj
);
1247 pr_warning("failed to load object '%s'\n", obj
->path
);
1251 static int check_path(const char *path
)
1253 struct statfs st_fs
;
1260 dname
= strdup(path
);
1264 dir
= dirname(dname
);
1265 if (statfs(dir
, &st_fs
)) {
1266 pr_warning("failed to statfs %s: %s\n", dir
, strerror(errno
));
1271 if (!err
&& st_fs
.f_type
!= BPF_FS_MAGIC
) {
1272 pr_warning("specified path %s is not on BPF FS\n", path
);
1279 int bpf_program__pin_instance(struct bpf_program
*prog
, const char *path
,
1284 err
= check_path(path
);
1289 pr_warning("invalid program pointer\n");
1293 if (instance
< 0 || instance
>= prog
->instances
.nr
) {
1294 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1295 instance
, prog
->section_name
, prog
->instances
.nr
);
1299 if (bpf_obj_pin(prog
->instances
.fds
[instance
], path
)) {
1300 pr_warning("failed to pin program: %s\n", strerror(errno
));
1303 pr_debug("pinned program '%s'\n", path
);
1308 static int make_dir(const char *path
)
1312 if (mkdir(path
, 0700) && errno
!= EEXIST
)
1316 pr_warning("failed to mkdir %s: %s\n", path
, strerror(-err
));
1320 int bpf_program__pin(struct bpf_program
*prog
, const char *path
)
1324 err
= check_path(path
);
1329 pr_warning("invalid program pointer\n");
1333 if (prog
->instances
.nr
<= 0) {
1334 pr_warning("no instances of prog %s to pin\n",
1335 prog
->section_name
);
1339 err
= make_dir(path
);
1343 for (i
= 0; i
< prog
->instances
.nr
; i
++) {
1347 len
= snprintf(buf
, PATH_MAX
, "%s/%d", path
, i
);
1350 else if (len
>= PATH_MAX
)
1351 return -ENAMETOOLONG
;
1353 err
= bpf_program__pin_instance(prog
, buf
, i
);
1361 int bpf_map__pin(struct bpf_map
*map
, const char *path
)
1365 err
= check_path(path
);
1370 pr_warning("invalid map pointer\n");
1374 if (bpf_obj_pin(map
->fd
, path
)) {
1375 pr_warning("failed to pin map: %s\n", strerror(errno
));
1379 pr_debug("pinned map '%s'\n", path
);
1383 int bpf_object__pin(struct bpf_object
*obj
, const char *path
)
1385 struct bpf_program
*prog
;
1386 struct bpf_map
*map
;
1393 pr_warning("object not yet loaded; load it first\n");
1397 err
= make_dir(path
);
1401 bpf_map__for_each(map
, obj
) {
1405 len
= snprintf(buf
, PATH_MAX
, "%s/%s", path
,
1406 bpf_map__name(map
));
1409 else if (len
>= PATH_MAX
)
1410 return -ENAMETOOLONG
;
1412 err
= bpf_map__pin(map
, buf
);
1417 bpf_object__for_each_program(prog
, obj
) {
1421 len
= snprintf(buf
, PATH_MAX
, "%s/%s", path
,
1422 prog
->section_name
);
1425 else if (len
>= PATH_MAX
)
1426 return -ENAMETOOLONG
;
1428 err
= bpf_program__pin(prog
, buf
);
1436 void bpf_object__close(struct bpf_object
*obj
)
1443 if (obj
->clear_priv
)
1444 obj
->clear_priv(obj
, obj
->priv
);
1446 bpf_object__elf_finish(obj
);
1447 bpf_object__unload(obj
);
1449 for (i
= 0; i
< obj
->nr_maps
; i
++) {
1450 zfree(&obj
->maps
[i
].name
);
1451 if (obj
->maps
[i
].clear_priv
)
1452 obj
->maps
[i
].clear_priv(&obj
->maps
[i
],
1454 obj
->maps
[i
].priv
= NULL
;
1455 obj
->maps
[i
].clear_priv
= NULL
;
1460 if (obj
->programs
&& obj
->nr_programs
) {
1461 for (i
= 0; i
< obj
->nr_programs
; i
++)
1462 bpf_program__exit(&obj
->programs
[i
]);
1464 zfree(&obj
->programs
);
1466 list_del(&obj
->list
);
1471 bpf_object__next(struct bpf_object
*prev
)
1473 struct bpf_object
*next
;
1476 next
= list_first_entry(&bpf_objects_list
,
1480 next
= list_next_entry(prev
, list
);
1482 /* Empty list is noticed here so don't need checking on entry. */
1483 if (&next
->list
== &bpf_objects_list
)
1489 const char *bpf_object__name(struct bpf_object
*obj
)
1491 return obj
? obj
->path
: ERR_PTR(-EINVAL
);
1494 unsigned int bpf_object__kversion(struct bpf_object
*obj
)
1496 return obj
? obj
->kern_version
: 0;
1499 int bpf_object__set_priv(struct bpf_object
*obj
, void *priv
,
1500 bpf_object_clear_priv_t clear_priv
)
1502 if (obj
->priv
&& obj
->clear_priv
)
1503 obj
->clear_priv(obj
, obj
->priv
);
1506 obj
->clear_priv
= clear_priv
;
1510 void *bpf_object__priv(struct bpf_object
*obj
)
1512 return obj
? obj
->priv
: ERR_PTR(-EINVAL
);
1515 struct bpf_program
*
1516 bpf_program__next(struct bpf_program
*prev
, struct bpf_object
*obj
)
1524 return &obj
->programs
[0];
1526 if (prev
->obj
!= obj
) {
1527 pr_warning("error: program handler doesn't match object\n");
1531 idx
= (prev
- obj
->programs
) + 1;
1532 if (idx
>= obj
->nr_programs
)
1534 return &obj
->programs
[idx
];
1537 int bpf_program__set_priv(struct bpf_program
*prog
, void *priv
,
1538 bpf_program_clear_priv_t clear_priv
)
1540 if (prog
->priv
&& prog
->clear_priv
)
1541 prog
->clear_priv(prog
, prog
->priv
);
1544 prog
->clear_priv
= clear_priv
;
1548 void *bpf_program__priv(struct bpf_program
*prog
)
1550 return prog
? prog
->priv
: ERR_PTR(-EINVAL
);
1553 const char *bpf_program__title(struct bpf_program
*prog
, bool needs_copy
)
1557 title
= prog
->section_name
;
1559 title
= strdup(title
);
1561 pr_warning("failed to strdup program title\n");
1562 return ERR_PTR(-ENOMEM
);
1569 int bpf_program__fd(struct bpf_program
*prog
)
1571 return bpf_program__nth_fd(prog
, 0);
1574 int bpf_program__set_prep(struct bpf_program
*prog
, int nr_instances
,
1575 bpf_program_prep_t prep
)
1579 if (nr_instances
<= 0 || !prep
)
1582 if (prog
->instances
.nr
> 0 || prog
->instances
.fds
) {
1583 pr_warning("Can't set pre-processor after loading\n");
1587 instances_fds
= malloc(sizeof(int) * nr_instances
);
1588 if (!instances_fds
) {
1589 pr_warning("alloc memory failed for fds\n");
1593 /* fill all fd with -1 */
1594 memset(instances_fds
, -1, sizeof(int) * nr_instances
);
1596 prog
->instances
.nr
= nr_instances
;
1597 prog
->instances
.fds
= instances_fds
;
1598 prog
->preprocessor
= prep
;
1602 int bpf_program__nth_fd(struct bpf_program
*prog
, int n
)
1606 if (n
>= prog
->instances
.nr
|| n
< 0) {
1607 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
1608 n
, prog
->section_name
, prog
->instances
.nr
);
1612 fd
= prog
->instances
.fds
[n
];
1614 pr_warning("%dth instance of program '%s' is invalid\n",
1615 n
, prog
->section_name
);
1622 void bpf_program__set_type(struct bpf_program
*prog
, enum bpf_prog_type type
)
1627 static bool bpf_program__is_type(struct bpf_program
*prog
,
1628 enum bpf_prog_type type
)
1630 return prog
? (prog
->type
== type
) : false;
1633 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \
1634 int bpf_program__set_##NAME(struct bpf_program *prog) \
1638 bpf_program__set_type(prog, TYPE); \
1642 bool bpf_program__is_##NAME(struct bpf_program *prog) \
1644 return bpf_program__is_type(prog, TYPE); \
1647 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
1648 BPF_PROG_TYPE_FNS(kprobe
, BPF_PROG_TYPE_KPROBE
);
1649 BPF_PROG_TYPE_FNS(sched_cls
, BPF_PROG_TYPE_SCHED_CLS
);
1650 BPF_PROG_TYPE_FNS(sched_act
, BPF_PROG_TYPE_SCHED_ACT
);
1651 BPF_PROG_TYPE_FNS(tracepoint
, BPF_PROG_TYPE_TRACEPOINT
);
1652 BPF_PROG_TYPE_FNS(xdp
, BPF_PROG_TYPE_XDP
);
1653 BPF_PROG_TYPE_FNS(perf_event
, BPF_PROG_TYPE_PERF_EVENT
);
1655 int bpf_map__fd(struct bpf_map
*map
)
1657 return map
? map
->fd
: -EINVAL
;
1660 const struct bpf_map_def
*bpf_map__def(struct bpf_map
*map
)
1662 return map
? &map
->def
: ERR_PTR(-EINVAL
);
1665 const char *bpf_map__name(struct bpf_map
*map
)
1667 return map
? map
->name
: NULL
;
1670 int bpf_map__set_priv(struct bpf_map
*map
, void *priv
,
1671 bpf_map_clear_priv_t clear_priv
)
1677 if (map
->clear_priv
)
1678 map
->clear_priv(map
, map
->priv
);
1682 map
->clear_priv
= clear_priv
;
1686 void *bpf_map__priv(struct bpf_map
*map
)
1688 return map
? map
->priv
: ERR_PTR(-EINVAL
);
1692 bpf_map__next(struct bpf_map
*prev
, struct bpf_object
*obj
)
1695 struct bpf_map
*s
, *e
;
1697 if (!obj
|| !obj
->maps
)
1701 e
= obj
->maps
+ obj
->nr_maps
;
1706 if ((prev
< s
) || (prev
>= e
)) {
1707 pr_warning("error in %s: map handler doesn't belong to object\n",
1712 idx
= (prev
- obj
->maps
) + 1;
1713 if (idx
>= obj
->nr_maps
)
1715 return &obj
->maps
[idx
];
1719 bpf_object__find_map_by_name(struct bpf_object
*obj
, const char *name
)
1721 struct bpf_map
*pos
;
1723 bpf_map__for_each(pos
, obj
) {
1724 if (pos
->name
&& !strcmp(pos
->name
, name
))
1731 bpf_object__find_map_by_offset(struct bpf_object
*obj
, size_t offset
)
1735 for (i
= 0; i
< obj
->nr_maps
; i
++) {
1736 if (obj
->maps
[i
].offset
== offset
)
1737 return &obj
->maps
[i
];
1739 return ERR_PTR(-ENOENT
);
1742 long libbpf_get_error(const void *ptr
)
1745 return PTR_ERR(ptr
);