]>
Commit | Line | Data |
---|---|---|
1bc38b8f | 1 | // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) |
6061a3d6 | 2 | |
1b76c13e WN |
3 | /* |
4 | * Common eBPF ELF object loading operations. | |
5 | * | |
6 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
7 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
8 | * Copyright (C) 2015 Huawei Inc. | |
f367540c | 9 | * Copyright (C) 2017 Nicira, Inc. |
d859900c | 10 | * Copyright (C) 2019 Isovalent, Inc. |
1b76c13e WN |
11 | */ |
12 | ||
b4269954 | 13 | #ifndef _GNU_SOURCE |
531b014e | 14 | #define _GNU_SOURCE |
b4269954 | 15 | #endif |
1b76c13e | 16 | #include <stdlib.h> |
b3f59d66 WN |
17 | #include <stdio.h> |
18 | #include <stdarg.h> | |
f367540c | 19 | #include <libgen.h> |
34090915 | 20 | #include <inttypes.h> |
b3f59d66 | 21 | #include <string.h> |
1b76c13e | 22 | #include <unistd.h> |
cdb2f920 | 23 | #include <endian.h> |
1a5e3fb1 WN |
24 | #include <fcntl.h> |
25 | #include <errno.h> | |
1b76c13e | 26 | #include <asm/unistd.h> |
e28ff1a8 | 27 | #include <linux/err.h> |
cb1e5e96 | 28 | #include <linux/kernel.h> |
1b76c13e | 29 | #include <linux/bpf.h> |
38d5d3b3 | 30 | #include <linux/btf.h> |
47eff617 | 31 | #include <linux/filter.h> |
9a208eff | 32 | #include <linux/list.h> |
f367540c | 33 | #include <linux/limits.h> |
438363c0 | 34 | #include <linux/perf_event.h> |
a64af0ef | 35 | #include <linux/ring_buffer.h> |
fb84b822 | 36 | #include <sys/epoll.h> |
63f2f5ee | 37 | #include <sys/ioctl.h> |
fb84b822 | 38 | #include <sys/mman.h> |
f367540c JS |
39 | #include <sys/stat.h> |
40 | #include <sys/types.h> | |
41 | #include <sys/vfs.h> | |
ddc7c304 | 42 | #include <sys/utsname.h> |
531b014e | 43 | #include <tools/libc_compat.h> |
1a5e3fb1 WN |
44 | #include <libelf.h> |
45 | #include <gelf.h> | |
1b76c13e WN |
46 | |
47 | #include "libbpf.h" | |
52d3352e | 48 | #include "bpf.h" |
8a138aed | 49 | #include "btf.h" |
6d41907c | 50 | #include "str_error.h" |
d7c4b398 | 51 | #include "libbpf_internal.h" |
ddc7c304 | 52 | #include "hashmap.h" |
b3f59d66 | 53 | |
9b16137a WN |
54 | #ifndef EM_BPF |
55 | #define EM_BPF 247 | |
56 | #endif | |
57 | ||
f367540c JS |
58 | #ifndef BPF_FS_MAGIC |
59 | #define BPF_FS_MAGIC 0xcafe4a11 | |
60 | #endif | |
61 | ||
ff466b58 AI |
62 | /* vsprintf() in __base_pr() uses nonliteral format string. It may break |
63 | * compilation if user enables corresponding warning. Disable it explicitly. | |
64 | */ | |
65 | #pragma GCC diagnostic ignored "-Wformat-nonliteral" | |
66 | ||
b3f59d66 WN |
67 | #define __printf(a, b) __attribute__((format(printf, a, b))) |
68 | ||
a8a1f7d0 SF |
69 | static int __base_pr(enum libbpf_print_level level, const char *format, |
70 | va_list args) | |
b3f59d66 | 71 | { |
6f1ae8b6 YS |
72 | if (level == LIBBPF_DEBUG) |
73 | return 0; | |
74 | ||
a8a1f7d0 | 75 | return vfprintf(stderr, format, args); |
b3f59d66 WN |
76 | } |
77 | ||
a8a1f7d0 | 78 | static libbpf_print_fn_t __libbpf_pr = __base_pr; |
b3f59d66 | 79 | |
e87fd8ba | 80 | libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) |
b3f59d66 | 81 | { |
e87fd8ba AN |
82 | libbpf_print_fn_t old_print_fn = __libbpf_pr; |
83 | ||
6f1ae8b6 | 84 | __libbpf_pr = fn; |
e87fd8ba | 85 | return old_print_fn; |
b3f59d66 | 86 | } |
1a5e3fb1 | 87 | |
8461ef8b YS |
88 | __printf(2, 3) |
89 | void libbpf_print(enum libbpf_print_level level, const char *format, ...) | |
90 | { | |
91 | va_list args; | |
92 | ||
6f1ae8b6 YS |
93 | if (!__libbpf_pr) |
94 | return; | |
95 | ||
8461ef8b | 96 | va_start(args, format); |
6f1ae8b6 | 97 | __libbpf_pr(level, format, args); |
8461ef8b YS |
98 | va_end(args); |
99 | } | |
100 | ||
6371ca3b WN |
101 | #define STRERR_BUFSIZE 128 |
102 | ||
6371ca3b WN |
103 | #define CHECK_ERR(action, err, out) do { \ |
104 | err = action; \ | |
105 | if (err) \ | |
106 | goto out; \ | |
107 | } while(0) | |
108 | ||
109 | ||
1a5e3fb1 WN |
110 | /* Copied from tools/perf/util/util.h */ |
111 | #ifndef zfree | |
112 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
113 | #endif | |
114 | ||
115 | #ifndef zclose | |
116 | # define zclose(fd) ({ \ | |
117 | int ___err = 0; \ | |
118 | if ((fd) >= 0) \ | |
119 | ___err = close((fd)); \ | |
120 | fd = -1; \ | |
121 | ___err; }) | |
122 | #endif | |
123 | ||
124 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
125 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
126 | #else | |
127 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
128 | #endif | |
129 | ||
34be1646 SL |
130 | static inline __u64 ptr_to_u64(const void *ptr) |
131 | { | |
132 | return (__u64) (unsigned long) ptr; | |
133 | } | |
134 | ||
47eff617 SF |
135 | struct bpf_capabilities { |
136 | /* v4.14: kernel support for program & map names. */ | |
137 | __u32 name:1; | |
8837fe5d DB |
138 | /* v5.2: kernel support for global data sections. */ |
139 | __u32 global_data:1; | |
d7c4b398 AN |
140 | /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ |
141 | __u32 btf_func:1; | |
142 | /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ | |
143 | __u32 btf_datasec:1; | |
47eff617 SF |
144 | }; |
145 | ||
a5b8bd47 WN |
146 | /* |
147 | * bpf_prog should be a better name but it has been used in | |
148 | * linux/filter.h. | |
149 | */ | |
150 | struct bpf_program { | |
151 | /* Index in elf obj file, for relocation use. */ | |
152 | int idx; | |
88cda1c9 | 153 | char *name; |
f0307a7e | 154 | int prog_ifindex; |
a5b8bd47 | 155 | char *section_name; |
33a2c75c SF |
156 | /* section_name with / replaced by _; makes recursive pinning |
157 | * in bpf_object__pin_programs easier | |
158 | */ | |
159 | char *pin_name; | |
a5b8bd47 | 160 | struct bpf_insn *insns; |
48cca7e4 | 161 | size_t insns_cnt, main_prog_cnt; |
5f44e4c8 | 162 | enum bpf_prog_type type; |
34090915 | 163 | |
48cca7e4 AS |
164 | struct reloc_desc { |
165 | enum { | |
166 | RELO_LD64, | |
167 | RELO_CALL, | |
d859900c | 168 | RELO_DATA, |
48cca7e4 | 169 | } type; |
34090915 | 170 | int insn_idx; |
48cca7e4 AS |
171 | union { |
172 | int map_idx; | |
173 | int text_off; | |
174 | }; | |
34090915 WN |
175 | } *reloc_desc; |
176 | int nr_reloc; | |
da11b417 | 177 | int log_level; |
55cffde2 | 178 | |
b580563e WN |
179 | struct { |
180 | int nr; | |
181 | int *fds; | |
182 | } instances; | |
183 | bpf_program_prep_t preprocessor; | |
aa9b1ac3 WN |
184 | |
185 | struct bpf_object *obj; | |
186 | void *priv; | |
187 | bpf_program_clear_priv_t clear_priv; | |
d7be143b AI |
188 | |
189 | enum bpf_attach_type expected_attach_type; | |
2993e051 YS |
190 | void *func_info; |
191 | __u32 func_info_rec_size; | |
f0187f0b | 192 | __u32 func_info_cnt; |
47eff617 SF |
193 | |
194 | struct bpf_capabilities *caps; | |
3d650141 MKL |
195 | |
196 | void *line_info; | |
197 | __u32 line_info_rec_size; | |
198 | __u32 line_info_cnt; | |
04656198 | 199 | __u32 prog_flags; |
a5b8bd47 WN |
200 | }; |
201 | ||
d859900c DB |
202 | enum libbpf_map_type { |
203 | LIBBPF_MAP_UNSPEC, | |
204 | LIBBPF_MAP_DATA, | |
205 | LIBBPF_MAP_BSS, | |
206 | LIBBPF_MAP_RODATA, | |
207 | }; | |
208 | ||
209 | static const char * const libbpf_type_to_btf_name[] = { | |
210 | [LIBBPF_MAP_DATA] = ".data", | |
211 | [LIBBPF_MAP_BSS] = ".bss", | |
212 | [LIBBPF_MAP_RODATA] = ".rodata", | |
213 | }; | |
214 | ||
9d759a9b WN |
215 | struct bpf_map { |
216 | int fd; | |
561bbcca | 217 | char *name; |
db48814b AN |
218 | int sec_idx; |
219 | size_t sec_offset; | |
f0307a7e | 220 | int map_ifindex; |
addb9fc9 | 221 | int inner_map_fd; |
9d759a9b | 222 | struct bpf_map_def def; |
5b891af7 MKL |
223 | __u32 btf_key_type_id; |
224 | __u32 btf_value_type_id; | |
9d759a9b WN |
225 | void *priv; |
226 | bpf_map_clear_priv_t clear_priv; | |
d859900c DB |
227 | enum libbpf_map_type libbpf_type; |
228 | }; | |
229 | ||
230 | struct bpf_secdata { | |
231 | void *rodata; | |
232 | void *data; | |
9d759a9b WN |
233 | }; |
234 | ||
9a208eff WN |
235 | static LIST_HEAD(bpf_objects_list); |
236 | ||
1a5e3fb1 | 237 | struct bpf_object { |
d859900c | 238 | char name[BPF_OBJ_NAME_LEN]; |
cb1e5e96 | 239 | char license[64]; |
438363c0 | 240 | __u32 kern_version; |
0b3d1efa | 241 | |
a5b8bd47 WN |
242 | struct bpf_program *programs; |
243 | size_t nr_programs; | |
9d759a9b WN |
244 | struct bpf_map *maps; |
245 | size_t nr_maps; | |
bf829271 | 246 | size_t maps_cap; |
d859900c | 247 | struct bpf_secdata sections; |
9d759a9b | 248 | |
52d3352e | 249 | bool loaded; |
9a94f277 | 250 | bool has_pseudo_calls; |
a5b8bd47 | 251 | |
1a5e3fb1 WN |
252 | /* |
253 | * Information when doing elf related work. Only valid if fd | |
254 | * is valid. | |
255 | */ | |
256 | struct { | |
257 | int fd; | |
6c956392 WN |
258 | void *obj_buf; |
259 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
260 | Elf *elf; |
261 | GElf_Ehdr ehdr; | |
bec7d68c | 262 | Elf_Data *symbols; |
d859900c DB |
263 | Elf_Data *data; |
264 | Elf_Data *rodata; | |
265 | Elf_Data *bss; | |
77ba9a5b | 266 | size_t strtabidx; |
b62f06e8 WN |
267 | struct { |
268 | GElf_Shdr shdr; | |
269 | Elf_Data *data; | |
270 | } *reloc; | |
271 | int nr_reloc; | |
666810e8 | 272 | int maps_shndx; |
abd29c93 | 273 | int btf_maps_shndx; |
48cca7e4 | 274 | int text_shndx; |
d859900c DB |
275 | int data_shndx; |
276 | int rodata_shndx; | |
277 | int bss_shndx; | |
1a5e3fb1 | 278 | } efile; |
9a208eff WN |
279 | /* |
280 | * All loaded bpf_object is linked in a list, which is | |
281 | * hidden to caller. bpf_objects__<func> handlers deal with | |
282 | * all objects. | |
283 | */ | |
284 | struct list_head list; | |
10931d24 | 285 | |
8a138aed | 286 | struct btf *btf; |
2993e051 | 287 | struct btf_ext *btf_ext; |
8a138aed | 288 | |
10931d24 WN |
289 | void *priv; |
290 | bpf_object_clear_priv_t clear_priv; | |
291 | ||
47eff617 SF |
292 | struct bpf_capabilities caps; |
293 | ||
1a5e3fb1 WN |
294 | char path[]; |
295 | }; | |
296 | #define obj_elf_valid(o) ((o)->efile.elf) | |
297 | ||
29cd77f4 | 298 | void bpf_program__unload(struct bpf_program *prog) |
55cffde2 | 299 | { |
b580563e WN |
300 | int i; |
301 | ||
55cffde2 WN |
302 | if (!prog) |
303 | return; | |
304 | ||
b580563e WN |
305 | /* |
306 | * If the object is opened but the program was never loaded, | |
307 | * it is possible that prog->instances.nr == -1. | |
308 | */ | |
309 | if (prog->instances.nr > 0) { | |
310 | for (i = 0; i < prog->instances.nr; i++) | |
311 | zclose(prog->instances.fds[i]); | |
312 | } else if (prog->instances.nr != -1) { | |
313 | pr_warning("Internal error: instances.nr is %d\n", | |
314 | prog->instances.nr); | |
315 | } | |
316 | ||
317 | prog->instances.nr = -1; | |
318 | zfree(&prog->instances.fds); | |
2993e051 | 319 | |
2993e051 | 320 | zfree(&prog->func_info); |
07a09d1b | 321 | zfree(&prog->line_info); |
55cffde2 WN |
322 | } |
323 | ||
a5b8bd47 WN |
324 | static void bpf_program__exit(struct bpf_program *prog) |
325 | { | |
326 | if (!prog) | |
327 | return; | |
328 | ||
aa9b1ac3 WN |
329 | if (prog->clear_priv) |
330 | prog->clear_priv(prog, prog->priv); | |
331 | ||
332 | prog->priv = NULL; | |
333 | prog->clear_priv = NULL; | |
334 | ||
55cffde2 | 335 | bpf_program__unload(prog); |
88cda1c9 | 336 | zfree(&prog->name); |
a5b8bd47 | 337 | zfree(&prog->section_name); |
33a2c75c | 338 | zfree(&prog->pin_name); |
a5b8bd47 | 339 | zfree(&prog->insns); |
34090915 WN |
340 | zfree(&prog->reloc_desc); |
341 | ||
342 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
343 | prog->insns_cnt = 0; |
344 | prog->idx = -1; | |
345 | } | |
346 | ||
33a2c75c SF |
347 | static char *__bpf_program__pin_name(struct bpf_program *prog) |
348 | { | |
349 | char *name, *p; | |
350 | ||
351 | name = p = strdup(prog->section_name); | |
352 | while ((p = strchr(p, '/'))) | |
353 | *p = '_'; | |
354 | ||
355 | return name; | |
356 | } | |
357 | ||
a5b8bd47 | 358 | static int |
88cda1c9 MKL |
359 | bpf_program__init(void *data, size_t size, char *section_name, int idx, |
360 | struct bpf_program *prog) | |
a5b8bd47 | 361 | { |
8ca990ce AN |
362 | const size_t bpf_insn_sz = sizeof(struct bpf_insn); |
363 | ||
364 | if (size == 0 || size % bpf_insn_sz) { | |
365 | pr_warning("corrupted section '%s', size: %zu\n", | |
366 | section_name, size); | |
a5b8bd47 WN |
367 | return -EINVAL; |
368 | } | |
369 | ||
1ad9cbb8 | 370 | memset(prog, 0, sizeof(*prog)); |
a5b8bd47 | 371 | |
88cda1c9 | 372 | prog->section_name = strdup(section_name); |
a5b8bd47 | 373 | if (!prog->section_name) { |
077c066a JDB |
374 | pr_warning("failed to alloc name for prog under section(%d) %s\n", |
375 | idx, section_name); | |
a5b8bd47 WN |
376 | goto errout; |
377 | } | |
378 | ||
33a2c75c SF |
379 | prog->pin_name = __bpf_program__pin_name(prog); |
380 | if (!prog->pin_name) { | |
381 | pr_warning("failed to alloc pin name for prog under section(%d) %s\n", | |
382 | idx, section_name); | |
383 | goto errout; | |
384 | } | |
385 | ||
a5b8bd47 WN |
386 | prog->insns = malloc(size); |
387 | if (!prog->insns) { | |
88cda1c9 MKL |
388 | pr_warning("failed to alloc insns for prog under section %s\n", |
389 | section_name); | |
a5b8bd47 WN |
390 | goto errout; |
391 | } | |
8ca990ce AN |
392 | prog->insns_cnt = size / bpf_insn_sz; |
393 | memcpy(prog->insns, data, size); | |
a5b8bd47 | 394 | prog->idx = idx; |
b580563e WN |
395 | prog->instances.fds = NULL; |
396 | prog->instances.nr = -1; | |
47ae7e3d | 397 | prog->type = BPF_PROG_TYPE_UNSPEC; |
a5b8bd47 WN |
398 | |
399 | return 0; | |
400 | errout: | |
401 | bpf_program__exit(prog); | |
402 | return -ENOMEM; | |
403 | } | |
404 | ||
405 | static int | |
406 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
88cda1c9 | 407 | char *section_name, int idx) |
a5b8bd47 WN |
408 | { |
409 | struct bpf_program prog, *progs; | |
410 | int nr_progs, err; | |
411 | ||
88cda1c9 | 412 | err = bpf_program__init(data, size, section_name, idx, &prog); |
a5b8bd47 WN |
413 | if (err) |
414 | return err; | |
415 | ||
47eff617 | 416 | prog.caps = &obj->caps; |
a5b8bd47 WN |
417 | progs = obj->programs; |
418 | nr_progs = obj->nr_programs; | |
419 | ||
531b014e | 420 | progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); |
a5b8bd47 WN |
421 | if (!progs) { |
422 | /* | |
423 | * In this case the original obj->programs | |
424 | * is still valid, so don't need special treat for | |
425 | * bpf_close_object(). | |
426 | */ | |
88cda1c9 MKL |
427 | pr_warning("failed to alloc a new program under section '%s'\n", |
428 | section_name); | |
a5b8bd47 WN |
429 | bpf_program__exit(&prog); |
430 | return -ENOMEM; | |
431 | } | |
432 | ||
433 | pr_debug("found program %s\n", prog.section_name); | |
434 | obj->programs = progs; | |
435 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 436 | prog.obj = obj; |
a5b8bd47 WN |
437 | progs[nr_progs] = prog; |
438 | return 0; | |
439 | } | |
440 | ||
88cda1c9 MKL |
441 | static int |
442 | bpf_object__init_prog_names(struct bpf_object *obj) | |
443 | { | |
444 | Elf_Data *symbols = obj->efile.symbols; | |
445 | struct bpf_program *prog; | |
446 | size_t pi, si; | |
447 | ||
448 | for (pi = 0; pi < obj->nr_programs; pi++) { | |
48cca7e4 | 449 | const char *name = NULL; |
88cda1c9 MKL |
450 | |
451 | prog = &obj->programs[pi]; | |
452 | ||
453 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; | |
454 | si++) { | |
455 | GElf_Sym sym; | |
456 | ||
457 | if (!gelf_getsym(symbols, si, &sym)) | |
458 | continue; | |
459 | if (sym.st_shndx != prog->idx) | |
460 | continue; | |
fe4d44b2 RG |
461 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) |
462 | continue; | |
88cda1c9 MKL |
463 | |
464 | name = elf_strptr(obj->efile.elf, | |
465 | obj->efile.strtabidx, | |
466 | sym.st_name); | |
467 | if (!name) { | |
468 | pr_warning("failed to get sym name string for prog %s\n", | |
469 | prog->section_name); | |
470 | return -LIBBPF_ERRNO__LIBELF; | |
471 | } | |
472 | } | |
473 | ||
9a94f277 JK |
474 | if (!name && prog->idx == obj->efile.text_shndx) |
475 | name = ".text"; | |
476 | ||
88cda1c9 MKL |
477 | if (!name) { |
478 | pr_warning("failed to find sym for prog %s\n", | |
479 | prog->section_name); | |
480 | return -EINVAL; | |
481 | } | |
9a94f277 | 482 | |
88cda1c9 MKL |
483 | prog->name = strdup(name); |
484 | if (!prog->name) { | |
485 | pr_warning("failed to allocate memory for prog sym %s\n", | |
486 | name); | |
487 | return -ENOMEM; | |
488 | } | |
489 | } | |
490 | ||
491 | return 0; | |
492 | } | |
493 | ||
6c956392 WN |
494 | static struct bpf_object *bpf_object__new(const char *path, |
495 | void *obj_buf, | |
496 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
497 | { |
498 | struct bpf_object *obj; | |
d859900c | 499 | char *end; |
1a5e3fb1 WN |
500 | |
501 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
502 | if (!obj) { | |
503 | pr_warning("alloc memory failed for %s\n", path); | |
6371ca3b | 504 | return ERR_PTR(-ENOMEM); |
1a5e3fb1 WN |
505 | } |
506 | ||
507 | strcpy(obj->path, path); | |
d859900c | 508 | /* Using basename() GNU version which doesn't modify arg. */ |
399dc65e | 509 | strncpy(obj->name, basename((void *)path), sizeof(obj->name) - 1); |
d859900c DB |
510 | end = strchr(obj->name, '.'); |
511 | if (end) | |
512 | *end = 0; | |
6c956392 | 513 | |
d859900c | 514 | obj->efile.fd = -1; |
6c956392 | 515 | /* |
76e1022b | 516 | * Caller of this function should also call |
6c956392 WN |
517 | * bpf_object__elf_finish() after data collection to return |
518 | * obj_buf to user. If not, we should duplicate the buffer to | |
519 | * avoid user freeing them before elf finish. | |
520 | */ | |
521 | obj->efile.obj_buf = obj_buf; | |
522 | obj->efile.obj_buf_sz = obj_buf_sz; | |
666810e8 | 523 | obj->efile.maps_shndx = -1; |
abd29c93 | 524 | obj->efile.btf_maps_shndx = -1; |
d859900c DB |
525 | obj->efile.data_shndx = -1; |
526 | obj->efile.rodata_shndx = -1; | |
527 | obj->efile.bss_shndx = -1; | |
6c956392 | 528 | |
52d3352e | 529 | obj->loaded = false; |
9a208eff WN |
530 | |
531 | INIT_LIST_HEAD(&obj->list); | |
532 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
533 | return obj; |
534 | } | |
535 | ||
536 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
537 | { | |
538 | if (!obj_elf_valid(obj)) | |
539 | return; | |
540 | ||
541 | if (obj->efile.elf) { | |
542 | elf_end(obj->efile.elf); | |
543 | obj->efile.elf = NULL; | |
544 | } | |
bec7d68c | 545 | obj->efile.symbols = NULL; |
d859900c DB |
546 | obj->efile.data = NULL; |
547 | obj->efile.rodata = NULL; | |
548 | obj->efile.bss = NULL; | |
b62f06e8 WN |
549 | |
550 | zfree(&obj->efile.reloc); | |
551 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 552 | zclose(obj->efile.fd); |
6c956392 WN |
553 | obj->efile.obj_buf = NULL; |
554 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
555 | } |
556 | ||
557 | static int bpf_object__elf_init(struct bpf_object *obj) | |
558 | { | |
559 | int err = 0; | |
560 | GElf_Ehdr *ep; | |
561 | ||
562 | if (obj_elf_valid(obj)) { | |
563 | pr_warning("elf init: internal error\n"); | |
6371ca3b | 564 | return -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
565 | } |
566 | ||
6c956392 WN |
567 | if (obj->efile.obj_buf_sz > 0) { |
568 | /* | |
569 | * obj_buf should have been validated by | |
570 | * bpf_object__open_buffer(). | |
571 | */ | |
572 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
573 | obj->efile.obj_buf_sz); | |
574 | } else { | |
575 | obj->efile.fd = open(obj->path, O_RDONLY); | |
576 | if (obj->efile.fd < 0) { | |
be5c5d4e | 577 | char errmsg[STRERR_BUFSIZE], *cp; |
1ce6a9fc | 578 | |
be5c5d4e AN |
579 | err = -errno; |
580 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
1ce6a9fc | 581 | pr_warning("failed to open %s: %s\n", obj->path, cp); |
be5c5d4e | 582 | return err; |
6c956392 WN |
583 | } |
584 | ||
585 | obj->efile.elf = elf_begin(obj->efile.fd, | |
76e1022b | 586 | LIBBPF_ELF_C_READ_MMAP, NULL); |
1a5e3fb1 WN |
587 | } |
588 | ||
1a5e3fb1 | 589 | if (!obj->efile.elf) { |
399dc65e | 590 | pr_warning("failed to open %s as ELF file\n", obj->path); |
6371ca3b | 591 | err = -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
592 | goto errout; |
593 | } | |
594 | ||
595 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
399dc65e | 596 | pr_warning("failed to get EHDR from %s\n", obj->path); |
6371ca3b | 597 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
598 | goto errout; |
599 | } | |
600 | ep = &obj->efile.ehdr; | |
601 | ||
9b16137a | 602 | /* Old LLVM set e_machine to EM_NONE */ |
76e1022b AN |
603 | if (ep->e_type != ET_REL || |
604 | (ep->e_machine && ep->e_machine != EM_BPF)) { | |
605 | pr_warning("%s is not an eBPF object file\n", obj->path); | |
6371ca3b | 606 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
607 | goto errout; |
608 | } | |
609 | ||
610 | return 0; | |
611 | errout: | |
612 | bpf_object__elf_finish(obj); | |
613 | return err; | |
614 | } | |
615 | ||
12ef5634 | 616 | static int bpf_object__check_endianness(struct bpf_object *obj) |
cc4228d5 | 617 | { |
cdb2f920 | 618 | #if __BYTE_ORDER == __LITTLE_ENDIAN |
12ef5634 AN |
619 | if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) |
620 | return 0; | |
cdb2f920 | 621 | #elif __BYTE_ORDER == __BIG_ENDIAN |
12ef5634 AN |
622 | if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) |
623 | return 0; | |
624 | #else | |
625 | # error "Unrecognized __BYTE_ORDER__" | |
626 | #endif | |
627 | pr_warning("endianness mismatch.\n"); | |
6371ca3b | 628 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
629 | } |
630 | ||
cb1e5e96 | 631 | static int |
399dc65e | 632 | bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) |
cb1e5e96 | 633 | { |
399dc65e | 634 | memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); |
cb1e5e96 WN |
635 | pr_debug("license of %s is %s\n", obj->path, obj->license); |
636 | return 0; | |
637 | } | |
638 | ||
639 | static int | |
399dc65e | 640 | bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) |
cb1e5e96 | 641 | { |
438363c0 | 642 | __u32 kver; |
cb1e5e96 WN |
643 | |
644 | if (size != sizeof(kver)) { | |
645 | pr_warning("invalid kver section in %s\n", obj->path); | |
6371ca3b | 646 | return -LIBBPF_ERRNO__FORMAT; |
cb1e5e96 WN |
647 | } |
648 | memcpy(&kver, data, sizeof(kver)); | |
649 | obj->kern_version = kver; | |
399dc65e | 650 | pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); |
cb1e5e96 WN |
651 | return 0; |
652 | } | |
653 | ||
4708bbda EL |
654 | static int compare_bpf_map(const void *_a, const void *_b) |
655 | { | |
656 | const struct bpf_map *a = _a; | |
657 | const struct bpf_map *b = _b; | |
9d759a9b | 658 | |
db48814b AN |
659 | if (a->sec_idx != b->sec_idx) |
660 | return a->sec_idx - b->sec_idx; | |
661 | return a->sec_offset - b->sec_offset; | |
0b3d1efa WN |
662 | } |
663 | ||
addb9fc9 NS |
664 | static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) |
665 | { | |
666 | if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || | |
667 | type == BPF_MAP_TYPE_HASH_OF_MAPS) | |
668 | return true; | |
669 | return false; | |
670 | } | |
671 | ||
1713d68b DB |
672 | static int bpf_object_search_section_size(const struct bpf_object *obj, |
673 | const char *name, size_t *d_size) | |
674 | { | |
675 | const GElf_Ehdr *ep = &obj->efile.ehdr; | |
676 | Elf *elf = obj->efile.elf; | |
677 | Elf_Scn *scn = NULL; | |
678 | int idx = 0; | |
679 | ||
680 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
681 | const char *sec_name; | |
682 | Elf_Data *data; | |
683 | GElf_Shdr sh; | |
684 | ||
685 | idx++; | |
686 | if (gelf_getshdr(scn, &sh) != &sh) { | |
687 | pr_warning("failed to get section(%d) header from %s\n", | |
688 | idx, obj->path); | |
689 | return -EIO; | |
690 | } | |
691 | ||
692 | sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
693 | if (!sec_name) { | |
694 | pr_warning("failed to get section(%d) name from %s\n", | |
695 | idx, obj->path); | |
696 | return -EIO; | |
697 | } | |
698 | ||
699 | if (strcmp(name, sec_name)) | |
700 | continue; | |
701 | ||
702 | data = elf_getdata(scn, 0); | |
703 | if (!data) { | |
704 | pr_warning("failed to get section(%d) data from %s(%s)\n", | |
705 | idx, name, obj->path); | |
706 | return -EIO; | |
707 | } | |
708 | ||
709 | *d_size = data->d_size; | |
710 | return 0; | |
711 | } | |
712 | ||
713 | return -ENOENT; | |
714 | } | |
715 | ||
716 | int bpf_object__section_size(const struct bpf_object *obj, const char *name, | |
717 | __u32 *size) | |
718 | { | |
719 | int ret = -ENOENT; | |
720 | size_t d_size; | |
721 | ||
722 | *size = 0; | |
723 | if (!name) { | |
724 | return -EINVAL; | |
725 | } else if (!strcmp(name, ".data")) { | |
726 | if (obj->efile.data) | |
727 | *size = obj->efile.data->d_size; | |
728 | } else if (!strcmp(name, ".bss")) { | |
729 | if (obj->efile.bss) | |
730 | *size = obj->efile.bss->d_size; | |
731 | } else if (!strcmp(name, ".rodata")) { | |
732 | if (obj->efile.rodata) | |
733 | *size = obj->efile.rodata->d_size; | |
734 | } else { | |
735 | ret = bpf_object_search_section_size(obj, name, &d_size); | |
736 | if (!ret) | |
737 | *size = d_size; | |
738 | } | |
739 | ||
740 | return *size ? 0 : ret; | |
741 | } | |
742 | ||
743 | int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, | |
744 | __u32 *off) | |
745 | { | |
746 | Elf_Data *symbols = obj->efile.symbols; | |
747 | const char *sname; | |
748 | size_t si; | |
749 | ||
750 | if (!name || !off) | |
751 | return -EINVAL; | |
752 | ||
753 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { | |
754 | GElf_Sym sym; | |
755 | ||
756 | if (!gelf_getsym(symbols, si, &sym)) | |
757 | continue; | |
758 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || | |
759 | GELF_ST_TYPE(sym.st_info) != STT_OBJECT) | |
760 | continue; | |
761 | ||
762 | sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
763 | sym.st_name); | |
764 | if (!sname) { | |
765 | pr_warning("failed to get sym name string for var %s\n", | |
766 | name); | |
767 | return -EIO; | |
768 | } | |
769 | if (strcmp(name, sname) == 0) { | |
770 | *off = sym.st_value; | |
771 | return 0; | |
772 | } | |
773 | } | |
774 | ||
775 | return -ENOENT; | |
776 | } | |
777 | ||
bf829271 | 778 | static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) |
d859900c | 779 | { |
bf829271 AN |
780 | struct bpf_map *new_maps; |
781 | size_t new_cap; | |
782 | int i; | |
783 | ||
784 | if (obj->nr_maps < obj->maps_cap) | |
785 | return &obj->maps[obj->nr_maps++]; | |
786 | ||
95064979 | 787 | new_cap = max((size_t)4, obj->maps_cap * 3 / 2); |
bf829271 AN |
788 | new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); |
789 | if (!new_maps) { | |
790 | pr_warning("alloc maps for object failed\n"); | |
791 | return ERR_PTR(-ENOMEM); | |
792 | } | |
793 | ||
794 | obj->maps_cap = new_cap; | |
795 | obj->maps = new_maps; | |
796 | ||
797 | /* zero out new maps */ | |
798 | memset(obj->maps + obj->nr_maps, 0, | |
799 | (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); | |
800 | /* | |
801 | * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) | |
802 | * when failure (zclose won't close negative fd)). | |
803 | */ | |
804 | for (i = obj->nr_maps; i < obj->maps_cap; i++) { | |
805 | obj->maps[i].fd = -1; | |
806 | obj->maps[i].inner_map_fd = -1; | |
807 | } | |
808 | ||
809 | return &obj->maps[obj->nr_maps++]; | |
d859900c DB |
810 | } |
811 | ||
812 | static int | |
bf829271 | 813 | bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, |
db48814b | 814 | int sec_idx, Elf_Data *data, void **data_buff) |
d859900c | 815 | { |
d859900c | 816 | char map_name[BPF_OBJ_NAME_LEN]; |
bf829271 AN |
817 | struct bpf_map_def *def; |
818 | struct bpf_map *map; | |
819 | ||
820 | map = bpf_object__add_map(obj); | |
821 | if (IS_ERR(map)) | |
822 | return PTR_ERR(map); | |
d859900c DB |
823 | |
824 | map->libbpf_type = type; | |
db48814b AN |
825 | map->sec_idx = sec_idx; |
826 | map->sec_offset = 0; | |
d859900c DB |
827 | snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, |
828 | libbpf_type_to_btf_name[type]); | |
829 | map->name = strdup(map_name); | |
830 | if (!map->name) { | |
831 | pr_warning("failed to alloc map name\n"); | |
832 | return -ENOMEM; | |
833 | } | |
db48814b AN |
834 | pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n", |
835 | map_name, map->sec_idx, map->sec_offset); | |
d859900c | 836 | |
bf829271 | 837 | def = &map->def; |
d859900c DB |
838 | def->type = BPF_MAP_TYPE_ARRAY; |
839 | def->key_size = sizeof(int); | |
840 | def->value_size = data->d_size; | |
841 | def->max_entries = 1; | |
399dc65e | 842 | def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0; |
d859900c DB |
843 | if (data_buff) { |
844 | *data_buff = malloc(data->d_size); | |
845 | if (!*data_buff) { | |
846 | zfree(&map->name); | |
847 | pr_warning("failed to alloc map content buffer\n"); | |
848 | return -ENOMEM; | |
849 | } | |
850 | memcpy(*data_buff, data->d_buf, data->d_size); | |
851 | } | |
852 | ||
e1d1dc46 | 853 | pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); |
d859900c DB |
854 | return 0; |
855 | } | |
856 | ||
bf829271 AN |
857 | static int bpf_object__init_global_data_maps(struct bpf_object *obj) |
858 | { | |
859 | int err; | |
860 | ||
861 | if (!obj->caps.global_data) | |
862 | return 0; | |
863 | /* | |
864 | * Populate obj->maps with libbpf internal maps. | |
865 | */ | |
866 | if (obj->efile.data_shndx >= 0) { | |
867 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, | |
db48814b | 868 | obj->efile.data_shndx, |
bf829271 AN |
869 | obj->efile.data, |
870 | &obj->sections.data); | |
871 | if (err) | |
872 | return err; | |
873 | } | |
874 | if (obj->efile.rodata_shndx >= 0) { | |
875 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, | |
db48814b | 876 | obj->efile.rodata_shndx, |
bf829271 AN |
877 | obj->efile.rodata, |
878 | &obj->sections.rodata); | |
879 | if (err) | |
880 | return err; | |
881 | } | |
882 | if (obj->efile.bss_shndx >= 0) { | |
883 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, | |
db48814b | 884 | obj->efile.bss_shndx, |
bf829271 AN |
885 | obj->efile.bss, NULL); |
886 | if (err) | |
887 | return err; | |
888 | } | |
889 | return 0; | |
890 | } | |
891 | ||
892 | static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) | |
561bbcca | 893 | { |
561bbcca | 894 | Elf_Data *symbols = obj->efile.symbols; |
bf829271 | 895 | int i, map_def_sz = 0, nr_maps = 0, nr_syms; |
d859900c | 896 | Elf_Data *data = NULL; |
bf829271 AN |
897 | Elf_Scn *scn; |
898 | ||
899 | if (obj->efile.maps_shndx < 0) | |
900 | return 0; | |
561bbcca | 901 | |
4708bbda EL |
902 | if (!symbols) |
903 | return -EINVAL; | |
904 | ||
bf829271 AN |
905 | scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); |
906 | if (scn) | |
907 | data = elf_getdata(scn, NULL); | |
908 | if (!scn || !data) { | |
909 | pr_warning("failed to get Elf_Data from map section %d\n", | |
910 | obj->efile.maps_shndx); | |
911 | return -EINVAL; | |
4708bbda | 912 | } |
561bbcca | 913 | |
4708bbda EL |
914 | /* |
915 | * Count number of maps. Each map has a name. | |
916 | * Array of maps is not supported: only the first element is | |
917 | * considered. | |
918 | * | |
919 | * TODO: Detect array of map and report error. | |
920 | */ | |
bf829271 AN |
921 | nr_syms = symbols->d_size / sizeof(GElf_Sym); |
922 | for (i = 0; i < nr_syms; i++) { | |
561bbcca | 923 | GElf_Sym sym; |
4708bbda EL |
924 | |
925 | if (!gelf_getsym(symbols, i, &sym)) | |
926 | continue; | |
927 | if (sym.st_shndx != obj->efile.maps_shndx) | |
928 | continue; | |
929 | nr_maps++; | |
930 | } | |
b13c5c14 | 931 | /* Assume equally sized map definitions */ |
bf829271 AN |
932 | pr_debug("maps in %s: %d maps in %zd bytes\n", |
933 | obj->path, nr_maps, data->d_size); | |
934 | ||
935 | map_def_sz = data->d_size / nr_maps; | |
936 | if (!data->d_size || (data->d_size % nr_maps) != 0) { | |
937 | pr_warning("unable to determine map definition size " | |
938 | "section %s, %d maps in %zd bytes\n", | |
939 | obj->path, nr_maps, data->d_size); | |
940 | return -EINVAL; | |
addb9fc9 | 941 | } |
4708bbda | 942 | |
bf829271 AN |
943 | /* Fill obj->maps using data in "maps" section. */ |
944 | for (i = 0; i < nr_syms; i++) { | |
4708bbda | 945 | GElf_Sym sym; |
561bbcca | 946 | const char *map_name; |
4708bbda | 947 | struct bpf_map_def *def; |
bf829271 | 948 | struct bpf_map *map; |
561bbcca WN |
949 | |
950 | if (!gelf_getsym(symbols, i, &sym)) | |
951 | continue; | |
666810e8 | 952 | if (sym.st_shndx != obj->efile.maps_shndx) |
561bbcca WN |
953 | continue; |
954 | ||
bf829271 AN |
955 | map = bpf_object__add_map(obj); |
956 | if (IS_ERR(map)) | |
957 | return PTR_ERR(map); | |
958 | ||
959 | map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
561bbcca | 960 | sym.st_name); |
c51829bb AN |
961 | if (!map_name) { |
962 | pr_warning("failed to get map #%d name sym string for obj %s\n", | |
bf829271 | 963 | i, obj->path); |
c51829bb AN |
964 | return -LIBBPF_ERRNO__FORMAT; |
965 | } | |
d859900c | 966 | |
bf829271 | 967 | map->libbpf_type = LIBBPF_MAP_UNSPEC; |
db48814b AN |
968 | map->sec_idx = sym.st_shndx; |
969 | map->sec_offset = sym.st_value; | |
970 | pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", | |
971 | map_name, map->sec_idx, map->sec_offset); | |
b13c5c14 | 972 | if (sym.st_value + map_def_sz > data->d_size) { |
4708bbda EL |
973 | pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", |
974 | obj->path, map_name); | |
975 | return -EINVAL; | |
561bbcca | 976 | } |
4708bbda | 977 | |
bf829271 AN |
978 | map->name = strdup(map_name); |
979 | if (!map->name) { | |
973170e6 WN |
980 | pr_warning("failed to alloc map name\n"); |
981 | return -ENOMEM; | |
982 | } | |
bf829271 | 983 | pr_debug("map %d is \"%s\"\n", i, map->name); |
4708bbda | 984 | def = (struct bpf_map_def *)(data->d_buf + sym.st_value); |
b13c5c14 CG |
985 | /* |
986 | * If the definition of the map in the object file fits in | |
987 | * bpf_map_def, copy it. Any extra fields in our version | |
988 | * of bpf_map_def will default to zero as a result of the | |
989 | * calloc above. | |
990 | */ | |
991 | if (map_def_sz <= sizeof(struct bpf_map_def)) { | |
bf829271 | 992 | memcpy(&map->def, def, map_def_sz); |
b13c5c14 CG |
993 | } else { |
994 | /* | |
995 | * Here the map structure being read is bigger than what | |
996 | * we expect, truncate if the excess bits are all zero. | |
997 | * If they are not zero, reject this map as | |
998 | * incompatible. | |
999 | */ | |
1000 | char *b; | |
1001 | for (b = ((char *)def) + sizeof(struct bpf_map_def); | |
1002 | b < ((char *)def) + map_def_sz; b++) { | |
1003 | if (*b != 0) { | |
1004 | pr_warning("maps section in %s: \"%s\" " | |
1005 | "has unrecognized, non-zero " | |
1006 | "options\n", | |
1007 | obj->path, map_name); | |
c034a177 JF |
1008 | if (strict) |
1009 | return -EINVAL; | |
b13c5c14 CG |
1010 | } |
1011 | } | |
bf829271 | 1012 | memcpy(&map->def, def, sizeof(struct bpf_map_def)); |
b13c5c14 | 1013 | } |
561bbcca | 1014 | } |
bf829271 AN |
1015 | return 0; |
1016 | } | |
4708bbda | 1017 | |
ddc7c304 AN |
1018 | static const struct btf_type * |
1019 | skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) | |
abd29c93 AN |
1020 | { |
1021 | const struct btf_type *t = btf__type_by_id(btf, id); | |
8837fe5d | 1022 | |
ddc7c304 AN |
1023 | if (res_id) |
1024 | *res_id = id; | |
1025 | ||
1026 | while (btf_is_mod(t) || btf_is_typedef(t)) { | |
1027 | if (res_id) | |
1028 | *res_id = t->type; | |
1029 | t = btf__type_by_id(btf, t->type); | |
abd29c93 | 1030 | } |
ddc7c304 AN |
1031 | |
1032 | return t; | |
abd29c93 AN |
1033 | } |
1034 | ||
ef99b02b AN |
1035 | /* |
1036 | * Fetch integer attribute of BTF map definition. Such attributes are | |
1037 | * represented using a pointer to an array, in which dimensionality of array | |
1038 | * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; | |
1039 | * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF | |
1040 | * type definition, while using only sizeof(void *) space in ELF data section. | |
1041 | */ | |
1042 | static bool get_map_field_int(const char *map_name, const struct btf *btf, | |
abd29c93 | 1043 | const struct btf_type *def, |
ef99b02b | 1044 | const struct btf_member *m, __u32 *res) { |
ddc7c304 | 1045 | const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); |
abd29c93 | 1046 | const char *name = btf__name_by_offset(btf, m->name_off); |
ef99b02b AN |
1047 | const struct btf_array *arr_info; |
1048 | const struct btf_type *arr_t; | |
abd29c93 | 1049 | |
b03bc685 | 1050 | if (!btf_is_ptr(t)) { |
ef99b02b | 1051 | pr_warning("map '%s': attr '%s': expected PTR, got %u.\n", |
b03bc685 | 1052 | map_name, name, btf_kind(t)); |
abd29c93 AN |
1053 | return false; |
1054 | } | |
ef99b02b AN |
1055 | |
1056 | arr_t = btf__type_by_id(btf, t->type); | |
1057 | if (!arr_t) { | |
1058 | pr_warning("map '%s': attr '%s': type [%u] not found.\n", | |
1059 | map_name, name, t->type); | |
abd29c93 AN |
1060 | return false; |
1061 | } | |
b03bc685 | 1062 | if (!btf_is_array(arr_t)) { |
ef99b02b | 1063 | pr_warning("map '%s': attr '%s': expected ARRAY, got %u.\n", |
b03bc685 | 1064 | map_name, name, btf_kind(arr_t)); |
abd29c93 AN |
1065 | return false; |
1066 | } | |
b03bc685 | 1067 | arr_info = btf_array(arr_t); |
ef99b02b | 1068 | *res = arr_info->nelems; |
abd29c93 AN |
1069 | return true; |
1070 | } | |
1071 | ||
1072 | static int bpf_object__init_user_btf_map(struct bpf_object *obj, | |
1073 | const struct btf_type *sec, | |
1074 | int var_idx, int sec_idx, | |
1075 | const Elf_Data *data, bool strict) | |
1076 | { | |
1077 | const struct btf_type *var, *def, *t; | |
1078 | const struct btf_var_secinfo *vi; | |
1079 | const struct btf_var *var_extra; | |
1080 | const struct btf_member *m; | |
abd29c93 AN |
1081 | const char *map_name; |
1082 | struct bpf_map *map; | |
1083 | int vlen, i; | |
1084 | ||
b03bc685 | 1085 | vi = btf_var_secinfos(sec) + var_idx; |
abd29c93 | 1086 | var = btf__type_by_id(obj->btf, vi->type); |
b03bc685 | 1087 | var_extra = btf_var(var); |
abd29c93 | 1088 | map_name = btf__name_by_offset(obj->btf, var->name_off); |
b03bc685 | 1089 | vlen = btf_vlen(var); |
abd29c93 AN |
1090 | |
1091 | if (map_name == NULL || map_name[0] == '\0') { | |
1092 | pr_warning("map #%d: empty name.\n", var_idx); | |
1093 | return -EINVAL; | |
1094 | } | |
1095 | if ((__u64)vi->offset + vi->size > data->d_size) { | |
1096 | pr_warning("map '%s' BTF data is corrupted.\n", map_name); | |
1097 | return -EINVAL; | |
1098 | } | |
b03bc685 | 1099 | if (!btf_is_var(var)) { |
abd29c93 | 1100 | pr_warning("map '%s': unexpected var kind %u.\n", |
b03bc685 | 1101 | map_name, btf_kind(var)); |
abd29c93 AN |
1102 | return -EINVAL; |
1103 | } | |
1104 | if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && | |
1105 | var_extra->linkage != BTF_VAR_STATIC) { | |
1106 | pr_warning("map '%s': unsupported var linkage %u.\n", | |
1107 | map_name, var_extra->linkage); | |
1108 | return -EOPNOTSUPP; | |
1109 | } | |
1110 | ||
ddc7c304 | 1111 | def = skip_mods_and_typedefs(obj->btf, var->type, NULL); |
b03bc685 | 1112 | if (!btf_is_struct(def)) { |
abd29c93 | 1113 | pr_warning("map '%s': unexpected def kind %u.\n", |
b03bc685 | 1114 | map_name, btf_kind(var)); |
abd29c93 AN |
1115 | return -EINVAL; |
1116 | } | |
1117 | if (def->size > vi->size) { | |
1118 | pr_warning("map '%s': invalid def size.\n", map_name); | |
1119 | return -EINVAL; | |
1120 | } | |
1121 | ||
1122 | map = bpf_object__add_map(obj); | |
1123 | if (IS_ERR(map)) | |
1124 | return PTR_ERR(map); | |
1125 | map->name = strdup(map_name); | |
1126 | if (!map->name) { | |
1127 | pr_warning("map '%s': failed to alloc map name.\n", map_name); | |
1128 | return -ENOMEM; | |
1129 | } | |
1130 | map->libbpf_type = LIBBPF_MAP_UNSPEC; | |
1131 | map->def.type = BPF_MAP_TYPE_UNSPEC; | |
1132 | map->sec_idx = sec_idx; | |
1133 | map->sec_offset = vi->offset; | |
1134 | pr_debug("map '%s': at sec_idx %d, offset %zu.\n", | |
1135 | map_name, map->sec_idx, map->sec_offset); | |
1136 | ||
b03bc685 AN |
1137 | vlen = btf_vlen(def); |
1138 | m = btf_members(def); | |
abd29c93 AN |
1139 | for (i = 0; i < vlen; i++, m++) { |
1140 | const char *name = btf__name_by_offset(obj->btf, m->name_off); | |
1141 | ||
1142 | if (!name) { | |
1143 | pr_warning("map '%s': invalid field #%d.\n", | |
1144 | map_name, i); | |
1145 | return -EINVAL; | |
1146 | } | |
1147 | if (strcmp(name, "type") == 0) { | |
1148 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1149 | &map->def.type)) |
abd29c93 AN |
1150 | return -EINVAL; |
1151 | pr_debug("map '%s': found type = %u.\n", | |
1152 | map_name, map->def.type); | |
1153 | } else if (strcmp(name, "max_entries") == 0) { | |
1154 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1155 | &map->def.max_entries)) |
abd29c93 AN |
1156 | return -EINVAL; |
1157 | pr_debug("map '%s': found max_entries = %u.\n", | |
1158 | map_name, map->def.max_entries); | |
1159 | } else if (strcmp(name, "map_flags") == 0) { | |
1160 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1161 | &map->def.map_flags)) |
abd29c93 AN |
1162 | return -EINVAL; |
1163 | pr_debug("map '%s': found map_flags = %u.\n", | |
1164 | map_name, map->def.map_flags); | |
1165 | } else if (strcmp(name, "key_size") == 0) { | |
1166 | __u32 sz; | |
1167 | ||
1168 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1169 | &sz)) |
abd29c93 AN |
1170 | return -EINVAL; |
1171 | pr_debug("map '%s': found key_size = %u.\n", | |
1172 | map_name, sz); | |
1173 | if (map->def.key_size && map->def.key_size != sz) { | |
900de4ac | 1174 | pr_warning("map '%s': conflicting key size %u != %u.\n", |
abd29c93 AN |
1175 | map_name, map->def.key_size, sz); |
1176 | return -EINVAL; | |
1177 | } | |
1178 | map->def.key_size = sz; | |
1179 | } else if (strcmp(name, "key") == 0) { | |
1180 | __s64 sz; | |
1181 | ||
1182 | t = btf__type_by_id(obj->btf, m->type); | |
1183 | if (!t) { | |
1184 | pr_warning("map '%s': key type [%d] not found.\n", | |
1185 | map_name, m->type); | |
1186 | return -EINVAL; | |
1187 | } | |
b03bc685 | 1188 | if (!btf_is_ptr(t)) { |
abd29c93 | 1189 | pr_warning("map '%s': key spec is not PTR: %u.\n", |
b03bc685 | 1190 | map_name, btf_kind(t)); |
abd29c93 AN |
1191 | return -EINVAL; |
1192 | } | |
1193 | sz = btf__resolve_size(obj->btf, t->type); | |
1194 | if (sz < 0) { | |
1195 | pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n", | |
1196 | map_name, t->type, sz); | |
1197 | return sz; | |
1198 | } | |
1199 | pr_debug("map '%s': found key [%u], sz = %lld.\n", | |
1200 | map_name, t->type, sz); | |
1201 | if (map->def.key_size && map->def.key_size != sz) { | |
900de4ac | 1202 | pr_warning("map '%s': conflicting key size %u != %lld.\n", |
abd29c93 AN |
1203 | map_name, map->def.key_size, sz); |
1204 | return -EINVAL; | |
1205 | } | |
1206 | map->def.key_size = sz; | |
1207 | map->btf_key_type_id = t->type; | |
1208 | } else if (strcmp(name, "value_size") == 0) { | |
1209 | __u32 sz; | |
1210 | ||
1211 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1212 | &sz)) |
abd29c93 AN |
1213 | return -EINVAL; |
1214 | pr_debug("map '%s': found value_size = %u.\n", | |
1215 | map_name, sz); | |
1216 | if (map->def.value_size && map->def.value_size != sz) { | |
900de4ac | 1217 | pr_warning("map '%s': conflicting value size %u != %u.\n", |
abd29c93 AN |
1218 | map_name, map->def.value_size, sz); |
1219 | return -EINVAL; | |
1220 | } | |
1221 | map->def.value_size = sz; | |
1222 | } else if (strcmp(name, "value") == 0) { | |
1223 | __s64 sz; | |
1224 | ||
1225 | t = btf__type_by_id(obj->btf, m->type); | |
1226 | if (!t) { | |
1227 | pr_warning("map '%s': value type [%d] not found.\n", | |
1228 | map_name, m->type); | |
1229 | return -EINVAL; | |
1230 | } | |
b03bc685 | 1231 | if (!btf_is_ptr(t)) { |
abd29c93 | 1232 | pr_warning("map '%s': value spec is not PTR: %u.\n", |
b03bc685 | 1233 | map_name, btf_kind(t)); |
abd29c93 AN |
1234 | return -EINVAL; |
1235 | } | |
1236 | sz = btf__resolve_size(obj->btf, t->type); | |
1237 | if (sz < 0) { | |
1238 | pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n", | |
1239 | map_name, t->type, sz); | |
1240 | return sz; | |
1241 | } | |
1242 | pr_debug("map '%s': found value [%u], sz = %lld.\n", | |
1243 | map_name, t->type, sz); | |
1244 | if (map->def.value_size && map->def.value_size != sz) { | |
900de4ac | 1245 | pr_warning("map '%s': conflicting value size %u != %lld.\n", |
abd29c93 AN |
1246 | map_name, map->def.value_size, sz); |
1247 | return -EINVAL; | |
1248 | } | |
1249 | map->def.value_size = sz; | |
1250 | map->btf_value_type_id = t->type; | |
1251 | } else { | |
1252 | if (strict) { | |
1253 | pr_warning("map '%s': unknown field '%s'.\n", | |
1254 | map_name, name); | |
1255 | return -ENOTSUP; | |
1256 | } | |
1257 | pr_debug("map '%s': ignoring unknown field '%s'.\n", | |
1258 | map_name, name); | |
1259 | } | |
1260 | } | |
1261 | ||
1262 | if (map->def.type == BPF_MAP_TYPE_UNSPEC) { | |
1263 | pr_warning("map '%s': map type isn't specified.\n", map_name); | |
1264 | return -EINVAL; | |
1265 | } | |
1266 | ||
1267 | return 0; | |
1268 | } | |
1269 | ||
1270 | static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) | |
1271 | { | |
1272 | const struct btf_type *sec = NULL; | |
1273 | int nr_types, i, vlen, err; | |
1274 | const struct btf_type *t; | |
1275 | const char *name; | |
1276 | Elf_Data *data; | |
1277 | Elf_Scn *scn; | |
1278 | ||
1279 | if (obj->efile.btf_maps_shndx < 0) | |
1280 | return 0; | |
1281 | ||
1282 | scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); | |
1283 | if (scn) | |
1284 | data = elf_getdata(scn, NULL); | |
1285 | if (!scn || !data) { | |
1286 | pr_warning("failed to get Elf_Data from map section %d (%s)\n", | |
1287 | obj->efile.maps_shndx, MAPS_ELF_SEC); | |
1288 | return -EINVAL; | |
1289 | } | |
1290 | ||
1291 | nr_types = btf__get_nr_types(obj->btf); | |
1292 | for (i = 1; i <= nr_types; i++) { | |
1293 | t = btf__type_by_id(obj->btf, i); | |
b03bc685 | 1294 | if (!btf_is_datasec(t)) |
abd29c93 AN |
1295 | continue; |
1296 | name = btf__name_by_offset(obj->btf, t->name_off); | |
1297 | if (strcmp(name, MAPS_ELF_SEC) == 0) { | |
1298 | sec = t; | |
1299 | break; | |
1300 | } | |
1301 | } | |
1302 | ||
1303 | if (!sec) { | |
1304 | pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC); | |
1305 | return -ENOENT; | |
1306 | } | |
1307 | ||
b03bc685 | 1308 | vlen = btf_vlen(sec); |
abd29c93 AN |
1309 | for (i = 0; i < vlen; i++) { |
1310 | err = bpf_object__init_user_btf_map(obj, sec, i, | |
1311 | obj->efile.btf_maps_shndx, | |
1312 | data, strict); | |
1313 | if (err) | |
1314 | return err; | |
1315 | } | |
1316 | ||
1317 | return 0; | |
1318 | } | |
1319 | ||
bf829271 AN |
1320 | static int bpf_object__init_maps(struct bpf_object *obj, int flags) |
1321 | { | |
1322 | bool strict = !(flags & MAPS_RELAX_COMPAT); | |
1323 | int err; | |
8837fe5d | 1324 | |
bf829271 AN |
1325 | err = bpf_object__init_user_maps(obj, strict); |
1326 | if (err) | |
1327 | return err; | |
1328 | ||
abd29c93 AN |
1329 | err = bpf_object__init_user_btf_maps(obj, strict); |
1330 | if (err) | |
1331 | return err; | |
1332 | ||
bf829271 AN |
1333 | err = bpf_object__init_global_data_maps(obj); |
1334 | if (err) | |
1335 | return err; | |
1336 | ||
1337 | if (obj->nr_maps) { | |
d859900c DB |
1338 | qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), |
1339 | compare_bpf_map); | |
bf829271 AN |
1340 | } |
1341 | return 0; | |
561bbcca WN |
1342 | } |
1343 | ||
e3d91b0c JDB |
1344 | static bool section_have_execinstr(struct bpf_object *obj, int idx) |
1345 | { | |
1346 | Elf_Scn *scn; | |
1347 | GElf_Shdr sh; | |
1348 | ||
1349 | scn = elf_getscn(obj->efile.elf, idx); | |
1350 | if (!scn) | |
1351 | return false; | |
1352 | ||
1353 | if (gelf_getshdr(scn, &sh) != &sh) | |
1354 | return false; | |
1355 | ||
1356 | if (sh.sh_flags & SHF_EXECINSTR) | |
1357 | return true; | |
1358 | ||
1359 | return false; | |
1360 | } | |
1361 | ||
d7c4b398 AN |
1362 | static void bpf_object__sanitize_btf(struct bpf_object *obj) |
1363 | { | |
1364 | bool has_datasec = obj->caps.btf_datasec; | |
1365 | bool has_func = obj->caps.btf_func; | |
1366 | struct btf *btf = obj->btf; | |
1367 | struct btf_type *t; | |
1368 | int i, j, vlen; | |
d7c4b398 AN |
1369 | |
1370 | if (!obj->btf || (has_func && has_datasec)) | |
1371 | return; | |
1372 | ||
1373 | for (i = 1; i <= btf__get_nr_types(btf); i++) { | |
1374 | t = (struct btf_type *)btf__type_by_id(btf, i); | |
d7c4b398 | 1375 | |
b03bc685 | 1376 | if (!has_datasec && btf_is_var(t)) { |
d7c4b398 AN |
1377 | /* replace VAR with INT */ |
1378 | t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); | |
1d4126c4 AN |
1379 | /* |
1380 | * using size = 1 is the safest choice, 4 will be too | |
1381 | * big and cause kernel BTF validation failure if | |
1382 | * original variable took less than 4 bytes | |
1383 | */ | |
1384 | t->size = 1; | |
708852dc | 1385 | *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); |
b03bc685 | 1386 | } else if (!has_datasec && btf_is_datasec(t)) { |
d7c4b398 | 1387 | /* replace DATASEC with STRUCT */ |
b03bc685 AN |
1388 | const struct btf_var_secinfo *v = btf_var_secinfos(t); |
1389 | struct btf_member *m = btf_members(t); | |
d7c4b398 AN |
1390 | struct btf_type *vt; |
1391 | char *name; | |
1392 | ||
1393 | name = (char *)btf__name_by_offset(btf, t->name_off); | |
1394 | while (*name) { | |
1395 | if (*name == '.') | |
1396 | *name = '_'; | |
1397 | name++; | |
1398 | } | |
1399 | ||
b03bc685 | 1400 | vlen = btf_vlen(t); |
d7c4b398 AN |
1401 | t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); |
1402 | for (j = 0; j < vlen; j++, v++, m++) { | |
1403 | /* order of field assignments is important */ | |
1404 | m->offset = v->offset * 8; | |
1405 | m->type = v->type; | |
1406 | /* preserve variable name as member name */ | |
1407 | vt = (void *)btf__type_by_id(btf, v->type); | |
1408 | m->name_off = vt->name_off; | |
1409 | } | |
b03bc685 | 1410 | } else if (!has_func && btf_is_func_proto(t)) { |
d7c4b398 | 1411 | /* replace FUNC_PROTO with ENUM */ |
b03bc685 | 1412 | vlen = btf_vlen(t); |
d7c4b398 AN |
1413 | t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); |
1414 | t->size = sizeof(__u32); /* kernel enforced */ | |
b03bc685 | 1415 | } else if (!has_func && btf_is_func(t)) { |
d7c4b398 AN |
1416 | /* replace FUNC with TYPEDEF */ |
1417 | t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); | |
1418 | } | |
1419 | } | |
1420 | } | |
1421 | ||
1422 | static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) | |
1423 | { | |
1424 | if (!obj->btf_ext) | |
1425 | return; | |
1426 | ||
1427 | if (!obj->caps.btf_func) { | |
1428 | btf_ext__free(obj->btf_ext); | |
1429 | obj->btf_ext = NULL; | |
1430 | } | |
1431 | } | |
1432 | ||
abd29c93 AN |
1433 | static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj) |
1434 | { | |
1435 | return obj->efile.btf_maps_shndx >= 0; | |
1436 | } | |
1437 | ||
063183bf | 1438 | static int bpf_object__init_btf(struct bpf_object *obj, |
9c6660d0 AN |
1439 | Elf_Data *btf_data, |
1440 | Elf_Data *btf_ext_data) | |
1441 | { | |
abd29c93 | 1442 | bool btf_required = bpf_object__is_btf_mandatory(obj); |
9c6660d0 AN |
1443 | int err = 0; |
1444 | ||
1445 | if (btf_data) { | |
1446 | obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); | |
1447 | if (IS_ERR(obj->btf)) { | |
1448 | pr_warning("Error loading ELF section %s: %d.\n", | |
1449 | BTF_ELF_SEC, err); | |
1450 | goto out; | |
1451 | } | |
1452 | err = btf__finalize_data(obj, obj->btf); | |
1453 | if (err) { | |
1454 | pr_warning("Error finalizing %s: %d.\n", | |
1455 | BTF_ELF_SEC, err); | |
1456 | goto out; | |
1457 | } | |
9c6660d0 AN |
1458 | } |
1459 | if (btf_ext_data) { | |
1460 | if (!obj->btf) { | |
1461 | pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", | |
1462 | BTF_EXT_ELF_SEC, BTF_ELF_SEC); | |
1463 | goto out; | |
1464 | } | |
1465 | obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, | |
1466 | btf_ext_data->d_size); | |
1467 | if (IS_ERR(obj->btf_ext)) { | |
1468 | pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", | |
1469 | BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); | |
1470 | obj->btf_ext = NULL; | |
1471 | goto out; | |
1472 | } | |
9c6660d0 AN |
1473 | } |
1474 | out: | |
1475 | if (err || IS_ERR(obj->btf)) { | |
abd29c93 AN |
1476 | if (btf_required) |
1477 | err = err ? : PTR_ERR(obj->btf); | |
1478 | else | |
1479 | err = 0; | |
9c6660d0 AN |
1480 | if (!IS_ERR_OR_NULL(obj->btf)) |
1481 | btf__free(obj->btf); | |
1482 | obj->btf = NULL; | |
1483 | } | |
abd29c93 AN |
1484 | if (btf_required && !obj->btf) { |
1485 | pr_warning("BTF is required, but is missing or corrupted.\n"); | |
1486 | return err == 0 ? -ENOENT : err; | |
1487 | } | |
9c6660d0 AN |
1488 | return 0; |
1489 | } | |
1490 | ||
063183bf AN |
1491 | static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) |
1492 | { | |
1493 | int err = 0; | |
1494 | ||
1495 | if (!obj->btf) | |
1496 | return 0; | |
1497 | ||
1498 | bpf_object__sanitize_btf(obj); | |
1499 | bpf_object__sanitize_btf_ext(obj); | |
1500 | ||
1501 | err = btf__load(obj->btf); | |
1502 | if (err) { | |
1503 | pr_warning("Error loading %s into kernel: %d.\n", | |
1504 | BTF_ELF_SEC, err); | |
1505 | btf__free(obj->btf); | |
1506 | obj->btf = NULL; | |
04efe591 AN |
1507 | /* btf_ext can't exist without btf, so free it as well */ |
1508 | if (obj->btf_ext) { | |
1509 | btf_ext__free(obj->btf_ext); | |
1510 | obj->btf_ext = NULL; | |
1511 | } | |
1512 | ||
abd29c93 AN |
1513 | if (bpf_object__is_btf_mandatory(obj)) |
1514 | return err; | |
063183bf AN |
1515 | } |
1516 | return 0; | |
1517 | } | |
1518 | ||
c034a177 | 1519 | static int bpf_object__elf_collect(struct bpf_object *obj, int flags) |
29603665 WN |
1520 | { |
1521 | Elf *elf = obj->efile.elf; | |
1522 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
f0187f0b | 1523 | Elf_Data *btf_ext_data = NULL; |
1713d68b | 1524 | Elf_Data *btf_data = NULL; |
29603665 | 1525 | Elf_Scn *scn = NULL; |
666810e8 | 1526 | int idx = 0, err = 0; |
29603665 WN |
1527 | |
1528 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
1529 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
399dc65e | 1530 | pr_warning("failed to get e_shstrndx from %s\n", obj->path); |
6371ca3b | 1531 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
1532 | } |
1533 | ||
1534 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
1535 | char *name; | |
1536 | GElf_Shdr sh; | |
1537 | Elf_Data *data; | |
1538 | ||
1539 | idx++; | |
1540 | if (gelf_getshdr(scn, &sh) != &sh) { | |
077c066a JDB |
1541 | pr_warning("failed to get section(%d) header from %s\n", |
1542 | idx, obj->path); | |
01b29d1d | 1543 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
1544 | } |
1545 | ||
1546 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
1547 | if (!name) { | |
077c066a JDB |
1548 | pr_warning("failed to get section(%d) name from %s\n", |
1549 | idx, obj->path); | |
01b29d1d | 1550 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
1551 | } |
1552 | ||
1553 | data = elf_getdata(scn, 0); | |
1554 | if (!data) { | |
077c066a JDB |
1555 | pr_warning("failed to get section(%d) data from %s(%s)\n", |
1556 | idx, name, obj->path); | |
01b29d1d | 1557 | return -LIBBPF_ERRNO__FORMAT; |
29603665 | 1558 | } |
077c066a JDB |
1559 | pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", |
1560 | idx, name, (unsigned long)data->d_size, | |
29603665 WN |
1561 | (int)sh.sh_link, (unsigned long)sh.sh_flags, |
1562 | (int)sh.sh_type); | |
cb1e5e96 | 1563 | |
1713d68b | 1564 | if (strcmp(name, "license") == 0) { |
cb1e5e96 WN |
1565 | err = bpf_object__init_license(obj, |
1566 | data->d_buf, | |
1567 | data->d_size); | |
01b29d1d AN |
1568 | if (err) |
1569 | return err; | |
1713d68b | 1570 | } else if (strcmp(name, "version") == 0) { |
cb1e5e96 WN |
1571 | err = bpf_object__init_kversion(obj, |
1572 | data->d_buf, | |
1573 | data->d_size); | |
01b29d1d AN |
1574 | if (err) |
1575 | return err; | |
1713d68b | 1576 | } else if (strcmp(name, "maps") == 0) { |
666810e8 | 1577 | obj->efile.maps_shndx = idx; |
abd29c93 AN |
1578 | } else if (strcmp(name, MAPS_ELF_SEC) == 0) { |
1579 | obj->efile.btf_maps_shndx = idx; | |
1713d68b DB |
1580 | } else if (strcmp(name, BTF_ELF_SEC) == 0) { |
1581 | btf_data = data; | |
2993e051 | 1582 | } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { |
f0187f0b | 1583 | btf_ext_data = data; |
8a138aed | 1584 | } else if (sh.sh_type == SHT_SYMTAB) { |
bec7d68c WN |
1585 | if (obj->efile.symbols) { |
1586 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
1587 | obj->path); | |
01b29d1d | 1588 | return -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 1589 | } |
01b29d1d AN |
1590 | obj->efile.symbols = data; |
1591 | obj->efile.strtabidx = sh.sh_link; | |
f8c7a4d4 JS |
1592 | } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { |
1593 | if (sh.sh_flags & SHF_EXECINSTR) { | |
1594 | if (strcmp(name, ".text") == 0) | |
1595 | obj->efile.text_shndx = idx; | |
1596 | err = bpf_object__add_program(obj, data->d_buf, | |
1597 | data->d_size, name, idx); | |
1598 | if (err) { | |
1599 | char errmsg[STRERR_BUFSIZE]; | |
1600 | char *cp = libbpf_strerror_r(-err, errmsg, | |
1601 | sizeof(errmsg)); | |
1602 | ||
1603 | pr_warning("failed to alloc program %s (%s): %s", | |
1604 | name, obj->path, cp); | |
01b29d1d | 1605 | return err; |
f8c7a4d4 | 1606 | } |
d859900c DB |
1607 | } else if (strcmp(name, ".data") == 0) { |
1608 | obj->efile.data = data; | |
1609 | obj->efile.data_shndx = idx; | |
1610 | } else if (strcmp(name, ".rodata") == 0) { | |
1611 | obj->efile.rodata = data; | |
1612 | obj->efile.rodata_shndx = idx; | |
1613 | } else { | |
1614 | pr_debug("skip section(%d) %s\n", idx, name); | |
a5b8bd47 | 1615 | } |
b62f06e8 | 1616 | } else if (sh.sh_type == SHT_REL) { |
01b29d1d | 1617 | int nr_reloc = obj->efile.nr_reloc; |
b62f06e8 | 1618 | void *reloc = obj->efile.reloc; |
e3d91b0c JDB |
1619 | int sec = sh.sh_info; /* points to other section */ |
1620 | ||
1621 | /* Only do relo for section with exec instructions */ | |
1622 | if (!section_have_execinstr(obj, sec)) { | |
1623 | pr_debug("skip relo %s(%d) for section(%d)\n", | |
1624 | name, idx, sec); | |
1625 | continue; | |
1626 | } | |
b62f06e8 | 1627 | |
01b29d1d | 1628 | reloc = reallocarray(reloc, nr_reloc + 1, |
531b014e | 1629 | sizeof(*obj->efile.reloc)); |
b62f06e8 WN |
1630 | if (!reloc) { |
1631 | pr_warning("realloc failed\n"); | |
01b29d1d AN |
1632 | return -ENOMEM; |
1633 | } | |
b62f06e8 | 1634 | |
01b29d1d AN |
1635 | obj->efile.reloc = reloc; |
1636 | obj->efile.nr_reloc++; | |
b62f06e8 | 1637 | |
01b29d1d AN |
1638 | obj->efile.reloc[nr_reloc].shdr = sh; |
1639 | obj->efile.reloc[nr_reloc].data = data; | |
d859900c DB |
1640 | } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { |
1641 | obj->efile.bss = data; | |
1642 | obj->efile.bss_shndx = idx; | |
077c066a JDB |
1643 | } else { |
1644 | pr_debug("skip section(%d) %s\n", idx, name); | |
bec7d68c | 1645 | } |
29603665 | 1646 | } |
561bbcca | 1647 | |
77ba9a5b WN |
1648 | if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { |
1649 | pr_warning("Corrupted ELF file: index of strtab invalid\n"); | |
f102154d | 1650 | return -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 1651 | } |
063183bf | 1652 | err = bpf_object__init_btf(obj, btf_data, btf_ext_data); |
bf829271 | 1653 | if (!err) |
c034a177 | 1654 | err = bpf_object__init_maps(obj, flags); |
063183bf AN |
1655 | if (!err) |
1656 | err = bpf_object__sanitize_and_load_btf(obj); | |
bf829271 AN |
1657 | if (!err) |
1658 | err = bpf_object__init_prog_names(obj); | |
29603665 WN |
1659 | return err; |
1660 | } | |
1661 | ||
34090915 WN |
1662 | static struct bpf_program * |
1663 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
1664 | { | |
1665 | struct bpf_program *prog; | |
1666 | size_t i; | |
1667 | ||
1668 | for (i = 0; i < obj->nr_programs; i++) { | |
1669 | prog = &obj->programs[i]; | |
1670 | if (prog->idx == idx) | |
1671 | return prog; | |
1672 | } | |
1673 | return NULL; | |
1674 | } | |
1675 | ||
6d4b198b | 1676 | struct bpf_program * |
a324aae3 AN |
1677 | bpf_object__find_program_by_title(const struct bpf_object *obj, |
1678 | const char *title) | |
6d4b198b JK |
1679 | { |
1680 | struct bpf_program *pos; | |
1681 | ||
1682 | bpf_object__for_each_program(pos, obj) { | |
1683 | if (pos->section_name && !strcmp(pos->section_name, title)) | |
1684 | return pos; | |
1685 | } | |
1686 | return NULL; | |
1687 | } | |
1688 | ||
d859900c DB |
1689 | static bool bpf_object__shndx_is_data(const struct bpf_object *obj, |
1690 | int shndx) | |
1691 | { | |
1692 | return shndx == obj->efile.data_shndx || | |
1693 | shndx == obj->efile.bss_shndx || | |
1694 | shndx == obj->efile.rodata_shndx; | |
1695 | } | |
1696 | ||
1697 | static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, | |
1698 | int shndx) | |
1699 | { | |
abd29c93 AN |
1700 | return shndx == obj->efile.maps_shndx || |
1701 | shndx == obj->efile.btf_maps_shndx; | |
d859900c DB |
1702 | } |
1703 | ||
1704 | static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, | |
1705 | int shndx) | |
1706 | { | |
1707 | return shndx == obj->efile.text_shndx || | |
1708 | bpf_object__shndx_is_maps(obj, shndx) || | |
1709 | bpf_object__shndx_is_data(obj, shndx); | |
1710 | } | |
1711 | ||
1712 | static enum libbpf_map_type | |
1713 | bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) | |
1714 | { | |
1715 | if (shndx == obj->efile.data_shndx) | |
1716 | return LIBBPF_MAP_DATA; | |
1717 | else if (shndx == obj->efile.bss_shndx) | |
1718 | return LIBBPF_MAP_BSS; | |
1719 | else if (shndx == obj->efile.rodata_shndx) | |
1720 | return LIBBPF_MAP_RODATA; | |
1721 | else | |
1722 | return LIBBPF_MAP_UNSPEC; | |
1723 | } | |
1724 | ||
34090915 | 1725 | static int |
48cca7e4 AS |
1726 | bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, |
1727 | Elf_Data *data, struct bpf_object *obj) | |
34090915 | 1728 | { |
48cca7e4 | 1729 | Elf_Data *symbols = obj->efile.symbols; |
48cca7e4 AS |
1730 | struct bpf_map *maps = obj->maps; |
1731 | size_t nr_maps = obj->nr_maps; | |
34090915 WN |
1732 | int i, nrels; |
1733 | ||
399dc65e | 1734 | pr_debug("collecting relocating info for: '%s'\n", prog->section_name); |
34090915 WN |
1735 | nrels = shdr->sh_size / shdr->sh_entsize; |
1736 | ||
1737 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
1738 | if (!prog->reloc_desc) { | |
1739 | pr_warning("failed to alloc memory in relocation\n"); | |
1740 | return -ENOMEM; | |
1741 | } | |
1742 | prog->nr_reloc = nrels; | |
1743 | ||
1744 | for (i = 0; i < nrels; i++) { | |
34090915 | 1745 | struct bpf_insn *insns = prog->insns; |
d859900c | 1746 | enum libbpf_map_type type; |
abd29c93 AN |
1747 | unsigned int insn_idx; |
1748 | unsigned int shdr_idx; | |
d859900c | 1749 | const char *name; |
34090915 | 1750 | size_t map_idx; |
abd29c93 AN |
1751 | GElf_Sym sym; |
1752 | GElf_Rel rel; | |
34090915 WN |
1753 | |
1754 | if (!gelf_getrel(data, i, &rel)) { | |
1755 | pr_warning("relocation: failed to get %d reloc\n", i); | |
6371ca3b | 1756 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
1757 | } |
1758 | ||
399dc65e | 1759 | if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { |
34090915 WN |
1760 | pr_warning("relocation: symbol %"PRIx64" not found\n", |
1761 | GELF_R_SYM(rel.r_info)); | |
6371ca3b | 1762 | return -LIBBPF_ERRNO__FORMAT; |
34090915 | 1763 | } |
d859900c DB |
1764 | |
1765 | name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
1766 | sym.st_name) ? : "<?>"; | |
1767 | ||
1768 | pr_debug("relo for %lld value %lld name %d (\'%s\')\n", | |
7d9890ef | 1769 | (long long) (rel.r_info >> 32), |
d859900c | 1770 | (long long) sym.st_value, sym.st_name, name); |
34090915 | 1771 | |
d859900c | 1772 | shdr_idx = sym.st_shndx; |
f2a3e4e9 AN |
1773 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); |
1774 | pr_debug("relocation: insn_idx=%u, shdr_idx=%u\n", | |
1775 | insn_idx, shdr_idx); | |
1776 | ||
1777 | if (shdr_idx >= SHN_LORESERVE) { | |
1778 | pr_warning("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n", | |
1779 | name, shdr_idx, insn_idx, | |
1780 | insns[insn_idx].code); | |
1781 | return -LIBBPF_ERRNO__RELOC; | |
1782 | } | |
d859900c DB |
1783 | if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { |
1784 | pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", | |
1785 | prog->section_name, shdr_idx); | |
666810e8 WN |
1786 | return -LIBBPF_ERRNO__RELOC; |
1787 | } | |
1788 | ||
48cca7e4 AS |
1789 | if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { |
1790 | if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { | |
1791 | pr_warning("incorrect bpf_call opcode\n"); | |
1792 | return -LIBBPF_ERRNO__RELOC; | |
1793 | } | |
1794 | prog->reloc_desc[i].type = RELO_CALL; | |
1795 | prog->reloc_desc[i].insn_idx = insn_idx; | |
1796 | prog->reloc_desc[i].text_off = sym.st_value; | |
9a94f277 | 1797 | obj->has_pseudo_calls = true; |
48cca7e4 AS |
1798 | continue; |
1799 | } | |
1800 | ||
34090915 WN |
1801 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { |
1802 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
1803 | insn_idx, insns[insn_idx].code); | |
6371ca3b | 1804 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
1805 | } |
1806 | ||
d859900c DB |
1807 | if (bpf_object__shndx_is_maps(obj, shdr_idx) || |
1808 | bpf_object__shndx_is_data(obj, shdr_idx)) { | |
1809 | type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); | |
8837fe5d DB |
1810 | if (type != LIBBPF_MAP_UNSPEC) { |
1811 | if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { | |
1812 | pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", | |
1813 | name, insn_idx, insns[insn_idx].code); | |
1814 | return -LIBBPF_ERRNO__RELOC; | |
1815 | } | |
1816 | if (!obj->caps.global_data) { | |
1817 | pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", | |
1818 | name, insn_idx); | |
1819 | return -LIBBPF_ERRNO__RELOC; | |
1820 | } | |
d859900c DB |
1821 | } |
1822 | ||
f8c7a4d4 | 1823 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { |
d859900c DB |
1824 | if (maps[map_idx].libbpf_type != type) |
1825 | continue; | |
1826 | if (type != LIBBPF_MAP_UNSPEC || | |
db48814b AN |
1827 | (maps[map_idx].sec_idx == sym.st_shndx && |
1828 | maps[map_idx].sec_offset == sym.st_value)) { | |
1829 | pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n", | |
1830 | map_idx, maps[map_idx].name, | |
1831 | maps[map_idx].sec_idx, | |
1832 | maps[map_idx].sec_offset, | |
1833 | insn_idx); | |
f8c7a4d4 JS |
1834 | break; |
1835 | } | |
94e5adec | 1836 | } |
94e5adec | 1837 | |
f8c7a4d4 | 1838 | if (map_idx >= nr_maps) { |
76e1022b | 1839 | pr_warning("bpf relocation: map_idx %d larger than %d\n", |
f8c7a4d4 JS |
1840 | (int)map_idx, (int)nr_maps - 1); |
1841 | return -LIBBPF_ERRNO__RELOC; | |
1842 | } | |
34090915 | 1843 | |
d859900c DB |
1844 | prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? |
1845 | RELO_DATA : RELO_LD64; | |
f8c7a4d4 JS |
1846 | prog->reloc_desc[i].insn_idx = insn_idx; |
1847 | prog->reloc_desc[i].map_idx = map_idx; | |
1848 | } | |
34090915 WN |
1849 | } |
1850 | return 0; | |
1851 | } | |
1852 | ||
abd29c93 | 1853 | static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) |
8a138aed MKL |
1854 | { |
1855 | struct bpf_map_def *def = &map->def; | |
d859900c | 1856 | __u32 key_type_id = 0, value_type_id = 0; |
96408c43 | 1857 | int ret; |
8a138aed | 1858 | |
abd29c93 AN |
1859 | /* if it's BTF-defined map, we don't need to search for type IDs */ |
1860 | if (map->sec_idx == obj->efile.btf_maps_shndx) | |
1861 | return 0; | |
1862 | ||
d859900c | 1863 | if (!bpf_map__is_internal(map)) { |
abd29c93 | 1864 | ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, |
d859900c DB |
1865 | def->value_size, &key_type_id, |
1866 | &value_type_id); | |
1867 | } else { | |
1868 | /* | |
1869 | * LLVM annotates global data differently in BTF, that is, | |
1870 | * only as '.data', '.bss' or '.rodata'. | |
1871 | */ | |
abd29c93 | 1872 | ret = btf__find_by_name(obj->btf, |
d859900c DB |
1873 | libbpf_type_to_btf_name[map->libbpf_type]); |
1874 | } | |
1875 | if (ret < 0) | |
96408c43 | 1876 | return ret; |
8a138aed | 1877 | |
96408c43 | 1878 | map->btf_key_type_id = key_type_id; |
d859900c DB |
1879 | map->btf_value_type_id = bpf_map__is_internal(map) ? |
1880 | ret : value_type_id; | |
8a138aed MKL |
1881 | return 0; |
1882 | } | |
1883 | ||
26736eb9 JK |
1884 | int bpf_map__reuse_fd(struct bpf_map *map, int fd) |
1885 | { | |
1886 | struct bpf_map_info info = {}; | |
1887 | __u32 len = sizeof(info); | |
1888 | int new_fd, err; | |
1889 | char *new_name; | |
1890 | ||
1891 | err = bpf_obj_get_info_by_fd(fd, &info, &len); | |
1892 | if (err) | |
1893 | return err; | |
1894 | ||
1895 | new_name = strdup(info.name); | |
1896 | if (!new_name) | |
1897 | return -errno; | |
1898 | ||
1899 | new_fd = open("/", O_RDONLY | O_CLOEXEC); | |
4e08326e THJ |
1900 | if (new_fd < 0) { |
1901 | err = -errno; | |
26736eb9 | 1902 | goto err_free_new_name; |
4e08326e | 1903 | } |
26736eb9 JK |
1904 | |
1905 | new_fd = dup3(fd, new_fd, O_CLOEXEC); | |
4e08326e THJ |
1906 | if (new_fd < 0) { |
1907 | err = -errno; | |
26736eb9 | 1908 | goto err_close_new_fd; |
4e08326e | 1909 | } |
26736eb9 JK |
1910 | |
1911 | err = zclose(map->fd); | |
4e08326e THJ |
1912 | if (err) { |
1913 | err = -errno; | |
26736eb9 | 1914 | goto err_close_new_fd; |
4e08326e | 1915 | } |
26736eb9 JK |
1916 | free(map->name); |
1917 | ||
1918 | map->fd = new_fd; | |
1919 | map->name = new_name; | |
1920 | map->def.type = info.type; | |
1921 | map->def.key_size = info.key_size; | |
1922 | map->def.value_size = info.value_size; | |
1923 | map->def.max_entries = info.max_entries; | |
1924 | map->def.map_flags = info.map_flags; | |
1925 | map->btf_key_type_id = info.btf_key_type_id; | |
1926 | map->btf_value_type_id = info.btf_value_type_id; | |
1927 | ||
1928 | return 0; | |
1929 | ||
1930 | err_close_new_fd: | |
1931 | close(new_fd); | |
1932 | err_free_new_name: | |
1933 | free(new_name); | |
4e08326e | 1934 | return err; |
26736eb9 JK |
1935 | } |
1936 | ||
1a11a4c7 AI |
1937 | int bpf_map__resize(struct bpf_map *map, __u32 max_entries) |
1938 | { | |
1939 | if (!map || !max_entries) | |
1940 | return -EINVAL; | |
1941 | ||
1942 | /* If map already created, its attributes can't be changed. */ | |
1943 | if (map->fd >= 0) | |
1944 | return -EBUSY; | |
1945 | ||
1946 | map->def.max_entries = max_entries; | |
1947 | ||
1948 | return 0; | |
1949 | } | |
1950 | ||
47eff617 SF |
1951 | static int |
1952 | bpf_object__probe_name(struct bpf_object *obj) | |
1953 | { | |
1954 | struct bpf_load_program_attr attr; | |
1955 | char *cp, errmsg[STRERR_BUFSIZE]; | |
1956 | struct bpf_insn insns[] = { | |
1957 | BPF_MOV64_IMM(BPF_REG_0, 0), | |
1958 | BPF_EXIT_INSN(), | |
1959 | }; | |
1960 | int ret; | |
1961 | ||
1962 | /* make sure basic loading works */ | |
1963 | ||
1964 | memset(&attr, 0, sizeof(attr)); | |
1965 | attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; | |
1966 | attr.insns = insns; | |
1967 | attr.insns_cnt = ARRAY_SIZE(insns); | |
1968 | attr.license = "GPL"; | |
1969 | ||
1970 | ret = bpf_load_program_xattr(&attr, NULL, 0); | |
1971 | if (ret < 0) { | |
1972 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
1973 | pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", | |
1974 | __func__, cp, errno); | |
1975 | return -errno; | |
1976 | } | |
1977 | close(ret); | |
1978 | ||
1979 | /* now try the same program, but with the name */ | |
1980 | ||
1981 | attr.name = "test"; | |
1982 | ret = bpf_load_program_xattr(&attr, NULL, 0); | |
1983 | if (ret >= 0) { | |
1984 | obj->caps.name = 1; | |
1985 | close(ret); | |
1986 | } | |
1987 | ||
1988 | return 0; | |
1989 | } | |
1990 | ||
8837fe5d DB |
1991 | static int |
1992 | bpf_object__probe_global_data(struct bpf_object *obj) | |
1993 | { | |
1994 | struct bpf_load_program_attr prg_attr; | |
1995 | struct bpf_create_map_attr map_attr; | |
1996 | char *cp, errmsg[STRERR_BUFSIZE]; | |
1997 | struct bpf_insn insns[] = { | |
1998 | BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), | |
1999 | BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), | |
2000 | BPF_MOV64_IMM(BPF_REG_0, 0), | |
2001 | BPF_EXIT_INSN(), | |
2002 | }; | |
2003 | int ret, map; | |
2004 | ||
2005 | memset(&map_attr, 0, sizeof(map_attr)); | |
2006 | map_attr.map_type = BPF_MAP_TYPE_ARRAY; | |
2007 | map_attr.key_size = sizeof(int); | |
2008 | map_attr.value_size = 32; | |
2009 | map_attr.max_entries = 1; | |
2010 | ||
2011 | map = bpf_create_map_xattr(&map_attr); | |
2012 | if (map < 0) { | |
2013 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
2014 | pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", | |
2015 | __func__, cp, errno); | |
2016 | return -errno; | |
2017 | } | |
2018 | ||
2019 | insns[0].imm = map; | |
2020 | ||
2021 | memset(&prg_attr, 0, sizeof(prg_attr)); | |
2022 | prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; | |
2023 | prg_attr.insns = insns; | |
2024 | prg_attr.insns_cnt = ARRAY_SIZE(insns); | |
2025 | prg_attr.license = "GPL"; | |
2026 | ||
2027 | ret = bpf_load_program_xattr(&prg_attr, NULL, 0); | |
2028 | if (ret >= 0) { | |
2029 | obj->caps.global_data = 1; | |
2030 | close(ret); | |
2031 | } | |
2032 | ||
2033 | close(map); | |
2034 | return 0; | |
2035 | } | |
2036 | ||
d7c4b398 AN |
2037 | static int bpf_object__probe_btf_func(struct bpf_object *obj) |
2038 | { | |
2039 | const char strs[] = "\0int\0x\0a"; | |
2040 | /* void x(int a) {} */ | |
2041 | __u32 types[] = { | |
2042 | /* int */ | |
2043 | BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ | |
2044 | /* FUNC_PROTO */ /* [2] */ | |
2045 | BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), | |
2046 | BTF_PARAM_ENC(7, 1), | |
2047 | /* FUNC x */ /* [3] */ | |
2048 | BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), | |
2049 | }; | |
cfd49210 | 2050 | int btf_fd; |
d7c4b398 | 2051 | |
cfd49210 MR |
2052 | btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), |
2053 | strs, sizeof(strs)); | |
2054 | if (btf_fd >= 0) { | |
d7c4b398 | 2055 | obj->caps.btf_func = 1; |
cfd49210 MR |
2056 | close(btf_fd); |
2057 | return 1; | |
2058 | } | |
2059 | ||
d7c4b398 AN |
2060 | return 0; |
2061 | } | |
2062 | ||
2063 | static int bpf_object__probe_btf_datasec(struct bpf_object *obj) | |
2064 | { | |
2065 | const char strs[] = "\0x\0.data"; | |
2066 | /* static int a; */ | |
2067 | __u32 types[] = { | |
2068 | /* int */ | |
2069 | BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ | |
2070 | /* VAR x */ /* [2] */ | |
2071 | BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), | |
2072 | BTF_VAR_STATIC, | |
2073 | /* DATASEC val */ /* [3] */ | |
2074 | BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), | |
2075 | BTF_VAR_SECINFO_ENC(2, 0, 4), | |
2076 | }; | |
cfd49210 | 2077 | int btf_fd; |
d7c4b398 | 2078 | |
cfd49210 MR |
2079 | btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), |
2080 | strs, sizeof(strs)); | |
2081 | if (btf_fd >= 0) { | |
d7c4b398 | 2082 | obj->caps.btf_datasec = 1; |
cfd49210 MR |
2083 | close(btf_fd); |
2084 | return 1; | |
2085 | } | |
2086 | ||
d7c4b398 AN |
2087 | return 0; |
2088 | } | |
2089 | ||
47eff617 SF |
2090 | static int |
2091 | bpf_object__probe_caps(struct bpf_object *obj) | |
2092 | { | |
8837fe5d DB |
2093 | int (*probe_fn[])(struct bpf_object *obj) = { |
2094 | bpf_object__probe_name, | |
2095 | bpf_object__probe_global_data, | |
d7c4b398 AN |
2096 | bpf_object__probe_btf_func, |
2097 | bpf_object__probe_btf_datasec, | |
8837fe5d DB |
2098 | }; |
2099 | int i, ret; | |
2100 | ||
2101 | for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { | |
2102 | ret = probe_fn[i](obj); | |
2103 | if (ret < 0) | |
15ea164e | 2104 | pr_debug("Probe #%d failed with %d.\n", i, ret); |
8837fe5d DB |
2105 | } |
2106 | ||
2107 | return 0; | |
47eff617 SF |
2108 | } |
2109 | ||
d859900c DB |
2110 | static int |
2111 | bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) | |
2112 | { | |
2113 | char *cp, errmsg[STRERR_BUFSIZE]; | |
2114 | int err, zero = 0; | |
2115 | __u8 *data; | |
2116 | ||
2117 | /* Nothing to do here since kernel already zero-initializes .bss map. */ | |
2118 | if (map->libbpf_type == LIBBPF_MAP_BSS) | |
2119 | return 0; | |
2120 | ||
2121 | data = map->libbpf_type == LIBBPF_MAP_DATA ? | |
2122 | obj->sections.data : obj->sections.rodata; | |
2123 | ||
2124 | err = bpf_map_update_elem(map->fd, &zero, data, 0); | |
2125 | /* Freeze .rodata map as read-only from syscall side. */ | |
2126 | if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { | |
2127 | err = bpf_map_freeze(map->fd); | |
2128 | if (err) { | |
2129 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
2130 | pr_warning("Error freezing map(%s) as read-only: %s\n", | |
2131 | map->name, cp); | |
2132 | err = 0; | |
2133 | } | |
2134 | } | |
2135 | return err; | |
2136 | } | |
2137 | ||
52d3352e WN |
2138 | static int |
2139 | bpf_object__create_maps(struct bpf_object *obj) | |
2140 | { | |
8a138aed | 2141 | struct bpf_create_map_attr create_attr = {}; |
d7ff34d5 | 2142 | int nr_cpus = 0; |
52d3352e | 2143 | unsigned int i; |
8a138aed | 2144 | int err; |
52d3352e | 2145 | |
9d759a9b | 2146 | for (i = 0; i < obj->nr_maps; i++) { |
8a138aed MKL |
2147 | struct bpf_map *map = &obj->maps[i]; |
2148 | struct bpf_map_def *def = &map->def; | |
1ce6a9fc | 2149 | char *cp, errmsg[STRERR_BUFSIZE]; |
8a138aed MKL |
2150 | int *pfd = &map->fd; |
2151 | ||
26736eb9 JK |
2152 | if (map->fd >= 0) { |
2153 | pr_debug("skip map create (preset) %s: fd=%d\n", | |
2154 | map->name, map->fd); | |
2155 | continue; | |
2156 | } | |
2157 | ||
94cb310c SF |
2158 | if (obj->caps.name) |
2159 | create_attr.name = map->name; | |
f0307a7e | 2160 | create_attr.map_ifindex = map->map_ifindex; |
8a138aed MKL |
2161 | create_attr.map_type = def->type; |
2162 | create_attr.map_flags = def->map_flags; | |
2163 | create_attr.key_size = def->key_size; | |
2164 | create_attr.value_size = def->value_size; | |
d7ff34d5 AN |
2165 | if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && |
2166 | !def->max_entries) { | |
2167 | if (!nr_cpus) | |
2168 | nr_cpus = libbpf_num_possible_cpus(); | |
2169 | if (nr_cpus < 0) { | |
2170 | pr_warning("failed to determine number of system CPUs: %d\n", | |
2171 | nr_cpus); | |
2172 | err = nr_cpus; | |
2173 | goto err_out; | |
2174 | } | |
2175 | pr_debug("map '%s': setting size to %d\n", | |
2176 | map->name, nr_cpus); | |
2177 | create_attr.max_entries = nr_cpus; | |
2178 | } else { | |
2179 | create_attr.max_entries = def->max_entries; | |
2180 | } | |
e55d54f4 | 2181 | create_attr.btf_fd = 0; |
61746dbe MKL |
2182 | create_attr.btf_key_type_id = 0; |
2183 | create_attr.btf_value_type_id = 0; | |
addb9fc9 NS |
2184 | if (bpf_map_type__is_map_in_map(def->type) && |
2185 | map->inner_map_fd >= 0) | |
2186 | create_attr.inner_map_fd = map->inner_map_fd; | |
8a138aed | 2187 | |
abd29c93 | 2188 | if (obj->btf && !bpf_map_find_btf_info(obj, map)) { |
8a138aed | 2189 | create_attr.btf_fd = btf__fd(obj->btf); |
61746dbe MKL |
2190 | create_attr.btf_key_type_id = map->btf_key_type_id; |
2191 | create_attr.btf_value_type_id = map->btf_value_type_id; | |
8a138aed MKL |
2192 | } |
2193 | ||
2194 | *pfd = bpf_create_map_xattr(&create_attr); | |
e55d54f4 AN |
2195 | if (*pfd < 0 && (create_attr.btf_key_type_id || |
2196 | create_attr.btf_value_type_id)) { | |
d7ff34d5 AN |
2197 | err = -errno; |
2198 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
8a138aed | 2199 | pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", |
d7ff34d5 | 2200 | map->name, cp, err); |
e55d54f4 | 2201 | create_attr.btf_fd = 0; |
61746dbe MKL |
2202 | create_attr.btf_key_type_id = 0; |
2203 | create_attr.btf_value_type_id = 0; | |
2204 | map->btf_key_type_id = 0; | |
2205 | map->btf_value_type_id = 0; | |
8a138aed MKL |
2206 | *pfd = bpf_create_map_xattr(&create_attr); |
2207 | } | |
2208 | ||
52d3352e WN |
2209 | if (*pfd < 0) { |
2210 | size_t j; | |
52d3352e | 2211 | |
d7ff34d5 | 2212 | err = -errno; |
d859900c | 2213 | err_out: |
d7ff34d5 AN |
2214 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); |
2215 | pr_warning("failed to create map (name: '%s'): %s(%d)\n", | |
2216 | map->name, cp, err); | |
52d3352e | 2217 | for (j = 0; j < i; j++) |
9d759a9b | 2218 | zclose(obj->maps[j].fd); |
52d3352e WN |
2219 | return err; |
2220 | } | |
d859900c DB |
2221 | |
2222 | if (bpf_map__is_internal(map)) { | |
2223 | err = bpf_object__populate_internal_map(obj, map); | |
2224 | if (err < 0) { | |
2225 | zclose(*pfd); | |
2226 | goto err_out; | |
2227 | } | |
2228 | } | |
2229 | ||
76e1022b | 2230 | pr_debug("created map %s: fd=%d\n", map->name, *pfd); |
52d3352e WN |
2231 | } |
2232 | ||
52d3352e WN |
2233 | return 0; |
2234 | } | |
2235 | ||
f0187f0b MKL |
2236 | static int |
2237 | check_btf_ext_reloc_err(struct bpf_program *prog, int err, | |
2238 | void *btf_prog_info, const char *info_name) | |
2239 | { | |
2240 | if (err != -ENOENT) { | |
2241 | pr_warning("Error in loading %s for sec %s.\n", | |
2242 | info_name, prog->section_name); | |
2243 | return err; | |
2244 | } | |
2245 | ||
2246 | /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ | |
2247 | ||
2248 | if (btf_prog_info) { | |
2249 | /* | |
2250 | * Some info has already been found but has problem | |
399dc65e | 2251 | * in the last btf_ext reloc. Must have to error out. |
f0187f0b MKL |
2252 | */ |
2253 | pr_warning("Error in relocating %s for sec %s.\n", | |
2254 | info_name, prog->section_name); | |
2255 | return err; | |
2256 | } | |
2257 | ||
399dc65e | 2258 | /* Have problem loading the very first info. Ignore the rest. */ |
f0187f0b MKL |
2259 | pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", |
2260 | info_name, prog->section_name, info_name); | |
2261 | return 0; | |
2262 | } | |
2263 | ||
2264 | static int | |
2265 | bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, | |
2266 | const char *section_name, __u32 insn_offset) | |
2267 | { | |
2268 | int err; | |
2269 | ||
2270 | if (!insn_offset || prog->func_info) { | |
2271 | /* | |
2272 | * !insn_offset => main program | |
2273 | * | |
2274 | * For sub prog, the main program's func_info has to | |
2275 | * be loaded first (i.e. prog->func_info != NULL) | |
2276 | */ | |
2277 | err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, | |
2278 | section_name, insn_offset, | |
2279 | &prog->func_info, | |
2280 | &prog->func_info_cnt); | |
2281 | if (err) | |
2282 | return check_btf_ext_reloc_err(prog, err, | |
2283 | prog->func_info, | |
2284 | "bpf_func_info"); | |
2285 | ||
2286 | prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); | |
2287 | } | |
2288 | ||
3d650141 MKL |
2289 | if (!insn_offset || prog->line_info) { |
2290 | err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, | |
2291 | section_name, insn_offset, | |
2292 | &prog->line_info, | |
2293 | &prog->line_info_cnt); | |
2294 | if (err) | |
2295 | return check_btf_ext_reloc_err(prog, err, | |
2296 | prog->line_info, | |
2297 | "bpf_line_info"); | |
2298 | ||
2299 | prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); | |
2300 | } | |
2301 | ||
f0187f0b MKL |
2302 | return 0; |
2303 | } | |
2304 | ||
ddc7c304 AN |
2305 | #define BPF_CORE_SPEC_MAX_LEN 64 |
2306 | ||
2307 | /* represents BPF CO-RE field or array element accessor */ | |
2308 | struct bpf_core_accessor { | |
2309 | __u32 type_id; /* struct/union type or array element type */ | |
2310 | __u32 idx; /* field index or array index */ | |
2311 | const char *name; /* field name or NULL for array accessor */ | |
2312 | }; | |
2313 | ||
2314 | struct bpf_core_spec { | |
2315 | const struct btf *btf; | |
2316 | /* high-level spec: named fields and array indices only */ | |
2317 | struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN]; | |
2318 | /* high-level spec length */ | |
2319 | int len; | |
2320 | /* raw, low-level spec: 1-to-1 with accessor spec string */ | |
2321 | int raw_spec[BPF_CORE_SPEC_MAX_LEN]; | |
2322 | /* raw spec length */ | |
2323 | int raw_len; | |
2324 | /* field byte offset represented by spec */ | |
2325 | __u32 offset; | |
2326 | }; | |
2327 | ||
2328 | static bool str_is_empty(const char *s) | |
2329 | { | |
2330 | return !s || !s[0]; | |
2331 | } | |
2332 | ||
2333 | /* | |
2334 | * Turn bpf_offset_reloc into a low- and high-level spec representation, | |
2335 | * validating correctness along the way, as well as calculating resulting | |
2336 | * field offset (in bytes), specified by accessor string. Low-level spec | |
2337 | * captures every single level of nestedness, including traversing anonymous | |
2338 | * struct/union members. High-level one only captures semantically meaningful | |
2339 | * "turning points": named fields and array indicies. | |
2340 | * E.g., for this case: | |
2341 | * | |
2342 | * struct sample { | |
2343 | * int __unimportant; | |
2344 | * struct { | |
2345 | * int __1; | |
2346 | * int __2; | |
2347 | * int a[7]; | |
2348 | * }; | |
2349 | * }; | |
2350 | * | |
2351 | * struct sample *s = ...; | |
2352 | * | |
2353 | * int x = &s->a[3]; // access string = '0:1:2:3' | |
2354 | * | |
2355 | * Low-level spec has 1:1 mapping with each element of access string (it's | |
2356 | * just a parsed access string representation): [0, 1, 2, 3]. | |
2357 | * | |
2358 | * High-level spec will capture only 3 points: | |
2359 | * - intial zero-index access by pointer (&s->... is the same as &s[0]...); | |
2360 | * - field 'a' access (corresponds to '2' in low-level spec); | |
2361 | * - array element #3 access (corresponds to '3' in low-level spec). | |
2362 | * | |
2363 | */ | |
2364 | static int bpf_core_spec_parse(const struct btf *btf, | |
2365 | __u32 type_id, | |
2366 | const char *spec_str, | |
2367 | struct bpf_core_spec *spec) | |
2368 | { | |
2369 | int access_idx, parsed_len, i; | |
2370 | const struct btf_type *t; | |
2371 | const char *name; | |
2372 | __u32 id; | |
2373 | __s64 sz; | |
2374 | ||
2375 | if (str_is_empty(spec_str) || *spec_str == ':') | |
2376 | return -EINVAL; | |
2377 | ||
2378 | memset(spec, 0, sizeof(*spec)); | |
2379 | spec->btf = btf; | |
2380 | ||
2381 | /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */ | |
2382 | while (*spec_str) { | |
2383 | if (*spec_str == ':') | |
2384 | ++spec_str; | |
2385 | if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) | |
2386 | return -EINVAL; | |
2387 | if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) | |
2388 | return -E2BIG; | |
2389 | spec_str += parsed_len; | |
2390 | spec->raw_spec[spec->raw_len++] = access_idx; | |
2391 | } | |
2392 | ||
2393 | if (spec->raw_len == 0) | |
2394 | return -EINVAL; | |
2395 | ||
2396 | /* first spec value is always reloc type array index */ | |
2397 | t = skip_mods_and_typedefs(btf, type_id, &id); | |
2398 | if (!t) | |
2399 | return -EINVAL; | |
2400 | ||
2401 | access_idx = spec->raw_spec[0]; | |
2402 | spec->spec[0].type_id = id; | |
2403 | spec->spec[0].idx = access_idx; | |
2404 | spec->len++; | |
2405 | ||
2406 | sz = btf__resolve_size(btf, id); | |
2407 | if (sz < 0) | |
2408 | return sz; | |
2409 | spec->offset = access_idx * sz; | |
2410 | ||
2411 | for (i = 1; i < spec->raw_len; i++) { | |
2412 | t = skip_mods_and_typedefs(btf, id, &id); | |
2413 | if (!t) | |
2414 | return -EINVAL; | |
2415 | ||
2416 | access_idx = spec->raw_spec[i]; | |
2417 | ||
2418 | if (btf_is_composite(t)) { | |
2419 | const struct btf_member *m; | |
2420 | __u32 offset; | |
2421 | ||
2422 | if (access_idx >= btf_vlen(t)) | |
2423 | return -EINVAL; | |
2424 | if (btf_member_bitfield_size(t, access_idx)) | |
2425 | return -EINVAL; | |
2426 | ||
2427 | offset = btf_member_bit_offset(t, access_idx); | |
2428 | if (offset % 8) | |
2429 | return -EINVAL; | |
2430 | spec->offset += offset / 8; | |
2431 | ||
2432 | m = btf_members(t) + access_idx; | |
2433 | if (m->name_off) { | |
2434 | name = btf__name_by_offset(btf, m->name_off); | |
2435 | if (str_is_empty(name)) | |
2436 | return -EINVAL; | |
2437 | ||
2438 | spec->spec[spec->len].type_id = id; | |
2439 | spec->spec[spec->len].idx = access_idx; | |
2440 | spec->spec[spec->len].name = name; | |
2441 | spec->len++; | |
2442 | } | |
2443 | ||
2444 | id = m->type; | |
2445 | } else if (btf_is_array(t)) { | |
2446 | const struct btf_array *a = btf_array(t); | |
2447 | ||
2448 | t = skip_mods_and_typedefs(btf, a->type, &id); | |
2449 | if (!t || access_idx >= a->nelems) | |
2450 | return -EINVAL; | |
2451 | ||
2452 | spec->spec[spec->len].type_id = id; | |
2453 | spec->spec[spec->len].idx = access_idx; | |
2454 | spec->len++; | |
2455 | ||
2456 | sz = btf__resolve_size(btf, id); | |
2457 | if (sz < 0) | |
2458 | return sz; | |
2459 | spec->offset += access_idx * sz; | |
2460 | } else { | |
2461 | pr_warning("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n", | |
2462 | type_id, spec_str, i, id, btf_kind(t)); | |
2463 | return -EINVAL; | |
2464 | } | |
2465 | } | |
2466 | ||
2467 | return 0; | |
2468 | } | |
2469 | ||
2470 | static bool bpf_core_is_flavor_sep(const char *s) | |
2471 | { | |
2472 | /* check X___Y name pattern, where X and Y are not underscores */ | |
2473 | return s[0] != '_' && /* X */ | |
2474 | s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ | |
2475 | s[4] != '_'; /* Y */ | |
2476 | } | |
2477 | ||
2478 | /* Given 'some_struct_name___with_flavor' return the length of a name prefix | |
2479 | * before last triple underscore. Struct name part after last triple | |
2480 | * underscore is ignored by BPF CO-RE relocation during relocation matching. | |
2481 | */ | |
2482 | static size_t bpf_core_essential_name_len(const char *name) | |
2483 | { | |
2484 | size_t n = strlen(name); | |
2485 | int i; | |
2486 | ||
2487 | for (i = n - 5; i >= 0; i--) { | |
2488 | if (bpf_core_is_flavor_sep(name + i)) | |
2489 | return i + 1; | |
2490 | } | |
2491 | return n; | |
2492 | } | |
2493 | ||
2494 | /* dynamically sized list of type IDs */ | |
2495 | struct ids_vec { | |
2496 | __u32 *data; | |
2497 | int len; | |
2498 | }; | |
2499 | ||
2500 | static void bpf_core_free_cands(struct ids_vec *cand_ids) | |
2501 | { | |
2502 | free(cand_ids->data); | |
2503 | free(cand_ids); | |
2504 | } | |
2505 | ||
2506 | static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf, | |
2507 | __u32 local_type_id, | |
2508 | const struct btf *targ_btf) | |
2509 | { | |
2510 | size_t local_essent_len, targ_essent_len; | |
2511 | const char *local_name, *targ_name; | |
2512 | const struct btf_type *t; | |
2513 | struct ids_vec *cand_ids; | |
2514 | __u32 *new_ids; | |
2515 | int i, err, n; | |
2516 | ||
2517 | t = btf__type_by_id(local_btf, local_type_id); | |
2518 | if (!t) | |
2519 | return ERR_PTR(-EINVAL); | |
2520 | ||
2521 | local_name = btf__name_by_offset(local_btf, t->name_off); | |
2522 | if (str_is_empty(local_name)) | |
2523 | return ERR_PTR(-EINVAL); | |
2524 | local_essent_len = bpf_core_essential_name_len(local_name); | |
2525 | ||
2526 | cand_ids = calloc(1, sizeof(*cand_ids)); | |
2527 | if (!cand_ids) | |
2528 | return ERR_PTR(-ENOMEM); | |
2529 | ||
2530 | n = btf__get_nr_types(targ_btf); | |
2531 | for (i = 1; i <= n; i++) { | |
2532 | t = btf__type_by_id(targ_btf, i); | |
2533 | targ_name = btf__name_by_offset(targ_btf, t->name_off); | |
2534 | if (str_is_empty(targ_name)) | |
2535 | continue; | |
2536 | ||
2537 | targ_essent_len = bpf_core_essential_name_len(targ_name); | |
2538 | if (targ_essent_len != local_essent_len) | |
2539 | continue; | |
2540 | ||
2541 | if (strncmp(local_name, targ_name, local_essent_len) == 0) { | |
2542 | pr_debug("[%d] %s: found candidate [%d] %s\n", | |
2543 | local_type_id, local_name, i, targ_name); | |
2544 | new_ids = realloc(cand_ids->data, cand_ids->len + 1); | |
2545 | if (!new_ids) { | |
2546 | err = -ENOMEM; | |
2547 | goto err_out; | |
2548 | } | |
2549 | cand_ids->data = new_ids; | |
2550 | cand_ids->data[cand_ids->len++] = i; | |
2551 | } | |
2552 | } | |
2553 | return cand_ids; | |
2554 | err_out: | |
2555 | bpf_core_free_cands(cand_ids); | |
2556 | return ERR_PTR(err); | |
2557 | } | |
2558 | ||
2559 | /* Check two types for compatibility, skipping const/volatile/restrict and | |
2560 | * typedefs, to ensure we are relocating offset to the compatible entities: | |
2561 | * - any two STRUCTs/UNIONs are compatible and can be mixed; | |
2562 | * - any two FWDs are compatible; | |
2563 | * - any two PTRs are always compatible; | |
2564 | * - for ENUMs, check sizes, names are ignored; | |
2565 | * - for INT, size and bitness should match, signedness is ignored; | |
2566 | * - for ARRAY, dimensionality is ignored, element types are checked for | |
2567 | * compatibility recursively; | |
2568 | * - everything else shouldn't be ever a target of relocation. | |
2569 | * These rules are not set in stone and probably will be adjusted as we get | |
2570 | * more experience with using BPF CO-RE relocations. | |
2571 | */ | |
2572 | static int bpf_core_fields_are_compat(const struct btf *local_btf, | |
2573 | __u32 local_id, | |
2574 | const struct btf *targ_btf, | |
2575 | __u32 targ_id) | |
2576 | { | |
2577 | const struct btf_type *local_type, *targ_type; | |
2578 | ||
2579 | recur: | |
2580 | local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); | |
2581 | targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); | |
2582 | if (!local_type || !targ_type) | |
2583 | return -EINVAL; | |
2584 | ||
2585 | if (btf_is_composite(local_type) && btf_is_composite(targ_type)) | |
2586 | return 1; | |
2587 | if (btf_kind(local_type) != btf_kind(targ_type)) | |
2588 | return 0; | |
2589 | ||
2590 | switch (btf_kind(local_type)) { | |
2591 | case BTF_KIND_FWD: | |
2592 | case BTF_KIND_PTR: | |
2593 | return 1; | |
2594 | case BTF_KIND_ENUM: | |
2595 | return local_type->size == targ_type->size; | |
2596 | case BTF_KIND_INT: | |
2597 | return btf_int_offset(local_type) == 0 && | |
2598 | btf_int_offset(targ_type) == 0 && | |
2599 | local_type->size == targ_type->size && | |
2600 | btf_int_bits(local_type) == btf_int_bits(targ_type); | |
2601 | case BTF_KIND_ARRAY: | |
2602 | local_id = btf_array(local_type)->type; | |
2603 | targ_id = btf_array(targ_type)->type; | |
2604 | goto recur; | |
2605 | default: | |
2606 | pr_warning("unexpected kind %d relocated, local [%d], target [%d]\n", | |
2607 | btf_kind(local_type), local_id, targ_id); | |
2608 | return 0; | |
2609 | } | |
2610 | } | |
2611 | ||
2612 | /* | |
2613 | * Given single high-level named field accessor in local type, find | |
2614 | * corresponding high-level accessor for a target type. Along the way, | |
2615 | * maintain low-level spec for target as well. Also keep updating target | |
2616 | * offset. | |
2617 | * | |
2618 | * Searching is performed through recursive exhaustive enumeration of all | |
2619 | * fields of a struct/union. If there are any anonymous (embedded) | |
2620 | * structs/unions, they are recursively searched as well. If field with | |
2621 | * desired name is found, check compatibility between local and target types, | |
2622 | * before returning result. | |
2623 | * | |
2624 | * 1 is returned, if field is found. | |
2625 | * 0 is returned if no compatible field is found. | |
2626 | * <0 is returned on error. | |
2627 | */ | |
2628 | static int bpf_core_match_member(const struct btf *local_btf, | |
2629 | const struct bpf_core_accessor *local_acc, | |
2630 | const struct btf *targ_btf, | |
2631 | __u32 targ_id, | |
2632 | struct bpf_core_spec *spec, | |
2633 | __u32 *next_targ_id) | |
2634 | { | |
2635 | const struct btf_type *local_type, *targ_type; | |
2636 | const struct btf_member *local_member, *m; | |
2637 | const char *local_name, *targ_name; | |
2638 | __u32 local_id; | |
2639 | int i, n, found; | |
2640 | ||
2641 | targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); | |
2642 | if (!targ_type) | |
2643 | return -EINVAL; | |
2644 | if (!btf_is_composite(targ_type)) | |
2645 | return 0; | |
2646 | ||
2647 | local_id = local_acc->type_id; | |
2648 | local_type = btf__type_by_id(local_btf, local_id); | |
2649 | local_member = btf_members(local_type) + local_acc->idx; | |
2650 | local_name = btf__name_by_offset(local_btf, local_member->name_off); | |
2651 | ||
2652 | n = btf_vlen(targ_type); | |
2653 | m = btf_members(targ_type); | |
2654 | for (i = 0; i < n; i++, m++) { | |
2655 | __u32 offset; | |
2656 | ||
2657 | /* bitfield relocations not supported */ | |
2658 | if (btf_member_bitfield_size(targ_type, i)) | |
2659 | continue; | |
2660 | offset = btf_member_bit_offset(targ_type, i); | |
2661 | if (offset % 8) | |
2662 | continue; | |
2663 | ||
2664 | /* too deep struct/union/array nesting */ | |
2665 | if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) | |
2666 | return -E2BIG; | |
2667 | ||
2668 | /* speculate this member will be the good one */ | |
2669 | spec->offset += offset / 8; | |
2670 | spec->raw_spec[spec->raw_len++] = i; | |
2671 | ||
2672 | targ_name = btf__name_by_offset(targ_btf, m->name_off); | |
2673 | if (str_is_empty(targ_name)) { | |
2674 | /* embedded struct/union, we need to go deeper */ | |
2675 | found = bpf_core_match_member(local_btf, local_acc, | |
2676 | targ_btf, m->type, | |
2677 | spec, next_targ_id); | |
2678 | if (found) /* either found or error */ | |
2679 | return found; | |
2680 | } else if (strcmp(local_name, targ_name) == 0) { | |
2681 | /* matching named field */ | |
2682 | struct bpf_core_accessor *targ_acc; | |
2683 | ||
2684 | targ_acc = &spec->spec[spec->len++]; | |
2685 | targ_acc->type_id = targ_id; | |
2686 | targ_acc->idx = i; | |
2687 | targ_acc->name = targ_name; | |
2688 | ||
2689 | *next_targ_id = m->type; | |
2690 | found = bpf_core_fields_are_compat(local_btf, | |
2691 | local_member->type, | |
2692 | targ_btf, m->type); | |
2693 | if (!found) | |
2694 | spec->len--; /* pop accessor */ | |
2695 | return found; | |
2696 | } | |
2697 | /* member turned out not to be what we looked for */ | |
2698 | spec->offset -= offset / 8; | |
2699 | spec->raw_len--; | |
2700 | } | |
2701 | ||
2702 | return 0; | |
2703 | } | |
2704 | ||
2705 | /* | |
2706 | * Try to match local spec to a target type and, if successful, produce full | |
2707 | * target spec (high-level, low-level + offset). | |
2708 | */ | |
2709 | static int bpf_core_spec_match(struct bpf_core_spec *local_spec, | |
2710 | const struct btf *targ_btf, __u32 targ_id, | |
2711 | struct bpf_core_spec *targ_spec) | |
2712 | { | |
2713 | const struct btf_type *targ_type; | |
2714 | const struct bpf_core_accessor *local_acc; | |
2715 | struct bpf_core_accessor *targ_acc; | |
2716 | int i, sz, matched; | |
2717 | ||
2718 | memset(targ_spec, 0, sizeof(*targ_spec)); | |
2719 | targ_spec->btf = targ_btf; | |
2720 | ||
2721 | local_acc = &local_spec->spec[0]; | |
2722 | targ_acc = &targ_spec->spec[0]; | |
2723 | ||
2724 | for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) { | |
2725 | targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, | |
2726 | &targ_id); | |
2727 | if (!targ_type) | |
2728 | return -EINVAL; | |
2729 | ||
2730 | if (local_acc->name) { | |
2731 | matched = bpf_core_match_member(local_spec->btf, | |
2732 | local_acc, | |
2733 | targ_btf, targ_id, | |
2734 | targ_spec, &targ_id); | |
2735 | if (matched <= 0) | |
2736 | return matched; | |
2737 | } else { | |
2738 | /* for i=0, targ_id is already treated as array element | |
2739 | * type (because it's the original struct), for others | |
2740 | * we should find array element type first | |
2741 | */ | |
2742 | if (i > 0) { | |
2743 | const struct btf_array *a; | |
2744 | ||
2745 | if (!btf_is_array(targ_type)) | |
2746 | return 0; | |
2747 | ||
2748 | a = btf_array(targ_type); | |
2749 | if (local_acc->idx >= a->nelems) | |
2750 | return 0; | |
2751 | if (!skip_mods_and_typedefs(targ_btf, a->type, | |
2752 | &targ_id)) | |
2753 | return -EINVAL; | |
2754 | } | |
2755 | ||
2756 | /* too deep struct/union/array nesting */ | |
2757 | if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN) | |
2758 | return -E2BIG; | |
2759 | ||
2760 | targ_acc->type_id = targ_id; | |
2761 | targ_acc->idx = local_acc->idx; | |
2762 | targ_acc->name = NULL; | |
2763 | targ_spec->len++; | |
2764 | targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; | |
2765 | targ_spec->raw_len++; | |
2766 | ||
2767 | sz = btf__resolve_size(targ_btf, targ_id); | |
2768 | if (sz < 0) | |
2769 | return sz; | |
2770 | targ_spec->offset += local_acc->idx * sz; | |
2771 | } | |
2772 | } | |
2773 | ||
2774 | return 1; | |
2775 | } | |
2776 | ||
2777 | /* | |
2778 | * Patch relocatable BPF instruction. | |
2779 | * Expected insn->imm value is provided for validation, as well as the new | |
2780 | * relocated value. | |
2781 | * | |
2782 | * Currently three kinds of BPF instructions are supported: | |
2783 | * 1. rX = <imm> (assignment with immediate operand); | |
2784 | * 2. rX += <imm> (arithmetic operations with immediate operand); | |
2785 | * 3. *(rX) = <imm> (indirect memory assignment with immediate operand). | |
2786 | * | |
2787 | * If actual insn->imm value is wrong, bail out. | |
2788 | */ | |
2789 | static int bpf_core_reloc_insn(struct bpf_program *prog, int insn_off, | |
2790 | __u32 orig_off, __u32 new_off) | |
2791 | { | |
2792 | struct bpf_insn *insn; | |
2793 | int insn_idx; | |
2794 | __u8 class; | |
2795 | ||
2796 | if (insn_off % sizeof(struct bpf_insn)) | |
2797 | return -EINVAL; | |
2798 | insn_idx = insn_off / sizeof(struct bpf_insn); | |
2799 | ||
2800 | insn = &prog->insns[insn_idx]; | |
2801 | class = BPF_CLASS(insn->code); | |
2802 | ||
2803 | if (class == BPF_ALU || class == BPF_ALU64) { | |
2804 | if (BPF_SRC(insn->code) != BPF_K) | |
2805 | return -EINVAL; | |
2806 | if (insn->imm != orig_off) | |
2807 | return -EINVAL; | |
2808 | insn->imm = new_off; | |
2809 | pr_debug("prog '%s': patched insn #%d (ALU/ALU64) imm %d -> %d\n", | |
2810 | bpf_program__title(prog, false), | |
2811 | insn_idx, orig_off, new_off); | |
2812 | } else { | |
2813 | pr_warning("prog '%s': trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n", | |
2814 | bpf_program__title(prog, false), | |
2815 | insn_idx, insn->code, insn->src_reg, insn->dst_reg, | |
2816 | insn->off, insn->imm); | |
2817 | return -EINVAL; | |
2818 | } | |
2819 | return 0; | |
2820 | } | |
2821 | ||
a1916a15 AN |
2822 | static struct btf *btf_load_raw(const char *path) |
2823 | { | |
2824 | struct btf *btf; | |
2825 | size_t read_cnt; | |
2826 | struct stat st; | |
2827 | void *data; | |
2828 | FILE *f; | |
2829 | ||
2830 | if (stat(path, &st)) | |
2831 | return ERR_PTR(-errno); | |
2832 | ||
2833 | data = malloc(st.st_size); | |
2834 | if (!data) | |
2835 | return ERR_PTR(-ENOMEM); | |
2836 | ||
2837 | f = fopen(path, "rb"); | |
2838 | if (!f) { | |
2839 | btf = ERR_PTR(-errno); | |
2840 | goto cleanup; | |
2841 | } | |
2842 | ||
2843 | read_cnt = fread(data, 1, st.st_size, f); | |
2844 | fclose(f); | |
2845 | if (read_cnt < st.st_size) { | |
2846 | btf = ERR_PTR(-EBADF); | |
2847 | goto cleanup; | |
2848 | } | |
2849 | ||
2850 | btf = btf__new(data, read_cnt); | |
2851 | ||
2852 | cleanup: | |
2853 | free(data); | |
2854 | return btf; | |
2855 | } | |
2856 | ||
ddc7c304 AN |
2857 | /* |
2858 | * Probe few well-known locations for vmlinux kernel image and try to load BTF | |
2859 | * data out of it to use for target BTF. | |
2860 | */ | |
2861 | static struct btf *bpf_core_find_kernel_btf(void) | |
2862 | { | |
a1916a15 AN |
2863 | struct { |
2864 | const char *path_fmt; | |
2865 | bool raw_btf; | |
2866 | } locations[] = { | |
2867 | /* try canonical vmlinux BTF through sysfs first */ | |
2868 | { "/sys/kernel/btf/vmlinux", true /* raw BTF */ }, | |
2869 | /* fall back to trying to find vmlinux ELF on disk otherwise */ | |
2870 | { "/boot/vmlinux-%1$s" }, | |
2871 | { "/lib/modules/%1$s/vmlinux-%1$s" }, | |
2872 | { "/lib/modules/%1$s/build/vmlinux" }, | |
2873 | { "/usr/lib/modules/%1$s/kernel/vmlinux" }, | |
2874 | { "/usr/lib/debug/boot/vmlinux-%1$s" }, | |
2875 | { "/usr/lib/debug/boot/vmlinux-%1$s.debug" }, | |
2876 | { "/usr/lib/debug/lib/modules/%1$s/vmlinux" }, | |
ddc7c304 AN |
2877 | }; |
2878 | char path[PATH_MAX + 1]; | |
2879 | struct utsname buf; | |
2880 | struct btf *btf; | |
2881 | int i; | |
2882 | ||
2883 | uname(&buf); | |
2884 | ||
2885 | for (i = 0; i < ARRAY_SIZE(locations); i++) { | |
a1916a15 | 2886 | snprintf(path, PATH_MAX, locations[i].path_fmt, buf.release); |
ddc7c304 AN |
2887 | |
2888 | if (access(path, R_OK)) | |
2889 | continue; | |
2890 | ||
a1916a15 AN |
2891 | if (locations[i].raw_btf) |
2892 | btf = btf_load_raw(path); | |
2893 | else | |
2894 | btf = btf__parse_elf(path, NULL); | |
2895 | ||
2896 | pr_debug("loading kernel BTF '%s': %ld\n", | |
2897 | path, IS_ERR(btf) ? PTR_ERR(btf) : 0); | |
ddc7c304 AN |
2898 | if (IS_ERR(btf)) |
2899 | continue; | |
2900 | ||
2901 | return btf; | |
2902 | } | |
2903 | ||
2904 | pr_warning("failed to find valid kernel BTF\n"); | |
2905 | return ERR_PTR(-ESRCH); | |
2906 | } | |
2907 | ||
2908 | /* Output spec definition in the format: | |
2909 | * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>, | |
2910 | * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b | |
2911 | */ | |
2912 | static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) | |
2913 | { | |
2914 | const struct btf_type *t; | |
2915 | const char *s; | |
2916 | __u32 type_id; | |
2917 | int i; | |
2918 | ||
2919 | type_id = spec->spec[0].type_id; | |
2920 | t = btf__type_by_id(spec->btf, type_id); | |
2921 | s = btf__name_by_offset(spec->btf, t->name_off); | |
2922 | libbpf_print(level, "[%u] %s + ", type_id, s); | |
2923 | ||
2924 | for (i = 0; i < spec->raw_len; i++) | |
2925 | libbpf_print(level, "%d%s", spec->raw_spec[i], | |
2926 | i == spec->raw_len - 1 ? " => " : ":"); | |
2927 | ||
2928 | libbpf_print(level, "%u @ &x", spec->offset); | |
2929 | ||
2930 | for (i = 0; i < spec->len; i++) { | |
2931 | if (spec->spec[i].name) | |
2932 | libbpf_print(level, ".%s", spec->spec[i].name); | |
2933 | else | |
2934 | libbpf_print(level, "[%u]", spec->spec[i].idx); | |
2935 | } | |
2936 | ||
2937 | } | |
2938 | ||
2939 | static size_t bpf_core_hash_fn(const void *key, void *ctx) | |
2940 | { | |
2941 | return (size_t)key; | |
2942 | } | |
2943 | ||
2944 | static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx) | |
2945 | { | |
2946 | return k1 == k2; | |
2947 | } | |
2948 | ||
2949 | static void *u32_as_hash_key(__u32 x) | |
2950 | { | |
2951 | return (void *)(uintptr_t)x; | |
2952 | } | |
2953 | ||
2954 | /* | |
2955 | * CO-RE relocate single instruction. | |
2956 | * | |
2957 | * The outline and important points of the algorithm: | |
2958 | * 1. For given local type, find corresponding candidate target types. | |
2959 | * Candidate type is a type with the same "essential" name, ignoring | |
2960 | * everything after last triple underscore (___). E.g., `sample`, | |
2961 | * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates | |
2962 | * for each other. Names with triple underscore are referred to as | |
2963 | * "flavors" and are useful, among other things, to allow to | |
2964 | * specify/support incompatible variations of the same kernel struct, which | |
2965 | * might differ between different kernel versions and/or build | |
2966 | * configurations. | |
2967 | * | |
2968 | * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C | |
2969 | * converter, when deduplicated BTF of a kernel still contains more than | |
2970 | * one different types with the same name. In that case, ___2, ___3, etc | |
2971 | * are appended starting from second name conflict. But start flavors are | |
2972 | * also useful to be defined "locally", in BPF program, to extract same | |
2973 | * data from incompatible changes between different kernel | |
2974 | * versions/configurations. For instance, to handle field renames between | |
2975 | * kernel versions, one can use two flavors of the struct name with the | |
2976 | * same common name and use conditional relocations to extract that field, | |
2977 | * depending on target kernel version. | |
2978 | * 2. For each candidate type, try to match local specification to this | |
2979 | * candidate target type. Matching involves finding corresponding | |
2980 | * high-level spec accessors, meaning that all named fields should match, | |
2981 | * as well as all array accesses should be within the actual bounds. Also, | |
2982 | * types should be compatible (see bpf_core_fields_are_compat for details). | |
2983 | * 3. It is supported and expected that there might be multiple flavors | |
2984 | * matching the spec. As long as all the specs resolve to the same set of | |
2985 | * offsets across all candidates, there is not error. If there is any | |
2986 | * ambiguity, CO-RE relocation will fail. This is necessary to accomodate | |
2987 | * imprefection of BTF deduplication, which can cause slight duplication of | |
2988 | * the same BTF type, if some directly or indirectly referenced (by | |
2989 | * pointer) type gets resolved to different actual types in different | |
2990 | * object files. If such situation occurs, deduplicated BTF will end up | |
2991 | * with two (or more) structurally identical types, which differ only in | |
2992 | * types they refer to through pointer. This should be OK in most cases and | |
2993 | * is not an error. | |
2994 | * 4. Candidate types search is performed by linearly scanning through all | |
2995 | * types in target BTF. It is anticipated that this is overall more | |
2996 | * efficient memory-wise and not significantly worse (if not better) | |
2997 | * CPU-wise compared to prebuilding a map from all local type names to | |
2998 | * a list of candidate type names. It's also sped up by caching resolved | |
2999 | * list of matching candidates per each local "root" type ID, that has at | |
3000 | * least one bpf_offset_reloc associated with it. This list is shared | |
3001 | * between multiple relocations for the same type ID and is updated as some | |
3002 | * of the candidates are pruned due to structural incompatibility. | |
3003 | */ | |
3004 | static int bpf_core_reloc_offset(struct bpf_program *prog, | |
3005 | const struct bpf_offset_reloc *relo, | |
3006 | int relo_idx, | |
3007 | const struct btf *local_btf, | |
3008 | const struct btf *targ_btf, | |
3009 | struct hashmap *cand_cache) | |
3010 | { | |
3011 | const char *prog_name = bpf_program__title(prog, false); | |
3012 | struct bpf_core_spec local_spec, cand_spec, targ_spec; | |
3013 | const void *type_key = u32_as_hash_key(relo->type_id); | |
3014 | const struct btf_type *local_type, *cand_type; | |
3015 | const char *local_name, *cand_name; | |
3016 | struct ids_vec *cand_ids; | |
3017 | __u32 local_id, cand_id; | |
3018 | const char *spec_str; | |
3019 | int i, j, err; | |
3020 | ||
3021 | local_id = relo->type_id; | |
3022 | local_type = btf__type_by_id(local_btf, local_id); | |
3023 | if (!local_type) | |
3024 | return -EINVAL; | |
3025 | ||
3026 | local_name = btf__name_by_offset(local_btf, local_type->name_off); | |
3027 | if (str_is_empty(local_name)) | |
3028 | return -EINVAL; | |
3029 | ||
3030 | spec_str = btf__name_by_offset(local_btf, relo->access_str_off); | |
3031 | if (str_is_empty(spec_str)) | |
3032 | return -EINVAL; | |
3033 | ||
3034 | err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec); | |
3035 | if (err) { | |
3036 | pr_warning("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n", | |
3037 | prog_name, relo_idx, local_id, local_name, spec_str, | |
3038 | err); | |
3039 | return -EINVAL; | |
3040 | } | |
3041 | ||
3042 | pr_debug("prog '%s': relo #%d: spec is ", prog_name, relo_idx); | |
3043 | bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec); | |
3044 | libbpf_print(LIBBPF_DEBUG, "\n"); | |
3045 | ||
3046 | if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) { | |
3047 | cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf); | |
3048 | if (IS_ERR(cand_ids)) { | |
3049 | pr_warning("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld", | |
3050 | prog_name, relo_idx, local_id, local_name, | |
3051 | PTR_ERR(cand_ids)); | |
3052 | return PTR_ERR(cand_ids); | |
3053 | } | |
3054 | err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL); | |
3055 | if (err) { | |
3056 | bpf_core_free_cands(cand_ids); | |
3057 | return err; | |
3058 | } | |
3059 | } | |
3060 | ||
3061 | for (i = 0, j = 0; i < cand_ids->len; i++) { | |
3062 | cand_id = cand_ids->data[i]; | |
3063 | cand_type = btf__type_by_id(targ_btf, cand_id); | |
3064 | cand_name = btf__name_by_offset(targ_btf, cand_type->name_off); | |
3065 | ||
3066 | err = bpf_core_spec_match(&local_spec, targ_btf, | |
3067 | cand_id, &cand_spec); | |
3068 | pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ", | |
3069 | prog_name, relo_idx, i, cand_name); | |
3070 | bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec); | |
3071 | libbpf_print(LIBBPF_DEBUG, ": %d\n", err); | |
3072 | if (err < 0) { | |
3073 | pr_warning("prog '%s': relo #%d: matching error: %d\n", | |
3074 | prog_name, relo_idx, err); | |
3075 | return err; | |
3076 | } | |
3077 | if (err == 0) | |
3078 | continue; | |
3079 | ||
3080 | if (j == 0) { | |
3081 | targ_spec = cand_spec; | |
3082 | } else if (cand_spec.offset != targ_spec.offset) { | |
3083 | /* if there are many candidates, they should all | |
3084 | * resolve to the same offset | |
3085 | */ | |
3086 | pr_warning("prog '%s': relo #%d: offset ambiguity: %u != %u\n", | |
3087 | prog_name, relo_idx, cand_spec.offset, | |
3088 | targ_spec.offset); | |
3089 | return -EINVAL; | |
3090 | } | |
3091 | ||
3092 | cand_ids->data[j++] = cand_spec.spec[0].type_id; | |
3093 | } | |
3094 | ||
3095 | cand_ids->len = j; | |
3096 | if (cand_ids->len == 0) { | |
3097 | pr_warning("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n", | |
3098 | prog_name, relo_idx, local_id, local_name, spec_str); | |
3099 | return -ESRCH; | |
3100 | } | |
3101 | ||
3102 | err = bpf_core_reloc_insn(prog, relo->insn_off, | |
3103 | local_spec.offset, targ_spec.offset); | |
3104 | if (err) { | |
3105 | pr_warning("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n", | |
3106 | prog_name, relo_idx, relo->insn_off, err); | |
3107 | return -EINVAL; | |
3108 | } | |
3109 | ||
3110 | return 0; | |
3111 | } | |
3112 | ||
3113 | static int | |
3114 | bpf_core_reloc_offsets(struct bpf_object *obj, const char *targ_btf_path) | |
3115 | { | |
3116 | const struct btf_ext_info_sec *sec; | |
3117 | const struct bpf_offset_reloc *rec; | |
3118 | const struct btf_ext_info *seg; | |
3119 | struct hashmap_entry *entry; | |
3120 | struct hashmap *cand_cache = NULL; | |
3121 | struct bpf_program *prog; | |
3122 | struct btf *targ_btf; | |
3123 | const char *sec_name; | |
3124 | int i, err = 0; | |
3125 | ||
3126 | if (targ_btf_path) | |
3127 | targ_btf = btf__parse_elf(targ_btf_path, NULL); | |
3128 | else | |
3129 | targ_btf = bpf_core_find_kernel_btf(); | |
3130 | if (IS_ERR(targ_btf)) { | |
3131 | pr_warning("failed to get target BTF: %ld\n", | |
3132 | PTR_ERR(targ_btf)); | |
3133 | return PTR_ERR(targ_btf); | |
3134 | } | |
3135 | ||
3136 | cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); | |
3137 | if (IS_ERR(cand_cache)) { | |
3138 | err = PTR_ERR(cand_cache); | |
3139 | goto out; | |
3140 | } | |
3141 | ||
3142 | seg = &obj->btf_ext->offset_reloc_info; | |
3143 | for_each_btf_ext_sec(seg, sec) { | |
3144 | sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); | |
3145 | if (str_is_empty(sec_name)) { | |
3146 | err = -EINVAL; | |
3147 | goto out; | |
3148 | } | |
3149 | prog = bpf_object__find_program_by_title(obj, sec_name); | |
3150 | if (!prog) { | |
3151 | pr_warning("failed to find program '%s' for CO-RE offset relocation\n", | |
3152 | sec_name); | |
3153 | err = -EINVAL; | |
3154 | goto out; | |
3155 | } | |
3156 | ||
3157 | pr_debug("prog '%s': performing %d CO-RE offset relocs\n", | |
3158 | sec_name, sec->num_info); | |
3159 | ||
3160 | for_each_btf_ext_rec(seg, sec, i, rec) { | |
3161 | err = bpf_core_reloc_offset(prog, rec, i, obj->btf, | |
3162 | targ_btf, cand_cache); | |
3163 | if (err) { | |
3164 | pr_warning("prog '%s': relo #%d: failed to relocate: %d\n", | |
3165 | sec_name, i, err); | |
3166 | goto out; | |
3167 | } | |
3168 | } | |
3169 | } | |
3170 | ||
3171 | out: | |
3172 | btf__free(targ_btf); | |
3173 | if (!IS_ERR_OR_NULL(cand_cache)) { | |
3174 | hashmap__for_each_entry(cand_cache, entry, i) { | |
3175 | bpf_core_free_cands(entry->value); | |
3176 | } | |
3177 | hashmap__free(cand_cache); | |
3178 | } | |
3179 | return err; | |
3180 | } | |
3181 | ||
3182 | static int | |
3183 | bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) | |
3184 | { | |
3185 | int err = 0; | |
3186 | ||
3187 | if (obj->btf_ext->offset_reloc_info.len) | |
3188 | err = bpf_core_reloc_offsets(obj, targ_btf_path); | |
3189 | ||
3190 | return err; | |
3191 | } | |
3192 | ||
48cca7e4 AS |
3193 | static int |
3194 | bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, | |
3195 | struct reloc_desc *relo) | |
3196 | { | |
3197 | struct bpf_insn *insn, *new_insn; | |
3198 | struct bpf_program *text; | |
3199 | size_t new_cnt; | |
2993e051 | 3200 | int err; |
48cca7e4 AS |
3201 | |
3202 | if (relo->type != RELO_CALL) | |
3203 | return -LIBBPF_ERRNO__RELOC; | |
3204 | ||
3205 | if (prog->idx == obj->efile.text_shndx) { | |
3206 | pr_warning("relo in .text insn %d into off %d\n", | |
3207 | relo->insn_idx, relo->text_off); | |
3208 | return -LIBBPF_ERRNO__RELOC; | |
3209 | } | |
3210 | ||
3211 | if (prog->main_prog_cnt == 0) { | |
3212 | text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); | |
3213 | if (!text) { | |
3214 | pr_warning("no .text section found yet relo into text exist\n"); | |
3215 | return -LIBBPF_ERRNO__RELOC; | |
3216 | } | |
3217 | new_cnt = prog->insns_cnt + text->insns_cnt; | |
531b014e | 3218 | new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); |
48cca7e4 AS |
3219 | if (!new_insn) { |
3220 | pr_warning("oom in prog realloc\n"); | |
3221 | return -ENOMEM; | |
3222 | } | |
2993e051 | 3223 | |
f0187f0b MKL |
3224 | if (obj->btf_ext) { |
3225 | err = bpf_program_reloc_btf_ext(prog, obj, | |
3226 | text->section_name, | |
3227 | prog->insns_cnt); | |
3228 | if (err) | |
2993e051 | 3229 | return err; |
2993e051 YS |
3230 | } |
3231 | ||
48cca7e4 AS |
3232 | memcpy(new_insn + prog->insns_cnt, text->insns, |
3233 | text->insns_cnt * sizeof(*insn)); | |
3234 | prog->insns = new_insn; | |
3235 | prog->main_prog_cnt = prog->insns_cnt; | |
3236 | prog->insns_cnt = new_cnt; | |
b1a2ce82 JC |
3237 | pr_debug("added %zd insn from %s to prog %s\n", |
3238 | text->insns_cnt, text->section_name, | |
3239 | prog->section_name); | |
48cca7e4 AS |
3240 | } |
3241 | insn = &prog->insns[relo->insn_idx]; | |
3242 | insn->imm += prog->main_prog_cnt - relo->insn_idx; | |
48cca7e4 AS |
3243 | return 0; |
3244 | } | |
3245 | ||
8a47a6c5 | 3246 | static int |
9d759a9b | 3247 | bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) |
8a47a6c5 | 3248 | { |
48cca7e4 | 3249 | int i, err; |
8a47a6c5 | 3250 | |
2993e051 YS |
3251 | if (!prog) |
3252 | return 0; | |
3253 | ||
f0187f0b MKL |
3254 | if (obj->btf_ext) { |
3255 | err = bpf_program_reloc_btf_ext(prog, obj, | |
3256 | prog->section_name, 0); | |
3257 | if (err) | |
2993e051 | 3258 | return err; |
2993e051 YS |
3259 | } |
3260 | ||
3261 | if (!prog->reloc_desc) | |
8a47a6c5 WN |
3262 | return 0; |
3263 | ||
3264 | for (i = 0; i < prog->nr_reloc; i++) { | |
d859900c DB |
3265 | if (prog->reloc_desc[i].type == RELO_LD64 || |
3266 | prog->reloc_desc[i].type == RELO_DATA) { | |
3267 | bool relo_data = prog->reloc_desc[i].type == RELO_DATA; | |
48cca7e4 AS |
3268 | struct bpf_insn *insns = prog->insns; |
3269 | int insn_idx, map_idx; | |
8a47a6c5 | 3270 | |
48cca7e4 AS |
3271 | insn_idx = prog->reloc_desc[i].insn_idx; |
3272 | map_idx = prog->reloc_desc[i].map_idx; | |
8a47a6c5 | 3273 | |
d859900c | 3274 | if (insn_idx + 1 >= (int)prog->insns_cnt) { |
48cca7e4 AS |
3275 | pr_warning("relocation out of range: '%s'\n", |
3276 | prog->section_name); | |
3277 | return -LIBBPF_ERRNO__RELOC; | |
3278 | } | |
d859900c DB |
3279 | |
3280 | if (!relo_data) { | |
3281 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
3282 | } else { | |
3283 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; | |
3284 | insns[insn_idx + 1].imm = insns[insn_idx].imm; | |
3285 | } | |
48cca7e4 | 3286 | insns[insn_idx].imm = obj->maps[map_idx].fd; |
f8c7a4d4 | 3287 | } else if (prog->reloc_desc[i].type == RELO_CALL) { |
48cca7e4 AS |
3288 | err = bpf_program__reloc_text(prog, obj, |
3289 | &prog->reloc_desc[i]); | |
3290 | if (err) | |
3291 | return err; | |
8a47a6c5 | 3292 | } |
8a47a6c5 WN |
3293 | } |
3294 | ||
3295 | zfree(&prog->reloc_desc); | |
3296 | prog->nr_reloc = 0; | |
3297 | return 0; | |
3298 | } | |
3299 | ||
8a47a6c5 | 3300 | static int |
ddc7c304 | 3301 | bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) |
8a47a6c5 WN |
3302 | { |
3303 | struct bpf_program *prog; | |
3304 | size_t i; | |
3305 | int err; | |
3306 | ||
ddc7c304 AN |
3307 | if (obj->btf_ext) { |
3308 | err = bpf_object__relocate_core(obj, targ_btf_path); | |
3309 | if (err) { | |
3310 | pr_warning("failed to perform CO-RE relocations: %d\n", | |
3311 | err); | |
3312 | return err; | |
3313 | } | |
3314 | } | |
8a47a6c5 WN |
3315 | for (i = 0; i < obj->nr_programs; i++) { |
3316 | prog = &obj->programs[i]; | |
3317 | ||
9d759a9b | 3318 | err = bpf_program__relocate(prog, obj); |
8a47a6c5 WN |
3319 | if (err) { |
3320 | pr_warning("failed to relocate '%s'\n", | |
3321 | prog->section_name); | |
3322 | return err; | |
3323 | } | |
3324 | } | |
3325 | return 0; | |
3326 | } | |
3327 | ||
34090915 WN |
3328 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
3329 | { | |
3330 | int i, err; | |
3331 | ||
3332 | if (!obj_elf_valid(obj)) { | |
3333 | pr_warning("Internal error: elf object is closed\n"); | |
6371ca3b | 3334 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
3335 | } |
3336 | ||
3337 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
3338 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
3339 | Elf_Data *data = obj->efile.reloc[i].data; | |
3340 | int idx = shdr->sh_info; | |
3341 | struct bpf_program *prog; | |
34090915 WN |
3342 | |
3343 | if (shdr->sh_type != SHT_REL) { | |
3344 | pr_warning("internal error at %d\n", __LINE__); | |
6371ca3b | 3345 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
3346 | } |
3347 | ||
3348 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
3349 | if (!prog) { | |
077c066a | 3350 | pr_warning("relocation failed: no section(%d)\n", idx); |
6371ca3b | 3351 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
3352 | } |
3353 | ||
399dc65e | 3354 | err = bpf_program__collect_reloc(prog, shdr, data, obj); |
34090915 | 3355 | if (err) |
6371ca3b | 3356 | return err; |
34090915 WN |
3357 | } |
3358 | return 0; | |
3359 | } | |
3360 | ||
55cffde2 | 3361 | static int |
2993e051 | 3362 | load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, |
f0187f0b | 3363 | char *license, __u32 kern_version, int *pfd) |
55cffde2 | 3364 | { |
d7be143b | 3365 | struct bpf_load_program_attr load_attr; |
1ce6a9fc | 3366 | char *cp, errmsg[STRERR_BUFSIZE]; |
da11b417 | 3367 | int log_buf_size = BPF_LOG_BUF_SIZE; |
55cffde2 | 3368 | char *log_buf; |
5d01ab7b | 3369 | int btf_fd, ret; |
55cffde2 | 3370 | |
fba01a06 AN |
3371 | if (!insns || !insns_cnt) |
3372 | return -EINVAL; | |
3373 | ||
d7be143b | 3374 | memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); |
2993e051 YS |
3375 | load_attr.prog_type = prog->type; |
3376 | load_attr.expected_attach_type = prog->expected_attach_type; | |
5b32a23e SF |
3377 | if (prog->caps->name) |
3378 | load_attr.name = prog->name; | |
d7be143b AI |
3379 | load_attr.insns = insns; |
3380 | load_attr.insns_cnt = insns_cnt; | |
3381 | load_attr.license = license; | |
3382 | load_attr.kern_version = kern_version; | |
2993e051 | 3383 | load_attr.prog_ifindex = prog->prog_ifindex; |
3415ec64 AN |
3384 | /* if .BTF.ext was loaded, kernel supports associated BTF for prog */ |
3385 | if (prog->obj->btf_ext) | |
3386 | btf_fd = bpf_object__btf_fd(prog->obj); | |
3387 | else | |
3388 | btf_fd = -1; | |
5d01ab7b | 3389 | load_attr.prog_btf_fd = btf_fd >= 0 ? btf_fd : 0; |
2993e051 YS |
3390 | load_attr.func_info = prog->func_info; |
3391 | load_attr.func_info_rec_size = prog->func_info_rec_size; | |
f0187f0b | 3392 | load_attr.func_info_cnt = prog->func_info_cnt; |
3d650141 MKL |
3393 | load_attr.line_info = prog->line_info; |
3394 | load_attr.line_info_rec_size = prog->line_info_rec_size; | |
3395 | load_attr.line_info_cnt = prog->line_info_cnt; | |
da11b417 | 3396 | load_attr.log_level = prog->log_level; |
04656198 | 3397 | load_attr.prog_flags = prog->prog_flags; |
55cffde2 | 3398 | |
da11b417 AS |
3399 | retry_load: |
3400 | log_buf = malloc(log_buf_size); | |
55cffde2 WN |
3401 | if (!log_buf) |
3402 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
3403 | ||
da11b417 | 3404 | ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); |
55cffde2 WN |
3405 | |
3406 | if (ret >= 0) { | |
da11b417 AS |
3407 | if (load_attr.log_level) |
3408 | pr_debug("verifier log:\n%s", log_buf); | |
55cffde2 WN |
3409 | *pfd = ret; |
3410 | ret = 0; | |
3411 | goto out; | |
3412 | } | |
3413 | ||
da11b417 AS |
3414 | if (errno == ENOSPC) { |
3415 | log_buf_size <<= 1; | |
3416 | free(log_buf); | |
3417 | goto retry_load; | |
3418 | } | |
6371ca3b | 3419 | ret = -LIBBPF_ERRNO__LOAD; |
24d6a808 | 3420 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 3421 | pr_warning("load bpf program failed: %s\n", cp); |
55cffde2 | 3422 | |
6371ca3b WN |
3423 | if (log_buf && log_buf[0] != '\0') { |
3424 | ret = -LIBBPF_ERRNO__VERIFY; | |
55cffde2 WN |
3425 | pr_warning("-- BEGIN DUMP LOG ---\n"); |
3426 | pr_warning("\n%s\n", log_buf); | |
3427 | pr_warning("-- END LOG --\n"); | |
d7be143b AI |
3428 | } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { |
3429 | pr_warning("Program too large (%zu insns), at most %d insns\n", | |
3430 | load_attr.insns_cnt, BPF_MAXINSNS); | |
705fa219 | 3431 | ret = -LIBBPF_ERRNO__PROG2BIG; |
6371ca3b | 3432 | } else { |
705fa219 | 3433 | /* Wrong program type? */ |
d7be143b | 3434 | if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { |
705fa219 WN |
3435 | int fd; |
3436 | ||
d7be143b AI |
3437 | load_attr.prog_type = BPF_PROG_TYPE_KPROBE; |
3438 | load_attr.expected_attach_type = 0; | |
3439 | fd = bpf_load_program_xattr(&load_attr, NULL, 0); | |
705fa219 WN |
3440 | if (fd >= 0) { |
3441 | close(fd); | |
3442 | ret = -LIBBPF_ERRNO__PROGTYPE; | |
3443 | goto out; | |
3444 | } | |
6371ca3b | 3445 | } |
705fa219 WN |
3446 | |
3447 | if (log_buf) | |
3448 | ret = -LIBBPF_ERRNO__KVER; | |
55cffde2 WN |
3449 | } |
3450 | ||
3451 | out: | |
3452 | free(log_buf); | |
3453 | return ret; | |
3454 | } | |
3455 | ||
29cd77f4 | 3456 | int |
55cffde2 | 3457 | bpf_program__load(struct bpf_program *prog, |
e5b0863c | 3458 | char *license, __u32 kern_version) |
55cffde2 | 3459 | { |
b580563e | 3460 | int err = 0, fd, i; |
55cffde2 | 3461 | |
b580563e WN |
3462 | if (prog->instances.nr < 0 || !prog->instances.fds) { |
3463 | if (prog->preprocessor) { | |
3464 | pr_warning("Internal error: can't load program '%s'\n", | |
3465 | prog->section_name); | |
3466 | return -LIBBPF_ERRNO__INTERNAL; | |
3467 | } | |
55cffde2 | 3468 | |
b580563e WN |
3469 | prog->instances.fds = malloc(sizeof(int)); |
3470 | if (!prog->instances.fds) { | |
3471 | pr_warning("Not enough memory for BPF fds\n"); | |
3472 | return -ENOMEM; | |
3473 | } | |
3474 | prog->instances.nr = 1; | |
3475 | prog->instances.fds[0] = -1; | |
3476 | } | |
3477 | ||
3478 | if (!prog->preprocessor) { | |
3479 | if (prog->instances.nr != 1) { | |
3480 | pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", | |
3481 | prog->section_name, prog->instances.nr); | |
3482 | } | |
2993e051 | 3483 | err = load_program(prog, prog->insns, prog->insns_cnt, |
f0187f0b | 3484 | license, kern_version, &fd); |
b580563e WN |
3485 | if (!err) |
3486 | prog->instances.fds[0] = fd; | |
3487 | goto out; | |
3488 | } | |
3489 | ||
3490 | for (i = 0; i < prog->instances.nr; i++) { | |
3491 | struct bpf_prog_prep_result result; | |
3492 | bpf_program_prep_t preprocessor = prog->preprocessor; | |
3493 | ||
1ad9cbb8 | 3494 | memset(&result, 0, sizeof(result)); |
b580563e WN |
3495 | err = preprocessor(prog, i, prog->insns, |
3496 | prog->insns_cnt, &result); | |
3497 | if (err) { | |
3498 | pr_warning("Preprocessing the %dth instance of program '%s' failed\n", | |
3499 | i, prog->section_name); | |
3500 | goto out; | |
3501 | } | |
3502 | ||
3503 | if (!result.new_insn_ptr || !result.new_insn_cnt) { | |
3504 | pr_debug("Skip loading the %dth instance of program '%s'\n", | |
3505 | i, prog->section_name); | |
3506 | prog->instances.fds[i] = -1; | |
3507 | if (result.pfd) | |
3508 | *result.pfd = -1; | |
3509 | continue; | |
3510 | } | |
3511 | ||
2993e051 | 3512 | err = load_program(prog, result.new_insn_ptr, |
b580563e | 3513 | result.new_insn_cnt, |
f0187f0b | 3514 | license, kern_version, &fd); |
b580563e WN |
3515 | |
3516 | if (err) { | |
3517 | pr_warning("Loading the %dth instance of program '%s' failed\n", | |
3518 | i, prog->section_name); | |
3519 | goto out; | |
3520 | } | |
3521 | ||
3522 | if (result.pfd) | |
3523 | *result.pfd = fd; | |
3524 | prog->instances.fds[i] = fd; | |
3525 | } | |
3526 | out: | |
55cffde2 WN |
3527 | if (err) |
3528 | pr_warning("failed to load program '%s'\n", | |
3529 | prog->section_name); | |
3530 | zfree(&prog->insns); | |
3531 | prog->insns_cnt = 0; | |
3532 | return err; | |
3533 | } | |
3534 | ||
a324aae3 AN |
3535 | static bool bpf_program__is_function_storage(const struct bpf_program *prog, |
3536 | const struct bpf_object *obj) | |
9a94f277 JK |
3537 | { |
3538 | return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; | |
3539 | } | |
3540 | ||
55cffde2 | 3541 | static int |
60276f98 | 3542 | bpf_object__load_progs(struct bpf_object *obj, int log_level) |
55cffde2 WN |
3543 | { |
3544 | size_t i; | |
3545 | int err; | |
3546 | ||
3547 | for (i = 0; i < obj->nr_programs; i++) { | |
9a94f277 | 3548 | if (bpf_program__is_function_storage(&obj->programs[i], obj)) |
48cca7e4 | 3549 | continue; |
501b125a | 3550 | obj->programs[i].log_level |= log_level; |
55cffde2 WN |
3551 | err = bpf_program__load(&obj->programs[i], |
3552 | obj->license, | |
3553 | obj->kern_version); | |
3554 | if (err) | |
3555 | return err; | |
3556 | } | |
3557 | return 0; | |
3558 | } | |
3559 | ||
17387dd5 JK |
3560 | static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) |
3561 | { | |
3562 | switch (type) { | |
3563 | case BPF_PROG_TYPE_SOCKET_FILTER: | |
3564 | case BPF_PROG_TYPE_SCHED_CLS: | |
3565 | case BPF_PROG_TYPE_SCHED_ACT: | |
3566 | case BPF_PROG_TYPE_XDP: | |
3567 | case BPF_PROG_TYPE_CGROUP_SKB: | |
3568 | case BPF_PROG_TYPE_CGROUP_SOCK: | |
3569 | case BPF_PROG_TYPE_LWT_IN: | |
3570 | case BPF_PROG_TYPE_LWT_OUT: | |
3571 | case BPF_PROG_TYPE_LWT_XMIT: | |
004d4b27 | 3572 | case BPF_PROG_TYPE_LWT_SEG6LOCAL: |
17387dd5 JK |
3573 | case BPF_PROG_TYPE_SOCK_OPS: |
3574 | case BPF_PROG_TYPE_SK_SKB: | |
3575 | case BPF_PROG_TYPE_CGROUP_DEVICE: | |
3576 | case BPF_PROG_TYPE_SK_MSG: | |
3577 | case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: | |
6bdd533c | 3578 | case BPF_PROG_TYPE_LIRC_MODE2: |
6bc8529c | 3579 | case BPF_PROG_TYPE_SK_REUSEPORT: |
c22fbae7 | 3580 | case BPF_PROG_TYPE_FLOW_DISSECTOR: |
17387dd5 | 3581 | case BPF_PROG_TYPE_UNSPEC: |
17387dd5 | 3582 | case BPF_PROG_TYPE_TRACEPOINT: |
17387dd5 | 3583 | case BPF_PROG_TYPE_RAW_TRACEPOINT: |
4635b0ae | 3584 | case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: |
47ae7e3d | 3585 | case BPF_PROG_TYPE_PERF_EVENT: |
063cc9f0 | 3586 | case BPF_PROG_TYPE_CGROUP_SYSCTL: |
4cdbfb59 | 3587 | case BPF_PROG_TYPE_CGROUP_SOCKOPT: |
47ae7e3d NS |
3588 | return false; |
3589 | case BPF_PROG_TYPE_KPROBE: | |
17387dd5 JK |
3590 | default: |
3591 | return true; | |
3592 | } | |
3593 | } | |
3594 | ||
3595 | static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) | |
cb1e5e96 | 3596 | { |
17387dd5 | 3597 | if (needs_kver && obj->kern_version == 0) { |
cb1e5e96 WN |
3598 | pr_warning("%s doesn't provide kernel version\n", |
3599 | obj->path); | |
6371ca3b | 3600 | return -LIBBPF_ERRNO__KVERSION; |
cb1e5e96 WN |
3601 | } |
3602 | return 0; | |
3603 | } | |
3604 | ||
1a5e3fb1 | 3605 | static struct bpf_object * |
17387dd5 | 3606 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, |
c034a177 | 3607 | bool needs_kver, int flags) |
1a5e3fb1 WN |
3608 | { |
3609 | struct bpf_object *obj; | |
6371ca3b | 3610 | int err; |
1a5e3fb1 WN |
3611 | |
3612 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
3613 | pr_warning("failed to init libelf for %s\n", path); | |
6371ca3b | 3614 | return ERR_PTR(-LIBBPF_ERRNO__LIBELF); |
1a5e3fb1 WN |
3615 | } |
3616 | ||
6c956392 | 3617 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
6371ca3b WN |
3618 | if (IS_ERR(obj)) |
3619 | return obj; | |
1a5e3fb1 | 3620 | |
6371ca3b WN |
3621 | CHECK_ERR(bpf_object__elf_init(obj), err, out); |
3622 | CHECK_ERR(bpf_object__check_endianness(obj), err, out); | |
8837fe5d | 3623 | CHECK_ERR(bpf_object__probe_caps(obj), err, out); |
c034a177 | 3624 | CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); |
6371ca3b | 3625 | CHECK_ERR(bpf_object__collect_reloc(obj), err, out); |
17387dd5 | 3626 | CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); |
1a5e3fb1 WN |
3627 | |
3628 | bpf_object__elf_finish(obj); | |
3629 | return obj; | |
3630 | out: | |
3631 | bpf_object__close(obj); | |
6371ca3b | 3632 | return ERR_PTR(err); |
1a5e3fb1 WN |
3633 | } |
3634 | ||
c034a177 JF |
3635 | struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, |
3636 | int flags) | |
1a5e3fb1 WN |
3637 | { |
3638 | /* param validation */ | |
07f2d4ea | 3639 | if (!attr->file) |
1a5e3fb1 WN |
3640 | return NULL; |
3641 | ||
07f2d4ea JK |
3642 | pr_debug("loading %s\n", attr->file); |
3643 | ||
3644 | return __bpf_object__open(attr->file, NULL, 0, | |
c034a177 JF |
3645 | bpf_prog_type__needs_kver(attr->prog_type), |
3646 | flags); | |
3647 | } | |
3648 | ||
3649 | struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) | |
3650 | { | |
3651 | return __bpf_object__open_xattr(attr, 0); | |
07f2d4ea JK |
3652 | } |
3653 | ||
3654 | struct bpf_object *bpf_object__open(const char *path) | |
3655 | { | |
3656 | struct bpf_object_open_attr attr = { | |
3657 | .file = path, | |
3658 | .prog_type = BPF_PROG_TYPE_UNSPEC, | |
3659 | }; | |
1a5e3fb1 | 3660 | |
07f2d4ea | 3661 | return bpf_object__open_xattr(&attr); |
6c956392 WN |
3662 | } |
3663 | ||
3664 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
acf860ae WN |
3665 | size_t obj_buf_sz, |
3666 | const char *name) | |
6c956392 | 3667 | { |
acf860ae WN |
3668 | char tmp_name[64]; |
3669 | ||
6c956392 WN |
3670 | /* param validation */ |
3671 | if (!obj_buf || obj_buf_sz <= 0) | |
3672 | return NULL; | |
3673 | ||
acf860ae WN |
3674 | if (!name) { |
3675 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
3676 | (unsigned long)obj_buf, | |
3677 | (unsigned long)obj_buf_sz); | |
acf860ae WN |
3678 | name = tmp_name; |
3679 | } | |
399dc65e | 3680 | pr_debug("loading object '%s' from buffer\n", name); |
6c956392 | 3681 | |
c034a177 | 3682 | return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); |
1a5e3fb1 WN |
3683 | } |
3684 | ||
52d3352e WN |
3685 | int bpf_object__unload(struct bpf_object *obj) |
3686 | { | |
3687 | size_t i; | |
3688 | ||
3689 | if (!obj) | |
3690 | return -EINVAL; | |
3691 | ||
9d759a9b WN |
3692 | for (i = 0; i < obj->nr_maps; i++) |
3693 | zclose(obj->maps[i].fd); | |
52d3352e | 3694 | |
55cffde2 WN |
3695 | for (i = 0; i < obj->nr_programs; i++) |
3696 | bpf_program__unload(&obj->programs[i]); | |
3697 | ||
52d3352e WN |
3698 | return 0; |
3699 | } | |
3700 | ||
60276f98 | 3701 | int bpf_object__load_xattr(struct bpf_object_load_attr *attr) |
52d3352e | 3702 | { |
60276f98 | 3703 | struct bpf_object *obj; |
6371ca3b WN |
3704 | int err; |
3705 | ||
60276f98 QM |
3706 | if (!attr) |
3707 | return -EINVAL; | |
3708 | obj = attr->obj; | |
52d3352e WN |
3709 | if (!obj) |
3710 | return -EINVAL; | |
3711 | ||
3712 | if (obj->loaded) { | |
3713 | pr_warning("object should not be loaded twice\n"); | |
3714 | return -EINVAL; | |
3715 | } | |
3716 | ||
3717 | obj->loaded = true; | |
6371ca3b WN |
3718 | |
3719 | CHECK_ERR(bpf_object__create_maps(obj), err, out); | |
ddc7c304 | 3720 | CHECK_ERR(bpf_object__relocate(obj, attr->target_btf_path), err, out); |
60276f98 | 3721 | CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out); |
52d3352e WN |
3722 | |
3723 | return 0; | |
3724 | out: | |
3725 | bpf_object__unload(obj); | |
3726 | pr_warning("failed to load object '%s'\n", obj->path); | |
6371ca3b | 3727 | return err; |
52d3352e WN |
3728 | } |
3729 | ||
60276f98 QM |
3730 | int bpf_object__load(struct bpf_object *obj) |
3731 | { | |
3732 | struct bpf_object_load_attr attr = { | |
3733 | .obj = obj, | |
3734 | }; | |
3735 | ||
3736 | return bpf_object__load_xattr(&attr); | |
3737 | } | |
3738 | ||
f367540c JS |
3739 | static int check_path(const char *path) |
3740 | { | |
1ce6a9fc | 3741 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
3742 | struct statfs st_fs; |
3743 | char *dname, *dir; | |
3744 | int err = 0; | |
3745 | ||
3746 | if (path == NULL) | |
3747 | return -EINVAL; | |
3748 | ||
3749 | dname = strdup(path); | |
3750 | if (dname == NULL) | |
3751 | return -ENOMEM; | |
3752 | ||
3753 | dir = dirname(dname); | |
3754 | if (statfs(dir, &st_fs)) { | |
24d6a808 | 3755 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 3756 | pr_warning("failed to statfs %s: %s\n", dir, cp); |
f367540c JS |
3757 | err = -errno; |
3758 | } | |
3759 | free(dname); | |
3760 | ||
3761 | if (!err && st_fs.f_type != BPF_FS_MAGIC) { | |
3762 | pr_warning("specified path %s is not on BPF FS\n", path); | |
3763 | err = -EINVAL; | |
3764 | } | |
3765 | ||
3766 | return err; | |
3767 | } | |
3768 | ||
3769 | int bpf_program__pin_instance(struct bpf_program *prog, const char *path, | |
3770 | int instance) | |
3771 | { | |
1ce6a9fc | 3772 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
3773 | int err; |
3774 | ||
3775 | err = check_path(path); | |
3776 | if (err) | |
3777 | return err; | |
3778 | ||
3779 | if (prog == NULL) { | |
3780 | pr_warning("invalid program pointer\n"); | |
3781 | return -EINVAL; | |
3782 | } | |
3783 | ||
3784 | if (instance < 0 || instance >= prog->instances.nr) { | |
3785 | pr_warning("invalid prog instance %d of prog %s (max %d)\n", | |
3786 | instance, prog->section_name, prog->instances.nr); | |
3787 | return -EINVAL; | |
3788 | } | |
3789 | ||
3790 | if (bpf_obj_pin(prog->instances.fds[instance], path)) { | |
24d6a808 | 3791 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 3792 | pr_warning("failed to pin program: %s\n", cp); |
f367540c JS |
3793 | return -errno; |
3794 | } | |
3795 | pr_debug("pinned program '%s'\n", path); | |
3796 | ||
3797 | return 0; | |
3798 | } | |
3799 | ||
0c19a9fb SF |
3800 | int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, |
3801 | int instance) | |
3802 | { | |
3803 | int err; | |
3804 | ||
3805 | err = check_path(path); | |
3806 | if (err) | |
3807 | return err; | |
3808 | ||
3809 | if (prog == NULL) { | |
3810 | pr_warning("invalid program pointer\n"); | |
3811 | return -EINVAL; | |
3812 | } | |
3813 | ||
3814 | if (instance < 0 || instance >= prog->instances.nr) { | |
3815 | pr_warning("invalid prog instance %d of prog %s (max %d)\n", | |
3816 | instance, prog->section_name, prog->instances.nr); | |
3817 | return -EINVAL; | |
3818 | } | |
3819 | ||
3820 | err = unlink(path); | |
3821 | if (err != 0) | |
3822 | return -errno; | |
3823 | pr_debug("unpinned program '%s'\n", path); | |
3824 | ||
3825 | return 0; | |
3826 | } | |
3827 | ||
f367540c JS |
3828 | static int make_dir(const char *path) |
3829 | { | |
1ce6a9fc | 3830 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
3831 | int err = 0; |
3832 | ||
3833 | if (mkdir(path, 0700) && errno != EEXIST) | |
3834 | err = -errno; | |
3835 | ||
1ce6a9fc | 3836 | if (err) { |
24d6a808 | 3837 | cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); |
1ce6a9fc TR |
3838 | pr_warning("failed to mkdir %s: %s\n", path, cp); |
3839 | } | |
f367540c JS |
3840 | return err; |
3841 | } | |
3842 | ||
3843 | int bpf_program__pin(struct bpf_program *prog, const char *path) | |
3844 | { | |
3845 | int i, err; | |
3846 | ||
3847 | err = check_path(path); | |
3848 | if (err) | |
3849 | return err; | |
3850 | ||
3851 | if (prog == NULL) { | |
3852 | pr_warning("invalid program pointer\n"); | |
3853 | return -EINVAL; | |
3854 | } | |
3855 | ||
3856 | if (prog->instances.nr <= 0) { | |
3857 | pr_warning("no instances of prog %s to pin\n", | |
3858 | prog->section_name); | |
3859 | return -EINVAL; | |
3860 | } | |
3861 | ||
fd734c5c SF |
3862 | if (prog->instances.nr == 1) { |
3863 | /* don't create subdirs when pinning single instance */ | |
3864 | return bpf_program__pin_instance(prog, path, 0); | |
3865 | } | |
3866 | ||
f367540c JS |
3867 | err = make_dir(path); |
3868 | if (err) | |
3869 | return err; | |
3870 | ||
0c19a9fb SF |
3871 | for (i = 0; i < prog->instances.nr; i++) { |
3872 | char buf[PATH_MAX]; | |
3873 | int len; | |
3874 | ||
3875 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
3876 | if (len < 0) { | |
3877 | err = -EINVAL; | |
3878 | goto err_unpin; | |
3879 | } else if (len >= PATH_MAX) { | |
3880 | err = -ENAMETOOLONG; | |
3881 | goto err_unpin; | |
3882 | } | |
3883 | ||
3884 | err = bpf_program__pin_instance(prog, buf, i); | |
3885 | if (err) | |
3886 | goto err_unpin; | |
3887 | } | |
3888 | ||
3889 | return 0; | |
3890 | ||
3891 | err_unpin: | |
3892 | for (i = i - 1; i >= 0; i--) { | |
3893 | char buf[PATH_MAX]; | |
3894 | int len; | |
3895 | ||
3896 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
3897 | if (len < 0) | |
3898 | continue; | |
3899 | else if (len >= PATH_MAX) | |
3900 | continue; | |
3901 | ||
3902 | bpf_program__unpin_instance(prog, buf, i); | |
3903 | } | |
3904 | ||
3905 | rmdir(path); | |
3906 | ||
3907 | return err; | |
3908 | } | |
3909 | ||
3910 | int bpf_program__unpin(struct bpf_program *prog, const char *path) | |
3911 | { | |
3912 | int i, err; | |
3913 | ||
3914 | err = check_path(path); | |
3915 | if (err) | |
3916 | return err; | |
3917 | ||
3918 | if (prog == NULL) { | |
3919 | pr_warning("invalid program pointer\n"); | |
3920 | return -EINVAL; | |
3921 | } | |
3922 | ||
3923 | if (prog->instances.nr <= 0) { | |
3924 | pr_warning("no instances of prog %s to pin\n", | |
3925 | prog->section_name); | |
3926 | return -EINVAL; | |
fd734c5c SF |
3927 | } |
3928 | ||
3929 | if (prog->instances.nr == 1) { | |
3930 | /* don't create subdirs when pinning single instance */ | |
3931 | return bpf_program__unpin_instance(prog, path, 0); | |
0c19a9fb SF |
3932 | } |
3933 | ||
f367540c JS |
3934 | for (i = 0; i < prog->instances.nr; i++) { |
3935 | char buf[PATH_MAX]; | |
3936 | int len; | |
3937 | ||
3938 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
3939 | if (len < 0) | |
3940 | return -EINVAL; | |
3941 | else if (len >= PATH_MAX) | |
3942 | return -ENAMETOOLONG; | |
3943 | ||
0c19a9fb | 3944 | err = bpf_program__unpin_instance(prog, buf, i); |
f367540c JS |
3945 | if (err) |
3946 | return err; | |
3947 | } | |
3948 | ||
0c19a9fb SF |
3949 | err = rmdir(path); |
3950 | if (err) | |
3951 | return -errno; | |
3952 | ||
f367540c JS |
3953 | return 0; |
3954 | } | |
3955 | ||
b6989f35 JS |
3956 | int bpf_map__pin(struct bpf_map *map, const char *path) |
3957 | { | |
1ce6a9fc | 3958 | char *cp, errmsg[STRERR_BUFSIZE]; |
b6989f35 JS |
3959 | int err; |
3960 | ||
3961 | err = check_path(path); | |
3962 | if (err) | |
3963 | return err; | |
3964 | ||
3965 | if (map == NULL) { | |
3966 | pr_warning("invalid map pointer\n"); | |
3967 | return -EINVAL; | |
3968 | } | |
3969 | ||
3970 | if (bpf_obj_pin(map->fd, path)) { | |
24d6a808 | 3971 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 3972 | pr_warning("failed to pin map: %s\n", cp); |
b6989f35 JS |
3973 | return -errno; |
3974 | } | |
3975 | ||
3976 | pr_debug("pinned map '%s'\n", path); | |
0c19a9fb | 3977 | |
b6989f35 JS |
3978 | return 0; |
3979 | } | |
3980 | ||
0c19a9fb SF |
3981 | int bpf_map__unpin(struct bpf_map *map, const char *path) |
3982 | { | |
3983 | int err; | |
3984 | ||
3985 | err = check_path(path); | |
3986 | if (err) | |
3987 | return err; | |
3988 | ||
3989 | if (map == NULL) { | |
3990 | pr_warning("invalid map pointer\n"); | |
3991 | return -EINVAL; | |
3992 | } | |
3993 | ||
3994 | err = unlink(path); | |
3995 | if (err != 0) | |
3996 | return -errno; | |
3997 | pr_debug("unpinned map '%s'\n", path); | |
3998 | ||
3999 | return 0; | |
4000 | } | |
4001 | ||
4002 | int bpf_object__pin_maps(struct bpf_object *obj, const char *path) | |
d5148d85 | 4003 | { |
d5148d85 JS |
4004 | struct bpf_map *map; |
4005 | int err; | |
4006 | ||
4007 | if (!obj) | |
4008 | return -ENOENT; | |
4009 | ||
4010 | if (!obj->loaded) { | |
4011 | pr_warning("object not yet loaded; load it first\n"); | |
4012 | return -ENOENT; | |
4013 | } | |
4014 | ||
4015 | err = make_dir(path); | |
4016 | if (err) | |
4017 | return err; | |
4018 | ||
f74a53d9 | 4019 | bpf_object__for_each_map(map, obj) { |
0c19a9fb SF |
4020 | char buf[PATH_MAX]; |
4021 | int len; | |
4022 | ||
4023 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
4024 | bpf_map__name(map)); | |
4025 | if (len < 0) { | |
4026 | err = -EINVAL; | |
4027 | goto err_unpin_maps; | |
4028 | } else if (len >= PATH_MAX) { | |
4029 | err = -ENAMETOOLONG; | |
4030 | goto err_unpin_maps; | |
4031 | } | |
4032 | ||
4033 | err = bpf_map__pin(map, buf); | |
4034 | if (err) | |
4035 | goto err_unpin_maps; | |
4036 | } | |
4037 | ||
4038 | return 0; | |
4039 | ||
4040 | err_unpin_maps: | |
4041 | while ((map = bpf_map__prev(map, obj))) { | |
4042 | char buf[PATH_MAX]; | |
4043 | int len; | |
4044 | ||
4045 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
4046 | bpf_map__name(map)); | |
4047 | if (len < 0) | |
4048 | continue; | |
4049 | else if (len >= PATH_MAX) | |
4050 | continue; | |
4051 | ||
4052 | bpf_map__unpin(map, buf); | |
4053 | } | |
4054 | ||
4055 | return err; | |
4056 | } | |
4057 | ||
4058 | int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) | |
4059 | { | |
4060 | struct bpf_map *map; | |
4061 | int err; | |
4062 | ||
4063 | if (!obj) | |
4064 | return -ENOENT; | |
4065 | ||
f74a53d9 | 4066 | bpf_object__for_each_map(map, obj) { |
d5148d85 JS |
4067 | char buf[PATH_MAX]; |
4068 | int len; | |
4069 | ||
4070 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
4071 | bpf_map__name(map)); | |
4072 | if (len < 0) | |
4073 | return -EINVAL; | |
4074 | else if (len >= PATH_MAX) | |
4075 | return -ENAMETOOLONG; | |
4076 | ||
0c19a9fb | 4077 | err = bpf_map__unpin(map, buf); |
d5148d85 JS |
4078 | if (err) |
4079 | return err; | |
4080 | } | |
4081 | ||
0c19a9fb SF |
4082 | return 0; |
4083 | } | |
4084 | ||
4085 | int bpf_object__pin_programs(struct bpf_object *obj, const char *path) | |
4086 | { | |
4087 | struct bpf_program *prog; | |
4088 | int err; | |
4089 | ||
4090 | if (!obj) | |
4091 | return -ENOENT; | |
4092 | ||
4093 | if (!obj->loaded) { | |
4094 | pr_warning("object not yet loaded; load it first\n"); | |
4095 | return -ENOENT; | |
4096 | } | |
4097 | ||
4098 | err = make_dir(path); | |
4099 | if (err) | |
4100 | return err; | |
4101 | ||
4102 | bpf_object__for_each_program(prog, obj) { | |
4103 | char buf[PATH_MAX]; | |
4104 | int len; | |
4105 | ||
4106 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 4107 | prog->pin_name); |
0c19a9fb SF |
4108 | if (len < 0) { |
4109 | err = -EINVAL; | |
4110 | goto err_unpin_programs; | |
4111 | } else if (len >= PATH_MAX) { | |
4112 | err = -ENAMETOOLONG; | |
4113 | goto err_unpin_programs; | |
4114 | } | |
4115 | ||
4116 | err = bpf_program__pin(prog, buf); | |
4117 | if (err) | |
4118 | goto err_unpin_programs; | |
4119 | } | |
4120 | ||
4121 | return 0; | |
4122 | ||
4123 | err_unpin_programs: | |
4124 | while ((prog = bpf_program__prev(prog, obj))) { | |
4125 | char buf[PATH_MAX]; | |
4126 | int len; | |
4127 | ||
4128 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 4129 | prog->pin_name); |
0c19a9fb SF |
4130 | if (len < 0) |
4131 | continue; | |
4132 | else if (len >= PATH_MAX) | |
4133 | continue; | |
4134 | ||
4135 | bpf_program__unpin(prog, buf); | |
4136 | } | |
4137 | ||
4138 | return err; | |
4139 | } | |
4140 | ||
4141 | int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) | |
4142 | { | |
4143 | struct bpf_program *prog; | |
4144 | int err; | |
4145 | ||
4146 | if (!obj) | |
4147 | return -ENOENT; | |
4148 | ||
d5148d85 JS |
4149 | bpf_object__for_each_program(prog, obj) { |
4150 | char buf[PATH_MAX]; | |
4151 | int len; | |
4152 | ||
4153 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 4154 | prog->pin_name); |
d5148d85 JS |
4155 | if (len < 0) |
4156 | return -EINVAL; | |
4157 | else if (len >= PATH_MAX) | |
4158 | return -ENAMETOOLONG; | |
4159 | ||
0c19a9fb | 4160 | err = bpf_program__unpin(prog, buf); |
d5148d85 JS |
4161 | if (err) |
4162 | return err; | |
4163 | } | |
4164 | ||
4165 | return 0; | |
4166 | } | |
4167 | ||
0c19a9fb SF |
4168 | int bpf_object__pin(struct bpf_object *obj, const char *path) |
4169 | { | |
4170 | int err; | |
4171 | ||
4172 | err = bpf_object__pin_maps(obj, path); | |
4173 | if (err) | |
4174 | return err; | |
4175 | ||
4176 | err = bpf_object__pin_programs(obj, path); | |
4177 | if (err) { | |
4178 | bpf_object__unpin_maps(obj, path); | |
4179 | return err; | |
4180 | } | |
4181 | ||
4182 | return 0; | |
4183 | } | |
4184 | ||
1a5e3fb1 WN |
4185 | void bpf_object__close(struct bpf_object *obj) |
4186 | { | |
a5b8bd47 WN |
4187 | size_t i; |
4188 | ||
1a5e3fb1 WN |
4189 | if (!obj) |
4190 | return; | |
4191 | ||
10931d24 WN |
4192 | if (obj->clear_priv) |
4193 | obj->clear_priv(obj, obj->priv); | |
4194 | ||
1a5e3fb1 | 4195 | bpf_object__elf_finish(obj); |
52d3352e | 4196 | bpf_object__unload(obj); |
8a138aed | 4197 | btf__free(obj->btf); |
2993e051 | 4198 | btf_ext__free(obj->btf_ext); |
1a5e3fb1 | 4199 | |
9d759a9b | 4200 | for (i = 0; i < obj->nr_maps; i++) { |
561bbcca | 4201 | zfree(&obj->maps[i].name); |
9d759a9b WN |
4202 | if (obj->maps[i].clear_priv) |
4203 | obj->maps[i].clear_priv(&obj->maps[i], | |
4204 | obj->maps[i].priv); | |
4205 | obj->maps[i].priv = NULL; | |
4206 | obj->maps[i].clear_priv = NULL; | |
4207 | } | |
d859900c DB |
4208 | |
4209 | zfree(&obj->sections.rodata); | |
4210 | zfree(&obj->sections.data); | |
9d759a9b WN |
4211 | zfree(&obj->maps); |
4212 | obj->nr_maps = 0; | |
a5b8bd47 WN |
4213 | |
4214 | if (obj->programs && obj->nr_programs) { | |
4215 | for (i = 0; i < obj->nr_programs; i++) | |
4216 | bpf_program__exit(&obj->programs[i]); | |
4217 | } | |
4218 | zfree(&obj->programs); | |
4219 | ||
9a208eff | 4220 | list_del(&obj->list); |
1a5e3fb1 WN |
4221 | free(obj); |
4222 | } | |
aa9b1ac3 | 4223 | |
9a208eff WN |
4224 | struct bpf_object * |
4225 | bpf_object__next(struct bpf_object *prev) | |
4226 | { | |
4227 | struct bpf_object *next; | |
4228 | ||
4229 | if (!prev) | |
4230 | next = list_first_entry(&bpf_objects_list, | |
4231 | struct bpf_object, | |
4232 | list); | |
4233 | else | |
4234 | next = list_next_entry(prev, list); | |
4235 | ||
4236 | /* Empty list is noticed here so don't need checking on entry. */ | |
4237 | if (&next->list == &bpf_objects_list) | |
4238 | return NULL; | |
4239 | ||
4240 | return next; | |
4241 | } | |
4242 | ||
a324aae3 | 4243 | const char *bpf_object__name(const struct bpf_object *obj) |
acf860ae | 4244 | { |
a7fe0450 | 4245 | return obj ? obj->path : ERR_PTR(-EINVAL); |
acf860ae WN |
4246 | } |
4247 | ||
a324aae3 | 4248 | unsigned int bpf_object__kversion(const struct bpf_object *obj) |
45825d8a | 4249 | { |
a7fe0450 | 4250 | return obj ? obj->kern_version : 0; |
45825d8a WN |
4251 | } |
4252 | ||
a324aae3 | 4253 | struct btf *bpf_object__btf(const struct bpf_object *obj) |
789f6bab AI |
4254 | { |
4255 | return obj ? obj->btf : NULL; | |
4256 | } | |
4257 | ||
8a138aed MKL |
4258 | int bpf_object__btf_fd(const struct bpf_object *obj) |
4259 | { | |
4260 | return obj->btf ? btf__fd(obj->btf) : -1; | |
4261 | } | |
4262 | ||
10931d24 WN |
4263 | int bpf_object__set_priv(struct bpf_object *obj, void *priv, |
4264 | bpf_object_clear_priv_t clear_priv) | |
4265 | { | |
4266 | if (obj->priv && obj->clear_priv) | |
4267 | obj->clear_priv(obj, obj->priv); | |
4268 | ||
4269 | obj->priv = priv; | |
4270 | obj->clear_priv = clear_priv; | |
4271 | return 0; | |
4272 | } | |
4273 | ||
a324aae3 | 4274 | void *bpf_object__priv(const struct bpf_object *obj) |
10931d24 WN |
4275 | { |
4276 | return obj ? obj->priv : ERR_PTR(-EINVAL); | |
4277 | } | |
4278 | ||
eac7d845 | 4279 | static struct bpf_program * |
a324aae3 AN |
4280 | __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, |
4281 | bool forward) | |
aa9b1ac3 | 4282 | { |
a83d6e76 | 4283 | size_t nr_programs = obj->nr_programs; |
0c19a9fb | 4284 | ssize_t idx; |
aa9b1ac3 | 4285 | |
a83d6e76 | 4286 | if (!nr_programs) |
aa9b1ac3 | 4287 | return NULL; |
aa9b1ac3 | 4288 | |
a83d6e76 MKL |
4289 | if (!p) |
4290 | /* Iter from the beginning */ | |
4291 | return forward ? &obj->programs[0] : | |
4292 | &obj->programs[nr_programs - 1]; | |
4293 | ||
0c19a9fb | 4294 | if (p->obj != obj) { |
aa9b1ac3 WN |
4295 | pr_warning("error: program handler doesn't match object\n"); |
4296 | return NULL; | |
4297 | } | |
4298 | ||
a83d6e76 | 4299 | idx = (p - obj->programs) + (forward ? 1 : -1); |
0c19a9fb | 4300 | if (idx >= obj->nr_programs || idx < 0) |
aa9b1ac3 WN |
4301 | return NULL; |
4302 | return &obj->programs[idx]; | |
4303 | } | |
4304 | ||
eac7d845 | 4305 | struct bpf_program * |
a324aae3 | 4306 | bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) |
eac7d845 JK |
4307 | { |
4308 | struct bpf_program *prog = prev; | |
4309 | ||
4310 | do { | |
a83d6e76 | 4311 | prog = __bpf_program__iter(prog, obj, true); |
0c19a9fb SF |
4312 | } while (prog && bpf_program__is_function_storage(prog, obj)); |
4313 | ||
4314 | return prog; | |
4315 | } | |
4316 | ||
4317 | struct bpf_program * | |
a324aae3 | 4318 | bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) |
0c19a9fb SF |
4319 | { |
4320 | struct bpf_program *prog = next; | |
4321 | ||
0c19a9fb | 4322 | do { |
a83d6e76 | 4323 | prog = __bpf_program__iter(prog, obj, false); |
eac7d845 JK |
4324 | } while (prog && bpf_program__is_function_storage(prog, obj)); |
4325 | ||
4326 | return prog; | |
4327 | } | |
4328 | ||
edb13ed4 ACM |
4329 | int bpf_program__set_priv(struct bpf_program *prog, void *priv, |
4330 | bpf_program_clear_priv_t clear_priv) | |
aa9b1ac3 WN |
4331 | { |
4332 | if (prog->priv && prog->clear_priv) | |
4333 | prog->clear_priv(prog, prog->priv); | |
4334 | ||
4335 | prog->priv = priv; | |
4336 | prog->clear_priv = clear_priv; | |
4337 | return 0; | |
4338 | } | |
4339 | ||
a324aae3 | 4340 | void *bpf_program__priv(const struct bpf_program *prog) |
aa9b1ac3 | 4341 | { |
be834ffb | 4342 | return prog ? prog->priv : ERR_PTR(-EINVAL); |
aa9b1ac3 WN |
4343 | } |
4344 | ||
9aba3613 JK |
4345 | void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) |
4346 | { | |
4347 | prog->prog_ifindex = ifindex; | |
4348 | } | |
4349 | ||
a324aae3 | 4350 | const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
4351 | { |
4352 | const char *title; | |
4353 | ||
4354 | title = prog->section_name; | |
715f8db9 | 4355 | if (needs_copy) { |
aa9b1ac3 WN |
4356 | title = strdup(title); |
4357 | if (!title) { | |
4358 | pr_warning("failed to strdup program title\n"); | |
6371ca3b | 4359 | return ERR_PTR(-ENOMEM); |
aa9b1ac3 WN |
4360 | } |
4361 | } | |
4362 | ||
4363 | return title; | |
4364 | } | |
4365 | ||
a324aae3 | 4366 | int bpf_program__fd(const struct bpf_program *prog) |
aa9b1ac3 | 4367 | { |
b580563e WN |
4368 | return bpf_program__nth_fd(prog, 0); |
4369 | } | |
4370 | ||
4371 | int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, | |
4372 | bpf_program_prep_t prep) | |
4373 | { | |
4374 | int *instances_fds; | |
4375 | ||
4376 | if (nr_instances <= 0 || !prep) | |
4377 | return -EINVAL; | |
4378 | ||
4379 | if (prog->instances.nr > 0 || prog->instances.fds) { | |
4380 | pr_warning("Can't set pre-processor after loading\n"); | |
4381 | return -EINVAL; | |
4382 | } | |
4383 | ||
4384 | instances_fds = malloc(sizeof(int) * nr_instances); | |
4385 | if (!instances_fds) { | |
4386 | pr_warning("alloc memory failed for fds\n"); | |
4387 | return -ENOMEM; | |
4388 | } | |
4389 | ||
4390 | /* fill all fd with -1 */ | |
4391 | memset(instances_fds, -1, sizeof(int) * nr_instances); | |
4392 | ||
4393 | prog->instances.nr = nr_instances; | |
4394 | prog->instances.fds = instances_fds; | |
4395 | prog->preprocessor = prep; | |
4396 | return 0; | |
4397 | } | |
4398 | ||
a324aae3 | 4399 | int bpf_program__nth_fd(const struct bpf_program *prog, int n) |
b580563e WN |
4400 | { |
4401 | int fd; | |
4402 | ||
1e960043 JK |
4403 | if (!prog) |
4404 | return -EINVAL; | |
4405 | ||
b580563e WN |
4406 | if (n >= prog->instances.nr || n < 0) { |
4407 | pr_warning("Can't get the %dth fd from program %s: only %d instances\n", | |
4408 | n, prog->section_name, prog->instances.nr); | |
4409 | return -EINVAL; | |
4410 | } | |
4411 | ||
4412 | fd = prog->instances.fds[n]; | |
4413 | if (fd < 0) { | |
4414 | pr_warning("%dth instance of program '%s' is invalid\n", | |
4415 | n, prog->section_name); | |
4416 | return -ENOENT; | |
4417 | } | |
4418 | ||
4419 | return fd; | |
aa9b1ac3 | 4420 | } |
9d759a9b | 4421 | |
dd26b7f5 | 4422 | void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) |
5f44e4c8 WN |
4423 | { |
4424 | prog->type = type; | |
4425 | } | |
4426 | ||
a324aae3 | 4427 | static bool bpf_program__is_type(const struct bpf_program *prog, |
5f44e4c8 WN |
4428 | enum bpf_prog_type type) |
4429 | { | |
4430 | return prog ? (prog->type == type) : false; | |
4431 | } | |
4432 | ||
a324aae3 AN |
4433 | #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ |
4434 | int bpf_program__set_##NAME(struct bpf_program *prog) \ | |
4435 | { \ | |
4436 | if (!prog) \ | |
4437 | return -EINVAL; \ | |
4438 | bpf_program__set_type(prog, TYPE); \ | |
4439 | return 0; \ | |
4440 | } \ | |
4441 | \ | |
4442 | bool bpf_program__is_##NAME(const struct bpf_program *prog) \ | |
4443 | { \ | |
4444 | return bpf_program__is_type(prog, TYPE); \ | |
4445 | } \ | |
ed794073 | 4446 | |
7803ba73 | 4447 | BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); |
ed794073 | 4448 | BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); |
7803ba73 JS |
4449 | BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); |
4450 | BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); | |
ed794073 | 4451 | BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); |
e14c93fd | 4452 | BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); |
7803ba73 JS |
4453 | BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); |
4454 | BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); | |
5f44e4c8 | 4455 | |
16962b24 JF |
4456 | void bpf_program__set_expected_attach_type(struct bpf_program *prog, |
4457 | enum bpf_attach_type type) | |
d7be143b AI |
4458 | { |
4459 | prog->expected_attach_type = type; | |
4460 | } | |
4461 | ||
36153532 AI |
4462 | #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ |
4463 | { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } | |
d7be143b | 4464 | |
956b620f | 4465 | /* Programs that can NOT be attached. */ |
36153532 | 4466 | #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) |
d7be143b | 4467 | |
956b620f AI |
4468 | /* Programs that can be attached. */ |
4469 | #define BPF_APROG_SEC(string, ptype, atype) \ | |
36153532 | 4470 | BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) |
81efee75 | 4471 | |
956b620f AI |
4472 | /* Programs that must specify expected attach type at load time. */ |
4473 | #define BPF_EAPROG_SEC(string, ptype, eatype) \ | |
36153532 | 4474 | BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) |
956b620f AI |
4475 | |
4476 | /* Programs that can be attached but attach type can't be identified by section | |
4477 | * name. Kept for backward compatibility. | |
4478 | */ | |
4479 | #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) | |
e50b0a6f | 4480 | |
583c9009 RG |
4481 | static const struct { |
4482 | const char *sec; | |
4483 | size_t len; | |
4484 | enum bpf_prog_type prog_type; | |
d7be143b | 4485 | enum bpf_attach_type expected_attach_type; |
36153532 | 4486 | int is_attachable; |
956b620f | 4487 | enum bpf_attach_type attach_type; |
583c9009 | 4488 | } section_names[] = { |
956b620f AI |
4489 | BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), |
4490 | BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), | |
4491 | BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), | |
4492 | BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), | |
4493 | BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), | |
4494 | BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), | |
4495 | BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), | |
4496 | BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), | |
4497 | BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), | |
4498 | BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), | |
4499 | BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), | |
4500 | BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), | |
4501 | BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), | |
bafa7afe AI |
4502 | BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, |
4503 | BPF_CGROUP_INET_INGRESS), | |
4504 | BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, | |
4505 | BPF_CGROUP_INET_EGRESS), | |
956b620f AI |
4506 | BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), |
4507 | BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, | |
4508 | BPF_CGROUP_INET_SOCK_CREATE), | |
4509 | BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, | |
4510 | BPF_CGROUP_INET4_POST_BIND), | |
4511 | BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, | |
4512 | BPF_CGROUP_INET6_POST_BIND), | |
4513 | BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, | |
4514 | BPF_CGROUP_DEVICE), | |
4515 | BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, | |
4516 | BPF_CGROUP_SOCK_OPS), | |
c6f6851b AI |
4517 | BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, |
4518 | BPF_SK_SKB_STREAM_PARSER), | |
4519 | BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, | |
4520 | BPF_SK_SKB_STREAM_VERDICT), | |
956b620f AI |
4521 | BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), |
4522 | BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, | |
4523 | BPF_SK_MSG_VERDICT), | |
4524 | BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, | |
4525 | BPF_LIRC_MODE2), | |
4526 | BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, | |
4527 | BPF_FLOW_DISSECTOR), | |
4528 | BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4529 | BPF_CGROUP_INET4_BIND), | |
4530 | BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4531 | BPF_CGROUP_INET6_BIND), | |
4532 | BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4533 | BPF_CGROUP_INET4_CONNECT), | |
4534 | BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4535 | BPF_CGROUP_INET6_CONNECT), | |
4536 | BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4537 | BPF_CGROUP_UDP4_SENDMSG), | |
4538 | BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4539 | BPF_CGROUP_UDP6_SENDMSG), | |
9bb59ac1 DB |
4540 | BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, |
4541 | BPF_CGROUP_UDP4_RECVMSG), | |
4542 | BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4543 | BPF_CGROUP_UDP6_RECVMSG), | |
063cc9f0 AI |
4544 | BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, |
4545 | BPF_CGROUP_SYSCTL), | |
4cdbfb59 SF |
4546 | BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, |
4547 | BPF_CGROUP_GETSOCKOPT), | |
4548 | BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, | |
4549 | BPF_CGROUP_SETSOCKOPT), | |
583c9009 | 4550 | }; |
d7be143b | 4551 | |
956b620f | 4552 | #undef BPF_PROG_SEC_IMPL |
583c9009 | 4553 | #undef BPF_PROG_SEC |
956b620f AI |
4554 | #undef BPF_APROG_SEC |
4555 | #undef BPF_EAPROG_SEC | |
4556 | #undef BPF_APROG_COMPAT | |
583c9009 | 4557 | |
c76e4c22 TS |
4558 | #define MAX_TYPE_NAME_SIZE 32 |
4559 | ||
4560 | static char *libbpf_get_type_names(bool attach_type) | |
4561 | { | |
4562 | int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; | |
4563 | char *buf; | |
4564 | ||
4565 | buf = malloc(len); | |
4566 | if (!buf) | |
4567 | return NULL; | |
4568 | ||
4569 | buf[0] = '\0'; | |
4570 | /* Forge string buf with all available names */ | |
4571 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { | |
4572 | if (attach_type && !section_names[i].is_attachable) | |
4573 | continue; | |
4574 | ||
4575 | if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { | |
4576 | free(buf); | |
4577 | return NULL; | |
4578 | } | |
4579 | strcat(buf, " "); | |
4580 | strcat(buf, section_names[i].sec); | |
4581 | } | |
4582 | ||
4583 | return buf; | |
4584 | } | |
4585 | ||
b60df2a0 JK |
4586 | int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, |
4587 | enum bpf_attach_type *expected_attach_type) | |
583c9009 | 4588 | { |
c76e4c22 | 4589 | char *type_names; |
583c9009 RG |
4590 | int i; |
4591 | ||
b60df2a0 JK |
4592 | if (!name) |
4593 | return -EINVAL; | |
583c9009 | 4594 | |
b60df2a0 JK |
4595 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { |
4596 | if (strncmp(name, section_names[i].sec, section_names[i].len)) | |
4597 | continue; | |
4598 | *prog_type = section_names[i].prog_type; | |
4599 | *expected_attach_type = section_names[i].expected_attach_type; | |
4600 | return 0; | |
4601 | } | |
c76e4c22 TS |
4602 | pr_warning("failed to guess program type based on ELF section name '%s'\n", name); |
4603 | type_names = libbpf_get_type_names(false); | |
4604 | if (type_names != NULL) { | |
4605 | pr_info("supported section(type) names are:%s\n", type_names); | |
4606 | free(type_names); | |
4607 | } | |
4608 | ||
b60df2a0 JK |
4609 | return -EINVAL; |
4610 | } | |
583c9009 | 4611 | |
956b620f AI |
4612 | int libbpf_attach_type_by_name(const char *name, |
4613 | enum bpf_attach_type *attach_type) | |
4614 | { | |
c76e4c22 | 4615 | char *type_names; |
956b620f AI |
4616 | int i; |
4617 | ||
4618 | if (!name) | |
4619 | return -EINVAL; | |
4620 | ||
4621 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { | |
4622 | if (strncmp(name, section_names[i].sec, section_names[i].len)) | |
4623 | continue; | |
36153532 | 4624 | if (!section_names[i].is_attachable) |
956b620f AI |
4625 | return -EINVAL; |
4626 | *attach_type = section_names[i].attach_type; | |
4627 | return 0; | |
4628 | } | |
c76e4c22 TS |
4629 | pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); |
4630 | type_names = libbpf_get_type_names(true); | |
4631 | if (type_names != NULL) { | |
4632 | pr_info("attachable section(type) names are:%s\n", type_names); | |
4633 | free(type_names); | |
4634 | } | |
4635 | ||
956b620f AI |
4636 | return -EINVAL; |
4637 | } | |
4638 | ||
b60df2a0 JK |
4639 | static int |
4640 | bpf_program__identify_section(struct bpf_program *prog, | |
4641 | enum bpf_prog_type *prog_type, | |
4642 | enum bpf_attach_type *expected_attach_type) | |
4643 | { | |
4644 | return libbpf_prog_type_by_name(prog->section_name, prog_type, | |
4645 | expected_attach_type); | |
583c9009 RG |
4646 | } |
4647 | ||
a324aae3 | 4648 | int bpf_map__fd(const struct bpf_map *map) |
9d759a9b | 4649 | { |
6e009e65 | 4650 | return map ? map->fd : -EINVAL; |
9d759a9b WN |
4651 | } |
4652 | ||
a324aae3 | 4653 | const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) |
9d759a9b | 4654 | { |
53897a78 | 4655 | return map ? &map->def : ERR_PTR(-EINVAL); |
9d759a9b WN |
4656 | } |
4657 | ||
a324aae3 | 4658 | const char *bpf_map__name(const struct bpf_map *map) |
561bbcca | 4659 | { |
009ad5d5 | 4660 | return map ? map->name : NULL; |
561bbcca WN |
4661 | } |
4662 | ||
5b891af7 | 4663 | __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) |
8a138aed | 4664 | { |
61746dbe | 4665 | return map ? map->btf_key_type_id : 0; |
8a138aed MKL |
4666 | } |
4667 | ||
5b891af7 | 4668 | __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) |
8a138aed | 4669 | { |
61746dbe | 4670 | return map ? map->btf_value_type_id : 0; |
8a138aed MKL |
4671 | } |
4672 | ||
edb13ed4 ACM |
4673 | int bpf_map__set_priv(struct bpf_map *map, void *priv, |
4674 | bpf_map_clear_priv_t clear_priv) | |
9d759a9b WN |
4675 | { |
4676 | if (!map) | |
4677 | return -EINVAL; | |
4678 | ||
4679 | if (map->priv) { | |
4680 | if (map->clear_priv) | |
4681 | map->clear_priv(map, map->priv); | |
4682 | } | |
4683 | ||
4684 | map->priv = priv; | |
4685 | map->clear_priv = clear_priv; | |
4686 | return 0; | |
4687 | } | |
4688 | ||
a324aae3 | 4689 | void *bpf_map__priv(const struct bpf_map *map) |
9d759a9b | 4690 | { |
b4cbfa56 | 4691 | return map ? map->priv : ERR_PTR(-EINVAL); |
9d759a9b WN |
4692 | } |
4693 | ||
a324aae3 | 4694 | bool bpf_map__is_offload_neutral(const struct bpf_map *map) |
f83fb22c JK |
4695 | { |
4696 | return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; | |
4697 | } | |
4698 | ||
a324aae3 | 4699 | bool bpf_map__is_internal(const struct bpf_map *map) |
d859900c DB |
4700 | { |
4701 | return map->libbpf_type != LIBBPF_MAP_UNSPEC; | |
4702 | } | |
4703 | ||
9aba3613 JK |
4704 | void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) |
4705 | { | |
4706 | map->map_ifindex = ifindex; | |
4707 | } | |
4708 | ||
addb9fc9 NS |
4709 | int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) |
4710 | { | |
4711 | if (!bpf_map_type__is_map_in_map(map->def.type)) { | |
4712 | pr_warning("error: unsupported map type\n"); | |
4713 | return -EINVAL; | |
4714 | } | |
4715 | if (map->inner_map_fd != -1) { | |
4716 | pr_warning("error: inner_map_fd already specified\n"); | |
4717 | return -EINVAL; | |
4718 | } | |
4719 | map->inner_map_fd = fd; | |
4720 | return 0; | |
4721 | } | |
4722 | ||
0c19a9fb | 4723 | static struct bpf_map * |
a324aae3 | 4724 | __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) |
9d759a9b | 4725 | { |
0c19a9fb | 4726 | ssize_t idx; |
9d759a9b WN |
4727 | struct bpf_map *s, *e; |
4728 | ||
4729 | if (!obj || !obj->maps) | |
4730 | return NULL; | |
4731 | ||
4732 | s = obj->maps; | |
4733 | e = obj->maps + obj->nr_maps; | |
4734 | ||
0c19a9fb | 4735 | if ((m < s) || (m >= e)) { |
9d759a9b WN |
4736 | pr_warning("error in %s: map handler doesn't belong to object\n", |
4737 | __func__); | |
4738 | return NULL; | |
4739 | } | |
4740 | ||
0c19a9fb SF |
4741 | idx = (m - obj->maps) + i; |
4742 | if (idx >= obj->nr_maps || idx < 0) | |
9d759a9b WN |
4743 | return NULL; |
4744 | return &obj->maps[idx]; | |
4745 | } | |
561bbcca | 4746 | |
0c19a9fb | 4747 | struct bpf_map * |
a324aae3 | 4748 | bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) |
0c19a9fb SF |
4749 | { |
4750 | if (prev == NULL) | |
4751 | return obj->maps; | |
4752 | ||
4753 | return __bpf_map__iter(prev, obj, 1); | |
4754 | } | |
4755 | ||
4756 | struct bpf_map * | |
a324aae3 | 4757 | bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) |
0c19a9fb SF |
4758 | { |
4759 | if (next == NULL) { | |
4760 | if (!obj->nr_maps) | |
4761 | return NULL; | |
4762 | return obj->maps + obj->nr_maps - 1; | |
4763 | } | |
4764 | ||
4765 | return __bpf_map__iter(next, obj, -1); | |
4766 | } | |
4767 | ||
561bbcca | 4768 | struct bpf_map * |
a324aae3 | 4769 | bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) |
561bbcca WN |
4770 | { |
4771 | struct bpf_map *pos; | |
4772 | ||
f74a53d9 | 4773 | bpf_object__for_each_map(pos, obj) { |
973170e6 | 4774 | if (pos->name && !strcmp(pos->name, name)) |
561bbcca WN |
4775 | return pos; |
4776 | } | |
4777 | return NULL; | |
4778 | } | |
5a6acad1 | 4779 | |
f3cea32d | 4780 | int |
a324aae3 | 4781 | bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) |
f3cea32d MF |
4782 | { |
4783 | return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); | |
4784 | } | |
4785 | ||
5a6acad1 WN |
4786 | struct bpf_map * |
4787 | bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) | |
4788 | { | |
db48814b | 4789 | return ERR_PTR(-ENOTSUP); |
5a6acad1 | 4790 | } |
e28ff1a8 JS |
4791 | |
4792 | long libbpf_get_error(const void *ptr) | |
4793 | { | |
d98363b5 | 4794 | return PTR_ERR_OR_ZERO(ptr); |
e28ff1a8 | 4795 | } |
6f6d33f3 JF |
4796 | |
4797 | int bpf_prog_load(const char *file, enum bpf_prog_type type, | |
4798 | struct bpf_object **pobj, int *prog_fd) | |
d7be143b AI |
4799 | { |
4800 | struct bpf_prog_load_attr attr; | |
4801 | ||
4802 | memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); | |
4803 | attr.file = file; | |
4804 | attr.prog_type = type; | |
4805 | attr.expected_attach_type = 0; | |
4806 | ||
4807 | return bpf_prog_load_xattr(&attr, pobj, prog_fd); | |
4808 | } | |
4809 | ||
4810 | int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, | |
4811 | struct bpf_object **pobj, int *prog_fd) | |
6f6d33f3 | 4812 | { |
33bae185 | 4813 | struct bpf_object_open_attr open_attr = {}; |
48cca7e4 | 4814 | struct bpf_program *prog, *first_prog = NULL; |
d7be143b AI |
4815 | enum bpf_attach_type expected_attach_type; |
4816 | enum bpf_prog_type prog_type; | |
6f6d33f3 | 4817 | struct bpf_object *obj; |
f0307a7e | 4818 | struct bpf_map *map; |
6f6d33f3 JF |
4819 | int err; |
4820 | ||
d7be143b AI |
4821 | if (!attr) |
4822 | return -EINVAL; | |
17387dd5 JK |
4823 | if (!attr->file) |
4824 | return -EINVAL; | |
d7be143b | 4825 | |
33bae185 LY |
4826 | open_attr.file = attr->file; |
4827 | open_attr.prog_type = attr->prog_type; | |
4828 | ||
07f2d4ea | 4829 | obj = bpf_object__open_xattr(&open_attr); |
3597683c | 4830 | if (IS_ERR_OR_NULL(obj)) |
6f6d33f3 JF |
4831 | return -ENOENT; |
4832 | ||
48cca7e4 AS |
4833 | bpf_object__for_each_program(prog, obj) { |
4834 | /* | |
4835 | * If type is not specified, try to guess it based on | |
4836 | * section name. | |
4837 | */ | |
d7be143b | 4838 | prog_type = attr->prog_type; |
f0307a7e | 4839 | prog->prog_ifindex = attr->ifindex; |
d7be143b AI |
4840 | expected_attach_type = attr->expected_attach_type; |
4841 | if (prog_type == BPF_PROG_TYPE_UNSPEC) { | |
b60df2a0 JK |
4842 | err = bpf_program__identify_section(prog, &prog_type, |
4843 | &expected_attach_type); | |
4844 | if (err < 0) { | |
48cca7e4 AS |
4845 | bpf_object__close(obj); |
4846 | return -EINVAL; | |
4847 | } | |
583c9009 | 4848 | } |
48cca7e4 | 4849 | |
d7be143b AI |
4850 | bpf_program__set_type(prog, prog_type); |
4851 | bpf_program__set_expected_attach_type(prog, | |
4852 | expected_attach_type); | |
4853 | ||
da11b417 | 4854 | prog->log_level = attr->log_level; |
04656198 | 4855 | prog->prog_flags = attr->prog_flags; |
69495d2a | 4856 | if (!first_prog) |
48cca7e4 AS |
4857 | first_prog = prog; |
4858 | } | |
4859 | ||
f74a53d9 | 4860 | bpf_object__for_each_map(map, obj) { |
f83fb22c JK |
4861 | if (!bpf_map__is_offload_neutral(map)) |
4862 | map->map_ifindex = attr->ifindex; | |
f0307a7e DB |
4863 | } |
4864 | ||
48cca7e4 AS |
4865 | if (!first_prog) { |
4866 | pr_warning("object file doesn't contain bpf program\n"); | |
4867 | bpf_object__close(obj); | |
4868 | return -ENOENT; | |
583c9009 RG |
4869 | } |
4870 | ||
6f6d33f3 JF |
4871 | err = bpf_object__load(obj); |
4872 | if (err) { | |
4873 | bpf_object__close(obj); | |
4874 | return -EINVAL; | |
4875 | } | |
4876 | ||
4877 | *pobj = obj; | |
48cca7e4 | 4878 | *prog_fd = bpf_program__fd(first_prog); |
6f6d33f3 JF |
4879 | return 0; |
4880 | } | |
d0cabbb0 | 4881 | |
1c2e9efc AN |
4882 | struct bpf_link { |
4883 | int (*destroy)(struct bpf_link *link); | |
4884 | }; | |
4885 | ||
4886 | int bpf_link__destroy(struct bpf_link *link) | |
4887 | { | |
4888 | int err; | |
4889 | ||
4890 | if (!link) | |
4891 | return 0; | |
4892 | ||
4893 | err = link->destroy(link); | |
4894 | free(link); | |
4895 | ||
4896 | return err; | |
4897 | } | |
4898 | ||
63f2f5ee AN |
4899 | struct bpf_link_fd { |
4900 | struct bpf_link link; /* has to be at the top of struct */ | |
4901 | int fd; /* hook FD */ | |
4902 | }; | |
4903 | ||
4904 | static int bpf_link__destroy_perf_event(struct bpf_link *link) | |
4905 | { | |
4906 | struct bpf_link_fd *l = (void *)link; | |
4907 | int err; | |
4908 | ||
4909 | err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0); | |
4910 | if (err) | |
4911 | err = -errno; | |
4912 | ||
4913 | close(l->fd); | |
4914 | return err; | |
4915 | } | |
4916 | ||
4917 | struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, | |
4918 | int pfd) | |
4919 | { | |
4920 | char errmsg[STRERR_BUFSIZE]; | |
4921 | struct bpf_link_fd *link; | |
4922 | int prog_fd, err; | |
4923 | ||
4924 | if (pfd < 0) { | |
4925 | pr_warning("program '%s': invalid perf event FD %d\n", | |
4926 | bpf_program__title(prog, false), pfd); | |
4927 | return ERR_PTR(-EINVAL); | |
4928 | } | |
4929 | prog_fd = bpf_program__fd(prog); | |
4930 | if (prog_fd < 0) { | |
4931 | pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n", | |
4932 | bpf_program__title(prog, false)); | |
4933 | return ERR_PTR(-EINVAL); | |
4934 | } | |
4935 | ||
4936 | link = malloc(sizeof(*link)); | |
4937 | if (!link) | |
4938 | return ERR_PTR(-ENOMEM); | |
4939 | link->link.destroy = &bpf_link__destroy_perf_event; | |
4940 | link->fd = pfd; | |
4941 | ||
4942 | if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { | |
4943 | err = -errno; | |
4944 | free(link); | |
4945 | pr_warning("program '%s': failed to attach to pfd %d: %s\n", | |
4946 | bpf_program__title(prog, false), pfd, | |
4947 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
4948 | return ERR_PTR(err); | |
4949 | } | |
4950 | if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { | |
4951 | err = -errno; | |
4952 | free(link); | |
4953 | pr_warning("program '%s': failed to enable pfd %d: %s\n", | |
4954 | bpf_program__title(prog, false), pfd, | |
4955 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
4956 | return ERR_PTR(err); | |
4957 | } | |
4958 | return (struct bpf_link *)link; | |
4959 | } | |
4960 | ||
b2650027 AN |
4961 | /* |
4962 | * this function is expected to parse integer in the range of [0, 2^31-1] from | |
4963 | * given file using scanf format string fmt. If actual parsed value is | |
4964 | * negative, the result might be indistinguishable from error | |
4965 | */ | |
4966 | static int parse_uint_from_file(const char *file, const char *fmt) | |
4967 | { | |
4968 | char buf[STRERR_BUFSIZE]; | |
4969 | int err, ret; | |
4970 | FILE *f; | |
4971 | ||
4972 | f = fopen(file, "r"); | |
4973 | if (!f) { | |
4974 | err = -errno; | |
4975 | pr_debug("failed to open '%s': %s\n", file, | |
4976 | libbpf_strerror_r(err, buf, sizeof(buf))); | |
4977 | return err; | |
4978 | } | |
4979 | err = fscanf(f, fmt, &ret); | |
4980 | if (err != 1) { | |
4981 | err = err == EOF ? -EIO : -errno; | |
4982 | pr_debug("failed to parse '%s': %s\n", file, | |
4983 | libbpf_strerror_r(err, buf, sizeof(buf))); | |
4984 | fclose(f); | |
4985 | return err; | |
4986 | } | |
4987 | fclose(f); | |
4988 | return ret; | |
4989 | } | |
4990 | ||
4991 | static int determine_kprobe_perf_type(void) | |
4992 | { | |
4993 | const char *file = "/sys/bus/event_source/devices/kprobe/type"; | |
4994 | ||
4995 | return parse_uint_from_file(file, "%d\n"); | |
4996 | } | |
4997 | ||
4998 | static int determine_uprobe_perf_type(void) | |
4999 | { | |
5000 | const char *file = "/sys/bus/event_source/devices/uprobe/type"; | |
5001 | ||
5002 | return parse_uint_from_file(file, "%d\n"); | |
5003 | } | |
5004 | ||
5005 | static int determine_kprobe_retprobe_bit(void) | |
5006 | { | |
5007 | const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; | |
5008 | ||
5009 | return parse_uint_from_file(file, "config:%d\n"); | |
5010 | } | |
5011 | ||
5012 | static int determine_uprobe_retprobe_bit(void) | |
5013 | { | |
5014 | const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; | |
5015 | ||
5016 | return parse_uint_from_file(file, "config:%d\n"); | |
5017 | } | |
5018 | ||
5019 | static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, | |
5020 | uint64_t offset, int pid) | |
5021 | { | |
5022 | struct perf_event_attr attr = {}; | |
5023 | char errmsg[STRERR_BUFSIZE]; | |
5024 | int type, pfd, err; | |
5025 | ||
5026 | type = uprobe ? determine_uprobe_perf_type() | |
5027 | : determine_kprobe_perf_type(); | |
5028 | if (type < 0) { | |
5029 | pr_warning("failed to determine %s perf type: %s\n", | |
5030 | uprobe ? "uprobe" : "kprobe", | |
5031 | libbpf_strerror_r(type, errmsg, sizeof(errmsg))); | |
5032 | return type; | |
5033 | } | |
5034 | if (retprobe) { | |
5035 | int bit = uprobe ? determine_uprobe_retprobe_bit() | |
5036 | : determine_kprobe_retprobe_bit(); | |
5037 | ||
5038 | if (bit < 0) { | |
5039 | pr_warning("failed to determine %s retprobe bit: %s\n", | |
5040 | uprobe ? "uprobe" : "kprobe", | |
5041 | libbpf_strerror_r(bit, errmsg, | |
5042 | sizeof(errmsg))); | |
5043 | return bit; | |
5044 | } | |
5045 | attr.config |= 1 << bit; | |
5046 | } | |
5047 | attr.size = sizeof(attr); | |
5048 | attr.type = type; | |
36db2a94 AN |
5049 | attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ |
5050 | attr.config2 = offset; /* kprobe_addr or probe_offset */ | |
b2650027 AN |
5051 | |
5052 | /* pid filter is meaningful only for uprobes */ | |
5053 | pfd = syscall(__NR_perf_event_open, &attr, | |
5054 | pid < 0 ? -1 : pid /* pid */, | |
5055 | pid == -1 ? 0 : -1 /* cpu */, | |
5056 | -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); | |
5057 | if (pfd < 0) { | |
5058 | err = -errno; | |
5059 | pr_warning("%s perf_event_open() failed: %s\n", | |
5060 | uprobe ? "uprobe" : "kprobe", | |
5061 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
5062 | return err; | |
5063 | } | |
5064 | return pfd; | |
5065 | } | |
5066 | ||
5067 | struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, | |
5068 | bool retprobe, | |
5069 | const char *func_name) | |
5070 | { | |
5071 | char errmsg[STRERR_BUFSIZE]; | |
5072 | struct bpf_link *link; | |
5073 | int pfd, err; | |
5074 | ||
5075 | pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, | |
5076 | 0 /* offset */, -1 /* pid */); | |
5077 | if (pfd < 0) { | |
5078 | pr_warning("program '%s': failed to create %s '%s' perf event: %s\n", | |
5079 | bpf_program__title(prog, false), | |
5080 | retprobe ? "kretprobe" : "kprobe", func_name, | |
5081 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
5082 | return ERR_PTR(pfd); | |
5083 | } | |
5084 | link = bpf_program__attach_perf_event(prog, pfd); | |
5085 | if (IS_ERR(link)) { | |
5086 | close(pfd); | |
5087 | err = PTR_ERR(link); | |
5088 | pr_warning("program '%s': failed to attach to %s '%s': %s\n", | |
5089 | bpf_program__title(prog, false), | |
5090 | retprobe ? "kretprobe" : "kprobe", func_name, | |
5091 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
5092 | return link; | |
5093 | } | |
5094 | return link; | |
5095 | } | |
5096 | ||
5097 | struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, | |
5098 | bool retprobe, pid_t pid, | |
5099 | const char *binary_path, | |
5100 | size_t func_offset) | |
5101 | { | |
5102 | char errmsg[STRERR_BUFSIZE]; | |
5103 | struct bpf_link *link; | |
5104 | int pfd, err; | |
5105 | ||
5106 | pfd = perf_event_open_probe(true /* uprobe */, retprobe, | |
5107 | binary_path, func_offset, pid); | |
5108 | if (pfd < 0) { | |
5109 | pr_warning("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", | |
5110 | bpf_program__title(prog, false), | |
5111 | retprobe ? "uretprobe" : "uprobe", | |
5112 | binary_path, func_offset, | |
5113 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
5114 | return ERR_PTR(pfd); | |
5115 | } | |
5116 | link = bpf_program__attach_perf_event(prog, pfd); | |
5117 | if (IS_ERR(link)) { | |
5118 | close(pfd); | |
5119 | err = PTR_ERR(link); | |
5120 | pr_warning("program '%s': failed to attach to %s '%s:0x%zx': %s\n", | |
5121 | bpf_program__title(prog, false), | |
5122 | retprobe ? "uretprobe" : "uprobe", | |
5123 | binary_path, func_offset, | |
5124 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
5125 | return link; | |
5126 | } | |
5127 | return link; | |
5128 | } | |
5129 | ||
f6de59c1 AN |
5130 | static int determine_tracepoint_id(const char *tp_category, |
5131 | const char *tp_name) | |
5132 | { | |
5133 | char file[PATH_MAX]; | |
5134 | int ret; | |
5135 | ||
5136 | ret = snprintf(file, sizeof(file), | |
5137 | "/sys/kernel/debug/tracing/events/%s/%s/id", | |
5138 | tp_category, tp_name); | |
5139 | if (ret < 0) | |
5140 | return -errno; | |
5141 | if (ret >= sizeof(file)) { | |
5142 | pr_debug("tracepoint %s/%s path is too long\n", | |
5143 | tp_category, tp_name); | |
5144 | return -E2BIG; | |
5145 | } | |
5146 | return parse_uint_from_file(file, "%d\n"); | |
5147 | } | |
5148 | ||
5149 | static int perf_event_open_tracepoint(const char *tp_category, | |
5150 | const char *tp_name) | |
5151 | { | |
5152 | struct perf_event_attr attr = {}; | |
5153 | char errmsg[STRERR_BUFSIZE]; | |
5154 | int tp_id, pfd, err; | |
5155 | ||
5156 | tp_id = determine_tracepoint_id(tp_category, tp_name); | |
5157 | if (tp_id < 0) { | |
5158 | pr_warning("failed to determine tracepoint '%s/%s' perf event ID: %s\n", | |
5159 | tp_category, tp_name, | |
5160 | libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); | |
5161 | return tp_id; | |
5162 | } | |
5163 | ||
5164 | attr.type = PERF_TYPE_TRACEPOINT; | |
5165 | attr.size = sizeof(attr); | |
5166 | attr.config = tp_id; | |
5167 | ||
5168 | pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, | |
5169 | -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); | |
5170 | if (pfd < 0) { | |
5171 | err = -errno; | |
5172 | pr_warning("tracepoint '%s/%s' perf_event_open() failed: %s\n", | |
5173 | tp_category, tp_name, | |
5174 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
5175 | return err; | |
5176 | } | |
5177 | return pfd; | |
5178 | } | |
5179 | ||
5180 | struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, | |
5181 | const char *tp_category, | |
5182 | const char *tp_name) | |
5183 | { | |
5184 | char errmsg[STRERR_BUFSIZE]; | |
5185 | struct bpf_link *link; | |
5186 | int pfd, err; | |
5187 | ||
5188 | pfd = perf_event_open_tracepoint(tp_category, tp_name); | |
5189 | if (pfd < 0) { | |
5190 | pr_warning("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", | |
5191 | bpf_program__title(prog, false), | |
5192 | tp_category, tp_name, | |
5193 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
5194 | return ERR_PTR(pfd); | |
5195 | } | |
5196 | link = bpf_program__attach_perf_event(prog, pfd); | |
5197 | if (IS_ERR(link)) { | |
5198 | close(pfd); | |
5199 | err = PTR_ERR(link); | |
5200 | pr_warning("program '%s': failed to attach to tracepoint '%s/%s': %s\n", | |
5201 | bpf_program__title(prog, false), | |
5202 | tp_category, tp_name, | |
5203 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
5204 | return link; | |
5205 | } | |
5206 | return link; | |
5207 | } | |
5208 | ||
84bf5e1f AN |
5209 | static int bpf_link__destroy_fd(struct bpf_link *link) |
5210 | { | |
5211 | struct bpf_link_fd *l = (void *)link; | |
5212 | ||
5213 | return close(l->fd); | |
5214 | } | |
5215 | ||
5216 | struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, | |
5217 | const char *tp_name) | |
5218 | { | |
5219 | char errmsg[STRERR_BUFSIZE]; | |
5220 | struct bpf_link_fd *link; | |
5221 | int prog_fd, pfd; | |
5222 | ||
5223 | prog_fd = bpf_program__fd(prog); | |
5224 | if (prog_fd < 0) { | |
5225 | pr_warning("program '%s': can't attach before loaded\n", | |
5226 | bpf_program__title(prog, false)); | |
5227 | return ERR_PTR(-EINVAL); | |
5228 | } | |
5229 | ||
5230 | link = malloc(sizeof(*link)); | |
5231 | if (!link) | |
5232 | return ERR_PTR(-ENOMEM); | |
5233 | link->link.destroy = &bpf_link__destroy_fd; | |
5234 | ||
5235 | pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); | |
5236 | if (pfd < 0) { | |
5237 | pfd = -errno; | |
5238 | free(link); | |
5239 | pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n", | |
5240 | bpf_program__title(prog, false), tp_name, | |
5241 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
5242 | return ERR_PTR(pfd); | |
5243 | } | |
5244 | link->fd = pfd; | |
5245 | return (struct bpf_link *)link; | |
5246 | } | |
5247 | ||
d0cabbb0 | 5248 | enum bpf_perf_event_ret |
3dca2115 DB |
5249 | bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, |
5250 | void **copy_mem, size_t *copy_size, | |
5251 | bpf_perf_event_print_t fn, void *private_data) | |
d0cabbb0 | 5252 | { |
3dca2115 | 5253 | struct perf_event_mmap_page *header = mmap_mem; |
a64af0ef | 5254 | __u64 data_head = ring_buffer_read_head(header); |
d0cabbb0 | 5255 | __u64 data_tail = header->data_tail; |
3dca2115 DB |
5256 | void *base = ((__u8 *)header) + page_size; |
5257 | int ret = LIBBPF_PERF_EVENT_CONT; | |
5258 | struct perf_event_header *ehdr; | |
5259 | size_t ehdr_size; | |
5260 | ||
5261 | while (data_head != data_tail) { | |
5262 | ehdr = base + (data_tail & (mmap_size - 1)); | |
5263 | ehdr_size = ehdr->size; | |
5264 | ||
5265 | if (((void *)ehdr) + ehdr_size > base + mmap_size) { | |
5266 | void *copy_start = ehdr; | |
5267 | size_t len_first = base + mmap_size - copy_start; | |
5268 | size_t len_secnd = ehdr_size - len_first; | |
5269 | ||
5270 | if (*copy_size < ehdr_size) { | |
5271 | free(*copy_mem); | |
5272 | *copy_mem = malloc(ehdr_size); | |
5273 | if (!*copy_mem) { | |
5274 | *copy_size = 0; | |
d0cabbb0 JK |
5275 | ret = LIBBPF_PERF_EVENT_ERROR; |
5276 | break; | |
5277 | } | |
3dca2115 | 5278 | *copy_size = ehdr_size; |
d0cabbb0 JK |
5279 | } |
5280 | ||
3dca2115 DB |
5281 | memcpy(*copy_mem, copy_start, len_first); |
5282 | memcpy(*copy_mem + len_first, base, len_secnd); | |
5283 | ehdr = *copy_mem; | |
d0cabbb0 JK |
5284 | } |
5285 | ||
3dca2115 DB |
5286 | ret = fn(ehdr, private_data); |
5287 | data_tail += ehdr_size; | |
d0cabbb0 JK |
5288 | if (ret != LIBBPF_PERF_EVENT_CONT) |
5289 | break; | |
d0cabbb0 JK |
5290 | } |
5291 | ||
a64af0ef | 5292 | ring_buffer_write_tail(header, data_tail); |
d0cabbb0 JK |
5293 | return ret; |
5294 | } | |
34be1646 | 5295 | |
fb84b822 AN |
5296 | struct perf_buffer; |
5297 | ||
5298 | struct perf_buffer_params { | |
5299 | struct perf_event_attr *attr; | |
5300 | /* if event_cb is specified, it takes precendence */ | |
5301 | perf_buffer_event_fn event_cb; | |
5302 | /* sample_cb and lost_cb are higher-level common-case callbacks */ | |
5303 | perf_buffer_sample_fn sample_cb; | |
5304 | perf_buffer_lost_fn lost_cb; | |
5305 | void *ctx; | |
5306 | int cpu_cnt; | |
5307 | int *cpus; | |
5308 | int *map_keys; | |
5309 | }; | |
5310 | ||
5311 | struct perf_cpu_buf { | |
5312 | struct perf_buffer *pb; | |
5313 | void *base; /* mmap()'ed memory */ | |
5314 | void *buf; /* for reconstructing segmented data */ | |
5315 | size_t buf_size; | |
5316 | int fd; | |
5317 | int cpu; | |
5318 | int map_key; | |
5319 | }; | |
5320 | ||
5321 | struct perf_buffer { | |
5322 | perf_buffer_event_fn event_cb; | |
5323 | perf_buffer_sample_fn sample_cb; | |
5324 | perf_buffer_lost_fn lost_cb; | |
5325 | void *ctx; /* passed into callbacks */ | |
5326 | ||
5327 | size_t page_size; | |
5328 | size_t mmap_size; | |
5329 | struct perf_cpu_buf **cpu_bufs; | |
5330 | struct epoll_event *events; | |
5331 | int cpu_cnt; | |
5332 | int epoll_fd; /* perf event FD */ | |
5333 | int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ | |
5334 | }; | |
5335 | ||
5336 | static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, | |
5337 | struct perf_cpu_buf *cpu_buf) | |
5338 | { | |
5339 | if (!cpu_buf) | |
5340 | return; | |
5341 | if (cpu_buf->base && | |
5342 | munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) | |
5343 | pr_warning("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); | |
5344 | if (cpu_buf->fd >= 0) { | |
5345 | ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); | |
5346 | close(cpu_buf->fd); | |
5347 | } | |
5348 | free(cpu_buf->buf); | |
5349 | free(cpu_buf); | |
5350 | } | |
5351 | ||
5352 | void perf_buffer__free(struct perf_buffer *pb) | |
5353 | { | |
5354 | int i; | |
5355 | ||
5356 | if (!pb) | |
5357 | return; | |
5358 | if (pb->cpu_bufs) { | |
5359 | for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) { | |
5360 | struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; | |
5361 | ||
5362 | bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); | |
5363 | perf_buffer__free_cpu_buf(pb, cpu_buf); | |
5364 | } | |
5365 | free(pb->cpu_bufs); | |
5366 | } | |
5367 | if (pb->epoll_fd >= 0) | |
5368 | close(pb->epoll_fd); | |
5369 | free(pb->events); | |
5370 | free(pb); | |
5371 | } | |
5372 | ||
5373 | static struct perf_cpu_buf * | |
5374 | perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, | |
5375 | int cpu, int map_key) | |
5376 | { | |
5377 | struct perf_cpu_buf *cpu_buf; | |
5378 | char msg[STRERR_BUFSIZE]; | |
5379 | int err; | |
5380 | ||
5381 | cpu_buf = calloc(1, sizeof(*cpu_buf)); | |
5382 | if (!cpu_buf) | |
5383 | return ERR_PTR(-ENOMEM); | |
5384 | ||
5385 | cpu_buf->pb = pb; | |
5386 | cpu_buf->cpu = cpu; | |
5387 | cpu_buf->map_key = map_key; | |
5388 | ||
5389 | cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, | |
5390 | -1, PERF_FLAG_FD_CLOEXEC); | |
5391 | if (cpu_buf->fd < 0) { | |
5392 | err = -errno; | |
5393 | pr_warning("failed to open perf buffer event on cpu #%d: %s\n", | |
5394 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
5395 | goto error; | |
5396 | } | |
5397 | ||
5398 | cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, | |
5399 | PROT_READ | PROT_WRITE, MAP_SHARED, | |
5400 | cpu_buf->fd, 0); | |
5401 | if (cpu_buf->base == MAP_FAILED) { | |
5402 | cpu_buf->base = NULL; | |
5403 | err = -errno; | |
5404 | pr_warning("failed to mmap perf buffer on cpu #%d: %s\n", | |
5405 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
5406 | goto error; | |
5407 | } | |
5408 | ||
5409 | if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { | |
5410 | err = -errno; | |
5411 | pr_warning("failed to enable perf buffer event on cpu #%d: %s\n", | |
5412 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
5413 | goto error; | |
5414 | } | |
5415 | ||
5416 | return cpu_buf; | |
5417 | ||
5418 | error: | |
5419 | perf_buffer__free_cpu_buf(pb, cpu_buf); | |
5420 | return (struct perf_cpu_buf *)ERR_PTR(err); | |
5421 | } | |
5422 | ||
5423 | static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, | |
5424 | struct perf_buffer_params *p); | |
5425 | ||
5426 | struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, | |
5427 | const struct perf_buffer_opts *opts) | |
5428 | { | |
5429 | struct perf_buffer_params p = {}; | |
4be6e05c ACM |
5430 | struct perf_event_attr attr = { 0, }; |
5431 | ||
5432 | attr.config = PERF_COUNT_SW_BPF_OUTPUT, | |
5433 | attr.type = PERF_TYPE_SOFTWARE; | |
5434 | attr.sample_type = PERF_SAMPLE_RAW; | |
5435 | attr.sample_period = 1; | |
5436 | attr.wakeup_events = 1; | |
fb84b822 AN |
5437 | |
5438 | p.attr = &attr; | |
5439 | p.sample_cb = opts ? opts->sample_cb : NULL; | |
5440 | p.lost_cb = opts ? opts->lost_cb : NULL; | |
5441 | p.ctx = opts ? opts->ctx : NULL; | |
5442 | ||
5443 | return __perf_buffer__new(map_fd, page_cnt, &p); | |
5444 | } | |
5445 | ||
5446 | struct perf_buffer * | |
5447 | perf_buffer__new_raw(int map_fd, size_t page_cnt, | |
5448 | const struct perf_buffer_raw_opts *opts) | |
5449 | { | |
5450 | struct perf_buffer_params p = {}; | |
5451 | ||
5452 | p.attr = opts->attr; | |
5453 | p.event_cb = opts->event_cb; | |
5454 | p.ctx = opts->ctx; | |
5455 | p.cpu_cnt = opts->cpu_cnt; | |
5456 | p.cpus = opts->cpus; | |
5457 | p.map_keys = opts->map_keys; | |
5458 | ||
5459 | return __perf_buffer__new(map_fd, page_cnt, &p); | |
5460 | } | |
5461 | ||
5462 | static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, | |
5463 | struct perf_buffer_params *p) | |
5464 | { | |
5465 | struct bpf_map_info map = {}; | |
5466 | char msg[STRERR_BUFSIZE]; | |
5467 | struct perf_buffer *pb; | |
5468 | __u32 map_info_len; | |
5469 | int err, i; | |
5470 | ||
5471 | if (page_cnt & (page_cnt - 1)) { | |
5472 | pr_warning("page count should be power of two, but is %zu\n", | |
5473 | page_cnt); | |
5474 | return ERR_PTR(-EINVAL); | |
5475 | } | |
5476 | ||
5477 | map_info_len = sizeof(map); | |
5478 | err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); | |
5479 | if (err) { | |
5480 | err = -errno; | |
5481 | pr_warning("failed to get map info for map FD %d: %s\n", | |
5482 | map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); | |
5483 | return ERR_PTR(err); | |
5484 | } | |
5485 | ||
5486 | if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { | |
5487 | pr_warning("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", | |
5488 | map.name); | |
5489 | return ERR_PTR(-EINVAL); | |
5490 | } | |
5491 | ||
5492 | pb = calloc(1, sizeof(*pb)); | |
5493 | if (!pb) | |
5494 | return ERR_PTR(-ENOMEM); | |
5495 | ||
5496 | pb->event_cb = p->event_cb; | |
5497 | pb->sample_cb = p->sample_cb; | |
5498 | pb->lost_cb = p->lost_cb; | |
5499 | pb->ctx = p->ctx; | |
5500 | ||
5501 | pb->page_size = getpagesize(); | |
5502 | pb->mmap_size = pb->page_size * page_cnt; | |
5503 | pb->map_fd = map_fd; | |
5504 | ||
5505 | pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); | |
5506 | if (pb->epoll_fd < 0) { | |
5507 | err = -errno; | |
5508 | pr_warning("failed to create epoll instance: %s\n", | |
5509 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
5510 | goto error; | |
5511 | } | |
5512 | ||
5513 | if (p->cpu_cnt > 0) { | |
5514 | pb->cpu_cnt = p->cpu_cnt; | |
5515 | } else { | |
5516 | pb->cpu_cnt = libbpf_num_possible_cpus(); | |
5517 | if (pb->cpu_cnt < 0) { | |
5518 | err = pb->cpu_cnt; | |
5519 | goto error; | |
5520 | } | |
5521 | if (map.max_entries < pb->cpu_cnt) | |
5522 | pb->cpu_cnt = map.max_entries; | |
5523 | } | |
5524 | ||
5525 | pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); | |
5526 | if (!pb->events) { | |
5527 | err = -ENOMEM; | |
5528 | pr_warning("failed to allocate events: out of memory\n"); | |
5529 | goto error; | |
5530 | } | |
5531 | pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); | |
5532 | if (!pb->cpu_bufs) { | |
5533 | err = -ENOMEM; | |
5534 | pr_warning("failed to allocate buffers: out of memory\n"); | |
5535 | goto error; | |
5536 | } | |
5537 | ||
5538 | for (i = 0; i < pb->cpu_cnt; i++) { | |
5539 | struct perf_cpu_buf *cpu_buf; | |
5540 | int cpu, map_key; | |
5541 | ||
5542 | cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; | |
5543 | map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; | |
5544 | ||
5545 | cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); | |
5546 | if (IS_ERR(cpu_buf)) { | |
5547 | err = PTR_ERR(cpu_buf); | |
5548 | goto error; | |
5549 | } | |
5550 | ||
5551 | pb->cpu_bufs[i] = cpu_buf; | |
5552 | ||
5553 | err = bpf_map_update_elem(pb->map_fd, &map_key, | |
5554 | &cpu_buf->fd, 0); | |
5555 | if (err) { | |
5556 | err = -errno; | |
5557 | pr_warning("failed to set cpu #%d, key %d -> perf FD %d: %s\n", | |
5558 | cpu, map_key, cpu_buf->fd, | |
5559 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
5560 | goto error; | |
5561 | } | |
5562 | ||
5563 | pb->events[i].events = EPOLLIN; | |
5564 | pb->events[i].data.ptr = cpu_buf; | |
5565 | if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, | |
5566 | &pb->events[i]) < 0) { | |
5567 | err = -errno; | |
5568 | pr_warning("failed to epoll_ctl cpu #%d perf FD %d: %s\n", | |
5569 | cpu, cpu_buf->fd, | |
5570 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
5571 | goto error; | |
5572 | } | |
5573 | } | |
5574 | ||
5575 | return pb; | |
5576 | ||
5577 | error: | |
5578 | if (pb) | |
5579 | perf_buffer__free(pb); | |
5580 | return ERR_PTR(err); | |
5581 | } | |
5582 | ||
5583 | struct perf_sample_raw { | |
5584 | struct perf_event_header header; | |
5585 | uint32_t size; | |
5586 | char data[0]; | |
5587 | }; | |
5588 | ||
5589 | struct perf_sample_lost { | |
5590 | struct perf_event_header header; | |
5591 | uint64_t id; | |
5592 | uint64_t lost; | |
5593 | uint64_t sample_id; | |
5594 | }; | |
5595 | ||
5596 | static enum bpf_perf_event_ret | |
5597 | perf_buffer__process_record(struct perf_event_header *e, void *ctx) | |
5598 | { | |
5599 | struct perf_cpu_buf *cpu_buf = ctx; | |
5600 | struct perf_buffer *pb = cpu_buf->pb; | |
5601 | void *data = e; | |
5602 | ||
5603 | /* user wants full control over parsing perf event */ | |
5604 | if (pb->event_cb) | |
5605 | return pb->event_cb(pb->ctx, cpu_buf->cpu, e); | |
5606 | ||
5607 | switch (e->type) { | |
5608 | case PERF_RECORD_SAMPLE: { | |
5609 | struct perf_sample_raw *s = data; | |
5610 | ||
5611 | if (pb->sample_cb) | |
5612 | pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); | |
5613 | break; | |
5614 | } | |
5615 | case PERF_RECORD_LOST: { | |
5616 | struct perf_sample_lost *s = data; | |
5617 | ||
5618 | if (pb->lost_cb) | |
5619 | pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); | |
5620 | break; | |
5621 | } | |
5622 | default: | |
5623 | pr_warning("unknown perf sample type %d\n", e->type); | |
5624 | return LIBBPF_PERF_EVENT_ERROR; | |
5625 | } | |
5626 | return LIBBPF_PERF_EVENT_CONT; | |
5627 | } | |
5628 | ||
5629 | static int perf_buffer__process_records(struct perf_buffer *pb, | |
5630 | struct perf_cpu_buf *cpu_buf) | |
5631 | { | |
5632 | enum bpf_perf_event_ret ret; | |
5633 | ||
5634 | ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, | |
5635 | pb->page_size, &cpu_buf->buf, | |
5636 | &cpu_buf->buf_size, | |
5637 | perf_buffer__process_record, cpu_buf); | |
5638 | if (ret != LIBBPF_PERF_EVENT_CONT) | |
5639 | return ret; | |
5640 | return 0; | |
5641 | } | |
5642 | ||
5643 | int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) | |
5644 | { | |
5645 | int i, cnt, err; | |
5646 | ||
5647 | cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); | |
5648 | for (i = 0; i < cnt; i++) { | |
5649 | struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; | |
5650 | ||
5651 | err = perf_buffer__process_records(pb, cpu_buf); | |
5652 | if (err) { | |
5653 | pr_warning("error while processing records: %d\n", err); | |
5654 | return err; | |
5655 | } | |
5656 | } | |
5657 | return cnt < 0 ? -errno : cnt; | |
5658 | } | |
5659 | ||
34be1646 SL |
5660 | struct bpf_prog_info_array_desc { |
5661 | int array_offset; /* e.g. offset of jited_prog_insns */ | |
5662 | int count_offset; /* e.g. offset of jited_prog_len */ | |
5663 | int size_offset; /* > 0: offset of rec size, | |
5664 | * < 0: fix size of -size_offset | |
5665 | */ | |
5666 | }; | |
5667 | ||
5668 | static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { | |
5669 | [BPF_PROG_INFO_JITED_INSNS] = { | |
5670 | offsetof(struct bpf_prog_info, jited_prog_insns), | |
5671 | offsetof(struct bpf_prog_info, jited_prog_len), | |
5672 | -1, | |
5673 | }, | |
5674 | [BPF_PROG_INFO_XLATED_INSNS] = { | |
5675 | offsetof(struct bpf_prog_info, xlated_prog_insns), | |
5676 | offsetof(struct bpf_prog_info, xlated_prog_len), | |
5677 | -1, | |
5678 | }, | |
5679 | [BPF_PROG_INFO_MAP_IDS] = { | |
5680 | offsetof(struct bpf_prog_info, map_ids), | |
5681 | offsetof(struct bpf_prog_info, nr_map_ids), | |
5682 | -(int)sizeof(__u32), | |
5683 | }, | |
5684 | [BPF_PROG_INFO_JITED_KSYMS] = { | |
5685 | offsetof(struct bpf_prog_info, jited_ksyms), | |
5686 | offsetof(struct bpf_prog_info, nr_jited_ksyms), | |
5687 | -(int)sizeof(__u64), | |
5688 | }, | |
5689 | [BPF_PROG_INFO_JITED_FUNC_LENS] = { | |
5690 | offsetof(struct bpf_prog_info, jited_func_lens), | |
5691 | offsetof(struct bpf_prog_info, nr_jited_func_lens), | |
5692 | -(int)sizeof(__u32), | |
5693 | }, | |
5694 | [BPF_PROG_INFO_FUNC_INFO] = { | |
5695 | offsetof(struct bpf_prog_info, func_info), | |
5696 | offsetof(struct bpf_prog_info, nr_func_info), | |
5697 | offsetof(struct bpf_prog_info, func_info_rec_size), | |
5698 | }, | |
5699 | [BPF_PROG_INFO_LINE_INFO] = { | |
5700 | offsetof(struct bpf_prog_info, line_info), | |
5701 | offsetof(struct bpf_prog_info, nr_line_info), | |
5702 | offsetof(struct bpf_prog_info, line_info_rec_size), | |
5703 | }, | |
5704 | [BPF_PROG_INFO_JITED_LINE_INFO] = { | |
5705 | offsetof(struct bpf_prog_info, jited_line_info), | |
5706 | offsetof(struct bpf_prog_info, nr_jited_line_info), | |
5707 | offsetof(struct bpf_prog_info, jited_line_info_rec_size), | |
5708 | }, | |
5709 | [BPF_PROG_INFO_PROG_TAGS] = { | |
5710 | offsetof(struct bpf_prog_info, prog_tags), | |
5711 | offsetof(struct bpf_prog_info, nr_prog_tags), | |
5712 | -(int)sizeof(__u8) * BPF_TAG_SIZE, | |
5713 | }, | |
5714 | ||
5715 | }; | |
5716 | ||
5717 | static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) | |
5718 | { | |
5719 | __u32 *array = (__u32 *)info; | |
5720 | ||
5721 | if (offset >= 0) | |
5722 | return array[offset / sizeof(__u32)]; | |
5723 | return -(int)offset; | |
5724 | } | |
5725 | ||
5726 | static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) | |
5727 | { | |
5728 | __u64 *array = (__u64 *)info; | |
5729 | ||
5730 | if (offset >= 0) | |
5731 | return array[offset / sizeof(__u64)]; | |
5732 | return -(int)offset; | |
5733 | } | |
5734 | ||
5735 | static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, | |
5736 | __u32 val) | |
5737 | { | |
5738 | __u32 *array = (__u32 *)info; | |
5739 | ||
5740 | if (offset >= 0) | |
5741 | array[offset / sizeof(__u32)] = val; | |
5742 | } | |
5743 | ||
5744 | static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, | |
5745 | __u64 val) | |
5746 | { | |
5747 | __u64 *array = (__u64 *)info; | |
5748 | ||
5749 | if (offset >= 0) | |
5750 | array[offset / sizeof(__u64)] = val; | |
5751 | } | |
5752 | ||
5753 | struct bpf_prog_info_linear * | |
5754 | bpf_program__get_prog_info_linear(int fd, __u64 arrays) | |
5755 | { | |
5756 | struct bpf_prog_info_linear *info_linear; | |
5757 | struct bpf_prog_info info = {}; | |
5758 | __u32 info_len = sizeof(info); | |
5759 | __u32 data_len = 0; | |
5760 | int i, err; | |
5761 | void *ptr; | |
5762 | ||
5763 | if (arrays >> BPF_PROG_INFO_LAST_ARRAY) | |
5764 | return ERR_PTR(-EINVAL); | |
5765 | ||
5766 | /* step 1: get array dimensions */ | |
5767 | err = bpf_obj_get_info_by_fd(fd, &info, &info_len); | |
5768 | if (err) { | |
5769 | pr_debug("can't get prog info: %s", strerror(errno)); | |
5770 | return ERR_PTR(-EFAULT); | |
5771 | } | |
5772 | ||
5773 | /* step 2: calculate total size of all arrays */ | |
5774 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5775 | bool include_array = (arrays & (1UL << i)) > 0; | |
5776 | struct bpf_prog_info_array_desc *desc; | |
5777 | __u32 count, size; | |
5778 | ||
5779 | desc = bpf_prog_info_array_desc + i; | |
5780 | ||
5781 | /* kernel is too old to support this field */ | |
5782 | if (info_len < desc->array_offset + sizeof(__u32) || | |
5783 | info_len < desc->count_offset + sizeof(__u32) || | |
5784 | (desc->size_offset > 0 && info_len < desc->size_offset)) | |
5785 | include_array = false; | |
5786 | ||
5787 | if (!include_array) { | |
5788 | arrays &= ~(1UL << i); /* clear the bit */ | |
5789 | continue; | |
5790 | } | |
5791 | ||
5792 | count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
5793 | size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
5794 | ||
5795 | data_len += count * size; | |
5796 | } | |
5797 | ||
5798 | /* step 3: allocate continuous memory */ | |
5799 | data_len = roundup(data_len, sizeof(__u64)); | |
5800 | info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); | |
5801 | if (!info_linear) | |
5802 | return ERR_PTR(-ENOMEM); | |
5803 | ||
5804 | /* step 4: fill data to info_linear->info */ | |
5805 | info_linear->arrays = arrays; | |
5806 | memset(&info_linear->info, 0, sizeof(info)); | |
5807 | ptr = info_linear->data; | |
5808 | ||
5809 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5810 | struct bpf_prog_info_array_desc *desc; | |
5811 | __u32 count, size; | |
5812 | ||
5813 | if ((arrays & (1UL << i)) == 0) | |
5814 | continue; | |
5815 | ||
5816 | desc = bpf_prog_info_array_desc + i; | |
5817 | count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
5818 | size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
5819 | bpf_prog_info_set_offset_u32(&info_linear->info, | |
5820 | desc->count_offset, count); | |
5821 | bpf_prog_info_set_offset_u32(&info_linear->info, | |
5822 | desc->size_offset, size); | |
5823 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
5824 | desc->array_offset, | |
5825 | ptr_to_u64(ptr)); | |
5826 | ptr += count * size; | |
5827 | } | |
5828 | ||
5829 | /* step 5: call syscall again to get required arrays */ | |
5830 | err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); | |
5831 | if (err) { | |
5832 | pr_debug("can't get prog info: %s", strerror(errno)); | |
5833 | free(info_linear); | |
5834 | return ERR_PTR(-EFAULT); | |
5835 | } | |
5836 | ||
5837 | /* step 6: verify the data */ | |
5838 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5839 | struct bpf_prog_info_array_desc *desc; | |
5840 | __u32 v1, v2; | |
5841 | ||
5842 | if ((arrays & (1UL << i)) == 0) | |
5843 | continue; | |
5844 | ||
5845 | desc = bpf_prog_info_array_desc + i; | |
5846 | v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
5847 | v2 = bpf_prog_info_read_offset_u32(&info_linear->info, | |
5848 | desc->count_offset); | |
5849 | if (v1 != v2) | |
5850 | pr_warning("%s: mismatch in element count\n", __func__); | |
5851 | ||
5852 | v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
5853 | v2 = bpf_prog_info_read_offset_u32(&info_linear->info, | |
5854 | desc->size_offset); | |
5855 | if (v1 != v2) | |
5856 | pr_warning("%s: mismatch in rec size\n", __func__); | |
5857 | } | |
5858 | ||
5859 | /* step 7: update info_len and data_len */ | |
5860 | info_linear->info_len = sizeof(struct bpf_prog_info); | |
5861 | info_linear->data_len = data_len; | |
5862 | ||
5863 | return info_linear; | |
5864 | } | |
5865 | ||
5866 | void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) | |
5867 | { | |
5868 | int i; | |
5869 | ||
5870 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5871 | struct bpf_prog_info_array_desc *desc; | |
5872 | __u64 addr, offs; | |
5873 | ||
5874 | if ((info_linear->arrays & (1UL << i)) == 0) | |
5875 | continue; | |
5876 | ||
5877 | desc = bpf_prog_info_array_desc + i; | |
5878 | addr = bpf_prog_info_read_offset_u64(&info_linear->info, | |
5879 | desc->array_offset); | |
5880 | offs = addr - ptr_to_u64(info_linear->data); | |
5881 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
5882 | desc->array_offset, offs); | |
5883 | } | |
5884 | } | |
5885 | ||
5886 | void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) | |
5887 | { | |
5888 | int i; | |
5889 | ||
5890 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5891 | struct bpf_prog_info_array_desc *desc; | |
5892 | __u64 addr, offs; | |
5893 | ||
5894 | if ((info_linear->arrays & (1UL << i)) == 0) | |
5895 | continue; | |
5896 | ||
5897 | desc = bpf_prog_info_array_desc + i; | |
5898 | offs = bpf_prog_info_read_offset_u64(&info_linear->info, | |
5899 | desc->array_offset); | |
5900 | addr = offs + ptr_to_u64(info_linear->data); | |
5901 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
5902 | desc->array_offset, addr); | |
5903 | } | |
5904 | } | |
6446b315 HL |
5905 | |
5906 | int libbpf_num_possible_cpus(void) | |
5907 | { | |
5908 | static const char *fcpu = "/sys/devices/system/cpu/possible"; | |
5909 | int len = 0, n = 0, il = 0, ir = 0; | |
5910 | unsigned int start = 0, end = 0; | |
56fbc241 | 5911 | int tmp_cpus = 0; |
6446b315 HL |
5912 | static int cpus; |
5913 | char buf[128]; | |
5914 | int error = 0; | |
5915 | int fd = -1; | |
5916 | ||
56fbc241 TC |
5917 | tmp_cpus = READ_ONCE(cpus); |
5918 | if (tmp_cpus > 0) | |
5919 | return tmp_cpus; | |
6446b315 HL |
5920 | |
5921 | fd = open(fcpu, O_RDONLY); | |
5922 | if (fd < 0) { | |
5923 | error = errno; | |
5924 | pr_warning("Failed to open file %s: %s\n", | |
5925 | fcpu, strerror(error)); | |
5926 | return -error; | |
5927 | } | |
5928 | len = read(fd, buf, sizeof(buf)); | |
5929 | close(fd); | |
5930 | if (len <= 0) { | |
5931 | error = len ? errno : EINVAL; | |
5932 | pr_warning("Failed to read # of possible cpus from %s: %s\n", | |
5933 | fcpu, strerror(error)); | |
5934 | return -error; | |
5935 | } | |
5936 | if (len == sizeof(buf)) { | |
5937 | pr_warning("File %s size overflow\n", fcpu); | |
5938 | return -EOVERFLOW; | |
5939 | } | |
5940 | buf[len] = '\0'; | |
5941 | ||
56fbc241 | 5942 | for (ir = 0, tmp_cpus = 0; ir <= len; ir++) { |
6446b315 HL |
5943 | /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ |
5944 | if (buf[ir] == ',' || buf[ir] == '\0') { | |
5945 | buf[ir] = '\0'; | |
5946 | n = sscanf(&buf[il], "%u-%u", &start, &end); | |
5947 | if (n <= 0) { | |
5948 | pr_warning("Failed to get # CPUs from %s\n", | |
5949 | &buf[il]); | |
5950 | return -EINVAL; | |
5951 | } else if (n == 1) { | |
5952 | end = start; | |
5953 | } | |
56fbc241 | 5954 | tmp_cpus += end - start + 1; |
6446b315 HL |
5955 | il = ir + 1; |
5956 | } | |
5957 | } | |
56fbc241 TC |
5958 | if (tmp_cpus <= 0) { |
5959 | pr_warning("Invalid #CPUs %d from %s\n", tmp_cpus, fcpu); | |
6446b315 HL |
5960 | return -EINVAL; |
5961 | } | |
56fbc241 TC |
5962 | |
5963 | WRITE_ONCE(cpus, tmp_cpus); | |
5964 | return tmp_cpus; | |
6446b315 | 5965 | } |