1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
6 * Copyright (C) 2015 Huawei Inc.
10 #include <bpf/libbpf.h>
12 #include <linux/err.h>
13 #include <linux/kernel.h>
14 #include <linux/string.h>
15 #include <linux/zalloc.h>
20 #include "bpf-loader.h"
21 #include "bpf-prologue.h"
22 #include "probe-event.h"
23 #include "probe-finder.h" // for MAX_PROBES
24 #include "parse-events.h"
25 #include "strfilter.h"
26 #include "llvm-utils.h"
27 #include "c++/clang-c.h"
29 static int libbpf_perf_print(enum libbpf_print_level level
__attribute__((unused
)),
30 const char *fmt
, va_list args
)
32 return veprintf(1, verbose
, pr_fmt(fmt
), args
);
35 struct bpf_prog_priv
{
39 struct perf_probe_event pev
;
41 struct bpf_insn
*insns_buf
;
46 static bool libbpf_initialized
;
49 bpf__prepare_load_buffer(void *obj_buf
, size_t obj_buf_sz
, const char *name
)
51 struct bpf_object
*obj
;
53 if (!libbpf_initialized
) {
54 libbpf_set_print(libbpf_perf_print
);
55 libbpf_initialized
= true;
58 obj
= bpf_object__open_buffer(obj_buf
, obj_buf_sz
, name
);
59 if (IS_ERR_OR_NULL(obj
)) {
60 pr_debug("bpf: failed to load buffer\n");
61 return ERR_PTR(-EINVAL
);
67 struct bpf_object
*bpf__prepare_load(const char *filename
, bool source
)
69 struct bpf_object
*obj
;
71 if (!libbpf_initialized
) {
72 libbpf_set_print(libbpf_perf_print
);
73 libbpf_initialized
= true;
82 err
= perf_clang__compile_bpf(filename
, &obj_buf
, &obj_buf_sz
);
83 perf_clang__cleanup();
85 pr_debug("bpf: builtin compilation failed: %d, try external compiler\n", err
);
86 err
= llvm__compile_bpf(filename
, &obj_buf
, &obj_buf_sz
);
88 return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE
);
90 pr_debug("bpf: successful builtin compilation\n");
91 obj
= bpf_object__open_buffer(obj_buf
, obj_buf_sz
, filename
);
93 if (!IS_ERR_OR_NULL(obj
) && llvm_param
.dump_obj
)
94 llvm__dump_obj(filename
, obj_buf
, obj_buf_sz
);
98 obj
= bpf_object__open(filename
);
100 if (IS_ERR_OR_NULL(obj
)) {
101 pr_debug("bpf: failed to load %s\n", filename
);
108 void bpf__clear(void)
110 struct bpf_object
*obj
, *tmp
;
112 bpf_object__for_each_safe(obj
, tmp
) {
114 bpf_object__close(obj
);
119 clear_prog_priv(struct bpf_program
*prog __maybe_unused
,
122 struct bpf_prog_priv
*priv
= _priv
;
124 cleanup_perf_probe_events(&priv
->pev
, 1);
125 zfree(&priv
->insns_buf
);
126 zfree(&priv
->type_mapping
);
127 zfree(&priv
->sys_name
);
128 zfree(&priv
->evt_name
);
133 prog_config__exec(const char *value
, struct perf_probe_event
*pev
)
136 pev
->target
= strdup(value
);
143 prog_config__module(const char *value
, struct perf_probe_event
*pev
)
145 pev
->uprobes
= false;
146 pev
->target
= strdup(value
);
153 prog_config__bool(const char *value
, bool *pbool
, bool invert
)
161 err
= strtobool(value
, &bool_value
);
165 *pbool
= invert
? !bool_value
: bool_value
;
170 prog_config__inlines(const char *value
,
171 struct perf_probe_event
*pev __maybe_unused
)
173 return prog_config__bool(value
, &probe_conf
.no_inlines
, true);
177 prog_config__force(const char *value
,
178 struct perf_probe_event
*pev __maybe_unused
)
180 return prog_config__bool(value
, &probe_conf
.force_add
, false);
187 int (*func
)(const char *, struct perf_probe_event
*);
188 } bpf_prog_config_terms
[] = {
191 .usage
= "exec=<full path of file>",
192 .desc
= "Set uprobe target",
193 .func
= prog_config__exec
,
197 .usage
= "module=<module name> ",
198 .desc
= "Set kprobe module",
199 .func
= prog_config__module
,
203 .usage
= "inlines=[yes|no] ",
204 .desc
= "Probe at inline symbol",
205 .func
= prog_config__inlines
,
209 .usage
= "force=[yes|no] ",
210 .desc
= "Forcibly add events with existing name",
211 .func
= prog_config__force
,
216 do_prog_config(const char *key
, const char *value
,
217 struct perf_probe_event
*pev
)
221 pr_debug("config bpf program: %s=%s\n", key
, value
);
222 for (i
= 0; i
< ARRAY_SIZE(bpf_prog_config_terms
); i
++)
223 if (strcmp(key
, bpf_prog_config_terms
[i
].key
) == 0)
224 return bpf_prog_config_terms
[i
].func(value
, pev
);
226 pr_debug("BPF: ERROR: invalid program config option: %s=%s\n",
229 pr_debug("\nHint: Valid options are:\n");
230 for (i
= 0; i
< ARRAY_SIZE(bpf_prog_config_terms
); i
++)
231 pr_debug("\t%s:\t%s\n", bpf_prog_config_terms
[i
].usage
,
232 bpf_prog_config_terms
[i
].desc
);
235 return -BPF_LOADER_ERRNO__PROGCONF_TERM
;
239 parse_prog_config_kvpair(const char *config_str
, struct perf_probe_event
*pev
)
241 char *text
= strdup(config_str
);
243 const char *main_str
= NULL
;
247 pr_debug("Not enough memory: dup config_str failed\n");
248 return ERR_PTR(-ENOMEM
);
252 while ((sep
= strchr(line
, ';'))) {
256 equ
= strchr(line
, '=');
258 pr_warning("WARNING: invalid config in BPF object: %s\n",
260 pr_warning("\tShould be 'key=value'.\n");
265 err
= do_prog_config(line
, equ
+ 1, pev
);
273 main_str
= config_str
+ (line
- text
);
276 return err
? ERR_PTR(err
) : main_str
;
280 parse_prog_config(const char *config_str
, const char **p_main_str
,
281 bool *is_tp
, struct perf_probe_event
*pev
)
284 const char *main_str
= parse_prog_config_kvpair(config_str
, pev
);
286 if (IS_ERR(main_str
))
287 return PTR_ERR(main_str
);
289 *p_main_str
= main_str
;
290 if (!strchr(main_str
, '=')) {
291 /* Is a tracepoint event? */
292 const char *s
= strchr(main_str
, ':');
295 pr_debug("bpf: '%s' is not a valid tracepoint\n",
297 return -BPF_LOADER_ERRNO__CONFIG
;
305 err
= parse_perf_probe_command(main_str
, pev
);
307 pr_debug("bpf: '%s' is not a valid config string\n",
309 /* parse failed, don't need clear pev. */
310 return -BPF_LOADER_ERRNO__CONFIG
;
316 config_bpf_program(struct bpf_program
*prog
)
318 struct perf_probe_event
*pev
= NULL
;
319 struct bpf_prog_priv
*priv
= NULL
;
320 const char *config_str
, *main_str
;
324 /* Initialize per-program probing setting */
325 probe_conf
.no_inlines
= false;
326 probe_conf
.force_add
= false;
328 config_str
= bpf_program__title(prog
, false);
329 if (IS_ERR(config_str
)) {
330 pr_debug("bpf: unable to get title for program\n");
331 return PTR_ERR(config_str
);
334 priv
= calloc(sizeof(*priv
), 1);
336 pr_debug("bpf: failed to alloc priv\n");
341 pr_debug("bpf: config program '%s'\n", config_str
);
342 err
= parse_prog_config(config_str
, &main_str
, &is_tp
, pev
);
347 char *s
= strchr(main_str
, ':');
350 priv
->sys_name
= strndup(main_str
, s
- main_str
);
351 priv
->evt_name
= strdup(s
+ 1);
355 if (pev
->group
&& strcmp(pev
->group
, PERF_BPF_PROBE_GROUP
)) {
356 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
357 config_str
, PERF_BPF_PROBE_GROUP
);
358 err
= -BPF_LOADER_ERRNO__GROUP
;
360 } else if (!pev
->group
)
361 pev
->group
= strdup(PERF_BPF_PROBE_GROUP
);
364 pr_debug("bpf: strdup failed\n");
370 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
372 err
= -BPF_LOADER_ERRNO__EVENTNAME
;
375 pr_debug("bpf: config '%s' is ok\n", config_str
);
378 err
= bpf_program__set_priv(prog
, priv
, clear_prog_priv
);
380 pr_debug("Failed to set priv for program '%s'\n", config_str
);
388 clear_perf_probe_event(pev
);
393 static int bpf__prepare_probe(void)
396 static bool initialized
= false;
399 * Make err static, so if init failed the first, bpf__prepare_probe()
400 * fails each time without calling init_probe_symbol_maps multiple
407 err
= init_probe_symbol_maps(false);
409 pr_debug("Failed to init_probe_symbol_maps\n");
410 probe_conf
.max_probes
= MAX_PROBES
;
415 preproc_gen_prologue(struct bpf_program
*prog
, int n
,
416 struct bpf_insn
*orig_insns
, int orig_insns_cnt
,
417 struct bpf_prog_prep_result
*res
)
419 struct bpf_prog_priv
*priv
= bpf_program__priv(prog
);
420 struct probe_trace_event
*tev
;
421 struct perf_probe_event
*pev
;
422 struct bpf_insn
*buf
;
423 size_t prologue_cnt
= 0;
426 if (IS_ERR(priv
) || !priv
|| priv
->is_tp
)
431 if (n
< 0 || n
>= priv
->nr_types
)
434 /* Find a tev belongs to that type */
435 for (i
= 0; i
< pev
->ntevs
; i
++) {
436 if (priv
->type_mapping
[i
] == n
)
440 if (i
>= pev
->ntevs
) {
441 pr_debug("Internal error: prologue type %d not found\n", n
);
442 return -BPF_LOADER_ERRNO__PROLOGUE
;
447 buf
= priv
->insns_buf
;
448 err
= bpf__gen_prologue(tev
->args
, tev
->nargs
,
450 BPF_MAXINSNS
- orig_insns_cnt
);
454 title
= bpf_program__title(prog
, false);
458 pr_debug("Failed to generate prologue for program %s\n",
463 memcpy(&buf
[prologue_cnt
], orig_insns
,
464 sizeof(struct bpf_insn
) * orig_insns_cnt
);
466 res
->new_insn_ptr
= buf
;
467 res
->new_insn_cnt
= prologue_cnt
+ orig_insns_cnt
;
472 pr_debug("Internal error in preproc_gen_prologue\n");
473 return -BPF_LOADER_ERRNO__PROLOGUE
;
477 * compare_tev_args is reflexive, transitive and antisymmetric.
478 * I can proof it but this margin is too narrow to contain.
480 static int compare_tev_args(const void *ptev1
, const void *ptev2
)
483 const struct probe_trace_event
*tev1
=
484 *(const struct probe_trace_event
**)ptev1
;
485 const struct probe_trace_event
*tev2
=
486 *(const struct probe_trace_event
**)ptev2
;
488 ret
= tev2
->nargs
- tev1
->nargs
;
492 for (i
= 0; i
< tev1
->nargs
; i
++) {
493 struct probe_trace_arg
*arg1
, *arg2
;
494 struct probe_trace_arg_ref
*ref1
, *ref2
;
496 arg1
= &tev1
->args
[i
];
497 arg2
= &tev2
->args
[i
];
499 ret
= strcmp(arg1
->value
, arg2
->value
);
506 while (ref1
&& ref2
) {
507 ret
= ref2
->offset
- ref1
->offset
;
516 return ref2
? 1 : -1;
523 * Assign a type number to each tevs in a pev.
524 * mapping is an array with same slots as tevs in that pev.
525 * nr_types will be set to number of types.
527 static int map_prologue(struct perf_probe_event
*pev
, int *mapping
,
531 struct probe_trace_event
**ptevs
;
533 size_t array_sz
= sizeof(*ptevs
) * pev
->ntevs
;
535 ptevs
= malloc(array_sz
);
537 pr_debug("Not enough memory: alloc ptevs failed\n");
541 pr_debug("In map_prologue, ntevs=%d\n", pev
->ntevs
);
542 for (i
= 0; i
< pev
->ntevs
; i
++)
543 ptevs
[i
] = &pev
->tevs
[i
];
545 qsort(ptevs
, pev
->ntevs
, sizeof(*ptevs
),
548 for (i
= 0; i
< pev
->ntevs
; i
++) {
551 n
= ptevs
[i
] - pev
->tevs
;
554 pr_debug("mapping[%d]=%d\n", n
, type
);
558 if (compare_tev_args(ptevs
+ i
, ptevs
+ i
- 1) == 0)
563 pr_debug("mapping[%d]=%d\n", n
, mapping
[n
]);
566 *nr_types
= type
+ 1;
571 static int hook_load_preprocessor(struct bpf_program
*prog
)
573 struct bpf_prog_priv
*priv
= bpf_program__priv(prog
);
574 struct perf_probe_event
*pev
;
575 bool need_prologue
= false;
578 if (IS_ERR(priv
) || !priv
) {
579 pr_debug("Internal error when hook preprocessor\n");
580 return -BPF_LOADER_ERRNO__INTERNAL
;
584 priv
->need_prologue
= false;
589 for (i
= 0; i
< pev
->ntevs
; i
++) {
590 struct probe_trace_event
*tev
= &pev
->tevs
[i
];
592 if (tev
->nargs
> 0) {
593 need_prologue
= true;
599 * Since all tevs don't have argument, we don't need generate
602 if (!need_prologue
) {
603 priv
->need_prologue
= false;
607 priv
->need_prologue
= true;
608 priv
->insns_buf
= malloc(sizeof(struct bpf_insn
) * BPF_MAXINSNS
);
609 if (!priv
->insns_buf
) {
610 pr_debug("Not enough memory: alloc insns_buf failed\n");
614 priv
->type_mapping
= malloc(sizeof(int) * pev
->ntevs
);
615 if (!priv
->type_mapping
) {
616 pr_debug("Not enough memory: alloc type_mapping failed\n");
619 memset(priv
->type_mapping
, -1,
620 sizeof(int) * pev
->ntevs
);
622 err
= map_prologue(pev
, priv
->type_mapping
, &priv
->nr_types
);
626 err
= bpf_program__set_prep(prog
, priv
->nr_types
,
627 preproc_gen_prologue
);
631 int bpf__probe(struct bpf_object
*obj
)
634 struct bpf_program
*prog
;
635 struct bpf_prog_priv
*priv
;
636 struct perf_probe_event
*pev
;
638 err
= bpf__prepare_probe();
640 pr_debug("bpf__prepare_probe failed\n");
644 bpf_object__for_each_program(prog
, obj
) {
645 err
= config_bpf_program(prog
);
649 priv
= bpf_program__priv(prog
);
650 if (IS_ERR(priv
) || !priv
) {
656 bpf_program__set_tracepoint(prog
);
660 bpf_program__set_kprobe(prog
);
663 err
= convert_perf_probe_events(pev
, 1);
665 pr_debug("bpf_probe: failed to convert perf probe events\n");
669 err
= apply_perf_probe_events(pev
, 1);
671 pr_debug("bpf_probe: failed to apply perf probe events\n");
676 * After probing, let's consider prologue, which
677 * adds program fetcher to BPF programs.
679 * hook_load_preprocessorr() hooks pre-processor
680 * to bpf_program, let it generate prologue
681 * dynamically during loading.
683 err
= hook_load_preprocessor(prog
);
688 return err
< 0 ? err
: 0;
691 #define EVENTS_WRITE_BUFSIZE 4096
692 int bpf__unprobe(struct bpf_object
*obj
)
695 struct bpf_program
*prog
;
697 bpf_object__for_each_program(prog
, obj
) {
698 struct bpf_prog_priv
*priv
= bpf_program__priv(prog
);
701 if (IS_ERR(priv
) || !priv
|| priv
->is_tp
)
704 for (i
= 0; i
< priv
->pev
.ntevs
; i
++) {
705 struct probe_trace_event
*tev
= &priv
->pev
.tevs
[i
];
706 char name_buf
[EVENTS_WRITE_BUFSIZE
];
707 struct strfilter
*delfilter
;
709 snprintf(name_buf
, EVENTS_WRITE_BUFSIZE
,
710 "%s:%s", tev
->group
, tev
->event
);
711 name_buf
[EVENTS_WRITE_BUFSIZE
- 1] = '\0';
713 delfilter
= strfilter__new(name_buf
, NULL
);
715 pr_debug("Failed to create filter for unprobing\n");
720 err
= del_perf_probe_events(delfilter
);
721 strfilter__delete(delfilter
);
723 pr_debug("Failed to delete %s\n", name_buf
);
732 int bpf__load(struct bpf_object
*obj
)
736 err
= bpf_object__load(obj
);
739 libbpf_strerror(err
, bf
, sizeof(bf
));
740 pr_debug("bpf: load objects failed: err=%d: (%s)\n", err
, bf
);
746 int bpf__foreach_event(struct bpf_object
*obj
,
747 bpf_prog_iter_callback_t func
,
750 struct bpf_program
*prog
;
753 bpf_object__for_each_program(prog
, obj
) {
754 struct bpf_prog_priv
*priv
= bpf_program__priv(prog
);
755 struct probe_trace_event
*tev
;
756 struct perf_probe_event
*pev
;
759 if (IS_ERR(priv
) || !priv
) {
760 pr_debug("bpf: failed to get private field\n");
761 return -BPF_LOADER_ERRNO__INTERNAL
;
765 fd
= bpf_program__fd(prog
);
766 err
= (*func
)(priv
->sys_name
, priv
->evt_name
, fd
, arg
);
768 pr_debug("bpf: tracepoint call back failed, stop iterate\n");
775 for (i
= 0; i
< pev
->ntevs
; i
++) {
778 if (priv
->need_prologue
) {
779 int type
= priv
->type_mapping
[i
];
781 fd
= bpf_program__nth_fd(prog
, type
);
783 fd
= bpf_program__fd(prog
);
787 pr_debug("bpf: failed to get file descriptor\n");
791 err
= (*func
)(tev
->group
, tev
->event
, fd
, arg
);
793 pr_debug("bpf: call back failed, stop iterate\n");
801 enum bpf_map_op_type
{
802 BPF_MAP_OP_SET_VALUE
,
803 BPF_MAP_OP_SET_EVSEL
,
806 enum bpf_map_key_type
{
812 struct list_head list
;
813 enum bpf_map_op_type op_type
;
814 enum bpf_map_key_type key_type
;
816 struct parse_events_array array
;
820 struct perf_evsel
*evsel
;
824 struct bpf_map_priv
{
825 struct list_head ops_list
;
829 bpf_map_op__delete(struct bpf_map_op
*op
)
831 if (!list_empty(&op
->list
))
832 list_del_init(&op
->list
);
833 if (op
->key_type
== BPF_MAP_KEY_RANGES
)
834 parse_events__clear_array(&op
->k
.array
);
839 bpf_map_priv__purge(struct bpf_map_priv
*priv
)
841 struct bpf_map_op
*pos
, *n
;
843 list_for_each_entry_safe(pos
, n
, &priv
->ops_list
, list
) {
844 list_del_init(&pos
->list
);
845 bpf_map_op__delete(pos
);
850 bpf_map_priv__clear(struct bpf_map
*map __maybe_unused
,
853 struct bpf_map_priv
*priv
= _priv
;
855 bpf_map_priv__purge(priv
);
860 bpf_map_op_setkey(struct bpf_map_op
*op
, struct parse_events_term
*term
)
862 op
->key_type
= BPF_MAP_KEY_ALL
;
866 if (term
->array
.nr_ranges
) {
867 size_t memsz
= term
->array
.nr_ranges
*
868 sizeof(op
->k
.array
.ranges
[0]);
870 op
->k
.array
.ranges
= memdup(term
->array
.ranges
, memsz
);
871 if (!op
->k
.array
.ranges
) {
872 pr_debug("Not enough memory to alloc indices for map\n");
875 op
->key_type
= BPF_MAP_KEY_RANGES
;
876 op
->k
.array
.nr_ranges
= term
->array
.nr_ranges
;
881 static struct bpf_map_op
*
882 bpf_map_op__new(struct parse_events_term
*term
)
884 struct bpf_map_op
*op
;
887 op
= zalloc(sizeof(*op
));
889 pr_debug("Failed to alloc bpf_map_op\n");
890 return ERR_PTR(-ENOMEM
);
892 INIT_LIST_HEAD(&op
->list
);
894 err
= bpf_map_op_setkey(op
, term
);
902 static struct bpf_map_op
*
903 bpf_map_op__clone(struct bpf_map_op
*op
)
905 struct bpf_map_op
*newop
;
907 newop
= memdup(op
, sizeof(*op
));
909 pr_debug("Failed to alloc bpf_map_op\n");
913 INIT_LIST_HEAD(&newop
->list
);
914 if (op
->key_type
== BPF_MAP_KEY_RANGES
) {
915 size_t memsz
= op
->k
.array
.nr_ranges
*
916 sizeof(op
->k
.array
.ranges
[0]);
918 newop
->k
.array
.ranges
= memdup(op
->k
.array
.ranges
, memsz
);
919 if (!newop
->k
.array
.ranges
) {
920 pr_debug("Failed to alloc indices for map\n");
929 static struct bpf_map_priv
*
930 bpf_map_priv__clone(struct bpf_map_priv
*priv
)
932 struct bpf_map_priv
*newpriv
;
933 struct bpf_map_op
*pos
, *newop
;
935 newpriv
= zalloc(sizeof(*newpriv
));
937 pr_debug("Not enough memory to alloc map private\n");
940 INIT_LIST_HEAD(&newpriv
->ops_list
);
942 list_for_each_entry(pos
, &priv
->ops_list
, list
) {
943 newop
= bpf_map_op__clone(pos
);
945 bpf_map_priv__purge(newpriv
);
948 list_add_tail(&newop
->list
, &newpriv
->ops_list
);
955 bpf_map__add_op(struct bpf_map
*map
, struct bpf_map_op
*op
)
957 const char *map_name
= bpf_map__name(map
);
958 struct bpf_map_priv
*priv
= bpf_map__priv(map
);
961 pr_debug("Failed to get private from map %s\n", map_name
);
962 return PTR_ERR(priv
);
966 priv
= zalloc(sizeof(*priv
));
968 pr_debug("Not enough memory to alloc map private\n");
971 INIT_LIST_HEAD(&priv
->ops_list
);
973 if (bpf_map__set_priv(map
, priv
, bpf_map_priv__clear
)) {
975 return -BPF_LOADER_ERRNO__INTERNAL
;
979 list_add_tail(&op
->list
, &priv
->ops_list
);
983 static struct bpf_map_op
*
984 bpf_map__add_newop(struct bpf_map
*map
, struct parse_events_term
*term
)
986 struct bpf_map_op
*op
;
989 op
= bpf_map_op__new(term
);
993 err
= bpf_map__add_op(map
, op
);
995 bpf_map_op__delete(op
);
1002 __bpf_map__config_value(struct bpf_map
*map
,
1003 struct parse_events_term
*term
)
1005 struct bpf_map_op
*op
;
1006 const char *map_name
= bpf_map__name(map
);
1007 const struct bpf_map_def
*def
= bpf_map__def(map
);
1010 pr_debug("Unable to get map definition from '%s'\n",
1012 return -BPF_LOADER_ERRNO__INTERNAL
;
1015 if (def
->type
!= BPF_MAP_TYPE_ARRAY
) {
1016 pr_debug("Map %s type is not BPF_MAP_TYPE_ARRAY\n",
1018 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE
;
1020 if (def
->key_size
< sizeof(unsigned int)) {
1021 pr_debug("Map %s has incorrect key size\n", map_name
);
1022 return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE
;
1024 switch (def
->value_size
) {
1031 pr_debug("Map %s has incorrect value size\n", map_name
);
1032 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE
;
1035 op
= bpf_map__add_newop(map
, term
);
1038 op
->op_type
= BPF_MAP_OP_SET_VALUE
;
1039 op
->v
.value
= term
->val
.num
;
1044 bpf_map__config_value(struct bpf_map
*map
,
1045 struct parse_events_term
*term
,
1046 struct perf_evlist
*evlist __maybe_unused
)
1048 if (!term
->err_val
) {
1049 pr_debug("Config value not set\n");
1050 return -BPF_LOADER_ERRNO__OBJCONF_CONF
;
1053 if (term
->type_val
!= PARSE_EVENTS__TERM_TYPE_NUM
) {
1054 pr_debug("ERROR: wrong value type for 'value'\n");
1055 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE
;
1058 return __bpf_map__config_value(map
, term
);
1062 __bpf_map__config_event(struct bpf_map
*map
,
1063 struct parse_events_term
*term
,
1064 struct perf_evlist
*evlist
)
1066 struct perf_evsel
*evsel
;
1067 const struct bpf_map_def
*def
;
1068 struct bpf_map_op
*op
;
1069 const char *map_name
= bpf_map__name(map
);
1071 evsel
= perf_evlist__find_evsel_by_str(evlist
, term
->val
.str
);
1073 pr_debug("Event (for '%s') '%s' doesn't exist\n",
1074 map_name
, term
->val
.str
);
1075 return -BPF_LOADER_ERRNO__OBJCONF_MAP_NOEVT
;
1078 def
= bpf_map__def(map
);
1080 pr_debug("Unable to get map definition from '%s'\n",
1082 return PTR_ERR(def
);
1086 * No need to check key_size and value_size:
1087 * kernel has already checked them.
1089 if (def
->type
!= BPF_MAP_TYPE_PERF_EVENT_ARRAY
) {
1090 pr_debug("Map %s type is not BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
1092 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE
;
1095 op
= bpf_map__add_newop(map
, term
);
1098 op
->op_type
= BPF_MAP_OP_SET_EVSEL
;
1099 op
->v
.evsel
= evsel
;
1104 bpf_map__config_event(struct bpf_map
*map
,
1105 struct parse_events_term
*term
,
1106 struct perf_evlist
*evlist
)
1108 if (!term
->err_val
) {
1109 pr_debug("Config value not set\n");
1110 return -BPF_LOADER_ERRNO__OBJCONF_CONF
;
1113 if (term
->type_val
!= PARSE_EVENTS__TERM_TYPE_STR
) {
1114 pr_debug("ERROR: wrong value type for 'event'\n");
1115 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE
;
1118 return __bpf_map__config_event(map
, term
, evlist
);
1121 struct bpf_obj_config__map_func
{
1122 const char *config_opt
;
1123 int (*config_func
)(struct bpf_map
*, struct parse_events_term
*,
1124 struct perf_evlist
*);
1127 struct bpf_obj_config__map_func bpf_obj_config__map_funcs
[] = {
1128 {"value", bpf_map__config_value
},
1129 {"event", bpf_map__config_event
},
1133 config_map_indices_range_check(struct parse_events_term
*term
,
1134 struct bpf_map
*map
,
1135 const char *map_name
)
1137 struct parse_events_array
*array
= &term
->array
;
1138 const struct bpf_map_def
*def
;
1141 if (!array
->nr_ranges
)
1143 if (!array
->ranges
) {
1144 pr_debug("ERROR: map %s: array->nr_ranges is %d but range array is NULL\n",
1145 map_name
, (int)array
->nr_ranges
);
1146 return -BPF_LOADER_ERRNO__INTERNAL
;
1149 def
= bpf_map__def(map
);
1151 pr_debug("ERROR: Unable to get map definition from '%s'\n",
1153 return -BPF_LOADER_ERRNO__INTERNAL
;
1156 for (i
= 0; i
< array
->nr_ranges
; i
++) {
1157 unsigned int start
= array
->ranges
[i
].start
;
1158 size_t length
= array
->ranges
[i
].length
;
1159 unsigned int idx
= start
+ length
- 1;
1161 if (idx
>= def
->max_entries
) {
1162 pr_debug("ERROR: index %d too large\n", idx
);
1163 return -BPF_LOADER_ERRNO__OBJCONF_MAP_IDX2BIG
;
1170 bpf__obj_config_map(struct bpf_object
*obj
,
1171 struct parse_events_term
*term
,
1172 struct perf_evlist
*evlist
,
1175 /* key is "map:<mapname>.<config opt>" */
1176 char *map_name
= strdup(term
->config
+ sizeof("map:") - 1);
1177 struct bpf_map
*map
;
1178 int err
= -BPF_LOADER_ERRNO__OBJCONF_OPT
;
1185 map_opt
= strchr(map_name
, '.');
1187 pr_debug("ERROR: Invalid map config: %s\n", map_name
);
1192 if (*map_opt
== '\0') {
1193 pr_debug("ERROR: Invalid map option: %s\n", term
->config
);
1197 map
= bpf_object__find_map_by_name(obj
, map_name
);
1199 pr_debug("ERROR: Map %s doesn't exist\n", map_name
);
1200 err
= -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST
;
1204 *key_scan_pos
+= strlen(map_opt
);
1205 err
= config_map_indices_range_check(term
, map
, map_name
);
1208 *key_scan_pos
-= strlen(map_opt
);
1210 for (i
= 0; i
< ARRAY_SIZE(bpf_obj_config__map_funcs
); i
++) {
1211 struct bpf_obj_config__map_func
*func
=
1212 &bpf_obj_config__map_funcs
[i
];
1214 if (strcmp(map_opt
, func
->config_opt
) == 0) {
1215 err
= func
->config_func(map
, term
, evlist
);
1220 pr_debug("ERROR: Invalid map config option '%s'\n", map_opt
);
1221 err
= -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT
;
1225 key_scan_pos
+= strlen(map_opt
);
1229 int bpf__config_obj(struct bpf_object
*obj
,
1230 struct parse_events_term
*term
,
1231 struct perf_evlist
*evlist
,
1234 int key_scan_pos
= 0;
1237 if (!obj
|| !term
|| !term
->config
)
1240 if (strstarts(term
->config
, "map:")) {
1241 key_scan_pos
= sizeof("map:") - 1;
1242 err
= bpf__obj_config_map(obj
, term
, evlist
, &key_scan_pos
);
1245 err
= -BPF_LOADER_ERRNO__OBJCONF_OPT
;
1248 *error_pos
= key_scan_pos
;
1253 typedef int (*map_config_func_t
)(const char *name
, int map_fd
,
1254 const struct bpf_map_def
*pdef
,
1255 struct bpf_map_op
*op
,
1256 void *pkey
, void *arg
);
1259 foreach_key_array_all(map_config_func_t func
,
1260 void *arg
, const char *name
,
1261 int map_fd
, const struct bpf_map_def
*pdef
,
1262 struct bpf_map_op
*op
)
1267 for (i
= 0; i
< pdef
->max_entries
; i
++) {
1268 err
= func(name
, map_fd
, pdef
, op
, &i
, arg
);
1270 pr_debug("ERROR: failed to insert value to %s[%u]\n",
1279 foreach_key_array_ranges(map_config_func_t func
, void *arg
,
1280 const char *name
, int map_fd
,
1281 const struct bpf_map_def
*pdef
,
1282 struct bpf_map_op
*op
)
1287 for (i
= 0; i
< op
->k
.array
.nr_ranges
; i
++) {
1288 unsigned int start
= op
->k
.array
.ranges
[i
].start
;
1289 size_t length
= op
->k
.array
.ranges
[i
].length
;
1291 for (j
= 0; j
< length
; j
++) {
1292 unsigned int idx
= start
+ j
;
1294 err
= func(name
, map_fd
, pdef
, op
, &idx
, arg
);
1296 pr_debug("ERROR: failed to insert value to %s[%u]\n",
1306 bpf_map_config_foreach_key(struct bpf_map
*map
,
1307 map_config_func_t func
,
1311 struct bpf_map_op
*op
;
1312 const struct bpf_map_def
*def
;
1313 const char *name
= bpf_map__name(map
);
1314 struct bpf_map_priv
*priv
= bpf_map__priv(map
);
1317 pr_debug("ERROR: failed to get private from map %s\n", name
);
1318 return -BPF_LOADER_ERRNO__INTERNAL
;
1320 if (!priv
|| list_empty(&priv
->ops_list
)) {
1321 pr_debug("INFO: nothing to config for map %s\n", name
);
1325 def
= bpf_map__def(map
);
1327 pr_debug("ERROR: failed to get definition from map %s\n", name
);
1328 return -BPF_LOADER_ERRNO__INTERNAL
;
1330 map_fd
= bpf_map__fd(map
);
1332 pr_debug("ERROR: failed to get fd from map %s\n", name
);
1336 list_for_each_entry(op
, &priv
->ops_list
, list
) {
1337 switch (def
->type
) {
1338 case BPF_MAP_TYPE_ARRAY
:
1339 case BPF_MAP_TYPE_PERF_EVENT_ARRAY
:
1340 switch (op
->key_type
) {
1341 case BPF_MAP_KEY_ALL
:
1342 err
= foreach_key_array_all(func
, arg
, name
,
1345 case BPF_MAP_KEY_RANGES
:
1346 err
= foreach_key_array_ranges(func
, arg
, name
,
1351 pr_debug("ERROR: keytype for map '%s' invalid\n",
1353 return -BPF_LOADER_ERRNO__INTERNAL
;
1359 pr_debug("ERROR: type of '%s' incorrect\n", name
);
1360 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE
;
1368 apply_config_value_for_key(int map_fd
, void *pkey
,
1369 size_t val_size
, u64 val
)
1375 u8 _val
= (u8
)(val
);
1376 err
= bpf_map_update_elem(map_fd
, pkey
, &_val
, BPF_ANY
);
1380 u16 _val
= (u16
)(val
);
1381 err
= bpf_map_update_elem(map_fd
, pkey
, &_val
, BPF_ANY
);
1385 u32 _val
= (u32
)(val
);
1386 err
= bpf_map_update_elem(map_fd
, pkey
, &_val
, BPF_ANY
);
1390 err
= bpf_map_update_elem(map_fd
, pkey
, &val
, BPF_ANY
);
1394 pr_debug("ERROR: invalid value size\n");
1395 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE
;
1403 apply_config_evsel_for_key(const char *name
, int map_fd
, void *pkey
,
1404 struct perf_evsel
*evsel
)
1406 struct xyarray
*xy
= evsel
->fd
;
1407 struct perf_event_attr
*attr
;
1408 unsigned int key
, events
;
1409 bool check_pass
= false;
1414 pr_debug("ERROR: evsel not ready for map %s\n", name
);
1415 return -BPF_LOADER_ERRNO__INTERNAL
;
1418 if (xy
->row_size
/ xy
->entry_size
!= 1) {
1419 pr_debug("ERROR: Dimension of target event is incorrect for map %s\n",
1421 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM
;
1424 attr
= &evsel
->attr
;
1425 if (attr
->inherit
) {
1426 pr_debug("ERROR: Can't put inherit event into map %s\n", name
);
1427 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH
;
1430 if (perf_evsel__is_bpf_output(evsel
))
1432 if (attr
->type
== PERF_TYPE_RAW
)
1434 if (attr
->type
== PERF_TYPE_HARDWARE
)
1437 pr_debug("ERROR: Event type is wrong for map %s\n", name
);
1438 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE
;
1441 events
= xy
->entries
/ (xy
->row_size
/ xy
->entry_size
);
1442 key
= *((unsigned int *)pkey
);
1443 if (key
>= events
) {
1444 pr_debug("ERROR: there is no event %d for map %s\n",
1446 return -BPF_LOADER_ERRNO__OBJCONF_MAP_MAPSIZE
;
1448 evt_fd
= xyarray__entry(xy
, key
, 0);
1449 err
= bpf_map_update_elem(map_fd
, pkey
, evt_fd
, BPF_ANY
);
1456 apply_obj_config_map_for_key(const char *name
, int map_fd
,
1457 const struct bpf_map_def
*pdef
,
1458 struct bpf_map_op
*op
,
1459 void *pkey
, void *arg __maybe_unused
)
1463 switch (op
->op_type
) {
1464 case BPF_MAP_OP_SET_VALUE
:
1465 err
= apply_config_value_for_key(map_fd
, pkey
,
1469 case BPF_MAP_OP_SET_EVSEL
:
1470 err
= apply_config_evsel_for_key(name
, map_fd
, pkey
,
1474 pr_debug("ERROR: unknown value type for '%s'\n", name
);
1475 err
= -BPF_LOADER_ERRNO__INTERNAL
;
1481 apply_obj_config_map(struct bpf_map
*map
)
1483 return bpf_map_config_foreach_key(map
,
1484 apply_obj_config_map_for_key
,
1489 apply_obj_config_object(struct bpf_object
*obj
)
1491 struct bpf_map
*map
;
1494 bpf_object__for_each_map(map
, obj
) {
1495 err
= apply_obj_config_map(map
);
1502 int bpf__apply_obj_config(void)
1504 struct bpf_object
*obj
, *tmp
;
1507 bpf_object__for_each_safe(obj
, tmp
) {
1508 err
= apply_obj_config_object(obj
);
1516 #define bpf__for_each_map(pos, obj, objtmp) \
1517 bpf_object__for_each_safe(obj, objtmp) \
1518 bpf_object__for_each_map(pos, obj)
1520 #define bpf__for_each_map_named(pos, obj, objtmp, name) \
1521 bpf__for_each_map(pos, obj, objtmp) \
1522 if (bpf_map__name(pos) && \
1524 bpf_map__name(pos)) == 0))
1526 struct perf_evsel
*bpf__setup_output_event(struct perf_evlist
*evlist
, const char *name
)
1528 struct bpf_map_priv
*tmpl_priv
= NULL
;
1529 struct bpf_object
*obj
, *tmp
;
1530 struct perf_evsel
*evsel
= NULL
;
1531 struct bpf_map
*map
;
1533 bool need_init
= false;
1535 bpf__for_each_map_named(map
, obj
, tmp
, name
) {
1536 struct bpf_map_priv
*priv
= bpf_map__priv(map
);
1539 return ERR_PTR(-BPF_LOADER_ERRNO__INTERNAL
);
1542 * No need to check map type: type should have been
1543 * verified by kernel.
1545 if (!need_init
&& !priv
)
1547 if (!tmpl_priv
&& priv
)
1555 char *event_definition
= NULL
;
1557 if (asprintf(&event_definition
, "bpf-output/no-inherit=1,name=%s/", name
) < 0)
1558 return ERR_PTR(-ENOMEM
);
1560 err
= parse_events(evlist
, event_definition
, NULL
);
1561 free(event_definition
);
1564 pr_debug("ERROR: failed to create the \"%s\" bpf-output event\n", name
);
1565 return ERR_PTR(-err
);
1568 evsel
= perf_evlist__last(evlist
);
1571 bpf__for_each_map_named(map
, obj
, tmp
, name
) {
1572 struct bpf_map_priv
*priv
= bpf_map__priv(map
);
1575 return ERR_PTR(-BPF_LOADER_ERRNO__INTERNAL
);
1580 priv
= bpf_map_priv__clone(tmpl_priv
);
1582 return ERR_PTR(-ENOMEM
);
1584 err
= bpf_map__set_priv(map
, priv
, bpf_map_priv__clear
);
1586 bpf_map_priv__clear(map
, priv
);
1587 return ERR_PTR(err
);
1590 struct bpf_map_op
*op
;
1592 op
= bpf_map__add_newop(map
, NULL
);
1594 return ERR_CAST(op
);
1595 op
->op_type
= BPF_MAP_OP_SET_EVSEL
;
1596 op
->v
.evsel
= evsel
;
1603 int bpf__setup_stdout(struct perf_evlist
*evlist
)
1605 struct perf_evsel
*evsel
= bpf__setup_output_event(evlist
, "__bpf_stdout__");
1606 return PTR_ERR_OR_ZERO(evsel
);
1609 #define ERRNO_OFFSET(e) ((e) - __BPF_LOADER_ERRNO__START)
1610 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
1611 #define NR_ERRNO (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
1613 static const char *bpf_loader_strerror_table
[NR_ERRNO
] = {
1614 [ERRCODE_OFFSET(CONFIG
)] = "Invalid config string",
1615 [ERRCODE_OFFSET(GROUP
)] = "Invalid group name",
1616 [ERRCODE_OFFSET(EVENTNAME
)] = "No event name found in config string",
1617 [ERRCODE_OFFSET(INTERNAL
)] = "BPF loader internal error",
1618 [ERRCODE_OFFSET(COMPILE
)] = "Error when compiling BPF scriptlet",
1619 [ERRCODE_OFFSET(PROGCONF_TERM
)] = "Invalid program config term in config string",
1620 [ERRCODE_OFFSET(PROLOGUE
)] = "Failed to generate prologue",
1621 [ERRCODE_OFFSET(PROLOGUE2BIG
)] = "Prologue too big for program",
1622 [ERRCODE_OFFSET(PROLOGUEOOB
)] = "Offset out of bound for prologue",
1623 [ERRCODE_OFFSET(OBJCONF_OPT
)] = "Invalid object config option",
1624 [ERRCODE_OFFSET(OBJCONF_CONF
)] = "Config value not set (missing '=')",
1625 [ERRCODE_OFFSET(OBJCONF_MAP_OPT
)] = "Invalid object map config option",
1626 [ERRCODE_OFFSET(OBJCONF_MAP_NOTEXIST
)] = "Target map doesn't exist",
1627 [ERRCODE_OFFSET(OBJCONF_MAP_VALUE
)] = "Incorrect value type for map",
1628 [ERRCODE_OFFSET(OBJCONF_MAP_TYPE
)] = "Incorrect map type",
1629 [ERRCODE_OFFSET(OBJCONF_MAP_KEYSIZE
)] = "Incorrect map key size",
1630 [ERRCODE_OFFSET(OBJCONF_MAP_VALUESIZE
)] = "Incorrect map value size",
1631 [ERRCODE_OFFSET(OBJCONF_MAP_NOEVT
)] = "Event not found for map setting",
1632 [ERRCODE_OFFSET(OBJCONF_MAP_MAPSIZE
)] = "Invalid map size for event setting",
1633 [ERRCODE_OFFSET(OBJCONF_MAP_EVTDIM
)] = "Event dimension too large",
1634 [ERRCODE_OFFSET(OBJCONF_MAP_EVTINH
)] = "Doesn't support inherit event",
1635 [ERRCODE_OFFSET(OBJCONF_MAP_EVTTYPE
)] = "Wrong event type for map",
1636 [ERRCODE_OFFSET(OBJCONF_MAP_IDX2BIG
)] = "Index too large",
1640 bpf_loader_strerror(int err
, char *buf
, size_t size
)
1642 char sbuf
[STRERR_BUFSIZE
];
1648 err
= err
> 0 ? err
: -err
;
1650 if (err
>= __LIBBPF_ERRNO__START
)
1651 return libbpf_strerror(err
, buf
, size
);
1653 if (err
>= __BPF_LOADER_ERRNO__START
&& err
< __BPF_LOADER_ERRNO__END
) {
1654 msg
= bpf_loader_strerror_table
[ERRNO_OFFSET(err
)];
1655 snprintf(buf
, size
, "%s", msg
);
1656 buf
[size
- 1] = '\0';
1660 if (err
>= __BPF_LOADER_ERRNO__END
)
1661 snprintf(buf
, size
, "Unknown bpf loader error %d", err
);
1663 snprintf(buf
, size
, "%s",
1664 str_error_r(err
, sbuf
, sizeof(sbuf
)));
1666 buf
[size
- 1] = '\0';
1670 #define bpf__strerror_head(err, buf, size) \
1671 char sbuf[STRERR_BUFSIZE], *emsg;\
1676 bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
1680 scnprintf(buf, size, "%s", emsg);\
1683 #define bpf__strerror_entry(val, fmt...)\
1685 scnprintf(buf, size, fmt);\
1689 #define bpf__strerror_end(buf, size)\
1691 buf[size - 1] = '\0';
1693 int bpf__strerror_prepare_load(const char *filename
, bool source
,
1694 int err
, char *buf
, size_t size
)
1699 n
= snprintf(buf
, size
, "Failed to load %s%s: ",
1700 filename
, source
? " from source" : "");
1702 buf
[size
- 1] = '\0';
1708 ret
= bpf_loader_strerror(err
, buf
, size
);
1709 buf
[size
- 1] = '\0';
1713 int bpf__strerror_probe(struct bpf_object
*obj __maybe_unused
,
1714 int err
, char *buf
, size_t size
)
1716 bpf__strerror_head(err
, buf
, size
);
1717 case BPF_LOADER_ERRNO__PROGCONF_TERM
: {
1718 scnprintf(buf
, size
, "%s (add -v to see detail)", emsg
);
1721 bpf__strerror_entry(EEXIST
, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'");
1722 bpf__strerror_entry(EACCES
, "You need to be root");
1723 bpf__strerror_entry(EPERM
, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
1724 bpf__strerror_entry(ENOENT
, "You need to check probing points in BPF file");
1725 bpf__strerror_end(buf
, size
);
1729 int bpf__strerror_load(struct bpf_object
*obj
,
1730 int err
, char *buf
, size_t size
)
1732 bpf__strerror_head(err
, buf
, size
);
1733 case LIBBPF_ERRNO__KVER
: {
1734 unsigned int obj_kver
= bpf_object__kversion(obj
);
1735 unsigned int real_kver
;
1737 if (fetch_kernel_version(&real_kver
, NULL
, 0)) {
1738 scnprintf(buf
, size
, "Unable to fetch kernel version");
1742 if (obj_kver
!= real_kver
) {
1743 scnprintf(buf
, size
,
1744 "'version' ("KVER_FMT
") doesn't match running kernel ("KVER_FMT
")",
1745 KVER_PARAM(obj_kver
),
1746 KVER_PARAM(real_kver
));
1750 scnprintf(buf
, size
, "Failed to load program for unknown reason");
1753 bpf__strerror_end(buf
, size
);
1757 int bpf__strerror_config_obj(struct bpf_object
*obj __maybe_unused
,
1758 struct parse_events_term
*term __maybe_unused
,
1759 struct perf_evlist
*evlist __maybe_unused
,
1760 int *error_pos __maybe_unused
, int err
,
1761 char *buf
, size_t size
)
1763 bpf__strerror_head(err
, buf
, size
);
1764 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE
,
1765 "Can't use this config term with this map type");
1766 bpf__strerror_end(buf
, size
);
1770 int bpf__strerror_apply_obj_config(int err
, char *buf
, size_t size
)
1772 bpf__strerror_head(err
, buf
, size
);
1773 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM
,
1774 "Cannot set event to BPF map in multi-thread tracing");
1775 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH
,
1776 "%s (Hint: use -i to turn off inherit)", emsg
);
1777 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE
,
1778 "Can only put raw, hardware and BPF output event into a BPF map");
1779 bpf__strerror_end(buf
, size
);
1783 int bpf__strerror_setup_output_event(struct perf_evlist
*evlist __maybe_unused
,
1784 int err
, char *buf
, size_t size
)
1786 bpf__strerror_head(err
, buf
, size
);
1787 bpf__strerror_end(buf
, size
);