4 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
5 * Copyright (C) 2015 Huawei Inc.
9 #include <bpf/libbpf.h>
11 #include <linux/err.h>
12 #include <linux/kernel.h>
13 #include <linux/string.h>
17 #include "bpf-loader.h"
18 #include "bpf-prologue.h"
19 #include "probe-event.h"
20 #include "probe-finder.h" // for MAX_PROBES
21 #include "parse-events.h"
22 #include "strfilter.h"
23 #include "llvm-utils.h"
24 #include "c++/clang-c.h"
26 #define DEFINE_PRINT_FN(name, level) \
27 static int libbpf_##name(const char *fmt, ...) \
32 va_start(args, fmt); \
33 ret = veprintf(level, verbose, pr_fmt(fmt), args);\
38 DEFINE_PRINT_FN(warning
, 1)
39 DEFINE_PRINT_FN(info
, 1)
40 DEFINE_PRINT_FN(debug
, 1)
42 struct bpf_prog_priv
{
46 struct perf_probe_event pev
;
48 struct bpf_insn
*insns_buf
;
53 static bool libbpf_initialized
;
56 bpf__prepare_load_buffer(void *obj_buf
, size_t obj_buf_sz
, const char *name
)
58 struct bpf_object
*obj
;
60 if (!libbpf_initialized
) {
61 libbpf_set_print(libbpf_warning
,
64 libbpf_initialized
= true;
67 obj
= bpf_object__open_buffer(obj_buf
, obj_buf_sz
, name
);
69 pr_debug("bpf: failed to load buffer\n");
70 return ERR_PTR(-EINVAL
);
76 struct bpf_object
*bpf__prepare_load(const char *filename
, bool source
)
78 struct bpf_object
*obj
;
80 if (!libbpf_initialized
) {
81 libbpf_set_print(libbpf_warning
,
84 libbpf_initialized
= true;
93 err
= perf_clang__compile_bpf(filename
, &obj_buf
, &obj_buf_sz
);
94 perf_clang__cleanup();
96 pr_warning("bpf: builtin compilation failed: %d, try external compiler\n", err
);
97 err
= llvm__compile_bpf(filename
, &obj_buf
, &obj_buf_sz
);
99 return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE
);
101 pr_debug("bpf: successfull builtin compilation\n");
102 obj
= bpf_object__open_buffer(obj_buf
, obj_buf_sz
, filename
);
104 if (!IS_ERR(obj
) && llvm_param
.dump_obj
)
105 llvm__dump_obj(filename
, obj_buf
, obj_buf_sz
);
109 obj
= bpf_object__open(filename
);
112 pr_debug("bpf: failed to load %s\n", filename
);
119 void bpf__clear(void)
121 struct bpf_object
*obj
, *tmp
;
123 bpf_object__for_each_safe(obj
, tmp
) {
125 bpf_object__close(obj
);
130 clear_prog_priv(struct bpf_program
*prog __maybe_unused
,
133 struct bpf_prog_priv
*priv
= _priv
;
135 cleanup_perf_probe_events(&priv
->pev
, 1);
136 zfree(&priv
->insns_buf
);
137 zfree(&priv
->type_mapping
);
138 zfree(&priv
->sys_name
);
139 zfree(&priv
->evt_name
);
144 prog_config__exec(const char *value
, struct perf_probe_event
*pev
)
147 pev
->target
= strdup(value
);
154 prog_config__module(const char *value
, struct perf_probe_event
*pev
)
156 pev
->uprobes
= false;
157 pev
->target
= strdup(value
);
164 prog_config__bool(const char *value
, bool *pbool
, bool invert
)
172 err
= strtobool(value
, &bool_value
);
176 *pbool
= invert
? !bool_value
: bool_value
;
181 prog_config__inlines(const char *value
,
182 struct perf_probe_event
*pev __maybe_unused
)
184 return prog_config__bool(value
, &probe_conf
.no_inlines
, true);
188 prog_config__force(const char *value
,
189 struct perf_probe_event
*pev __maybe_unused
)
191 return prog_config__bool(value
, &probe_conf
.force_add
, false);
198 int (*func
)(const char *, struct perf_probe_event
*);
199 } bpf_prog_config_terms
[] = {
202 .usage
= "exec=<full path of file>",
203 .desc
= "Set uprobe target",
204 .func
= prog_config__exec
,
208 .usage
= "module=<module name> ",
209 .desc
= "Set kprobe module",
210 .func
= prog_config__module
,
214 .usage
= "inlines=[yes|no] ",
215 .desc
= "Probe at inline symbol",
216 .func
= prog_config__inlines
,
220 .usage
= "force=[yes|no] ",
221 .desc
= "Forcibly add events with existing name",
222 .func
= prog_config__force
,
227 do_prog_config(const char *key
, const char *value
,
228 struct perf_probe_event
*pev
)
232 pr_debug("config bpf program: %s=%s\n", key
, value
);
233 for (i
= 0; i
< ARRAY_SIZE(bpf_prog_config_terms
); i
++)
234 if (strcmp(key
, bpf_prog_config_terms
[i
].key
) == 0)
235 return bpf_prog_config_terms
[i
].func(value
, pev
);
237 pr_debug("BPF: ERROR: invalid program config option: %s=%s\n",
240 pr_debug("\nHint: Valid options are:\n");
241 for (i
= 0; i
< ARRAY_SIZE(bpf_prog_config_terms
); i
++)
242 pr_debug("\t%s:\t%s\n", bpf_prog_config_terms
[i
].usage
,
243 bpf_prog_config_terms
[i
].desc
);
246 return -BPF_LOADER_ERRNO__PROGCONF_TERM
;
250 parse_prog_config_kvpair(const char *config_str
, struct perf_probe_event
*pev
)
252 char *text
= strdup(config_str
);
254 const char *main_str
= NULL
;
258 pr_debug("Not enough memory: dup config_str failed\n");
259 return ERR_PTR(-ENOMEM
);
263 while ((sep
= strchr(line
, ';'))) {
267 equ
= strchr(line
, '=');
269 pr_warning("WARNING: invalid config in BPF object: %s\n",
271 pr_warning("\tShould be 'key=value'.\n");
276 err
= do_prog_config(line
, equ
+ 1, pev
);
284 main_str
= config_str
+ (line
- text
);
287 return err
? ERR_PTR(err
) : main_str
;
291 parse_prog_config(const char *config_str
, const char **p_main_str
,
292 bool *is_tp
, struct perf_probe_event
*pev
)
295 const char *main_str
= parse_prog_config_kvpair(config_str
, pev
);
297 if (IS_ERR(main_str
))
298 return PTR_ERR(main_str
);
300 *p_main_str
= main_str
;
301 if (!strchr(main_str
, '=')) {
302 /* Is a tracepoint event? */
303 const char *s
= strchr(main_str
, ':');
306 pr_debug("bpf: '%s' is not a valid tracepoint\n",
308 return -BPF_LOADER_ERRNO__CONFIG
;
316 err
= parse_perf_probe_command(main_str
, pev
);
318 pr_debug("bpf: '%s' is not a valid config string\n",
320 /* parse failed, don't need clear pev. */
321 return -BPF_LOADER_ERRNO__CONFIG
;
327 config_bpf_program(struct bpf_program
*prog
)
329 struct perf_probe_event
*pev
= NULL
;
330 struct bpf_prog_priv
*priv
= NULL
;
331 const char *config_str
, *main_str
;
335 /* Initialize per-program probing setting */
336 probe_conf
.no_inlines
= false;
337 probe_conf
.force_add
= false;
339 config_str
= bpf_program__title(prog
, false);
340 if (IS_ERR(config_str
)) {
341 pr_debug("bpf: unable to get title for program\n");
342 return PTR_ERR(config_str
);
345 priv
= calloc(sizeof(*priv
), 1);
347 pr_debug("bpf: failed to alloc priv\n");
352 pr_debug("bpf: config program '%s'\n", config_str
);
353 err
= parse_prog_config(config_str
, &main_str
, &is_tp
, pev
);
358 char *s
= strchr(main_str
, ':');
361 priv
->sys_name
= strndup(main_str
, s
- main_str
);
362 priv
->evt_name
= strdup(s
+ 1);
366 if (pev
->group
&& strcmp(pev
->group
, PERF_BPF_PROBE_GROUP
)) {
367 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
368 config_str
, PERF_BPF_PROBE_GROUP
);
369 err
= -BPF_LOADER_ERRNO__GROUP
;
371 } else if (!pev
->group
)
372 pev
->group
= strdup(PERF_BPF_PROBE_GROUP
);
375 pr_debug("bpf: strdup failed\n");
381 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
383 err
= -BPF_LOADER_ERRNO__EVENTNAME
;
386 pr_debug("bpf: config '%s' is ok\n", config_str
);
389 err
= bpf_program__set_priv(prog
, priv
, clear_prog_priv
);
391 pr_debug("Failed to set priv for program '%s'\n", config_str
);
399 clear_perf_probe_event(pev
);
404 static int bpf__prepare_probe(void)
407 static bool initialized
= false;
410 * Make err static, so if init failed the first, bpf__prepare_probe()
411 * fails each time without calling init_probe_symbol_maps multiple
418 err
= init_probe_symbol_maps(false);
420 pr_debug("Failed to init_probe_symbol_maps\n");
421 probe_conf
.max_probes
= MAX_PROBES
;
426 preproc_gen_prologue(struct bpf_program
*prog
, int n
,
427 struct bpf_insn
*orig_insns
, int orig_insns_cnt
,
428 struct bpf_prog_prep_result
*res
)
430 struct bpf_prog_priv
*priv
= bpf_program__priv(prog
);
431 struct probe_trace_event
*tev
;
432 struct perf_probe_event
*pev
;
433 struct bpf_insn
*buf
;
434 size_t prologue_cnt
= 0;
437 if (IS_ERR(priv
) || !priv
|| priv
->is_tp
)
442 if (n
< 0 || n
>= priv
->nr_types
)
445 /* Find a tev belongs to that type */
446 for (i
= 0; i
< pev
->ntevs
; i
++) {
447 if (priv
->type_mapping
[i
] == n
)
451 if (i
>= pev
->ntevs
) {
452 pr_debug("Internal error: prologue type %d not found\n", n
);
453 return -BPF_LOADER_ERRNO__PROLOGUE
;
458 buf
= priv
->insns_buf
;
459 err
= bpf__gen_prologue(tev
->args
, tev
->nargs
,
461 BPF_MAXINSNS
- orig_insns_cnt
);
465 title
= bpf_program__title(prog
, false);
469 pr_debug("Failed to generate prologue for program %s\n",
474 memcpy(&buf
[prologue_cnt
], orig_insns
,
475 sizeof(struct bpf_insn
) * orig_insns_cnt
);
477 res
->new_insn_ptr
= buf
;
478 res
->new_insn_cnt
= prologue_cnt
+ orig_insns_cnt
;
483 pr_debug("Internal error in preproc_gen_prologue\n");
484 return -BPF_LOADER_ERRNO__PROLOGUE
;
488 * compare_tev_args is reflexive, transitive and antisymmetric.
489 * I can proof it but this margin is too narrow to contain.
491 static int compare_tev_args(const void *ptev1
, const void *ptev2
)
494 const struct probe_trace_event
*tev1
=
495 *(const struct probe_trace_event
**)ptev1
;
496 const struct probe_trace_event
*tev2
=
497 *(const struct probe_trace_event
**)ptev2
;
499 ret
= tev2
->nargs
- tev1
->nargs
;
503 for (i
= 0; i
< tev1
->nargs
; i
++) {
504 struct probe_trace_arg
*arg1
, *arg2
;
505 struct probe_trace_arg_ref
*ref1
, *ref2
;
507 arg1
= &tev1
->args
[i
];
508 arg2
= &tev2
->args
[i
];
510 ret
= strcmp(arg1
->value
, arg2
->value
);
517 while (ref1
&& ref2
) {
518 ret
= ref2
->offset
- ref1
->offset
;
527 return ref2
? 1 : -1;
534 * Assign a type number to each tevs in a pev.
535 * mapping is an array with same slots as tevs in that pev.
536 * nr_types will be set to number of types.
538 static int map_prologue(struct perf_probe_event
*pev
, int *mapping
,
542 struct probe_trace_event
**ptevs
;
544 size_t array_sz
= sizeof(*ptevs
) * pev
->ntevs
;
546 ptevs
= malloc(array_sz
);
548 pr_debug("Not enough memory: alloc ptevs failed\n");
552 pr_debug("In map_prologue, ntevs=%d\n", pev
->ntevs
);
553 for (i
= 0; i
< pev
->ntevs
; i
++)
554 ptevs
[i
] = &pev
->tevs
[i
];
556 qsort(ptevs
, pev
->ntevs
, sizeof(*ptevs
),
559 for (i
= 0; i
< pev
->ntevs
; i
++) {
562 n
= ptevs
[i
] - pev
->tevs
;
565 pr_debug("mapping[%d]=%d\n", n
, type
);
569 if (compare_tev_args(ptevs
+ i
, ptevs
+ i
- 1) == 0)
574 pr_debug("mapping[%d]=%d\n", n
, mapping
[n
]);
577 *nr_types
= type
+ 1;
582 static int hook_load_preprocessor(struct bpf_program
*prog
)
584 struct bpf_prog_priv
*priv
= bpf_program__priv(prog
);
585 struct perf_probe_event
*pev
;
586 bool need_prologue
= false;
589 if (IS_ERR(priv
) || !priv
) {
590 pr_debug("Internal error when hook preprocessor\n");
591 return -BPF_LOADER_ERRNO__INTERNAL
;
595 priv
->need_prologue
= false;
600 for (i
= 0; i
< pev
->ntevs
; i
++) {
601 struct probe_trace_event
*tev
= &pev
->tevs
[i
];
603 if (tev
->nargs
> 0) {
604 need_prologue
= true;
610 * Since all tevs don't have argument, we don't need generate
613 if (!need_prologue
) {
614 priv
->need_prologue
= false;
618 priv
->need_prologue
= true;
619 priv
->insns_buf
= malloc(sizeof(struct bpf_insn
) * BPF_MAXINSNS
);
620 if (!priv
->insns_buf
) {
621 pr_debug("Not enough memory: alloc insns_buf failed\n");
625 priv
->type_mapping
= malloc(sizeof(int) * pev
->ntevs
);
626 if (!priv
->type_mapping
) {
627 pr_debug("Not enough memory: alloc type_mapping failed\n");
630 memset(priv
->type_mapping
, -1,
631 sizeof(int) * pev
->ntevs
);
633 err
= map_prologue(pev
, priv
->type_mapping
, &priv
->nr_types
);
637 err
= bpf_program__set_prep(prog
, priv
->nr_types
,
638 preproc_gen_prologue
);
642 int bpf__probe(struct bpf_object
*obj
)
645 struct bpf_program
*prog
;
646 struct bpf_prog_priv
*priv
;
647 struct perf_probe_event
*pev
;
649 err
= bpf__prepare_probe();
651 pr_debug("bpf__prepare_probe failed\n");
655 bpf_object__for_each_program(prog
, obj
) {
656 err
= config_bpf_program(prog
);
660 priv
= bpf_program__priv(prog
);
661 if (IS_ERR(priv
) || !priv
) {
667 bpf_program__set_tracepoint(prog
);
671 bpf_program__set_kprobe(prog
);
674 err
= convert_perf_probe_events(pev
, 1);
676 pr_debug("bpf_probe: failed to convert perf probe events\n");
680 err
= apply_perf_probe_events(pev
, 1);
682 pr_debug("bpf_probe: failed to apply perf probe events\n");
687 * After probing, let's consider prologue, which
688 * adds program fetcher to BPF programs.
690 * hook_load_preprocessorr() hooks pre-processor
691 * to bpf_program, let it generate prologue
692 * dynamically during loading.
694 err
= hook_load_preprocessor(prog
);
699 return err
< 0 ? err
: 0;
702 #define EVENTS_WRITE_BUFSIZE 4096
703 int bpf__unprobe(struct bpf_object
*obj
)
706 struct bpf_program
*prog
;
708 bpf_object__for_each_program(prog
, obj
) {
709 struct bpf_prog_priv
*priv
= bpf_program__priv(prog
);
712 if (IS_ERR(priv
) || !priv
|| priv
->is_tp
)
715 for (i
= 0; i
< priv
->pev
.ntevs
; i
++) {
716 struct probe_trace_event
*tev
= &priv
->pev
.tevs
[i
];
717 char name_buf
[EVENTS_WRITE_BUFSIZE
];
718 struct strfilter
*delfilter
;
720 snprintf(name_buf
, EVENTS_WRITE_BUFSIZE
,
721 "%s:%s", tev
->group
, tev
->event
);
722 name_buf
[EVENTS_WRITE_BUFSIZE
- 1] = '\0';
724 delfilter
= strfilter__new(name_buf
, NULL
);
726 pr_debug("Failed to create filter for unprobing\n");
731 err
= del_perf_probe_events(delfilter
);
732 strfilter__delete(delfilter
);
734 pr_debug("Failed to delete %s\n", name_buf
);
743 int bpf__load(struct bpf_object
*obj
)
747 err
= bpf_object__load(obj
);
749 pr_debug("bpf: load objects failed\n");
755 int bpf__foreach_event(struct bpf_object
*obj
,
756 bpf_prog_iter_callback_t func
,
759 struct bpf_program
*prog
;
762 bpf_object__for_each_program(prog
, obj
) {
763 struct bpf_prog_priv
*priv
= bpf_program__priv(prog
);
764 struct probe_trace_event
*tev
;
765 struct perf_probe_event
*pev
;
768 if (IS_ERR(priv
) || !priv
) {
769 pr_debug("bpf: failed to get private field\n");
770 return -BPF_LOADER_ERRNO__INTERNAL
;
774 fd
= bpf_program__fd(prog
);
775 err
= (*func
)(priv
->sys_name
, priv
->evt_name
, fd
, arg
);
777 pr_debug("bpf: tracepoint call back failed, stop iterate\n");
784 for (i
= 0; i
< pev
->ntevs
; i
++) {
787 if (priv
->need_prologue
) {
788 int type
= priv
->type_mapping
[i
];
790 fd
= bpf_program__nth_fd(prog
, type
);
792 fd
= bpf_program__fd(prog
);
796 pr_debug("bpf: failed to get file descriptor\n");
800 err
= (*func
)(tev
->group
, tev
->event
, fd
, arg
);
802 pr_debug("bpf: call back failed, stop iterate\n");
810 enum bpf_map_op_type
{
811 BPF_MAP_OP_SET_VALUE
,
812 BPF_MAP_OP_SET_EVSEL
,
815 enum bpf_map_key_type
{
821 struct list_head list
;
822 enum bpf_map_op_type op_type
;
823 enum bpf_map_key_type key_type
;
825 struct parse_events_array array
;
829 struct perf_evsel
*evsel
;
833 struct bpf_map_priv
{
834 struct list_head ops_list
;
838 bpf_map_op__delete(struct bpf_map_op
*op
)
840 if (!list_empty(&op
->list
))
842 if (op
->key_type
== BPF_MAP_KEY_RANGES
)
843 parse_events__clear_array(&op
->k
.array
);
848 bpf_map_priv__purge(struct bpf_map_priv
*priv
)
850 struct bpf_map_op
*pos
, *n
;
852 list_for_each_entry_safe(pos
, n
, &priv
->ops_list
, list
) {
853 list_del_init(&pos
->list
);
854 bpf_map_op__delete(pos
);
859 bpf_map_priv__clear(struct bpf_map
*map __maybe_unused
,
862 struct bpf_map_priv
*priv
= _priv
;
864 bpf_map_priv__purge(priv
);
869 bpf_map_op_setkey(struct bpf_map_op
*op
, struct parse_events_term
*term
)
871 op
->key_type
= BPF_MAP_KEY_ALL
;
875 if (term
->array
.nr_ranges
) {
876 size_t memsz
= term
->array
.nr_ranges
*
877 sizeof(op
->k
.array
.ranges
[0]);
879 op
->k
.array
.ranges
= memdup(term
->array
.ranges
, memsz
);
880 if (!op
->k
.array
.ranges
) {
881 pr_debug("Not enough memory to alloc indices for map\n");
884 op
->key_type
= BPF_MAP_KEY_RANGES
;
885 op
->k
.array
.nr_ranges
= term
->array
.nr_ranges
;
890 static struct bpf_map_op
*
891 bpf_map_op__new(struct parse_events_term
*term
)
893 struct bpf_map_op
*op
;
896 op
= zalloc(sizeof(*op
));
898 pr_debug("Failed to alloc bpf_map_op\n");
899 return ERR_PTR(-ENOMEM
);
901 INIT_LIST_HEAD(&op
->list
);
903 err
= bpf_map_op_setkey(op
, term
);
911 static struct bpf_map_op
*
912 bpf_map_op__clone(struct bpf_map_op
*op
)
914 struct bpf_map_op
*newop
;
916 newop
= memdup(op
, sizeof(*op
));
918 pr_debug("Failed to alloc bpf_map_op\n");
922 INIT_LIST_HEAD(&newop
->list
);
923 if (op
->key_type
== BPF_MAP_KEY_RANGES
) {
924 size_t memsz
= op
->k
.array
.nr_ranges
*
925 sizeof(op
->k
.array
.ranges
[0]);
927 newop
->k
.array
.ranges
= memdup(op
->k
.array
.ranges
, memsz
);
928 if (!newop
->k
.array
.ranges
) {
929 pr_debug("Failed to alloc indices for map\n");
938 static struct bpf_map_priv
*
939 bpf_map_priv__clone(struct bpf_map_priv
*priv
)
941 struct bpf_map_priv
*newpriv
;
942 struct bpf_map_op
*pos
, *newop
;
944 newpriv
= zalloc(sizeof(*newpriv
));
946 pr_debug("Not enough memory to alloc map private\n");
949 INIT_LIST_HEAD(&newpriv
->ops_list
);
951 list_for_each_entry(pos
, &priv
->ops_list
, list
) {
952 newop
= bpf_map_op__clone(pos
);
954 bpf_map_priv__purge(newpriv
);
957 list_add_tail(&newop
->list
, &newpriv
->ops_list
);
964 bpf_map__add_op(struct bpf_map
*map
, struct bpf_map_op
*op
)
966 const char *map_name
= bpf_map__name(map
);
967 struct bpf_map_priv
*priv
= bpf_map__priv(map
);
970 pr_debug("Failed to get private from map %s\n", map_name
);
971 return PTR_ERR(priv
);
975 priv
= zalloc(sizeof(*priv
));
977 pr_debug("Not enough memory to alloc map private\n");
980 INIT_LIST_HEAD(&priv
->ops_list
);
982 if (bpf_map__set_priv(map
, priv
, bpf_map_priv__clear
)) {
984 return -BPF_LOADER_ERRNO__INTERNAL
;
988 list_add_tail(&op
->list
, &priv
->ops_list
);
992 static struct bpf_map_op
*
993 bpf_map__add_newop(struct bpf_map
*map
, struct parse_events_term
*term
)
995 struct bpf_map_op
*op
;
998 op
= bpf_map_op__new(term
);
1002 err
= bpf_map__add_op(map
, op
);
1004 bpf_map_op__delete(op
);
1005 return ERR_PTR(err
);
1011 __bpf_map__config_value(struct bpf_map
*map
,
1012 struct parse_events_term
*term
)
1014 struct bpf_map_op
*op
;
1015 const char *map_name
= bpf_map__name(map
);
1016 const struct bpf_map_def
*def
= bpf_map__def(map
);
1019 pr_debug("Unable to get map definition from '%s'\n",
1021 return -BPF_LOADER_ERRNO__INTERNAL
;
1024 if (def
->type
!= BPF_MAP_TYPE_ARRAY
) {
1025 pr_debug("Map %s type is not BPF_MAP_TYPE_ARRAY\n",
1027 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE
;
1029 if (def
->key_size
< sizeof(unsigned int)) {
1030 pr_debug("Map %s has incorrect key size\n", map_name
);
1031 return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE
;
1033 switch (def
->value_size
) {
1040 pr_debug("Map %s has incorrect value size\n", map_name
);
1041 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE
;
1044 op
= bpf_map__add_newop(map
, term
);
1047 op
->op_type
= BPF_MAP_OP_SET_VALUE
;
1048 op
->v
.value
= term
->val
.num
;
1053 bpf_map__config_value(struct bpf_map
*map
,
1054 struct parse_events_term
*term
,
1055 struct perf_evlist
*evlist __maybe_unused
)
1057 if (!term
->err_val
) {
1058 pr_debug("Config value not set\n");
1059 return -BPF_LOADER_ERRNO__OBJCONF_CONF
;
1062 if (term
->type_val
!= PARSE_EVENTS__TERM_TYPE_NUM
) {
1063 pr_debug("ERROR: wrong value type for 'value'\n");
1064 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE
;
1067 return __bpf_map__config_value(map
, term
);
1071 __bpf_map__config_event(struct bpf_map
*map
,
1072 struct parse_events_term
*term
,
1073 struct perf_evlist
*evlist
)
1075 struct perf_evsel
*evsel
;
1076 const struct bpf_map_def
*def
;
1077 struct bpf_map_op
*op
;
1078 const char *map_name
= bpf_map__name(map
);
1080 evsel
= perf_evlist__find_evsel_by_str(evlist
, term
->val
.str
);
1082 pr_debug("Event (for '%s') '%s' doesn't exist\n",
1083 map_name
, term
->val
.str
);
1084 return -BPF_LOADER_ERRNO__OBJCONF_MAP_NOEVT
;
1087 def
= bpf_map__def(map
);
1089 pr_debug("Unable to get map definition from '%s'\n",
1091 return PTR_ERR(def
);
1095 * No need to check key_size and value_size:
1096 * kernel has already checked them.
1098 if (def
->type
!= BPF_MAP_TYPE_PERF_EVENT_ARRAY
) {
1099 pr_debug("Map %s type is not BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
1101 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE
;
1104 op
= bpf_map__add_newop(map
, term
);
1107 op
->op_type
= BPF_MAP_OP_SET_EVSEL
;
1108 op
->v
.evsel
= evsel
;
1113 bpf_map__config_event(struct bpf_map
*map
,
1114 struct parse_events_term
*term
,
1115 struct perf_evlist
*evlist
)
1117 if (!term
->err_val
) {
1118 pr_debug("Config value not set\n");
1119 return -BPF_LOADER_ERRNO__OBJCONF_CONF
;
1122 if (term
->type_val
!= PARSE_EVENTS__TERM_TYPE_STR
) {
1123 pr_debug("ERROR: wrong value type for 'event'\n");
1124 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE
;
1127 return __bpf_map__config_event(map
, term
, evlist
);
1130 struct bpf_obj_config__map_func
{
1131 const char *config_opt
;
1132 int (*config_func
)(struct bpf_map
*, struct parse_events_term
*,
1133 struct perf_evlist
*);
1136 struct bpf_obj_config__map_func bpf_obj_config__map_funcs
[] = {
1137 {"value", bpf_map__config_value
},
1138 {"event", bpf_map__config_event
},
1142 config_map_indices_range_check(struct parse_events_term
*term
,
1143 struct bpf_map
*map
,
1144 const char *map_name
)
1146 struct parse_events_array
*array
= &term
->array
;
1147 const struct bpf_map_def
*def
;
1150 if (!array
->nr_ranges
)
1152 if (!array
->ranges
) {
1153 pr_debug("ERROR: map %s: array->nr_ranges is %d but range array is NULL\n",
1154 map_name
, (int)array
->nr_ranges
);
1155 return -BPF_LOADER_ERRNO__INTERNAL
;
1158 def
= bpf_map__def(map
);
1160 pr_debug("ERROR: Unable to get map definition from '%s'\n",
1162 return -BPF_LOADER_ERRNO__INTERNAL
;
1165 for (i
= 0; i
< array
->nr_ranges
; i
++) {
1166 unsigned int start
= array
->ranges
[i
].start
;
1167 size_t length
= array
->ranges
[i
].length
;
1168 unsigned int idx
= start
+ length
- 1;
1170 if (idx
>= def
->max_entries
) {
1171 pr_debug("ERROR: index %d too large\n", idx
);
1172 return -BPF_LOADER_ERRNO__OBJCONF_MAP_IDX2BIG
;
1179 bpf__obj_config_map(struct bpf_object
*obj
,
1180 struct parse_events_term
*term
,
1181 struct perf_evlist
*evlist
,
1184 /* key is "map:<mapname>.<config opt>" */
1185 char *map_name
= strdup(term
->config
+ sizeof("map:") - 1);
1186 struct bpf_map
*map
;
1187 int err
= -BPF_LOADER_ERRNO__OBJCONF_OPT
;
1194 map_opt
= strchr(map_name
, '.');
1196 pr_debug("ERROR: Invalid map config: %s\n", map_name
);
1201 if (*map_opt
== '\0') {
1202 pr_debug("ERROR: Invalid map option: %s\n", term
->config
);
1206 map
= bpf_object__find_map_by_name(obj
, map_name
);
1208 pr_debug("ERROR: Map %s doesn't exist\n", map_name
);
1209 err
= -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST
;
1213 *key_scan_pos
+= strlen(map_opt
);
1214 err
= config_map_indices_range_check(term
, map
, map_name
);
1217 *key_scan_pos
-= strlen(map_opt
);
1219 for (i
= 0; i
< ARRAY_SIZE(bpf_obj_config__map_funcs
); i
++) {
1220 struct bpf_obj_config__map_func
*func
=
1221 &bpf_obj_config__map_funcs
[i
];
1223 if (strcmp(map_opt
, func
->config_opt
) == 0) {
1224 err
= func
->config_func(map
, term
, evlist
);
1229 pr_debug("ERROR: Invalid map config option '%s'\n", map_opt
);
1230 err
= -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT
;
1234 key_scan_pos
+= strlen(map_opt
);
1238 int bpf__config_obj(struct bpf_object
*obj
,
1239 struct parse_events_term
*term
,
1240 struct perf_evlist
*evlist
,
1243 int key_scan_pos
= 0;
1246 if (!obj
|| !term
|| !term
->config
)
1249 if (!prefixcmp(term
->config
, "map:")) {
1250 key_scan_pos
= sizeof("map:") - 1;
1251 err
= bpf__obj_config_map(obj
, term
, evlist
, &key_scan_pos
);
1254 err
= -BPF_LOADER_ERRNO__OBJCONF_OPT
;
1257 *error_pos
= key_scan_pos
;
1262 typedef int (*map_config_func_t
)(const char *name
, int map_fd
,
1263 const struct bpf_map_def
*pdef
,
1264 struct bpf_map_op
*op
,
1265 void *pkey
, void *arg
);
1268 foreach_key_array_all(map_config_func_t func
,
1269 void *arg
, const char *name
,
1270 int map_fd
, const struct bpf_map_def
*pdef
,
1271 struct bpf_map_op
*op
)
1276 for (i
= 0; i
< pdef
->max_entries
; i
++) {
1277 err
= func(name
, map_fd
, pdef
, op
, &i
, arg
);
1279 pr_debug("ERROR: failed to insert value to %s[%u]\n",
1288 foreach_key_array_ranges(map_config_func_t func
, void *arg
,
1289 const char *name
, int map_fd
,
1290 const struct bpf_map_def
*pdef
,
1291 struct bpf_map_op
*op
)
1296 for (i
= 0; i
< op
->k
.array
.nr_ranges
; i
++) {
1297 unsigned int start
= op
->k
.array
.ranges
[i
].start
;
1298 size_t length
= op
->k
.array
.ranges
[i
].length
;
1300 for (j
= 0; j
< length
; j
++) {
1301 unsigned int idx
= start
+ j
;
1303 err
= func(name
, map_fd
, pdef
, op
, &idx
, arg
);
1305 pr_debug("ERROR: failed to insert value to %s[%u]\n",
1315 bpf_map_config_foreach_key(struct bpf_map
*map
,
1316 map_config_func_t func
,
1320 struct bpf_map_op
*op
;
1321 const struct bpf_map_def
*def
;
1322 const char *name
= bpf_map__name(map
);
1323 struct bpf_map_priv
*priv
= bpf_map__priv(map
);
1326 pr_debug("ERROR: failed to get private from map %s\n", name
);
1327 return -BPF_LOADER_ERRNO__INTERNAL
;
1329 if (!priv
|| list_empty(&priv
->ops_list
)) {
1330 pr_debug("INFO: nothing to config for map %s\n", name
);
1334 def
= bpf_map__def(map
);
1336 pr_debug("ERROR: failed to get definition from map %s\n", name
);
1337 return -BPF_LOADER_ERRNO__INTERNAL
;
1339 map_fd
= bpf_map__fd(map
);
1341 pr_debug("ERROR: failed to get fd from map %s\n", name
);
1345 list_for_each_entry(op
, &priv
->ops_list
, list
) {
1346 switch (def
->type
) {
1347 case BPF_MAP_TYPE_ARRAY
:
1348 case BPF_MAP_TYPE_PERF_EVENT_ARRAY
:
1349 switch (op
->key_type
) {
1350 case BPF_MAP_KEY_ALL
:
1351 err
= foreach_key_array_all(func
, arg
, name
,
1354 case BPF_MAP_KEY_RANGES
:
1355 err
= foreach_key_array_ranges(func
, arg
, name
,
1360 pr_debug("ERROR: keytype for map '%s' invalid\n",
1362 return -BPF_LOADER_ERRNO__INTERNAL
;
1368 pr_debug("ERROR: type of '%s' incorrect\n", name
);
1369 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE
;
1377 apply_config_value_for_key(int map_fd
, void *pkey
,
1378 size_t val_size
, u64 val
)
1384 u8 _val
= (u8
)(val
);
1385 err
= bpf_map_update_elem(map_fd
, pkey
, &_val
, BPF_ANY
);
1389 u16 _val
= (u16
)(val
);
1390 err
= bpf_map_update_elem(map_fd
, pkey
, &_val
, BPF_ANY
);
1394 u32 _val
= (u32
)(val
);
1395 err
= bpf_map_update_elem(map_fd
, pkey
, &_val
, BPF_ANY
);
1399 err
= bpf_map_update_elem(map_fd
, pkey
, &val
, BPF_ANY
);
1403 pr_debug("ERROR: invalid value size\n");
1404 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE
;
1412 apply_config_evsel_for_key(const char *name
, int map_fd
, void *pkey
,
1413 struct perf_evsel
*evsel
)
1415 struct xyarray
*xy
= evsel
->fd
;
1416 struct perf_event_attr
*attr
;
1417 unsigned int key
, events
;
1418 bool check_pass
= false;
1423 pr_debug("ERROR: evsel not ready for map %s\n", name
);
1424 return -BPF_LOADER_ERRNO__INTERNAL
;
1427 if (xy
->row_size
/ xy
->entry_size
!= 1) {
1428 pr_debug("ERROR: Dimension of target event is incorrect for map %s\n",
1430 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM
;
1433 attr
= &evsel
->attr
;
1434 if (attr
->inherit
) {
1435 pr_debug("ERROR: Can't put inherit event into map %s\n", name
);
1436 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH
;
1439 if (perf_evsel__is_bpf_output(evsel
))
1441 if (attr
->type
== PERF_TYPE_RAW
)
1443 if (attr
->type
== PERF_TYPE_HARDWARE
)
1446 pr_debug("ERROR: Event type is wrong for map %s\n", name
);
1447 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE
;
1450 events
= xy
->entries
/ (xy
->row_size
/ xy
->entry_size
);
1451 key
= *((unsigned int *)pkey
);
1452 if (key
>= events
) {
1453 pr_debug("ERROR: there is no event %d for map %s\n",
1455 return -BPF_LOADER_ERRNO__OBJCONF_MAP_MAPSIZE
;
1457 evt_fd
= xyarray__entry(xy
, key
, 0);
1458 err
= bpf_map_update_elem(map_fd
, pkey
, evt_fd
, BPF_ANY
);
1465 apply_obj_config_map_for_key(const char *name
, int map_fd
,
1466 const struct bpf_map_def
*pdef
,
1467 struct bpf_map_op
*op
,
1468 void *pkey
, void *arg __maybe_unused
)
1472 switch (op
->op_type
) {
1473 case BPF_MAP_OP_SET_VALUE
:
1474 err
= apply_config_value_for_key(map_fd
, pkey
,
1478 case BPF_MAP_OP_SET_EVSEL
:
1479 err
= apply_config_evsel_for_key(name
, map_fd
, pkey
,
1483 pr_debug("ERROR: unknown value type for '%s'\n", name
);
1484 err
= -BPF_LOADER_ERRNO__INTERNAL
;
1490 apply_obj_config_map(struct bpf_map
*map
)
1492 return bpf_map_config_foreach_key(map
,
1493 apply_obj_config_map_for_key
,
1498 apply_obj_config_object(struct bpf_object
*obj
)
1500 struct bpf_map
*map
;
1503 bpf_map__for_each(map
, obj
) {
1504 err
= apply_obj_config_map(map
);
1511 int bpf__apply_obj_config(void)
1513 struct bpf_object
*obj
, *tmp
;
1516 bpf_object__for_each_safe(obj
, tmp
) {
1517 err
= apply_obj_config_object(obj
);
1525 #define bpf__for_each_map(pos, obj, objtmp) \
1526 bpf_object__for_each_safe(obj, objtmp) \
1527 bpf_map__for_each(pos, obj)
1529 #define bpf__for_each_stdout_map(pos, obj, objtmp) \
1530 bpf__for_each_map(pos, obj, objtmp) \
1531 if (bpf_map__name(pos) && \
1532 (strcmp("__bpf_stdout__", \
1533 bpf_map__name(pos)) == 0))
1535 int bpf__setup_stdout(struct perf_evlist
*evlist __maybe_unused
)
1537 struct bpf_map_priv
*tmpl_priv
= NULL
;
1538 struct bpf_object
*obj
, *tmp
;
1539 struct perf_evsel
*evsel
= NULL
;
1540 struct bpf_map
*map
;
1542 bool need_init
= false;
1544 bpf__for_each_stdout_map(map
, obj
, tmp
) {
1545 struct bpf_map_priv
*priv
= bpf_map__priv(map
);
1548 return -BPF_LOADER_ERRNO__INTERNAL
;
1551 * No need to check map type: type should have been
1552 * verified by kernel.
1554 if (!need_init
&& !priv
)
1556 if (!tmpl_priv
&& priv
)
1564 err
= parse_events(evlist
, "bpf-output/no-inherit=1,name=__bpf_stdout__/",
1567 pr_debug("ERROR: failed to create bpf-output event\n");
1571 evsel
= perf_evlist__last(evlist
);
1574 bpf__for_each_stdout_map(map
, obj
, tmp
) {
1575 struct bpf_map_priv
*priv
= bpf_map__priv(map
);
1578 return -BPF_LOADER_ERRNO__INTERNAL
;
1583 priv
= bpf_map_priv__clone(tmpl_priv
);
1587 err
= bpf_map__set_priv(map
, priv
, bpf_map_priv__clear
);
1589 bpf_map_priv__clear(map
, priv
);
1593 struct bpf_map_op
*op
;
1595 op
= bpf_map__add_newop(map
, NULL
);
1598 op
->op_type
= BPF_MAP_OP_SET_EVSEL
;
1599 op
->v
.evsel
= evsel
;
1606 #define ERRNO_OFFSET(e) ((e) - __BPF_LOADER_ERRNO__START)
1607 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
1608 #define NR_ERRNO (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
1610 static const char *bpf_loader_strerror_table
[NR_ERRNO
] = {
1611 [ERRCODE_OFFSET(CONFIG
)] = "Invalid config string",
1612 [ERRCODE_OFFSET(GROUP
)] = "Invalid group name",
1613 [ERRCODE_OFFSET(EVENTNAME
)] = "No event name found in config string",
1614 [ERRCODE_OFFSET(INTERNAL
)] = "BPF loader internal error",
1615 [ERRCODE_OFFSET(COMPILE
)] = "Error when compiling BPF scriptlet",
1616 [ERRCODE_OFFSET(PROGCONF_TERM
)] = "Invalid program config term in config string",
1617 [ERRCODE_OFFSET(PROLOGUE
)] = "Failed to generate prologue",
1618 [ERRCODE_OFFSET(PROLOGUE2BIG
)] = "Prologue too big for program",
1619 [ERRCODE_OFFSET(PROLOGUEOOB
)] = "Offset out of bound for prologue",
1620 [ERRCODE_OFFSET(OBJCONF_OPT
)] = "Invalid object config option",
1621 [ERRCODE_OFFSET(OBJCONF_CONF
)] = "Config value not set (missing '=')",
1622 [ERRCODE_OFFSET(OBJCONF_MAP_OPT
)] = "Invalid object map config option",
1623 [ERRCODE_OFFSET(OBJCONF_MAP_NOTEXIST
)] = "Target map doesn't exist",
1624 [ERRCODE_OFFSET(OBJCONF_MAP_VALUE
)] = "Incorrect value type for map",
1625 [ERRCODE_OFFSET(OBJCONF_MAP_TYPE
)] = "Incorrect map type",
1626 [ERRCODE_OFFSET(OBJCONF_MAP_KEYSIZE
)] = "Incorrect map key size",
1627 [ERRCODE_OFFSET(OBJCONF_MAP_VALUESIZE
)] = "Incorrect map value size",
1628 [ERRCODE_OFFSET(OBJCONF_MAP_NOEVT
)] = "Event not found for map setting",
1629 [ERRCODE_OFFSET(OBJCONF_MAP_MAPSIZE
)] = "Invalid map size for event setting",
1630 [ERRCODE_OFFSET(OBJCONF_MAP_EVTDIM
)] = "Event dimension too large",
1631 [ERRCODE_OFFSET(OBJCONF_MAP_EVTINH
)] = "Doesn't support inherit event",
1632 [ERRCODE_OFFSET(OBJCONF_MAP_EVTTYPE
)] = "Wrong event type for map",
1633 [ERRCODE_OFFSET(OBJCONF_MAP_IDX2BIG
)] = "Index too large",
1637 bpf_loader_strerror(int err
, char *buf
, size_t size
)
1639 char sbuf
[STRERR_BUFSIZE
];
1645 err
= err
> 0 ? err
: -err
;
1647 if (err
>= __LIBBPF_ERRNO__START
)
1648 return libbpf_strerror(err
, buf
, size
);
1650 if (err
>= __BPF_LOADER_ERRNO__START
&& err
< __BPF_LOADER_ERRNO__END
) {
1651 msg
= bpf_loader_strerror_table
[ERRNO_OFFSET(err
)];
1652 snprintf(buf
, size
, "%s", msg
);
1653 buf
[size
- 1] = '\0';
1657 if (err
>= __BPF_LOADER_ERRNO__END
)
1658 snprintf(buf
, size
, "Unknown bpf loader error %d", err
);
1660 snprintf(buf
, size
, "%s",
1661 str_error_r(err
, sbuf
, sizeof(sbuf
)));
1663 buf
[size
- 1] = '\0';
1667 #define bpf__strerror_head(err, buf, size) \
1668 char sbuf[STRERR_BUFSIZE], *emsg;\
1673 bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
1677 scnprintf(buf, size, "%s", emsg);\
1680 #define bpf__strerror_entry(val, fmt...)\
1682 scnprintf(buf, size, fmt);\
1686 #define bpf__strerror_end(buf, size)\
1688 buf[size - 1] = '\0';
1690 int bpf__strerror_prepare_load(const char *filename
, bool source
,
1691 int err
, char *buf
, size_t size
)
1696 n
= snprintf(buf
, size
, "Failed to load %s%s: ",
1697 filename
, source
? " from source" : "");
1699 buf
[size
- 1] = '\0';
1705 ret
= bpf_loader_strerror(err
, buf
, size
);
1706 buf
[size
- 1] = '\0';
1710 int bpf__strerror_probe(struct bpf_object
*obj __maybe_unused
,
1711 int err
, char *buf
, size_t size
)
1713 bpf__strerror_head(err
, buf
, size
);
1714 case BPF_LOADER_ERRNO__PROGCONF_TERM
: {
1715 scnprintf(buf
, size
, "%s (add -v to see detail)", emsg
);
1718 bpf__strerror_entry(EEXIST
, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'");
1719 bpf__strerror_entry(EACCES
, "You need to be root");
1720 bpf__strerror_entry(EPERM
, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
1721 bpf__strerror_entry(ENOENT
, "You need to check probing points in BPF file");
1722 bpf__strerror_end(buf
, size
);
1726 int bpf__strerror_load(struct bpf_object
*obj
,
1727 int err
, char *buf
, size_t size
)
1729 bpf__strerror_head(err
, buf
, size
);
1730 case LIBBPF_ERRNO__KVER
: {
1731 unsigned int obj_kver
= bpf_object__kversion(obj
);
1732 unsigned int real_kver
;
1734 if (fetch_kernel_version(&real_kver
, NULL
, 0)) {
1735 scnprintf(buf
, size
, "Unable to fetch kernel version");
1739 if (obj_kver
!= real_kver
) {
1740 scnprintf(buf
, size
,
1741 "'version' ("KVER_FMT
") doesn't match running kernel ("KVER_FMT
")",
1742 KVER_PARAM(obj_kver
),
1743 KVER_PARAM(real_kver
));
1747 scnprintf(buf
, size
, "Failed to load program for unknown reason");
1750 bpf__strerror_end(buf
, size
);
1754 int bpf__strerror_config_obj(struct bpf_object
*obj __maybe_unused
,
1755 struct parse_events_term
*term __maybe_unused
,
1756 struct perf_evlist
*evlist __maybe_unused
,
1757 int *error_pos __maybe_unused
, int err
,
1758 char *buf
, size_t size
)
1760 bpf__strerror_head(err
, buf
, size
);
1761 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE
,
1762 "Can't use this config term with this map type");
1763 bpf__strerror_end(buf
, size
);
1767 int bpf__strerror_apply_obj_config(int err
, char *buf
, size_t size
)
1769 bpf__strerror_head(err
, buf
, size
);
1770 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM
,
1771 "Cannot set event to BPF map in multi-thread tracing");
1772 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH
,
1773 "%s (Hint: use -i to turn off inherit)", emsg
);
1774 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE
,
1775 "Can only put raw, hardware and BPF output event into a BPF map");
1776 bpf__strerror_end(buf
, size
);
1780 int bpf__strerror_setup_stdout(struct perf_evlist
*evlist __maybe_unused
,
1781 int err
, char *buf
, size_t size
)
1783 bpf__strerror_head(err
, buf
, size
);
1784 bpf__strerror_end(buf
, size
);