1 // SPDX-License-Identifier: GPL-2.0
6 #include <bpf/libbpf.h>
9 #include <linux/string.h>
10 #include <internal/lib.h>
11 #include <symbol/kallsyms.h>
12 #include "bpf-event.h"
22 #include "util/synthetic-events.h"
24 struct btf
* __weak
btf__load_from_kernel_by_id(__u32 id
)
27 #pragma GCC diagnostic push
28 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
29 int err
= btf__get_from_id(id
, &btf
);
30 #pragma GCC diagnostic pop
32 return err
? ERR_PTR(err
) : btf
;
35 #define ptr_to_u64(ptr) ((__u64)(unsigned long)(ptr))
37 static int snprintf_hex(char *buf
, size_t size
, unsigned char *data
, size_t len
)
42 for (i
= 0; i
< len
; i
++)
43 ret
+= snprintf(buf
+ ret
, size
- ret
, "%02x", data
[i
]);
47 static int machine__process_bpf_event_load(struct machine
*machine
,
48 union perf_event
*event
,
49 struct perf_sample
*sample __maybe_unused
)
51 struct bpf_prog_info_linear
*info_linear
;
52 struct bpf_prog_info_node
*info_node
;
53 struct perf_env
*env
= machine
->env
;
54 int id
= event
->bpf
.id
;
57 /* perf-record, no need to handle bpf-event */
61 info_node
= perf_env__find_bpf_prog_info(env
, id
);
64 info_linear
= info_node
->info_linear
;
66 for (i
= 0; i
< info_linear
->info
.nr_jited_ksyms
; i
++) {
67 u64
*addrs
= (u64
*)(uintptr_t)(info_linear
->info
.jited_ksyms
);
69 struct map
*map
= maps__find(&machine
->kmaps
, addr
);
72 map
->dso
->binary_type
= DSO_BINARY_TYPE__BPF_PROG_INFO
;
73 map
->dso
->bpf_prog
.id
= id
;
74 map
->dso
->bpf_prog
.sub_id
= i
;
75 map
->dso
->bpf_prog
.env
= env
;
81 int machine__process_bpf(struct machine
*machine
, union perf_event
*event
,
82 struct perf_sample
*sample
)
85 perf_event__fprintf_bpf(event
, stdout
);
87 switch (event
->bpf
.type
) {
88 case PERF_BPF_EVENT_PROG_LOAD
:
89 return machine__process_bpf_event_load(machine
, event
, sample
);
91 case PERF_BPF_EVENT_PROG_UNLOAD
:
93 * Do not free bpf_prog_info and btf of the program here,
94 * as annotation still need them. They will be freed at
95 * the end of the session.
99 pr_debug("unexpected bpf event type of %d\n", event
->bpf
.type
);
105 static int perf_env__fetch_btf(struct perf_env
*env
,
109 struct btf_node
*node
;
113 data
= btf__get_raw_data(btf
, &data_size
);
115 node
= malloc(data_size
+ sizeof(struct btf_node
));
120 node
->data_size
= data_size
;
121 memcpy(node
->data
, data
, data_size
);
123 perf_env__insert_btf(env
, node
);
127 static int synthesize_bpf_prog_name(char *buf
, int size
,
128 struct bpf_prog_info
*info
,
132 u8 (*prog_tags
)[BPF_TAG_SIZE
] = (void *)(uintptr_t)(info
->prog_tags
);
133 void *func_infos
= (void *)(uintptr_t)(info
->func_info
);
134 u32 sub_prog_cnt
= info
->nr_jited_ksyms
;
135 const struct bpf_func_info
*finfo
;
136 const char *short_name
= NULL
;
137 const struct btf_type
*t
;
140 name_len
= snprintf(buf
, size
, "bpf_prog_");
141 name_len
+= snprintf_hex(buf
+ name_len
, size
- name_len
,
142 prog_tags
[sub_id
], BPF_TAG_SIZE
);
144 finfo
= func_infos
+ sub_id
* info
->func_info_rec_size
;
145 t
= btf__type_by_id(btf
, finfo
->type_id
);
146 short_name
= btf__name_by_offset(btf
, t
->name_off
);
147 } else if (sub_id
== 0 && sub_prog_cnt
== 1) {
150 short_name
= info
->name
;
154 name_len
+= snprintf(buf
+ name_len
, size
- name_len
,
160 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf
161 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And
162 * one PERF_RECORD_KSYMBOL is generated for each sub program.
167 * -2 for lack of kernel support.
169 static int perf_event__synthesize_one_bpf_prog(struct perf_session
*session
,
170 perf_event__handler_t process
,
171 struct machine
*machine
,
173 union perf_event
*event
,
174 struct record_opts
*opts
)
176 struct perf_record_ksymbol
*ksymbol_event
= &event
->ksymbol
;
177 struct perf_record_bpf_event
*bpf_event
= &event
->bpf
;
178 struct bpf_prog_info_linear
*info_linear
;
179 struct perf_tool
*tool
= session
->tool
;
180 struct bpf_prog_info_node
*info_node
;
181 struct bpf_prog_info
*info
;
182 struct btf
*btf
= NULL
;
183 struct perf_env
*env
;
189 * for perf-record and perf-report use header.env;
190 * otherwise, use global perf_env.
192 env
= session
->data
? &session
->header
.env
: &perf_env
;
194 arrays
= 1UL << BPF_PROG_INFO_JITED_KSYMS
;
195 arrays
|= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS
;
196 arrays
|= 1UL << BPF_PROG_INFO_FUNC_INFO
;
197 arrays
|= 1UL << BPF_PROG_INFO_PROG_TAGS
;
198 arrays
|= 1UL << BPF_PROG_INFO_JITED_INSNS
;
199 arrays
|= 1UL << BPF_PROG_INFO_LINE_INFO
;
200 arrays
|= 1UL << BPF_PROG_INFO_JITED_LINE_INFO
;
202 info_linear
= bpf_program__get_prog_info_linear(fd
, arrays
);
203 if (IS_ERR_OR_NULL(info_linear
)) {
205 pr_debug("%s: failed to get BPF program info. aborting\n", __func__
);
209 if (info_linear
->info_len
< offsetof(struct bpf_prog_info
, prog_tags
)) {
211 pr_debug("%s: the kernel is too old, aborting\n", __func__
);
215 info
= &info_linear
->info
;
216 if (!info
->jited_ksyms
) {
221 /* number of ksyms, func_lengths, and tags should match */
222 sub_prog_cnt
= info
->nr_jited_ksyms
;
223 if (sub_prog_cnt
!= info
->nr_prog_tags
||
224 sub_prog_cnt
!= info
->nr_jited_func_lens
) {
229 /* check BTF func info support */
230 if (info
->btf_id
&& info
->nr_func_info
&& info
->func_info_rec_size
) {
231 /* btf func info number should be same as sub_prog_cnt */
232 if (sub_prog_cnt
!= info
->nr_func_info
) {
233 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__
);
237 btf
= btf__load_from_kernel_by_id(info
->btf_id
);
238 if (libbpf_get_error(btf
)) {
239 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__
, info
->btf_id
);
243 perf_env__fetch_btf(env
, info
->btf_id
, btf
);
246 /* Synthesize PERF_RECORD_KSYMBOL */
247 for (i
= 0; i
< sub_prog_cnt
; i
++) {
248 __u32
*prog_lens
= (__u32
*)(uintptr_t)(info
->jited_func_lens
);
249 __u64
*prog_addrs
= (__u64
*)(uintptr_t)(info
->jited_ksyms
);
252 *ksymbol_event
= (struct perf_record_ksymbol
) {
254 .type
= PERF_RECORD_KSYMBOL
,
255 .size
= offsetof(struct perf_record_ksymbol
, name
),
257 .addr
= prog_addrs
[i
],
259 .ksym_type
= PERF_RECORD_KSYMBOL_TYPE_BPF
,
263 name_len
= synthesize_bpf_prog_name(ksymbol_event
->name
,
264 KSYM_NAME_LEN
, info
, btf
, i
);
265 ksymbol_event
->header
.size
+= PERF_ALIGN(name_len
+ 1,
268 memset((void *)event
+ event
->header
.size
, 0, machine
->id_hdr_size
);
269 event
->header
.size
+= machine
->id_hdr_size
;
270 err
= perf_tool__process_synth_event(tool
, event
,
274 if (!opts
->no_bpf_event
) {
275 /* Synthesize PERF_RECORD_BPF_EVENT */
276 *bpf_event
= (struct perf_record_bpf_event
) {
278 .type
= PERF_RECORD_BPF_EVENT
,
279 .size
= sizeof(struct perf_record_bpf_event
),
281 .type
= PERF_BPF_EVENT_PROG_LOAD
,
285 memcpy(bpf_event
->tag
, info
->tag
, BPF_TAG_SIZE
);
286 memset((void *)event
+ event
->header
.size
, 0, machine
->id_hdr_size
);
287 event
->header
.size
+= machine
->id_hdr_size
;
289 /* save bpf_prog_info to env */
290 info_node
= malloc(sizeof(struct bpf_prog_info_node
));
296 info_node
->info_linear
= info_linear
;
297 perf_env__insert_bpf_prog_info(env
, info_node
);
301 * process after saving bpf_prog_info to env, so that
302 * required information is ready for look up
304 err
= perf_tool__process_synth_event(tool
, event
,
314 struct kallsyms_parse
{
315 union perf_event
*event
;
316 perf_event__handler_t process
;
317 struct machine
*machine
;
318 struct perf_tool
*tool
;
322 process_bpf_image(char *name
, u64 addr
, struct kallsyms_parse
*data
)
324 struct machine
*machine
= data
->machine
;
325 union perf_event
*event
= data
->event
;
326 struct perf_record_ksymbol
*ksymbol
;
329 ksymbol
= &event
->ksymbol
;
331 *ksymbol
= (struct perf_record_ksymbol
) {
333 .type
= PERF_RECORD_KSYMBOL
,
334 .size
= offsetof(struct perf_record_ksymbol
, name
),
338 .ksym_type
= PERF_RECORD_KSYMBOL_TYPE_BPF
,
342 len
= scnprintf(ksymbol
->name
, KSYM_NAME_LEN
, "%s", name
);
343 ksymbol
->header
.size
+= PERF_ALIGN(len
+ 1, sizeof(u64
));
344 memset((void *) event
+ event
->header
.size
, 0, machine
->id_hdr_size
);
345 event
->header
.size
+= machine
->id_hdr_size
;
347 return perf_tool__process_synth_event(data
->tool
, event
, machine
,
352 kallsyms_process_symbol(void *data
, const char *_name
,
353 char type __maybe_unused
, u64 start
)
355 char disp
[KSYM_NAME_LEN
];
360 module
= strchr(_name
, '\t');
364 /* We are going after [bpf] module ... */
365 if (strcmp(module
+ 1, "[bpf]"))
368 name
= memdup(_name
, (module
- _name
) + 1);
372 name
[module
- _name
] = 0;
374 /* .. and only for trampolines and dispatchers */
375 if ((sscanf(name
, "bpf_trampoline_%lu", &id
) == 1) ||
376 (sscanf(name
, "bpf_dispatcher_%s", disp
) == 1))
377 err
= process_bpf_image(name
, start
, data
);
383 int perf_event__synthesize_bpf_events(struct perf_session
*session
,
384 perf_event__handler_t process
,
385 struct machine
*machine
,
386 struct record_opts
*opts
)
388 const char *kallsyms_filename
= "/proc/kallsyms";
389 struct kallsyms_parse arg
;
390 union perf_event
*event
;
395 event
= malloc(sizeof(event
->bpf
) + KSYM_NAME_LEN
+ machine
->id_hdr_size
);
399 /* Synthesize all the bpf programs in system. */
401 err
= bpf_prog_get_next_id(id
, &id
);
403 if (errno
== ENOENT
) {
407 pr_debug("%s: can't get next program: %s%s\n",
408 __func__
, strerror(errno
),
409 errno
== EINVAL
? " -- kernel too old?" : "");
410 /* don't report error on old kernel or EPERM */
411 err
= (errno
== EINVAL
|| errno
== EPERM
) ? 0 : -1;
414 fd
= bpf_prog_get_fd_by_id(id
);
416 pr_debug("%s: failed to get fd for prog_id %u\n",
421 err
= perf_event__synthesize_one_bpf_prog(session
, process
,
426 /* do not return error for old kernel */
433 /* Synthesize all the bpf images - trampolines/dispatchers. */
434 if (symbol_conf
.kallsyms_name
!= NULL
)
435 kallsyms_filename
= symbol_conf
.kallsyms_name
;
437 arg
= (struct kallsyms_parse
) {
441 .tool
= session
->tool
,
444 if (kallsyms__parse(kallsyms_filename
, &arg
, kallsyms_process_symbol
)) {
445 pr_err("%s: failed to synthesize bpf images: %s\n",
446 __func__
, strerror(errno
));
453 static void perf_env__add_bpf_info(struct perf_env
*env
, u32 id
)
455 struct bpf_prog_info_linear
*info_linear
;
456 struct bpf_prog_info_node
*info_node
;
457 struct btf
*btf
= NULL
;
462 fd
= bpf_prog_get_fd_by_id(id
);
466 arrays
= 1UL << BPF_PROG_INFO_JITED_KSYMS
;
467 arrays
|= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS
;
468 arrays
|= 1UL << BPF_PROG_INFO_FUNC_INFO
;
469 arrays
|= 1UL << BPF_PROG_INFO_PROG_TAGS
;
470 arrays
|= 1UL << BPF_PROG_INFO_JITED_INSNS
;
471 arrays
|= 1UL << BPF_PROG_INFO_LINE_INFO
;
472 arrays
|= 1UL << BPF_PROG_INFO_JITED_LINE_INFO
;
474 info_linear
= bpf_program__get_prog_info_linear(fd
, arrays
);
475 if (IS_ERR_OR_NULL(info_linear
)) {
476 pr_debug("%s: failed to get BPF program info. aborting\n", __func__
);
480 btf_id
= info_linear
->info
.btf_id
;
482 info_node
= malloc(sizeof(struct bpf_prog_info_node
));
484 info_node
->info_linear
= info_linear
;
485 perf_env__insert_bpf_prog_info(env
, info_node
);
492 btf
= btf__load_from_kernel_by_id(btf_id
);
493 if (libbpf_get_error(btf
)) {
494 pr_debug("%s: failed to get BTF of id %u, aborting\n",
498 perf_env__fetch_btf(env
, btf_id
, btf
);
505 static int bpf_event__sb_cb(union perf_event
*event
, void *data
)
507 struct perf_env
*env
= data
;
509 if (event
->header
.type
!= PERF_RECORD_BPF_EVENT
)
512 switch (event
->bpf
.type
) {
513 case PERF_BPF_EVENT_PROG_LOAD
:
514 perf_env__add_bpf_info(env
, event
->bpf
.id
);
516 case PERF_BPF_EVENT_PROG_UNLOAD
:
518 * Do not free bpf_prog_info and btf of the program here,
519 * as annotation still need them. They will be freed at
520 * the end of the session.
524 pr_debug("unexpected bpf event type of %d\n", event
->bpf
.type
);
531 int evlist__add_bpf_sb_event(struct evlist
*evlist
, struct perf_env
*env
)
533 struct perf_event_attr attr
= {
534 .type
= PERF_TYPE_SOFTWARE
,
535 .config
= PERF_COUNT_SW_DUMMY
,
539 .size
= sizeof(attr
), /* to capture ABI version */
543 * Older gcc versions don't support designated initializers, like above,
544 * for unnamed union members, such as the following:
546 attr
.wakeup_watermark
= 1;
548 return evlist__add_sb_event(evlist
, &attr
, bpf_event__sb_cb
, env
);
551 void bpf_event__print_bpf_prog_info(struct bpf_prog_info
*info
,
552 struct perf_env
*env
,
555 __u32
*prog_lens
= (__u32
*)(uintptr_t)(info
->jited_func_lens
);
556 __u64
*prog_addrs
= (__u64
*)(uintptr_t)(info
->jited_ksyms
);
557 char name
[KSYM_NAME_LEN
];
558 struct btf
*btf
= NULL
;
561 sub_prog_cnt
= info
->nr_jited_ksyms
;
562 if (sub_prog_cnt
!= info
->nr_prog_tags
||
563 sub_prog_cnt
!= info
->nr_jited_func_lens
)
567 struct btf_node
*node
;
569 node
= perf_env__find_btf(env
, info
->btf_id
);
571 btf
= btf__new((__u8
*)(node
->data
),
575 if (sub_prog_cnt
== 1) {
576 synthesize_bpf_prog_name(name
, KSYM_NAME_LEN
, info
, btf
, 0);
577 fprintf(fp
, "# bpf_prog_info %u: %s addr 0x%llx size %u\n",
578 info
->id
, name
, prog_addrs
[0], prog_lens
[0]);
582 fprintf(fp
, "# bpf_prog_info %u:\n", info
->id
);
583 for (i
= 0; i
< sub_prog_cnt
; i
++) {
584 synthesize_bpf_prog_name(name
, KSYM_NAME_LEN
, info
, btf
, i
);
586 fprintf(fp
, "# \tsub_prog %u: %s addr 0x%llx size %u\n",
587 i
, name
, prog_addrs
[i
], prog_lens
[i
]);