1 // SPDX-License-Identifier: GPL-2.0
13 #include <linux/bpf.h>
14 #include <linux/filter.h>
15 #include <linux/perf_event.h>
16 #include <linux/netlink.h>
17 #include <linux/rtnetlink.h>
18 #include <linux/types.h>
19 #include <sys/types.h>
20 #include <sys/socket.h>
21 #include <sys/syscall.h>
22 #include <sys/ioctl.h>
31 #define DEBUGFS "/sys/kernel/debug/tracing/"
33 static char license
[128];
34 static int kern_version
;
35 static bool processed_sec
[128];
36 char bpf_log_buf
[BPF_LOG_BUF_SIZE
];
38 int prog_fd
[MAX_PROGS
];
39 int event_fd
[MAX_PROGS
];
41 int prog_array_fd
= -1;
43 struct bpf_map_data map_data
[MAX_MAPS
];
44 int map_data_count
= 0;
46 static int populate_prog_array(const char *event
, int prog_fd
)
48 int ind
= atoi(event
), err
;
50 err
= bpf_map_update_elem(prog_array_fd
, &ind
, &prog_fd
, BPF_ANY
);
52 printf("failed to store prog_fd in prog_array\n");
58 static int load_and_attach(const char *event
, struct bpf_insn
*prog
, int size
)
60 bool is_socket
= strncmp(event
, "socket", 6) == 0;
61 bool is_kprobe
= strncmp(event
, "kprobe/", 7) == 0;
62 bool is_kretprobe
= strncmp(event
, "kretprobe/", 10) == 0;
63 bool is_tracepoint
= strncmp(event
, "tracepoint/", 11) == 0;
64 bool is_xdp
= strncmp(event
, "xdp", 3) == 0;
65 bool is_perf_event
= strncmp(event
, "perf_event", 10) == 0;
66 bool is_cgroup_skb
= strncmp(event
, "cgroup/skb", 10) == 0;
67 bool is_cgroup_sk
= strncmp(event
, "cgroup/sock", 11) == 0;
68 bool is_sockops
= strncmp(event
, "sockops", 7) == 0;
69 bool is_sk_skb
= strncmp(event
, "sk_skb", 6) == 0;
70 size_t insns_cnt
= size
/ sizeof(struct bpf_insn
);
71 enum bpf_prog_type prog_type
;
74 struct perf_event_attr attr
= {};
76 attr
.type
= PERF_TYPE_TRACEPOINT
;
77 attr
.sample_type
= PERF_SAMPLE_RAW
;
78 attr
.sample_period
= 1;
79 attr
.wakeup_events
= 1;
82 prog_type
= BPF_PROG_TYPE_SOCKET_FILTER
;
83 } else if (is_kprobe
|| is_kretprobe
) {
84 prog_type
= BPF_PROG_TYPE_KPROBE
;
85 } else if (is_tracepoint
) {
86 prog_type
= BPF_PROG_TYPE_TRACEPOINT
;
88 prog_type
= BPF_PROG_TYPE_XDP
;
89 } else if (is_perf_event
) {
90 prog_type
= BPF_PROG_TYPE_PERF_EVENT
;
91 } else if (is_cgroup_skb
) {
92 prog_type
= BPF_PROG_TYPE_CGROUP_SKB
;
93 } else if (is_cgroup_sk
) {
94 prog_type
= BPF_PROG_TYPE_CGROUP_SOCK
;
95 } else if (is_sockops
) {
96 prog_type
= BPF_PROG_TYPE_SOCK_OPS
;
97 } else if (is_sk_skb
) {
98 prog_type
= BPF_PROG_TYPE_SK_SKB
;
100 printf("Unknown event '%s'\n", event
);
104 fd
= bpf_load_program(prog_type
, prog
, insns_cnt
, license
, kern_version
,
105 bpf_log_buf
, BPF_LOG_BUF_SIZE
);
107 printf("bpf_load_program() err=%d\n%s", errno
, bpf_log_buf
);
111 prog_fd
[prog_cnt
++] = fd
;
113 if (is_xdp
|| is_perf_event
|| is_cgroup_skb
|| is_cgroup_sk
)
116 if (is_socket
|| is_sockops
|| is_sk_skb
) {
124 if (!isdigit(*event
)) {
125 printf("invalid prog number\n");
128 return populate_prog_array(event
, fd
);
131 if (is_kprobe
|| is_kretprobe
) {
138 printf("event name cannot be empty\n");
143 return populate_prog_array(event
, fd
);
145 snprintf(buf
, sizeof(buf
),
146 "echo '%c:%s %s' >> /sys/kernel/debug/tracing/kprobe_events",
147 is_kprobe
? 'p' : 'r', event
, event
);
150 printf("failed to create kprobe '%s' error '%s'\n",
151 event
, strerror(errno
));
155 strcpy(buf
, DEBUGFS
);
156 strcat(buf
, "events/kprobes/");
159 } else if (is_tracepoint
) {
163 printf("event name cannot be empty\n");
166 strcpy(buf
, DEBUGFS
);
167 strcat(buf
, "events/");
172 efd
= open(buf
, O_RDONLY
, 0);
174 printf("failed to open event %s\n", event
);
178 err
= read(efd
, buf
, sizeof(buf
));
179 if (err
< 0 || err
>= sizeof(buf
)) {
180 printf("read from '%s' failed '%s'\n", event
, strerror(errno
));
190 efd
= sys_perf_event_open(&attr
, -1/*pid*/, 0/*cpu*/, -1/*group_fd*/, 0);
192 printf("event %d fd %d err %s\n", id
, efd
, strerror(errno
));
195 event_fd
[prog_cnt
- 1] = efd
;
196 ioctl(efd
, PERF_EVENT_IOC_ENABLE
, 0);
197 ioctl(efd
, PERF_EVENT_IOC_SET_BPF
, fd
);
202 static int load_maps(struct bpf_map_data
*maps
, int nr_maps
,
203 fixup_map_cb fixup_map
)
207 for (i
= 0; i
< nr_maps
; i
++) {
209 fixup_map(&maps
[i
], i
);
210 /* Allow userspace to assign map FD prior to creation */
211 if (maps
[i
].fd
!= -1) {
212 map_fd
[i
] = maps
[i
].fd
;
217 numa_node
= maps
[i
].def
.map_flags
& BPF_F_NUMA_NODE
?
218 maps
[i
].def
.numa_node
: -1;
220 if (maps
[i
].def
.type
== BPF_MAP_TYPE_ARRAY_OF_MAPS
||
221 maps
[i
].def
.type
== BPF_MAP_TYPE_HASH_OF_MAPS
) {
222 int inner_map_fd
= map_fd
[maps
[i
].def
.inner_map_idx
];
224 map_fd
[i
] = bpf_create_map_in_map_node(maps
[i
].def
.type
,
226 maps
[i
].def
.key_size
,
228 maps
[i
].def
.max_entries
,
229 maps
[i
].def
.map_flags
,
232 map_fd
[i
] = bpf_create_map_node(maps
[i
].def
.type
,
234 maps
[i
].def
.key_size
,
235 maps
[i
].def
.value_size
,
236 maps
[i
].def
.max_entries
,
237 maps
[i
].def
.map_flags
,
241 printf("failed to create a map: %d %s\n",
242 errno
, strerror(errno
));
245 maps
[i
].fd
= map_fd
[i
];
247 if (maps
[i
].def
.type
== BPF_MAP_TYPE_PROG_ARRAY
)
248 prog_array_fd
= map_fd
[i
];
253 static int get_sec(Elf
*elf
, int i
, GElf_Ehdr
*ehdr
, char **shname
,
254 GElf_Shdr
*shdr
, Elf_Data
**data
)
258 scn
= elf_getscn(elf
, i
);
262 if (gelf_getshdr(scn
, shdr
) != shdr
)
265 *shname
= elf_strptr(elf
, ehdr
->e_shstrndx
, shdr
->sh_name
);
266 if (!*shname
|| !shdr
->sh_size
)
269 *data
= elf_getdata(scn
, 0);
270 if (!*data
|| elf_getdata(scn
, *data
) != NULL
)
276 static int parse_relo_and_apply(Elf_Data
*data
, Elf_Data
*symbols
,
277 GElf_Shdr
*shdr
, struct bpf_insn
*insn
,
278 struct bpf_map_data
*maps
, int nr_maps
)
282 nrels
= shdr
->sh_size
/ shdr
->sh_entsize
;
284 for (i
= 0; i
< nrels
; i
++) {
287 unsigned int insn_idx
;
291 gelf_getrel(data
, i
, &rel
);
293 insn_idx
= rel
.r_offset
/ sizeof(struct bpf_insn
);
295 gelf_getsym(symbols
, GELF_R_SYM(rel
.r_info
), &sym
);
297 if (insn
[insn_idx
].code
!= (BPF_LD
| BPF_IMM
| BPF_DW
)) {
298 printf("invalid relo for insn[%d].code 0x%x\n",
299 insn_idx
, insn
[insn_idx
].code
);
302 insn
[insn_idx
].src_reg
= BPF_PSEUDO_MAP_FD
;
304 /* Match FD relocation against recorded map_data[] offset */
305 for (map_idx
= 0; map_idx
< nr_maps
; map_idx
++) {
306 if (maps
[map_idx
].elf_offset
== sym
.st_value
) {
312 insn
[insn_idx
].imm
= maps
[map_idx
].fd
;
314 printf("invalid relo for insn[%d] no map_data match\n",
323 static int cmp_symbols(const void *l
, const void *r
)
325 const GElf_Sym
*lsym
= (const GElf_Sym
*)l
;
326 const GElf_Sym
*rsym
= (const GElf_Sym
*)r
;
328 if (lsym
->st_value
< rsym
->st_value
)
330 else if (lsym
->st_value
> rsym
->st_value
)
336 static int load_elf_maps_section(struct bpf_map_data
*maps
, int maps_shndx
,
337 Elf
*elf
, Elf_Data
*symbols
, int strtabidx
)
339 int map_sz_elf
, map_sz_copy
;
340 bool validate_zero
= false;
352 /* Get data for maps section via elf index */
353 scn
= elf_getscn(elf
, maps_shndx
);
355 data_maps
= elf_getdata(scn
, NULL
);
356 if (!scn
|| !data_maps
) {
357 printf("Failed to get Elf_Data from maps section %d\n",
362 /* For each map get corrosponding symbol table entry */
363 sym
= calloc(MAX_MAPS
+1, sizeof(GElf_Sym
));
364 for (i
= 0, nr_maps
= 0; i
< symbols
->d_size
/ sizeof(GElf_Sym
); i
++) {
365 assert(nr_maps
< MAX_MAPS
+1);
366 if (!gelf_getsym(symbols
, i
, &sym
[nr_maps
]))
368 if (sym
[nr_maps
].st_shndx
!= maps_shndx
)
370 /* Only increment iif maps section */
374 /* Align to map_fd[] order, via sort on offset in sym.st_value */
375 qsort(sym
, nr_maps
, sizeof(GElf_Sym
), cmp_symbols
);
377 /* Keeping compatible with ELF maps section changes
378 * ------------------------------------------------
379 * The program size of struct bpf_map_def is known by loader
380 * code, but struct stored in ELF file can be different.
382 * Unfortunately sym[i].st_size is zero. To calculate the
383 * struct size stored in the ELF file, assume all struct have
384 * the same size, and simply divide with number of map
387 map_sz_elf
= data_maps
->d_size
/ nr_maps
;
388 map_sz_copy
= sizeof(struct bpf_map_def
);
389 if (map_sz_elf
< map_sz_copy
) {
391 * Backward compat, loading older ELF file with
392 * smaller struct, keeping remaining bytes zero.
394 map_sz_copy
= map_sz_elf
;
395 } else if (map_sz_elf
> map_sz_copy
) {
397 * Forward compat, loading newer ELF file with larger
398 * struct with unknown features. Assume zero means
399 * feature not used. Thus, validate rest of struct
402 validate_zero
= true;
405 /* Memcpy relevant part of ELF maps data to loader maps */
406 for (i
= 0; i
< nr_maps
; i
++) {
407 unsigned char *addr
, *end
;
408 struct bpf_map_def
*def
;
409 const char *map_name
;
412 map_name
= elf_strptr(elf
, strtabidx
, sym
[i
].st_name
);
413 maps
[i
].name
= strdup(map_name
);
415 printf("strdup(%s): %s(%d)\n", map_name
,
416 strerror(errno
), errno
);
421 /* Symbol value is offset into ELF maps section data area */
422 offset
= sym
[i
].st_value
;
423 def
= (struct bpf_map_def
*)(data_maps
->d_buf
+ offset
);
424 maps
[i
].elf_offset
= offset
;
425 memset(&maps
[i
].def
, 0, sizeof(struct bpf_map_def
));
426 memcpy(&maps
[i
].def
, def
, map_sz_copy
);
428 /* Verify no newer features were requested */
430 addr
= (unsigned char*) def
+ map_sz_copy
;
431 end
= (unsigned char*) def
+ map_sz_elf
;
432 for (; addr
< end
; addr
++) {
445 static int do_load_bpf_file(const char *path
, fixup_map_cb fixup_map
)
447 int fd
, i
, ret
, maps_shndx
= -1, strtabidx
= -1;
450 GElf_Shdr shdr
, shdr_prog
;
451 Elf_Data
*data
, *data_prog
, *data_maps
= NULL
, *symbols
= NULL
;
452 char *shname
, *shname_prog
;
455 /* reset global variables */
457 memset(license
, 0, sizeof(license
));
458 memset(processed_sec
, 0, sizeof(processed_sec
));
460 if (elf_version(EV_CURRENT
) == EV_NONE
)
463 fd
= open(path
, O_RDONLY
, 0);
467 elf
= elf_begin(fd
, ELF_C_READ
, NULL
);
472 if (gelf_getehdr(elf
, &ehdr
) != &ehdr
)
475 /* clear all kprobes */
476 i
= system("echo \"\" > /sys/kernel/debug/tracing/kprobe_events");
478 /* scan over all elf sections to get license and map info */
479 for (i
= 1; i
< ehdr
.e_shnum
; i
++) {
481 if (get_sec(elf
, i
, &ehdr
, &shname
, &shdr
, &data
))
484 if (0) /* helpful for llvm debugging */
485 printf("section %d:%s data %p size %zd link %d flags %d\n",
486 i
, shname
, data
->d_buf
, data
->d_size
,
487 shdr
.sh_link
, (int) shdr
.sh_flags
);
489 if (strcmp(shname
, "license") == 0) {
490 processed_sec
[i
] = true;
491 memcpy(license
, data
->d_buf
, data
->d_size
);
492 } else if (strcmp(shname
, "version") == 0) {
493 processed_sec
[i
] = true;
494 if (data
->d_size
!= sizeof(int)) {
495 printf("invalid size of version section %zd\n",
499 memcpy(&kern_version
, data
->d_buf
, sizeof(int));
500 } else if (strcmp(shname
, "maps") == 0) {
505 for (j
= 0; j
< MAX_MAPS
; j
++)
507 } else if (shdr
.sh_type
== SHT_SYMTAB
) {
508 strtabidx
= shdr
.sh_link
;
516 printf("missing SHT_SYMTAB section\n");
521 nr_maps
= load_elf_maps_section(map_data
, maps_shndx
,
522 elf
, symbols
, strtabidx
);
524 printf("Error: Failed loading ELF maps (errno:%d):%s\n",
525 nr_maps
, strerror(-nr_maps
));
529 if (load_maps(map_data
, nr_maps
, fixup_map
))
531 map_data_count
= nr_maps
;
533 processed_sec
[maps_shndx
] = true;
536 /* process all relo sections, and rewrite bpf insns for maps */
537 for (i
= 1; i
< ehdr
.e_shnum
; i
++) {
538 if (processed_sec
[i
])
541 if (get_sec(elf
, i
, &ehdr
, &shname
, &shdr
, &data
))
544 if (shdr
.sh_type
== SHT_REL
) {
545 struct bpf_insn
*insns
;
547 /* locate prog sec that need map fixup (relocations) */
548 if (get_sec(elf
, shdr
.sh_info
, &ehdr
, &shname_prog
,
549 &shdr_prog
, &data_prog
))
552 if (shdr_prog
.sh_type
!= SHT_PROGBITS
||
553 !(shdr_prog
.sh_flags
& SHF_EXECINSTR
))
556 insns
= (struct bpf_insn
*) data_prog
->d_buf
;
557 processed_sec
[i
] = true; /* relo section */
559 if (parse_relo_and_apply(data
, symbols
, &shdr
, insns
,
566 for (i
= 1; i
< ehdr
.e_shnum
; i
++) {
568 if (processed_sec
[i
])
571 if (get_sec(elf
, i
, &ehdr
, &shname
, &shdr
, &data
))
574 if (memcmp(shname
, "kprobe/", 7) == 0 ||
575 memcmp(shname
, "kretprobe/", 10) == 0 ||
576 memcmp(shname
, "tracepoint/", 11) == 0 ||
577 memcmp(shname
, "xdp", 3) == 0 ||
578 memcmp(shname
, "perf_event", 10) == 0 ||
579 memcmp(shname
, "socket", 6) == 0 ||
580 memcmp(shname
, "cgroup/", 7) == 0 ||
581 memcmp(shname
, "sockops", 7) == 0 ||
582 memcmp(shname
, "sk_skb", 6) == 0) {
583 ret
= load_and_attach(shname
, data
->d_buf
,
596 int load_bpf_file(char *path
)
598 return do_load_bpf_file(path
, NULL
);
601 int load_bpf_file_fixup_map(const char *path
, fixup_map_cb fixup_map
)
603 return do_load_bpf_file(path
, fixup_map
);
606 void read_trace_pipe(void)
610 trace_fd
= open(DEBUGFS
"trace_pipe", O_RDONLY
, 0);
615 static char buf
[4096];
618 sz
= read(trace_fd
, buf
, sizeof(buf
));
626 #define MAX_SYMS 300000
627 static struct ksym syms
[MAX_SYMS
];
630 static int ksym_cmp(const void *p1
, const void *p2
)
632 return ((struct ksym
*)p1
)->addr
- ((struct ksym
*)p2
)->addr
;
635 int load_kallsyms(void)
637 FILE *f
= fopen("/proc/kallsyms", "r");
638 char func
[256], buf
[256];
647 if (!fgets(buf
, sizeof(buf
), f
))
649 if (sscanf(buf
, "%p %c %s", &addr
, &symbol
, func
) != 3)
653 syms
[i
].addr
= (long) addr
;
654 syms
[i
].name
= strdup(func
);
658 qsort(syms
, sym_cnt
, sizeof(struct ksym
), ksym_cmp
);
662 struct ksym
*ksym_search(long key
)
664 int start
= 0, end
= sym_cnt
;
667 while (start
< end
) {
668 size_t mid
= start
+ (end
- start
) / 2;
670 result
= key
- syms
[mid
].addr
;
679 if (start
>= 1 && syms
[start
- 1].addr
< key
&&
680 key
< syms
[start
].addr
)
682 return &syms
[start
- 1];
684 /* out of range. return _stext */
688 int set_link_xdp_fd(int ifindex
, int fd
, __u32 flags
)
690 struct sockaddr_nl sa
;
691 int sock
, seq
= 0, len
, ret
= -1;
693 struct nlattr
*nla
, *nla_xdp
;
696 struct ifinfomsg ifinfo
;
700 struct nlmsgerr
*err
;
702 memset(&sa
, 0, sizeof(sa
));
703 sa
.nl_family
= AF_NETLINK
;
705 sock
= socket(AF_NETLINK
, SOCK_RAW
, NETLINK_ROUTE
);
707 printf("open netlink socket: %s\n", strerror(errno
));
711 if (bind(sock
, (struct sockaddr
*)&sa
, sizeof(sa
)) < 0) {
712 printf("bind to netlink: %s\n", strerror(errno
));
716 memset(&req
, 0, sizeof(req
));
717 req
.nh
.nlmsg_len
= NLMSG_LENGTH(sizeof(struct ifinfomsg
));
718 req
.nh
.nlmsg_flags
= NLM_F_REQUEST
| NLM_F_ACK
;
719 req
.nh
.nlmsg_type
= RTM_SETLINK
;
720 req
.nh
.nlmsg_pid
= 0;
721 req
.nh
.nlmsg_seq
= ++seq
;
722 req
.ifinfo
.ifi_family
= AF_UNSPEC
;
723 req
.ifinfo
.ifi_index
= ifindex
;
725 /* started nested attribute for XDP */
726 nla
= (struct nlattr
*)(((char *)&req
)
727 + NLMSG_ALIGN(req
.nh
.nlmsg_len
));
728 nla
->nla_type
= NLA_F_NESTED
| 43/*IFLA_XDP*/;
729 nla
->nla_len
= NLA_HDRLEN
;
732 nla_xdp
= (struct nlattr
*)((char *)nla
+ nla
->nla_len
);
733 nla_xdp
->nla_type
= 1/*IFLA_XDP_FD*/;
734 nla_xdp
->nla_len
= NLA_HDRLEN
+ sizeof(int);
735 memcpy((char *)nla_xdp
+ NLA_HDRLEN
, &fd
, sizeof(fd
));
736 nla
->nla_len
+= nla_xdp
->nla_len
;
738 /* if user passed in any flags, add those too */
740 nla_xdp
= (struct nlattr
*)((char *)nla
+ nla
->nla_len
);
741 nla_xdp
->nla_type
= 3/*IFLA_XDP_FLAGS*/;
742 nla_xdp
->nla_len
= NLA_HDRLEN
+ sizeof(flags
);
743 memcpy((char *)nla_xdp
+ NLA_HDRLEN
, &flags
, sizeof(flags
));
744 nla
->nla_len
+= nla_xdp
->nla_len
;
747 req
.nh
.nlmsg_len
+= NLA_ALIGN(nla
->nla_len
);
749 if (send(sock
, &req
, req
.nh
.nlmsg_len
, 0) < 0) {
750 printf("send to netlink: %s\n", strerror(errno
));
754 len
= recv(sock
, buf
, sizeof(buf
), 0);
756 printf("recv from netlink: %s\n", strerror(errno
));
760 for (nh
= (struct nlmsghdr
*)buf
; NLMSG_OK(nh
, len
);
761 nh
= NLMSG_NEXT(nh
, len
)) {
762 if (nh
->nlmsg_pid
!= getpid()) {
763 printf("Wrong pid %d, expected %d\n",
764 nh
->nlmsg_pid
, getpid());
767 if (nh
->nlmsg_seq
!= seq
) {
768 printf("Wrong seq %d, expected %d\n",
772 switch (nh
->nlmsg_type
) {
774 err
= (struct nlmsgerr
*)NLMSG_DATA(nh
);
777 printf("nlmsg error %s\n", strerror(-err
->error
));