12 #include <linux/bpf.h>
13 #include <linux/filter.h>
14 #include <linux/perf_event.h>
15 #include <linux/netlink.h>
16 #include <linux/rtnetlink.h>
17 #include <linux/types.h>
18 #include <sys/types.h>
19 #include <sys/socket.h>
20 #include <sys/syscall.h>
21 #include <sys/ioctl.h>
30 #define DEBUGFS "/sys/kernel/debug/tracing/"
32 static char license
[128];
33 static int kern_version
;
34 static bool processed_sec
[128];
35 char bpf_log_buf
[BPF_LOG_BUF_SIZE
];
37 int prog_fd
[MAX_PROGS
];
38 int event_fd
[MAX_PROGS
];
40 int prog_array_fd
= -1;
42 static int populate_prog_array(const char *event
, int prog_fd
)
44 int ind
= atoi(event
), err
;
46 err
= bpf_map_update_elem(prog_array_fd
, &ind
, &prog_fd
, BPF_ANY
);
48 printf("failed to store prog_fd in prog_array\n");
54 static int load_and_attach(const char *event
, struct bpf_insn
*prog
, int size
)
56 bool is_socket
= strncmp(event
, "socket", 6) == 0;
57 bool is_kprobe
= strncmp(event
, "kprobe/", 7) == 0;
58 bool is_kretprobe
= strncmp(event
, "kretprobe/", 10) == 0;
59 bool is_tracepoint
= strncmp(event
, "tracepoint/", 11) == 0;
60 bool is_xdp
= strncmp(event
, "xdp", 3) == 0;
61 bool is_perf_event
= strncmp(event
, "perf_event", 10) == 0;
62 bool is_cgroup_skb
= strncmp(event
, "cgroup/skb", 10) == 0;
63 bool is_cgroup_sk
= strncmp(event
, "cgroup/sock", 11) == 0;
64 size_t insns_cnt
= size
/ sizeof(struct bpf_insn
);
65 enum bpf_prog_type prog_type
;
68 struct perf_event_attr attr
= {};
70 attr
.type
= PERF_TYPE_TRACEPOINT
;
71 attr
.sample_type
= PERF_SAMPLE_RAW
;
72 attr
.sample_period
= 1;
73 attr
.wakeup_events
= 1;
76 prog_type
= BPF_PROG_TYPE_SOCKET_FILTER
;
77 } else if (is_kprobe
|| is_kretprobe
) {
78 prog_type
= BPF_PROG_TYPE_KPROBE
;
79 } else if (is_tracepoint
) {
80 prog_type
= BPF_PROG_TYPE_TRACEPOINT
;
82 prog_type
= BPF_PROG_TYPE_XDP
;
83 } else if (is_perf_event
) {
84 prog_type
= BPF_PROG_TYPE_PERF_EVENT
;
85 } else if (is_cgroup_skb
) {
86 prog_type
= BPF_PROG_TYPE_CGROUP_SKB
;
87 } else if (is_cgroup_sk
) {
88 prog_type
= BPF_PROG_TYPE_CGROUP_SOCK
;
90 printf("Unknown event '%s'\n", event
);
94 fd
= bpf_load_program(prog_type
, prog
, insns_cnt
, license
, kern_version
,
95 bpf_log_buf
, BPF_LOG_BUF_SIZE
);
97 printf("bpf_load_program() err=%d\n%s", errno
, bpf_log_buf
);
101 prog_fd
[prog_cnt
++] = fd
;
103 if (is_xdp
|| is_perf_event
|| is_cgroup_skb
|| is_cgroup_sk
)
111 if (!isdigit(*event
)) {
112 printf("invalid prog number\n");
115 return populate_prog_array(event
, fd
);
118 if (is_kprobe
|| is_kretprobe
) {
125 printf("event name cannot be empty\n");
130 return populate_prog_array(event
, fd
);
132 snprintf(buf
, sizeof(buf
),
133 "echo '%c:%s %s' >> /sys/kernel/debug/tracing/kprobe_events",
134 is_kprobe
? 'p' : 'r', event
, event
);
137 printf("failed to create kprobe '%s' error '%s'\n",
138 event
, strerror(errno
));
142 strcpy(buf
, DEBUGFS
);
143 strcat(buf
, "events/kprobes/");
146 } else if (is_tracepoint
) {
150 printf("event name cannot be empty\n");
153 strcpy(buf
, DEBUGFS
);
154 strcat(buf
, "events/");
159 efd
= open(buf
, O_RDONLY
, 0);
161 printf("failed to open event %s\n", event
);
165 err
= read(efd
, buf
, sizeof(buf
));
166 if (err
< 0 || err
>= sizeof(buf
)) {
167 printf("read from '%s' failed '%s'\n", event
, strerror(errno
));
177 efd
= sys_perf_event_open(&attr
, -1/*pid*/, 0/*cpu*/, -1/*group_fd*/, 0);
179 printf("event %d fd %d err %s\n", id
, efd
, strerror(errno
));
182 event_fd
[prog_cnt
- 1] = efd
;
183 ioctl(efd
, PERF_EVENT_IOC_ENABLE
, 0);
184 ioctl(efd
, PERF_EVENT_IOC_SET_BPF
, fd
);
189 static int load_maps(struct bpf_map_def
*maps
, int nr_maps
,
190 const char **map_names
, fixup_map_cb fixup_map
)
194 * Warning: Using "maps" pointing to ELF data_maps->d_buf as
195 * an array of struct bpf_map_def is a wrong assumption about
196 * the ELF maps section format.
198 for (i
= 0; i
< nr_maps
; i
++) {
200 fixup_map(&maps
[i
], map_names
[i
], i
);
202 if (maps
[i
].type
== BPF_MAP_TYPE_ARRAY_OF_MAPS
||
203 maps
[i
].type
== BPF_MAP_TYPE_HASH_OF_MAPS
) {
204 int inner_map_fd
= map_fd
[maps
[i
].inner_map_idx
];
206 map_fd
[i
] = bpf_create_map_in_map(maps
[i
].type
,
212 map_fd
[i
] = bpf_create_map(maps
[i
].type
,
219 printf("failed to create a map: %d %s\n",
220 errno
, strerror(errno
));
224 if (maps
[i
].type
== BPF_MAP_TYPE_PROG_ARRAY
)
225 prog_array_fd
= map_fd
[i
];
230 static int get_sec(Elf
*elf
, int i
, GElf_Ehdr
*ehdr
, char **shname
,
231 GElf_Shdr
*shdr
, Elf_Data
**data
)
235 scn
= elf_getscn(elf
, i
);
239 if (gelf_getshdr(scn
, shdr
) != shdr
)
242 *shname
= elf_strptr(elf
, ehdr
->e_shstrndx
, shdr
->sh_name
);
243 if (!*shname
|| !shdr
->sh_size
)
246 *data
= elf_getdata(scn
, 0);
247 if (!*data
|| elf_getdata(scn
, *data
) != NULL
)
253 static int parse_relo_and_apply(Elf_Data
*data
, Elf_Data
*symbols
,
254 GElf_Shdr
*shdr
, struct bpf_insn
*insn
)
258 nrels
= shdr
->sh_size
/ shdr
->sh_entsize
;
260 for (i
= 0; i
< nrels
; i
++) {
263 unsigned int insn_idx
;
265 gelf_getrel(data
, i
, &rel
);
267 insn_idx
= rel
.r_offset
/ sizeof(struct bpf_insn
);
269 gelf_getsym(symbols
, GELF_R_SYM(rel
.r_info
), &sym
);
271 if (insn
[insn_idx
].code
!= (BPF_LD
| BPF_IMM
| BPF_DW
)) {
272 printf("invalid relo for insn[%d].code 0x%x\n",
273 insn_idx
, insn
[insn_idx
].code
);
276 insn
[insn_idx
].src_reg
= BPF_PSEUDO_MAP_FD
;
278 * Warning: Using sizeof(struct bpf_map_def) here is a
279 * wrong assumption about ELF maps section format
281 insn
[insn_idx
].imm
= map_fd
[sym
.st_value
/ sizeof(struct bpf_map_def
)];
287 static int cmp_symbols(const void *l
, const void *r
)
289 const GElf_Sym
*lsym
= (const GElf_Sym
*)l
;
290 const GElf_Sym
*rsym
= (const GElf_Sym
*)r
;
292 if (lsym
->st_value
< rsym
->st_value
)
294 else if (lsym
->st_value
> rsym
->st_value
)
300 static int get_sorted_map_names(Elf
*elf
, Elf_Data
*symbols
, int maps_shndx
,
301 int strtabidx
, char **map_names
)
303 GElf_Sym map_symbols
[MAX_MAPS
];
306 for (i
= 0; i
< symbols
->d_size
/ sizeof(GElf_Sym
); i
++) {
307 assert(nr_maps
< MAX_MAPS
);
308 if (!gelf_getsym(symbols
, i
, &map_symbols
[nr_maps
]))
310 if (map_symbols
[nr_maps
].st_shndx
!= maps_shndx
)
315 qsort(map_symbols
, nr_maps
, sizeof(GElf_Sym
), cmp_symbols
);
317 for (i
= 0; i
< nr_maps
; i
++) {
320 map_name
= elf_strptr(elf
, strtabidx
, map_symbols
[i
].st_name
);
322 printf("cannot get map symbol\n");
326 map_names
[i
] = strdup(map_name
);
328 printf("strdup(%s): %s(%d)\n", map_name
,
329 strerror(errno
), errno
);
337 static int do_load_bpf_file(const char *path
, fixup_map_cb fixup_map
)
339 int fd
, i
, ret
, maps_shndx
= -1, strtabidx
= -1;
342 GElf_Shdr shdr
, shdr_prog
;
343 Elf_Data
*data
, *data_prog
, *data_maps
= NULL
, *symbols
= NULL
;
344 char *shname
, *shname_prog
, *map_names
[MAX_MAPS
] = { NULL
};
346 /* reset global variables */
348 memset(license
, 0, sizeof(license
));
349 memset(processed_sec
, 0, sizeof(processed_sec
));
351 if (elf_version(EV_CURRENT
) == EV_NONE
)
354 fd
= open(path
, O_RDONLY
, 0);
358 elf
= elf_begin(fd
, ELF_C_READ
, NULL
);
363 if (gelf_getehdr(elf
, &ehdr
) != &ehdr
)
366 /* clear all kprobes */
367 i
= system("echo \"\" > /sys/kernel/debug/tracing/kprobe_events");
369 /* scan over all elf sections to get license and map info */
370 for (i
= 1; i
< ehdr
.e_shnum
; i
++) {
372 if (get_sec(elf
, i
, &ehdr
, &shname
, &shdr
, &data
))
375 if (0) /* helpful for llvm debugging */
376 printf("section %d:%s data %p size %zd link %d flags %d\n",
377 i
, shname
, data
->d_buf
, data
->d_size
,
378 shdr
.sh_link
, (int) shdr
.sh_flags
);
380 if (strcmp(shname
, "license") == 0) {
381 processed_sec
[i
] = true;
382 memcpy(license
, data
->d_buf
, data
->d_size
);
383 } else if (strcmp(shname
, "version") == 0) {
384 processed_sec
[i
] = true;
385 if (data
->d_size
!= sizeof(int)) {
386 printf("invalid size of version section %zd\n",
390 memcpy(&kern_version
, data
->d_buf
, sizeof(int));
391 } else if (strcmp(shname
, "maps") == 0) {
394 } else if (shdr
.sh_type
== SHT_SYMTAB
) {
395 strtabidx
= shdr
.sh_link
;
403 printf("missing SHT_SYMTAB section\n");
411 nr_maps
= get_sorted_map_names(elf
, symbols
, maps_shndx
,
412 strtabidx
, map_names
);
416 /* Deduce map struct size stored in ELF maps section */
417 prog_elf_map_sz
= data_maps
->d_size
/ nr_maps
;
418 if (prog_elf_map_sz
!= sizeof(struct bpf_map_def
)) {
419 printf("Error: ELF maps sec wrong size (%d/%lu),"
420 " old kern.o file?\n",
421 prog_elf_map_sz
, sizeof(struct bpf_map_def
));
426 if (load_maps(data_maps
->d_buf
, nr_maps
,
427 (const char **)map_names
, fixup_map
))
430 processed_sec
[maps_shndx
] = true;
433 /* load programs that need map fixup (relocations) */
434 for (i
= 1; i
< ehdr
.e_shnum
; i
++) {
435 if (processed_sec
[i
])
438 if (get_sec(elf
, i
, &ehdr
, &shname
, &shdr
, &data
))
440 if (shdr
.sh_type
== SHT_REL
) {
441 struct bpf_insn
*insns
;
443 if (get_sec(elf
, shdr
.sh_info
, &ehdr
, &shname_prog
,
444 &shdr_prog
, &data_prog
))
447 if (shdr_prog
.sh_type
!= SHT_PROGBITS
||
448 !(shdr_prog
.sh_flags
& SHF_EXECINSTR
))
451 insns
= (struct bpf_insn
*) data_prog
->d_buf
;
453 processed_sec
[shdr
.sh_info
] = true;
454 processed_sec
[i
] = true;
456 if (parse_relo_and_apply(data
, symbols
, &shdr
, insns
))
459 if (memcmp(shname_prog
, "kprobe/", 7) == 0 ||
460 memcmp(shname_prog
, "kretprobe/", 10) == 0 ||
461 memcmp(shname_prog
, "tracepoint/", 11) == 0 ||
462 memcmp(shname_prog
, "xdp", 3) == 0 ||
463 memcmp(shname_prog
, "perf_event", 10) == 0 ||
464 memcmp(shname_prog
, "socket", 6) == 0 ||
465 memcmp(shname_prog
, "cgroup/", 7) == 0)
466 load_and_attach(shname_prog
, insns
, data_prog
->d_size
);
470 /* load programs that don't use maps */
471 for (i
= 1; i
< ehdr
.e_shnum
; i
++) {
473 if (processed_sec
[i
])
476 if (get_sec(elf
, i
, &ehdr
, &shname
, &shdr
, &data
))
479 if (memcmp(shname
, "kprobe/", 7) == 0 ||
480 memcmp(shname
, "kretprobe/", 10) == 0 ||
481 memcmp(shname
, "tracepoint/", 11) == 0 ||
482 memcmp(shname
, "xdp", 3) == 0 ||
483 memcmp(shname
, "perf_event", 10) == 0 ||
484 memcmp(shname
, "socket", 6) == 0 ||
485 memcmp(shname
, "cgroup/", 7) == 0)
486 load_and_attach(shname
, data
->d_buf
, data
->d_size
);
491 for (i
= 0; i
< MAX_MAPS
; i
++)
497 int load_bpf_file(char *path
)
499 return do_load_bpf_file(path
, NULL
);
502 int load_bpf_file_fixup_map(const char *path
, fixup_map_cb fixup_map
)
504 return do_load_bpf_file(path
, fixup_map
);
507 void read_trace_pipe(void)
511 trace_fd
= open(DEBUGFS
"trace_pipe", O_RDONLY
, 0);
516 static char buf
[4096];
519 sz
= read(trace_fd
, buf
, sizeof(buf
));
527 #define MAX_SYMS 300000
528 static struct ksym syms
[MAX_SYMS
];
531 static int ksym_cmp(const void *p1
, const void *p2
)
533 return ((struct ksym
*)p1
)->addr
- ((struct ksym
*)p2
)->addr
;
536 int load_kallsyms(void)
538 FILE *f
= fopen("/proc/kallsyms", "r");
539 char func
[256], buf
[256];
548 if (!fgets(buf
, sizeof(buf
), f
))
550 if (sscanf(buf
, "%p %c %s", &addr
, &symbol
, func
) != 3)
554 syms
[i
].addr
= (long) addr
;
555 syms
[i
].name
= strdup(func
);
559 qsort(syms
, sym_cnt
, sizeof(struct ksym
), ksym_cmp
);
563 struct ksym
*ksym_search(long key
)
565 int start
= 0, end
= sym_cnt
;
568 while (start
< end
) {
569 size_t mid
= start
+ (end
- start
) / 2;
571 result
= key
- syms
[mid
].addr
;
580 if (start
>= 1 && syms
[start
- 1].addr
< key
&&
581 key
< syms
[start
].addr
)
583 return &syms
[start
- 1];
585 /* out of range. return _stext */
589 int set_link_xdp_fd(int ifindex
, int fd
, __u32 flags
)
591 struct sockaddr_nl sa
;
592 int sock
, seq
= 0, len
, ret
= -1;
594 struct nlattr
*nla
, *nla_xdp
;
597 struct ifinfomsg ifinfo
;
601 struct nlmsgerr
*err
;
603 memset(&sa
, 0, sizeof(sa
));
604 sa
.nl_family
= AF_NETLINK
;
606 sock
= socket(AF_NETLINK
, SOCK_RAW
, NETLINK_ROUTE
);
608 printf("open netlink socket: %s\n", strerror(errno
));
612 if (bind(sock
, (struct sockaddr
*)&sa
, sizeof(sa
)) < 0) {
613 printf("bind to netlink: %s\n", strerror(errno
));
617 memset(&req
, 0, sizeof(req
));
618 req
.nh
.nlmsg_len
= NLMSG_LENGTH(sizeof(struct ifinfomsg
));
619 req
.nh
.nlmsg_flags
= NLM_F_REQUEST
| NLM_F_ACK
;
620 req
.nh
.nlmsg_type
= RTM_SETLINK
;
621 req
.nh
.nlmsg_pid
= 0;
622 req
.nh
.nlmsg_seq
= ++seq
;
623 req
.ifinfo
.ifi_family
= AF_UNSPEC
;
624 req
.ifinfo
.ifi_index
= ifindex
;
626 /* started nested attribute for XDP */
627 nla
= (struct nlattr
*)(((char *)&req
)
628 + NLMSG_ALIGN(req
.nh
.nlmsg_len
));
629 nla
->nla_type
= NLA_F_NESTED
| 43/*IFLA_XDP*/;
630 nla
->nla_len
= NLA_HDRLEN
;
633 nla_xdp
= (struct nlattr
*)((char *)nla
+ nla
->nla_len
);
634 nla_xdp
->nla_type
= 1/*IFLA_XDP_FD*/;
635 nla_xdp
->nla_len
= NLA_HDRLEN
+ sizeof(int);
636 memcpy((char *)nla_xdp
+ NLA_HDRLEN
, &fd
, sizeof(fd
));
637 nla
->nla_len
+= nla_xdp
->nla_len
;
639 /* if user passed in any flags, add those too */
641 nla_xdp
= (struct nlattr
*)((char *)nla
+ nla
->nla_len
);
642 nla_xdp
->nla_type
= 3/*IFLA_XDP_FLAGS*/;
643 nla_xdp
->nla_len
= NLA_HDRLEN
+ sizeof(flags
);
644 memcpy((char *)nla_xdp
+ NLA_HDRLEN
, &flags
, sizeof(flags
));
645 nla
->nla_len
+= nla_xdp
->nla_len
;
648 req
.nh
.nlmsg_len
+= NLA_ALIGN(nla
->nla_len
);
650 if (send(sock
, &req
, req
.nh
.nlmsg_len
, 0) < 0) {
651 printf("send to netlink: %s\n", strerror(errno
));
655 len
= recv(sock
, buf
, sizeof(buf
), 0);
657 printf("recv from netlink: %s\n", strerror(errno
));
661 for (nh
= (struct nlmsghdr
*)buf
; NLMSG_OK(nh
, len
);
662 nh
= NLMSG_NEXT(nh
, len
)) {
663 if (nh
->nlmsg_pid
!= getpid()) {
664 printf("Wrong pid %d, expected %d\n",
665 nh
->nlmsg_pid
, getpid());
668 if (nh
->nlmsg_seq
!= seq
) {
669 printf("Wrong seq %d, expected %d\n",
673 switch (nh
->nlmsg_type
) {
675 err
= (struct nlmsgerr
*)NLMSG_DATA(nh
);
678 printf("nlmsg error %s\n", strerror(-err
->error
));