12 #include <linux/bpf.h>
13 #include <linux/filter.h>
14 #include <linux/perf_event.h>
15 #include <linux/netlink.h>
16 #include <linux/rtnetlink.h>
17 #include <sys/types.h>
18 #include <sys/socket.h>
19 #include <sys/syscall.h>
20 #include <sys/ioctl.h>
28 #define DEBUGFS "/sys/kernel/debug/tracing/"
30 static char license
[128];
31 static int kern_version
;
32 static bool processed_sec
[128];
33 char bpf_log_buf
[BPF_LOG_BUF_SIZE
];
35 int prog_fd
[MAX_PROGS
];
36 int event_fd
[MAX_PROGS
];
38 int prog_array_fd
= -1;
42 unsigned int key_size
;
43 unsigned int value_size
;
44 unsigned int max_entries
;
45 unsigned int map_flags
;
48 static int populate_prog_array(const char *event
, int prog_fd
)
50 int ind
= atoi(event
), err
;
52 err
= bpf_map_update_elem(prog_array_fd
, &ind
, &prog_fd
, BPF_ANY
);
54 printf("failed to store prog_fd in prog_array\n");
60 static int load_and_attach(const char *event
, struct bpf_insn
*prog
, int size
)
62 bool is_socket
= strncmp(event
, "socket", 6) == 0;
63 bool is_kprobe
= strncmp(event
, "kprobe/", 7) == 0;
64 bool is_kretprobe
= strncmp(event
, "kretprobe/", 10) == 0;
65 bool is_tracepoint
= strncmp(event
, "tracepoint/", 11) == 0;
66 bool is_xdp
= strncmp(event
, "xdp", 3) == 0;
67 bool is_perf_event
= strncmp(event
, "perf_event", 10) == 0;
68 bool is_cgroup_skb
= strncmp(event
, "cgroup/skb", 10) == 0;
69 bool is_cgroup_sk
= strncmp(event
, "cgroup/sock", 11) == 0;
70 size_t insns_cnt
= size
/ sizeof(struct bpf_insn
);
71 enum bpf_prog_type prog_type
;
74 struct perf_event_attr attr
= {};
76 attr
.type
= PERF_TYPE_TRACEPOINT
;
77 attr
.sample_type
= PERF_SAMPLE_RAW
;
78 attr
.sample_period
= 1;
79 attr
.wakeup_events
= 1;
82 prog_type
= BPF_PROG_TYPE_SOCKET_FILTER
;
83 } else if (is_kprobe
|| is_kretprobe
) {
84 prog_type
= BPF_PROG_TYPE_KPROBE
;
85 } else if (is_tracepoint
) {
86 prog_type
= BPF_PROG_TYPE_TRACEPOINT
;
88 prog_type
= BPF_PROG_TYPE_XDP
;
89 } else if (is_perf_event
) {
90 prog_type
= BPF_PROG_TYPE_PERF_EVENT
;
91 } else if (is_cgroup_skb
) {
92 prog_type
= BPF_PROG_TYPE_CGROUP_SKB
;
93 } else if (is_cgroup_sk
) {
94 prog_type
= BPF_PROG_TYPE_CGROUP_SOCK
;
96 printf("Unknown event '%s'\n", event
);
100 fd
= bpf_load_program(prog_type
, prog
, insns_cnt
, license
, kern_version
,
101 bpf_log_buf
, BPF_LOG_BUF_SIZE
);
103 printf("bpf_load_program() err=%d\n%s", errno
, bpf_log_buf
);
107 prog_fd
[prog_cnt
++] = fd
;
109 if (is_xdp
|| is_perf_event
|| is_cgroup_skb
|| is_cgroup_sk
)
117 if (!isdigit(*event
)) {
118 printf("invalid prog number\n");
121 return populate_prog_array(event
, fd
);
124 if (is_kprobe
|| is_kretprobe
) {
131 printf("event name cannot be empty\n");
136 return populate_prog_array(event
, fd
);
138 snprintf(buf
, sizeof(buf
),
139 "echo '%c:%s %s' >> /sys/kernel/debug/tracing/kprobe_events",
140 is_kprobe
? 'p' : 'r', event
, event
);
143 printf("failed to create kprobe '%s' error '%s'\n",
144 event
, strerror(errno
));
148 strcpy(buf
, DEBUGFS
);
149 strcat(buf
, "events/kprobes/");
152 } else if (is_tracepoint
) {
156 printf("event name cannot be empty\n");
159 strcpy(buf
, DEBUGFS
);
160 strcat(buf
, "events/");
165 efd
= open(buf
, O_RDONLY
, 0);
167 printf("failed to open event %s\n", event
);
171 err
= read(efd
, buf
, sizeof(buf
));
172 if (err
< 0 || err
>= sizeof(buf
)) {
173 printf("read from '%s' failed '%s'\n", event
, strerror(errno
));
183 efd
= sys_perf_event_open(&attr
, -1/*pid*/, 0/*cpu*/, -1/*group_fd*/, 0);
185 printf("event %d fd %d err %s\n", id
, efd
, strerror(errno
));
188 event_fd
[prog_cnt
- 1] = efd
;
189 ioctl(efd
, PERF_EVENT_IOC_ENABLE
, 0);
190 ioctl(efd
, PERF_EVENT_IOC_SET_BPF
, fd
);
195 static int load_maps(struct bpf_map_def
*maps
, int len
)
199 for (i
= 0; i
< len
/ sizeof(struct bpf_map_def
); i
++) {
201 map_fd
[i
] = bpf_create_map(maps
[i
].type
,
207 printf("failed to create a map: %d %s\n",
208 errno
, strerror(errno
));
212 if (maps
[i
].type
== BPF_MAP_TYPE_PROG_ARRAY
)
213 prog_array_fd
= map_fd
[i
];
218 static int get_sec(Elf
*elf
, int i
, GElf_Ehdr
*ehdr
, char **shname
,
219 GElf_Shdr
*shdr
, Elf_Data
**data
)
223 scn
= elf_getscn(elf
, i
);
227 if (gelf_getshdr(scn
, shdr
) != shdr
)
230 *shname
= elf_strptr(elf
, ehdr
->e_shstrndx
, shdr
->sh_name
);
231 if (!*shname
|| !shdr
->sh_size
)
234 *data
= elf_getdata(scn
, 0);
235 if (!*data
|| elf_getdata(scn
, *data
) != NULL
)
241 static int parse_relo_and_apply(Elf_Data
*data
, Elf_Data
*symbols
,
242 GElf_Shdr
*shdr
, struct bpf_insn
*insn
)
246 nrels
= shdr
->sh_size
/ shdr
->sh_entsize
;
248 for (i
= 0; i
< nrels
; i
++) {
251 unsigned int insn_idx
;
253 gelf_getrel(data
, i
, &rel
);
255 insn_idx
= rel
.r_offset
/ sizeof(struct bpf_insn
);
257 gelf_getsym(symbols
, GELF_R_SYM(rel
.r_info
), &sym
);
259 if (insn
[insn_idx
].code
!= (BPF_LD
| BPF_IMM
| BPF_DW
)) {
260 printf("invalid relo for insn[%d].code 0x%x\n",
261 insn_idx
, insn
[insn_idx
].code
);
264 insn
[insn_idx
].src_reg
= BPF_PSEUDO_MAP_FD
;
265 insn
[insn_idx
].imm
= map_fd
[sym
.st_value
/ sizeof(struct bpf_map_def
)];
271 int load_bpf_file(char *path
)
276 GElf_Shdr shdr
, shdr_prog
;
277 Elf_Data
*data
, *data_prog
, *symbols
= NULL
;
278 char *shname
, *shname_prog
;
280 if (elf_version(EV_CURRENT
) == EV_NONE
)
283 fd
= open(path
, O_RDONLY
, 0);
287 elf
= elf_begin(fd
, ELF_C_READ
, NULL
);
292 if (gelf_getehdr(elf
, &ehdr
) != &ehdr
)
295 /* clear all kprobes */
296 i
= system("echo \"\" > /sys/kernel/debug/tracing/kprobe_events");
298 /* scan over all elf sections to get license and map info */
299 for (i
= 1; i
< ehdr
.e_shnum
; i
++) {
301 if (get_sec(elf
, i
, &ehdr
, &shname
, &shdr
, &data
))
304 if (0) /* helpful for llvm debugging */
305 printf("section %d:%s data %p size %zd link %d flags %d\n",
306 i
, shname
, data
->d_buf
, data
->d_size
,
307 shdr
.sh_link
, (int) shdr
.sh_flags
);
309 if (strcmp(shname
, "license") == 0) {
310 processed_sec
[i
] = true;
311 memcpy(license
, data
->d_buf
, data
->d_size
);
312 } else if (strcmp(shname
, "version") == 0) {
313 processed_sec
[i
] = true;
314 if (data
->d_size
!= sizeof(int)) {
315 printf("invalid size of version section %zd\n",
319 memcpy(&kern_version
, data
->d_buf
, sizeof(int));
320 } else if (strcmp(shname
, "maps") == 0) {
321 processed_sec
[i
] = true;
322 if (load_maps(data
->d_buf
, data
->d_size
))
324 } else if (shdr
.sh_type
== SHT_SYMTAB
) {
329 /* load programs that need map fixup (relocations) */
330 for (i
= 1; i
< ehdr
.e_shnum
; i
++) {
332 if (get_sec(elf
, i
, &ehdr
, &shname
, &shdr
, &data
))
334 if (shdr
.sh_type
== SHT_REL
) {
335 struct bpf_insn
*insns
;
337 if (get_sec(elf
, shdr
.sh_info
, &ehdr
, &shname_prog
,
338 &shdr_prog
, &data_prog
))
341 if (shdr_prog
.sh_type
!= SHT_PROGBITS
||
342 !(shdr_prog
.sh_flags
& SHF_EXECINSTR
))
345 insns
= (struct bpf_insn
*) data_prog
->d_buf
;
347 processed_sec
[shdr
.sh_info
] = true;
348 processed_sec
[i
] = true;
350 if (parse_relo_and_apply(data
, symbols
, &shdr
, insns
))
353 if (memcmp(shname_prog
, "kprobe/", 7) == 0 ||
354 memcmp(shname_prog
, "kretprobe/", 10) == 0 ||
355 memcmp(shname_prog
, "tracepoint/", 11) == 0 ||
356 memcmp(shname_prog
, "xdp", 3) == 0 ||
357 memcmp(shname_prog
, "perf_event", 10) == 0 ||
358 memcmp(shname_prog
, "socket", 6) == 0 ||
359 memcmp(shname_prog
, "cgroup/", 7) == 0)
360 load_and_attach(shname_prog
, insns
, data_prog
->d_size
);
364 /* load programs that don't use maps */
365 for (i
= 1; i
< ehdr
.e_shnum
; i
++) {
367 if (processed_sec
[i
])
370 if (get_sec(elf
, i
, &ehdr
, &shname
, &shdr
, &data
))
373 if (memcmp(shname
, "kprobe/", 7) == 0 ||
374 memcmp(shname
, "kretprobe/", 10) == 0 ||
375 memcmp(shname
, "tracepoint/", 11) == 0 ||
376 memcmp(shname
, "xdp", 3) == 0 ||
377 memcmp(shname
, "perf_event", 10) == 0 ||
378 memcmp(shname
, "socket", 6) == 0 ||
379 memcmp(shname
, "cgroup/", 7) == 0)
380 load_and_attach(shname
, data
->d_buf
, data
->d_size
);
387 void read_trace_pipe(void)
391 trace_fd
= open(DEBUGFS
"trace_pipe", O_RDONLY
, 0);
396 static char buf
[4096];
399 sz
= read(trace_fd
, buf
, sizeof(buf
));
407 #define MAX_SYMS 300000
408 static struct ksym syms
[MAX_SYMS
];
411 static int ksym_cmp(const void *p1
, const void *p2
)
413 return ((struct ksym
*)p1
)->addr
- ((struct ksym
*)p2
)->addr
;
416 int load_kallsyms(void)
418 FILE *f
= fopen("/proc/kallsyms", "r");
419 char func
[256], buf
[256];
428 if (!fgets(buf
, sizeof(buf
), f
))
430 if (sscanf(buf
, "%p %c %s", &addr
, &symbol
, func
) != 3)
434 syms
[i
].addr
= (long) addr
;
435 syms
[i
].name
= strdup(func
);
439 qsort(syms
, sym_cnt
, sizeof(struct ksym
), ksym_cmp
);
443 struct ksym
*ksym_search(long key
)
445 int start
= 0, end
= sym_cnt
;
448 while (start
< end
) {
449 size_t mid
= start
+ (end
- start
) / 2;
451 result
= key
- syms
[mid
].addr
;
460 if (start
>= 1 && syms
[start
- 1].addr
< key
&&
461 key
< syms
[start
].addr
)
463 return &syms
[start
- 1];
465 /* out of range. return _stext */
469 int set_link_xdp_fd(int ifindex
, int fd
)
471 struct sockaddr_nl sa
;
472 int sock
, seq
= 0, len
, ret
= -1;
474 struct nlattr
*nla
, *nla_xdp
;
477 struct ifinfomsg ifinfo
;
481 struct nlmsgerr
*err
;
483 memset(&sa
, 0, sizeof(sa
));
484 sa
.nl_family
= AF_NETLINK
;
486 sock
= socket(AF_NETLINK
, SOCK_RAW
, NETLINK_ROUTE
);
488 printf("open netlink socket: %s\n", strerror(errno
));
492 if (bind(sock
, (struct sockaddr
*)&sa
, sizeof(sa
)) < 0) {
493 printf("bind to netlink: %s\n", strerror(errno
));
497 memset(&req
, 0, sizeof(req
));
498 req
.nh
.nlmsg_len
= NLMSG_LENGTH(sizeof(struct ifinfomsg
));
499 req
.nh
.nlmsg_flags
= NLM_F_REQUEST
| NLM_F_ACK
;
500 req
.nh
.nlmsg_type
= RTM_SETLINK
;
501 req
.nh
.nlmsg_pid
= 0;
502 req
.nh
.nlmsg_seq
= ++seq
;
503 req
.ifinfo
.ifi_family
= AF_UNSPEC
;
504 req
.ifinfo
.ifi_index
= ifindex
;
505 nla
= (struct nlattr
*)(((char *)&req
)
506 + NLMSG_ALIGN(req
.nh
.nlmsg_len
));
507 nla
->nla_type
= NLA_F_NESTED
| 43/*IFLA_XDP*/;
509 nla_xdp
= (struct nlattr
*)((char *)nla
+ NLA_HDRLEN
);
510 nla_xdp
->nla_type
= 1/*IFLA_XDP_FD*/;
511 nla_xdp
->nla_len
= NLA_HDRLEN
+ sizeof(int);
512 memcpy((char *)nla_xdp
+ NLA_HDRLEN
, &fd
, sizeof(fd
));
513 nla
->nla_len
= NLA_HDRLEN
+ nla_xdp
->nla_len
;
515 req
.nh
.nlmsg_len
+= NLA_ALIGN(nla
->nla_len
);
517 if (send(sock
, &req
, req
.nh
.nlmsg_len
, 0) < 0) {
518 printf("send to netlink: %s\n", strerror(errno
));
522 len
= recv(sock
, buf
, sizeof(buf
), 0);
524 printf("recv from netlink: %s\n", strerror(errno
));
528 for (nh
= (struct nlmsghdr
*)buf
; NLMSG_OK(nh
, len
);
529 nh
= NLMSG_NEXT(nh
, len
)) {
530 if (nh
->nlmsg_pid
!= getpid()) {
531 printf("Wrong pid %d, expected %d\n",
532 nh
->nlmsg_pid
, getpid());
535 if (nh
->nlmsg_seq
!= seq
) {
536 printf("Wrong seq %d, expected %d\n",
540 switch (nh
->nlmsg_type
) {
542 err
= (struct nlmsgerr
*)NLMSG_DATA(nh
);
545 printf("nlmsg error %s\n", strerror(-err
->error
));