]> git.proxmox.com Git - mirror_iproute2.git/blob - lib/bpf.c
projects / mirror_iproute2.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'iproute2-master' into iproute2-next
[mirror_iproute2.git] / lib / bpf.c
1 /*
2 * bpf.c BPF common code
3 *
4 * This program is free software; you can distribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Authors: Daniel Borkmann <daniel@iogearbox.net>
10 * Jiri Pirko <jiri@resnulli.us>
11 * Alexei Starovoitov <ast@kernel.org>
12 */
13
14 #include <stdio.h>
15 #include <stdlib.h>
16 #include <unistd.h>
17 #include <string.h>
18 #include <stdbool.h>
19 #include <stdint.h>
20 #include <errno.h>
21 #include <fcntl.h>
22 #include <stdarg.h>
23 #include <limits.h>
24 #include <assert.h>
25
26 #ifdef HAVE_ELF
27 #include <libelf.h>
28 #include <gelf.h>
29 #endif
30
31 #include <sys/types.h>
32 #include <sys/stat.h>
33 #include <sys/un.h>
34 #include <sys/vfs.h>
35 #include <sys/mount.h>
36 #include <sys/syscall.h>
37 #include <sys/sendfile.h>
38 #include <sys/resource.h>
39
40 #include <arpa/inet.h>
41
42 #include "utils.h"
43 #include "json_print.h"
44
45 #include "bpf_util.h"
46 #include "bpf_elf.h"
47 #include "bpf_scm.h"
48
49 struct bpf_prog_meta {
50 const char *type;
51 const char *subdir;
52 const char *section;
53 bool may_uds_export;
54 };
55
56 static const enum bpf_prog_type __bpf_types[] = {
57 BPF_PROG_TYPE_SCHED_CLS,
58 BPF_PROG_TYPE_SCHED_ACT,
59 BPF_PROG_TYPE_XDP,
60 BPF_PROG_TYPE_LWT_IN,
61 BPF_PROG_TYPE_LWT_OUT,
62 BPF_PROG_TYPE_LWT_XMIT,
63 };
64
65 static const struct bpf_prog_meta __bpf_prog_meta[] = {
66 [BPF_PROG_TYPE_SCHED_CLS] = {
67 .type = "cls",
68 .subdir = "tc",
69 .section = ELF_SECTION_CLASSIFIER,
70 .may_uds_export = true,
71 },
72 [BPF_PROG_TYPE_SCHED_ACT] = {
73 .type = "act",
74 .subdir = "tc",
75 .section = ELF_SECTION_ACTION,
76 .may_uds_export = true,
77 },
78 [BPF_PROG_TYPE_XDP] = {
79 .type = "xdp",
80 .subdir = "xdp",
81 .section = ELF_SECTION_PROG,
82 },
83 [BPF_PROG_TYPE_LWT_IN] = {
84 .type = "lwt_in",
85 .subdir = "ip",
86 .section = ELF_SECTION_PROG,
87 },
88 [BPF_PROG_TYPE_LWT_OUT] = {
89 .type = "lwt_out",
90 .subdir = "ip",
91 .section = ELF_SECTION_PROG,
92 },
93 [BPF_PROG_TYPE_LWT_XMIT] = {
94 .type = "lwt_xmit",
95 .subdir = "ip",
96 .section = ELF_SECTION_PROG,
97 },
98 };
99
100 static const char *bpf_prog_to_subdir(enum bpf_prog_type type)
101 {
102 assert(type < ARRAY_SIZE(__bpf_prog_meta) &&
103 __bpf_prog_meta[type].subdir);
104 return __bpf_prog_meta[type].subdir;
105 }
106
107 const char *bpf_prog_to_default_section(enum bpf_prog_type type)
108 {
109 assert(type < ARRAY_SIZE(__bpf_prog_meta) &&
110 __bpf_prog_meta[type].section);
111 return __bpf_prog_meta[type].section;
112 }
113
114 #ifdef HAVE_ELF
115 static int bpf_obj_open(const char *path, enum bpf_prog_type type,
116 const char *sec, __u32 ifindex, bool verbose);
117 #else
118 static int bpf_obj_open(const char *path, enum bpf_prog_type type,
119 const char *sec, __u32 ifindex, bool verbose)
120 {
121 fprintf(stderr, "No ELF library support compiled in.\n");
122 errno = ENOSYS;
123 return -1;
124 }
125 #endif
126
127 static inline __u64 bpf_ptr_to_u64(const void *ptr)
128 {
129 return (__u64)(unsigned long)ptr;
130 }
131
132 static int bpf(int cmd, union bpf_attr *attr, unsigned int size)
133 {
134 #ifdef __NR_bpf
135 return syscall(__NR_bpf, cmd, attr, size);
136 #else
137 fprintf(stderr, "No bpf syscall, kernel headers too old?\n");
138 errno = ENOSYS;
139 return -1;
140 #endif
141 }
142
143 static int bpf_map_update(int fd, const void *key, const void *value,
144 uint64_t flags)
145 {
146 union bpf_attr attr = {};
147
148 attr.map_fd = fd;
149 attr.key = bpf_ptr_to_u64(key);
150 attr.value = bpf_ptr_to_u64(value);
151 attr.flags = flags;
152
153 return bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
154 }
155
156 static int bpf_prog_fd_by_id(uint32_t id)
157 {
158 union bpf_attr attr = {};
159
160 attr.prog_id = id;
161
162 return bpf(BPF_PROG_GET_FD_BY_ID, &attr, sizeof(attr));
163 }
164
165 static int bpf_prog_info_by_fd(int fd, struct bpf_prog_info *info,
166 uint32_t *info_len)
167 {
168 union bpf_attr attr = {};
169 int ret;
170
171 attr.info.bpf_fd = fd;
172 attr.info.info = bpf_ptr_to_u64(info);
173 attr.info.info_len = *info_len;
174
175 *info_len = 0;
176 ret = bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr));
177 if (!ret)
178 *info_len = attr.info.info_len;
179
180 return ret;
181 }
182
183 int bpf_dump_prog_info(FILE *f, uint32_t id)
184 {
185 struct bpf_prog_info info = {};
186 uint32_t len = sizeof(info);
187 int fd, ret, dump_ok = 0;
188 SPRINT_BUF(tmp);
189
190 open_json_object("prog");
191 print_uint(PRINT_ANY, "id", "id %u ", id);
192
193 fd = bpf_prog_fd_by_id(id);
194 if (fd < 0)
195 goto out;
196
197 ret = bpf_prog_info_by_fd(fd, &info, &len);
198 if (!ret && len) {
199 int jited = !!info.jited_prog_len;
200
201 print_string(PRINT_ANY, "tag", "tag %s ",
202 hexstring_n2a(info.tag, sizeof(info.tag),
203 tmp, sizeof(tmp)));
204 print_uint(PRINT_JSON, "jited", NULL, jited);
205 if (jited && !is_json_context())
206 fprintf(f, "jited ");
207 dump_ok = 1;
208 }
209
210 close(fd);
211 out:
212 close_json_object();
213 return dump_ok;
214 }
215
216 static int bpf_parse_string(char *arg, bool from_file, __u16 *bpf_len,
217 char **bpf_string, bool *need_release,
218 const char separator)
219 {
220 char sp;
221
222 if (from_file) {
223 size_t tmp_len, op_len = sizeof("65535 255 255 4294967295,");
224 char *tmp_string, *pos, c_prev = ' ';
225 FILE *fp;
226 int c;
227
228 tmp_len = sizeof("4096,") + BPF_MAXINSNS * op_len;
229 tmp_string = pos = calloc(1, tmp_len);
230 if (tmp_string == NULL)
231 return -ENOMEM;
232
233 fp = fopen(arg, "r");
234 if (fp == NULL) {
235 perror("Cannot fopen");
236 free(tmp_string);
237 return -ENOENT;
238 }
239
240 while ((c = fgetc(fp)) != EOF) {
241 switch (c) {
242 case '\n':
243 if (c_prev != ',')
244 *(pos++) = ',';
245 c_prev = ',';
246 break;
247 case ' ':
248 case '\t':
249 if (c_prev != ' ')
250 *(pos++) = c;
251 c_prev = ' ';
252 break;
253 default:
254 *(pos++) = c;
255 c_prev = c;
256 }
257 if (pos - tmp_string == tmp_len)
258 break;
259 }
260
261 if (!feof(fp)) {
262 free(tmp_string);
263 fclose(fp);
264 return -E2BIG;
265 }
266
267 fclose(fp);
268 *pos = 0;
269
270 *need_release = true;
271 *bpf_string = tmp_string;
272 } else {
273 *need_release = false;
274 *bpf_string = arg;
275 }
276
277 if (sscanf(*bpf_string, "%hu%c", bpf_len, &sp) != 2 ||
278 sp != separator) {
279 if (*need_release)
280 free(*bpf_string);
281 return -EINVAL;
282 }
283
284 return 0;
285 }
286
287 static int bpf_ops_parse(int argc, char **argv, struct sock_filter *bpf_ops,
288 bool from_file)
289 {
290 char *bpf_string, *token, separator = ',';
291 int ret = 0, i = 0;
292 bool need_release;
293 __u16 bpf_len = 0;
294
295 if (argc < 1)
296 return -EINVAL;
297 if (bpf_parse_string(argv[0], from_file, &bpf_len, &bpf_string,
298 &need_release, separator))
299 return -EINVAL;
300 if (bpf_len == 0 || bpf_len > BPF_MAXINSNS) {
301 ret = -EINVAL;
302 goto out;
303 }
304
305 token = bpf_string;
306 while ((token = strchr(token, separator)) && (++token)[0]) {
307 if (i >= bpf_len) {
308 fprintf(stderr, "Real program length exceeds encoded length parameter!\n");
309 ret = -EINVAL;
310 goto out;
311 }
312
313 if (sscanf(token, "%hu %hhu %hhu %u,",
314 &bpf_ops[i].code, &bpf_ops[i].jt,
315 &bpf_ops[i].jf, &bpf_ops[i].k) != 4) {
316 fprintf(stderr, "Error at instruction %d!\n", i);
317 ret = -EINVAL;
318 goto out;
319 }
320
321 i++;
322 }
323
324 if (i != bpf_len) {
325 fprintf(stderr, "Parsed program length is less than encoded length parameter!\n");
326 ret = -EINVAL;
327 goto out;
328 }
329 ret = bpf_len;
330 out:
331 if (need_release)
332 free(bpf_string);
333
334 return ret;
335 }
336
337 void bpf_print_ops(FILE *f, struct rtattr *bpf_ops, __u16 len)
338 {
339 struct sock_filter *ops = RTA_DATA(bpf_ops);
340 int i;
341
342 if (len == 0)
343 return;
344
345 fprintf(f, "bytecode \'%u,", len);
346
347 for (i = 0; i < len - 1; i++)
348 fprintf(f, "%hu %hhu %hhu %u,", ops[i].code, ops[i].jt,
349 ops[i].jf, ops[i].k);
350
351 fprintf(f, "%hu %hhu %hhu %u\'", ops[i].code, ops[i].jt,
352 ops[i].jf, ops[i].k);
353 }
354
355 static void bpf_map_pin_report(const struct bpf_elf_map *pin,
356 const struct bpf_elf_map *obj)
357 {
358 fprintf(stderr, "Map specification differs from pinned file!\n");
359
360 if (obj->type != pin->type)
361 fprintf(stderr, " - Type: %u (obj) != %u (pin)\n",
362 obj->type, pin->type);
363 if (obj->size_key != pin->size_key)
364 fprintf(stderr, " - Size key: %u (obj) != %u (pin)\n",
365 obj->size_key, pin->size_key);
366 if (obj->size_value != pin->size_value)
367 fprintf(stderr, " - Size value: %u (obj) != %u (pin)\n",
368 obj->size_value, pin->size_value);
369 if (obj->max_elem != pin->max_elem)
370 fprintf(stderr, " - Max elems: %u (obj) != %u (pin)\n",
371 obj->max_elem, pin->max_elem);
372 if (obj->flags != pin->flags)
373 fprintf(stderr, " - Flags: %#x (obj) != %#x (pin)\n",
374 obj->flags, pin->flags);
375
376 fprintf(stderr, "\n");
377 }
378
379 struct bpf_prog_data {
380 unsigned int type;
381 unsigned int jited;
382 };
383
384 struct bpf_map_ext {
385 struct bpf_prog_data owner;
386 };
387
388 static int bpf_derive_elf_map_from_fdinfo(int fd, struct bpf_elf_map *map,
389 struct bpf_map_ext *ext)
390 {
391 unsigned int val, owner_type = 0, owner_jited = 0;
392 char file[PATH_MAX], buff[4096];
393 FILE *fp;
394
395 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
396 memset(map, 0, sizeof(*map));
397
398 fp = fopen(file, "r");
399 if (!fp) {
400 fprintf(stderr, "No procfs support?!\n");
401 return -EIO;
402 }
403
404 while (fgets(buff, sizeof(buff), fp)) {
405 if (sscanf(buff, "map_type:\t%u", &val) == 1)
406 map->type = val;
407 else if (sscanf(buff, "key_size:\t%u", &val) == 1)
408 map->size_key = val;
409 else if (sscanf(buff, "value_size:\t%u", &val) == 1)
410 map->size_value = val;
411 else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
412 map->max_elem = val;
413 else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
414 map->flags = val;
415 else if (sscanf(buff, "owner_prog_type:\t%i", &val) == 1)
416 owner_type = val;
417 else if (sscanf(buff, "owner_jited:\t%i", &val) == 1)
418 owner_jited = val;
419 }
420
421 fclose(fp);
422 if (ext) {
423 memset(ext, 0, sizeof(*ext));
424 ext->owner.type = owner_type;
425 ext->owner.jited = owner_jited;
426 }
427
428 return 0;
429 }
430
431 static int bpf_map_selfcheck_pinned(int fd, const struct bpf_elf_map *map,
432 struct bpf_map_ext *ext, int length,
433 enum bpf_prog_type type)
434 {
435 struct bpf_elf_map tmp, zero = {};
436 int ret;
437
438 ret = bpf_derive_elf_map_from_fdinfo(fd, &tmp, ext);
439 if (ret < 0)
440 return ret;
441
442 /* The decision to reject this is on kernel side eventually, but
443 * at least give the user a chance to know what's wrong.
444 */
445 if (ext->owner.type && ext->owner.type != type)
446 fprintf(stderr, "Program array map owner types differ: %u (obj) != %u (pin)\n",
447 type, ext->owner.type);
448
449 if (!memcmp(&tmp, map, length)) {
450 return 0;
451 } else {
452 /* If kernel doesn't have eBPF-related fdinfo, we cannot do much,
453 * so just accept it. We know we do have an eBPF fd and in this
454 * case, everything is 0. It is guaranteed that no such map exists
455 * since map type of 0 is unloadable BPF_MAP_TYPE_UNSPEC.
456 */
457 if (!memcmp(&tmp, &zero, length))
458 return 0;
459
460 bpf_map_pin_report(&tmp, map);
461 return -EINVAL;
462 }
463 }
464
465 static int bpf_mnt_fs(const char *target)
466 {
467 bool bind_done = false;
468
469 while (mount("", target, "none", MS_PRIVATE | MS_REC, NULL)) {
470 if (errno != EINVAL || bind_done) {
471 fprintf(stderr, "mount --make-private %s failed: %s\n",
472 target, strerror(errno));
473 return -1;
474 }
475
476 if (mount(target, target, "none", MS_BIND, NULL)) {
477 fprintf(stderr, "mount --bind %s %s failed: %s\n",
478 target, target, strerror(errno));
479 return -1;
480 }
481
482 bind_done = true;
483 }
484
485 if (mount("bpf", target, "bpf", 0, "mode=0700")) {
486 fprintf(stderr, "mount -t bpf bpf %s failed: %s\n",
487 target, strerror(errno));
488 return -1;
489 }
490
491 return 0;
492 }
493
494 static int bpf_mnt_check_target(const char *target)
495 {
496 struct stat sb = {};
497 int ret;
498
499 ret = stat(target, &sb);
500 if (ret) {
501 ret = mkdir(target, S_IRWXU);
502 if (ret) {
503 fprintf(stderr, "mkdir %s failed: %s\n", target,
504 strerror(errno));
505 return ret;
506 }
507 }
508
509 return 0;
510 }
511
512 static int bpf_valid_mntpt(const char *mnt, unsigned long magic)
513 {
514 struct statfs st_fs;
515
516 if (statfs(mnt, &st_fs) < 0)
517 return -ENOENT;
518 if ((unsigned long)st_fs.f_type != magic)
519 return -ENOENT;
520
521 return 0;
522 }
523
524 static const char *bpf_find_mntpt_single(unsigned long magic, char *mnt,
525 int len, const char *mntpt)
526 {
527 int ret;
528
529 ret = bpf_valid_mntpt(mntpt, magic);
530 if (!ret) {
531 strlcpy(mnt, mntpt, len);
532 return mnt;
533 }
534
535 return NULL;
536 }
537
538 static const char *bpf_find_mntpt(const char *fstype, unsigned long magic,
539 char *mnt, int len,
540 const char * const *known_mnts)
541 {
542 const char * const *ptr;
543 char type[100];
544 FILE *fp;
545
546 if (known_mnts) {
547 ptr = known_mnts;
548 while (*ptr) {
549 if (bpf_find_mntpt_single(magic, mnt, len, *ptr))
550 return mnt;
551 ptr++;
552 }
553 }
554
555 if (len != PATH_MAX)
556 return NULL;
557
558 fp = fopen("/proc/mounts", "r");
559 if (fp == NULL)
560 return NULL;
561
562 while (fscanf(fp, "%*s %" textify(PATH_MAX) "s %99s %*s %*d %*d\n",
563 mnt, type) == 2) {
564 if (strcmp(type, fstype) == 0)
565 break;
566 }
567
568 fclose(fp);
569 if (strcmp(type, fstype) != 0)
570 return NULL;
571
572 return mnt;
573 }
574
575 int bpf_trace_pipe(void)
576 {
577 char tracefs_mnt[PATH_MAX] = TRACE_DIR_MNT;
578 static const char * const tracefs_known_mnts[] = {
579 TRACE_DIR_MNT,
580 "/sys/kernel/debug/tracing",
581 "/tracing",
582 "/trace",
583 0,
584 };
585 int fd_in, fd_out = STDERR_FILENO;
586 char tpipe[PATH_MAX];
587 const char *mnt;
588
589 mnt = bpf_find_mntpt("tracefs", TRACEFS_MAGIC, tracefs_mnt,
590 sizeof(tracefs_mnt), tracefs_known_mnts);
591 if (!mnt) {
592 fprintf(stderr, "tracefs not mounted?\n");
593 return -1;
594 }
595
596 snprintf(tpipe, sizeof(tpipe), "%s/trace_pipe", mnt);
597
598 fd_in = open(tpipe, O_RDONLY);
599 if (fd_in < 0)
600 return -1;
601
602 fprintf(stderr, "Running! Hang up with ^C!\n\n");
603 while (1) {
604 static char buff[4096];
605 ssize_t ret;
606
607 ret = read(fd_in, buff, sizeof(buff));
608 if (ret > 0 && write(fd_out, buff, ret) == ret)
609 continue;
610 break;
611 }
612
613 close(fd_in);
614 return -1;
615 }
616
617 static int bpf_gen_global(const char *bpf_sub_dir)
618 {
619 char bpf_glo_dir[PATH_MAX];
620 int ret;
621
622 snprintf(bpf_glo_dir, sizeof(bpf_glo_dir), "%s/%s/",
623 bpf_sub_dir, BPF_DIR_GLOBALS);
624
625 ret = mkdir(bpf_glo_dir, S_IRWXU);
626 if (ret && errno != EEXIST) {
627 fprintf(stderr, "mkdir %s failed: %s\n", bpf_glo_dir,
628 strerror(errno));
629 return ret;
630 }
631
632 return 0;
633 }
634
635 static int bpf_gen_master(const char *base, const char *name)
636 {
637 char bpf_sub_dir[PATH_MAX];
638 int ret;
639
640 snprintf(bpf_sub_dir, sizeof(bpf_sub_dir), "%s%s/", base, name);
641
642 ret = mkdir(bpf_sub_dir, S_IRWXU);
643 if (ret && errno != EEXIST) {
644 fprintf(stderr, "mkdir %s failed: %s\n", bpf_sub_dir,
645 strerror(errno));
646 return ret;
647 }
648
649 return bpf_gen_global(bpf_sub_dir);
650 }
651
652 static int bpf_slave_via_bind_mnt(const char *full_name,
653 const char *full_link)
654 {
655 int ret;
656
657 ret = mkdir(full_name, S_IRWXU);
658 if (ret) {
659 assert(errno != EEXIST);
660 fprintf(stderr, "mkdir %s failed: %s\n", full_name,
661 strerror(errno));
662 return ret;
663 }
664
665 ret = mount(full_link, full_name, "none", MS_BIND, NULL);
666 if (ret) {
667 rmdir(full_name);
668 fprintf(stderr, "mount --bind %s %s failed: %s\n",
669 full_link, full_name, strerror(errno));
670 }
671
672 return ret;
673 }
674
675 static int bpf_gen_slave(const char *base, const char *name,
676 const char *link)
677 {
678 char bpf_lnk_dir[PATH_MAX];
679 char bpf_sub_dir[PATH_MAX];
680 struct stat sb = {};
681 int ret;
682
683 snprintf(bpf_lnk_dir, sizeof(bpf_lnk_dir), "%s%s/", base, link);
684 snprintf(bpf_sub_dir, sizeof(bpf_sub_dir), "%s%s", base, name);
685
686 ret = symlink(bpf_lnk_dir, bpf_sub_dir);
687 if (ret) {
688 if (errno != EEXIST) {
689 if (errno != EPERM) {
690 fprintf(stderr, "symlink %s failed: %s\n",
691 bpf_sub_dir, strerror(errno));
692 return ret;
693 }
694
695 return bpf_slave_via_bind_mnt(bpf_sub_dir,
696 bpf_lnk_dir);
697 }
698
699 ret = lstat(bpf_sub_dir, &sb);
700 if (ret) {
701 fprintf(stderr, "lstat %s failed: %s\n",
702 bpf_sub_dir, strerror(errno));
703 return ret;
704 }
705
706 if ((sb.st_mode & S_IFMT) != S_IFLNK)
707 return bpf_gen_global(bpf_sub_dir);
708 }
709
710 return 0;
711 }
712
713 static int bpf_gen_hierarchy(const char *base)
714 {
715 int ret, i;
716
717 ret = bpf_gen_master(base, bpf_prog_to_subdir(__bpf_types[0]));
718 for (i = 1; i < ARRAY_SIZE(__bpf_types) && !ret; i++)
719 ret = bpf_gen_slave(base,
720 bpf_prog_to_subdir(__bpf_types[i]),
721 bpf_prog_to_subdir(__bpf_types[0]));
722 return ret;
723 }
724
725 static const char *bpf_get_work_dir(enum bpf_prog_type type)
726 {
727 static char bpf_tmp[PATH_MAX] = BPF_DIR_MNT;
728 static char bpf_wrk_dir[PATH_MAX];
729 static const char *mnt;
730 static bool bpf_mnt_cached;
731 const char *mnt_env = getenv(BPF_ENV_MNT);
732 static const char * const bpf_known_mnts[] = {
733 BPF_DIR_MNT,
734 "/bpf",
735 0,
736 };
737 int ret;
738
739 if (bpf_mnt_cached) {
740 const char *out = mnt;
741
742 if (out && type) {
743 snprintf(bpf_tmp, sizeof(bpf_tmp), "%s%s/",
744 out, bpf_prog_to_subdir(type));
745 out = bpf_tmp;
746 }
747 return out;
748 }
749
750 if (mnt_env)
751 mnt = bpf_find_mntpt_single(BPF_FS_MAGIC, bpf_tmp,
752 sizeof(bpf_tmp), mnt_env);
753 else
754 mnt = bpf_find_mntpt("bpf", BPF_FS_MAGIC, bpf_tmp,
755 sizeof(bpf_tmp), bpf_known_mnts);
756 if (!mnt) {
757 mnt = mnt_env ? : BPF_DIR_MNT;
758 ret = bpf_mnt_check_target(mnt);
759 if (!ret)
760 ret = bpf_mnt_fs(mnt);
761 if (ret) {
762 mnt = NULL;
763 goto out;
764 }
765 }
766
767 snprintf(bpf_wrk_dir, sizeof(bpf_wrk_dir), "%s/", mnt);
768
769 ret = bpf_gen_hierarchy(bpf_wrk_dir);
770 if (ret) {
771 mnt = NULL;
772 goto out;
773 }
774
775 mnt = bpf_wrk_dir;
776 out:
777 bpf_mnt_cached = true;
778 return mnt;
779 }
780
781 static int bpf_obj_get(const char *pathname, enum bpf_prog_type type)
782 {
783 union bpf_attr attr = {};
784 char tmp[PATH_MAX];
785
786 if (strlen(pathname) > 2 && pathname[0] == 'm' &&
787 pathname[1] == ':' && bpf_get_work_dir(type)) {
788 snprintf(tmp, sizeof(tmp), "%s/%s",
789 bpf_get_work_dir(type), pathname + 2);
790 pathname = tmp;
791 }
792
793 attr.pathname = bpf_ptr_to_u64(pathname);
794
795 return bpf(BPF_OBJ_GET, &attr, sizeof(attr));
796 }
797
798 static int bpf_obj_pinned(const char *pathname, enum bpf_prog_type type)
799 {
800 int prog_fd = bpf_obj_get(pathname, type);
801
802 if (prog_fd < 0)
803 fprintf(stderr, "Couldn\'t retrieve pinned program \'%s\': %s\n",
804 pathname, strerror(errno));
805 return prog_fd;
806 }
807
808 static int bpf_do_parse(struct bpf_cfg_in *cfg, const bool *opt_tbl)
809 {
810 const char *file, *section, *uds_name;
811 bool verbose = false;
812 int i, ret, argc;
813 char **argv;
814
815 argv = cfg->argv;
816 argc = cfg->argc;
817
818 if (opt_tbl[CBPF_BYTECODE] &&
819 (matches(*argv, "bytecode") == 0 ||
820 strcmp(*argv, "bc") == 0)) {
821 cfg->mode = CBPF_BYTECODE;
822 } else if (opt_tbl[CBPF_FILE] &&
823 (matches(*argv, "bytecode-file") == 0 ||
824 strcmp(*argv, "bcf") == 0)) {
825 cfg->mode = CBPF_FILE;
826 } else if (opt_tbl[EBPF_OBJECT] &&
827 (matches(*argv, "object-file") == 0 ||
828 strcmp(*argv, "obj") == 0)) {
829 cfg->mode = EBPF_OBJECT;
830 } else if (opt_tbl[EBPF_PINNED] &&
831 (matches(*argv, "object-pinned") == 0 ||
832 matches(*argv, "pinned") == 0 ||
833 matches(*argv, "fd") == 0)) {
834 cfg->mode = EBPF_PINNED;
835 } else {
836 fprintf(stderr, "What mode is \"%s\"?\n", *argv);
837 return -1;
838 }
839
840 NEXT_ARG();
841 file = section = uds_name = NULL;
842 if (cfg->mode == EBPF_OBJECT || cfg->mode == EBPF_PINNED) {
843 file = *argv;
844 NEXT_ARG_FWD();
845
846 if (cfg->type == BPF_PROG_TYPE_UNSPEC) {
847 if (argc > 0 && matches(*argv, "type") == 0) {
848 NEXT_ARG();
849 for (i = 0; i < ARRAY_SIZE(__bpf_prog_meta);
850 i++) {
851 if (!__bpf_prog_meta[i].type)
852 continue;
853 if (!matches(*argv,
854 __bpf_prog_meta[i].type)) {
855 cfg->type = i;
856 break;
857 }
858 }
859
860 if (cfg->type == BPF_PROG_TYPE_UNSPEC) {
861 fprintf(stderr, "What type is \"%s\"?\n",
862 *argv);
863 return -1;
864 }
865 NEXT_ARG_FWD();
866 } else {
867 cfg->type = BPF_PROG_TYPE_SCHED_CLS;
868 }
869 }
870
871 section = bpf_prog_to_default_section(cfg->type);
872 if (argc > 0 && matches(*argv, "section") == 0) {
873 NEXT_ARG();
874 section = *argv;
875 NEXT_ARG_FWD();
876 }
877
878 if (__bpf_prog_meta[cfg->type].may_uds_export) {
879 uds_name = getenv(BPF_ENV_UDS);
880 if (argc > 0 && !uds_name &&
881 matches(*argv, "export") == 0) {
882 NEXT_ARG();
883 uds_name = *argv;
884 NEXT_ARG_FWD();
885 }
886 }
887
888 if (argc > 0 && matches(*argv, "verbose") == 0) {
889 verbose = true;
890 NEXT_ARG_FWD();
891 }
892
893 PREV_ARG();
894 }
895
896 if (cfg->mode == CBPF_BYTECODE || cfg->mode == CBPF_FILE) {
897 ret = bpf_ops_parse(argc, argv, cfg->opcodes,
898 cfg->mode == CBPF_FILE);
899 cfg->n_opcodes = ret;
900 } else if (cfg->mode == EBPF_OBJECT) {
901 ret = 0; /* program will be loaded by load stage */
902 } else if (cfg->mode == EBPF_PINNED) {
903 ret = bpf_obj_pinned(file, cfg->type);
904 cfg->prog_fd = ret;
905 } else {
906 return -1;
907 }
908
909 cfg->object = file;
910 cfg->section = section;
911 cfg->uds = uds_name;
912 cfg->argc = argc;
913 cfg->argv = argv;
914 cfg->verbose = verbose;
915
916 return ret;
917 }
918
919 static int bpf_do_load(struct bpf_cfg_in *cfg)
920 {
921 if (cfg->mode == EBPF_OBJECT) {
922 cfg->prog_fd = bpf_obj_open(cfg->object, cfg->type,
923 cfg->section, cfg->ifindex,
924 cfg->verbose);
925 return cfg->prog_fd;
926 }
927 return 0;
928 }
929
930 int bpf_load_common(struct bpf_cfg_in *cfg, const struct bpf_cfg_ops *ops,
931 void *nl)
932 {
933 char annotation[256];
934 int ret;
935
936 ret = bpf_do_load(cfg);
937 if (ret < 0)
938 return ret;
939
940 if (cfg->mode == CBPF_BYTECODE || cfg->mode == CBPF_FILE)
941 ops->cbpf_cb(nl, cfg->opcodes, cfg->n_opcodes);
942 if (cfg->mode == EBPF_OBJECT || cfg->mode == EBPF_PINNED) {
943 snprintf(annotation, sizeof(annotation), "%s:[%s]",
944 basename(cfg->object), cfg->mode == EBPF_PINNED ?
945 "*fsobj" : cfg->section);
946 ops->ebpf_cb(nl, cfg->prog_fd, annotation);
947 }
948
949 return 0;
950 }
951
952 int bpf_parse_common(struct bpf_cfg_in *cfg, const struct bpf_cfg_ops *ops)
953 {
954 bool opt_tbl[BPF_MODE_MAX] = {};
955
956 if (ops->cbpf_cb) {
957 opt_tbl[CBPF_BYTECODE] = true;
958 opt_tbl[CBPF_FILE] = true;
959 }
960
961 if (ops->ebpf_cb) {
962 opt_tbl[EBPF_OBJECT] = true;
963 opt_tbl[EBPF_PINNED] = true;
964 }
965
966 return bpf_do_parse(cfg, opt_tbl);
967 }
968
969 int bpf_parse_and_load_common(struct bpf_cfg_in *cfg,
970 const struct bpf_cfg_ops *ops, void *nl)
971 {
972 int ret;
973
974 ret = bpf_parse_common(cfg, ops);
975 if (ret < 0)
976 return ret;
977
978 return bpf_load_common(cfg, ops, nl);
979 }
980
981 int bpf_graft_map(const char *map_path, uint32_t *key, int argc, char **argv)
982 {
983 const bool opt_tbl[BPF_MODE_MAX] = {
984 [EBPF_OBJECT] = true,
985 [EBPF_PINNED] = true,
986 };
987 const struct bpf_elf_map test = {
988 .type = BPF_MAP_TYPE_PROG_ARRAY,
989 .size_key = sizeof(int),
990 .size_value = sizeof(int),
991 };
992 struct bpf_cfg_in cfg = {
993 .type = BPF_PROG_TYPE_UNSPEC,
994 .argc = argc,
995 .argv = argv,
996 };
997 struct bpf_map_ext ext = {};
998 int ret, prog_fd, map_fd;
999 uint32_t map_key;
1000
1001 ret = bpf_do_parse(&cfg, opt_tbl);
1002 if (ret < 0)
1003 return ret;
1004
1005 ret = bpf_do_load(&cfg);
1006 if (ret < 0)
1007 return ret;
1008
1009 prog_fd = cfg.prog_fd;
1010
1011 if (key) {
1012 map_key = *key;
1013 } else {
1014 ret = sscanf(cfg.section, "%*i/%i", &map_key);
1015 if (ret != 1) {
1016 fprintf(stderr, "Couldn\'t infer map key from section name! Please provide \'key\' argument!\n");
1017 ret = -EINVAL;
1018 goto out_prog;
1019 }
1020 }
1021
1022 map_fd = bpf_obj_get(map_path, cfg.type);
1023 if (map_fd < 0) {
1024 fprintf(stderr, "Couldn\'t retrieve pinned map \'%s\': %s\n",
1025 map_path, strerror(errno));
1026 ret = map_fd;
1027 goto out_prog;
1028 }
1029
1030 ret = bpf_map_selfcheck_pinned(map_fd, &test, &ext,
1031 offsetof(struct bpf_elf_map, max_elem),
1032 cfg.type);
1033 if (ret < 0) {
1034 fprintf(stderr, "Map \'%s\' self-check failed!\n", map_path);
1035 goto out_map;
1036 }
1037
1038 ret = bpf_map_update(map_fd, &map_key, &prog_fd, BPF_ANY);
1039 if (ret < 0)
1040 fprintf(stderr, "Map update failed: %s\n", strerror(errno));
1041 out_map:
1042 close(map_fd);
1043 out_prog:
1044 close(prog_fd);
1045 return ret;
1046 }
1047
1048 int bpf_prog_attach_fd(int prog_fd, int target_fd, enum bpf_attach_type type)
1049 {
1050 union bpf_attr attr = {};
1051
1052 attr.target_fd = target_fd;
1053 attr.attach_bpf_fd = prog_fd;
1054 attr.attach_type = type;
1055
1056 return bpf(BPF_PROG_ATTACH, &attr, sizeof(attr));
1057 }
1058
1059 int bpf_prog_detach_fd(int target_fd, enum bpf_attach_type type)
1060 {
1061 union bpf_attr attr = {};
1062
1063 attr.target_fd = target_fd;
1064 attr.attach_type = type;
1065
1066 return bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
1067 }
1068
1069 static int bpf_prog_load_dev(enum bpf_prog_type type,
1070 const struct bpf_insn *insns, size_t size_insns,
1071 const char *license, __u32 ifindex,
1072 char *log, size_t size_log)
1073 {
1074 union bpf_attr attr = {};
1075
1076 attr.prog_type = type;
1077 attr.insns = bpf_ptr_to_u64(insns);
1078 attr.insn_cnt = size_insns / sizeof(struct bpf_insn);
1079 attr.license = bpf_ptr_to_u64(license);
1080 attr.prog_ifindex = ifindex;
1081
1082 if (size_log > 0) {
1083 attr.log_buf = bpf_ptr_to_u64(log);
1084 attr.log_size = size_log;
1085 attr.log_level = 1;
1086 }
1087
1088 return bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
1089 }
1090
1091 int bpf_prog_load(enum bpf_prog_type type, const struct bpf_insn *insns,
1092 size_t size_insns, const char *license, char *log,
1093 size_t size_log)
1094 {
1095 return bpf_prog_load_dev(type, insns, size_insns, license, 0,
1096 log, size_log);
1097 }
1098
1099 #ifdef HAVE_ELF
1100 struct bpf_elf_prog {
1101 enum bpf_prog_type type;
1102 const struct bpf_insn *insns;
1103 size_t size;
1104 const char *license;
1105 };
1106
1107 struct bpf_hash_entry {
1108 unsigned int pinning;
1109 const char *subpath;
1110 struct bpf_hash_entry *next;
1111 };
1112
1113 struct bpf_config {
1114 unsigned int jit_enabled;
1115 };
1116
1117 struct bpf_elf_ctx {
1118 struct bpf_config cfg;
1119 Elf *elf_fd;
1120 GElf_Ehdr elf_hdr;
1121 Elf_Data *sym_tab;
1122 Elf_Data *str_tab;
1123 int obj_fd;
1124 int map_fds[ELF_MAX_MAPS];
1125 struct bpf_elf_map maps[ELF_MAX_MAPS];
1126 struct bpf_map_ext maps_ext[ELF_MAX_MAPS];
1127 int sym_num;
1128 int map_num;
1129 int map_len;
1130 bool *sec_done;
1131 int sec_maps;
1132 char license[ELF_MAX_LICENSE_LEN];
1133 enum bpf_prog_type type;
1134 __u32 ifindex;
1135 bool verbose;
1136 struct bpf_elf_st stat;
1137 struct bpf_hash_entry *ht[256];
1138 char *log;
1139 size_t log_size;
1140 };
1141
1142 struct bpf_elf_sec_data {
1143 GElf_Shdr sec_hdr;
1144 Elf_Data *sec_data;
1145 const char *sec_name;
1146 };
1147
1148 struct bpf_map_data {
1149 int *fds;
1150 const char *obj;
1151 struct bpf_elf_st *st;
1152 struct bpf_elf_map *ent;
1153 };
1154
1155 static __check_format_string(2, 3) void
1156 bpf_dump_error(struct bpf_elf_ctx *ctx, const char *format, ...)
1157 {
1158 va_list vl;
1159
1160 va_start(vl, format);
1161 vfprintf(stderr, format, vl);
1162 va_end(vl);
1163
1164 if (ctx->log && ctx->log[0]) {
1165 if (ctx->verbose) {
1166 fprintf(stderr, "%s\n", ctx->log);
1167 } else {
1168 unsigned int off = 0, len = strlen(ctx->log);
1169
1170 if (len > BPF_MAX_LOG) {
1171 off = len - BPF_MAX_LOG;
1172 fprintf(stderr, "Skipped %u bytes, use \'verb\' option for the full verbose log.\n[...]\n",
1173 off);
1174 }
1175 fprintf(stderr, "%s\n", ctx->log + off);
1176 }
1177
1178 memset(ctx->log, 0, ctx->log_size);
1179 }
1180 }
1181
1182 static int bpf_log_realloc(struct bpf_elf_ctx *ctx)
1183 {
1184 const size_t log_max = UINT_MAX >> 8;
1185 size_t log_size = ctx->log_size;
1186 char *ptr;
1187
1188 if (!ctx->log) {
1189 log_size = 65536;
1190 } else if (log_size < log_max) {
1191 log_size <<= 1;
1192 if (log_size > log_max)
1193 log_size = log_max;
1194 } else {
1195 return -EINVAL;
1196 }
1197
1198 ptr = realloc(ctx->log, log_size);
1199 if (!ptr)
1200 return -ENOMEM;
1201
1202 ptr[0] = 0;
1203 ctx->log = ptr;
1204 ctx->log_size = log_size;
1205
1206 return 0;
1207 }
1208
1209 static int bpf_map_create(enum bpf_map_type type, uint32_t size_key,
1210 uint32_t size_value, uint32_t max_elem,
1211 uint32_t flags, int inner_fd, uint32_t ifindex)
1212 {
1213 union bpf_attr attr = {};
1214
1215 attr.map_type = type;
1216 attr.key_size = size_key;
1217 attr.value_size = inner_fd ? sizeof(int) : size_value;
1218 attr.max_entries = max_elem;
1219 attr.map_flags = flags;
1220 attr.inner_map_fd = inner_fd;
1221 attr.map_ifindex = ifindex;
1222
1223 return bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
1224 }
1225
1226 static int bpf_obj_pin(int fd, const char *pathname)
1227 {
1228 union bpf_attr attr = {};
1229
1230 attr.pathname = bpf_ptr_to_u64(pathname);
1231 attr.bpf_fd = fd;
1232
1233 return bpf(BPF_OBJ_PIN, &attr, sizeof(attr));
1234 }
1235
1236 static int bpf_obj_hash(const char *object, uint8_t *out, size_t len)
1237 {
1238 struct sockaddr_alg alg = {
1239 .salg_family = AF_ALG,
1240 .salg_type = "hash",
1241 .salg_name = "sha1",
1242 };
1243 int ret, cfd, ofd, ffd;
1244 struct stat stbuff;
1245 ssize_t size;
1246
1247 if (!object || len != 20)
1248 return -EINVAL;
1249
1250 cfd = socket(AF_ALG, SOCK_SEQPACKET, 0);
1251 if (cfd < 0) {
1252 fprintf(stderr, "Cannot get AF_ALG socket: %s\n",
1253 strerror(errno));
1254 return cfd;
1255 }
1256
1257 ret = bind(cfd, (struct sockaddr *)&alg, sizeof(alg));
1258 if (ret < 0) {
1259 fprintf(stderr, "Error binding socket: %s\n", strerror(errno));
1260 goto out_cfd;
1261 }
1262
1263 ofd = accept(cfd, NULL, 0);
1264 if (ofd < 0) {
1265 fprintf(stderr, "Error accepting socket: %s\n",
1266 strerror(errno));
1267 ret = ofd;
1268 goto out_cfd;
1269 }
1270
1271 ffd = open(object, O_RDONLY);
1272 if (ffd < 0) {
1273 fprintf(stderr, "Error opening object %s: %s\n",
1274 object, strerror(errno));
1275 ret = ffd;
1276 goto out_ofd;
1277 }
1278
1279 ret = fstat(ffd, &stbuff);
1280 if (ret < 0) {
1281 fprintf(stderr, "Error doing fstat: %s\n",
1282 strerror(errno));
1283 goto out_ffd;
1284 }
1285
1286 size = sendfile(ofd, ffd, NULL, stbuff.st_size);
1287 if (size != stbuff.st_size) {
1288 fprintf(stderr, "Error from sendfile (%zd vs %zu bytes): %s\n",
1289 size, stbuff.st_size, strerror(errno));
1290 ret = -1;
1291 goto out_ffd;
1292 }
1293
1294 size = read(ofd, out, len);
1295 if (size != len) {
1296 fprintf(stderr, "Error from read (%zd vs %zu bytes): %s\n",
1297 size, len, strerror(errno));
1298 ret = -1;
1299 } else {
1300 ret = 0;
1301 }
1302 out_ffd:
1303 close(ffd);
1304 out_ofd:
1305 close(ofd);
1306 out_cfd:
1307 close(cfd);
1308 return ret;
1309 }
1310
1311 static const char *bpf_get_obj_uid(const char *pathname)
1312 {
1313 static bool bpf_uid_cached;
1314 static char bpf_uid[64];
1315 uint8_t tmp[20];
1316 int ret;
1317
1318 if (bpf_uid_cached)
1319 goto done;
1320
1321 ret = bpf_obj_hash(pathname, tmp, sizeof(tmp));
1322 if (ret) {
1323 fprintf(stderr, "Object hashing failed!\n");
1324 return NULL;
1325 }
1326
1327 hexstring_n2a(tmp, sizeof(tmp), bpf_uid, sizeof(bpf_uid));
1328 bpf_uid_cached = true;
1329 done:
1330 return bpf_uid;
1331 }
1332
1333 static int bpf_init_env(const char *pathname)
1334 {
1335 struct rlimit limit = {
1336 .rlim_cur = RLIM_INFINITY,
1337 .rlim_max = RLIM_INFINITY,
1338 };
1339
1340 /* Don't bother in case we fail! */
1341 setrlimit(RLIMIT_MEMLOCK, &limit);
1342
1343 if (!bpf_get_work_dir(BPF_PROG_TYPE_UNSPEC)) {
1344 fprintf(stderr, "Continuing without mounted eBPF fs. Too old kernel?\n");
1345 return 0;
1346 }
1347
1348 if (!bpf_get_obj_uid(pathname))
1349 return -1;
1350
1351 return 0;
1352 }
1353
1354 static const char *bpf_custom_pinning(const struct bpf_elf_ctx *ctx,
1355 uint32_t pinning)
1356 {
1357 struct bpf_hash_entry *entry;
1358
1359 entry = ctx->ht[pinning & (ARRAY_SIZE(ctx->ht) - 1)];
1360 while (entry && entry->pinning != pinning)
1361 entry = entry->next;
1362
1363 return entry ? entry->subpath : NULL;
1364 }
1365
1366 static bool bpf_no_pinning(const struct bpf_elf_ctx *ctx,
1367 uint32_t pinning)
1368 {
1369 switch (pinning) {
1370 case PIN_OBJECT_NS:
1371 case PIN_GLOBAL_NS:
1372 return false;
1373 case PIN_NONE:
1374 return true;
1375 default:
1376 return !bpf_custom_pinning(ctx, pinning);
1377 }
1378 }
1379
1380 static void bpf_make_pathname(char *pathname, size_t len, const char *name,
1381 const struct bpf_elf_ctx *ctx, uint32_t pinning)
1382 {
1383 switch (pinning) {
1384 case PIN_OBJECT_NS:
1385 snprintf(pathname, len, "%s/%s/%s",
1386 bpf_get_work_dir(ctx->type),
1387 bpf_get_obj_uid(NULL), name);
1388 break;
1389 case PIN_GLOBAL_NS:
1390 snprintf(pathname, len, "%s/%s/%s",
1391 bpf_get_work_dir(ctx->type),
1392 BPF_DIR_GLOBALS, name);
1393 break;
1394 default:
1395 snprintf(pathname, len, "%s/../%s/%s",
1396 bpf_get_work_dir(ctx->type),
1397 bpf_custom_pinning(ctx, pinning), name);
1398 break;
1399 }
1400 }
1401
1402 static int bpf_probe_pinned(const char *name, const struct bpf_elf_ctx *ctx,
1403 uint32_t pinning)
1404 {
1405 char pathname[PATH_MAX];
1406
1407 if (bpf_no_pinning(ctx, pinning) || !bpf_get_work_dir(ctx->type))
1408 return 0;
1409
1410 bpf_make_pathname(pathname, sizeof(pathname), name, ctx, pinning);
1411 return bpf_obj_get(pathname, ctx->type);
1412 }
1413
1414 static int bpf_make_obj_path(const struct bpf_elf_ctx *ctx)
1415 {
1416 char tmp[PATH_MAX];
1417 int ret;
1418
1419 snprintf(tmp, sizeof(tmp), "%s/%s", bpf_get_work_dir(ctx->type),
1420 bpf_get_obj_uid(NULL));
1421
1422 ret = mkdir(tmp, S_IRWXU);
1423 if (ret && errno != EEXIST) {
1424 fprintf(stderr, "mkdir %s failed: %s\n", tmp, strerror(errno));
1425 return ret;
1426 }
1427
1428 return 0;
1429 }
1430
1431 static int bpf_make_custom_path(const struct bpf_elf_ctx *ctx,
1432 const char *todo)
1433 {
1434 char tmp[PATH_MAX], rem[PATH_MAX], *sub;
1435 int ret;
1436
1437 snprintf(tmp, sizeof(tmp), "%s/../", bpf_get_work_dir(ctx->type));
1438 snprintf(rem, sizeof(rem), "%s/", todo);
1439 sub = strtok(rem, "/");
1440
1441 while (sub) {
1442 if (strlen(tmp) + strlen(sub) + 2 > PATH_MAX)
1443 return -EINVAL;
1444
1445 strcat(tmp, sub);
1446 strcat(tmp, "/");
1447
1448 ret = mkdir(tmp, S_IRWXU);
1449 if (ret && errno != EEXIST) {
1450 fprintf(stderr, "mkdir %s failed: %s\n", tmp,
1451 strerror(errno));
1452 return ret;
1453 }
1454
1455 sub = strtok(NULL, "/");
1456 }
1457
1458 return 0;
1459 }
1460
1461 static int bpf_place_pinned(int fd, const char *name,
1462 const struct bpf_elf_ctx *ctx, uint32_t pinning)
1463 {
1464 char pathname[PATH_MAX];
1465 const char *tmp;
1466 int ret = 0;
1467
1468 if (bpf_no_pinning(ctx, pinning) || !bpf_get_work_dir(ctx->type))
1469 return 0;
1470
1471 if (pinning == PIN_OBJECT_NS)
1472 ret = bpf_make_obj_path(ctx);
1473 else if ((tmp = bpf_custom_pinning(ctx, pinning)))
1474 ret = bpf_make_custom_path(ctx, tmp);
1475 if (ret < 0)
1476 return ret;
1477
1478 bpf_make_pathname(pathname, sizeof(pathname), name, ctx, pinning);
1479 return bpf_obj_pin(fd, pathname);
1480 }
1481
1482 static void bpf_prog_report(int fd, const char *section,
1483 const struct bpf_elf_prog *prog,
1484 struct bpf_elf_ctx *ctx)
1485 {
1486 unsigned int insns = prog->size / sizeof(struct bpf_insn);
1487
1488 fprintf(stderr, "\nProg section \'%s\' %s%s (%d)!\n", section,
1489 fd < 0 ? "rejected: " : "loaded",
1490 fd < 0 ? strerror(errno) : "",
1491 fd < 0 ? errno : fd);
1492
1493 fprintf(stderr, " - Type: %u\n", prog->type);
1494 fprintf(stderr, " - Instructions: %u (%u over limit)\n",
1495 insns, insns > BPF_MAXINSNS ? insns - BPF_MAXINSNS : 0);
1496 fprintf(stderr, " - License: %s\n\n", prog->license);
1497
1498 bpf_dump_error(ctx, "Verifier analysis:\n\n");
1499 }
1500
1501 static int bpf_prog_attach(const char *section,
1502 const struct bpf_elf_prog *prog,
1503 struct bpf_elf_ctx *ctx)
1504 {
1505 int tries = 0, fd;
1506 retry:
1507 errno = 0;
1508 fd = bpf_prog_load_dev(prog->type, prog->insns, prog->size,
1509 prog->license, ctx->ifindex,
1510 ctx->log, ctx->log_size);
1511 if (fd < 0 || ctx->verbose) {
1512 /* The verifier log is pretty chatty, sometimes so chatty
1513 * on larger programs, that we could fail to dump everything
1514 * into our buffer. Still, try to give a debuggable error
1515 * log for the user, so enlarge it and re-fail.
1516 */
1517 if (fd < 0 && (errno == ENOSPC || !ctx->log_size)) {
1518 if (tries++ < 10 && !bpf_log_realloc(ctx))
1519 goto retry;
1520
1521 fprintf(stderr, "Log buffer too small to dump verifier log %zu bytes (%d tries)!\n",
1522 ctx->log_size, tries);
1523 return fd;
1524 }
1525
1526 bpf_prog_report(fd, section, prog, ctx);
1527 }
1528
1529 return fd;
1530 }
1531
1532 static void bpf_map_report(int fd, const char *name,
1533 const struct bpf_elf_map *map,
1534 struct bpf_elf_ctx *ctx, int inner_fd)
1535 {
1536 fprintf(stderr, "Map object \'%s\' %s%s (%d)!\n", name,
1537 fd < 0 ? "rejected: " : "loaded",
1538 fd < 0 ? strerror(errno) : "",
1539 fd < 0 ? errno : fd);
1540
1541 fprintf(stderr, " - Type: %u\n", map->type);
1542 fprintf(stderr, " - Identifier: %u\n", map->id);
1543 fprintf(stderr, " - Pinning: %u\n", map->pinning);
1544 fprintf(stderr, " - Size key: %u\n", map->size_key);
1545 fprintf(stderr, " - Size value: %u\n",
1546 inner_fd ? (int)sizeof(int) : map->size_value);
1547 fprintf(stderr, " - Max elems: %u\n", map->max_elem);
1548 fprintf(stderr, " - Flags: %#x\n\n", map->flags);
1549 }
1550
1551 static int bpf_find_map_id(const struct bpf_elf_ctx *ctx, uint32_t id)
1552 {
1553 int i;
1554
1555 for (i = 0; i < ctx->map_num; i++) {
1556 if (ctx->maps[i].id != id)
1557 continue;
1558 if (ctx->map_fds[i] < 0)
1559 return -EINVAL;
1560
1561 return ctx->map_fds[i];
1562 }
1563
1564 return -ENOENT;
1565 }
1566
1567 static void bpf_report_map_in_map(int outer_fd, uint32_t idx)
1568 {
1569 struct bpf_elf_map outer_map;
1570 int ret;
1571
1572 fprintf(stderr, "Cannot insert map into map! ");
1573
1574 ret = bpf_derive_elf_map_from_fdinfo(outer_fd, &outer_map, NULL);
1575 if (!ret) {
1576 if (idx >= outer_map.max_elem &&
1577 outer_map.type == BPF_MAP_TYPE_ARRAY_OF_MAPS) {
1578 fprintf(stderr, "Outer map has %u elements, index %u is invalid!\n",
1579 outer_map.max_elem, idx);
1580 return;
1581 }
1582 }
1583
1584 fprintf(stderr, "Different map specs used for outer and inner map?\n");
1585 }
1586
1587 static bool bpf_is_map_in_map_type(const struct bpf_elf_map *map)
1588 {
1589 return map->type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1590 map->type == BPF_MAP_TYPE_HASH_OF_MAPS;
1591 }
1592
1593 static int bpf_map_attach(const char *name, struct bpf_elf_ctx *ctx,
1594 const struct bpf_elf_map *map, struct bpf_map_ext *ext,
1595 int *have_map_in_map)
1596 {
1597 int fd, ret, map_inner_fd = 0;
1598
1599 fd = bpf_probe_pinned(name, ctx, map->pinning);
1600 if (fd > 0) {
1601 ret = bpf_map_selfcheck_pinned(fd, map, ext,
1602 offsetof(struct bpf_elf_map,
1603 id), ctx->type);
1604 if (ret < 0) {
1605 close(fd);
1606 fprintf(stderr, "Map \'%s\' self-check failed!\n",
1607 name);
1608 return ret;
1609 }
1610 if (ctx->verbose)
1611 fprintf(stderr, "Map \'%s\' loaded as pinned!\n",
1612 name);
1613 return fd;
1614 }
1615
1616 if (have_map_in_map && bpf_is_map_in_map_type(map)) {
1617 (*have_map_in_map)++;
1618 if (map->inner_id)
1619 return 0;
1620 fprintf(stderr, "Map \'%s\' cannot be created since no inner map ID defined!\n",
1621 name);
1622 return -EINVAL;
1623 }
1624
1625 if (!have_map_in_map && bpf_is_map_in_map_type(map)) {
1626 map_inner_fd = bpf_find_map_id(ctx, map->inner_id);
1627 if (map_inner_fd < 0) {
1628 fprintf(stderr, "Map \'%s\' cannot be loaded. Inner map with ID %u not found!\n",
1629 name, map->inner_id);
1630 return -EINVAL;
1631 }
1632 }
1633
1634 errno = 0;
1635 fd = bpf_map_create(map->type, map->size_key, map->size_value,
1636 map->max_elem, map->flags, map_inner_fd,
1637 ctx->ifindex);
1638
1639 if (fd < 0 || ctx->verbose) {
1640 bpf_map_report(fd, name, map, ctx, map_inner_fd);
1641 if (fd < 0)
1642 return fd;
1643 }
1644
1645 ret = bpf_place_pinned(fd, name, ctx, map->pinning);
1646 if (ret < 0 && errno != EEXIST) {
1647 fprintf(stderr, "Could not pin %s map: %s\n", name,
1648 strerror(errno));
1649 close(fd);
1650 return ret;
1651 }
1652
1653 return fd;
1654 }
1655
1656 static const char *bpf_str_tab_name(const struct bpf_elf_ctx *ctx,
1657 const GElf_Sym *sym)
1658 {
1659 return ctx->str_tab->d_buf + sym->st_name;
1660 }
1661
1662 static const char *bpf_map_fetch_name(struct bpf_elf_ctx *ctx, int which)
1663 {
1664 GElf_Sym sym;
1665 int i;
1666
1667 for (i = 0; i < ctx->sym_num; i++) {
1668 if (gelf_getsym(ctx->sym_tab, i, &sym) != &sym)
1669 continue;
1670
1671 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1672 GELF_ST_TYPE(sym.st_info) != STT_NOTYPE ||
1673 sym.st_shndx != ctx->sec_maps ||
1674 sym.st_value / ctx->map_len != which)
1675 continue;
1676
1677 return bpf_str_tab_name(ctx, &sym);
1678 }
1679
1680 return NULL;
1681 }
1682
1683 static int bpf_maps_attach_all(struct bpf_elf_ctx *ctx)
1684 {
1685 int i, j, ret, fd, inner_fd, inner_idx, have_map_in_map = 0;
1686 const char *map_name;
1687
1688 for (i = 0; i < ctx->map_num; i++) {
1689 map_name = bpf_map_fetch_name(ctx, i);
1690 if (!map_name)
1691 return -EIO;
1692
1693 fd = bpf_map_attach(map_name, ctx, &ctx->maps[i],
1694 &ctx->maps_ext[i], &have_map_in_map);
1695 if (fd < 0)
1696 return fd;
1697
1698 ctx->map_fds[i] = !fd ? -1 : fd;
1699 }
1700
1701 for (i = 0; have_map_in_map && i < ctx->map_num; i++) {
1702 if (ctx->map_fds[i] >= 0)
1703 continue;
1704
1705 map_name = bpf_map_fetch_name(ctx, i);
1706 if (!map_name)
1707 return -EIO;
1708
1709 fd = bpf_map_attach(map_name, ctx, &ctx->maps[i],
1710 &ctx->maps_ext[i], NULL);
1711 if (fd < 0)
1712 return fd;
1713
1714 ctx->map_fds[i] = fd;
1715 }
1716
1717 for (i = 0; have_map_in_map && i < ctx->map_num; i++) {
1718 if (!ctx->maps[i].id ||
1719 ctx->maps[i].inner_id ||
1720 ctx->maps[i].inner_idx == -1)
1721 continue;
1722
1723 inner_fd = ctx->map_fds[i];
1724 inner_idx = ctx->maps[i].inner_idx;
1725
1726 for (j = 0; j < ctx->map_num; j++) {
1727 if (!bpf_is_map_in_map_type(&ctx->maps[j]))
1728 continue;
1729 if (ctx->maps[j].inner_id != ctx->maps[i].id)
1730 continue;
1731
1732 ret = bpf_map_update(ctx->map_fds[j], &inner_idx,
1733 &inner_fd, BPF_ANY);
1734 if (ret < 0) {
1735 bpf_report_map_in_map(ctx->map_fds[j],
1736 inner_idx);
1737 return ret;
1738 }
1739 }
1740 }
1741
1742 return 0;
1743 }
1744
1745 static int bpf_map_num_sym(struct bpf_elf_ctx *ctx)
1746 {
1747 int i, num = 0;
1748 GElf_Sym sym;
1749
1750 for (i = 0; i < ctx->sym_num; i++) {
1751 if (gelf_getsym(ctx->sym_tab, i, &sym) != &sym)
1752 continue;
1753
1754 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1755 GELF_ST_TYPE(sym.st_info) != STT_NOTYPE ||
1756 sym.st_shndx != ctx->sec_maps)
1757 continue;
1758 num++;
1759 }
1760
1761 return num;
1762 }
1763
1764 static int bpf_fill_section_data(struct bpf_elf_ctx *ctx, int section,
1765 struct bpf_elf_sec_data *data)
1766 {
1767 Elf_Data *sec_edata;
1768 GElf_Shdr sec_hdr;
1769 Elf_Scn *sec_fd;
1770 char *sec_name;
1771
1772 memset(data, 0, sizeof(*data));
1773
1774 sec_fd = elf_getscn(ctx->elf_fd, section);
1775 if (!sec_fd)
1776 return -EINVAL;
1777 if (gelf_getshdr(sec_fd, &sec_hdr) != &sec_hdr)
1778 return -EIO;
1779
1780 sec_name = elf_strptr(ctx->elf_fd, ctx->elf_hdr.e_shstrndx,
1781 sec_hdr.sh_name);
1782 if (!sec_name || !sec_hdr.sh_size)
1783 return -ENOENT;
1784
1785 sec_edata = elf_getdata(sec_fd, NULL);
1786 if (!sec_edata || elf_getdata(sec_fd, sec_edata))
1787 return -EIO;
1788
1789 memcpy(&data->sec_hdr, &sec_hdr, sizeof(sec_hdr));
1790
1791 data->sec_name = sec_name;
1792 data->sec_data = sec_edata;
1793 return 0;
1794 }
1795
1796 struct bpf_elf_map_min {
1797 __u32 type;
1798 __u32 size_key;
1799 __u32 size_value;
1800 __u32 max_elem;
1801 };
1802
1803 static int bpf_fetch_maps_begin(struct bpf_elf_ctx *ctx, int section,
1804 struct bpf_elf_sec_data *data)
1805 {
1806 ctx->map_num = data->sec_data->d_size;
1807 ctx->sec_maps = section;
1808 ctx->sec_done[section] = true;
1809
1810 if (ctx->map_num > sizeof(ctx->maps)) {
1811 fprintf(stderr, "Too many BPF maps in ELF section!\n");
1812 return -ENOMEM;
1813 }
1814
1815 memcpy(ctx->maps, data->sec_data->d_buf, ctx->map_num);
1816 return 0;
1817 }
1818
1819 static int bpf_map_verify_all_offs(struct bpf_elf_ctx *ctx, int end)
1820 {
1821 GElf_Sym sym;
1822 int off, i;
1823
1824 for (off = 0; off < end; off += ctx->map_len) {
1825 /* Order doesn't need to be linear here, hence we walk
1826 * the table again.
1827 */
1828 for (i = 0; i < ctx->sym_num; i++) {
1829 if (gelf_getsym(ctx->sym_tab, i, &sym) != &sym)
1830 continue;
1831 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1832 GELF_ST_TYPE(sym.st_info) != STT_NOTYPE ||
1833 sym.st_shndx != ctx->sec_maps)
1834 continue;
1835 if (sym.st_value == off)
1836 break;
1837 if (i == ctx->sym_num - 1)
1838 return -1;
1839 }
1840 }
1841
1842 return off == end ? 0 : -1;
1843 }
1844
1845 static int bpf_fetch_maps_end(struct bpf_elf_ctx *ctx)
1846 {
1847 struct bpf_elf_map fixup[ARRAY_SIZE(ctx->maps)] = {};
1848 int i, sym_num = bpf_map_num_sym(ctx);
1849 __u8 *buff;
1850
1851 if (sym_num == 0 || sym_num > ARRAY_SIZE(ctx->maps)) {
1852 fprintf(stderr, "%u maps not supported in current map section!\n",
1853 sym_num);
1854 return -EINVAL;
1855 }
1856
1857 if (ctx->map_num % sym_num != 0 ||
1858 ctx->map_num % sizeof(__u32) != 0) {
1859 fprintf(stderr, "Number BPF map symbols are not multiple of struct bpf_elf_map!\n");
1860 return -EINVAL;
1861 }
1862
1863 ctx->map_len = ctx->map_num / sym_num;
1864 if (bpf_map_verify_all_offs(ctx, ctx->map_num)) {
1865 fprintf(stderr, "Different struct bpf_elf_map in use!\n");
1866 return -EINVAL;
1867 }
1868
1869 if (ctx->map_len == sizeof(struct bpf_elf_map)) {
1870 ctx->map_num = sym_num;
1871 return 0;
1872 } else if (ctx->map_len > sizeof(struct bpf_elf_map)) {
1873 fprintf(stderr, "struct bpf_elf_map not supported, coming from future version?\n");
1874 return -EINVAL;
1875 } else if (ctx->map_len < sizeof(struct bpf_elf_map_min)) {
1876 fprintf(stderr, "struct bpf_elf_map too small, not supported!\n");
1877 return -EINVAL;
1878 }
1879
1880 ctx->map_num = sym_num;
1881 for (i = 0, buff = (void *)ctx->maps; i < ctx->map_num;
1882 i++, buff += ctx->map_len) {
1883 /* The fixup leaves the rest of the members as zero, which
1884 * is fine currently, but option exist to set some other
1885 * default value as well when needed in future.
1886 */
1887 memcpy(&fixup[i], buff, ctx->map_len);
1888 }
1889
1890 memcpy(ctx->maps, fixup, sizeof(fixup));
1891
1892 printf("Note: %zu bytes struct bpf_elf_map fixup performed due to size mismatch!\n",
1893 sizeof(struct bpf_elf_map) - ctx->map_len);
1894 return 0;
1895 }
1896
1897 static int bpf_fetch_license(struct bpf_elf_ctx *ctx, int section,
1898 struct bpf_elf_sec_data *data)
1899 {
1900 if (data->sec_data->d_size > sizeof(ctx->license))
1901 return -ENOMEM;
1902
1903 memcpy(ctx->license, data->sec_data->d_buf, data->sec_data->d_size);
1904 ctx->sec_done[section] = true;
1905 return 0;
1906 }
1907
1908 static int bpf_fetch_symtab(struct bpf_elf_ctx *ctx, int section,
1909 struct bpf_elf_sec_data *data)
1910 {
1911 ctx->sym_tab = data->sec_data;
1912 ctx->sym_num = data->sec_hdr.sh_size / data->sec_hdr.sh_entsize;
1913 ctx->sec_done[section] = true;
1914 return 0;
1915 }
1916
1917 static int bpf_fetch_strtab(struct bpf_elf_ctx *ctx, int section,
1918 struct bpf_elf_sec_data *data)
1919 {
1920 ctx->str_tab = data->sec_data;
1921 ctx->sec_done[section] = true;
1922 return 0;
1923 }
1924
1925 static bool bpf_has_map_data(const struct bpf_elf_ctx *ctx)
1926 {
1927 return ctx->sym_tab && ctx->str_tab && ctx->sec_maps;
1928 }
1929
1930 static int bpf_fetch_ancillary(struct bpf_elf_ctx *ctx)
1931 {
1932 struct bpf_elf_sec_data data;
1933 int i, ret = -1;
1934
1935 for (i = 1; i < ctx->elf_hdr.e_shnum; i++) {
1936 ret = bpf_fill_section_data(ctx, i, &data);
1937 if (ret < 0)
1938 continue;
1939
1940 if (data.sec_hdr.sh_type == SHT_PROGBITS &&
1941 !strcmp(data.sec_name, ELF_SECTION_MAPS))
1942 ret = bpf_fetch_maps_begin(ctx, i, &data);
1943 else if (data.sec_hdr.sh_type == SHT_PROGBITS &&
1944 !strcmp(data.sec_name, ELF_SECTION_LICENSE))
1945 ret = bpf_fetch_license(ctx, i, &data);
1946 else if (data.sec_hdr.sh_type == SHT_SYMTAB &&
1947 !strcmp(data.sec_name, ".symtab"))
1948 ret = bpf_fetch_symtab(ctx, i, &data);
1949 else if (data.sec_hdr.sh_type == SHT_STRTAB &&
1950 !strcmp(data.sec_name, ".strtab"))
1951 ret = bpf_fetch_strtab(ctx, i, &data);
1952 if (ret < 0) {
1953 fprintf(stderr, "Error parsing section %d! Perhaps check with readelf -a?\n",
1954 i);
1955 return ret;
1956 }
1957 }
1958
1959 if (bpf_has_map_data(ctx)) {
1960 ret = bpf_fetch_maps_end(ctx);
1961 if (ret < 0) {
1962 fprintf(stderr, "Error fixing up map structure, incompatible struct bpf_elf_map used?\n");
1963 return ret;
1964 }
1965
1966 ret = bpf_maps_attach_all(ctx);
1967 if (ret < 0) {
1968 fprintf(stderr, "Error loading maps into kernel!\n");
1969 return ret;
1970 }
1971 }
1972
1973 return ret;
1974 }
1975
1976 static int bpf_fetch_prog(struct bpf_elf_ctx *ctx, const char *section,
1977 bool *sseen)
1978 {
1979 struct bpf_elf_sec_data data;
1980 struct bpf_elf_prog prog;
1981 int ret, i, fd = -1;
1982
1983 for (i = 1; i < ctx->elf_hdr.e_shnum; i++) {
1984 if (ctx->sec_done[i])
1985 continue;
1986
1987 ret = bpf_fill_section_data(ctx, i, &data);
1988 if (ret < 0 ||
1989 !(data.sec_hdr.sh_type == SHT_PROGBITS &&
1990 data.sec_hdr.sh_flags & SHF_EXECINSTR &&
1991 !strcmp(data.sec_name, section)))
1992 continue;
1993
1994 *sseen = true;
1995
1996 memset(&prog, 0, sizeof(prog));
1997 prog.type = ctx->type;
1998 prog.insns = data.sec_data->d_buf;
1999 prog.size = data.sec_data->d_size;
2000 prog.license = ctx->license;
2001
2002 fd = bpf_prog_attach(section, &prog, ctx);
2003 if (fd < 0)
2004 return fd;
2005
2006 ctx->sec_done[i] = true;
2007 break;
2008 }
2009
2010 return fd;
2011 }
2012
2013 struct bpf_tail_call_props {
2014 unsigned int total;
2015 unsigned int jited;
2016 };
2017
2018 static int bpf_apply_relo_data(struct bpf_elf_ctx *ctx,
2019 struct bpf_elf_sec_data *data_relo,
2020 struct bpf_elf_sec_data *data_insn,
2021 struct bpf_tail_call_props *props)
2022 {
2023 Elf_Data *idata = data_insn->sec_data;
2024 GElf_Shdr *rhdr = &data_relo->sec_hdr;
2025 int relo_ent, relo_num = rhdr->sh_size / rhdr->sh_entsize;
2026 struct bpf_insn *insns = idata->d_buf;
2027 unsigned int num_insns = idata->d_size / sizeof(*insns);
2028
2029 for (relo_ent = 0; relo_ent < relo_num; relo_ent++) {
2030 unsigned int ioff, rmap;
2031 GElf_Rel relo;
2032 GElf_Sym sym;
2033
2034 if (gelf_getrel(data_relo->sec_data, relo_ent, &relo) != &relo)
2035 return -EIO;
2036
2037 ioff = relo.r_offset / sizeof(struct bpf_insn);
2038 if (ioff >= num_insns ||
2039 insns[ioff].code != (BPF_LD | BPF_IMM | BPF_DW)) {
2040 fprintf(stderr, "ELF contains relo data for non ld64 instruction at offset %u! Compiler bug?!\n",
2041 ioff);
2042 fprintf(stderr, " - Current section: %s\n", data_relo->sec_name);
2043 if (ioff < num_insns &&
2044 insns[ioff].code == (BPF_JMP | BPF_CALL))
2045 fprintf(stderr, " - Try to annotate functions with always_inline attribute!\n");
2046 return -EINVAL;
2047 }
2048
2049 if (gelf_getsym(ctx->sym_tab, GELF_R_SYM(relo.r_info), &sym) != &sym)
2050 return -EIO;
2051 if (sym.st_shndx != ctx->sec_maps) {
2052 fprintf(stderr, "ELF contains non-map related relo data in entry %u pointing to section %u! Compiler bug?!\n",
2053 relo_ent, sym.st_shndx);
2054 return -EIO;
2055 }
2056
2057 rmap = sym.st_value / ctx->map_len;
2058 if (rmap >= ARRAY_SIZE(ctx->map_fds))
2059 return -EINVAL;
2060 if (!ctx->map_fds[rmap])
2061 return -EINVAL;
2062 if (ctx->maps[rmap].type == BPF_MAP_TYPE_PROG_ARRAY) {
2063 props->total++;
2064 if (ctx->maps_ext[rmap].owner.jited ||
2065 (ctx->maps_ext[rmap].owner.type == 0 &&
2066 ctx->cfg.jit_enabled))
2067 props->jited++;
2068 }
2069
2070 if (ctx->verbose)
2071 fprintf(stderr, "Map \'%s\' (%d) injected into prog section \'%s\' at offset %u!\n",
2072 bpf_str_tab_name(ctx, &sym), ctx->map_fds[rmap],
2073 data_insn->sec_name, ioff);
2074
2075 insns[ioff].src_reg = BPF_PSEUDO_MAP_FD;
2076 insns[ioff].imm = ctx->map_fds[rmap];
2077 }
2078
2079 return 0;
2080 }
2081
2082 static int bpf_fetch_prog_relo(struct bpf_elf_ctx *ctx, const char *section,
2083 bool *lderr, bool *sseen)
2084 {
2085 struct bpf_elf_sec_data data_relo, data_insn;
2086 struct bpf_elf_prog prog;
2087 int ret, idx, i, fd = -1;
2088
2089 for (i = 1; i < ctx->elf_hdr.e_shnum; i++) {
2090 struct bpf_tail_call_props props = {};
2091
2092 ret = bpf_fill_section_data(ctx, i, &data_relo);
2093 if (ret < 0 || data_relo.sec_hdr.sh_type != SHT_REL)
2094 continue;
2095
2096 idx = data_relo.sec_hdr.sh_info;
2097
2098 ret = bpf_fill_section_data(ctx, idx, &data_insn);
2099 if (ret < 0 ||
2100 !(data_insn.sec_hdr.sh_type == SHT_PROGBITS &&
2101 data_insn.sec_hdr.sh_flags & SHF_EXECINSTR &&
2102 !strcmp(data_insn.sec_name, section)))
2103 continue;
2104
2105 *sseen = true;
2106
2107 ret = bpf_apply_relo_data(ctx, &data_relo, &data_insn, &props);
2108 if (ret < 0) {
2109 *lderr = true;
2110 return ret;
2111 }
2112
2113 memset(&prog, 0, sizeof(prog));
2114 prog.type = ctx->type;
2115 prog.insns = data_insn.sec_data->d_buf;
2116 prog.size = data_insn.sec_data->d_size;
2117 prog.license = ctx->license;
2118
2119 fd = bpf_prog_attach(section, &prog, ctx);
2120 if (fd < 0) {
2121 *lderr = true;
2122 if (props.total) {
2123 if (ctx->cfg.jit_enabled &&
2124 props.total != props.jited)
2125 fprintf(stderr, "JIT enabled, but only %u/%u tail call maps in the program have JITed owner!\n",
2126 props.jited, props.total);
2127 if (!ctx->cfg.jit_enabled &&
2128 props.jited)
2129 fprintf(stderr, "JIT disabled, but %u/%u tail call maps in the program have JITed owner!\n",
2130 props.jited, props.total);
2131 }
2132 return fd;
2133 }
2134
2135 ctx->sec_done[i] = true;
2136 ctx->sec_done[idx] = true;
2137 break;
2138 }
2139
2140 return fd;
2141 }
2142
2143 static int bpf_fetch_prog_sec(struct bpf_elf_ctx *ctx, const char *section)
2144 {
2145 bool lderr = false, sseen = false;
2146 int ret = -1;
2147
2148 if (bpf_has_map_data(ctx))
2149 ret = bpf_fetch_prog_relo(ctx, section, &lderr, &sseen);
2150 if (ret < 0 && !lderr)
2151 ret = bpf_fetch_prog(ctx, section, &sseen);
2152 if (ret < 0 && !sseen)
2153 fprintf(stderr, "Program section \'%s\' not found in ELF file!\n",
2154 section);
2155 return ret;
2156 }
2157
2158 static int bpf_find_map_by_id(struct bpf_elf_ctx *ctx, uint32_t id)
2159 {
2160 int i;
2161
2162 for (i = 0; i < ARRAY_SIZE(ctx->map_fds); i++)
2163 if (ctx->map_fds[i] && ctx->maps[i].id == id &&
2164 ctx->maps[i].type == BPF_MAP_TYPE_PROG_ARRAY)
2165 return i;
2166 return -1;
2167 }
2168
2169 struct bpf_jited_aux {
2170 int prog_fd;
2171 int map_fd;
2172 struct bpf_prog_data prog;
2173 struct bpf_map_ext map;
2174 };
2175
2176 static int bpf_derive_prog_from_fdinfo(int fd, struct bpf_prog_data *prog)
2177 {
2178 char file[PATH_MAX], buff[4096];
2179 unsigned int val;
2180 FILE *fp;
2181
2182 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
2183 memset(prog, 0, sizeof(*prog));
2184
2185 fp = fopen(file, "r");
2186 if (!fp) {
2187 fprintf(stderr, "No procfs support?!\n");
2188 return -EIO;
2189 }
2190
2191 while (fgets(buff, sizeof(buff), fp)) {
2192 if (sscanf(buff, "prog_type:\t%u", &val) == 1)
2193 prog->type = val;
2194 else if (sscanf(buff, "prog_jited:\t%u", &val) == 1)
2195 prog->jited = val;
2196 }
2197
2198 fclose(fp);
2199 return 0;
2200 }
2201
2202 static int bpf_tail_call_get_aux(struct bpf_jited_aux *aux)
2203 {
2204 struct bpf_elf_map tmp;
2205 int ret;
2206
2207 ret = bpf_derive_elf_map_from_fdinfo(aux->map_fd, &tmp, &aux->map);
2208 if (!ret)
2209 ret = bpf_derive_prog_from_fdinfo(aux->prog_fd, &aux->prog);
2210
2211 return ret;
2212 }
2213
2214 static int bpf_fill_prog_arrays(struct bpf_elf_ctx *ctx)
2215 {
2216 struct bpf_elf_sec_data data;
2217 uint32_t map_id, key_id;
2218 int fd, i, ret, idx;
2219
2220 for (i = 1; i < ctx->elf_hdr.e_shnum; i++) {
2221 if (ctx->sec_done[i])
2222 continue;
2223
2224 ret = bpf_fill_section_data(ctx, i, &data);
2225 if (ret < 0)
2226 continue;
2227
2228 ret = sscanf(data.sec_name, "%i/%i", &map_id, &key_id);
2229 if (ret != 2)
2230 continue;
2231
2232 idx = bpf_find_map_by_id(ctx, map_id);
2233 if (idx < 0)
2234 continue;
2235
2236 fd = bpf_fetch_prog_sec(ctx, data.sec_name);
2237 if (fd < 0)
2238 return -EIO;
2239
2240 ret = bpf_map_update(ctx->map_fds[idx], &key_id,
2241 &fd, BPF_ANY);
2242 if (ret < 0) {
2243 struct bpf_jited_aux aux = {};
2244
2245 ret = -errno;
2246 if (errno == E2BIG) {
2247 fprintf(stderr, "Tail call key %u for map %u out of bounds?\n",
2248 key_id, map_id);
2249 return ret;
2250 }
2251
2252 aux.map_fd = ctx->map_fds[idx];
2253 aux.prog_fd = fd;
2254
2255 if (bpf_tail_call_get_aux(&aux))
2256 return ret;
2257 if (!aux.map.owner.type)
2258 return ret;
2259
2260 if (aux.prog.type != aux.map.owner.type)
2261 fprintf(stderr, "Tail call map owned by prog type %u, but prog type is %u!\n",
2262 aux.map.owner.type, aux.prog.type);
2263 if (aux.prog.jited != aux.map.owner.jited)
2264 fprintf(stderr, "Tail call map %s jited, but prog %s!\n",
2265 aux.map.owner.jited ? "is" : "not",
2266 aux.prog.jited ? "is" : "not");
2267 return ret;
2268 }
2269
2270 ctx->sec_done[i] = true;
2271 }
2272
2273 return 0;
2274 }
2275
2276 static void bpf_save_finfo(struct bpf_elf_ctx *ctx)
2277 {
2278 struct stat st;
2279 int ret;
2280
2281 memset(&ctx->stat, 0, sizeof(ctx->stat));
2282
2283 ret = fstat(ctx->obj_fd, &st);
2284 if (ret < 0) {
2285 fprintf(stderr, "Stat of elf file failed: %s\n",
2286 strerror(errno));
2287 return;
2288 }
2289
2290 ctx->stat.st_dev = st.st_dev;
2291 ctx->stat.st_ino = st.st_ino;
2292 }
2293
2294 static int bpf_read_pin_mapping(FILE *fp, uint32_t *id, char *path)
2295 {
2296 char buff[PATH_MAX];
2297
2298 while (fgets(buff, sizeof(buff), fp)) {
2299 char *ptr = buff;
2300
2301 while (*ptr == ' ' || *ptr == '\t')
2302 ptr++;
2303
2304 if (*ptr == '#' || *ptr == '\n' || *ptr == 0)
2305 continue;
2306
2307 if (sscanf(ptr, "%i %s\n", id, path) != 2 &&
2308 sscanf(ptr, "%i %s #", id, path) != 2) {
2309 strcpy(path, ptr);
2310 return -1;
2311 }
2312
2313 return 1;
2314 }
2315
2316 return 0;
2317 }
2318
2319 static bool bpf_pinning_reserved(uint32_t pinning)
2320 {
2321 switch (pinning) {
2322 case PIN_NONE:
2323 case PIN_OBJECT_NS:
2324 case PIN_GLOBAL_NS:
2325 return true;
2326 default:
2327 return false;
2328 }
2329 }
2330
2331 static void bpf_hash_init(struct bpf_elf_ctx *ctx, const char *db_file)
2332 {
2333 struct bpf_hash_entry *entry;
2334 char subpath[PATH_MAX] = {};
2335 uint32_t pinning;
2336 FILE *fp;
2337 int ret;
2338
2339 fp = fopen(db_file, "r");
2340 if (!fp)
2341 return;
2342
2343 while ((ret = bpf_read_pin_mapping(fp, &pinning, subpath))) {
2344 if (ret == -1) {
2345 fprintf(stderr, "Database %s is corrupted at: %s\n",
2346 db_file, subpath);
2347 fclose(fp);
2348 return;
2349 }
2350
2351 if (bpf_pinning_reserved(pinning)) {
2352 fprintf(stderr, "Database %s, id %u is reserved - ignoring!\n",
2353 db_file, pinning);
2354 continue;
2355 }
2356
2357 entry = malloc(sizeof(*entry));
2358 if (!entry) {
2359 fprintf(stderr, "No memory left for db entry!\n");
2360 continue;
2361 }
2362
2363 entry->pinning = pinning;
2364 entry->subpath = strdup(subpath);
2365 if (!entry->subpath) {
2366 fprintf(stderr, "No memory left for db entry!\n");
2367 free(entry);
2368 continue;
2369 }
2370
2371 entry->next = ctx->ht[pinning & (ARRAY_SIZE(ctx->ht) - 1)];
2372 ctx->ht[pinning & (ARRAY_SIZE(ctx->ht) - 1)] = entry;
2373 }
2374
2375 fclose(fp);
2376 }
2377
2378 static void bpf_hash_destroy(struct bpf_elf_ctx *ctx)
2379 {
2380 struct bpf_hash_entry *entry;
2381 int i;
2382
2383 for (i = 0; i < ARRAY_SIZE(ctx->ht); i++) {
2384 while ((entry = ctx->ht[i]) != NULL) {
2385 ctx->ht[i] = entry->next;
2386 free((char *)entry->subpath);
2387 free(entry);
2388 }
2389 }
2390 }
2391
2392 static int bpf_elf_check_ehdr(const struct bpf_elf_ctx *ctx)
2393 {
2394 if (ctx->elf_hdr.e_type != ET_REL ||
2395 (ctx->elf_hdr.e_machine != EM_NONE &&
2396 ctx->elf_hdr.e_machine != EM_BPF) ||
2397 ctx->elf_hdr.e_version != EV_CURRENT) {
2398 fprintf(stderr, "ELF format error, ELF file not for eBPF?\n");
2399 return -EINVAL;
2400 }
2401
2402 switch (ctx->elf_hdr.e_ident[EI_DATA]) {
2403 default:
2404 fprintf(stderr, "ELF format error, wrong endianness info?\n");
2405 return -EINVAL;
2406 case ELFDATA2LSB:
2407 if (htons(1) == 1) {
2408 fprintf(stderr,
2409 "We are big endian, eBPF object is little endian!\n");
2410 return -EIO;
2411 }
2412 break;
2413 case ELFDATA2MSB:
2414 if (htons(1) != 1) {
2415 fprintf(stderr,
2416 "We are little endian, eBPF object is big endian!\n");
2417 return -EIO;
2418 }
2419 break;
2420 }
2421
2422 return 0;
2423 }
2424
2425 static void bpf_get_cfg(struct bpf_elf_ctx *ctx)
2426 {
2427 static const char *path_jit = "/proc/sys/net/core/bpf_jit_enable";
2428 int fd;
2429
2430 fd = open(path_jit, O_RDONLY);
2431 if (fd > 0) {
2432 char tmp[16] = {};
2433
2434 if (read(fd, tmp, sizeof(tmp)) > 0)
2435 ctx->cfg.jit_enabled = atoi(tmp);
2436 close(fd);
2437 }
2438 }
2439
2440 static int bpf_elf_ctx_init(struct bpf_elf_ctx *ctx, const char *pathname,
2441 enum bpf_prog_type type, __u32 ifindex,
2442 bool verbose)
2443 {
2444 int ret = -EINVAL;
2445
2446 if (elf_version(EV_CURRENT) == EV_NONE ||
2447 bpf_init_env(pathname))
2448 return ret;
2449
2450 memset(ctx, 0, sizeof(*ctx));
2451 bpf_get_cfg(ctx);
2452 ctx->verbose = verbose;
2453 ctx->type = type;
2454 ctx->ifindex = ifindex;
2455
2456 ctx->obj_fd = open(pathname, O_RDONLY);
2457 if (ctx->obj_fd < 0)
2458 return ctx->obj_fd;
2459
2460 ctx->elf_fd = elf_begin(ctx->obj_fd, ELF_C_READ, NULL);
2461 if (!ctx->elf_fd) {
2462 ret = -EINVAL;
2463 goto out_fd;
2464 }
2465
2466 if (elf_kind(ctx->elf_fd) != ELF_K_ELF) {
2467 ret = -EINVAL;
2468 goto out_fd;
2469 }
2470
2471 if (gelf_getehdr(ctx->elf_fd, &ctx->elf_hdr) !=
2472 &ctx->elf_hdr) {
2473 ret = -EIO;
2474 goto out_elf;
2475 }
2476
2477 ret = bpf_elf_check_ehdr(ctx);
2478 if (ret < 0)
2479 goto out_elf;
2480
2481 ctx->sec_done = calloc(ctx->elf_hdr.e_shnum,
2482 sizeof(*(ctx->sec_done)));
2483 if (!ctx->sec_done) {
2484 ret = -ENOMEM;
2485 goto out_elf;
2486 }
2487
2488 if (ctx->verbose && bpf_log_realloc(ctx)) {
2489 ret = -ENOMEM;
2490 goto out_free;
2491 }
2492
2493 bpf_save_finfo(ctx);
2494 bpf_hash_init(ctx, CONFDIR "/bpf_pinning");
2495
2496 return 0;
2497 out_free:
2498 free(ctx->sec_done);
2499 out_elf:
2500 elf_end(ctx->elf_fd);
2501 out_fd:
2502 close(ctx->obj_fd);
2503 return ret;
2504 }
2505
2506 static int bpf_maps_count(struct bpf_elf_ctx *ctx)
2507 {
2508 int i, count = 0;
2509
2510 for (i = 0; i < ARRAY_SIZE(ctx->map_fds); i++) {
2511 if (!ctx->map_fds[i])
2512 break;
2513 count++;
2514 }
2515
2516 return count;
2517 }
2518
2519 static void bpf_maps_teardown(struct bpf_elf_ctx *ctx)
2520 {
2521 int i;
2522
2523 for (i = 0; i < ARRAY_SIZE(ctx->map_fds); i++) {
2524 if (ctx->map_fds[i])
2525 close(ctx->map_fds[i]);
2526 }
2527 }
2528
2529 static void bpf_elf_ctx_destroy(struct bpf_elf_ctx *ctx, bool failure)
2530 {
2531 if (failure)
2532 bpf_maps_teardown(ctx);
2533
2534 bpf_hash_destroy(ctx);
2535
2536 free(ctx->sec_done);
2537 free(ctx->log);
2538
2539 elf_end(ctx->elf_fd);
2540 close(ctx->obj_fd);
2541 }
2542
2543 static struct bpf_elf_ctx __ctx;
2544
2545 static int bpf_obj_open(const char *pathname, enum bpf_prog_type type,
2546 const char *section, __u32 ifindex, bool verbose)
2547 {
2548 struct bpf_elf_ctx *ctx = &__ctx;
2549 int fd = 0, ret;
2550
2551 ret = bpf_elf_ctx_init(ctx, pathname, type, ifindex, verbose);
2552 if (ret < 0) {
2553 fprintf(stderr, "Cannot initialize ELF context!\n");
2554 return ret;
2555 }
2556
2557 ret = bpf_fetch_ancillary(ctx);
2558 if (ret < 0) {
2559 fprintf(stderr, "Error fetching ELF ancillary data!\n");
2560 goto out;
2561 }
2562
2563 fd = bpf_fetch_prog_sec(ctx, section);
2564 if (fd < 0) {
2565 fprintf(stderr, "Error fetching program/map!\n");
2566 ret = fd;
2567 goto out;
2568 }
2569
2570 ret = bpf_fill_prog_arrays(ctx);
2571 if (ret < 0)
2572 fprintf(stderr, "Error filling program arrays!\n");
2573 out:
2574 bpf_elf_ctx_destroy(ctx, ret < 0);
2575 if (ret < 0) {
2576 if (fd)
2577 close(fd);
2578 return ret;
2579 }
2580
2581 return fd;
2582 }
2583
2584 static int
2585 bpf_map_set_send(int fd, struct sockaddr_un *addr, unsigned int addr_len,
2586 const struct bpf_map_data *aux, unsigned int entries)
2587 {
2588 struct bpf_map_set_msg msg = {
2589 .aux.uds_ver = BPF_SCM_AUX_VER,
2590 .aux.num_ent = entries,
2591 };
2592 int *cmsg_buf, min_fd;
2593 char *amsg_buf;
2594 int i;
2595
2596 strlcpy(msg.aux.obj_name, aux->obj, sizeof(msg.aux.obj_name));
2597 memcpy(&msg.aux.obj_st, aux->st, sizeof(msg.aux.obj_st));
2598
2599 cmsg_buf = bpf_map_set_init(&msg, addr, addr_len);
2600 amsg_buf = (char *)msg.aux.ent;
2601
2602 for (i = 0; i < entries; i += min_fd) {
2603 int ret;
2604
2605 min_fd = min(BPF_SCM_MAX_FDS * 1U, entries - i);
2606 bpf_map_set_init_single(&msg, min_fd);
2607
2608 memcpy(cmsg_buf, &aux->fds[i], sizeof(aux->fds[0]) * min_fd);
2609 memcpy(amsg_buf, &aux->ent[i], sizeof(aux->ent[0]) * min_fd);
2610
2611 ret = sendmsg(fd, &msg.hdr, 0);
2612 if (ret <= 0)
2613 return ret ? : -1;
2614 }
2615
2616 return 0;
2617 }
2618
2619 static int
2620 bpf_map_set_recv(int fd, int *fds, struct bpf_map_aux *aux,
2621 unsigned int entries)
2622 {
2623 struct bpf_map_set_msg msg;
2624 int *cmsg_buf, min_fd;
2625 char *amsg_buf, *mmsg_buf;
2626 unsigned int needed = 1;
2627 int i;
2628
2629 cmsg_buf = bpf_map_set_init(&msg, NULL, 0);
2630 amsg_buf = (char *)msg.aux.ent;
2631 mmsg_buf = (char *)&msg.aux;
2632
2633 for (i = 0; i < min(entries, needed); i += min_fd) {
2634 struct cmsghdr *cmsg;
2635 int ret;
2636
2637 min_fd = min(entries, entries - i);
2638 bpf_map_set_init_single(&msg, min_fd);
2639
2640 ret = recvmsg(fd, &msg.hdr, 0);
2641 if (ret <= 0)
2642 return ret ? : -1;
2643
2644 cmsg = CMSG_FIRSTHDR(&msg.hdr);
2645 if (!cmsg || cmsg->cmsg_type != SCM_RIGHTS)
2646 return -EINVAL;
2647 if (msg.hdr.msg_flags & MSG_CTRUNC)
2648 return -EIO;
2649 if (msg.aux.uds_ver != BPF_SCM_AUX_VER)
2650 return -ENOSYS;
2651
2652 min_fd = (cmsg->cmsg_len - sizeof(*cmsg)) / sizeof(fd);
2653 if (min_fd > entries || min_fd <= 0)
2654 return -EINVAL;
2655
2656 memcpy(&fds[i], cmsg_buf, sizeof(fds[0]) * min_fd);
2657 memcpy(&aux->ent[i], amsg_buf, sizeof(aux->ent[0]) * min_fd);
2658 memcpy(aux, mmsg_buf, offsetof(struct bpf_map_aux, ent));
2659
2660 needed = aux->num_ent;
2661 }
2662
2663 return 0;
2664 }
2665
2666 int bpf_send_map_fds(const char *path, const char *obj)
2667 {
2668 struct bpf_elf_ctx *ctx = &__ctx;
2669 struct sockaddr_un addr = { .sun_family = AF_UNIX };
2670 struct bpf_map_data bpf_aux = {
2671 .fds = ctx->map_fds,
2672 .ent = ctx->maps,
2673 .st = &ctx->stat,
2674 .obj = obj,
2675 };
2676 int fd, ret;
2677
2678 fd = socket(AF_UNIX, SOCK_DGRAM, 0);
2679 if (fd < 0) {
2680 fprintf(stderr, "Cannot open socket: %s\n",
2681 strerror(errno));
2682 return -1;
2683 }
2684
2685 strlcpy(addr.sun_path, path, sizeof(addr.sun_path));
2686
2687 ret = connect(fd, (struct sockaddr *)&addr, sizeof(addr));
2688 if (ret < 0) {
2689 fprintf(stderr, "Cannot connect to %s: %s\n",
2690 path, strerror(errno));
2691 return -1;
2692 }
2693
2694 ret = bpf_map_set_send(fd, &addr, sizeof(addr), &bpf_aux,
2695 bpf_maps_count(ctx));
2696 if (ret < 0)
2697 fprintf(stderr, "Cannot send fds to %s: %s\n",
2698 path, strerror(errno));
2699
2700 bpf_maps_teardown(ctx);
2701 close(fd);
2702 return ret;
2703 }
2704
2705 int bpf_recv_map_fds(const char *path, int *fds, struct bpf_map_aux *aux,
2706 unsigned int entries)
2707 {
2708 struct sockaddr_un addr = { .sun_family = AF_UNIX };
2709 int fd, ret;
2710
2711 fd = socket(AF_UNIX, SOCK_DGRAM, 0);
2712 if (fd < 0) {
2713 fprintf(stderr, "Cannot open socket: %s\n",
2714 strerror(errno));
2715 return -1;
2716 }
2717
2718 strlcpy(addr.sun_path, path, sizeof(addr.sun_path));
2719
2720 ret = bind(fd, (struct sockaddr *)&addr, sizeof(addr));
2721 if (ret < 0) {
2722 fprintf(stderr, "Cannot bind to socket: %s\n",
2723 strerror(errno));
2724 return -1;
2725 }
2726
2727 ret = bpf_map_set_recv(fd, fds, aux, entries);
2728 if (ret < 0)
2729 fprintf(stderr, "Cannot recv fds from %s: %s\n",
2730 path, strerror(errno));
2731
2732 unlink(addr.sun_path);
2733 close(fd);
2734 return ret;
2735 }
2736 #endif /* HAVE_ELF */
Upstream mirror of iproute2