]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - tools/perf/util/bpf-loader.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:


dbbd17ca6d6f2671edae88069063419bf0e9259b
[mirror_ubuntu-artful-kernel.git] / tools / perf / util / bpf-loader.c
1 /*
2 * bpf-loader.c
3 *
4 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
5 * Copyright (C) 2015 Huawei Inc.
6 */
7
8 #include <linux/bpf.h>
9 #include <bpf/libbpf.h>
10 #include <bpf/bpf.h>
11 #include <linux/err.h>
12 #include <linux/string.h>
13 #include "perf.h"
14 #include "debug.h"
15 #include "bpf-loader.h"
16 #include "bpf-prologue.h"
17 #include "llvm-utils.h"
18 #include "probe-event.h"
19 #include "probe-finder.h" // for MAX_PROBES
20 #include "llvm-utils.h"
21
22 #define DEFINE_PRINT_FN(name, level) \
23 static int libbpf_##name(const char *fmt, ...) \
24 { \
25 va_list args; \
26 int ret; \
27 \
28 va_start(args, fmt); \
29 ret = veprintf(level, verbose, pr_fmt(fmt), args);\
30 va_end(args); \
31 return ret; \
32 }
33
34 DEFINE_PRINT_FN(warning, 1)
35 DEFINE_PRINT_FN(info, 1)
36 DEFINE_PRINT_FN(debug, 1)
37
38 struct bpf_prog_priv {
39 struct perf_probe_event pev;
40 bool need_prologue;
41 struct bpf_insn *insns_buf;
42 int nr_types;
43 int *type_mapping;
44 };
45
46 static bool libbpf_initialized;
47
48 struct bpf_object *
49 bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
50 {
51 struct bpf_object *obj;
52
53 if (!libbpf_initialized) {
54 libbpf_set_print(libbpf_warning,
55 libbpf_info,
56 libbpf_debug);
57 libbpf_initialized = true;
58 }
59
60 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name);
61 if (IS_ERR(obj)) {
62 pr_debug("bpf: failed to load buffer\n");
63 return ERR_PTR(-EINVAL);
64 }
65
66 return obj;
67 }
68
69 struct bpf_object *bpf__prepare_load(const char *filename, bool source)
70 {
71 struct bpf_object *obj;
72
73 if (!libbpf_initialized) {
74 libbpf_set_print(libbpf_warning,
75 libbpf_info,
76 libbpf_debug);
77 libbpf_initialized = true;
78 }
79
80 if (source) {
81 int err;
82 void *obj_buf;
83 size_t obj_buf_sz;
84
85 err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
86 if (err)
87 return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
88 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename);
89 free(obj_buf);
90 } else
91 obj = bpf_object__open(filename);
92
93 if (IS_ERR(obj)) {
94 pr_debug("bpf: failed to load %s\n", filename);
95 return obj;
96 }
97
98 return obj;
99 }
100
101 void bpf__clear(void)
102 {
103 struct bpf_object *obj, *tmp;
104
105 bpf_object__for_each_safe(obj, tmp) {
106 bpf__unprobe(obj);
107 bpf_object__close(obj);
108 }
109 }
110
111 static void
112 clear_prog_priv(struct bpf_program *prog __maybe_unused,
113 void *_priv)
114 {
115 struct bpf_prog_priv *priv = _priv;
116
117 cleanup_perf_probe_events(&priv->pev, 1);
118 zfree(&priv->insns_buf);
119 zfree(&priv->type_mapping);
120 free(priv);
121 }
122
123 static int
124 prog_config__exec(const char *value, struct perf_probe_event *pev)
125 {
126 pev->uprobes = true;
127 pev->target = strdup(value);
128 if (!pev->target)
129 return -ENOMEM;
130 return 0;
131 }
132
133 static int
134 prog_config__module(const char *value, struct perf_probe_event *pev)
135 {
136 pev->uprobes = false;
137 pev->target = strdup(value);
138 if (!pev->target)
139 return -ENOMEM;
140 return 0;
141 }
142
143 static int
144 prog_config__bool(const char *value, bool *pbool, bool invert)
145 {
146 int err;
147 bool bool_value;
148
149 if (!pbool)
150 return -EINVAL;
151
152 err = strtobool(value, &bool_value);
153 if (err)
154 return err;
155
156 *pbool = invert ? !bool_value : bool_value;
157 return 0;
158 }
159
160 static int
161 prog_config__inlines(const char *value,
162 struct perf_probe_event *pev __maybe_unused)
163 {
164 return prog_config__bool(value, &probe_conf.no_inlines, true);
165 }
166
167 static int
168 prog_config__force(const char *value,
169 struct perf_probe_event *pev __maybe_unused)
170 {
171 return prog_config__bool(value, &probe_conf.force_add, false);
172 }
173
174 static struct {
175 const char *key;
176 const char *usage;
177 const char *desc;
178 int (*func)(const char *, struct perf_probe_event *);
179 } bpf_prog_config_terms[] = {
180 {
181 .key = "exec",
182 .usage = "exec=<full path of file>",
183 .desc = "Set uprobe target",
184 .func = prog_config__exec,
185 },
186 {
187 .key = "module",
188 .usage = "module=<module name> ",
189 .desc = "Set kprobe module",
190 .func = prog_config__module,
191 },
192 {
193 .key = "inlines",
194 .usage = "inlines=[yes|no] ",
195 .desc = "Probe at inline symbol",
196 .func = prog_config__inlines,
197 },
198 {
199 .key = "force",
200 .usage = "force=[yes|no] ",
201 .desc = "Forcibly add events with existing name",
202 .func = prog_config__force,
203 },
204 };
205
206 static int
207 do_prog_config(const char *key, const char *value,
208 struct perf_probe_event *pev)
209 {
210 unsigned int i;
211
212 pr_debug("config bpf program: %s=%s\n", key, value);
213 for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
214 if (strcmp(key, bpf_prog_config_terms[i].key) == 0)
215 return bpf_prog_config_terms[i].func(value, pev);
216
217 pr_debug("BPF: ERROR: invalid program config option: %s=%s\n",
218 key, value);
219
220 pr_debug("\nHint: Valid options are:\n");
221 for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
222 pr_debug("\t%s:\t%s\n", bpf_prog_config_terms[i].usage,
223 bpf_prog_config_terms[i].desc);
224 pr_debug("\n");
225
226 return -BPF_LOADER_ERRNO__PROGCONF_TERM;
227 }
228
229 static const char *
230 parse_prog_config_kvpair(const char *config_str, struct perf_probe_event *pev)
231 {
232 char *text = strdup(config_str);
233 char *sep, *line;
234 const char *main_str = NULL;
235 int err = 0;
236
237 if (!text) {
238 pr_debug("No enough memory: dup config_str failed\n");
239 return ERR_PTR(-ENOMEM);
240 }
241
242 line = text;
243 while ((sep = strchr(line, ';'))) {
244 char *equ;
245
246 *sep = '\0';
247 equ = strchr(line, '=');
248 if (!equ) {
249 pr_warning("WARNING: invalid config in BPF object: %s\n",
250 line);
251 pr_warning("\tShould be 'key=value'.\n");
252 goto nextline;
253 }
254 *equ = '\0';
255
256 err = do_prog_config(line, equ + 1, pev);
257 if (err)
258 break;
259 nextline:
260 line = sep + 1;
261 }
262
263 if (!err)
264 main_str = config_str + (line - text);
265 free(text);
266
267 return err ? ERR_PTR(err) : main_str;
268 }
269
270 static int
271 parse_prog_config(const char *config_str, struct perf_probe_event *pev)
272 {
273 int err;
274 const char *main_str = parse_prog_config_kvpair(config_str, pev);
275
276 if (IS_ERR(main_str))
277 return PTR_ERR(main_str);
278
279 err = parse_perf_probe_command(main_str, pev);
280 if (err < 0) {
281 pr_debug("bpf: '%s' is not a valid config string\n",
282 config_str);
283 /* parse failed, don't need clear pev. */
284 return -BPF_LOADER_ERRNO__CONFIG;
285 }
286 return 0;
287 }
288
289 static int
290 config_bpf_program(struct bpf_program *prog)
291 {
292 struct perf_probe_event *pev = NULL;
293 struct bpf_prog_priv *priv = NULL;
294 const char *config_str;
295 int err;
296
297 /* Initialize per-program probing setting */
298 probe_conf.no_inlines = false;
299 probe_conf.force_add = false;
300
301 config_str = bpf_program__title(prog, false);
302 if (IS_ERR(config_str)) {
303 pr_debug("bpf: unable to get title for program\n");
304 return PTR_ERR(config_str);
305 }
306
307 priv = calloc(sizeof(*priv), 1);
308 if (!priv) {
309 pr_debug("bpf: failed to alloc priv\n");
310 return -ENOMEM;
311 }
312 pev = &priv->pev;
313
314 pr_debug("bpf: config program '%s'\n", config_str);
315 err = parse_prog_config(config_str, pev);
316 if (err)
317 goto errout;
318
319 if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
320 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
321 config_str, PERF_BPF_PROBE_GROUP);
322 err = -BPF_LOADER_ERRNO__GROUP;
323 goto errout;
324 } else if (!pev->group)
325 pev->group = strdup(PERF_BPF_PROBE_GROUP);
326
327 if (!pev->group) {
328 pr_debug("bpf: strdup failed\n");
329 err = -ENOMEM;
330 goto errout;
331 }
332
333 if (!pev->event) {
334 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
335 config_str);
336 err = -BPF_LOADER_ERRNO__EVENTNAME;
337 goto errout;
338 }
339 pr_debug("bpf: config '%s' is ok\n", config_str);
340
341 err = bpf_program__set_private(prog, priv, clear_prog_priv);
342 if (err) {
343 pr_debug("Failed to set priv for program '%s'\n", config_str);
344 goto errout;
345 }
346
347 return 0;
348
349 errout:
350 if (pev)
351 clear_perf_probe_event(pev);
352 free(priv);
353 return err;
354 }
355
356 static int bpf__prepare_probe(void)
357 {
358 static int err = 0;
359 static bool initialized = false;
360
361 /*
362 * Make err static, so if init failed the first, bpf__prepare_probe()
363 * fails each time without calling init_probe_symbol_maps multiple
364 * times.
365 */
366 if (initialized)
367 return err;
368
369 initialized = true;
370 err = init_probe_symbol_maps(false);
371 if (err < 0)
372 pr_debug("Failed to init_probe_symbol_maps\n");
373 probe_conf.max_probes = MAX_PROBES;
374 return err;
375 }
376
377 static int
378 preproc_gen_prologue(struct bpf_program *prog, int n,
379 struct bpf_insn *orig_insns, int orig_insns_cnt,
380 struct bpf_prog_prep_result *res)
381 {
382 struct probe_trace_event *tev;
383 struct perf_probe_event *pev;
384 struct bpf_prog_priv *priv;
385 struct bpf_insn *buf;
386 size_t prologue_cnt = 0;
387 int i, err;
388
389 err = bpf_program__get_private(prog, (void **)&priv);
390 if (err || !priv)
391 goto errout;
392
393 pev = &priv->pev;
394
395 if (n < 0 || n >= priv->nr_types)
396 goto errout;
397
398 /* Find a tev belongs to that type */
399 for (i = 0; i < pev->ntevs; i++) {
400 if (priv->type_mapping[i] == n)
401 break;
402 }
403
404 if (i >= pev->ntevs) {
405 pr_debug("Internal error: prologue type %d not found\n", n);
406 return -BPF_LOADER_ERRNO__PROLOGUE;
407 }
408
409 tev = &pev->tevs[i];
410
411 buf = priv->insns_buf;
412 err = bpf__gen_prologue(tev->args, tev->nargs,
413 buf, &prologue_cnt,
414 BPF_MAXINSNS - orig_insns_cnt);
415 if (err) {
416 const char *title;
417
418 title = bpf_program__title(prog, false);
419 if (!title)
420 title = "[unknown]";
421
422 pr_debug("Failed to generate prologue for program %s\n",
423 title);
424 return err;
425 }
426
427 memcpy(&buf[prologue_cnt], orig_insns,
428 sizeof(struct bpf_insn) * orig_insns_cnt);
429
430 res->new_insn_ptr = buf;
431 res->new_insn_cnt = prologue_cnt + orig_insns_cnt;
432 res->pfd = NULL;
433 return 0;
434
435 errout:
436 pr_debug("Internal error in preproc_gen_prologue\n");
437 return -BPF_LOADER_ERRNO__PROLOGUE;
438 }
439
440 /*
441 * compare_tev_args is reflexive, transitive and antisymmetric.
442 * I can proof it but this margin is too narrow to contain.
443 */
444 static int compare_tev_args(const void *ptev1, const void *ptev2)
445 {
446 int i, ret;
447 const struct probe_trace_event *tev1 =
448 *(const struct probe_trace_event **)ptev1;
449 const struct probe_trace_event *tev2 =
450 *(const struct probe_trace_event **)ptev2;
451
452 ret = tev2->nargs - tev1->nargs;
453 if (ret)
454 return ret;
455
456 for (i = 0; i < tev1->nargs; i++) {
457 struct probe_trace_arg *arg1, *arg2;
458 struct probe_trace_arg_ref *ref1, *ref2;
459
460 arg1 = &tev1->args[i];
461 arg2 = &tev2->args[i];
462
463 ret = strcmp(arg1->value, arg2->value);
464 if (ret)
465 return ret;
466
467 ref1 = arg1->ref;
468 ref2 = arg2->ref;
469
470 while (ref1 && ref2) {
471 ret = ref2->offset - ref1->offset;
472 if (ret)
473 return ret;
474
475 ref1 = ref1->next;
476 ref2 = ref2->next;
477 }
478
479 if (ref1 || ref2)
480 return ref2 ? 1 : -1;
481 }
482
483 return 0;
484 }
485
486 /*
487 * Assign a type number to each tevs in a pev.
488 * mapping is an array with same slots as tevs in that pev.
489 * nr_types will be set to number of types.
490 */
491 static int map_prologue(struct perf_probe_event *pev, int *mapping,
492 int *nr_types)
493 {
494 int i, type = 0;
495 struct probe_trace_event **ptevs;
496
497 size_t array_sz = sizeof(*ptevs) * pev->ntevs;
498
499 ptevs = malloc(array_sz);
500 if (!ptevs) {
501 pr_debug("No ehough memory: alloc ptevs failed\n");
502 return -ENOMEM;
503 }
504
505 pr_debug("In map_prologue, ntevs=%d\n", pev->ntevs);
506 for (i = 0; i < pev->ntevs; i++)
507 ptevs[i] = &pev->tevs[i];
508
509 qsort(ptevs, pev->ntevs, sizeof(*ptevs),
510 compare_tev_args);
511
512 for (i = 0; i < pev->ntevs; i++) {
513 int n;
514
515 n = ptevs[i] - pev->tevs;
516 if (i == 0) {
517 mapping[n] = type;
518 pr_debug("mapping[%d]=%d\n", n, type);
519 continue;
520 }
521
522 if (compare_tev_args(ptevs + i, ptevs + i - 1) == 0)
523 mapping[n] = type;
524 else
525 mapping[n] = ++type;
526
527 pr_debug("mapping[%d]=%d\n", n, mapping[n]);
528 }
529 free(ptevs);
530 *nr_types = type + 1;
531
532 return 0;
533 }
534
535 static int hook_load_preprocessor(struct bpf_program *prog)
536 {
537 struct perf_probe_event *pev;
538 struct bpf_prog_priv *priv;
539 bool need_prologue = false;
540 int err, i;
541
542 err = bpf_program__get_private(prog, (void **)&priv);
543 if (err || !priv) {
544 pr_debug("Internal error when hook preprocessor\n");
545 return -BPF_LOADER_ERRNO__INTERNAL;
546 }
547
548 pev = &priv->pev;
549 for (i = 0; i < pev->ntevs; i++) {
550 struct probe_trace_event *tev = &pev->tevs[i];
551
552 if (tev->nargs > 0) {
553 need_prologue = true;
554 break;
555 }
556 }
557
558 /*
559 * Since all tevs don't have argument, we don't need generate
560 * prologue.
561 */
562 if (!need_prologue) {
563 priv->need_prologue = false;
564 return 0;
565 }
566
567 priv->need_prologue = true;
568 priv->insns_buf = malloc(sizeof(struct bpf_insn) * BPF_MAXINSNS);
569 if (!priv->insns_buf) {
570 pr_debug("No enough memory: alloc insns_buf failed\n");
571 return -ENOMEM;
572 }
573
574 priv->type_mapping = malloc(sizeof(int) * pev->ntevs);
575 if (!priv->type_mapping) {
576 pr_debug("No enough memory: alloc type_mapping failed\n");
577 return -ENOMEM;
578 }
579 memset(priv->type_mapping, -1,
580 sizeof(int) * pev->ntevs);
581
582 err = map_prologue(pev, priv->type_mapping, &priv->nr_types);
583 if (err)
584 return err;
585
586 err = bpf_program__set_prep(prog, priv->nr_types,
587 preproc_gen_prologue);
588 return err;
589 }
590
591 int bpf__probe(struct bpf_object *obj)
592 {
593 int err = 0;
594 struct bpf_program *prog;
595 struct bpf_prog_priv *priv;
596 struct perf_probe_event *pev;
597
598 err = bpf__prepare_probe();
599 if (err) {
600 pr_debug("bpf__prepare_probe failed\n");
601 return err;
602 }
603
604 bpf_object__for_each_program(prog, obj) {
605 err = config_bpf_program(prog);
606 if (err)
607 goto out;
608
609 err = bpf_program__get_private(prog, (void **)&priv);
610 if (err || !priv)
611 goto out;
612 pev = &priv->pev;
613
614 err = convert_perf_probe_events(pev, 1);
615 if (err < 0) {
616 pr_debug("bpf_probe: failed to convert perf probe events");
617 goto out;
618 }
619
620 err = apply_perf_probe_events(pev, 1);
621 if (err < 0) {
622 pr_debug("bpf_probe: failed to apply perf probe events");
623 goto out;
624 }
625
626 /*
627 * After probing, let's consider prologue, which
628 * adds program fetcher to BPF programs.
629 *
630 * hook_load_preprocessorr() hooks pre-processor
631 * to bpf_program, let it generate prologue
632 * dynamically during loading.
633 */
634 err = hook_load_preprocessor(prog);
635 if (err)
636 goto out;
637 }
638 out:
639 return err < 0 ? err : 0;
640 }
641
642 #define EVENTS_WRITE_BUFSIZE 4096
643 int bpf__unprobe(struct bpf_object *obj)
644 {
645 int err, ret = 0;
646 struct bpf_program *prog;
647 struct bpf_prog_priv *priv;
648
649 bpf_object__for_each_program(prog, obj) {
650 int i;
651
652 err = bpf_program__get_private(prog, (void **)&priv);
653 if (err || !priv)
654 continue;
655
656 for (i = 0; i < priv->pev.ntevs; i++) {
657 struct probe_trace_event *tev = &priv->pev.tevs[i];
658 char name_buf[EVENTS_WRITE_BUFSIZE];
659 struct strfilter *delfilter;
660
661 snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
662 "%s:%s", tev->group, tev->event);
663 name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
664
665 delfilter = strfilter__new(name_buf, NULL);
666 if (!delfilter) {
667 pr_debug("Failed to create filter for unprobing\n");
668 ret = -ENOMEM;
669 continue;
670 }
671
672 err = del_perf_probe_events(delfilter);
673 strfilter__delete(delfilter);
674 if (err) {
675 pr_debug("Failed to delete %s\n", name_buf);
676 ret = err;
677 continue;
678 }
679 }
680 }
681 return ret;
682 }
683
684 int bpf__load(struct bpf_object *obj)
685 {
686 int err;
687
688 err = bpf_object__load(obj);
689 if (err) {
690 pr_debug("bpf: load objects failed\n");
691 return err;
692 }
693 return 0;
694 }
695
696 int bpf__foreach_tev(struct bpf_object *obj,
697 bpf_prog_iter_callback_t func,
698 void *arg)
699 {
700 struct bpf_program *prog;
701 int err;
702
703 bpf_object__for_each_program(prog, obj) {
704 struct probe_trace_event *tev;
705 struct perf_probe_event *pev;
706 struct bpf_prog_priv *priv;
707 int i, fd;
708
709 err = bpf_program__get_private(prog,
710 (void **)&priv);
711 if (err || !priv) {
712 pr_debug("bpf: failed to get private field\n");
713 return -BPF_LOADER_ERRNO__INTERNAL;
714 }
715
716 pev = &priv->pev;
717 for (i = 0; i < pev->ntevs; i++) {
718 tev = &pev->tevs[i];
719
720 if (priv->need_prologue) {
721 int type = priv->type_mapping[i];
722
723 fd = bpf_program__nth_fd(prog, type);
724 } else {
725 fd = bpf_program__fd(prog);
726 }
727
728 if (fd < 0) {
729 pr_debug("bpf: failed to get file descriptor\n");
730 return fd;
731 }
732
733 err = (*func)(tev, fd, arg);
734 if (err) {
735 pr_debug("bpf: call back failed, stop iterate\n");
736 return err;
737 }
738 }
739 }
740 return 0;
741 }
742
743 enum bpf_map_op_type {
744 BPF_MAP_OP_SET_VALUE,
745 };
746
747 enum bpf_map_key_type {
748 BPF_MAP_KEY_ALL,
749 };
750
751 struct bpf_map_op {
752 struct list_head list;
753 enum bpf_map_op_type op_type;
754 enum bpf_map_key_type key_type;
755 union {
756 u64 value;
757 } v;
758 };
759
760 struct bpf_map_priv {
761 struct list_head ops_list;
762 };
763
764 static void
765 bpf_map_op__delete(struct bpf_map_op *op)
766 {
767 if (!list_empty(&op->list))
768 list_del(&op->list);
769 free(op);
770 }
771
772 static void
773 bpf_map_priv__purge(struct bpf_map_priv *priv)
774 {
775 struct bpf_map_op *pos, *n;
776
777 list_for_each_entry_safe(pos, n, &priv->ops_list, list) {
778 list_del_init(&pos->list);
779 bpf_map_op__delete(pos);
780 }
781 }
782
783 static void
784 bpf_map_priv__clear(struct bpf_map *map __maybe_unused,
785 void *_priv)
786 {
787 struct bpf_map_priv *priv = _priv;
788
789 bpf_map_priv__purge(priv);
790 free(priv);
791 }
792
793 static struct bpf_map_op *
794 bpf_map_op__new(void)
795 {
796 struct bpf_map_op *op;
797
798 op = zalloc(sizeof(*op));
799 if (!op) {
800 pr_debug("Failed to alloc bpf_map_op\n");
801 return ERR_PTR(-ENOMEM);
802 }
803 INIT_LIST_HEAD(&op->list);
804
805 op->key_type = BPF_MAP_KEY_ALL;
806 return op;
807 }
808
809 static int
810 bpf_map__add_op(struct bpf_map *map, struct bpf_map_op *op)
811 {
812 struct bpf_map_priv *priv;
813 const char *map_name;
814 int err;
815
816 map_name = bpf_map__get_name(map);
817 err = bpf_map__get_private(map, (void **)&priv);
818 if (err) {
819 pr_debug("Failed to get private from map %s\n", map_name);
820 return err;
821 }
822
823 if (!priv) {
824 priv = zalloc(sizeof(*priv));
825 if (!priv) {
826 pr_debug("No enough memory to alloc map private\n");
827 return -ENOMEM;
828 }
829 INIT_LIST_HEAD(&priv->ops_list);
830
831 if (bpf_map__set_private(map, priv, bpf_map_priv__clear)) {
832 free(priv);
833 return -BPF_LOADER_ERRNO__INTERNAL;
834 }
835 }
836
837 list_add_tail(&op->list, &priv->ops_list);
838 return 0;
839 }
840
841 static int
842 __bpf_map__config_value(struct bpf_map *map,
843 struct parse_events_term *term)
844 {
845 struct bpf_map_def def;
846 struct bpf_map_op *op;
847 const char *map_name;
848 int err;
849
850 map_name = bpf_map__get_name(map);
851
852 err = bpf_map__get_def(map, &def);
853 if (err) {
854 pr_debug("Unable to get map definition from '%s'\n",
855 map_name);
856 return -BPF_LOADER_ERRNO__INTERNAL;
857 }
858
859 if (def.type != BPF_MAP_TYPE_ARRAY) {
860 pr_debug("Map %s type is not BPF_MAP_TYPE_ARRAY\n",
861 map_name);
862 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
863 }
864 if (def.key_size < sizeof(unsigned int)) {
865 pr_debug("Map %s has incorrect key size\n", map_name);
866 return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE;
867 }
868 switch (def.value_size) {
869 case 1:
870 case 2:
871 case 4:
872 case 8:
873 break;
874 default:
875 pr_debug("Map %s has incorrect value size\n", map_name);
876 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
877 }
878
879 op = bpf_map_op__new();
880 if (IS_ERR(op))
881 return PTR_ERR(op);
882 op->op_type = BPF_MAP_OP_SET_VALUE;
883 op->v.value = term->val.num;
884
885 err = bpf_map__add_op(map, op);
886 if (err)
887 bpf_map_op__delete(op);
888 return err;
889 }
890
891 static int
892 bpf_map__config_value(struct bpf_map *map,
893 struct parse_events_term *term,
894 struct perf_evlist *evlist __maybe_unused)
895 {
896 if (!term->err_val) {
897 pr_debug("Config value not set\n");
898 return -BPF_LOADER_ERRNO__OBJCONF_CONF;
899 }
900
901 if (term->type_val != PARSE_EVENTS__TERM_TYPE_NUM) {
902 pr_debug("ERROR: wrong value type\n");
903 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE;
904 }
905
906 return __bpf_map__config_value(map, term);
907 }
908
909 struct bpf_obj_config__map_func {
910 const char *config_opt;
911 int (*config_func)(struct bpf_map *, struct parse_events_term *,
912 struct perf_evlist *);
913 };
914
915 struct bpf_obj_config__map_func bpf_obj_config__map_funcs[] = {
916 {"value", bpf_map__config_value},
917 };
918
919 static int
920 bpf__obj_config_map(struct bpf_object *obj,
921 struct parse_events_term *term,
922 struct perf_evlist *evlist,
923 int *key_scan_pos)
924 {
925 /* key is "map:<mapname>.<config opt>" */
926 char *map_name = strdup(term->config + sizeof("map:") - 1);
927 struct bpf_map *map;
928 int err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
929 char *map_opt;
930 size_t i;
931
932 if (!map_name)
933 return -ENOMEM;
934
935 map_opt = strchr(map_name, '.');
936 if (!map_opt) {
937 pr_debug("ERROR: Invalid map config: %s\n", map_name);
938 goto out;
939 }
940
941 *map_opt++ = '\0';
942 if (*map_opt == '\0') {
943 pr_debug("ERROR: Invalid map option: %s\n", term->config);
944 goto out;
945 }
946
947 map = bpf_object__get_map_by_name(obj, map_name);
948 if (!map) {
949 pr_debug("ERROR: Map %s doesn't exist\n", map_name);
950 err = -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST;
951 goto out;
952 }
953
954 *key_scan_pos += map_opt - map_name;
955 for (i = 0; i < ARRAY_SIZE(bpf_obj_config__map_funcs); i++) {
956 struct bpf_obj_config__map_func *func =
957 &bpf_obj_config__map_funcs[i];
958
959 if (strcmp(map_opt, func->config_opt) == 0) {
960 err = func->config_func(map, term, evlist);
961 goto out;
962 }
963 }
964
965 pr_debug("ERROR: Invalid map config option '%s'\n", map_opt);
966 err = -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT;
967 out:
968 free(map_name);
969 if (!err)
970 key_scan_pos += strlen(map_opt);
971 return err;
972 }
973
974 int bpf__config_obj(struct bpf_object *obj,
975 struct parse_events_term *term,
976 struct perf_evlist *evlist,
977 int *error_pos)
978 {
979 int key_scan_pos = 0;
980 int err;
981
982 if (!obj || !term || !term->config)
983 return -EINVAL;
984
985 if (!prefixcmp(term->config, "map:")) {
986 key_scan_pos = sizeof("map:") - 1;
987 err = bpf__obj_config_map(obj, term, evlist, &key_scan_pos);
988 goto out;
989 }
990 err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
991 out:
992 if (error_pos)
993 *error_pos = key_scan_pos;
994 return err;
995
996 }
997
998 typedef int (*map_config_func_t)(const char *name, int map_fd,
999 struct bpf_map_def *pdef,
1000 struct bpf_map_op *op,
1001 void *pkey, void *arg);
1002
1003 static int
1004 foreach_key_array_all(map_config_func_t func,
1005 void *arg, const char *name,
1006 int map_fd, struct bpf_map_def *pdef,
1007 struct bpf_map_op *op)
1008 {
1009 unsigned int i;
1010 int err;
1011
1012 for (i = 0; i < pdef->max_entries; i++) {
1013 err = func(name, map_fd, pdef, op, &i, arg);
1014 if (err) {
1015 pr_debug("ERROR: failed to insert value to %s[%u]\n",
1016 name, i);
1017 return err;
1018 }
1019 }
1020 return 0;
1021 }
1022
1023 static int
1024 bpf_map_config_foreach_key(struct bpf_map *map,
1025 map_config_func_t func,
1026 void *arg)
1027 {
1028 int err, map_fd;
1029 const char *name;
1030 struct bpf_map_op *op;
1031 struct bpf_map_def def;
1032 struct bpf_map_priv *priv;
1033
1034 name = bpf_map__get_name(map);
1035
1036 err = bpf_map__get_private(map, (void **)&priv);
1037 if (err) {
1038 pr_debug("ERROR: failed to get private from map %s\n", name);
1039 return -BPF_LOADER_ERRNO__INTERNAL;
1040 }
1041 if (!priv || list_empty(&priv->ops_list)) {
1042 pr_debug("INFO: nothing to config for map %s\n", name);
1043 return 0;
1044 }
1045
1046 err = bpf_map__get_def(map, &def);
1047 if (err) {
1048 pr_debug("ERROR: failed to get definition from map %s\n", name);
1049 return -BPF_LOADER_ERRNO__INTERNAL;
1050 }
1051 map_fd = bpf_map__get_fd(map);
1052 if (map_fd < 0) {
1053 pr_debug("ERROR: failed to get fd from map %s\n", name);
1054 return map_fd;
1055 }
1056
1057 list_for_each_entry(op, &priv->ops_list, list) {
1058 switch (def.type) {
1059 case BPF_MAP_TYPE_ARRAY:
1060 switch (op->key_type) {
1061 case BPF_MAP_KEY_ALL:
1062 err = foreach_key_array_all(func, arg, name,
1063 map_fd, &def, op);
1064 if (err)
1065 return err;
1066 break;
1067 default:
1068 pr_debug("ERROR: keytype for map '%s' invalid\n",
1069 name);
1070 return -BPF_LOADER_ERRNO__INTERNAL;
1071 }
1072 break;
1073 default:
1074 pr_debug("ERROR: type of '%s' incorrect\n", name);
1075 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1076 }
1077 }
1078
1079 return 0;
1080 }
1081
1082 static int
1083 apply_config_value_for_key(int map_fd, void *pkey,
1084 size_t val_size, u64 val)
1085 {
1086 int err = 0;
1087
1088 switch (val_size) {
1089 case 1: {
1090 u8 _val = (u8)(val);
1091 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1092 break;
1093 }
1094 case 2: {
1095 u16 _val = (u16)(val);
1096 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1097 break;
1098 }
1099 case 4: {
1100 u32 _val = (u32)(val);
1101 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1102 break;
1103 }
1104 case 8: {
1105 err = bpf_map_update_elem(map_fd, pkey, &val, BPF_ANY);
1106 break;
1107 }
1108 default:
1109 pr_debug("ERROR: invalid value size\n");
1110 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
1111 }
1112 if (err && errno)
1113 err = -errno;
1114 return err;
1115 }
1116
1117 static int
1118 apply_obj_config_map_for_key(const char *name, int map_fd,
1119 struct bpf_map_def *pdef __maybe_unused,
1120 struct bpf_map_op *op,
1121 void *pkey, void *arg __maybe_unused)
1122 {
1123 int err;
1124
1125 switch (op->op_type) {
1126 case BPF_MAP_OP_SET_VALUE:
1127 err = apply_config_value_for_key(map_fd, pkey,
1128 pdef->value_size,
1129 op->v.value);
1130 break;
1131 default:
1132 pr_debug("ERROR: unknown value type for '%s'\n", name);
1133 err = -BPF_LOADER_ERRNO__INTERNAL;
1134 }
1135 return err;
1136 }
1137
1138 static int
1139 apply_obj_config_map(struct bpf_map *map)
1140 {
1141 return bpf_map_config_foreach_key(map,
1142 apply_obj_config_map_for_key,
1143 NULL);
1144 }
1145
1146 static int
1147 apply_obj_config_object(struct bpf_object *obj)
1148 {
1149 struct bpf_map *map;
1150 int err;
1151
1152 bpf_map__for_each(map, obj) {
1153 err = apply_obj_config_map(map);
1154 if (err)
1155 return err;
1156 }
1157 return 0;
1158 }
1159
1160 int bpf__apply_obj_config(void)
1161 {
1162 struct bpf_object *obj, *tmp;
1163 int err;
1164
1165 bpf_object__for_each_safe(obj, tmp) {
1166 err = apply_obj_config_object(obj);
1167 if (err)
1168 return err;
1169 }
1170
1171 return 0;
1172 }
1173
1174 #define ERRNO_OFFSET(e) ((e) - __BPF_LOADER_ERRNO__START)
1175 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
1176 #define NR_ERRNO (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
1177
1178 static const char *bpf_loader_strerror_table[NR_ERRNO] = {
1179 [ERRCODE_OFFSET(CONFIG)] = "Invalid config string",
1180 [ERRCODE_OFFSET(GROUP)] = "Invalid group name",
1181 [ERRCODE_OFFSET(EVENTNAME)] = "No event name found in config string",
1182 [ERRCODE_OFFSET(INTERNAL)] = "BPF loader internal error",
1183 [ERRCODE_OFFSET(COMPILE)] = "Error when compiling BPF scriptlet",
1184 [ERRCODE_OFFSET(PROGCONF_TERM)] = "Invalid program config term in config string",
1185 [ERRCODE_OFFSET(PROLOGUE)] = "Failed to generate prologue",
1186 [ERRCODE_OFFSET(PROLOGUE2BIG)] = "Prologue too big for program",
1187 [ERRCODE_OFFSET(PROLOGUEOOB)] = "Offset out of bound for prologue",
1188 [ERRCODE_OFFSET(OBJCONF_OPT)] = "Invalid object config option",
1189 [ERRCODE_OFFSET(OBJCONF_CONF)] = "Config value not set (missing '=')",
1190 [ERRCODE_OFFSET(OBJCONF_MAP_OPT)] = "Invalid object map config option",
1191 [ERRCODE_OFFSET(OBJCONF_MAP_NOTEXIST)] = "Target map doesn't exist",
1192 [ERRCODE_OFFSET(OBJCONF_MAP_VALUE)] = "Incorrect value type for map",
1193 [ERRCODE_OFFSET(OBJCONF_MAP_TYPE)] = "Incorrect map type",
1194 [ERRCODE_OFFSET(OBJCONF_MAP_KEYSIZE)] = "Incorrect map key size",
1195 [ERRCODE_OFFSET(OBJCONF_MAP_VALUESIZE)] = "Incorrect map value size",
1196 };
1197
1198 static int
1199 bpf_loader_strerror(int err, char *buf, size_t size)
1200 {
1201 char sbuf[STRERR_BUFSIZE];
1202 const char *msg;
1203
1204 if (!buf || !size)
1205 return -1;
1206
1207 err = err > 0 ? err : -err;
1208
1209 if (err >= __LIBBPF_ERRNO__START)
1210 return libbpf_strerror(err, buf, size);
1211
1212 if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
1213 msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
1214 snprintf(buf, size, "%s", msg);
1215 buf[size - 1] = '\0';
1216 return 0;
1217 }
1218
1219 if (err >= __BPF_LOADER_ERRNO__END)
1220 snprintf(buf, size, "Unknown bpf loader error %d", err);
1221 else
1222 snprintf(buf, size, "%s",
1223 strerror_r(err, sbuf, sizeof(sbuf)));
1224
1225 buf[size - 1] = '\0';
1226 return -1;
1227 }
1228
1229 #define bpf__strerror_head(err, buf, size) \
1230 char sbuf[STRERR_BUFSIZE], *emsg;\
1231 if (!size)\
1232 return 0;\
1233 if (err < 0)\
1234 err = -err;\
1235 bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
1236 emsg = sbuf;\
1237 switch (err) {\
1238 default:\
1239 scnprintf(buf, size, "%s", emsg);\
1240 break;
1241
1242 #define bpf__strerror_entry(val, fmt...)\
1243 case val: {\
1244 scnprintf(buf, size, fmt);\
1245 break;\
1246 }
1247
1248 #define bpf__strerror_end(buf, size)\
1249 }\
1250 buf[size - 1] = '\0';
1251
1252 int bpf__strerror_prepare_load(const char *filename, bool source,
1253 int err, char *buf, size_t size)
1254 {
1255 size_t n;
1256 int ret;
1257
1258 n = snprintf(buf, size, "Failed to load %s%s: ",
1259 filename, source ? " from source" : "");
1260 if (n >= size) {
1261 buf[size - 1] = '\0';
1262 return 0;
1263 }
1264 buf += n;
1265 size -= n;
1266
1267 ret = bpf_loader_strerror(err, buf, size);
1268 buf[size - 1] = '\0';
1269 return ret;
1270 }
1271
1272 int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
1273 int err, char *buf, size_t size)
1274 {
1275 bpf__strerror_head(err, buf, size);
1276 case BPF_LOADER_ERRNO__PROGCONF_TERM: {
1277 scnprintf(buf, size, "%s (add -v to see detail)", emsg);
1278 break;
1279 }
1280 bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'");
1281 bpf__strerror_entry(EACCES, "You need to be root");
1282 bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
1283 bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
1284 bpf__strerror_end(buf, size);
1285 return 0;
1286 }
1287
1288 int bpf__strerror_load(struct bpf_object *obj,
1289 int err, char *buf, size_t size)
1290 {
1291 bpf__strerror_head(err, buf, size);
1292 case LIBBPF_ERRNO__KVER: {
1293 unsigned int obj_kver = bpf_object__get_kversion(obj);
1294 unsigned int real_kver;
1295
1296 if (fetch_kernel_version(&real_kver, NULL, 0)) {
1297 scnprintf(buf, size, "Unable to fetch kernel version");
1298 break;
1299 }
1300
1301 if (obj_kver != real_kver) {
1302 scnprintf(buf, size,
1303 "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
1304 KVER_PARAM(obj_kver),
1305 KVER_PARAM(real_kver));
1306 break;
1307 }
1308
1309 scnprintf(buf, size, "Failed to load program for unknown reason");
1310 break;
1311 }
1312 bpf__strerror_end(buf, size);
1313 return 0;
1314 }
1315
1316 int bpf__strerror_config_obj(struct bpf_object *obj __maybe_unused,
1317 struct parse_events_term *term __maybe_unused,
1318 struct perf_evlist *evlist __maybe_unused,
1319 int *error_pos __maybe_unused, int err,
1320 char *buf, size_t size)
1321 {
1322 bpf__strerror_head(err, buf, size);
1323 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE,
1324 "Can't use this config term with this map type");
1325 bpf__strerror_end(buf, size);
1326 return 0;
1327 }
1328
1329 int bpf__strerror_apply_obj_config(int err, char *buf, size_t size)
1330 {
1331 bpf__strerror_head(err, buf, size);
1332 bpf__strerror_end(buf, size);
1333 return 0;
1334 }
Ubuntu Artful kernel mirror