]>
Commit | Line | Data |
---|---|---|
69d262a9 WN |
1 | /* |
2 | * bpf-loader.c | |
3 | * | |
4 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
5 | * Copyright (C) 2015 Huawei Inc. | |
6 | */ | |
7 | ||
8 | #include <bpf/libbpf.h> | |
9 | #include <linux/err.h> | |
10 | #include "perf.h" | |
11 | #include "debug.h" | |
12 | #include "bpf-loader.h" | |
aa3abf30 WN |
13 | #include "probe-event.h" |
14 | #include "probe-finder.h" // for MAX_PROBES | |
d509db04 | 15 | #include "llvm-utils.h" |
69d262a9 WN |
16 | |
17 | #define DEFINE_PRINT_FN(name, level) \ | |
18 | static int libbpf_##name(const char *fmt, ...) \ | |
19 | { \ | |
20 | va_list args; \ | |
21 | int ret; \ | |
22 | \ | |
23 | va_start(args, fmt); \ | |
24 | ret = veprintf(level, verbose, pr_fmt(fmt), args);\ | |
25 | va_end(args); \ | |
26 | return ret; \ | |
27 | } | |
28 | ||
7a011946 WN |
29 | DEFINE_PRINT_FN(warning, 1) |
30 | DEFINE_PRINT_FN(info, 1) | |
69d262a9 WN |
31 | DEFINE_PRINT_FN(debug, 1) |
32 | ||
aa3abf30 WN |
33 | struct bpf_prog_priv { |
34 | struct perf_probe_event pev; | |
35 | }; | |
36 | ||
ba1fae43 WN |
37 | static bool libbpf_initialized; |
38 | ||
39 | struct bpf_object * | |
40 | bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name) | |
41 | { | |
42 | struct bpf_object *obj; | |
43 | ||
44 | if (!libbpf_initialized) { | |
45 | libbpf_set_print(libbpf_warning, | |
46 | libbpf_info, | |
47 | libbpf_debug); | |
48 | libbpf_initialized = true; | |
49 | } | |
50 | ||
51 | obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name); | |
52 | if (IS_ERR(obj)) { | |
53 | pr_debug("bpf: failed to load buffer\n"); | |
54 | return ERR_PTR(-EINVAL); | |
55 | } | |
56 | ||
57 | return obj; | |
58 | } | |
59 | ||
d509db04 | 60 | struct bpf_object *bpf__prepare_load(const char *filename, bool source) |
69d262a9 WN |
61 | { |
62 | struct bpf_object *obj; | |
69d262a9 WN |
63 | |
64 | if (!libbpf_initialized) { | |
65 | libbpf_set_print(libbpf_warning, | |
66 | libbpf_info, | |
67 | libbpf_debug); | |
68 | libbpf_initialized = true; | |
69 | } | |
70 | ||
d509db04 WN |
71 | if (source) { |
72 | int err; | |
73 | void *obj_buf; | |
74 | size_t obj_buf_sz; | |
75 | ||
76 | err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz); | |
77 | if (err) | |
d3e0ce39 | 78 | return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE); |
d509db04 WN |
79 | obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename); |
80 | free(obj_buf); | |
81 | } else | |
82 | obj = bpf_object__open(filename); | |
83 | ||
6371ca3b | 84 | if (IS_ERR(obj)) { |
69d262a9 | 85 | pr_debug("bpf: failed to load %s\n", filename); |
6371ca3b | 86 | return obj; |
69d262a9 WN |
87 | } |
88 | ||
89 | return obj; | |
90 | } | |
91 | ||
92 | void bpf__clear(void) | |
93 | { | |
94 | struct bpf_object *obj, *tmp; | |
95 | ||
aa3abf30 WN |
96 | bpf_object__for_each_safe(obj, tmp) { |
97 | bpf__unprobe(obj); | |
69d262a9 | 98 | bpf_object__close(obj); |
aa3abf30 WN |
99 | } |
100 | } | |
101 | ||
102 | static void | |
103 | bpf_prog_priv__clear(struct bpf_program *prog __maybe_unused, | |
104 | void *_priv) | |
105 | { | |
106 | struct bpf_prog_priv *priv = _priv; | |
107 | ||
108 | cleanup_perf_probe_events(&priv->pev, 1); | |
109 | free(priv); | |
110 | } | |
111 | ||
112 | static int | |
113 | config_bpf_program(struct bpf_program *prog) | |
114 | { | |
115 | struct perf_probe_event *pev = NULL; | |
116 | struct bpf_prog_priv *priv = NULL; | |
117 | const char *config_str; | |
118 | int err; | |
119 | ||
120 | config_str = bpf_program__title(prog, false); | |
6371ca3b | 121 | if (IS_ERR(config_str)) { |
aa3abf30 | 122 | pr_debug("bpf: unable to get title for program\n"); |
6371ca3b | 123 | return PTR_ERR(config_str); |
aa3abf30 WN |
124 | } |
125 | ||
126 | priv = calloc(sizeof(*priv), 1); | |
127 | if (!priv) { | |
128 | pr_debug("bpf: failed to alloc priv\n"); | |
129 | return -ENOMEM; | |
130 | } | |
131 | pev = &priv->pev; | |
132 | ||
133 | pr_debug("bpf: config program '%s'\n", config_str); | |
134 | err = parse_perf_probe_command(config_str, pev); | |
135 | if (err < 0) { | |
136 | pr_debug("bpf: '%s' is not a valid config string\n", | |
137 | config_str); | |
d3e0ce39 | 138 | err = -BPF_LOADER_ERRNO__CONFIG; |
aa3abf30 WN |
139 | goto errout; |
140 | } | |
141 | ||
142 | if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) { | |
143 | pr_debug("bpf: '%s': group for event is set and not '%s'.\n", | |
144 | config_str, PERF_BPF_PROBE_GROUP); | |
d3e0ce39 | 145 | err = -BPF_LOADER_ERRNO__GROUP; |
aa3abf30 WN |
146 | goto errout; |
147 | } else if (!pev->group) | |
148 | pev->group = strdup(PERF_BPF_PROBE_GROUP); | |
149 | ||
150 | if (!pev->group) { | |
151 | pr_debug("bpf: strdup failed\n"); | |
152 | err = -ENOMEM; | |
153 | goto errout; | |
154 | } | |
155 | ||
156 | if (!pev->event) { | |
d3e0ce39 | 157 | pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n", |
aa3abf30 | 158 | config_str); |
d3e0ce39 | 159 | err = -BPF_LOADER_ERRNO__EVENTNAME; |
aa3abf30 WN |
160 | goto errout; |
161 | } | |
162 | pr_debug("bpf: config '%s' is ok\n", config_str); | |
163 | ||
164 | err = bpf_program__set_private(prog, priv, bpf_prog_priv__clear); | |
165 | if (err) { | |
166 | pr_debug("Failed to set priv for program '%s'\n", config_str); | |
167 | goto errout; | |
168 | } | |
169 | ||
170 | return 0; | |
171 | ||
172 | errout: | |
173 | if (pev) | |
174 | clear_perf_probe_event(pev); | |
175 | free(priv); | |
176 | return err; | |
177 | } | |
178 | ||
179 | static int bpf__prepare_probe(void) | |
180 | { | |
181 | static int err = 0; | |
182 | static bool initialized = false; | |
183 | ||
184 | /* | |
185 | * Make err static, so if init failed the first, bpf__prepare_probe() | |
186 | * fails each time without calling init_probe_symbol_maps multiple | |
187 | * times. | |
188 | */ | |
189 | if (initialized) | |
190 | return err; | |
191 | ||
192 | initialized = true; | |
193 | err = init_probe_symbol_maps(false); | |
194 | if (err < 0) | |
195 | pr_debug("Failed to init_probe_symbol_maps\n"); | |
196 | probe_conf.max_probes = MAX_PROBES; | |
197 | return err; | |
198 | } | |
199 | ||
200 | int bpf__probe(struct bpf_object *obj) | |
201 | { | |
202 | int err = 0; | |
203 | struct bpf_program *prog; | |
204 | struct bpf_prog_priv *priv; | |
205 | struct perf_probe_event *pev; | |
206 | ||
207 | err = bpf__prepare_probe(); | |
208 | if (err) { | |
209 | pr_debug("bpf__prepare_probe failed\n"); | |
210 | return err; | |
211 | } | |
212 | ||
213 | bpf_object__for_each_program(prog, obj) { | |
214 | err = config_bpf_program(prog); | |
215 | if (err) | |
216 | goto out; | |
217 | ||
218 | err = bpf_program__get_private(prog, (void **)&priv); | |
219 | if (err || !priv) | |
220 | goto out; | |
221 | pev = &priv->pev; | |
222 | ||
223 | err = convert_perf_probe_events(pev, 1); | |
224 | if (err < 0) { | |
225 | pr_debug("bpf_probe: failed to convert perf probe events"); | |
226 | goto out; | |
227 | } | |
228 | ||
229 | err = apply_perf_probe_events(pev, 1); | |
230 | if (err < 0) { | |
231 | pr_debug("bpf_probe: failed to apply perf probe events"); | |
232 | goto out; | |
233 | } | |
234 | } | |
235 | out: | |
236 | return err < 0 ? err : 0; | |
237 | } | |
238 | ||
239 | #define EVENTS_WRITE_BUFSIZE 4096 | |
240 | int bpf__unprobe(struct bpf_object *obj) | |
241 | { | |
242 | int err, ret = 0; | |
243 | struct bpf_program *prog; | |
244 | struct bpf_prog_priv *priv; | |
245 | ||
246 | bpf_object__for_each_program(prog, obj) { | |
247 | int i; | |
248 | ||
249 | err = bpf_program__get_private(prog, (void **)&priv); | |
250 | if (err || !priv) | |
251 | continue; | |
252 | ||
253 | for (i = 0; i < priv->pev.ntevs; i++) { | |
254 | struct probe_trace_event *tev = &priv->pev.tevs[i]; | |
255 | char name_buf[EVENTS_WRITE_BUFSIZE]; | |
256 | struct strfilter *delfilter; | |
257 | ||
258 | snprintf(name_buf, EVENTS_WRITE_BUFSIZE, | |
259 | "%s:%s", tev->group, tev->event); | |
260 | name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0'; | |
261 | ||
262 | delfilter = strfilter__new(name_buf, NULL); | |
263 | if (!delfilter) { | |
264 | pr_debug("Failed to create filter for unprobing\n"); | |
265 | ret = -ENOMEM; | |
266 | continue; | |
267 | } | |
268 | ||
269 | err = del_perf_probe_events(delfilter); | |
270 | strfilter__delete(delfilter); | |
271 | if (err) { | |
272 | pr_debug("Failed to delete %s\n", name_buf); | |
273 | ret = err; | |
274 | continue; | |
275 | } | |
276 | } | |
277 | } | |
278 | return ret; | |
279 | } | |
280 | ||
1e5e3ee8 WN |
281 | int bpf__load(struct bpf_object *obj) |
282 | { | |
283 | int err; | |
284 | ||
285 | err = bpf_object__load(obj); | |
286 | if (err) { | |
287 | pr_debug("bpf: load objects failed\n"); | |
288 | return err; | |
289 | } | |
290 | return 0; | |
291 | } | |
292 | ||
4edf30e3 WN |
293 | int bpf__foreach_tev(struct bpf_object *obj, |
294 | bpf_prog_iter_callback_t func, | |
295 | void *arg) | |
296 | { | |
297 | struct bpf_program *prog; | |
298 | int err; | |
299 | ||
300 | bpf_object__for_each_program(prog, obj) { | |
301 | struct probe_trace_event *tev; | |
302 | struct perf_probe_event *pev; | |
303 | struct bpf_prog_priv *priv; | |
304 | int i, fd; | |
305 | ||
306 | err = bpf_program__get_private(prog, | |
307 | (void **)&priv); | |
308 | if (err || !priv) { | |
309 | pr_debug("bpf: failed to get private field\n"); | |
d3e0ce39 | 310 | return -BPF_LOADER_ERRNO__INTERNAL; |
4edf30e3 WN |
311 | } |
312 | ||
313 | pev = &priv->pev; | |
314 | for (i = 0; i < pev->ntevs; i++) { | |
315 | tev = &pev->tevs[i]; | |
316 | ||
317 | fd = bpf_program__fd(prog); | |
318 | if (fd < 0) { | |
319 | pr_debug("bpf: failed to get file descriptor\n"); | |
320 | return fd; | |
321 | } | |
322 | ||
323 | err = (*func)(tev, fd, arg); | |
324 | if (err) { | |
325 | pr_debug("bpf: call back failed, stop iterate\n"); | |
326 | return err; | |
327 | } | |
328 | } | |
329 | } | |
330 | return 0; | |
331 | } | |
332 | ||
d3e0ce39 WN |
333 | #define ERRNO_OFFSET(e) ((e) - __BPF_LOADER_ERRNO__START) |
334 | #define ERRCODE_OFFSET(c) ERRNO_OFFSET(BPF_LOADER_ERRNO__##c) | |
335 | #define NR_ERRNO (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START) | |
336 | ||
337 | static const char *bpf_loader_strerror_table[NR_ERRNO] = { | |
338 | [ERRCODE_OFFSET(CONFIG)] = "Invalid config string", | |
339 | [ERRCODE_OFFSET(GROUP)] = "Invalid group name", | |
340 | [ERRCODE_OFFSET(EVENTNAME)] = "No event name found in config string", | |
341 | [ERRCODE_OFFSET(INTERNAL)] = "BPF loader internal error", | |
342 | [ERRCODE_OFFSET(COMPILE)] = "Error when compiling BPF scriptlet", | |
343 | }; | |
344 | ||
6371ca3b WN |
345 | static int |
346 | bpf_loader_strerror(int err, char *buf, size_t size) | |
347 | { | |
348 | char sbuf[STRERR_BUFSIZE]; | |
349 | const char *msg; | |
350 | ||
351 | if (!buf || !size) | |
352 | return -1; | |
353 | ||
354 | err = err > 0 ? err : -err; | |
355 | ||
356 | if (err >= __LIBBPF_ERRNO__START) | |
357 | return libbpf_strerror(err, buf, size); | |
358 | ||
d3e0ce39 WN |
359 | if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) { |
360 | msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)]; | |
361 | snprintf(buf, size, "%s", msg); | |
362 | buf[size - 1] = '\0'; | |
363 | return 0; | |
364 | } | |
365 | ||
366 | if (err >= __BPF_LOADER_ERRNO__END) | |
367 | snprintf(buf, size, "Unknown bpf loader error %d", err); | |
368 | else | |
369 | snprintf(buf, size, "%s", | |
370 | strerror_r(err, sbuf, sizeof(sbuf))); | |
371 | ||
6371ca3b | 372 | buf[size - 1] = '\0'; |
d3e0ce39 | 373 | return -1; |
6371ca3b WN |
374 | } |
375 | ||
aa3abf30 WN |
376 | #define bpf__strerror_head(err, buf, size) \ |
377 | char sbuf[STRERR_BUFSIZE], *emsg;\ | |
378 | if (!size)\ | |
379 | return 0;\ | |
380 | if (err < 0)\ | |
381 | err = -err;\ | |
6371ca3b WN |
382 | bpf_loader_strerror(err, sbuf, sizeof(sbuf));\ |
383 | emsg = sbuf;\ | |
aa3abf30 WN |
384 | switch (err) {\ |
385 | default:\ | |
386 | scnprintf(buf, size, "%s", emsg);\ | |
387 | break; | |
388 | ||
389 | #define bpf__strerror_entry(val, fmt...)\ | |
390 | case val: {\ | |
391 | scnprintf(buf, size, fmt);\ | |
392 | break;\ | |
393 | } | |
394 | ||
395 | #define bpf__strerror_end(buf, size)\ | |
396 | }\ | |
397 | buf[size - 1] = '\0'; | |
398 | ||
d3e0ce39 WN |
399 | int bpf__strerror_prepare_load(const char *filename, bool source, |
400 | int err, char *buf, size_t size) | |
401 | { | |
402 | size_t n; | |
403 | int ret; | |
404 | ||
405 | n = snprintf(buf, size, "Failed to load %s%s: ", | |
406 | filename, source ? " from source" : ""); | |
407 | if (n >= size) { | |
408 | buf[size - 1] = '\0'; | |
409 | return 0; | |
410 | } | |
411 | buf += n; | |
412 | size -= n; | |
413 | ||
414 | ret = bpf_loader_strerror(err, buf, size); | |
415 | buf[size - 1] = '\0'; | |
416 | return ret; | |
417 | } | |
418 | ||
aa3abf30 WN |
419 | int bpf__strerror_probe(struct bpf_object *obj __maybe_unused, |
420 | int err, char *buf, size_t size) | |
421 | { | |
422 | bpf__strerror_head(err, buf, size); | |
423 | bpf__strerror_entry(EEXIST, "Probe point exist. Try use 'perf probe -d \"*\"'"); | |
d3e0ce39 WN |
424 | bpf__strerror_entry(EACCES, "You need to be root"); |
425 | bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0"); | |
426 | bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file"); | |
aa3abf30 WN |
427 | bpf__strerror_end(buf, size); |
428 | return 0; | |
69d262a9 | 429 | } |
1e5e3ee8 | 430 | |
d3e0ce39 | 431 | int bpf__strerror_load(struct bpf_object *obj, |
1e5e3ee8 WN |
432 | int err, char *buf, size_t size) |
433 | { | |
434 | bpf__strerror_head(err, buf, size); | |
d3e0ce39 WN |
435 | case LIBBPF_ERRNO__KVER: { |
436 | unsigned int obj_kver = bpf_object__get_kversion(obj); | |
437 | unsigned int real_kver; | |
438 | ||
439 | if (fetch_kernel_version(&real_kver, NULL, 0)) { | |
440 | scnprintf(buf, size, "Unable to fetch kernel version"); | |
441 | break; | |
442 | } | |
443 | ||
444 | if (obj_kver != real_kver) { | |
445 | scnprintf(buf, size, | |
446 | "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")", | |
447 | KVER_PARAM(obj_kver), | |
448 | KVER_PARAM(real_kver)); | |
449 | break; | |
450 | } | |
451 | ||
452 | scnprintf(buf, size, "Failed to load program for unknown reason"); | |
453 | break; | |
454 | } | |
1e5e3ee8 WN |
455 | bpf__strerror_end(buf, size); |
456 | return 0; | |
457 | } |