]>
Commit | Line | Data |
---|---|---|
a39e17b2 JK |
1 | /* |
2 | * Copyright (C) 2017 Netronome Systems, Inc. | |
3 | * | |
4 | * This software is licensed under the GNU General License Version 2, | |
5 | * June 1991 as shown in the file COPYING in the top-level directory of this | |
6 | * source tree. | |
7 | * | |
8 | * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" | |
9 | * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, | |
10 | * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS | |
11 | * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE | |
12 | * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME | |
13 | * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. | |
14 | */ | |
15 | ||
ab3f0063 JK |
16 | #include <linux/bpf.h> |
17 | #include <linux/bpf_verifier.h> | |
18 | #include <linux/bug.h> | |
675fc275 | 19 | #include <linux/kdev_t.h> |
ab3f0063 JK |
20 | #include <linux/list.h> |
21 | #include <linux/netdevice.h> | |
22 | #include <linux/printk.h> | |
675fc275 | 23 | #include <linux/proc_ns.h> |
ab3f0063 | 24 | #include <linux/rtnetlink.h> |
e0d3974a | 25 | #include <linux/rwsem.h> |
ab3f0063 | 26 | |
a3884572 JK |
27 | /* Protects bpf_prog_offload_devs, bpf_map_offload_devs and offload members |
28 | * of all progs. | |
e0d3974a JK |
29 | * RTNL lock cannot be taken when holding this lock. |
30 | */ | |
31 | static DECLARE_RWSEM(bpf_devs_lock); | |
ab3f0063 | 32 | static LIST_HEAD(bpf_prog_offload_devs); |
a3884572 | 33 | static LIST_HEAD(bpf_map_offload_devs); |
ab3f0063 | 34 | |
5bc2d55c JK |
35 | static int bpf_dev_offload_check(struct net_device *netdev) |
36 | { | |
37 | if (!netdev) | |
38 | return -EINVAL; | |
39 | if (!netdev->netdev_ops->ndo_bpf) | |
40 | return -EOPNOTSUPP; | |
41 | return 0; | |
42 | } | |
43 | ||
ab3f0063 JK |
44 | int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr) |
45 | { | |
0a9c1991 | 46 | struct bpf_prog_offload *offload; |
5bc2d55c | 47 | int err; |
ab3f0063 | 48 | |
649f11dc JK |
49 | if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS && |
50 | attr->prog_type != BPF_PROG_TYPE_XDP) | |
51 | return -EINVAL; | |
ab3f0063 JK |
52 | |
53 | if (attr->prog_flags) | |
54 | return -EINVAL; | |
55 | ||
56 | offload = kzalloc(sizeof(*offload), GFP_USER); | |
57 | if (!offload) | |
58 | return -ENOMEM; | |
59 | ||
60 | offload->prog = prog; | |
ab3f0063 | 61 | |
e0d3974a JK |
62 | offload->netdev = dev_get_by_index(current->nsproxy->net_ns, |
63 | attr->prog_ifindex); | |
5bc2d55c JK |
64 | err = bpf_dev_offload_check(offload->netdev); |
65 | if (err) | |
66 | goto err_maybe_put; | |
ab3f0063 | 67 | |
e0d3974a | 68 | down_write(&bpf_devs_lock); |
5bc2d55c JK |
69 | if (offload->netdev->reg_state != NETREG_REGISTERED) { |
70 | err = -EINVAL; | |
e0d3974a | 71 | goto err_unlock; |
5bc2d55c | 72 | } |
ab3f0063 JK |
73 | prog->aux->offload = offload; |
74 | list_add_tail(&offload->offloads, &bpf_prog_offload_devs); | |
e0d3974a JK |
75 | dev_put(offload->netdev); |
76 | up_write(&bpf_devs_lock); | |
ab3f0063 JK |
77 | |
78 | return 0; | |
e0d3974a JK |
79 | err_unlock: |
80 | up_write(&bpf_devs_lock); | |
5bc2d55c JK |
81 | err_maybe_put: |
82 | if (offload->netdev) | |
83 | dev_put(offload->netdev); | |
e0d3974a | 84 | kfree(offload); |
5bc2d55c | 85 | return err; |
ab3f0063 JK |
86 | } |
87 | ||
88 | static int __bpf_offload_ndo(struct bpf_prog *prog, enum bpf_netdev_command cmd, | |
89 | struct netdev_bpf *data) | |
90 | { | |
0a9c1991 | 91 | struct bpf_prog_offload *offload = prog->aux->offload; |
ce3b9db4 | 92 | struct net_device *netdev; |
ab3f0063 JK |
93 | |
94 | ASSERT_RTNL(); | |
95 | ||
ce3b9db4 | 96 | if (!offload) |
ab3f0063 | 97 | return -ENODEV; |
ce3b9db4 | 98 | netdev = offload->netdev; |
ab3f0063 JK |
99 | |
100 | data->command = cmd; | |
101 | ||
102 | return netdev->netdev_ops->ndo_bpf(netdev, data); | |
103 | } | |
104 | ||
105 | int bpf_prog_offload_verifier_prep(struct bpf_verifier_env *env) | |
106 | { | |
107 | struct netdev_bpf data = {}; | |
108 | int err; | |
109 | ||
110 | data.verifier.prog = env->prog; | |
111 | ||
112 | rtnl_lock(); | |
113 | err = __bpf_offload_ndo(env->prog, BPF_OFFLOAD_VERIFIER_PREP, &data); | |
114 | if (err) | |
115 | goto exit_unlock; | |
116 | ||
cae1927c | 117 | env->prog->aux->offload->dev_ops = data.verifier.ops; |
ab3f0063 | 118 | env->prog->aux->offload->dev_state = true; |
ab3f0063 JK |
119 | exit_unlock: |
120 | rtnl_unlock(); | |
121 | return err; | |
122 | } | |
123 | ||
cae1927c JK |
124 | int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, |
125 | int insn_idx, int prev_insn_idx) | |
126 | { | |
0a9c1991 | 127 | struct bpf_prog_offload *offload; |
cae1927c JK |
128 | int ret = -ENODEV; |
129 | ||
130 | down_read(&bpf_devs_lock); | |
131 | offload = env->prog->aux->offload; | |
ce3b9db4 | 132 | if (offload) |
cae1927c JK |
133 | ret = offload->dev_ops->insn_hook(env, insn_idx, prev_insn_idx); |
134 | up_read(&bpf_devs_lock); | |
135 | ||
136 | return ret; | |
137 | } | |
138 | ||
ab3f0063 JK |
139 | static void __bpf_prog_offload_destroy(struct bpf_prog *prog) |
140 | { | |
0a9c1991 | 141 | struct bpf_prog_offload *offload = prog->aux->offload; |
ab3f0063 JK |
142 | struct netdev_bpf data = {}; |
143 | ||
144 | data.offload.prog = prog; | |
145 | ||
ab3f0063 JK |
146 | if (offload->dev_state) |
147 | WARN_ON(__bpf_offload_ndo(prog, BPF_OFFLOAD_DESTROY, &data)); | |
148 | ||
ad8ad79f JK |
149 | /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ |
150 | bpf_prog_free_id(prog, true); | |
151 | ||
ab3f0063 | 152 | list_del_init(&offload->offloads); |
ce3b9db4 JK |
153 | kfree(offload); |
154 | prog->aux->offload = NULL; | |
ab3f0063 JK |
155 | } |
156 | ||
157 | void bpf_prog_offload_destroy(struct bpf_prog *prog) | |
158 | { | |
ab3f0063 | 159 | rtnl_lock(); |
e0d3974a | 160 | down_write(&bpf_devs_lock); |
ce3b9db4 JK |
161 | if (prog->aux->offload) |
162 | __bpf_prog_offload_destroy(prog); | |
e0d3974a | 163 | up_write(&bpf_devs_lock); |
ab3f0063 | 164 | rtnl_unlock(); |
ab3f0063 JK |
165 | } |
166 | ||
167 | static int bpf_prog_offload_translate(struct bpf_prog *prog) | |
168 | { | |
ab3f0063 JK |
169 | struct netdev_bpf data = {}; |
170 | int ret; | |
171 | ||
172 | data.offload.prog = prog; | |
173 | ||
ab3f0063 JK |
174 | rtnl_lock(); |
175 | ret = __bpf_offload_ndo(prog, BPF_OFFLOAD_TRANSLATE, &data); | |
176 | rtnl_unlock(); | |
177 | ||
178 | return ret; | |
179 | } | |
180 | ||
181 | static unsigned int bpf_prog_warn_on_exec(const void *ctx, | |
182 | const struct bpf_insn *insn) | |
183 | { | |
184 | WARN(1, "attempt to execute device eBPF program on the host!"); | |
185 | return 0; | |
186 | } | |
187 | ||
188 | int bpf_prog_offload_compile(struct bpf_prog *prog) | |
189 | { | |
190 | prog->bpf_func = bpf_prog_warn_on_exec; | |
191 | ||
192 | return bpf_prog_offload_translate(prog); | |
193 | } | |
194 | ||
675fc275 JK |
195 | struct ns_get_path_bpf_prog_args { |
196 | struct bpf_prog *prog; | |
197 | struct bpf_prog_info *info; | |
198 | }; | |
199 | ||
200 | static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data) | |
201 | { | |
202 | struct ns_get_path_bpf_prog_args *args = private_data; | |
203 | struct bpf_prog_aux *aux = args->prog->aux; | |
204 | struct ns_common *ns; | |
205 | struct net *net; | |
206 | ||
207 | rtnl_lock(); | |
208 | down_read(&bpf_devs_lock); | |
209 | ||
210 | if (aux->offload) { | |
211 | args->info->ifindex = aux->offload->netdev->ifindex; | |
212 | net = dev_net(aux->offload->netdev); | |
213 | get_net(net); | |
214 | ns = &net->ns; | |
215 | } else { | |
216 | args->info->ifindex = 0; | |
217 | ns = NULL; | |
218 | } | |
219 | ||
220 | up_read(&bpf_devs_lock); | |
221 | rtnl_unlock(); | |
222 | ||
223 | return ns; | |
224 | } | |
225 | ||
226 | int bpf_prog_offload_info_fill(struct bpf_prog_info *info, | |
227 | struct bpf_prog *prog) | |
228 | { | |
229 | struct ns_get_path_bpf_prog_args args = { | |
230 | .prog = prog, | |
231 | .info = info, | |
232 | }; | |
fcfb126d | 233 | struct bpf_prog_aux *aux = prog->aux; |
675fc275 JK |
234 | struct inode *ns_inode; |
235 | struct path ns_path; | |
fcfb126d | 236 | char __user *uinsns; |
675fc275 | 237 | void *res; |
fcfb126d | 238 | u32 ulen; |
675fc275 JK |
239 | |
240 | res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args); | |
241 | if (IS_ERR(res)) { | |
242 | if (!info->ifindex) | |
243 | return -ENODEV; | |
244 | return PTR_ERR(res); | |
245 | } | |
246 | ||
fcfb126d JW |
247 | down_read(&bpf_devs_lock); |
248 | ||
249 | if (!aux->offload) { | |
250 | up_read(&bpf_devs_lock); | |
251 | return -ENODEV; | |
252 | } | |
253 | ||
254 | ulen = info->jited_prog_len; | |
255 | info->jited_prog_len = aux->offload->jited_len; | |
256 | if (info->jited_prog_len & ulen) { | |
257 | uinsns = u64_to_user_ptr(info->jited_prog_insns); | |
258 | ulen = min_t(u32, info->jited_prog_len, ulen); | |
259 | if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) { | |
260 | up_read(&bpf_devs_lock); | |
261 | return -EFAULT; | |
262 | } | |
263 | } | |
264 | ||
265 | up_read(&bpf_devs_lock); | |
266 | ||
675fc275 JK |
267 | ns_inode = ns_path.dentry->d_inode; |
268 | info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); | |
269 | info->netns_ino = ns_inode->i_ino; | |
270 | path_put(&ns_path); | |
271 | ||
272 | return 0; | |
273 | } | |
274 | ||
ab3f0063 JK |
275 | const struct bpf_prog_ops bpf_offload_prog_ops = { |
276 | }; | |
277 | ||
a3884572 JK |
278 | static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap, |
279 | enum bpf_netdev_command cmd) | |
280 | { | |
281 | struct netdev_bpf data = {}; | |
282 | struct net_device *netdev; | |
283 | ||
284 | ASSERT_RTNL(); | |
285 | ||
286 | data.command = cmd; | |
287 | data.offmap = offmap; | |
288 | /* Caller must make sure netdev is valid */ | |
289 | netdev = offmap->netdev; | |
290 | ||
291 | return netdev->netdev_ops->ndo_bpf(netdev, &data); | |
292 | } | |
293 | ||
294 | struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) | |
295 | { | |
296 | struct net *net = current->nsproxy->net_ns; | |
297 | struct bpf_offloaded_map *offmap; | |
298 | int err; | |
299 | ||
300 | if (!capable(CAP_SYS_ADMIN)) | |
301 | return ERR_PTR(-EPERM); | |
7a0ef693 JK |
302 | if (attr->map_type != BPF_MAP_TYPE_ARRAY && |
303 | attr->map_type != BPF_MAP_TYPE_HASH) | |
a3884572 JK |
304 | return ERR_PTR(-EINVAL); |
305 | ||
306 | offmap = kzalloc(sizeof(*offmap), GFP_USER); | |
307 | if (!offmap) | |
308 | return ERR_PTR(-ENOMEM); | |
309 | ||
310 | bpf_map_init_from_attr(&offmap->map, attr); | |
311 | ||
312 | rtnl_lock(); | |
313 | down_write(&bpf_devs_lock); | |
314 | offmap->netdev = __dev_get_by_index(net, attr->map_ifindex); | |
315 | err = bpf_dev_offload_check(offmap->netdev); | |
316 | if (err) | |
317 | goto err_unlock; | |
318 | ||
319 | err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC); | |
320 | if (err) | |
321 | goto err_unlock; | |
322 | ||
323 | list_add_tail(&offmap->offloads, &bpf_map_offload_devs); | |
324 | up_write(&bpf_devs_lock); | |
325 | rtnl_unlock(); | |
326 | ||
327 | return &offmap->map; | |
328 | ||
329 | err_unlock: | |
330 | up_write(&bpf_devs_lock); | |
331 | rtnl_unlock(); | |
332 | kfree(offmap); | |
333 | return ERR_PTR(err); | |
334 | } | |
335 | ||
336 | static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap) | |
337 | { | |
338 | WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE)); | |
339 | /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */ | |
340 | bpf_map_free_id(&offmap->map, true); | |
341 | list_del_init(&offmap->offloads); | |
342 | offmap->netdev = NULL; | |
343 | } | |
344 | ||
345 | void bpf_map_offload_map_free(struct bpf_map *map) | |
346 | { | |
347 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
348 | ||
349 | rtnl_lock(); | |
350 | down_write(&bpf_devs_lock); | |
351 | if (offmap->netdev) | |
352 | __bpf_map_offload_destroy(offmap); | |
353 | up_write(&bpf_devs_lock); | |
354 | rtnl_unlock(); | |
355 | ||
356 | kfree(offmap); | |
357 | } | |
358 | ||
359 | int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value) | |
360 | { | |
361 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
362 | int ret = -ENODEV; | |
363 | ||
364 | down_read(&bpf_devs_lock); | |
365 | if (offmap->netdev) | |
366 | ret = offmap->dev_ops->map_lookup_elem(offmap, key, value); | |
367 | up_read(&bpf_devs_lock); | |
368 | ||
369 | return ret; | |
370 | } | |
371 | ||
372 | int bpf_map_offload_update_elem(struct bpf_map *map, | |
373 | void *key, void *value, u64 flags) | |
374 | { | |
375 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
376 | int ret = -ENODEV; | |
377 | ||
378 | if (unlikely(flags > BPF_EXIST)) | |
379 | return -EINVAL; | |
380 | ||
381 | down_read(&bpf_devs_lock); | |
382 | if (offmap->netdev) | |
383 | ret = offmap->dev_ops->map_update_elem(offmap, key, value, | |
384 | flags); | |
385 | up_read(&bpf_devs_lock); | |
386 | ||
387 | return ret; | |
388 | } | |
389 | ||
390 | int bpf_map_offload_delete_elem(struct bpf_map *map, void *key) | |
391 | { | |
392 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
393 | int ret = -ENODEV; | |
394 | ||
395 | down_read(&bpf_devs_lock); | |
396 | if (offmap->netdev) | |
397 | ret = offmap->dev_ops->map_delete_elem(offmap, key); | |
398 | up_read(&bpf_devs_lock); | |
399 | ||
400 | return ret; | |
401 | } | |
402 | ||
403 | int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key) | |
404 | { | |
405 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
406 | int ret = -ENODEV; | |
407 | ||
408 | down_read(&bpf_devs_lock); | |
409 | if (offmap->netdev) | |
410 | ret = offmap->dev_ops->map_get_next_key(offmap, key, next_key); | |
411 | up_read(&bpf_devs_lock); | |
412 | ||
413 | return ret; | |
414 | } | |
415 | ||
52775b33 JK |
416 | struct ns_get_path_bpf_map_args { |
417 | struct bpf_offloaded_map *offmap; | |
418 | struct bpf_map_info *info; | |
419 | }; | |
420 | ||
421 | static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data) | |
422 | { | |
423 | struct ns_get_path_bpf_map_args *args = private_data; | |
424 | struct ns_common *ns; | |
425 | struct net *net; | |
426 | ||
427 | rtnl_lock(); | |
428 | down_read(&bpf_devs_lock); | |
429 | ||
430 | if (args->offmap->netdev) { | |
431 | args->info->ifindex = args->offmap->netdev->ifindex; | |
432 | net = dev_net(args->offmap->netdev); | |
433 | get_net(net); | |
434 | ns = &net->ns; | |
435 | } else { | |
436 | args->info->ifindex = 0; | |
437 | ns = NULL; | |
438 | } | |
439 | ||
440 | up_read(&bpf_devs_lock); | |
441 | rtnl_unlock(); | |
442 | ||
443 | return ns; | |
444 | } | |
445 | ||
446 | int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map) | |
447 | { | |
448 | struct ns_get_path_bpf_map_args args = { | |
449 | .offmap = map_to_offmap(map), | |
450 | .info = info, | |
451 | }; | |
452 | struct inode *ns_inode; | |
453 | struct path ns_path; | |
454 | void *res; | |
455 | ||
456 | res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args); | |
457 | if (IS_ERR(res)) { | |
458 | if (!info->ifindex) | |
459 | return -ENODEV; | |
460 | return PTR_ERR(res); | |
461 | } | |
462 | ||
463 | ns_inode = ns_path.dentry->d_inode; | |
464 | info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); | |
465 | info->netns_ino = ns_inode->i_ino; | |
466 | path_put(&ns_path); | |
467 | ||
468 | return 0; | |
469 | } | |
470 | ||
a3884572 JK |
471 | bool bpf_offload_dev_match(struct bpf_prog *prog, struct bpf_map *map) |
472 | { | |
473 | struct bpf_offloaded_map *offmap; | |
474 | struct bpf_prog_offload *offload; | |
475 | bool ret; | |
476 | ||
0a2d28ff | 477 | if (!bpf_prog_is_dev_bound(prog->aux) || !bpf_map_is_dev_bound(map)) |
a3884572 | 478 | return false; |
a3884572 JK |
479 | |
480 | down_read(&bpf_devs_lock); | |
481 | offload = prog->aux->offload; | |
482 | offmap = map_to_offmap(map); | |
483 | ||
484 | ret = offload && offload->netdev == offmap->netdev; | |
485 | up_read(&bpf_devs_lock); | |
486 | ||
487 | return ret; | |
488 | } | |
489 | ||
490 | static void bpf_offload_orphan_all_progs(struct net_device *netdev) | |
491 | { | |
492 | struct bpf_prog_offload *offload, *tmp; | |
493 | ||
494 | list_for_each_entry_safe(offload, tmp, &bpf_prog_offload_devs, offloads) | |
495 | if (offload->netdev == netdev) | |
496 | __bpf_prog_offload_destroy(offload->prog); | |
497 | } | |
498 | ||
499 | static void bpf_offload_orphan_all_maps(struct net_device *netdev) | |
500 | { | |
501 | struct bpf_offloaded_map *offmap, *tmp; | |
502 | ||
503 | list_for_each_entry_safe(offmap, tmp, &bpf_map_offload_devs, offloads) | |
504 | if (offmap->netdev == netdev) | |
505 | __bpf_map_offload_destroy(offmap); | |
506 | } | |
507 | ||
ab3f0063 JK |
508 | static int bpf_offload_notification(struct notifier_block *notifier, |
509 | ulong event, void *ptr) | |
510 | { | |
511 | struct net_device *netdev = netdev_notifier_info_to_dev(ptr); | |
ab3f0063 JK |
512 | |
513 | ASSERT_RTNL(); | |
514 | ||
515 | switch (event) { | |
516 | case NETDEV_UNREGISTER: | |
62c71b45 JK |
517 | /* ignore namespace changes */ |
518 | if (netdev->reg_state != NETREG_UNREGISTERING) | |
519 | break; | |
520 | ||
e0d3974a | 521 | down_write(&bpf_devs_lock); |
a3884572 JK |
522 | bpf_offload_orphan_all_progs(netdev); |
523 | bpf_offload_orphan_all_maps(netdev); | |
e0d3974a | 524 | up_write(&bpf_devs_lock); |
ab3f0063 JK |
525 | break; |
526 | default: | |
527 | break; | |
528 | } | |
529 | return NOTIFY_OK; | |
530 | } | |
531 | ||
532 | static struct notifier_block bpf_offload_notifier = { | |
533 | .notifier_call = bpf_offload_notification, | |
534 | }; | |
535 | ||
536 | static int __init bpf_offload_init(void) | |
537 | { | |
538 | register_netdevice_notifier(&bpf_offload_notifier); | |
539 | return 0; | |
540 | } | |
541 | ||
542 | subsys_initcall(bpf_offload_init); |