]>
Commit | Line | Data |
---|---|---|
a39e17b2 | 1 | /* |
0cd3cbed | 2 | * Copyright (C) 2017-2018 Netronome Systems, Inc. |
a39e17b2 JK |
3 | * |
4 | * This software is licensed under the GNU General License Version 2, | |
5 | * June 1991 as shown in the file COPYING in the top-level directory of this | |
6 | * source tree. | |
7 | * | |
8 | * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" | |
9 | * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, | |
10 | * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS | |
11 | * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE | |
12 | * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME | |
13 | * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. | |
14 | */ | |
15 | ||
ab3f0063 JK |
16 | #include <linux/bpf.h> |
17 | #include <linux/bpf_verifier.h> | |
18 | #include <linux/bug.h> | |
675fc275 | 19 | #include <linux/kdev_t.h> |
ab3f0063 | 20 | #include <linux/list.h> |
9fd7c555 | 21 | #include <linux/lockdep.h> |
ab3f0063 JK |
22 | #include <linux/netdevice.h> |
23 | #include <linux/printk.h> | |
675fc275 | 24 | #include <linux/proc_ns.h> |
9fd7c555 | 25 | #include <linux/rhashtable.h> |
ab3f0063 | 26 | #include <linux/rtnetlink.h> |
e0d3974a | 27 | #include <linux/rwsem.h> |
ab3f0063 | 28 | |
9fd7c555 | 29 | /* Protects offdevs, members of bpf_offload_netdev and offload members |
a3884572 | 30 | * of all progs. |
e0d3974a JK |
31 | * RTNL lock cannot be taken when holding this lock. |
32 | */ | |
33 | static DECLARE_RWSEM(bpf_devs_lock); | |
9fd7c555 | 34 | |
602144c2 | 35 | struct bpf_offload_dev { |
1385d755 | 36 | const struct bpf_prog_offload_ops *ops; |
602144c2 | 37 | struct list_head netdevs; |
dd27c2e3 | 38 | void *priv; |
602144c2 JK |
39 | }; |
40 | ||
9fd7c555 JK |
41 | struct bpf_offload_netdev { |
42 | struct rhash_head l; | |
43 | struct net_device *netdev; | |
602144c2 | 44 | struct bpf_offload_dev *offdev; |
9fd7c555 JK |
45 | struct list_head progs; |
46 | struct list_head maps; | |
602144c2 | 47 | struct list_head offdev_netdevs; |
9fd7c555 JK |
48 | }; |
49 | ||
50 | static const struct rhashtable_params offdevs_params = { | |
51 | .nelem_hint = 4, | |
52 | .key_len = sizeof(struct net_device *), | |
53 | .key_offset = offsetof(struct bpf_offload_netdev, netdev), | |
54 | .head_offset = offsetof(struct bpf_offload_netdev, l), | |
55 | .automatic_shrinking = true, | |
56 | }; | |
57 | ||
58 | static struct rhashtable offdevs; | |
59 | static bool offdevs_inited; | |
ab3f0063 | 60 | |
5bc2d55c JK |
61 | static int bpf_dev_offload_check(struct net_device *netdev) |
62 | { | |
63 | if (!netdev) | |
64 | return -EINVAL; | |
65 | if (!netdev->netdev_ops->ndo_bpf) | |
66 | return -EOPNOTSUPP; | |
67 | return 0; | |
68 | } | |
69 | ||
9fd7c555 JK |
70 | static struct bpf_offload_netdev * |
71 | bpf_offload_find_netdev(struct net_device *netdev) | |
72 | { | |
73 | lockdep_assert_held(&bpf_devs_lock); | |
74 | ||
75 | if (!offdevs_inited) | |
76 | return NULL; | |
77 | return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); | |
78 | } | |
79 | ||
ab3f0063 JK |
80 | int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr) |
81 | { | |
9fd7c555 | 82 | struct bpf_offload_netdev *ondev; |
0a9c1991 | 83 | struct bpf_prog_offload *offload; |
5bc2d55c | 84 | int err; |
ab3f0063 | 85 | |
649f11dc JK |
86 | if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS && |
87 | attr->prog_type != BPF_PROG_TYPE_XDP) | |
88 | return -EINVAL; | |
ab3f0063 JK |
89 | |
90 | if (attr->prog_flags) | |
91 | return -EINVAL; | |
92 | ||
93 | offload = kzalloc(sizeof(*offload), GFP_USER); | |
94 | if (!offload) | |
95 | return -ENOMEM; | |
96 | ||
97 | offload->prog = prog; | |
ab3f0063 | 98 | |
e0d3974a JK |
99 | offload->netdev = dev_get_by_index(current->nsproxy->net_ns, |
100 | attr->prog_ifindex); | |
5bc2d55c JK |
101 | err = bpf_dev_offload_check(offload->netdev); |
102 | if (err) | |
103 | goto err_maybe_put; | |
ab3f0063 | 104 | |
e0d3974a | 105 | down_write(&bpf_devs_lock); |
9fd7c555 JK |
106 | ondev = bpf_offload_find_netdev(offload->netdev); |
107 | if (!ondev) { | |
5bc2d55c | 108 | err = -EINVAL; |
e0d3974a | 109 | goto err_unlock; |
5bc2d55c | 110 | } |
341b3e7b | 111 | offload->offdev = ondev->offdev; |
ab3f0063 | 112 | prog->aux->offload = offload; |
9fd7c555 | 113 | list_add_tail(&offload->offloads, &ondev->progs); |
e0d3974a JK |
114 | dev_put(offload->netdev); |
115 | up_write(&bpf_devs_lock); | |
ab3f0063 JK |
116 | |
117 | return 0; | |
e0d3974a JK |
118 | err_unlock: |
119 | up_write(&bpf_devs_lock); | |
5bc2d55c JK |
120 | err_maybe_put: |
121 | if (offload->netdev) | |
122 | dev_put(offload->netdev); | |
e0d3974a | 123 | kfree(offload); |
5bc2d55c | 124 | return err; |
ab3f0063 JK |
125 | } |
126 | ||
a40a2632 | 127 | int bpf_prog_offload_verifier_prep(struct bpf_prog *prog) |
ab3f0063 | 128 | { |
00db12c3 QM |
129 | struct bpf_prog_offload *offload; |
130 | int ret = -ENODEV; | |
ab3f0063 | 131 | |
00db12c3 | 132 | down_read(&bpf_devs_lock); |
a40a2632 | 133 | offload = prog->aux->offload; |
592ee43f | 134 | if (offload) { |
16a8cb5c | 135 | ret = offload->offdev->ops->prepare(prog); |
592ee43f CIK |
136 | offload->dev_state = !ret; |
137 | } | |
00db12c3 | 138 | up_read(&bpf_devs_lock); |
ab3f0063 | 139 | |
00db12c3 | 140 | return ret; |
ab3f0063 JK |
141 | } |
142 | ||
cae1927c JK |
143 | int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, |
144 | int insn_idx, int prev_insn_idx) | |
145 | { | |
0a9c1991 | 146 | struct bpf_prog_offload *offload; |
cae1927c JK |
147 | int ret = -ENODEV; |
148 | ||
149 | down_read(&bpf_devs_lock); | |
150 | offload = env->prog->aux->offload; | |
ce3b9db4 | 151 | if (offload) |
341b3e7b QM |
152 | ret = offload->offdev->ops->insn_hook(env, insn_idx, |
153 | prev_insn_idx); | |
cae1927c JK |
154 | up_read(&bpf_devs_lock); |
155 | ||
156 | return ret; | |
157 | } | |
158 | ||
c941ce9c QM |
159 | int bpf_prog_offload_finalize(struct bpf_verifier_env *env) |
160 | { | |
161 | struct bpf_prog_offload *offload; | |
162 | int ret = -ENODEV; | |
163 | ||
164 | down_read(&bpf_devs_lock); | |
165 | offload = env->prog->aux->offload; | |
166 | if (offload) { | |
6dc18fa6 QM |
167 | if (offload->offdev->ops->finalize) |
168 | ret = offload->offdev->ops->finalize(env); | |
c941ce9c QM |
169 | else |
170 | ret = 0; | |
171 | } | |
172 | up_read(&bpf_devs_lock); | |
173 | ||
174 | return ret; | |
175 | } | |
176 | ||
08ca90af JK |
177 | void |
178 | bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off, | |
179 | struct bpf_insn *insn) | |
180 | { | |
181 | const struct bpf_prog_offload_ops *ops; | |
182 | struct bpf_prog_offload *offload; | |
183 | int ret = -EOPNOTSUPP; | |
184 | ||
185 | down_read(&bpf_devs_lock); | |
186 | offload = env->prog->aux->offload; | |
187 | if (offload) { | |
188 | ops = offload->offdev->ops; | |
189 | if (!offload->opt_failed && ops->replace_insn) | |
190 | ret = ops->replace_insn(env, off, insn); | |
191 | offload->opt_failed |= ret; | |
192 | } | |
193 | up_read(&bpf_devs_lock); | |
194 | } | |
195 | ||
196 | void | |
197 | bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) | |
198 | { | |
199 | struct bpf_prog_offload *offload; | |
200 | int ret = -EOPNOTSUPP; | |
201 | ||
202 | down_read(&bpf_devs_lock); | |
203 | offload = env->prog->aux->offload; | |
204 | if (offload) { | |
205 | if (!offload->opt_failed && offload->offdev->ops->remove_insns) | |
206 | ret = offload->offdev->ops->remove_insns(env, off, cnt); | |
207 | offload->opt_failed |= ret; | |
208 | } | |
209 | up_read(&bpf_devs_lock); | |
210 | } | |
211 | ||
ab3f0063 JK |
212 | static void __bpf_prog_offload_destroy(struct bpf_prog *prog) |
213 | { | |
0a9c1991 | 214 | struct bpf_prog_offload *offload = prog->aux->offload; |
ab3f0063 | 215 | |
ab3f0063 | 216 | if (offload->dev_state) |
eb911947 | 217 | offload->offdev->ops->destroy(prog); |
ab3f0063 | 218 | |
ad8ad79f JK |
219 | /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ |
220 | bpf_prog_free_id(prog, true); | |
221 | ||
ab3f0063 | 222 | list_del_init(&offload->offloads); |
ce3b9db4 JK |
223 | kfree(offload); |
224 | prog->aux->offload = NULL; | |
ab3f0063 JK |
225 | } |
226 | ||
227 | void bpf_prog_offload_destroy(struct bpf_prog *prog) | |
228 | { | |
e0d3974a | 229 | down_write(&bpf_devs_lock); |
ce3b9db4 JK |
230 | if (prog->aux->offload) |
231 | __bpf_prog_offload_destroy(prog); | |
e0d3974a | 232 | up_write(&bpf_devs_lock); |
ab3f0063 JK |
233 | } |
234 | ||
235 | static int bpf_prog_offload_translate(struct bpf_prog *prog) | |
236 | { | |
b07ade27 QM |
237 | struct bpf_prog_offload *offload; |
238 | int ret = -ENODEV; | |
ab3f0063 | 239 | |
b07ade27 QM |
240 | down_read(&bpf_devs_lock); |
241 | offload = prog->aux->offload; | |
242 | if (offload) | |
16a8cb5c | 243 | ret = offload->offdev->ops->translate(prog); |
b07ade27 | 244 | up_read(&bpf_devs_lock); |
ab3f0063 JK |
245 | |
246 | return ret; | |
247 | } | |
248 | ||
249 | static unsigned int bpf_prog_warn_on_exec(const void *ctx, | |
250 | const struct bpf_insn *insn) | |
251 | { | |
252 | WARN(1, "attempt to execute device eBPF program on the host!"); | |
253 | return 0; | |
254 | } | |
255 | ||
256 | int bpf_prog_offload_compile(struct bpf_prog *prog) | |
257 | { | |
258 | prog->bpf_func = bpf_prog_warn_on_exec; | |
259 | ||
260 | return bpf_prog_offload_translate(prog); | |
261 | } | |
262 | ||
675fc275 JK |
263 | struct ns_get_path_bpf_prog_args { |
264 | struct bpf_prog *prog; | |
265 | struct bpf_prog_info *info; | |
266 | }; | |
267 | ||
268 | static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data) | |
269 | { | |
270 | struct ns_get_path_bpf_prog_args *args = private_data; | |
271 | struct bpf_prog_aux *aux = args->prog->aux; | |
272 | struct ns_common *ns; | |
273 | struct net *net; | |
274 | ||
275 | rtnl_lock(); | |
276 | down_read(&bpf_devs_lock); | |
277 | ||
278 | if (aux->offload) { | |
279 | args->info->ifindex = aux->offload->netdev->ifindex; | |
280 | net = dev_net(aux->offload->netdev); | |
281 | get_net(net); | |
282 | ns = &net->ns; | |
283 | } else { | |
284 | args->info->ifindex = 0; | |
285 | ns = NULL; | |
286 | } | |
287 | ||
288 | up_read(&bpf_devs_lock); | |
289 | rtnl_unlock(); | |
290 | ||
291 | return ns; | |
292 | } | |
293 | ||
294 | int bpf_prog_offload_info_fill(struct bpf_prog_info *info, | |
295 | struct bpf_prog *prog) | |
296 | { | |
297 | struct ns_get_path_bpf_prog_args args = { | |
298 | .prog = prog, | |
299 | .info = info, | |
300 | }; | |
fcfb126d | 301 | struct bpf_prog_aux *aux = prog->aux; |
675fc275 JK |
302 | struct inode *ns_inode; |
303 | struct path ns_path; | |
fcfb126d | 304 | char __user *uinsns; |
675fc275 | 305 | void *res; |
fcfb126d | 306 | u32 ulen; |
675fc275 JK |
307 | |
308 | res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args); | |
309 | if (IS_ERR(res)) { | |
310 | if (!info->ifindex) | |
311 | return -ENODEV; | |
312 | return PTR_ERR(res); | |
313 | } | |
314 | ||
fcfb126d JW |
315 | down_read(&bpf_devs_lock); |
316 | ||
317 | if (!aux->offload) { | |
318 | up_read(&bpf_devs_lock); | |
319 | return -ENODEV; | |
320 | } | |
321 | ||
322 | ulen = info->jited_prog_len; | |
323 | info->jited_prog_len = aux->offload->jited_len; | |
324 | if (info->jited_prog_len & ulen) { | |
325 | uinsns = u64_to_user_ptr(info->jited_prog_insns); | |
326 | ulen = min_t(u32, info->jited_prog_len, ulen); | |
327 | if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) { | |
328 | up_read(&bpf_devs_lock); | |
329 | return -EFAULT; | |
330 | } | |
331 | } | |
332 | ||
333 | up_read(&bpf_devs_lock); | |
334 | ||
675fc275 JK |
335 | ns_inode = ns_path.dentry->d_inode; |
336 | info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); | |
337 | info->netns_ino = ns_inode->i_ino; | |
338 | path_put(&ns_path); | |
339 | ||
340 | return 0; | |
341 | } | |
342 | ||
ab3f0063 JK |
343 | const struct bpf_prog_ops bpf_offload_prog_ops = { |
344 | }; | |
345 | ||
a3884572 JK |
346 | static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap, |
347 | enum bpf_netdev_command cmd) | |
348 | { | |
349 | struct netdev_bpf data = {}; | |
350 | struct net_device *netdev; | |
351 | ||
352 | ASSERT_RTNL(); | |
353 | ||
354 | data.command = cmd; | |
355 | data.offmap = offmap; | |
356 | /* Caller must make sure netdev is valid */ | |
357 | netdev = offmap->netdev; | |
358 | ||
359 | return netdev->netdev_ops->ndo_bpf(netdev, &data); | |
360 | } | |
361 | ||
362 | struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) | |
363 | { | |
364 | struct net *net = current->nsproxy->net_ns; | |
9fd7c555 | 365 | struct bpf_offload_netdev *ondev; |
a3884572 JK |
366 | struct bpf_offloaded_map *offmap; |
367 | int err; | |
368 | ||
369 | if (!capable(CAP_SYS_ADMIN)) | |
370 | return ERR_PTR(-EPERM); | |
7a0ef693 JK |
371 | if (attr->map_type != BPF_MAP_TYPE_ARRAY && |
372 | attr->map_type != BPF_MAP_TYPE_HASH) | |
a3884572 JK |
373 | return ERR_PTR(-EINVAL); |
374 | ||
375 | offmap = kzalloc(sizeof(*offmap), GFP_USER); | |
376 | if (!offmap) | |
377 | return ERR_PTR(-ENOMEM); | |
378 | ||
379 | bpf_map_init_from_attr(&offmap->map, attr); | |
380 | ||
381 | rtnl_lock(); | |
382 | down_write(&bpf_devs_lock); | |
383 | offmap->netdev = __dev_get_by_index(net, attr->map_ifindex); | |
384 | err = bpf_dev_offload_check(offmap->netdev); | |
385 | if (err) | |
386 | goto err_unlock; | |
387 | ||
9fd7c555 JK |
388 | ondev = bpf_offload_find_netdev(offmap->netdev); |
389 | if (!ondev) { | |
390 | err = -EINVAL; | |
391 | goto err_unlock; | |
392 | } | |
393 | ||
a3884572 JK |
394 | err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC); |
395 | if (err) | |
396 | goto err_unlock; | |
397 | ||
9fd7c555 | 398 | list_add_tail(&offmap->offloads, &ondev->maps); |
a3884572 JK |
399 | up_write(&bpf_devs_lock); |
400 | rtnl_unlock(); | |
401 | ||
402 | return &offmap->map; | |
403 | ||
404 | err_unlock: | |
405 | up_write(&bpf_devs_lock); | |
406 | rtnl_unlock(); | |
407 | kfree(offmap); | |
408 | return ERR_PTR(err); | |
409 | } | |
410 | ||
411 | static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap) | |
412 | { | |
413 | WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE)); | |
414 | /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */ | |
415 | bpf_map_free_id(&offmap->map, true); | |
416 | list_del_init(&offmap->offloads); | |
417 | offmap->netdev = NULL; | |
418 | } | |
419 | ||
420 | void bpf_map_offload_map_free(struct bpf_map *map) | |
421 | { | |
422 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
423 | ||
424 | rtnl_lock(); | |
425 | down_write(&bpf_devs_lock); | |
426 | if (offmap->netdev) | |
427 | __bpf_map_offload_destroy(offmap); | |
428 | up_write(&bpf_devs_lock); | |
429 | rtnl_unlock(); | |
430 | ||
431 | kfree(offmap); | |
432 | } | |
433 | ||
434 | int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value) | |
435 | { | |
436 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
437 | int ret = -ENODEV; | |
438 | ||
439 | down_read(&bpf_devs_lock); | |
440 | if (offmap->netdev) | |
441 | ret = offmap->dev_ops->map_lookup_elem(offmap, key, value); | |
442 | up_read(&bpf_devs_lock); | |
443 | ||
444 | return ret; | |
445 | } | |
446 | ||
447 | int bpf_map_offload_update_elem(struct bpf_map *map, | |
448 | void *key, void *value, u64 flags) | |
449 | { | |
450 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
451 | int ret = -ENODEV; | |
452 | ||
453 | if (unlikely(flags > BPF_EXIST)) | |
454 | return -EINVAL; | |
455 | ||
456 | down_read(&bpf_devs_lock); | |
457 | if (offmap->netdev) | |
458 | ret = offmap->dev_ops->map_update_elem(offmap, key, value, | |
459 | flags); | |
460 | up_read(&bpf_devs_lock); | |
461 | ||
462 | return ret; | |
463 | } | |
464 | ||
465 | int bpf_map_offload_delete_elem(struct bpf_map *map, void *key) | |
466 | { | |
467 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
468 | int ret = -ENODEV; | |
469 | ||
470 | down_read(&bpf_devs_lock); | |
471 | if (offmap->netdev) | |
472 | ret = offmap->dev_ops->map_delete_elem(offmap, key); | |
473 | up_read(&bpf_devs_lock); | |
474 | ||
475 | return ret; | |
476 | } | |
477 | ||
478 | int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key) | |
479 | { | |
480 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
481 | int ret = -ENODEV; | |
482 | ||
483 | down_read(&bpf_devs_lock); | |
484 | if (offmap->netdev) | |
485 | ret = offmap->dev_ops->map_get_next_key(offmap, key, next_key); | |
486 | up_read(&bpf_devs_lock); | |
487 | ||
488 | return ret; | |
489 | } | |
490 | ||
52775b33 JK |
491 | struct ns_get_path_bpf_map_args { |
492 | struct bpf_offloaded_map *offmap; | |
493 | struct bpf_map_info *info; | |
494 | }; | |
495 | ||
496 | static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data) | |
497 | { | |
498 | struct ns_get_path_bpf_map_args *args = private_data; | |
499 | struct ns_common *ns; | |
500 | struct net *net; | |
501 | ||
502 | rtnl_lock(); | |
503 | down_read(&bpf_devs_lock); | |
504 | ||
505 | if (args->offmap->netdev) { | |
506 | args->info->ifindex = args->offmap->netdev->ifindex; | |
507 | net = dev_net(args->offmap->netdev); | |
508 | get_net(net); | |
509 | ns = &net->ns; | |
510 | } else { | |
511 | args->info->ifindex = 0; | |
512 | ns = NULL; | |
513 | } | |
514 | ||
515 | up_read(&bpf_devs_lock); | |
516 | rtnl_unlock(); | |
517 | ||
518 | return ns; | |
519 | } | |
520 | ||
521 | int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map) | |
522 | { | |
523 | struct ns_get_path_bpf_map_args args = { | |
524 | .offmap = map_to_offmap(map), | |
525 | .info = info, | |
526 | }; | |
527 | struct inode *ns_inode; | |
528 | struct path ns_path; | |
529 | void *res; | |
530 | ||
531 | res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args); | |
532 | if (IS_ERR(res)) { | |
533 | if (!info->ifindex) | |
534 | return -ENODEV; | |
535 | return PTR_ERR(res); | |
536 | } | |
537 | ||
538 | ns_inode = ns_path.dentry->d_inode; | |
539 | info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); | |
540 | info->netns_ino = ns_inode->i_ino; | |
541 | path_put(&ns_path); | |
542 | ||
543 | return 0; | |
544 | } | |
545 | ||
fd4f227d JK |
546 | static bool __bpf_offload_dev_match(struct bpf_prog *prog, |
547 | struct net_device *netdev) | |
a3884572 | 548 | { |
fd4f227d | 549 | struct bpf_offload_netdev *ondev1, *ondev2; |
a3884572 | 550 | struct bpf_prog_offload *offload; |
a3884572 | 551 | |
0cd3cbed | 552 | if (!bpf_prog_is_dev_bound(prog->aux)) |
a3884572 | 553 | return false; |
a3884572 | 554 | |
a3884572 | 555 | offload = prog->aux->offload; |
fd4f227d JK |
556 | if (!offload) |
557 | return false; | |
558 | if (offload->netdev == netdev) | |
559 | return true; | |
560 | ||
561 | ondev1 = bpf_offload_find_netdev(offload->netdev); | |
562 | ondev2 = bpf_offload_find_netdev(netdev); | |
563 | ||
564 | return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev; | |
565 | } | |
566 | ||
567 | bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev) | |
568 | { | |
569 | bool ret; | |
570 | ||
571 | down_read(&bpf_devs_lock); | |
572 | ret = __bpf_offload_dev_match(prog, netdev); | |
573 | up_read(&bpf_devs_lock); | |
574 | ||
575 | return ret; | |
576 | } | |
577 | EXPORT_SYMBOL_GPL(bpf_offload_dev_match); | |
578 | ||
579 | bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map) | |
580 | { | |
581 | struct bpf_offloaded_map *offmap; | |
582 | bool ret; | |
583 | ||
584 | if (!bpf_map_is_dev_bound(map)) | |
585 | return bpf_map_offload_neutral(map); | |
a3884572 JK |
586 | offmap = map_to_offmap(map); |
587 | ||
fd4f227d JK |
588 | down_read(&bpf_devs_lock); |
589 | ret = __bpf_offload_dev_match(prog, offmap->netdev); | |
a3884572 JK |
590 | up_read(&bpf_devs_lock); |
591 | ||
592 | return ret; | |
593 | } | |
594 | ||
602144c2 JK |
595 | int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, |
596 | struct net_device *netdev) | |
a3884572 | 597 | { |
9fd7c555 JK |
598 | struct bpf_offload_netdev *ondev; |
599 | int err; | |
a3884572 | 600 | |
9fd7c555 JK |
601 | ondev = kzalloc(sizeof(*ondev), GFP_KERNEL); |
602 | if (!ondev) | |
603 | return -ENOMEM; | |
604 | ||
605 | ondev->netdev = netdev; | |
602144c2 | 606 | ondev->offdev = offdev; |
9fd7c555 JK |
607 | INIT_LIST_HEAD(&ondev->progs); |
608 | INIT_LIST_HEAD(&ondev->maps); | |
609 | ||
610 | down_write(&bpf_devs_lock); | |
611 | err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params); | |
612 | if (err) { | |
613 | netdev_warn(netdev, "failed to register for BPF offload\n"); | |
614 | goto err_unlock_free; | |
615 | } | |
a3884572 | 616 | |
602144c2 | 617 | list_add(&ondev->offdev_netdevs, &offdev->netdevs); |
9fd7c555 JK |
618 | up_write(&bpf_devs_lock); |
619 | return 0; | |
620 | ||
621 | err_unlock_free: | |
622 | up_write(&bpf_devs_lock); | |
623 | kfree(ondev); | |
624 | return err; | |
a3884572 | 625 | } |
9fd7c555 | 626 | EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register); |
a3884572 | 627 | |
602144c2 JK |
628 | void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, |
629 | struct net_device *netdev) | |
ab3f0063 | 630 | { |
602144c2 | 631 | struct bpf_offload_netdev *ondev, *altdev; |
9fd7c555 JK |
632 | struct bpf_offloaded_map *offmap, *mtmp; |
633 | struct bpf_prog_offload *offload, *ptmp; | |
ab3f0063 JK |
634 | |
635 | ASSERT_RTNL(); | |
636 | ||
9fd7c555 JK |
637 | down_write(&bpf_devs_lock); |
638 | ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); | |
639 | if (WARN_ON(!ondev)) | |
640 | goto unlock; | |
ab3f0063 | 641 | |
9fd7c555 | 642 | WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params)); |
602144c2 JK |
643 | list_del(&ondev->offdev_netdevs); |
644 | ||
645 | /* Try to move the objects to another netdev of the device */ | |
646 | altdev = list_first_entry_or_null(&offdev->netdevs, | |
647 | struct bpf_offload_netdev, | |
648 | offdev_netdevs); | |
649 | if (altdev) { | |
650 | list_for_each_entry(offload, &ondev->progs, offloads) | |
651 | offload->netdev = altdev->netdev; | |
652 | list_splice_init(&ondev->progs, &altdev->progs); | |
653 | ||
654 | list_for_each_entry(offmap, &ondev->maps, offloads) | |
655 | offmap->netdev = altdev->netdev; | |
656 | list_splice_init(&ondev->maps, &altdev->maps); | |
657 | } else { | |
658 | list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads) | |
659 | __bpf_prog_offload_destroy(offload->prog); | |
660 | list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads) | |
661 | __bpf_map_offload_destroy(offmap); | |
662 | } | |
ab3f0063 | 663 | |
9fd7c555 JK |
664 | WARN_ON(!list_empty(&ondev->progs)); |
665 | WARN_ON(!list_empty(&ondev->maps)); | |
666 | kfree(ondev); | |
667 | unlock: | |
668 | up_write(&bpf_devs_lock); | |
669 | } | |
670 | EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister); | |
602144c2 | 671 | |
1385d755 | 672 | struct bpf_offload_dev * |
dd27c2e3 | 673 | bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv) |
602144c2 JK |
674 | { |
675 | struct bpf_offload_dev *offdev; | |
676 | int err; | |
677 | ||
678 | down_write(&bpf_devs_lock); | |
679 | if (!offdevs_inited) { | |
680 | err = rhashtable_init(&offdevs, &offdevs_params); | |
d0fbb51d DC |
681 | if (err) { |
682 | up_write(&bpf_devs_lock); | |
602144c2 | 683 | return ERR_PTR(err); |
d0fbb51d | 684 | } |
602144c2 JK |
685 | offdevs_inited = true; |
686 | } | |
687 | up_write(&bpf_devs_lock); | |
688 | ||
689 | offdev = kzalloc(sizeof(*offdev), GFP_KERNEL); | |
690 | if (!offdev) | |
691 | return ERR_PTR(-ENOMEM); | |
692 | ||
1385d755 | 693 | offdev->ops = ops; |
dd27c2e3 | 694 | offdev->priv = priv; |
602144c2 JK |
695 | INIT_LIST_HEAD(&offdev->netdevs); |
696 | ||
697 | return offdev; | |
698 | } | |
699 | EXPORT_SYMBOL_GPL(bpf_offload_dev_create); | |
700 | ||
701 | void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev) | |
702 | { | |
703 | WARN_ON(!list_empty(&offdev->netdevs)); | |
704 | kfree(offdev); | |
705 | } | |
706 | EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy); | |
dd27c2e3 JK |
707 | |
708 | void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev) | |
709 | { | |
710 | return offdev->priv; | |
711 | } | |
712 | EXPORT_SYMBOL_GPL(bpf_offload_dev_priv); |