]>
Commit | Line | Data |
---|---|---|
a39e17b2 | 1 | /* |
0cd3cbed | 2 | * Copyright (C) 2017-2018 Netronome Systems, Inc. |
a39e17b2 JK |
3 | * |
4 | * This software is licensed under the GNU General License Version 2, | |
5 | * June 1991 as shown in the file COPYING in the top-level directory of this | |
6 | * source tree. | |
7 | * | |
8 | * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" | |
9 | * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, | |
10 | * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS | |
11 | * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE | |
12 | * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME | |
13 | * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. | |
14 | */ | |
15 | ||
ab3f0063 JK |
16 | #include <linux/bpf.h> |
17 | #include <linux/bpf_verifier.h> | |
18 | #include <linux/bug.h> | |
675fc275 | 19 | #include <linux/kdev_t.h> |
ab3f0063 | 20 | #include <linux/list.h> |
9fd7c555 | 21 | #include <linux/lockdep.h> |
ab3f0063 JK |
22 | #include <linux/netdevice.h> |
23 | #include <linux/printk.h> | |
675fc275 | 24 | #include <linux/proc_ns.h> |
9fd7c555 | 25 | #include <linux/rhashtable.h> |
ab3f0063 | 26 | #include <linux/rtnetlink.h> |
e0d3974a | 27 | #include <linux/rwsem.h> |
ab3f0063 | 28 | |
9fd7c555 | 29 | /* Protects offdevs, members of bpf_offload_netdev and offload members |
a3884572 | 30 | * of all progs. |
e0d3974a JK |
31 | * RTNL lock cannot be taken when holding this lock. |
32 | */ | |
33 | static DECLARE_RWSEM(bpf_devs_lock); | |
9fd7c555 | 34 | |
602144c2 | 35 | struct bpf_offload_dev { |
1385d755 | 36 | const struct bpf_prog_offload_ops *ops; |
602144c2 JK |
37 | struct list_head netdevs; |
38 | }; | |
39 | ||
9fd7c555 JK |
40 | struct bpf_offload_netdev { |
41 | struct rhash_head l; | |
42 | struct net_device *netdev; | |
602144c2 | 43 | struct bpf_offload_dev *offdev; |
9fd7c555 JK |
44 | struct list_head progs; |
45 | struct list_head maps; | |
602144c2 | 46 | struct list_head offdev_netdevs; |
9fd7c555 JK |
47 | }; |
48 | ||
49 | static const struct rhashtable_params offdevs_params = { | |
50 | .nelem_hint = 4, | |
51 | .key_len = sizeof(struct net_device *), | |
52 | .key_offset = offsetof(struct bpf_offload_netdev, netdev), | |
53 | .head_offset = offsetof(struct bpf_offload_netdev, l), | |
54 | .automatic_shrinking = true, | |
55 | }; | |
56 | ||
57 | static struct rhashtable offdevs; | |
58 | static bool offdevs_inited; | |
ab3f0063 | 59 | |
5bc2d55c JK |
60 | static int bpf_dev_offload_check(struct net_device *netdev) |
61 | { | |
62 | if (!netdev) | |
63 | return -EINVAL; | |
64 | if (!netdev->netdev_ops->ndo_bpf) | |
65 | return -EOPNOTSUPP; | |
66 | return 0; | |
67 | } | |
68 | ||
9fd7c555 JK |
69 | static struct bpf_offload_netdev * |
70 | bpf_offload_find_netdev(struct net_device *netdev) | |
71 | { | |
72 | lockdep_assert_held(&bpf_devs_lock); | |
73 | ||
74 | if (!offdevs_inited) | |
75 | return NULL; | |
76 | return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); | |
77 | } | |
78 | ||
ab3f0063 JK |
79 | int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr) |
80 | { | |
9fd7c555 | 81 | struct bpf_offload_netdev *ondev; |
0a9c1991 | 82 | struct bpf_prog_offload *offload; |
5bc2d55c | 83 | int err; |
ab3f0063 | 84 | |
649f11dc JK |
85 | if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS && |
86 | attr->prog_type != BPF_PROG_TYPE_XDP) | |
87 | return -EINVAL; | |
ab3f0063 JK |
88 | |
89 | if (attr->prog_flags) | |
90 | return -EINVAL; | |
91 | ||
92 | offload = kzalloc(sizeof(*offload), GFP_USER); | |
93 | if (!offload) | |
94 | return -ENOMEM; | |
95 | ||
96 | offload->prog = prog; | |
ab3f0063 | 97 | |
e0d3974a JK |
98 | offload->netdev = dev_get_by_index(current->nsproxy->net_ns, |
99 | attr->prog_ifindex); | |
5bc2d55c JK |
100 | err = bpf_dev_offload_check(offload->netdev); |
101 | if (err) | |
102 | goto err_maybe_put; | |
ab3f0063 | 103 | |
e0d3974a | 104 | down_write(&bpf_devs_lock); |
9fd7c555 JK |
105 | ondev = bpf_offload_find_netdev(offload->netdev); |
106 | if (!ondev) { | |
5bc2d55c | 107 | err = -EINVAL; |
e0d3974a | 108 | goto err_unlock; |
5bc2d55c | 109 | } |
341b3e7b | 110 | offload->offdev = ondev->offdev; |
ab3f0063 | 111 | prog->aux->offload = offload; |
9fd7c555 | 112 | list_add_tail(&offload->offloads, &ondev->progs); |
e0d3974a JK |
113 | dev_put(offload->netdev); |
114 | up_write(&bpf_devs_lock); | |
ab3f0063 JK |
115 | |
116 | return 0; | |
e0d3974a JK |
117 | err_unlock: |
118 | up_write(&bpf_devs_lock); | |
5bc2d55c JK |
119 | err_maybe_put: |
120 | if (offload->netdev) | |
121 | dev_put(offload->netdev); | |
e0d3974a | 122 | kfree(offload); |
5bc2d55c | 123 | return err; |
ab3f0063 JK |
124 | } |
125 | ||
ab3f0063 JK |
126 | int bpf_prog_offload_verifier_prep(struct bpf_verifier_env *env) |
127 | { | |
00db12c3 QM |
128 | struct bpf_prog_offload *offload; |
129 | int ret = -ENODEV; | |
ab3f0063 | 130 | |
00db12c3 QM |
131 | down_read(&bpf_devs_lock); |
132 | offload = env->prog->aux->offload; | |
133 | if (offload) | |
134 | ret = offload->offdev->ops->prepare(offload->netdev, env); | |
135 | offload->dev_state = !ret; | |
136 | up_read(&bpf_devs_lock); | |
ab3f0063 | 137 | |
00db12c3 | 138 | return ret; |
ab3f0063 JK |
139 | } |
140 | ||
cae1927c JK |
141 | int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, |
142 | int insn_idx, int prev_insn_idx) | |
143 | { | |
0a9c1991 | 144 | struct bpf_prog_offload *offload; |
cae1927c JK |
145 | int ret = -ENODEV; |
146 | ||
147 | down_read(&bpf_devs_lock); | |
148 | offload = env->prog->aux->offload; | |
ce3b9db4 | 149 | if (offload) |
341b3e7b QM |
150 | ret = offload->offdev->ops->insn_hook(env, insn_idx, |
151 | prev_insn_idx); | |
cae1927c JK |
152 | up_read(&bpf_devs_lock); |
153 | ||
154 | return ret; | |
155 | } | |
156 | ||
c941ce9c QM |
157 | int bpf_prog_offload_finalize(struct bpf_verifier_env *env) |
158 | { | |
159 | struct bpf_prog_offload *offload; | |
160 | int ret = -ENODEV; | |
161 | ||
162 | down_read(&bpf_devs_lock); | |
163 | offload = env->prog->aux->offload; | |
164 | if (offload) { | |
6dc18fa6 QM |
165 | if (offload->offdev->ops->finalize) |
166 | ret = offload->offdev->ops->finalize(env); | |
c941ce9c QM |
167 | else |
168 | ret = 0; | |
169 | } | |
170 | up_read(&bpf_devs_lock); | |
171 | ||
172 | return ret; | |
173 | } | |
174 | ||
ab3f0063 JK |
175 | static void __bpf_prog_offload_destroy(struct bpf_prog *prog) |
176 | { | |
0a9c1991 | 177 | struct bpf_prog_offload *offload = prog->aux->offload; |
ab3f0063 | 178 | |
ab3f0063 | 179 | if (offload->dev_state) |
eb911947 | 180 | offload->offdev->ops->destroy(prog); |
ab3f0063 | 181 | |
ad8ad79f JK |
182 | /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ |
183 | bpf_prog_free_id(prog, true); | |
184 | ||
ab3f0063 | 185 | list_del_init(&offload->offloads); |
ce3b9db4 JK |
186 | kfree(offload); |
187 | prog->aux->offload = NULL; | |
ab3f0063 JK |
188 | } |
189 | ||
190 | void bpf_prog_offload_destroy(struct bpf_prog *prog) | |
191 | { | |
e0d3974a | 192 | down_write(&bpf_devs_lock); |
ce3b9db4 JK |
193 | if (prog->aux->offload) |
194 | __bpf_prog_offload_destroy(prog); | |
e0d3974a | 195 | up_write(&bpf_devs_lock); |
ab3f0063 JK |
196 | } |
197 | ||
198 | static int bpf_prog_offload_translate(struct bpf_prog *prog) | |
199 | { | |
b07ade27 QM |
200 | struct bpf_prog_offload *offload; |
201 | int ret = -ENODEV; | |
ab3f0063 | 202 | |
b07ade27 QM |
203 | down_read(&bpf_devs_lock); |
204 | offload = prog->aux->offload; | |
205 | if (offload) | |
206 | ret = offload->offdev->ops->translate(offload->netdev, prog); | |
207 | up_read(&bpf_devs_lock); | |
ab3f0063 JK |
208 | |
209 | return ret; | |
210 | } | |
211 | ||
212 | static unsigned int bpf_prog_warn_on_exec(const void *ctx, | |
213 | const struct bpf_insn *insn) | |
214 | { | |
215 | WARN(1, "attempt to execute device eBPF program on the host!"); | |
216 | return 0; | |
217 | } | |
218 | ||
219 | int bpf_prog_offload_compile(struct bpf_prog *prog) | |
220 | { | |
221 | prog->bpf_func = bpf_prog_warn_on_exec; | |
222 | ||
223 | return bpf_prog_offload_translate(prog); | |
224 | } | |
225 | ||
675fc275 JK |
226 | struct ns_get_path_bpf_prog_args { |
227 | struct bpf_prog *prog; | |
228 | struct bpf_prog_info *info; | |
229 | }; | |
230 | ||
231 | static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data) | |
232 | { | |
233 | struct ns_get_path_bpf_prog_args *args = private_data; | |
234 | struct bpf_prog_aux *aux = args->prog->aux; | |
235 | struct ns_common *ns; | |
236 | struct net *net; | |
237 | ||
238 | rtnl_lock(); | |
239 | down_read(&bpf_devs_lock); | |
240 | ||
241 | if (aux->offload) { | |
242 | args->info->ifindex = aux->offload->netdev->ifindex; | |
243 | net = dev_net(aux->offload->netdev); | |
244 | get_net(net); | |
245 | ns = &net->ns; | |
246 | } else { | |
247 | args->info->ifindex = 0; | |
248 | ns = NULL; | |
249 | } | |
250 | ||
251 | up_read(&bpf_devs_lock); | |
252 | rtnl_unlock(); | |
253 | ||
254 | return ns; | |
255 | } | |
256 | ||
257 | int bpf_prog_offload_info_fill(struct bpf_prog_info *info, | |
258 | struct bpf_prog *prog) | |
259 | { | |
260 | struct ns_get_path_bpf_prog_args args = { | |
261 | .prog = prog, | |
262 | .info = info, | |
263 | }; | |
fcfb126d | 264 | struct bpf_prog_aux *aux = prog->aux; |
675fc275 JK |
265 | struct inode *ns_inode; |
266 | struct path ns_path; | |
fcfb126d | 267 | char __user *uinsns; |
675fc275 | 268 | void *res; |
fcfb126d | 269 | u32 ulen; |
675fc275 JK |
270 | |
271 | res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args); | |
272 | if (IS_ERR(res)) { | |
273 | if (!info->ifindex) | |
274 | return -ENODEV; | |
275 | return PTR_ERR(res); | |
276 | } | |
277 | ||
fcfb126d JW |
278 | down_read(&bpf_devs_lock); |
279 | ||
280 | if (!aux->offload) { | |
281 | up_read(&bpf_devs_lock); | |
282 | return -ENODEV; | |
283 | } | |
284 | ||
285 | ulen = info->jited_prog_len; | |
286 | info->jited_prog_len = aux->offload->jited_len; | |
287 | if (info->jited_prog_len & ulen) { | |
288 | uinsns = u64_to_user_ptr(info->jited_prog_insns); | |
289 | ulen = min_t(u32, info->jited_prog_len, ulen); | |
290 | if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) { | |
291 | up_read(&bpf_devs_lock); | |
292 | return -EFAULT; | |
293 | } | |
294 | } | |
295 | ||
296 | up_read(&bpf_devs_lock); | |
297 | ||
675fc275 JK |
298 | ns_inode = ns_path.dentry->d_inode; |
299 | info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); | |
300 | info->netns_ino = ns_inode->i_ino; | |
301 | path_put(&ns_path); | |
302 | ||
303 | return 0; | |
304 | } | |
305 | ||
ab3f0063 JK |
306 | const struct bpf_prog_ops bpf_offload_prog_ops = { |
307 | }; | |
308 | ||
a3884572 JK |
309 | static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap, |
310 | enum bpf_netdev_command cmd) | |
311 | { | |
312 | struct netdev_bpf data = {}; | |
313 | struct net_device *netdev; | |
314 | ||
315 | ASSERT_RTNL(); | |
316 | ||
317 | data.command = cmd; | |
318 | data.offmap = offmap; | |
319 | /* Caller must make sure netdev is valid */ | |
320 | netdev = offmap->netdev; | |
321 | ||
322 | return netdev->netdev_ops->ndo_bpf(netdev, &data); | |
323 | } | |
324 | ||
325 | struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) | |
326 | { | |
327 | struct net *net = current->nsproxy->net_ns; | |
9fd7c555 | 328 | struct bpf_offload_netdev *ondev; |
a3884572 JK |
329 | struct bpf_offloaded_map *offmap; |
330 | int err; | |
331 | ||
332 | if (!capable(CAP_SYS_ADMIN)) | |
333 | return ERR_PTR(-EPERM); | |
7a0ef693 JK |
334 | if (attr->map_type != BPF_MAP_TYPE_ARRAY && |
335 | attr->map_type != BPF_MAP_TYPE_HASH) | |
a3884572 JK |
336 | return ERR_PTR(-EINVAL); |
337 | ||
338 | offmap = kzalloc(sizeof(*offmap), GFP_USER); | |
339 | if (!offmap) | |
340 | return ERR_PTR(-ENOMEM); | |
341 | ||
342 | bpf_map_init_from_attr(&offmap->map, attr); | |
343 | ||
344 | rtnl_lock(); | |
345 | down_write(&bpf_devs_lock); | |
346 | offmap->netdev = __dev_get_by_index(net, attr->map_ifindex); | |
347 | err = bpf_dev_offload_check(offmap->netdev); | |
348 | if (err) | |
349 | goto err_unlock; | |
350 | ||
9fd7c555 JK |
351 | ondev = bpf_offload_find_netdev(offmap->netdev); |
352 | if (!ondev) { | |
353 | err = -EINVAL; | |
354 | goto err_unlock; | |
355 | } | |
356 | ||
a3884572 JK |
357 | err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC); |
358 | if (err) | |
359 | goto err_unlock; | |
360 | ||
9fd7c555 | 361 | list_add_tail(&offmap->offloads, &ondev->maps); |
a3884572 JK |
362 | up_write(&bpf_devs_lock); |
363 | rtnl_unlock(); | |
364 | ||
365 | return &offmap->map; | |
366 | ||
367 | err_unlock: | |
368 | up_write(&bpf_devs_lock); | |
369 | rtnl_unlock(); | |
370 | kfree(offmap); | |
371 | return ERR_PTR(err); | |
372 | } | |
373 | ||
374 | static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap) | |
375 | { | |
376 | WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE)); | |
377 | /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */ | |
378 | bpf_map_free_id(&offmap->map, true); | |
379 | list_del_init(&offmap->offloads); | |
380 | offmap->netdev = NULL; | |
381 | } | |
382 | ||
383 | void bpf_map_offload_map_free(struct bpf_map *map) | |
384 | { | |
385 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
386 | ||
387 | rtnl_lock(); | |
388 | down_write(&bpf_devs_lock); | |
389 | if (offmap->netdev) | |
390 | __bpf_map_offload_destroy(offmap); | |
391 | up_write(&bpf_devs_lock); | |
392 | rtnl_unlock(); | |
393 | ||
394 | kfree(offmap); | |
395 | } | |
396 | ||
397 | int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value) | |
398 | { | |
399 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
400 | int ret = -ENODEV; | |
401 | ||
402 | down_read(&bpf_devs_lock); | |
403 | if (offmap->netdev) | |
404 | ret = offmap->dev_ops->map_lookup_elem(offmap, key, value); | |
405 | up_read(&bpf_devs_lock); | |
406 | ||
407 | return ret; | |
408 | } | |
409 | ||
410 | int bpf_map_offload_update_elem(struct bpf_map *map, | |
411 | void *key, void *value, u64 flags) | |
412 | { | |
413 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
414 | int ret = -ENODEV; | |
415 | ||
416 | if (unlikely(flags > BPF_EXIST)) | |
417 | return -EINVAL; | |
418 | ||
419 | down_read(&bpf_devs_lock); | |
420 | if (offmap->netdev) | |
421 | ret = offmap->dev_ops->map_update_elem(offmap, key, value, | |
422 | flags); | |
423 | up_read(&bpf_devs_lock); | |
424 | ||
425 | return ret; | |
426 | } | |
427 | ||
428 | int bpf_map_offload_delete_elem(struct bpf_map *map, void *key) | |
429 | { | |
430 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
431 | int ret = -ENODEV; | |
432 | ||
433 | down_read(&bpf_devs_lock); | |
434 | if (offmap->netdev) | |
435 | ret = offmap->dev_ops->map_delete_elem(offmap, key); | |
436 | up_read(&bpf_devs_lock); | |
437 | ||
438 | return ret; | |
439 | } | |
440 | ||
441 | int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key) | |
442 | { | |
443 | struct bpf_offloaded_map *offmap = map_to_offmap(map); | |
444 | int ret = -ENODEV; | |
445 | ||
446 | down_read(&bpf_devs_lock); | |
447 | if (offmap->netdev) | |
448 | ret = offmap->dev_ops->map_get_next_key(offmap, key, next_key); | |
449 | up_read(&bpf_devs_lock); | |
450 | ||
451 | return ret; | |
452 | } | |
453 | ||
52775b33 JK |
454 | struct ns_get_path_bpf_map_args { |
455 | struct bpf_offloaded_map *offmap; | |
456 | struct bpf_map_info *info; | |
457 | }; | |
458 | ||
459 | static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data) | |
460 | { | |
461 | struct ns_get_path_bpf_map_args *args = private_data; | |
462 | struct ns_common *ns; | |
463 | struct net *net; | |
464 | ||
465 | rtnl_lock(); | |
466 | down_read(&bpf_devs_lock); | |
467 | ||
468 | if (args->offmap->netdev) { | |
469 | args->info->ifindex = args->offmap->netdev->ifindex; | |
470 | net = dev_net(args->offmap->netdev); | |
471 | get_net(net); | |
472 | ns = &net->ns; | |
473 | } else { | |
474 | args->info->ifindex = 0; | |
475 | ns = NULL; | |
476 | } | |
477 | ||
478 | up_read(&bpf_devs_lock); | |
479 | rtnl_unlock(); | |
480 | ||
481 | return ns; | |
482 | } | |
483 | ||
484 | int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map) | |
485 | { | |
486 | struct ns_get_path_bpf_map_args args = { | |
487 | .offmap = map_to_offmap(map), | |
488 | .info = info, | |
489 | }; | |
490 | struct inode *ns_inode; | |
491 | struct path ns_path; | |
492 | void *res; | |
493 | ||
494 | res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args); | |
495 | if (IS_ERR(res)) { | |
496 | if (!info->ifindex) | |
497 | return -ENODEV; | |
498 | return PTR_ERR(res); | |
499 | } | |
500 | ||
501 | ns_inode = ns_path.dentry->d_inode; | |
502 | info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); | |
503 | info->netns_ino = ns_inode->i_ino; | |
504 | path_put(&ns_path); | |
505 | ||
506 | return 0; | |
507 | } | |
508 | ||
fd4f227d JK |
509 | static bool __bpf_offload_dev_match(struct bpf_prog *prog, |
510 | struct net_device *netdev) | |
a3884572 | 511 | { |
fd4f227d | 512 | struct bpf_offload_netdev *ondev1, *ondev2; |
a3884572 | 513 | struct bpf_prog_offload *offload; |
a3884572 | 514 | |
0cd3cbed | 515 | if (!bpf_prog_is_dev_bound(prog->aux)) |
a3884572 | 516 | return false; |
a3884572 | 517 | |
a3884572 | 518 | offload = prog->aux->offload; |
fd4f227d JK |
519 | if (!offload) |
520 | return false; | |
521 | if (offload->netdev == netdev) | |
522 | return true; | |
523 | ||
524 | ondev1 = bpf_offload_find_netdev(offload->netdev); | |
525 | ondev2 = bpf_offload_find_netdev(netdev); | |
526 | ||
527 | return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev; | |
528 | } | |
529 | ||
530 | bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev) | |
531 | { | |
532 | bool ret; | |
533 | ||
534 | down_read(&bpf_devs_lock); | |
535 | ret = __bpf_offload_dev_match(prog, netdev); | |
536 | up_read(&bpf_devs_lock); | |
537 | ||
538 | return ret; | |
539 | } | |
540 | EXPORT_SYMBOL_GPL(bpf_offload_dev_match); | |
541 | ||
542 | bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map) | |
543 | { | |
544 | struct bpf_offloaded_map *offmap; | |
545 | bool ret; | |
546 | ||
547 | if (!bpf_map_is_dev_bound(map)) | |
548 | return bpf_map_offload_neutral(map); | |
a3884572 JK |
549 | offmap = map_to_offmap(map); |
550 | ||
fd4f227d JK |
551 | down_read(&bpf_devs_lock); |
552 | ret = __bpf_offload_dev_match(prog, offmap->netdev); | |
a3884572 JK |
553 | up_read(&bpf_devs_lock); |
554 | ||
555 | return ret; | |
556 | } | |
557 | ||
602144c2 JK |
558 | int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, |
559 | struct net_device *netdev) | |
a3884572 | 560 | { |
9fd7c555 JK |
561 | struct bpf_offload_netdev *ondev; |
562 | int err; | |
a3884572 | 563 | |
9fd7c555 JK |
564 | ondev = kzalloc(sizeof(*ondev), GFP_KERNEL); |
565 | if (!ondev) | |
566 | return -ENOMEM; | |
567 | ||
568 | ondev->netdev = netdev; | |
602144c2 | 569 | ondev->offdev = offdev; |
9fd7c555 JK |
570 | INIT_LIST_HEAD(&ondev->progs); |
571 | INIT_LIST_HEAD(&ondev->maps); | |
572 | ||
573 | down_write(&bpf_devs_lock); | |
574 | err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params); | |
575 | if (err) { | |
576 | netdev_warn(netdev, "failed to register for BPF offload\n"); | |
577 | goto err_unlock_free; | |
578 | } | |
a3884572 | 579 | |
602144c2 | 580 | list_add(&ondev->offdev_netdevs, &offdev->netdevs); |
9fd7c555 JK |
581 | up_write(&bpf_devs_lock); |
582 | return 0; | |
583 | ||
584 | err_unlock_free: | |
585 | up_write(&bpf_devs_lock); | |
586 | kfree(ondev); | |
587 | return err; | |
a3884572 | 588 | } |
9fd7c555 | 589 | EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register); |
a3884572 | 590 | |
602144c2 JK |
591 | void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, |
592 | struct net_device *netdev) | |
ab3f0063 | 593 | { |
602144c2 | 594 | struct bpf_offload_netdev *ondev, *altdev; |
9fd7c555 JK |
595 | struct bpf_offloaded_map *offmap, *mtmp; |
596 | struct bpf_prog_offload *offload, *ptmp; | |
ab3f0063 JK |
597 | |
598 | ASSERT_RTNL(); | |
599 | ||
9fd7c555 JK |
600 | down_write(&bpf_devs_lock); |
601 | ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); | |
602 | if (WARN_ON(!ondev)) | |
603 | goto unlock; | |
ab3f0063 | 604 | |
9fd7c555 | 605 | WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params)); |
602144c2 JK |
606 | list_del(&ondev->offdev_netdevs); |
607 | ||
608 | /* Try to move the objects to another netdev of the device */ | |
609 | altdev = list_first_entry_or_null(&offdev->netdevs, | |
610 | struct bpf_offload_netdev, | |
611 | offdev_netdevs); | |
612 | if (altdev) { | |
613 | list_for_each_entry(offload, &ondev->progs, offloads) | |
614 | offload->netdev = altdev->netdev; | |
615 | list_splice_init(&ondev->progs, &altdev->progs); | |
616 | ||
617 | list_for_each_entry(offmap, &ondev->maps, offloads) | |
618 | offmap->netdev = altdev->netdev; | |
619 | list_splice_init(&ondev->maps, &altdev->maps); | |
620 | } else { | |
621 | list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads) | |
622 | __bpf_prog_offload_destroy(offload->prog); | |
623 | list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads) | |
624 | __bpf_map_offload_destroy(offmap); | |
625 | } | |
ab3f0063 | 626 | |
9fd7c555 JK |
627 | WARN_ON(!list_empty(&ondev->progs)); |
628 | WARN_ON(!list_empty(&ondev->maps)); | |
629 | kfree(ondev); | |
630 | unlock: | |
631 | up_write(&bpf_devs_lock); | |
632 | } | |
633 | EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister); | |
602144c2 | 634 | |
1385d755 QM |
635 | struct bpf_offload_dev * |
636 | bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops) | |
602144c2 JK |
637 | { |
638 | struct bpf_offload_dev *offdev; | |
639 | int err; | |
640 | ||
641 | down_write(&bpf_devs_lock); | |
642 | if (!offdevs_inited) { | |
643 | err = rhashtable_init(&offdevs, &offdevs_params); | |
644 | if (err) | |
645 | return ERR_PTR(err); | |
646 | offdevs_inited = true; | |
647 | } | |
648 | up_write(&bpf_devs_lock); | |
649 | ||
650 | offdev = kzalloc(sizeof(*offdev), GFP_KERNEL); | |
651 | if (!offdev) | |
652 | return ERR_PTR(-ENOMEM); | |
653 | ||
1385d755 | 654 | offdev->ops = ops; |
602144c2 JK |
655 | INIT_LIST_HEAD(&offdev->netdevs); |
656 | ||
657 | return offdev; | |
658 | } | |
659 | EXPORT_SYMBOL_GPL(bpf_offload_dev_create); | |
660 | ||
661 | void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev) | |
662 | { | |
663 | WARN_ON(!list_empty(&offdev->netdevs)); | |
664 | kfree(offdev); | |
665 | } | |
666 | EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy); |