]>
Commit | Line | Data |
---|---|---|
a39e17b2 JK |
1 | /* |
2 | * Copyright (C) 2017 Netronome Systems, Inc. | |
3 | * | |
4 | * This software is licensed under the GNU General License Version 2, | |
5 | * June 1991 as shown in the file COPYING in the top-level directory of this | |
6 | * source tree. | |
7 | * | |
8 | * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" | |
9 | * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, | |
10 | * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS | |
11 | * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE | |
12 | * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME | |
13 | * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. | |
14 | */ | |
15 | ||
ab3f0063 JK |
16 | #include <linux/bpf.h> |
17 | #include <linux/bpf_verifier.h> | |
18 | #include <linux/bug.h> | |
675fc275 | 19 | #include <linux/kdev_t.h> |
ab3f0063 JK |
20 | #include <linux/list.h> |
21 | #include <linux/netdevice.h> | |
22 | #include <linux/printk.h> | |
675fc275 | 23 | #include <linux/proc_ns.h> |
ab3f0063 | 24 | #include <linux/rtnetlink.h> |
e0d3974a | 25 | #include <linux/rwsem.h> |
ab3f0063 | 26 | |
e0d3974a JK |
27 | /* Protects bpf_prog_offload_devs and offload members of all progs. |
28 | * RTNL lock cannot be taken when holding this lock. | |
29 | */ | |
30 | static DECLARE_RWSEM(bpf_devs_lock); | |
ab3f0063 JK |
31 | static LIST_HEAD(bpf_prog_offload_devs); |
32 | ||
33 | int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr) | |
34 | { | |
0a9c1991 | 35 | struct bpf_prog_offload *offload; |
ab3f0063 | 36 | |
649f11dc JK |
37 | if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS && |
38 | attr->prog_type != BPF_PROG_TYPE_XDP) | |
39 | return -EINVAL; | |
ab3f0063 JK |
40 | |
41 | if (attr->prog_flags) | |
42 | return -EINVAL; | |
43 | ||
44 | offload = kzalloc(sizeof(*offload), GFP_USER); | |
45 | if (!offload) | |
46 | return -ENOMEM; | |
47 | ||
48 | offload->prog = prog; | |
ab3f0063 | 49 | |
e0d3974a JK |
50 | offload->netdev = dev_get_by_index(current->nsproxy->net_ns, |
51 | attr->prog_ifindex); | |
52 | if (!offload->netdev) | |
53 | goto err_free; | |
ab3f0063 | 54 | |
e0d3974a JK |
55 | down_write(&bpf_devs_lock); |
56 | if (offload->netdev->reg_state != NETREG_REGISTERED) | |
57 | goto err_unlock; | |
ab3f0063 JK |
58 | prog->aux->offload = offload; |
59 | list_add_tail(&offload->offloads, &bpf_prog_offload_devs); | |
e0d3974a JK |
60 | dev_put(offload->netdev); |
61 | up_write(&bpf_devs_lock); | |
ab3f0063 JK |
62 | |
63 | return 0; | |
e0d3974a JK |
64 | err_unlock: |
65 | up_write(&bpf_devs_lock); | |
66 | dev_put(offload->netdev); | |
67 | err_free: | |
68 | kfree(offload); | |
69 | return -EINVAL; | |
ab3f0063 JK |
70 | } |
71 | ||
72 | static int __bpf_offload_ndo(struct bpf_prog *prog, enum bpf_netdev_command cmd, | |
73 | struct netdev_bpf *data) | |
74 | { | |
0a9c1991 | 75 | struct bpf_prog_offload *offload = prog->aux->offload; |
ce3b9db4 | 76 | struct net_device *netdev; |
ab3f0063 JK |
77 | |
78 | ASSERT_RTNL(); | |
79 | ||
ce3b9db4 | 80 | if (!offload) |
ab3f0063 | 81 | return -ENODEV; |
ce3b9db4 | 82 | netdev = offload->netdev; |
ab3f0063 JK |
83 | if (!netdev->netdev_ops->ndo_bpf) |
84 | return -EOPNOTSUPP; | |
85 | ||
86 | data->command = cmd; | |
87 | ||
88 | return netdev->netdev_ops->ndo_bpf(netdev, data); | |
89 | } | |
90 | ||
91 | int bpf_prog_offload_verifier_prep(struct bpf_verifier_env *env) | |
92 | { | |
93 | struct netdev_bpf data = {}; | |
94 | int err; | |
95 | ||
96 | data.verifier.prog = env->prog; | |
97 | ||
98 | rtnl_lock(); | |
99 | err = __bpf_offload_ndo(env->prog, BPF_OFFLOAD_VERIFIER_PREP, &data); | |
100 | if (err) | |
101 | goto exit_unlock; | |
102 | ||
cae1927c | 103 | env->prog->aux->offload->dev_ops = data.verifier.ops; |
ab3f0063 | 104 | env->prog->aux->offload->dev_state = true; |
ab3f0063 JK |
105 | exit_unlock: |
106 | rtnl_unlock(); | |
107 | return err; | |
108 | } | |
109 | ||
cae1927c JK |
110 | int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, |
111 | int insn_idx, int prev_insn_idx) | |
112 | { | |
0a9c1991 | 113 | struct bpf_prog_offload *offload; |
cae1927c JK |
114 | int ret = -ENODEV; |
115 | ||
116 | down_read(&bpf_devs_lock); | |
117 | offload = env->prog->aux->offload; | |
ce3b9db4 | 118 | if (offload) |
cae1927c JK |
119 | ret = offload->dev_ops->insn_hook(env, insn_idx, prev_insn_idx); |
120 | up_read(&bpf_devs_lock); | |
121 | ||
122 | return ret; | |
123 | } | |
124 | ||
ab3f0063 JK |
125 | static void __bpf_prog_offload_destroy(struct bpf_prog *prog) |
126 | { | |
0a9c1991 | 127 | struct bpf_prog_offload *offload = prog->aux->offload; |
ab3f0063 JK |
128 | struct netdev_bpf data = {}; |
129 | ||
130 | data.offload.prog = prog; | |
131 | ||
ab3f0063 JK |
132 | if (offload->dev_state) |
133 | WARN_ON(__bpf_offload_ndo(prog, BPF_OFFLOAD_DESTROY, &data)); | |
134 | ||
ad8ad79f JK |
135 | /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ |
136 | bpf_prog_free_id(prog, true); | |
137 | ||
ab3f0063 | 138 | list_del_init(&offload->offloads); |
ce3b9db4 JK |
139 | kfree(offload); |
140 | prog->aux->offload = NULL; | |
ab3f0063 JK |
141 | } |
142 | ||
143 | void bpf_prog_offload_destroy(struct bpf_prog *prog) | |
144 | { | |
ab3f0063 | 145 | rtnl_lock(); |
e0d3974a | 146 | down_write(&bpf_devs_lock); |
ce3b9db4 JK |
147 | if (prog->aux->offload) |
148 | __bpf_prog_offload_destroy(prog); | |
e0d3974a | 149 | up_write(&bpf_devs_lock); |
ab3f0063 | 150 | rtnl_unlock(); |
ab3f0063 JK |
151 | } |
152 | ||
153 | static int bpf_prog_offload_translate(struct bpf_prog *prog) | |
154 | { | |
ab3f0063 JK |
155 | struct netdev_bpf data = {}; |
156 | int ret; | |
157 | ||
158 | data.offload.prog = prog; | |
159 | ||
ab3f0063 JK |
160 | rtnl_lock(); |
161 | ret = __bpf_offload_ndo(prog, BPF_OFFLOAD_TRANSLATE, &data); | |
162 | rtnl_unlock(); | |
163 | ||
164 | return ret; | |
165 | } | |
166 | ||
167 | static unsigned int bpf_prog_warn_on_exec(const void *ctx, | |
168 | const struct bpf_insn *insn) | |
169 | { | |
170 | WARN(1, "attempt to execute device eBPF program on the host!"); | |
171 | return 0; | |
172 | } | |
173 | ||
174 | int bpf_prog_offload_compile(struct bpf_prog *prog) | |
175 | { | |
176 | prog->bpf_func = bpf_prog_warn_on_exec; | |
177 | ||
178 | return bpf_prog_offload_translate(prog); | |
179 | } | |
180 | ||
675fc275 JK |
181 | struct ns_get_path_bpf_prog_args { |
182 | struct bpf_prog *prog; | |
183 | struct bpf_prog_info *info; | |
184 | }; | |
185 | ||
186 | static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data) | |
187 | { | |
188 | struct ns_get_path_bpf_prog_args *args = private_data; | |
189 | struct bpf_prog_aux *aux = args->prog->aux; | |
190 | struct ns_common *ns; | |
191 | struct net *net; | |
192 | ||
193 | rtnl_lock(); | |
194 | down_read(&bpf_devs_lock); | |
195 | ||
196 | if (aux->offload) { | |
197 | args->info->ifindex = aux->offload->netdev->ifindex; | |
198 | net = dev_net(aux->offload->netdev); | |
199 | get_net(net); | |
200 | ns = &net->ns; | |
201 | } else { | |
202 | args->info->ifindex = 0; | |
203 | ns = NULL; | |
204 | } | |
205 | ||
206 | up_read(&bpf_devs_lock); | |
207 | rtnl_unlock(); | |
208 | ||
209 | return ns; | |
210 | } | |
211 | ||
212 | int bpf_prog_offload_info_fill(struct bpf_prog_info *info, | |
213 | struct bpf_prog *prog) | |
214 | { | |
215 | struct ns_get_path_bpf_prog_args args = { | |
216 | .prog = prog, | |
217 | .info = info, | |
218 | }; | |
219 | struct inode *ns_inode; | |
220 | struct path ns_path; | |
221 | void *res; | |
222 | ||
223 | res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args); | |
224 | if (IS_ERR(res)) { | |
225 | if (!info->ifindex) | |
226 | return -ENODEV; | |
227 | return PTR_ERR(res); | |
228 | } | |
229 | ||
230 | ns_inode = ns_path.dentry->d_inode; | |
231 | info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); | |
232 | info->netns_ino = ns_inode->i_ino; | |
233 | path_put(&ns_path); | |
234 | ||
235 | return 0; | |
236 | } | |
237 | ||
ab3f0063 JK |
238 | const struct bpf_prog_ops bpf_offload_prog_ops = { |
239 | }; | |
240 | ||
241 | static int bpf_offload_notification(struct notifier_block *notifier, | |
242 | ulong event, void *ptr) | |
243 | { | |
244 | struct net_device *netdev = netdev_notifier_info_to_dev(ptr); | |
0a9c1991 | 245 | struct bpf_prog_offload *offload, *tmp; |
ab3f0063 JK |
246 | |
247 | ASSERT_RTNL(); | |
248 | ||
249 | switch (event) { | |
250 | case NETDEV_UNREGISTER: | |
62c71b45 JK |
251 | /* ignore namespace changes */ |
252 | if (netdev->reg_state != NETREG_UNREGISTERING) | |
253 | break; | |
254 | ||
e0d3974a | 255 | down_write(&bpf_devs_lock); |
ab3f0063 JK |
256 | list_for_each_entry_safe(offload, tmp, &bpf_prog_offload_devs, |
257 | offloads) { | |
258 | if (offload->netdev == netdev) | |
259 | __bpf_prog_offload_destroy(offload->prog); | |
260 | } | |
e0d3974a | 261 | up_write(&bpf_devs_lock); |
ab3f0063 JK |
262 | break; |
263 | default: | |
264 | break; | |
265 | } | |
266 | return NOTIFY_OK; | |
267 | } | |
268 | ||
269 | static struct notifier_block bpf_offload_notifier = { | |
270 | .notifier_call = bpf_offload_notification, | |
271 | }; | |
272 | ||
273 | static int __init bpf_offload_init(void) | |
274 | { | |
275 | register_netdevice_notifier(&bpf_offload_notifier); | |
276 | return 0; | |
277 | } | |
278 | ||
279 | subsys_initcall(bpf_offload_init); |