]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - net/core/filter.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bcm2835-camera: Correct port_parameter_get return value
[mirror_ubuntu-zesty-kernel.git] / net / core / filter.c
CommitLineData
1da177e4
LT		 1/*
2 * Linux Socket Filter - Kernel level socket filtering
3 *
bd4cf0ed
AS		 4 * Based on the design of the Berkeley Packet Filter. The new
5 * internal format has been designed by PLUMgrid:
1da177e4 6 *
bd4cf0ed
AS		 7 * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
8 *
9 * Authors:
10 *
11 * Jay Schulist <jschlst@samba.org>
12 * Alexei Starovoitov <ast@plumgrid.com>
13 * Daniel Borkmann <dborkman@redhat.com>
1da177e4
LT		 14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 *
20 * Andi Kleen - Fix a few bad bugs and races.
4df95ff4 21 * Kris Katterjohn - Added many additional checks in bpf_check_classic()
1da177e4
LT		 22 */
23
24#include <linux/module.h>
25#include <linux/types.h>
1da177e4
LT		 26#include <linux/mm.h>
27#include <linux/fcntl.h>
28#include <linux/socket.h>
29#include <linux/in.h>
30#include <linux/inet.h>
31#include <linux/netdevice.h>
32#include <linux/if_packet.h>
c491680f 33#include <linux/if_arp.h>
5a0e3ad6 34#include <linux/gfp.h>
1da177e4
LT		 35#include <net/ip.h>
36#include <net/protocol.h>
4738c1db 37#include <net/netlink.h>
1da177e4
LT		 38#include <linux/skbuff.h>
39#include <net/sock.h>
10b89ee4 40#include <net/flow_dissector.h>
1da177e4
LT		 41#include <linux/errno.h>
42#include <linux/timer.h>
7c0f6ba6 43#include <linux/uaccess.h>
40daafc8 44#include <asm/unaligned.h>
1da177e4 45#include <linux/filter.h>
86e4ca66 46#include <linux/ratelimit.h>
46b325c7 47#include <linux/seccomp.h>
f3335031 48#include <linux/if_vlan.h>
89aa0758 49#include <linux/bpf.h>
d691f9e8 50#include <net/sch_generic.h>
8d20aabe 51#include <net/cls_cgroup.h>
d3aa45ce 52#include <net/dst_metadata.h>
c46646d0 53#include <net/dst.h>
538950a1 54#include <net/sock_reuseport.h>
1da177e4 55
43db6d65 56/**
f4979fce 57 * sk_filter_trim_cap - run a packet through a socket filter
43db6d65
SH		 58 * @sk: sock associated with &sk_buff
59 * @skb: buffer to filter
f4979fce 60 * @cap: limit on how short the eBPF program may trim the packet
43db6d65 61 *
ff936a04
AS		 62 * Run the eBPF program and then cut skb->data to correct size returned by
63 * the program. If pkt_len is 0 we toss packet. If skb->len is smaller
43db6d65 64 * than pkt_len we keep whole skb->data. This is the socket level
ff936a04 65 * wrapper to BPF_PROG_RUN. It returns 0 if the packet should
43db6d65
SH		 66 * be accepted or -EPERM if the packet should be tossed.
67 *
68 */
f4979fce 69int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap)
43db6d65
SH		 70{
71 int err;
72 struct sk_filter *filter;
73
c93bdd0e
MG		 74 /*
75 * If the skb was allocated from pfmemalloc reserves, only
76 * allow SOCK_MEMALLOC sockets to use it as this socket is
77 * helping free memory
78 */
79 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
80 return -ENOMEM;
81
c11cd3a6
DM		 82 err = BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb);
83 if (err)
84 return err;
85
43db6d65
SH		 86 err = security_sock_rcv_skb(sk, skb);
87 if (err)
88 return err;
89
80f8f102
ED		 90 rcu_read_lock();
91 filter = rcu_dereference(sk->sk_filter);
43db6d65 92 if (filter) {
ff936a04 93 unsigned int pkt_len = bpf_prog_run_save_cb(filter->prog, skb);
f4979fce 94 err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM;
43db6d65 95 }
80f8f102 96 rcu_read_unlock();
43db6d65
SH		 97
98 return err;
99}
f4979fce 100EXPORT_SYMBOL(sk_filter_trim_cap);
43db6d65 101
f3694e00 102BPF_CALL_1(__skb_get_pay_offset, struct sk_buff *, skb)
bd4cf0ed 103{
f3694e00 104 return skb_get_poff(skb);
bd4cf0ed
AS		 105}
106
f3694e00 107BPF_CALL_3(__skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x)
bd4cf0ed 108{
bd4cf0ed
AS		 109 struct nlattr *nla;
110
111 if (skb_is_nonlinear(skb))
112 return 0;
113
05ab8f26
MK		 114 if (skb->len < sizeof(struct nlattr))
115 return 0;
116
30743837 117 if (a > skb->len - sizeof(struct nlattr))
bd4cf0ed
AS		 118 return 0;
119
30743837 120 nla = nla_find((struct nlattr *) &skb->data[a], skb->len - a, x);
bd4cf0ed
AS		 121 if (nla)
122 return (void *) nla - (void *) skb->data;
123
124 return 0;
125}
126
f3694e00 127BPF_CALL_3(__skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x)
bd4cf0ed 128{
bd4cf0ed
AS		 129 struct nlattr *nla;
130
131 if (skb_is_nonlinear(skb))
132 return 0;
133
05ab8f26
MK		 134 if (skb->len < sizeof(struct nlattr))
135 return 0;
136
30743837 137 if (a > skb->len - sizeof(struct nlattr))
bd4cf0ed
AS		 138 return 0;
139
30743837
DB		 140 nla = (struct nlattr *) &skb->data[a];
141 if (nla->nla_len > skb->len - a)
bd4cf0ed
AS		 142 return 0;
143
30743837 144 nla = nla_find_nested(nla, x);
bd4cf0ed
AS		 145 if (nla)
146 return (void *) nla - (void *) skb->data;
147
148 return 0;
149}
150
f3694e00 151BPF_CALL_0(__get_raw_cpu_id)
bd4cf0ed
AS		 152{
153 return raw_smp_processor_id();
154}
155
80b48c44
DB		 156static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = {
157 .func = __get_raw_cpu_id,
158 .gpl_only = false,
159 .ret_type = RET_INTEGER,
160};
161
9bac3d6d
AS		 162static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg,
163 struct bpf_insn *insn_buf)
164{
165 struct bpf_insn *insn = insn_buf;
166
167 switch (skb_field) {
168 case SKF_AD_MARK:
169 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
170
171 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
172 offsetof(struct sk_buff, mark));
173 break;
174
175 case SKF_AD_PKTTYPE:
176 *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET());
177 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX);
178#ifdef __BIG_ENDIAN_BITFIELD
179 *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5);
180#endif
181 break;
182
183 case SKF_AD_QUEUE:
184 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
185
186 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
187 offsetof(struct sk_buff, queue_mapping));
188 break;
c2497395 189
c2497395
AS		 190 case SKF_AD_VLAN_TAG:
191 case SKF_AD_VLAN_TAG_PRESENT:
192 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
193 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
194
195 /* dst_reg = *(u16 *) (src_reg + offsetof(vlan_tci)) */
196 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
197 offsetof(struct sk_buff, vlan_tci));
198 if (skb_field == SKF_AD_VLAN_TAG) {
199 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg,
200 ~VLAN_TAG_PRESENT);
201 } else {
202 /* dst_reg >>= 12 */
203 *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 12);
204 /* dst_reg &= 1 */
205 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, 1);
206 }
207 break;
9bac3d6d
AS		 208 }
209
210 return insn - insn_buf;
211}
212
bd4cf0ed 213static bool convert_bpf_extensions(struct sock_filter *fp,
2695fb55 214 struct bpf_insn **insnp)
bd4cf0ed 215{
2695fb55 216 struct bpf_insn *insn = *insnp;
9bac3d6d 217 u32 cnt;
bd4cf0ed
AS		 218
219 switch (fp->k) {
220 case SKF_AD_OFF + SKF_AD_PROTOCOL:
0b8c707d
DB		 221 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
222
223 /* A = *(u16 *) (CTX + offsetof(protocol)) */
224 *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
225 offsetof(struct sk_buff, protocol));
226 /* A = ntohs(A) [emitting a nop or swap16] */
227 *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
bd4cf0ed
AS		 228 break;
229
230 case SKF_AD_OFF + SKF_AD_PKTTYPE:
9bac3d6d
AS		 231 cnt = convert_skb_access(SKF_AD_PKTTYPE, BPF_REG_A, BPF_REG_CTX, insn);
232 insn += cnt - 1;
bd4cf0ed
AS		 233 break;
234
235 case SKF_AD_OFF + SKF_AD_IFINDEX:
236 case SKF_AD_OFF + SKF_AD_HATYPE:
bd4cf0ed
AS		 237 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
238 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2);
f8f6d679 239
f035a515 240 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
f8f6d679
DB		 241 BPF_REG_TMP, BPF_REG_CTX,
242 offsetof(struct sk_buff, dev));
243 /* if (tmp != 0) goto pc + 1 */
244 *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_TMP, 0, 1);
245 *insn++ = BPF_EXIT_INSN();
246 if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX)
247 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_TMP,
248 offsetof(struct net_device, ifindex));
249 else
250 *insn = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_TMP,
251 offsetof(struct net_device, type));
bd4cf0ed
AS		 252 break;
253
254 case SKF_AD_OFF + SKF_AD_MARK:
9bac3d6d
AS		 255 cnt = convert_skb_access(SKF_AD_MARK, BPF_REG_A, BPF_REG_CTX, insn);
256 insn += cnt - 1;
bd4cf0ed
AS		 257 break;
258
259 case SKF_AD_OFF + SKF_AD_RXHASH:
260 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
261
9739eef1
AS		 262 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX,
263 offsetof(struct sk_buff, hash));
bd4cf0ed
AS		 264 break;
265
266 case SKF_AD_OFF + SKF_AD_QUEUE:
9bac3d6d
AS		 267 cnt = convert_skb_access(SKF_AD_QUEUE, BPF_REG_A, BPF_REG_CTX, insn);
268 insn += cnt - 1;
bd4cf0ed
AS		 269 break;
270
271 case SKF_AD_OFF + SKF_AD_VLAN_TAG:
c2497395
AS		 272 cnt = convert_skb_access(SKF_AD_VLAN_TAG,
273 BPF_REG_A, BPF_REG_CTX, insn);
274 insn += cnt - 1;
275 break;
bd4cf0ed 276
c2497395
AS		 277 case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT:
278 cnt = convert_skb_access(SKF_AD_VLAN_TAG_PRESENT,
279 BPF_REG_A, BPF_REG_CTX, insn);
280 insn += cnt - 1;
bd4cf0ed
AS		 281 break;
282
27cd5452
MS		 283 case SKF_AD_OFF + SKF_AD_VLAN_TPID:
284 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_proto) != 2);
285
286 /* A = *(u16 *) (CTX + offsetof(vlan_proto)) */
287 *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
288 offsetof(struct sk_buff, vlan_proto));
289 /* A = ntohs(A) [emitting a nop or swap16] */
290 *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
291 break;
292
bd4cf0ed
AS		 293 case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
294 case SKF_AD_OFF + SKF_AD_NLATTR:
295 case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
296 case SKF_AD_OFF + SKF_AD_CPU:
4cd3675e 297 case SKF_AD_OFF + SKF_AD_RANDOM:
e430f34e 298 /* arg1 = CTX */
f8f6d679 299 *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
bd4cf0ed 300 /* arg2 = A */
f8f6d679 301 *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_A);
bd4cf0ed 302 /* arg3 = X */
f8f6d679 303 *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_X);
e430f34e 304 /* Emit call(arg1=CTX, arg2=A, arg3=X) */
bd4cf0ed
AS		 305 switch (fp->k) {
306 case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
f8f6d679 307 *insn = BPF_EMIT_CALL(__skb_get_pay_offset);
bd4cf0ed
AS		 308 break;
309 case SKF_AD_OFF + SKF_AD_NLATTR:
f8f6d679 310 *insn = BPF_EMIT_CALL(__skb_get_nlattr);
bd4cf0ed
AS		 311 break;
312 case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
f8f6d679 313 *insn = BPF_EMIT_CALL(__skb_get_nlattr_nest);
bd4cf0ed
AS		 314 break;
315 case SKF_AD_OFF + SKF_AD_CPU:
f8f6d679 316 *insn = BPF_EMIT_CALL(__get_raw_cpu_id);
bd4cf0ed 317 break;
4cd3675e 318 case SKF_AD_OFF + SKF_AD_RANDOM:
3ad00405
DB		 319 *insn = BPF_EMIT_CALL(bpf_user_rnd_u32);
320 bpf_user_rnd_init_once();
4cd3675e 321 break;
bd4cf0ed
AS		 322 }
323 break;
324
325 case SKF_AD_OFF + SKF_AD_ALU_XOR_X:
9739eef1
AS		 326 /* A ^= X */
327 *insn = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_X);
bd4cf0ed
AS		 328 break;
329
330 default:
331 /* This is just a dummy call to avoid letting the compiler
332 * evict __bpf_call_base() as an optimization. Placed here
333 * where no-one bothers.
334 */
335 BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0);
336 return false;
337 }
338
339 *insnp = insn;
340 return true;
341}
342
343/**
8fb575ca 344 * bpf_convert_filter - convert filter program
bd4cf0ed
AS		 345 * @prog: the user passed filter program
346 * @len: the length of the user passed filter program
347 * @new_prog: buffer where converted program will be stored
348 * @new_len: pointer to store length of converted program
349 *
350 * Remap 'sock_filter' style BPF instruction set to 'sock_filter_ext' style.
351 * Conversion workflow:
352 *
353 * 1) First pass for calculating the new program length:
8fb575ca 354 * bpf_convert_filter(old_prog, old_len, NULL, &new_len)
bd4cf0ed
AS		 355 *
356 * 2) 2nd pass to remap in two passes: 1st pass finds new
357 * jump offsets, 2nd pass remapping:
2695fb55 358 * new_prog = kmalloc(sizeof(struct bpf_insn) * new_len);
8fb575ca 359 * bpf_convert_filter(old_prog, old_len, new_prog, &new_len);
bd4cf0ed 360 */
d9e12f42
NS		 361static int bpf_convert_filter(struct sock_filter *prog, int len,
362 struct bpf_insn *new_prog, int *new_len)
bd4cf0ed
AS		 363{
364 int new_flen = 0, pass = 0, target, i;
2695fb55 365 struct bpf_insn *new_insn;
bd4cf0ed
AS		 366 struct sock_filter *fp;
367 int *addrs = NULL;
368 u8 bpf_src;
369
370 BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK);
30743837 371 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
bd4cf0ed 372
6f9a093b 373 if (len <= 0 || len > BPF_MAXINSNS)
bd4cf0ed
AS		 374 return -EINVAL;
375
376 if (new_prog) {
658da937
DB		 377 addrs = kcalloc(len, sizeof(*addrs),
378 GFP_KERNEL | __GFP_NOWARN);
bd4cf0ed
AS		 379 if (!addrs)
380 return -ENOMEM;
381 }
382
383do_pass:
384 new_insn = new_prog;
385 fp = prog;
386
8b614aeb
DB		 387 /* Classic BPF related prologue emission. */
388 if (new_insn) {
389 /* Classic BPF expects A and X to be reset first. These need
390 * to be guaranteed to be the first two instructions.
391 */
392 *new_insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
393 *new_insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_X, BPF_REG_X);
394
395 /* All programs must keep CTX in callee saved BPF_REG_CTX.
396 * In eBPF case it's done by the compiler, here we need to
397 * do this ourself. Initial CTX is present in BPF_REG_ARG1.
398 */
399 *new_insn++ = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1);
400 } else {
401 new_insn += 3;
402 }
bd4cf0ed
AS		 403
404 for (i = 0; i < len; fp++, i++) {
2695fb55
AS		 405 struct bpf_insn tmp_insns[6] = { };
406 struct bpf_insn *insn = tmp_insns;
bd4cf0ed
AS		 407
408 if (addrs)
409 addrs[i] = new_insn - new_prog;
410
411 switch (fp->code) {
412 /* All arithmetic insns and skb loads map as-is. */
413 case BPF_ALU | BPF_ADD | BPF_X:
414 case BPF_ALU | BPF_ADD | BPF_K:
415 case BPF_ALU | BPF_SUB | BPF_X:
416 case BPF_ALU | BPF_SUB | BPF_K:
417 case BPF_ALU | BPF_AND | BPF_X:
418 case BPF_ALU | BPF_AND | BPF_K:
419 case BPF_ALU | BPF_OR | BPF_X:
420 case BPF_ALU | BPF_OR | BPF_K:
421 case BPF_ALU | BPF_LSH | BPF_X:
422 case BPF_ALU | BPF_LSH | BPF_K:
423 case BPF_ALU | BPF_RSH | BPF_X:
424 case BPF_ALU | BPF_RSH | BPF_K:
425 case BPF_ALU | BPF_XOR | BPF_X:
426 case BPF_ALU | BPF_XOR | BPF_K:
427 case BPF_ALU | BPF_MUL | BPF_X:
428 case BPF_ALU | BPF_MUL | BPF_K:
429 case BPF_ALU | BPF_DIV | BPF_X:
430 case BPF_ALU | BPF_DIV | BPF_K:
431 case BPF_ALU | BPF_MOD | BPF_X:
432 case BPF_ALU | BPF_MOD | BPF_K:
433 case BPF_ALU | BPF_NEG:
434 case BPF_LD | BPF_ABS | BPF_W:
435 case BPF_LD | BPF_ABS | BPF_H:
436 case BPF_LD | BPF_ABS | BPF_B:
437 case BPF_LD | BPF_IND | BPF_W:
438 case BPF_LD | BPF_IND | BPF_H:
439 case BPF_LD | BPF_IND | BPF_B:
440 /* Check for overloaded BPF extension and
441 * directly convert it if found, otherwise
442 * just move on with mapping.
443 */
444 if (BPF_CLASS(fp->code) == BPF_LD &&
445 BPF_MODE(fp->code) == BPF_ABS &&
446 convert_bpf_extensions(fp, &insn))
447 break;
448
f8f6d679 449 *insn = BPF_RAW_INSN(fp->code, BPF_REG_A, BPF_REG_X, 0, fp->k);
bd4cf0ed
AS		 450 break;
451
f8f6d679
DB		 452 /* Jump transformation cannot use BPF block macros
453 * everywhere as offset calculation and target updates
454 * require a bit more work than the rest, i.e. jump
455 * opcodes map as-is, but offsets need adjustment.
456 */
457
458#define BPF_EMIT_JMP \
bd4cf0ed
AS		 459 do { \
460 if (target >= len || target < 0) \
461 goto err; \
462 insn->off = addrs ? addrs[target] - addrs[i] - 1 : 0; \
463 /* Adjust pc relative offset for 2nd or 3rd insn. */ \
464 insn->off -= insn - tmp_insns; \
465 } while (0)
466
f8f6d679
DB		 467 case BPF_JMP | BPF_JA:
468 target = i + fp->k + 1;
469 insn->code = fp->code;
470 BPF_EMIT_JMP;
bd4cf0ed
AS		 471 break;
472
473 case BPF_JMP | BPF_JEQ | BPF_K:
474 case BPF_JMP | BPF_JEQ | BPF_X:
475 case BPF_JMP | BPF_JSET | BPF_K:
476 case BPF_JMP | BPF_JSET | BPF_X:
477 case BPF_JMP | BPF_JGT | BPF_K:
478 case BPF_JMP | BPF_JGT | BPF_X:
479 case BPF_JMP | BPF_JGE | BPF_K:
480 case BPF_JMP | BPF_JGE | BPF_X:
481 if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) {
482 /* BPF immediates are signed, zero extend
483 * immediate into tmp register and use it
484 * in compare insn.
485 */
f8f6d679 486 *insn++ = BPF_MOV32_IMM(BPF_REG_TMP, fp->k);
bd4cf0ed 487
e430f34e
AS		 488 insn->dst_reg = BPF_REG_A;
489 insn->src_reg = BPF_REG_TMP;
bd4cf0ed
AS		 490 bpf_src = BPF_X;
491 } else {
e430f34e 492 insn->dst_reg = BPF_REG_A;
bd4cf0ed
AS		 493 insn->imm = fp->k;
494 bpf_src = BPF_SRC(fp->code);
19539ce7 495 insn->src_reg = bpf_src == BPF_X ? BPF_REG_X : 0;
1da177e4 496 }
bd4cf0ed
AS		 497
498 /* Common case where 'jump_false' is next insn. */
499 if (fp->jf == 0) {
500 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
501 target = i + fp->jt + 1;
f8f6d679 502 BPF_EMIT_JMP;
bd4cf0ed 503 break;
1da177e4 504 }
bd4cf0ed
AS		 505
506 /* Convert JEQ into JNE when 'jump_true' is next insn. */
507 if (fp->jt == 0 && BPF_OP(fp->code) == BPF_JEQ) {
508 insn->code = BPF_JMP | BPF_JNE | bpf_src;
509 target = i + fp->jf + 1;
f8f6d679 510 BPF_EMIT_JMP;
bd4cf0ed 511 break;
0b05b2a4 512 }
bd4cf0ed
AS		 513
514 /* Other jumps are mapped into two insns: Jxx and JA. */
515 target = i + fp->jt + 1;
516 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
f8f6d679 517 BPF_EMIT_JMP;
bd4cf0ed
AS		 518 insn++;
519
520 insn->code = BPF_JMP | BPF_JA;
521 target = i + fp->jf + 1;
f8f6d679 522 BPF_EMIT_JMP;
bd4cf0ed
AS		 523 break;
524
525 /* ldxb 4 * ([14] & 0xf) is remaped into 6 insns. */
526 case BPF_LDX | BPF_MSH | BPF_B:
9739eef1 527 /* tmp = A */
f8f6d679 528 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_A);
1268e253 529 /* A = BPF_R0 = *(u8 *) (skb->data + K) */
f8f6d679 530 *insn++ = BPF_LD_ABS(BPF_B, fp->k);
9739eef1 531 /* A &= 0xf */
f8f6d679 532 *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf);
9739eef1 533 /* A <<= 2 */
f8f6d679 534 *insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2);
9739eef1 535 /* X = A */
f8f6d679 536 *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
9739eef1 537 /* A = tmp */
f8f6d679 538 *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP);
bd4cf0ed
AS		 539 break;
540
6205b9cf
DB		 541 /* RET_K is remaped into 2 insns. RET_A case doesn't need an
542 * extra mov as BPF_REG_0 is already mapped into BPF_REG_A.
543 */
bd4cf0ed
AS		 544 case BPF_RET | BPF_A:
545 case BPF_RET | BPF_K:
6205b9cf
DB		 546 if (BPF_RVAL(fp->code) == BPF_K)
547 *insn++ = BPF_MOV32_RAW(BPF_K, BPF_REG_0,
548 0, fp->k);
9739eef1 549 *insn = BPF_EXIT_INSN();
bd4cf0ed
AS		 550 break;
551
552 /* Store to stack. */
553 case BPF_ST:
554 case BPF_STX:
f8f6d679
DB		 555 *insn = BPF_STX_MEM(BPF_W, BPF_REG_FP, BPF_CLASS(fp->code) ==
556 BPF_ST ? BPF_REG_A : BPF_REG_X,
557 -(BPF_MEMWORDS - fp->k) * 4);
bd4cf0ed
AS		 558 break;
559
560 /* Load from stack. */
561 case BPF_LD | BPF_MEM:
562 case BPF_LDX | BPF_MEM:
f8f6d679
DB		 563 *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ?
564 BPF_REG_A : BPF_REG_X, BPF_REG_FP,
565 -(BPF_MEMWORDS - fp->k) * 4);
bd4cf0ed
AS		 566 break;
567
568 /* A = K or X = K */
569 case BPF_LD | BPF_IMM:
570 case BPF_LDX | BPF_IMM:
f8f6d679
DB		 571 *insn = BPF_MOV32_IMM(BPF_CLASS(fp->code) == BPF_LD ?
572 BPF_REG_A : BPF_REG_X, fp->k);
bd4cf0ed
AS		 573 break;
574
575 /* X = A */
576 case BPF_MISC | BPF_TAX:
f8f6d679 577 *insn = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
bd4cf0ed
AS		 578 break;
579
580 /* A = X */
581 case BPF_MISC | BPF_TXA:
f8f6d679 582 *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_X);
bd4cf0ed
AS		 583 break;
584
585 /* A = skb->len or X = skb->len */
586 case BPF_LD | BPF_W | BPF_LEN:
587 case BPF_LDX | BPF_W | BPF_LEN:
f8f6d679
DB		 588 *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ?
589 BPF_REG_A : BPF_REG_X, BPF_REG_CTX,
590 offsetof(struct sk_buff, len));
bd4cf0ed
AS		 591 break;
592
f8f6d679 593 /* Access seccomp_data fields. */
bd4cf0ed 594 case BPF_LDX | BPF_ABS | BPF_W:
9739eef1
AS		 595 /* A = *(u32 *) (ctx + K) */
596 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX, fp->k);
bd4cf0ed
AS		 597 break;
598
ca9f1fd2 599 /* Unknown instruction. */
1da177e4 600 default:
bd4cf0ed 601 goto err;
1da177e4 602 }
bd4cf0ed
AS		 603
604 insn++;
605 if (new_prog)
606 memcpy(new_insn, tmp_insns,
607 sizeof(*insn) * (insn - tmp_insns));
bd4cf0ed 608 new_insn += insn - tmp_insns;
1da177e4
LT		 609 }
610
bd4cf0ed
AS		 611 if (!new_prog) {
612 /* Only calculating new length. */
613 *new_len = new_insn - new_prog;
614 return 0;
615 }
616
617 pass++;
618 if (new_flen != new_insn - new_prog) {
619 new_flen = new_insn - new_prog;
620 if (pass > 2)
621 goto err;
bd4cf0ed
AS		 622 goto do_pass;
623 }
624
625 kfree(addrs);
626 BUG_ON(*new_len != new_flen);
1da177e4 627 return 0;
bd4cf0ed
AS		 628err:
629 kfree(addrs);
630 return -EINVAL;
1da177e4
LT		 631}
632
bd4cf0ed 633/* Security:
bd4cf0ed 634 *
2d5311e4 635 * As we dont want to clear mem[] array for each packet going through
8ea6e345 636 * __bpf_prog_run(), we check that filter loaded by user never try to read
2d5311e4 637 * a cell if not previously written, and we check all branches to be sure
25985edc 638 * a malicious user doesn't try to abuse us.
2d5311e4 639 */
ec31a05c 640static int check_load_and_stores(const struct sock_filter *filter, int flen)
2d5311e4 641{
34805931 642 u16 *masks, memvalid = 0; /* One bit per cell, 16 cells */
2d5311e4
ED		 643 int pc, ret = 0;
644
645 BUILD_BUG_ON(BPF_MEMWORDS > 16);
34805931 646
99e72a0f 647 masks = kmalloc_array(flen, sizeof(*masks), GFP_KERNEL);
2d5311e4
ED		 648 if (!masks)
649 return -ENOMEM;
34805931 650
2d5311e4
ED		 651 memset(masks, 0xff, flen * sizeof(*masks));
652
653 for (pc = 0; pc < flen; pc++) {
654 memvalid &= masks[pc];
655
656 switch (filter[pc].code) {
34805931
DB		 657 case BPF_ST:
658 case BPF_STX:
2d5311e4
ED		 659 memvalid |= (1 << filter[pc].k);
660 break;
34805931
DB		 661 case BPF_LD | BPF_MEM:
662 case BPF_LDX | BPF_MEM:
2d5311e4
ED		 663 if (!(memvalid & (1 << filter[pc].k))) {
664 ret = -EINVAL;
665 goto error;
666 }
667 break;
34805931
DB		 668 case BPF_JMP | BPF_JA:
669 /* A jump must set masks on target */
2d5311e4
ED		 670 masks[pc + 1 + filter[pc].k] &= memvalid;
671 memvalid = ~0;
672 break;
34805931
DB		 673 case BPF_JMP | BPF_JEQ | BPF_K:
674 case BPF_JMP | BPF_JEQ | BPF_X:
675 case BPF_JMP | BPF_JGE | BPF_K:
676 case BPF_JMP | BPF_JGE | BPF_X:
677 case BPF_JMP | BPF_JGT | BPF_K:
678 case BPF_JMP | BPF_JGT | BPF_X:
679 case BPF_JMP | BPF_JSET | BPF_K:
680 case BPF_JMP | BPF_JSET | BPF_X:
681 /* A jump must set masks on targets */
2d5311e4
ED		 682 masks[pc + 1 + filter[pc].jt] &= memvalid;
683 masks[pc + 1 + filter[pc].jf] &= memvalid;
684 memvalid = ~0;
685 break;
686 }
687 }
688error:
689 kfree(masks);
690 return ret;
691}
692
34805931
DB		 693static bool chk_code_allowed(u16 code_to_probe)
694{
695 static const bool codes[] = {
696 /* 32 bit ALU operations */
697 [BPF_ALU | BPF_ADD | BPF_K] = true,
698 [BPF_ALU | BPF_ADD | BPF_X] = true,
699 [BPF_ALU | BPF_SUB | BPF_K] = true,
700 [BPF_ALU | BPF_SUB | BPF_X] = true,
701 [BPF_ALU | BPF_MUL | BPF_K] = true,
702 [BPF_ALU | BPF_MUL | BPF_X] = true,
703 [BPF_ALU | BPF_DIV | BPF_K] = true,
704 [BPF_ALU | BPF_DIV | BPF_X] = true,
705 [BPF_ALU | BPF_MOD | BPF_K] = true,
706 [BPF_ALU | BPF_MOD | BPF_X] = true,
707 [BPF_ALU | BPF_AND | BPF_K] = true,
708 [BPF_ALU | BPF_AND | BPF_X] = true,
709 [BPF_ALU | BPF_OR | BPF_K] = true,
710 [BPF_ALU | BPF_OR | BPF_X] = true,
711 [BPF_ALU | BPF_XOR | BPF_K] = true,
712 [BPF_ALU | BPF_XOR | BPF_X] = true,
713 [BPF_ALU | BPF_LSH | BPF_K] = true,
714 [BPF_ALU | BPF_LSH | BPF_X] = true,
715 [BPF_ALU | BPF_RSH | BPF_K] = true,
716 [BPF_ALU | BPF_RSH | BPF_X] = true,
717 [BPF_ALU | BPF_NEG] = true,
718 /* Load instructions */
719 [BPF_LD | BPF_W | BPF_ABS] = true,
720 [BPF_LD | BPF_H | BPF_ABS] = true,
721 [BPF_LD | BPF_B | BPF_ABS] = true,
722 [BPF_LD | BPF_W | BPF_LEN] = true,
723 [BPF_LD | BPF_W | BPF_IND] = true,
724 [BPF_LD | BPF_H | BPF_IND] = true,
725 [BPF_LD | BPF_B | BPF_IND] = true,
726 [BPF_LD | BPF_IMM] = true,
727 [BPF_LD | BPF_MEM] = true,
728 [BPF_LDX | BPF_W | BPF_LEN] = true,
729 [BPF_LDX | BPF_B | BPF_MSH] = true,
730 [BPF_LDX | BPF_IMM] = true,
731 [BPF_LDX | BPF_MEM] = true,
732 /* Store instructions */
733 [BPF_ST] = true,
734 [BPF_STX] = true,
735 /* Misc instructions */
736 [BPF_MISC | BPF_TAX] = true,
737 [BPF_MISC | BPF_TXA] = true,
738 /* Return instructions */
739 [BPF_RET | BPF_K] = true,
740 [BPF_RET | BPF_A] = true,
741 /* Jump instructions */
742 [BPF_JMP | BPF_JA] = true,
743 [BPF_JMP | BPF_JEQ | BPF_K] = true,
744 [BPF_JMP | BPF_JEQ | BPF_X] = true,
745 [BPF_JMP | BPF_JGE | BPF_K] = true,
746 [BPF_JMP | BPF_JGE | BPF_X] = true,
747 [BPF_JMP | BPF_JGT | BPF_K] = true,
748 [BPF_JMP | BPF_JGT | BPF_X] = true,
749 [BPF_JMP | BPF_JSET | BPF_K] = true,
750 [BPF_JMP | BPF_JSET | BPF_X] = true,
751 };
752
753 if (code_to_probe >= ARRAY_SIZE(codes))
754 return false;
755
756 return codes[code_to_probe];
757}
758
f7bd9e36
DB		 759static bool bpf_check_basics_ok(const struct sock_filter *filter,
760 unsigned int flen)
761{
762 if (filter == NULL)
763 return false;
764 if (flen == 0 || flen > BPF_MAXINSNS)
765 return false;
766
767 return true;
768}
769
1da177e4 770/**
4df95ff4 771 * bpf_check_classic - verify socket filter code
1da177e4
LT		 772 * @filter: filter to verify
773 * @flen: length of filter
774 *
775 * Check the user's filter code. If we let some ugly
776 * filter code slip through kaboom! The filter must contain
93699863
KK		 777 * no references or jumps that are out of range, no illegal
778 * instructions, and must end with a RET instruction.
1da177e4 779 *
7b11f69f
KK		 780 * All jumps are forward as they are not signed.
781 *
782 * Returns 0 if the rule set is legal or -EINVAL if not.
1da177e4 783 */
d9e12f42
NS		 784static int bpf_check_classic(const struct sock_filter *filter,
785 unsigned int flen)
1da177e4 786{
aa1113d9 787 bool anc_found;
34805931 788 int pc;
1da177e4 789
34805931 790 /* Check the filter code now */
1da177e4 791 for (pc = 0; pc < flen; pc++) {
ec31a05c 792 const struct sock_filter *ftest = &filter[pc];
93699863 793
34805931
DB		 794 /* May we actually operate on this code? */
795 if (!chk_code_allowed(ftest->code))
cba328fc 796 return -EINVAL;
34805931 797
93699863 798 /* Some instructions need special checks */
34805931
DB		 799 switch (ftest->code) {
800 case BPF_ALU | BPF_DIV | BPF_K:
801 case BPF_ALU | BPF_MOD | BPF_K:
802 /* Check for division by zero */
b6069a95
ED		 803 if (ftest->k == 0)
804 return -EINVAL;
805 break;
229394e8
RV		 806 case BPF_ALU | BPF_LSH | BPF_K:
807 case BPF_ALU | BPF_RSH | BPF_K:
808 if (ftest->k >= 32)
809 return -EINVAL;
810 break;
34805931
DB		 811 case BPF_LD | BPF_MEM:
812 case BPF_LDX | BPF_MEM:
813 case BPF_ST:
814 case BPF_STX:
815 /* Check for invalid memory addresses */
93699863
KK		 816 if (ftest->k >= BPF_MEMWORDS)
817 return -EINVAL;
818 break;
34805931
DB		 819 case BPF_JMP | BPF_JA:
820 /* Note, the large ftest->k might cause loops.
93699863
KK		 821 * Compare this with conditional jumps below,
822 * where offsets are limited. --ANK (981016)
823 */
34805931 824 if (ftest->k >= (unsigned int)(flen - pc - 1))
93699863 825 return -EINVAL;
01f2f3f6 826 break;
34805931
DB		 827 case BPF_JMP | BPF_JEQ | BPF_K:
828 case BPF_JMP | BPF_JEQ | BPF_X:
829 case BPF_JMP | BPF_JGE | BPF_K:
830 case BPF_JMP | BPF_JGE | BPF_X:
831 case BPF_JMP | BPF_JGT | BPF_K:
832 case BPF_JMP | BPF_JGT | BPF_X:
833 case BPF_JMP | BPF_JSET | BPF_K:
834 case BPF_JMP | BPF_JSET | BPF_X:
835 /* Both conditionals must be safe */
e35bedf3 836 if (pc + ftest->jt + 1 >= flen ||
93699863
KK		 837 pc + ftest->jf + 1 >= flen)
838 return -EINVAL;
cba328fc 839 break;
34805931
DB		 840 case BPF_LD | BPF_W | BPF_ABS:
841 case BPF_LD | BPF_H | BPF_ABS:
842 case BPF_LD | BPF_B | BPF_ABS:
aa1113d9 843 anc_found = false;
34805931
DB		 844 if (bpf_anc_helper(ftest) & BPF_ANC)
845 anc_found = true;
846 /* Ancillary operation unknown or unsupported */
aa1113d9
DB		 847 if (anc_found == false && ftest->k >= SKF_AD_OFF)
848 return -EINVAL;
01f2f3f6
HPP		 849 }
850 }
93699863 851
34805931 852 /* Last instruction must be a RET code */
01f2f3f6 853 switch (filter[flen - 1].code) {
34805931
DB		 854 case BPF_RET | BPF_K:
855 case BPF_RET | BPF_A:
2d5311e4 856 return check_load_and_stores(filter, flen);
cba328fc 857 }
34805931 858
cba328fc 859 return -EINVAL;
1da177e4
LT		 860}
861
7ae457c1
AS		 862static int bpf_prog_store_orig_filter(struct bpf_prog *fp,
863 const struct sock_fprog *fprog)
a3ea269b 864{
009937e7 865 unsigned int fsize = bpf_classic_proglen(fprog);
a3ea269b
DB		 866 struct sock_fprog_kern *fkprog;
867
868 fp->orig_prog = kmalloc(sizeof(*fkprog), GFP_KERNEL);
869 if (!fp->orig_prog)
870 return -ENOMEM;
871
872 fkprog = fp->orig_prog;
873 fkprog->len = fprog->len;
658da937
DB		 874
875 fkprog->filter = kmemdup(fp->insns, fsize,
876 GFP_KERNEL | __GFP_NOWARN);
a3ea269b
DB		 877 if (!fkprog->filter) {
878 kfree(fp->orig_prog);
879 return -ENOMEM;
880 }
881
882 return 0;
883}
884
7ae457c1 885static void bpf_release_orig_filter(struct bpf_prog *fp)
a3ea269b
DB		 886{
887 struct sock_fprog_kern *fprog = fp->orig_prog;
888
889 if (fprog) {
890 kfree(fprog->filter);
891 kfree(fprog);
892 }
893}
894
7ae457c1
AS		 895static void __bpf_prog_release(struct bpf_prog *prog)
896{
24701ece 897 if (prog->type == BPF_PROG_TYPE_SOCKET_FILTER) {
89aa0758
AS		 898 bpf_prog_put(prog);
899 } else {
900 bpf_release_orig_filter(prog);
901 bpf_prog_free(prog);
902 }
7ae457c1
AS		 903}
904
34c5bd66
PN		 905static void __sk_filter_release(struct sk_filter *fp)
906{
7ae457c1
AS		 907 __bpf_prog_release(fp->prog);
908 kfree(fp);
34c5bd66
PN		 909}
910
47e958ea 911/**
46bcf14f 912 * sk_filter_release_rcu - Release a socket filter by rcu_head
47e958ea
PE		 913 * @rcu: rcu_head that contains the sk_filter to free
914 */
fbc907f0 915static void sk_filter_release_rcu(struct rcu_head *rcu)
47e958ea
PE		 916{
917 struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu);
918
34c5bd66 919 __sk_filter_release(fp);
47e958ea 920}
fbc907f0
DB		 921
922/**
923 * sk_filter_release - release a socket filter
924 * @fp: filter to remove
925 *
926 * Remove a filter from a socket and release its resources.
927 */
928static void sk_filter_release(struct sk_filter *fp)
929{
930 if (atomic_dec_and_test(&fp->refcnt))
931 call_rcu(&fp->rcu, sk_filter_release_rcu);
932}
933
934void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
935{
7ae457c1 936 u32 filter_size = bpf_prog_size(fp->prog->len);
fbc907f0 937
278571ba
AS		 938 atomic_sub(filter_size, &sk->sk_omem_alloc);
939 sk_filter_release(fp);
fbc907f0 940}
47e958ea 941
278571ba
AS		 942/* try to charge the socket memory if there is space available
943 * return true on success
944 */
945bool sk_filter_charge(struct sock *sk, struct sk_filter *fp)
bd4cf0ed 946{
7ae457c1 947 u32 filter_size = bpf_prog_size(fp->prog->len);
278571ba
AS		 948
949 /* same check as in sock_kmalloc() */
950 if (filter_size <= sysctl_optmem_max &&
951 atomic_read(&sk->sk_omem_alloc) + filter_size < sysctl_optmem_max) {
952 atomic_inc(&fp->refcnt);
953 atomic_add(filter_size, &sk->sk_omem_alloc);
954 return true;
bd4cf0ed 955 }
278571ba 956 return false;
bd4cf0ed
AS		 957}
958
7ae457c1 959static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp)
bd4cf0ed
AS		 960{
961 struct sock_filter *old_prog;
7ae457c1 962 struct bpf_prog *old_fp;
34805931 963 int err, new_len, old_len = fp->len;
bd4cf0ed
AS		 964
965 /* We are free to overwrite insns et al right here as it
966 * won't be used at this point in time anymore internally
967 * after the migration to the internal BPF instruction
968 * representation.
969 */
970 BUILD_BUG_ON(sizeof(struct sock_filter) !=
2695fb55 971 sizeof(struct bpf_insn));
bd4cf0ed 972
bd4cf0ed
AS		 973 /* Conversion cannot happen on overlapping memory areas,
974 * so we need to keep the user BPF around until the 2nd
975 * pass. At this time, the user BPF is stored in fp->insns.
976 */
977 old_prog = kmemdup(fp->insns, old_len * sizeof(struct sock_filter),
658da937 978 GFP_KERNEL | __GFP_NOWARN);
bd4cf0ed
AS		 979 if (!old_prog) {
980 err = -ENOMEM;
981 goto out_err;
982 }
983
984 /* 1st pass: calculate the new program length. */
8fb575ca 985 err = bpf_convert_filter(old_prog, old_len, NULL, &new_len);
bd4cf0ed
AS		 986 if (err)
987 goto out_err_free;
988
989 /* Expand fp for appending the new filter representation. */
990 old_fp = fp;
60a3b225 991 fp = bpf_prog_realloc(old_fp, bpf_prog_size(new_len), 0);
bd4cf0ed
AS		 992 if (!fp) {
993 /* The old_fp is still around in case we couldn't
994 * allocate new memory, so uncharge on that one.
995 */
996 fp = old_fp;
997 err = -ENOMEM;
998 goto out_err_free;
999 }
1000
bd4cf0ed
AS		 1001 fp->len = new_len;
1002
2695fb55 1003 /* 2nd pass: remap sock_filter insns into bpf_insn insns. */
8fb575ca 1004 err = bpf_convert_filter(old_prog, old_len, fp->insnsi, &new_len);
bd4cf0ed 1005 if (err)
8fb575ca 1006 /* 2nd bpf_convert_filter() can fail only if it fails
bd4cf0ed
AS		 1007 * to allocate memory, remapping must succeed. Note,
1008 * that at this time old_fp has already been released
278571ba 1009 * by krealloc().
bd4cf0ed
AS		 1010 */
1011 goto out_err_free;
1012
d1c55ab5
DB		 1013 /* We are guaranteed to never error here with cBPF to eBPF
1014 * transitions, since there's no issue with type compatibility
1015 * checks on program arrays.
1016 */
1017 fp = bpf_prog_select_runtime(fp, &err);
5fe821a9 1018
bd4cf0ed
AS		 1019 kfree(old_prog);
1020 return fp;
1021
1022out_err_free:
1023 kfree(old_prog);
1024out_err:
7ae457c1 1025 __bpf_prog_release(fp);
bd4cf0ed
AS		 1026 return ERR_PTR(err);
1027}
1028
ac67eb2c
DB		 1029static struct bpf_prog *bpf_prepare_filter(struct bpf_prog *fp,
1030 bpf_aux_classic_check_t trans)
302d6637
JP		 1031{
1032 int err;
1033
bd4cf0ed 1034 fp->bpf_func = NULL;
a91263d5 1035 fp->jited = 0;
302d6637 1036
4df95ff4 1037 err = bpf_check_classic(fp->insns, fp->len);
418c96ac 1038 if (err) {
7ae457c1 1039 __bpf_prog_release(fp);
bd4cf0ed 1040 return ERR_PTR(err);
418c96ac 1041 }
302d6637 1042
4ae92bc7
NS		 1043 /* There might be additional checks and transformations
1044 * needed on classic filters, f.e. in case of seccomp.
1045 */
1046 if (trans) {
1047 err = trans(fp->insns, fp->len);
1048 if (err) {
1049 __bpf_prog_release(fp);
1050 return ERR_PTR(err);
1051 }
1052 }
1053
bd4cf0ed
AS		 1054 /* Probe if we can JIT compile the filter and if so, do
1055 * the compilation of the filter.
1056 */
302d6637 1057 bpf_jit_compile(fp);
bd4cf0ed
AS		 1058
1059 /* JIT compiler couldn't process this filter, so do the
1060 * internal BPF translation for the optimized interpreter.
1061 */
5fe821a9 1062 if (!fp->jited)
7ae457c1 1063 fp = bpf_migrate_filter(fp);
bd4cf0ed
AS		 1064
1065 return fp;
302d6637
JP		 1066}
1067
1068/**
7ae457c1 1069 * bpf_prog_create - create an unattached filter
c6c4b97c 1070 * @pfp: the unattached filter that is created
677a9fd3 1071 * @fprog: the filter program
302d6637 1072 *
c6c4b97c 1073 * Create a filter independent of any socket. We first run some
302d6637
JP		 1074 * sanity checks on it to make sure it does not explode on us later.
1075 * If an error occurs or there is insufficient memory for the filter
1076 * a negative errno code is returned. On success the return is zero.
1077 */
7ae457c1 1078int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog)
302d6637 1079{
009937e7 1080 unsigned int fsize = bpf_classic_proglen(fprog);
7ae457c1 1081 struct bpf_prog *fp;
302d6637
JP		 1082
1083 /* Make sure new filter is there and in the right amounts. */
f7bd9e36 1084 if (!bpf_check_basics_ok(fprog->filter, fprog->len))
302d6637
JP		 1085 return -EINVAL;
1086
60a3b225 1087 fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
302d6637
JP		 1088 if (!fp)
1089 return -ENOMEM;
a3ea269b 1090
302d6637
JP		 1091 memcpy(fp->insns, fprog->filter, fsize);
1092
302d6637 1093 fp->len = fprog->len;
a3ea269b
DB		 1094 /* Since unattached filters are not copied back to user
1095 * space through sk_get_filter(), we do not need to hold
1096 * a copy here, and can spare us the work.
1097 */
1098 fp->orig_prog = NULL;
302d6637 1099
7ae457c1 1100 /* bpf_prepare_filter() already takes care of freeing
bd4cf0ed
AS		 1101 * memory in case something goes wrong.
1102 */
4ae92bc7 1103 fp = bpf_prepare_filter(fp, NULL);
bd4cf0ed
AS		 1104 if (IS_ERR(fp))
1105 return PTR_ERR(fp);
302d6637
JP		 1106
1107 *pfp = fp;
1108 return 0;
302d6637 1109}
7ae457c1 1110EXPORT_SYMBOL_GPL(bpf_prog_create);
302d6637 1111
ac67eb2c
DB		 1112/**
1113 * bpf_prog_create_from_user - create an unattached filter from user buffer
1114 * @pfp: the unattached filter that is created
1115 * @fprog: the filter program
1116 * @trans: post-classic verifier transformation handler
bab18991 1117 * @save_orig: save classic BPF program
ac67eb2c
DB		 1118 *
1119 * This function effectively does the same as bpf_prog_create(), only
1120 * that it builds up its insns buffer from user space provided buffer.
1121 * It also allows for passing a bpf_aux_classic_check_t handler.
1122 */
1123int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
bab18991 1124 bpf_aux_classic_check_t trans, bool save_orig)
ac67eb2c
DB		 1125{
1126 unsigned int fsize = bpf_classic_proglen(fprog);
1127 struct bpf_prog *fp;
bab18991 1128 int err;
ac67eb2c
DB		 1129
1130 /* Make sure new filter is there and in the right amounts. */
f7bd9e36 1131 if (!bpf_check_basics_ok(fprog->filter, fprog->len))
ac67eb2c
DB		 1132 return -EINVAL;
1133
1134 fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
1135 if (!fp)
1136 return -ENOMEM;
1137
1138 if (copy_from_user(fp->insns, fprog->filter, fsize)) {
1139 __bpf_prog_free(fp);
1140 return -EFAULT;
1141 }
1142
1143 fp->len = fprog->len;
ac67eb2c
DB		 1144 fp->orig_prog = NULL;
1145
bab18991
DB		 1146 if (save_orig) {
1147 err = bpf_prog_store_orig_filter(fp, fprog);
1148 if (err) {
1149 __bpf_prog_free(fp);
1150 return -ENOMEM;
1151 }
1152 }
1153
ac67eb2c
DB		 1154 /* bpf_prepare_filter() already takes care of freeing
1155 * memory in case something goes wrong.
1156 */
1157 fp = bpf_prepare_filter(fp, trans);
1158 if (IS_ERR(fp))
1159 return PTR_ERR(fp);
1160
1161 *pfp = fp;
1162 return 0;
1163}
2ea273d7 1164EXPORT_SYMBOL_GPL(bpf_prog_create_from_user);
ac67eb2c 1165
7ae457c1 1166void bpf_prog_destroy(struct bpf_prog *fp)
302d6637 1167{
7ae457c1 1168 __bpf_prog_release(fp);
302d6637 1169}
7ae457c1 1170EXPORT_SYMBOL_GPL(bpf_prog_destroy);
302d6637 1171
8ced425e 1172static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk)
49b31e57
DB		 1173{
1174 struct sk_filter *fp, *old_fp;
1175
1176 fp = kmalloc(sizeof(*fp), GFP_KERNEL);
1177 if (!fp)
1178 return -ENOMEM;
1179
1180 fp->prog = prog;
1181 atomic_set(&fp->refcnt, 0);
1182
1183 if (!sk_filter_charge(sk, fp)) {
1184 kfree(fp);
1185 return -ENOMEM;
1186 }
1187
8ced425e
HFS		 1188 old_fp = rcu_dereference_protected(sk->sk_filter,
1189 lockdep_sock_is_held(sk));
49b31e57 1190 rcu_assign_pointer(sk->sk_filter, fp);
8ced425e 1191
49b31e57
DB		 1192 if (old_fp)
1193 sk_filter_uncharge(sk, old_fp);
1194
1195 return 0;
1196}
1197
538950a1
CG		 1198static int __reuseport_attach_prog(struct bpf_prog *prog, struct sock *sk)
1199{
1200 struct bpf_prog *old_prog;
1201 int err;
1202
1203 if (bpf_prog_size(prog->len) > sysctl_optmem_max)
1204 return -ENOMEM;
1205
fa463497 1206 if (sk_unhashed(sk) && sk->sk_reuseport) {
538950a1
CG		 1207 err = reuseport_alloc(sk);
1208 if (err)
1209 return err;
1210 } else if (!rcu_access_pointer(sk->sk_reuseport_cb)) {
1211 /* The socket wasn't bound with SO_REUSEPORT */
1212 return -EINVAL;
1213 }
1214
1215 old_prog = reuseport_attach_prog(sk, prog);
1216 if (old_prog)
1217 bpf_prog_destroy(old_prog);
1218
1219 return 0;
1220}
1221
1222static
1223struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk)
1da177e4 1224{
009937e7 1225 unsigned int fsize = bpf_classic_proglen(fprog);
7ae457c1 1226 struct bpf_prog *prog;
1da177e4
LT		 1227 int err;
1228
d59577b6 1229 if (sock_flag(sk, SOCK_FILTER_LOCKED))
538950a1 1230 return ERR_PTR(-EPERM);
d59577b6 1231
1da177e4 1232 /* Make sure new filter is there and in the right amounts. */
f7bd9e36 1233 if (!bpf_check_basics_ok(fprog->filter, fprog->len))
538950a1 1234 return ERR_PTR(-EINVAL);
1da177e4 1235
f7bd9e36 1236 prog = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
7ae457c1 1237 if (!prog)
538950a1 1238 return ERR_PTR(-ENOMEM);
a3ea269b 1239
7ae457c1 1240 if (copy_from_user(prog->insns, fprog->filter, fsize)) {
c0d1379a 1241 __bpf_prog_free(prog);
538950a1 1242 return ERR_PTR(-EFAULT);
1da177e4
LT		 1243 }
1244
7ae457c1 1245 prog->len = fprog->len;
1da177e4 1246
7ae457c1 1247 err = bpf_prog_store_orig_filter(prog, fprog);
a3ea269b 1248 if (err) {
c0d1379a 1249 __bpf_prog_free(prog);
538950a1 1250 return ERR_PTR(-ENOMEM);
a3ea269b
DB		 1251 }
1252
7ae457c1 1253 /* bpf_prepare_filter() already takes care of freeing
bd4cf0ed
AS		 1254 * memory in case something goes wrong.
1255 */
538950a1
CG		 1256 return bpf_prepare_filter(prog, NULL);
1257}
1258
1259/**
1260 * sk_attach_filter - attach a socket filter
1261 * @fprog: the filter program
1262 * @sk: the socket to use
1263 *
1264 * Attach the user's filter code. We first run some sanity checks on
1265 * it to make sure it does not explode on us later. If an error
1266 * occurs or there is insufficient memory for the filter a negative
1267 * errno code is returned. On success the return is zero.
1268 */
8ced425e 1269int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
538950a1
CG		 1270{
1271 struct bpf_prog *prog = __get_filter(fprog, sk);
1272 int err;
1273
7ae457c1
AS		 1274 if (IS_ERR(prog))
1275 return PTR_ERR(prog);
1276
8ced425e 1277 err = __sk_attach_prog(prog, sk);
49b31e57 1278 if (err < 0) {
7ae457c1 1279 __bpf_prog_release(prog);
49b31e57 1280 return err;
278571ba
AS		 1281 }
1282
d3904b73 1283 return 0;
1da177e4 1284}
8ced425e 1285EXPORT_SYMBOL_GPL(sk_attach_filter);
1da177e4 1286
538950a1 1287int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk)
89aa0758 1288{
538950a1 1289 struct bpf_prog *prog = __get_filter(fprog, sk);
49b31e57 1290 int err;
89aa0758 1291
538950a1
CG		 1292 if (IS_ERR(prog))
1293 return PTR_ERR(prog);
1294
1295 err = __reuseport_attach_prog(prog, sk);
1296 if (err < 0) {
1297 __bpf_prog_release(prog);
1298 return err;
1299 }
1300
1301 return 0;
1302}
1303
1304static struct bpf_prog *__get_bpf(u32 ufd, struct sock *sk)
1305{
89aa0758 1306 if (sock_flag(sk, SOCK_FILTER_LOCKED))
538950a1 1307 return ERR_PTR(-EPERM);
89aa0758 1308
113214be 1309 return bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER);
538950a1
CG		 1310}
1311
1312int sk_attach_bpf(u32 ufd, struct sock *sk)
1313{
1314 struct bpf_prog *prog = __get_bpf(ufd, sk);
1315 int err;
1316
1317 if (IS_ERR(prog))
1318 return PTR_ERR(prog);
1319
8ced425e 1320 err = __sk_attach_prog(prog, sk);
49b31e57 1321 if (err < 0) {
89aa0758 1322 bpf_prog_put(prog);
49b31e57 1323 return err;
89aa0758
AS		 1324 }
1325
89aa0758
AS		 1326 return 0;
1327}
1328
538950a1
CG		 1329int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk)
1330{
1331 struct bpf_prog *prog = __get_bpf(ufd, sk);
1332 int err;
1333
1334 if (IS_ERR(prog))
1335 return PTR_ERR(prog);
1336
1337 err = __reuseport_attach_prog(prog, sk);
1338 if (err < 0) {
1339 bpf_prog_put(prog);
1340 return err;
1341 }
1342
1343 return 0;
1344}
1345
21cafc1d
DB		 1346struct bpf_scratchpad {
1347 union {
1348 __be32 diff[MAX_BPF_STACK / sizeof(__be32)];
1349 u8 buff[MAX_BPF_STACK];
1350 };
1351};
1352
1353static DEFINE_PER_CPU(struct bpf_scratchpad, bpf_sp);
91bc4822 1354
5293efe6
DB		 1355static inline int __bpf_try_make_writable(struct sk_buff *skb,
1356 unsigned int write_len)
1357{
1358 return skb_ensure_writable(skb, write_len);
1359}
1360
db58ba45
AS		 1361static inline int bpf_try_make_writable(struct sk_buff *skb,
1362 unsigned int write_len)
1363{
5293efe6 1364 int err = __bpf_try_make_writable(skb, write_len);
db58ba45 1365
0ed661d5 1366 bpf_compute_data_end(skb);
db58ba45
AS		 1367 return err;
1368}
1369
36bbef52
DB		 1370static int bpf_try_make_head_writable(struct sk_buff *skb)
1371{
1372 return bpf_try_make_writable(skb, skb_headlen(skb));
1373}
1374
a2bfe6bf
DB		 1375static inline void bpf_push_mac_rcsum(struct sk_buff *skb)
1376{
1377 if (skb_at_tc_ingress(skb))
1378 skb_postpush_rcsum(skb, skb_mac_header(skb), skb->mac_len);
1379}
1380
8065694e
DB		 1381static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
1382{
1383 if (skb_at_tc_ingress(skb))
1384 skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
1385}
1386
f3694e00
DB		 1387BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
1388 const void *, from, u32, len, u64, flags)
608cd71a 1389{
608cd71a
AS		 1390 void *ptr;
1391
8afd54c8 1392 if (unlikely(flags & ~(BPF_F_RECOMPUTE_CSUM | BPF_F_INVALIDATE_HASH)))
781c53bc 1393 return -EINVAL;
0ed661d5 1394 if (unlikely(offset > 0xffff))
608cd71a 1395 return -EFAULT;
db58ba45 1396 if (unlikely(bpf_try_make_writable(skb, offset + len)))
608cd71a
AS		 1397 return -EFAULT;
1398
0ed661d5 1399 ptr = skb->data + offset;
781c53bc 1400 if (flags & BPF_F_RECOMPUTE_CSUM)
479ffccc 1401 __skb_postpull_rcsum(skb, ptr, len, offset);
608cd71a
AS		 1402
1403 memcpy(ptr, from, len);
1404
781c53bc 1405 if (flags & BPF_F_RECOMPUTE_CSUM)
479ffccc 1406 __skb_postpush_rcsum(skb, ptr, len, offset);
8afd54c8
DB		 1407 if (flags & BPF_F_INVALIDATE_HASH)
1408 skb_clear_hash(skb);
f8ffad69 1409
608cd71a
AS		 1410 return 0;
1411}
1412
577c50aa 1413static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
608cd71a
AS		 1414 .func = bpf_skb_store_bytes,
1415 .gpl_only = false,
1416 .ret_type = RET_INTEGER,
1417 .arg1_type = ARG_PTR_TO_CTX,
1418 .arg2_type = ARG_ANYTHING,
1419 .arg3_type = ARG_PTR_TO_STACK,
1420 .arg4_type = ARG_CONST_STACK_SIZE,
91bc4822
AS		 1421 .arg5_type = ARG_ANYTHING,
1422};
1423
f3694e00
DB		 1424BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
1425 void *, to, u32, len)
05c74e5e 1426{
05c74e5e
DB		 1427 void *ptr;
1428
0ed661d5 1429 if (unlikely(offset > 0xffff))
074f528e 1430 goto err_clear;
05c74e5e
DB		 1431
1432 ptr = skb_header_pointer(skb, offset, len, to);
1433 if (unlikely(!ptr))
074f528e 1434 goto err_clear;
05c74e5e
DB		 1435 if (ptr != to)
1436 memcpy(to, ptr, len);
1437
1438 return 0;
074f528e
DB		 1439err_clear:
1440 memset(to, 0, len);
1441 return -EFAULT;
05c74e5e
DB		 1442}
1443
577c50aa 1444static const struct bpf_func_proto bpf_skb_load_bytes_proto = {
05c74e5e
DB		 1445 .func = bpf_skb_load_bytes,
1446 .gpl_only = false,
1447 .ret_type = RET_INTEGER,
1448 .arg1_type = ARG_PTR_TO_CTX,
1449 .arg2_type = ARG_ANYTHING,
074f528e 1450 .arg3_type = ARG_PTR_TO_RAW_STACK,
05c74e5e
DB		 1451 .arg4_type = ARG_CONST_STACK_SIZE,
1452};
1453
36bbef52
DB		 1454BPF_CALL_2(bpf_skb_pull_data, struct sk_buff *, skb, u32, len)
1455{
1456 /* Idea is the following: should the needed direct read/write
1457 * test fail during runtime, we can pull in more data and redo
1458 * again, since implicitly, we invalidate previous checks here.
1459 *
1460 * Or, since we know how much we need to make read/writeable,
1461 * this can be done once at the program beginning for direct
1462 * access case. By this we overcome limitations of only current
1463 * headroom being accessible.
1464 */
1465 return bpf_try_make_writable(skb, len ? : skb_headlen(skb));
1466}
1467
1468static const struct bpf_func_proto bpf_skb_pull_data_proto = {
1469 .func = bpf_skb_pull_data,
1470 .gpl_only = false,
1471 .ret_type = RET_INTEGER,
1472 .arg1_type = ARG_PTR_TO_CTX,
1473 .arg2_type = ARG_ANYTHING,
1474};
1475
f3694e00
DB		 1476BPF_CALL_5(bpf_l3_csum_replace, struct sk_buff *, skb, u32, offset,
1477 u64, from, u64, to, u64, flags)
91bc4822 1478{
0ed661d5 1479 __sum16 *ptr;
91bc4822 1480
781c53bc
DB		 1481 if (unlikely(flags & ~(BPF_F_HDR_FIELD_MASK)))
1482 return -EINVAL;
0ed661d5 1483 if (unlikely(offset > 0xffff || offset & 1))
91bc4822 1484 return -EFAULT;
0ed661d5 1485 if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
91bc4822
AS		 1486 return -EFAULT;
1487
0ed661d5 1488 ptr = (__sum16 *)(skb->data + offset);
781c53bc 1489 switch (flags & BPF_F_HDR_FIELD_MASK) {
8050c0f0
DB		 1490 case 0:
1491 if (unlikely(from != 0))
1492 return -EINVAL;
1493
1494 csum_replace_by_diff(ptr, to);
1495 break;
91bc4822
AS		 1496 case 2:
1497 csum_replace2(ptr, from, to);
1498 break;
1499 case 4:
1500 csum_replace4(ptr, from, to);
1501 break;
1502 default:
1503 return -EINVAL;
1504 }
1505
91bc4822
AS		 1506 return 0;
1507}
1508
577c50aa 1509static const struct bpf_func_proto bpf_l3_csum_replace_proto = {
91bc4822
AS		 1510 .func = bpf_l3_csum_replace,
1511 .gpl_only = false,
1512 .ret_type = RET_INTEGER,
1513 .arg1_type = ARG_PTR_TO_CTX,
1514 .arg2_type = ARG_ANYTHING,
1515 .arg3_type = ARG_ANYTHING,
1516 .arg4_type = ARG_ANYTHING,
1517 .arg5_type = ARG_ANYTHING,
1518};
1519
f3694e00
DB		 1520BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset,
1521 u64, from, u64, to, u64, flags)
91bc4822 1522{
781c53bc 1523 bool is_pseudo = flags & BPF_F_PSEUDO_HDR;
2f72959a 1524 bool is_mmzero = flags & BPF_F_MARK_MANGLED_0;
0ed661d5 1525 __sum16 *ptr;
91bc4822 1526
2f72959a
DB		 1527 if (unlikely(flags & ~(BPF_F_MARK_MANGLED_0 | BPF_F_PSEUDO_HDR |
1528 BPF_F_HDR_FIELD_MASK)))
781c53bc 1529 return -EINVAL;
0ed661d5 1530 if (unlikely(offset > 0xffff || offset & 1))
91bc4822 1531 return -EFAULT;
0ed661d5 1532 if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
91bc4822
AS		 1533 return -EFAULT;
1534
0ed661d5 1535 ptr = (__sum16 *)(skb->data + offset);
2f72959a
DB		 1536 if (is_mmzero && !*ptr)
1537 return 0;
91bc4822 1538
781c53bc 1539 switch (flags & BPF_F_HDR_FIELD_MASK) {
7d672345
DB		 1540 case 0:
1541 if (unlikely(from != 0))
1542 return -EINVAL;
1543
1544 inet_proto_csum_replace_by_diff(ptr, skb, to, is_pseudo);
1545 break;
91bc4822
AS		 1546 case 2:
1547 inet_proto_csum_replace2(ptr, skb, from, to, is_pseudo);
1548 break;
1549 case 4:
1550 inet_proto_csum_replace4(ptr, skb, from, to, is_pseudo);
1551 break;
1552 default:
1553 return -EINVAL;
1554 }
1555
2f72959a
DB		 1556 if (is_mmzero && !*ptr)
1557 *ptr = CSUM_MANGLED_0;
91bc4822
AS		 1558 return 0;
1559}
1560
577c50aa 1561static const struct bpf_func_proto bpf_l4_csum_replace_proto = {
91bc4822
AS		 1562 .func = bpf_l4_csum_replace,
1563 .gpl_only = false,
1564 .ret_type = RET_INTEGER,
1565 .arg1_type = ARG_PTR_TO_CTX,
1566 .arg2_type = ARG_ANYTHING,
1567 .arg3_type = ARG_ANYTHING,
1568 .arg4_type = ARG_ANYTHING,
1569 .arg5_type = ARG_ANYTHING,
608cd71a
AS		 1570};
1571
f3694e00
DB		 1572BPF_CALL_5(bpf_csum_diff, __be32 *, from, u32, from_size,
1573 __be32 *, to, u32, to_size, __wsum, seed)
7d672345 1574{
21cafc1d 1575 struct bpf_scratchpad *sp = this_cpu_ptr(&bpf_sp);
f3694e00 1576 u32 diff_size = from_size + to_size;
7d672345
DB		 1577 int i, j = 0;
1578
1579 /* This is quite flexible, some examples:
1580 *
1581 * from_size == 0, to_size > 0, seed := csum --> pushing data
1582 * from_size > 0, to_size == 0, seed := csum --> pulling data
1583 * from_size > 0, to_size > 0, seed := 0 --> diffing data
1584 *
1585 * Even for diffing, from_size and to_size don't need to be equal.
1586 */
1587 if (unlikely(((from_size | to_size) & (sizeof(__be32) - 1)) ||
1588 diff_size > sizeof(sp->diff)))
1589 return -EINVAL;
1590
1591 for (i = 0; i < from_size / sizeof(__be32); i++, j++)
1592 sp->diff[j] = ~from[i];
1593 for (i = 0; i < to_size / sizeof(__be32); i++, j++)
1594 sp->diff[j] = to[i];
1595
1596 return csum_partial(sp->diff, diff_size, seed);
1597}
1598
577c50aa 1599static const struct bpf_func_proto bpf_csum_diff_proto = {
7d672345
DB		 1600 .func = bpf_csum_diff,
1601 .gpl_only = false,
36bbef52 1602 .pkt_access = true,
7d672345
DB		 1603 .ret_type = RET_INTEGER,
1604 .arg1_type = ARG_PTR_TO_STACK,
1605 .arg2_type = ARG_CONST_STACK_SIZE_OR_ZERO,
1606 .arg3_type = ARG_PTR_TO_STACK,
1607 .arg4_type = ARG_CONST_STACK_SIZE_OR_ZERO,
1608 .arg5_type = ARG_ANYTHING,
1609};
1610
36bbef52
DB		 1611BPF_CALL_2(bpf_csum_update, struct sk_buff *, skb, __wsum, csum)
1612{
1613 /* The interface is to be used in combination with bpf_csum_diff()
1614 * for direct packet writes. csum rotation for alignment as well
1615 * as emulating csum_sub() can be done from the eBPF program.
1616 */
1617 if (skb->ip_summed == CHECKSUM_COMPLETE)
1618 return (skb->csum = csum_add(skb->csum, csum));
1619
1620 return -ENOTSUPP;
1621}
1622
1623static const struct bpf_func_proto bpf_csum_update_proto = {
1624 .func = bpf_csum_update,
1625 .gpl_only = false,
1626 .ret_type = RET_INTEGER,
1627 .arg1_type = ARG_PTR_TO_CTX,
1628 .arg2_type = ARG_ANYTHING,
1629};
1630
a70b506e
DB		 1631static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb)
1632{
a70b506e
DB		 1633 return dev_forward_skb(dev, skb);
1634}
1635
4e3264d2
MKL		 1636static inline int __bpf_rx_skb_no_mac(struct net_device *dev,
1637 struct sk_buff *skb)
1638{
1639 int ret = ____dev_forward_skb(dev, skb);
1640
1641 if (likely(!ret)) {
1642 skb->dev = dev;
1643 ret = netif_rx(skb);
1644 }
1645
1646 return ret;
1647}
1648
a70b506e
DB		 1649static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb)
1650{
1651 int ret;
1652
1653 if (unlikely(__this_cpu_read(xmit_recursion) > XMIT_RECURSION_LIMIT)) {
1654 net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
1655 kfree_skb(skb);
1656 return -ENETDOWN;
1657 }
1658
1659 skb->dev = dev;
1660
1661 __this_cpu_inc(xmit_recursion);
1662 ret = dev_queue_xmit(skb);
1663 __this_cpu_dec(xmit_recursion);
1664
1665 return ret;
1666}
1667
4e3264d2
MKL		 1668static int __bpf_redirect_no_mac(struct sk_buff *skb, struct net_device *dev,
1669 u32 flags)
1670{
1671 /* skb->mac_len is not set on normal egress */
1672 unsigned int mlen = skb->network_header - skb->mac_header;
1673
1674 __skb_pull(skb, mlen);
1675
1676 /* At ingress, the mac header has already been pulled once.
1677 * At egress, skb_pospull_rcsum has to be done in case that
1678 * the skb is originated from ingress (i.e. a forwarded skb)
1679 * to ensure that rcsum starts at net header.
1680 */
1681 if (!skb_at_tc_ingress(skb))
1682 skb_postpull_rcsum(skb, skb_mac_header(skb), mlen);
1683 skb_pop_mac_header(skb);
1684 skb_reset_mac_len(skb);
1685 return flags & BPF_F_INGRESS ?
1686 __bpf_rx_skb_no_mac(dev, skb) : __bpf_tx_skb(dev, skb);
1687}
1688
1689static int __bpf_redirect_common(struct sk_buff *skb, struct net_device *dev,
1690 u32 flags)
1691{
3a0af8fd
TG		 1692 /* Verify that a link layer header is carried */
1693 if (unlikely(skb->mac_header >= skb->network_header)) {
1694 kfree_skb(skb);
1695 return -ERANGE;
1696 }
1697
4e3264d2
MKL		 1698 bpf_push_mac_rcsum(skb);
1699 return flags & BPF_F_INGRESS ?
1700 __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb);
1701}
1702
1703static int __bpf_redirect(struct sk_buff *skb, struct net_device *dev,
1704 u32 flags)
1705{
c491680f 1706 if (dev_is_mac_header_xmit(dev))
4e3264d2 1707 return __bpf_redirect_common(skb, dev, flags);
c491680f
DB		 1708 else
1709 return __bpf_redirect_no_mac(skb, dev, flags);
4e3264d2
MKL		 1710}
1711
f3694e00 1712BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags)
3896d655 1713{
3896d655 1714 struct net_device *dev;
36bbef52
DB		 1715 struct sk_buff *clone;
1716 int ret;
3896d655 1717
781c53bc
DB		 1718 if (unlikely(flags & ~(BPF_F_INGRESS)))
1719 return -EINVAL;
1720
3896d655
AS		 1721 dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex);
1722 if (unlikely(!dev))
1723 return -EINVAL;
1724
36bbef52
DB		 1725 clone = skb_clone(skb, GFP_ATOMIC);
1726 if (unlikely(!clone))
3896d655
AS		 1727 return -ENOMEM;
1728
36bbef52
DB		 1729 /* For direct write, we need to keep the invariant that the skbs
1730 * we're dealing with need to be uncloned. Should uncloning fail
1731 * here, we need to free the just generated clone to unclone once
1732 * again.
1733 */
1734 ret = bpf_try_make_head_writable(skb);
1735 if (unlikely(ret)) {
1736 kfree_skb(clone);
1737 return -ENOMEM;
1738 }
1739
4e3264d2 1740 return __bpf_redirect(clone, dev, flags);
3896d655
AS		 1741}
1742
577c50aa 1743static const struct bpf_func_proto bpf_clone_redirect_proto = {
3896d655
AS		 1744 .func = bpf_clone_redirect,
1745 .gpl_only = false,
1746 .ret_type = RET_INTEGER,
1747 .arg1_type = ARG_PTR_TO_CTX,
1748 .arg2_type = ARG_ANYTHING,
1749 .arg3_type = ARG_ANYTHING,
1750};
1751
27b29f63
AS		 1752struct redirect_info {
1753 u32 ifindex;
1754 u32 flags;
1755};
1756
1757static DEFINE_PER_CPU(struct redirect_info, redirect_info);
781c53bc 1758
f3694e00 1759BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
27b29f63
AS		 1760{
1761 struct redirect_info *ri = this_cpu_ptr(&redirect_info);
1762
781c53bc
DB		 1763 if (unlikely(flags & ~(BPF_F_INGRESS)))
1764 return TC_ACT_SHOT;
1765
27b29f63
AS		 1766 ri->ifindex = ifindex;
1767 ri->flags = flags;
781c53bc 1768
27b29f63
AS		 1769 return TC_ACT_REDIRECT;
1770}
1771
1772int skb_do_redirect(struct sk_buff *skb)
1773{
1774 struct redirect_info *ri = this_cpu_ptr(&redirect_info);
1775 struct net_device *dev;
1776
1777 dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->ifindex);
1778 ri->ifindex = 0;
1779 if (unlikely(!dev)) {
1780 kfree_skb(skb);
1781 return -EINVAL;
1782 }
1783
4e3264d2 1784 return __bpf_redirect(skb, dev, ri->flags);
27b29f63
AS		 1785}
1786
577c50aa 1787static const struct bpf_func_proto bpf_redirect_proto = {
27b29f63
AS		 1788 .func = bpf_redirect,
1789 .gpl_only = false,
1790 .ret_type = RET_INTEGER,
1791 .arg1_type = ARG_ANYTHING,
1792 .arg2_type = ARG_ANYTHING,
1793};
1794
f3694e00 1795BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb)
8d20aabe 1796{
f3694e00 1797 return task_get_classid(skb);
8d20aabe
DB		 1798}
1799
1800static const struct bpf_func_proto bpf_get_cgroup_classid_proto = {
1801 .func = bpf_get_cgroup_classid,
1802 .gpl_only = false,
1803 .ret_type = RET_INTEGER,
1804 .arg1_type = ARG_PTR_TO_CTX,
1805};
1806
f3694e00 1807BPF_CALL_1(bpf_get_route_realm, const struct sk_buff *, skb)
c46646d0 1808{
f3694e00 1809 return dst_tclassid(skb);
c46646d0
DB		 1810}
1811
1812static const struct bpf_func_proto bpf_get_route_realm_proto = {
1813 .func = bpf_get_route_realm,
1814 .gpl_only = false,
1815 .ret_type = RET_INTEGER,
1816 .arg1_type = ARG_PTR_TO_CTX,
1817};
1818
f3694e00 1819BPF_CALL_1(bpf_get_hash_recalc, struct sk_buff *, skb)
13c5c240
DB		 1820{
1821 /* If skb_clear_hash() was called due to mangling, we can
1822 * trigger SW recalculation here. Later access to hash
1823 * can then use the inline skb->hash via context directly
1824 * instead of calling this helper again.
1825 */
f3694e00 1826 return skb_get_hash(skb);
13c5c240
DB		 1827}
1828
1829static const struct bpf_func_proto bpf_get_hash_recalc_proto = {
1830 .func = bpf_get_hash_recalc,
1831 .gpl_only = false,
1832 .ret_type = RET_INTEGER,
1833 .arg1_type = ARG_PTR_TO_CTX,
1834};
1835
7a4b28c6
DB		 1836BPF_CALL_1(bpf_set_hash_invalid, struct sk_buff *, skb)
1837{
1838 /* After all direct packet write, this can be used once for
1839 * triggering a lazy recalc on next skb_get_hash() invocation.
1840 */
1841 skb_clear_hash(skb);
1842 return 0;
1843}
1844
1845static const struct bpf_func_proto bpf_set_hash_invalid_proto = {
1846 .func = bpf_set_hash_invalid,
1847 .gpl_only = false,
1848 .ret_type = RET_INTEGER,
1849 .arg1_type = ARG_PTR_TO_CTX,
1850};
1851
f3694e00
DB		 1852BPF_CALL_3(bpf_skb_vlan_push, struct sk_buff *, skb, __be16, vlan_proto,
1853 u16, vlan_tci)
4e10df9a 1854{
db58ba45 1855 int ret;
4e10df9a
AS		 1856
1857 if (unlikely(vlan_proto != htons(ETH_P_8021Q) &&
1858 vlan_proto != htons(ETH_P_8021AD)))
1859 vlan_proto = htons(ETH_P_8021Q);
1860
8065694e 1861 bpf_push_mac_rcsum(skb);
db58ba45 1862 ret = skb_vlan_push(skb, vlan_proto, vlan_tci);
8065694e
DB		 1863 bpf_pull_mac_rcsum(skb);
1864
db58ba45
AS		 1865 bpf_compute_data_end(skb);
1866 return ret;
4e10df9a
AS		 1867}
1868
1869const struct bpf_func_proto bpf_skb_vlan_push_proto = {
1870 .func = bpf_skb_vlan_push,
1871 .gpl_only = false,
1872 .ret_type = RET_INTEGER,
1873 .arg1_type = ARG_PTR_TO_CTX,
1874 .arg2_type = ARG_ANYTHING,
1875 .arg3_type = ARG_ANYTHING,
1876};
4d9c5c53 1877EXPORT_SYMBOL_GPL(bpf_skb_vlan_push_proto);
4e10df9a 1878
f3694e00 1879BPF_CALL_1(bpf_skb_vlan_pop, struct sk_buff *, skb)
4e10df9a 1880{
db58ba45 1881 int ret;
4e10df9a 1882
8065694e 1883 bpf_push_mac_rcsum(skb);
db58ba45 1884 ret = skb_vlan_pop(skb);
8065694e
DB		 1885 bpf_pull_mac_rcsum(skb);
1886
db58ba45
AS		 1887 bpf_compute_data_end(skb);
1888 return ret;
4e10df9a
AS		 1889}
1890
1891const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
1892 .func = bpf_skb_vlan_pop,
1893 .gpl_only = false,
1894 .ret_type = RET_INTEGER,
1895 .arg1_type = ARG_PTR_TO_CTX,
1896};
4d9c5c53 1897EXPORT_SYMBOL_GPL(bpf_skb_vlan_pop_proto);
4e10df9a 1898
6578171a
DB		 1899static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len)
1900{
1901 /* Caller already did skb_cow() with len as headroom,
1902 * so no need to do it here.
1903 */
1904 skb_push(skb, len);
1905 memmove(skb->data, skb->data + len, off);
1906 memset(skb->data + off, 0, len);
1907
1908 /* No skb_postpush_rcsum(skb, skb->data + off, len)
1909 * needed here as it does not change the skb->csum
1910 * result for checksum complete when summing over
1911 * zeroed blocks.
1912 */
1913 return 0;
1914}
1915
1916static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len)
1917{
1918 /* skb_ensure_writable() is not needed here, as we're
1919 * already working on an uncloned skb.
1920 */
1921 if (unlikely(!pskb_may_pull(skb, off + len)))
1922 return -ENOMEM;
1923
1924 skb_postpull_rcsum(skb, skb->data + off, len);
1925 memmove(skb->data + len, skb->data, off);
1926 __skb_pull(skb, len);
1927
1928 return 0;
1929}
1930
1931static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len)
1932{
1933 bool trans_same = skb->transport_header == skb->network_header;
1934 int ret;
1935
1936 /* There's no need for __skb_push()/__skb_pull() pair to
1937 * get to the start of the mac header as we're guaranteed
1938 * to always start from here under eBPF.
1939 */
1940 ret = bpf_skb_generic_push(skb, off, len);
1941 if (likely(!ret)) {
1942 skb->mac_header -= len;
1943 skb->network_header -= len;
1944 if (trans_same)
1945 skb->transport_header = skb->network_header;
1946 }
1947
1948 return ret;
1949}
1950
1951static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len)
1952{
1953 bool trans_same = skb->transport_header == skb->network_header;
1954 int ret;
1955
1956 /* Same here, __skb_push()/__skb_pull() pair not needed. */
1957 ret = bpf_skb_generic_pop(skb, off, len);
1958 if (likely(!ret)) {
1959 skb->mac_header += len;
1960 skb->network_header += len;
1961 if (trans_same)
1962 skb->transport_header = skb->network_header;
1963 }
1964
1965 return ret;
1966}
1967
1968static int bpf_skb_proto_4_to_6(struct sk_buff *skb)
1969{
1970 const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
1971 u32 off = skb->network_header - skb->mac_header;
1972 int ret;
1973
1974 ret = skb_cow(skb, len_diff);
1975 if (unlikely(ret < 0))
1976 return ret;
1977
1978 ret = bpf_skb_net_hdr_push(skb, off, len_diff);
1979 if (unlikely(ret < 0))
1980 return ret;
1981
1982 if (skb_is_gso(skb)) {
1983 /* SKB_GSO_UDP stays as is. SKB_GSO_TCPV4 needs to
1984 * be changed into SKB_GSO_TCPV6.
1985 */
1986 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1987 skb_shinfo(skb)->gso_type &= ~SKB_GSO_TCPV4;
1988 skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
1989 }
1990
1991 /* Due to IPv6 header, MSS needs to be downgraded. */
1992 skb_shinfo(skb)->gso_size -= len_diff;
1993 /* Header must be checked, and gso_segs recomputed. */
1994 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1995 skb_shinfo(skb)->gso_segs = 0;
1996 }
1997
1998 skb->protocol = htons(ETH_P_IPV6);
1999 skb_clear_hash(skb);
2000
2001 return 0;
2002}
2003
2004static int bpf_skb_proto_6_to_4(struct sk_buff *skb)
2005{
2006 const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
2007 u32 off = skb->network_header - skb->mac_header;
2008 int ret;
2009
2010 ret = skb_unclone(skb, GFP_ATOMIC);
2011 if (unlikely(ret < 0))
2012 return ret;
2013
2014 ret = bpf_skb_net_hdr_pop(skb, off, len_diff);
2015 if (unlikely(ret < 0))
2016 return ret;
2017
2018 if (skb_is_gso(skb)) {
2019 /* SKB_GSO_UDP stays as is. SKB_GSO_TCPV6 needs to
2020 * be changed into SKB_GSO_TCPV4.
2021 */
2022 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
2023 skb_shinfo(skb)->gso_type &= ~SKB_GSO_TCPV6;
2024 skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4;
2025 }
2026
2027 /* Due to IPv4 header, MSS can be upgraded. */
2028 skb_shinfo(skb)->gso_size += len_diff;
2029 /* Header must be checked, and gso_segs recomputed. */
2030 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2031 skb_shinfo(skb)->gso_segs = 0;
2032 }
2033
2034 skb->protocol = htons(ETH_P_IP);
2035 skb_clear_hash(skb);
2036
2037 return 0;
2038}
2039
2040static int bpf_skb_proto_xlat(struct sk_buff *skb, __be16 to_proto)
2041{
2042 __be16 from_proto = skb->protocol;
2043
2044 if (from_proto == htons(ETH_P_IP) &&
2045 to_proto == htons(ETH_P_IPV6))
2046 return bpf_skb_proto_4_to_6(skb);
2047
2048 if (from_proto == htons(ETH_P_IPV6) &&
2049 to_proto == htons(ETH_P_IP))
2050 return bpf_skb_proto_6_to_4(skb);
2051
2052 return -ENOTSUPP;
2053}
2054
f3694e00
DB		 2055BPF_CALL_3(bpf_skb_change_proto, struct sk_buff *, skb, __be16, proto,
2056 u64, flags)
6578171a 2057{
6578171a
DB		 2058 int ret;
2059
2060 if (unlikely(flags))
2061 return -EINVAL;
2062
2063 /* General idea is that this helper does the basic groundwork
2064 * needed for changing the protocol, and eBPF program fills the
2065 * rest through bpf_skb_store_bytes(), bpf_lX_csum_replace()
2066 * and other helpers, rather than passing a raw buffer here.
2067 *
2068 * The rationale is to keep this minimal and without a need to
2069 * deal with raw packet data. F.e. even if we would pass buffers
2070 * here, the program still needs to call the bpf_lX_csum_replace()
2071 * helpers anyway. Plus, this way we keep also separation of
2072 * concerns, since f.e. bpf_skb_store_bytes() should only take
2073 * care of stores.
2074 *
2075 * Currently, additional options and extension header space are
2076 * not supported, but flags register is reserved so we can adapt
2077 * that. For offloads, we mark packet as dodgy, so that headers
2078 * need to be verified first.
2079 */
2080 ret = bpf_skb_proto_xlat(skb, proto);
2081 bpf_compute_data_end(skb);
2082 return ret;
2083}
2084
2085static const struct bpf_func_proto bpf_skb_change_proto_proto = {
2086 .func = bpf_skb_change_proto,
2087 .gpl_only = false,
2088 .ret_type = RET_INTEGER,
2089 .arg1_type = ARG_PTR_TO_CTX,
2090 .arg2_type = ARG_ANYTHING,
2091 .arg3_type = ARG_ANYTHING,
2092};
2093
f3694e00 2094BPF_CALL_2(bpf_skb_change_type, struct sk_buff *, skb, u32, pkt_type)
d2485c42 2095{
d2485c42 2096 /* We only allow a restricted subset to be changed for now. */
45c7fffa
DB		 2097 if (unlikely(!skb_pkt_type_ok(skb->pkt_type) ||
2098 !skb_pkt_type_ok(pkt_type)))
d2485c42
DB		 2099 return -EINVAL;
2100
2101 skb->pkt_type = pkt_type;
2102 return 0;
2103}
2104
2105static const struct bpf_func_proto bpf_skb_change_type_proto = {
2106 .func = bpf_skb_change_type,
2107 .gpl_only = false,
2108 .ret_type = RET_INTEGER,
2109 .arg1_type = ARG_PTR_TO_CTX,
2110 .arg2_type = ARG_ANYTHING,
2111};
2112
5293efe6
DB		 2113static u32 __bpf_skb_min_len(const struct sk_buff *skb)
2114{
2115 u32 min_len = skb_network_offset(skb);
2116
2117 if (skb_transport_header_was_set(skb))
2118 min_len = skb_transport_offset(skb);
2119 if (skb->ip_summed == CHECKSUM_PARTIAL)
2120 min_len = skb_checksum_start_offset(skb) +
2121 skb->csum_offset + sizeof(__sum16);
2122 return min_len;
2123}
2124
2125static u32 __bpf_skb_max_len(const struct sk_buff *skb)
2126{
6088b582 2127 return skb->dev->mtu + skb->dev->hard_header_len;
5293efe6
DB		 2128}
2129
2130static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len)
2131{
2132 unsigned int old_len = skb->len;
2133 int ret;
2134
2135 ret = __skb_grow_rcsum(skb, new_len);
2136 if (!ret)
2137 memset(skb->data + old_len, 0, new_len - old_len);
2138 return ret;
2139}
2140
2141static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len)
2142{
2143 return __skb_trim_rcsum(skb, new_len);
2144}
2145
f3694e00
DB		 2146BPF_CALL_3(bpf_skb_change_tail, struct sk_buff *, skb, u32, new_len,
2147 u64, flags)
5293efe6 2148{
5293efe6
DB		 2149 u32 max_len = __bpf_skb_max_len(skb);
2150 u32 min_len = __bpf_skb_min_len(skb);
5293efe6
DB		 2151 int ret;
2152
2153 if (unlikely(flags || new_len > max_len || new_len < min_len))
2154 return -EINVAL;
2155 if (skb->encapsulation)
2156 return -ENOTSUPP;
2157
2158 /* The basic idea of this helper is that it's performing the
2159 * needed work to either grow or trim an skb, and eBPF program
2160 * rewrites the rest via helpers like bpf_skb_store_bytes(),
2161 * bpf_lX_csum_replace() and others rather than passing a raw
2162 * buffer here. This one is a slow path helper and intended
2163 * for replies with control messages.
2164 *
2165 * Like in bpf_skb_change_proto(), we want to keep this rather
2166 * minimal and without protocol specifics so that we are able
2167 * to separate concerns as in bpf_skb_store_bytes() should only
2168 * be the one responsible for writing buffers.
2169 *
2170 * It's really expected to be a slow path operation here for
2171 * control message replies, so we're implicitly linearizing,
2172 * uncloning and drop offloads from the skb by this.
2173 */
2174 ret = __bpf_try_make_writable(skb, skb->len);
2175 if (!ret) {
2176 if (new_len > skb->len)
2177 ret = bpf_skb_grow_rcsum(skb, new_len);
2178 else if (new_len < skb->len)
2179 ret = bpf_skb_trim_rcsum(skb, new_len);
2180 if (!ret && skb_is_gso(skb))
2181 skb_gso_reset(skb);
2182 }
2183
2184 bpf_compute_data_end(skb);
2185 return ret;
2186}
2187
2188static const struct bpf_func_proto bpf_skb_change_tail_proto = {
2189 .func = bpf_skb_change_tail,
2190 .gpl_only = false,
2191 .ret_type = RET_INTEGER,
2192 .arg1_type = ARG_PTR_TO_CTX,
2193 .arg2_type = ARG_ANYTHING,
2194 .arg3_type = ARG_ANYTHING,
2195};
2196
3a0af8fd
TG		 2197BPF_CALL_3(bpf_skb_change_head, struct sk_buff *, skb, u32, head_room,
2198 u64, flags)
2199{
2200 u32 max_len = __bpf_skb_max_len(skb);
2201 u32 new_len = skb->len + head_room;
2202 int ret;
2203
2204 if (unlikely(flags || (!skb_is_gso(skb) && new_len > max_len) ||
2205 new_len < skb->len))
2206 return -EINVAL;
2207
2208 ret = skb_cow(skb, head_room);
2209 if (likely(!ret)) {
2210 /* Idea for this helper is that we currently only
2211 * allow to expand on mac header. This means that
2212 * skb->protocol network header, etc, stay as is.
2213 * Compared to bpf_skb_change_tail(), we're more
2214 * flexible due to not needing to linearize or
2215 * reset GSO. Intention for this helper is to be
2216 * used by an L3 skb that needs to push mac header
2217 * for redirection into L2 device.
2218 */
2219 __skb_push(skb, head_room);
2220 memset(skb->data, 0, head_room);
2221 skb_reset_mac_header(skb);
2222 }
2223
2224 bpf_compute_data_end(skb);
2225 return 0;
2226}
2227
2228static const struct bpf_func_proto bpf_skb_change_head_proto = {
2229 .func = bpf_skb_change_head,
2230 .gpl_only = false,
2231 .ret_type = RET_INTEGER,
2232 .arg1_type = ARG_PTR_TO_CTX,
2233 .arg2_type = ARG_ANYTHING,
2234 .arg3_type = ARG_ANYTHING,
2235};
2236
17bedab2
MKL		 2237BPF_CALL_2(bpf_xdp_adjust_head, struct xdp_buff *, xdp, int, offset)
2238{
2239 void *data = xdp->data + offset;
2240
2241 if (unlikely(data < xdp->data_hard_start ||
2242 data > xdp->data_end - ETH_HLEN))
2243 return -EINVAL;
2244
2245 xdp->data = data;
2246
2247 return 0;
2248}
2249
2250static const struct bpf_func_proto bpf_xdp_adjust_head_proto = {
2251 .func = bpf_xdp_adjust_head,
2252 .gpl_only = false,
2253 .ret_type = RET_INTEGER,
2254 .arg1_type = ARG_PTR_TO_CTX,
2255 .arg2_type = ARG_ANYTHING,
2256};
2257
2258bool bpf_helper_changes_pkt_data(void *func)
4e10df9a 2259{
36bbef52
DB		 2260 if (func == bpf_skb_vlan_push ||
2261 func == bpf_skb_vlan_pop ||
2262 func == bpf_skb_store_bytes ||
2263 func == bpf_skb_change_proto ||
3a0af8fd 2264 func == bpf_skb_change_head ||
36bbef52
DB		 2265 func == bpf_skb_change_tail ||
2266 func == bpf_skb_pull_data ||
2267 func == bpf_l3_csum_replace ||
17bedab2
MKL		 2268 func == bpf_l4_csum_replace ||
2269 func == bpf_xdp_adjust_head)
3697649f
DB		 2270 return true;
2271
4e10df9a
AS		 2272 return false;
2273}
2274
555c8a86 2275static unsigned long bpf_skb_copy(void *dst_buff, const void *skb,
aa7145c1 2276 unsigned long off, unsigned long len)
555c8a86 2277{
aa7145c1 2278 void *ptr = skb_header_pointer(skb, off, len, dst_buff);
555c8a86
DB		 2279
2280 if (unlikely(!ptr))
2281 return len;
2282 if (ptr != dst_buff)
2283 memcpy(dst_buff, ptr, len);
2284
2285 return 0;
2286}
2287
f3694e00
DB		 2288BPF_CALL_5(bpf_skb_event_output, struct sk_buff *, skb, struct bpf_map *, map,
2289 u64, flags, void *, meta, u64, meta_size)
555c8a86 2290{
555c8a86 2291 u64 skb_size = (flags & BPF_F_CTXLEN_MASK) >> 32;
555c8a86
DB		 2292
2293 if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
2294 return -EINVAL;
2295 if (unlikely(skb_size > skb->len))
2296 return -EFAULT;
2297
2298 return bpf_event_output(map, flags, meta, meta_size, skb, skb_size,
2299 bpf_skb_copy);
2300}
2301
2302static const struct bpf_func_proto bpf_skb_event_output_proto = {
2303 .func = bpf_skb_event_output,
2304 .gpl_only = true,
2305 .ret_type = RET_INTEGER,
2306 .arg1_type = ARG_PTR_TO_CTX,
2307 .arg2_type = ARG_CONST_MAP_PTR,
2308 .arg3_type = ARG_ANYTHING,
2309 .arg4_type = ARG_PTR_TO_STACK,
2310 .arg5_type = ARG_CONST_STACK_SIZE,
2311};
2312
c6c33454
DB		 2313static unsigned short bpf_tunnel_key_af(u64 flags)
2314{
2315 return flags & BPF_F_TUNINFO_IPV6 ? AF_INET6 : AF_INET;
2316}
2317
f3694e00
DB		 2318BPF_CALL_4(bpf_skb_get_tunnel_key, struct sk_buff *, skb, struct bpf_tunnel_key *, to,
2319 u32, size, u64, flags)
d3aa45ce 2320{
c6c33454
DB		 2321 const struct ip_tunnel_info *info = skb_tunnel_info(skb);
2322 u8 compat[sizeof(struct bpf_tunnel_key)];
074f528e
DB		 2323 void *to_orig = to;
2324 int err;
d3aa45ce 2325
074f528e
DB		 2326 if (unlikely(!info || (flags & ~(BPF_F_TUNINFO_IPV6)))) {
2327 err = -EINVAL;
2328 goto err_clear;
2329 }
2330 if (ip_tunnel_info_af(info) != bpf_tunnel_key_af(flags)) {
2331 err = -EPROTO;
2332 goto err_clear;
2333 }
c6c33454 2334 if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
074f528e 2335 err = -EINVAL;
c6c33454 2336 switch (size) {
4018ab18 2337 case offsetof(struct bpf_tunnel_key, tunnel_label):
c0e760c9 2338 case offsetof(struct bpf_tunnel_key, tunnel_ext):
4018ab18 2339 goto set_compat;
c6c33454
DB		 2340 case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
2341 /* Fixup deprecated structure layouts here, so we have
2342 * a common path later on.
2343 */
2344 if (ip_tunnel_info_af(info) != AF_INET)
074f528e 2345 goto err_clear;
4018ab18 2346set_compat:
c6c33454
DB		 2347 to = (struct bpf_tunnel_key *)compat;
2348 break;
2349 default:
074f528e 2350 goto err_clear;
c6c33454
DB		 2351 }
2352 }
d3aa45ce
AS		 2353
2354 to->tunnel_id = be64_to_cpu(info->key.tun_id);
c6c33454
DB		 2355 to->tunnel_tos = info->key.tos;
2356 to->tunnel_ttl = info->key.ttl;
2357
4018ab18 2358 if (flags & BPF_F_TUNINFO_IPV6) {
c6c33454
DB		 2359 memcpy(to->remote_ipv6, &info->key.u.ipv6.src,
2360 sizeof(to->remote_ipv6));
4018ab18
DB		 2361 to->tunnel_label = be32_to_cpu(info->key.label);
2362 } else {
c6c33454 2363 to->remote_ipv4 = be32_to_cpu(info->key.u.ipv4.src);
4018ab18 2364 }
c6c33454
DB		 2365
2366 if (unlikely(size != sizeof(struct bpf_tunnel_key)))
074f528e 2367 memcpy(to_orig, to, size);
d3aa45ce
AS		 2368
2369 return 0;
074f528e
DB		 2370err_clear:
2371 memset(to_orig, 0, size);
2372 return err;
d3aa45ce
AS		 2373}
2374
577c50aa 2375static const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = {
d3aa45ce
AS		 2376 .func = bpf_skb_get_tunnel_key,
2377 .gpl_only = false,
2378 .ret_type = RET_INTEGER,
2379 .arg1_type = ARG_PTR_TO_CTX,
074f528e 2380 .arg2_type = ARG_PTR_TO_RAW_STACK,
d3aa45ce
AS		 2381 .arg3_type = ARG_CONST_STACK_SIZE,
2382 .arg4_type = ARG_ANYTHING,
2383};
2384
f3694e00 2385BPF_CALL_3(bpf_skb_get_tunnel_opt, struct sk_buff *, skb, u8 *, to, u32, size)
14ca0751 2386{
14ca0751 2387 const struct ip_tunnel_info *info = skb_tunnel_info(skb);
074f528e 2388 int err;
14ca0751
DB		 2389
2390 if (unlikely(!info ||
074f528e
DB		 2391 !(info->key.tun_flags & TUNNEL_OPTIONS_PRESENT))) {
2392 err = -ENOENT;
2393 goto err_clear;
2394 }
2395 if (unlikely(size < info->options_len)) {
2396 err = -ENOMEM;
2397 goto err_clear;
2398 }
14ca0751
DB		 2399
2400 ip_tunnel_info_opts_get(to, info);
074f528e
DB		 2401 if (size > info->options_len)
2402 memset(to + info->options_len, 0, size - info->options_len);
14ca0751
DB		 2403
2404 return info->options_len;
074f528e
DB		 2405err_clear:
2406 memset(to, 0, size);
2407 return err;
14ca0751
DB		 2408}
2409
2410static const struct bpf_func_proto bpf_skb_get_tunnel_opt_proto = {
2411 .func = bpf_skb_get_tunnel_opt,
2412 .gpl_only = false,
2413 .ret_type = RET_INTEGER,
2414 .arg1_type = ARG_PTR_TO_CTX,
074f528e 2415 .arg2_type = ARG_PTR_TO_RAW_STACK,
14ca0751
DB		 2416 .arg3_type = ARG_CONST_STACK_SIZE,
2417};
2418
d3aa45ce
AS		 2419static struct metadata_dst __percpu *md_dst;
2420
f3694e00
DB		 2421BPF_CALL_4(bpf_skb_set_tunnel_key, struct sk_buff *, skb,
2422 const struct bpf_tunnel_key *, from, u32, size, u64, flags)
d3aa45ce 2423{
d3aa45ce 2424 struct metadata_dst *md = this_cpu_ptr(md_dst);
c6c33454 2425 u8 compat[sizeof(struct bpf_tunnel_key)];
d3aa45ce
AS		 2426 struct ip_tunnel_info *info;
2427
22080870
DB		 2428 if (unlikely(flags & ~(BPF_F_TUNINFO_IPV6 | BPF_F_ZERO_CSUM_TX |
2429 BPF_F_DONT_FRAGMENT)))
d3aa45ce 2430 return -EINVAL;
c6c33454
DB		 2431 if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
2432 switch (size) {
4018ab18 2433 case offsetof(struct bpf_tunnel_key, tunnel_label):
c0e760c9 2434 case offsetof(struct bpf_tunnel_key, tunnel_ext):
c6c33454
DB		 2435 case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
2436 /* Fixup deprecated structure layouts here, so we have
2437 * a common path later on.
2438 */
2439 memcpy(compat, from, size);
2440 memset(compat + size, 0, sizeof(compat) - size);
f3694e00 2441 from = (const struct bpf_tunnel_key *) compat;
c6c33454
DB		 2442 break;
2443 default:
2444 return -EINVAL;
2445 }
2446 }
c0e760c9
DB		 2447 if (unlikely((!(flags & BPF_F_TUNINFO_IPV6) && from->tunnel_label) ||
2448 from->tunnel_ext))
4018ab18 2449 return -EINVAL;
d3aa45ce
AS		 2450
2451 skb_dst_drop(skb);
2452 dst_hold((struct dst_entry *) md);
2453 skb_dst_set(skb, (struct dst_entry *) md);
2454
2455 info = &md->u.tun_info;
2456 info->mode = IP_TUNNEL_INFO_TX;
c6c33454 2457
db3c6139 2458 info->key.tun_flags = TUNNEL_KEY | TUNNEL_CSUM | TUNNEL_NOCACHE;
22080870
DB		 2459 if (flags & BPF_F_DONT_FRAGMENT)
2460 info->key.tun_flags |= TUNNEL_DONT_FRAGMENT;
2461
d3aa45ce 2462 info->key.tun_id = cpu_to_be64(from->tunnel_id);
c6c33454
DB		 2463 info->key.tos = from->tunnel_tos;
2464 info->key.ttl = from->tunnel_ttl;
2465
2466 if (flags & BPF_F_TUNINFO_IPV6) {
2467 info->mode |= IP_TUNNEL_INFO_IPV6;
2468 memcpy(&info->key.u.ipv6.dst, from->remote_ipv6,
2469 sizeof(from->remote_ipv6));
4018ab18
DB		 2470 info->key.label = cpu_to_be32(from->tunnel_label) &
2471 IPV6_FLOWLABEL_MASK;
c6c33454
DB		 2472 } else {
2473 info->key.u.ipv4.dst = cpu_to_be32(from->remote_ipv4);
2da897e5
DB		 2474 if (flags & BPF_F_ZERO_CSUM_TX)
2475 info->key.tun_flags &= ~TUNNEL_CSUM;
c6c33454 2476 }
d3aa45ce
AS		 2477
2478 return 0;
2479}
2480
577c50aa 2481static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = {
d3aa45ce
AS		 2482 .func = bpf_skb_set_tunnel_key,
2483 .gpl_only = false,
2484 .ret_type = RET_INTEGER,
2485 .arg1_type = ARG_PTR_TO_CTX,
2486 .arg2_type = ARG_PTR_TO_STACK,
2487 .arg3_type = ARG_CONST_STACK_SIZE,
2488 .arg4_type = ARG_ANYTHING,
2489};
2490
f3694e00
DB		 2491BPF_CALL_3(bpf_skb_set_tunnel_opt, struct sk_buff *, skb,
2492 const u8 *, from, u32, size)
14ca0751 2493{
14ca0751
DB		 2494 struct ip_tunnel_info *info = skb_tunnel_info(skb);
2495 const struct metadata_dst *md = this_cpu_ptr(md_dst);
2496
2497 if (unlikely(info != &md->u.tun_info || (size & (sizeof(u32) - 1))))
2498 return -EINVAL;
fca5fdf6 2499 if (unlikely(size > IP_TUNNEL_OPTS_MAX))
14ca0751
DB		 2500 return -ENOMEM;
2501
2502 ip_tunnel_info_opts_set(info, from, size);
2503
2504 return 0;
2505}
2506
2507static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = {
2508 .func = bpf_skb_set_tunnel_opt,
2509 .gpl_only = false,
2510 .ret_type = RET_INTEGER,
2511 .arg1_type = ARG_PTR_TO_CTX,
2512 .arg2_type = ARG_PTR_TO_STACK,
2513 .arg3_type = ARG_CONST_STACK_SIZE,
2514};
2515
2516static const struct bpf_func_proto *
2517bpf_get_skb_set_tunnel_proto(enum bpf_func_id which)
d3aa45ce
AS		 2518{
2519 if (!md_dst) {
14ca0751
DB		 2520 /* Race is not possible, since it's called from verifier
2521 * that is holding verifier mutex.
d3aa45ce 2522 */
fca5fdf6 2523 md_dst = metadata_dst_alloc_percpu(IP_TUNNEL_OPTS_MAX,
14ca0751 2524 GFP_KERNEL);
d3aa45ce
AS		 2525 if (!md_dst)
2526 return NULL;
2527 }
14ca0751
DB		 2528
2529 switch (which) {
2530 case BPF_FUNC_skb_set_tunnel_key:
2531 return &bpf_skb_set_tunnel_key_proto;
2532 case BPF_FUNC_skb_set_tunnel_opt:
2533 return &bpf_skb_set_tunnel_opt_proto;
2534 default:
2535 return NULL;
2536 }
d3aa45ce
AS		 2537}
2538
f3694e00
DB		 2539BPF_CALL_3(bpf_skb_under_cgroup, struct sk_buff *, skb, struct bpf_map *, map,
2540 u32, idx)
4a482f34 2541{
4a482f34
MKL		 2542 struct bpf_array *array = container_of(map, struct bpf_array, map);
2543 struct cgroup *cgrp;
2544 struct sock *sk;
4a482f34 2545
2d48c5f9 2546 sk = skb_to_full_sk(skb);
4a482f34
MKL		 2547 if (!sk || !sk_fullsock(sk))
2548 return -ENOENT;
f3694e00 2549 if (unlikely(idx >= array->map.max_entries))
4a482f34
MKL		 2550 return -E2BIG;
2551
f3694e00 2552 cgrp = READ_ONCE(array->ptrs[idx]);
4a482f34
MKL		 2553 if (unlikely(!cgrp))
2554 return -EAGAIN;
2555
54fd9c2d 2556 return sk_under_cgroup_hierarchy(sk, cgrp);
4a482f34
MKL		 2557}
2558
747ea55e
DB		 2559static const struct bpf_func_proto bpf_skb_under_cgroup_proto = {
2560 .func = bpf_skb_under_cgroup,
4a482f34
MKL		 2561 .gpl_only = false,
2562 .ret_type = RET_INTEGER,
2563 .arg1_type = ARG_PTR_TO_CTX,
2564 .arg2_type = ARG_CONST_MAP_PTR,
2565 .arg3_type = ARG_ANYTHING,
2566};
4a482f34 2567
4de16969
DB		 2568static unsigned long bpf_xdp_copy(void *dst_buff, const void *src_buff,
2569 unsigned long off, unsigned long len)
2570{
2571 memcpy(dst_buff, src_buff + off, len);
2572 return 0;
2573}
2574
f3694e00
DB		 2575BPF_CALL_5(bpf_xdp_event_output, struct xdp_buff *, xdp, struct bpf_map *, map,
2576 u64, flags, void *, meta, u64, meta_size)
4de16969 2577{
4de16969 2578 u64 xdp_size = (flags & BPF_F_CTXLEN_MASK) >> 32;
4de16969
DB		 2579
2580 if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
2581 return -EINVAL;
2582 if (unlikely(xdp_size > (unsigned long)(xdp->data_end - xdp->data)))
2583 return -EFAULT;
2584
2585 return bpf_event_output(map, flags, meta, meta_size, xdp, xdp_size,
2586 bpf_xdp_copy);
2587}
2588
2589static const struct bpf_func_proto bpf_xdp_event_output_proto = {
2590 .func = bpf_xdp_event_output,
2591 .gpl_only = true,
2592 .ret_type = RET_INTEGER,
2593 .arg1_type = ARG_PTR_TO_CTX,
2594 .arg2_type = ARG_CONST_MAP_PTR,
2595 .arg3_type = ARG_ANYTHING,
2596 .arg4_type = ARG_PTR_TO_STACK,
2597 .arg5_type = ARG_CONST_STACK_SIZE,
2598};
2599
d4052c4a
DB		 2600static const struct bpf_func_proto *
2601sk_filter_func_proto(enum bpf_func_id func_id)
89aa0758
AS		 2602{
2603 switch (func_id) {
2604 case BPF_FUNC_map_lookup_elem:
2605 return &bpf_map_lookup_elem_proto;
2606 case BPF_FUNC_map_update_elem:
2607 return &bpf_map_update_elem_proto;
2608 case BPF_FUNC_map_delete_elem:
2609 return &bpf_map_delete_elem_proto;
03e69b50
DB		 2610 case BPF_FUNC_get_prandom_u32:
2611 return &bpf_get_prandom_u32_proto;
c04167ce 2612 case BPF_FUNC_get_smp_processor_id:
80b48c44 2613 return &bpf_get_raw_smp_processor_id_proto;
2d0e30c3
DB		 2614 case BPF_FUNC_get_numa_node_id:
2615 return &bpf_get_numa_node_id_proto;
04fd61ab
AS		 2616 case BPF_FUNC_tail_call:
2617 return &bpf_tail_call_proto;
17ca8cbf
DB		 2618 case BPF_FUNC_ktime_get_ns:
2619 return &bpf_ktime_get_ns_proto;
0756ea3e 2620 case BPF_FUNC_trace_printk:
1be7f75d
AS		 2621 if (capable(CAP_SYS_ADMIN))
2622 return bpf_get_trace_printk_proto();
89aa0758
AS		 2623 default:
2624 return NULL;
2625 }
2626}
2627
608cd71a
AS		 2628static const struct bpf_func_proto *
2629tc_cls_act_func_proto(enum bpf_func_id func_id)
2630{
2631 switch (func_id) {
2632 case BPF_FUNC_skb_store_bytes:
2633 return &bpf_skb_store_bytes_proto;
05c74e5e
DB		 2634 case BPF_FUNC_skb_load_bytes:
2635 return &bpf_skb_load_bytes_proto;
36bbef52
DB		 2636 case BPF_FUNC_skb_pull_data:
2637 return &bpf_skb_pull_data_proto;
7d672345
DB		 2638 case BPF_FUNC_csum_diff:
2639 return &bpf_csum_diff_proto;
36bbef52
DB		 2640 case BPF_FUNC_csum_update:
2641 return &bpf_csum_update_proto;
91bc4822
AS		 2642 case BPF_FUNC_l3_csum_replace:
2643 return &bpf_l3_csum_replace_proto;
2644 case BPF_FUNC_l4_csum_replace:
2645 return &bpf_l4_csum_replace_proto;
3896d655
AS		 2646 case BPF_FUNC_clone_redirect:
2647 return &bpf_clone_redirect_proto;
8d20aabe
DB		 2648 case BPF_FUNC_get_cgroup_classid:
2649 return &bpf_get_cgroup_classid_proto;
4e10df9a
AS		 2650 case BPF_FUNC_skb_vlan_push:
2651 return &bpf_skb_vlan_push_proto;
2652 case BPF_FUNC_skb_vlan_pop:
2653 return &bpf_skb_vlan_pop_proto;
6578171a
DB		 2654 case BPF_FUNC_skb_change_proto:
2655 return &bpf_skb_change_proto_proto;
d2485c42
DB		 2656 case BPF_FUNC_skb_change_type:
2657 return &bpf_skb_change_type_proto;
5293efe6
DB		 2658 case BPF_FUNC_skb_change_tail:
2659 return &bpf_skb_change_tail_proto;
d3aa45ce
AS		 2660 case BPF_FUNC_skb_get_tunnel_key:
2661 return &bpf_skb_get_tunnel_key_proto;
2662 case BPF_FUNC_skb_set_tunnel_key:
14ca0751
DB		 2663 return bpf_get_skb_set_tunnel_proto(func_id);
2664 case BPF_FUNC_skb_get_tunnel_opt:
2665 return &bpf_skb_get_tunnel_opt_proto;
2666 case BPF_FUNC_skb_set_tunnel_opt:
2667 return bpf_get_skb_set_tunnel_proto(func_id);
27b29f63
AS		 2668 case BPF_FUNC_redirect:
2669 return &bpf_redirect_proto;
c46646d0
DB		 2670 case BPF_FUNC_get_route_realm:
2671 return &bpf_get_route_realm_proto;
13c5c240
DB		 2672 case BPF_FUNC_get_hash_recalc:
2673 return &bpf_get_hash_recalc_proto;
7a4b28c6
DB		 2674 case BPF_FUNC_set_hash_invalid:
2675 return &bpf_set_hash_invalid_proto;
bd570ff9 2676 case BPF_FUNC_perf_event_output:
555c8a86 2677 return &bpf_skb_event_output_proto;
80b48c44
DB		 2678 case BPF_FUNC_get_smp_processor_id:
2679 return &bpf_get_smp_processor_id_proto;
747ea55e
DB		 2680 case BPF_FUNC_skb_under_cgroup:
2681 return &bpf_skb_under_cgroup_proto;
608cd71a
AS		 2682 default:
2683 return sk_filter_func_proto(func_id);
2684 }
2685}
2686
6a773a15
BB		 2687static const struct bpf_func_proto *
2688xdp_func_proto(enum bpf_func_id func_id)
2689{
4de16969
DB		 2690 switch (func_id) {
2691 case BPF_FUNC_perf_event_output:
2692 return &bpf_xdp_event_output_proto;
669dc4d7
DB		 2693 case BPF_FUNC_get_smp_processor_id:
2694 return &bpf_get_smp_processor_id_proto;
17bedab2
MKL		 2695 case BPF_FUNC_xdp_adjust_head:
2696 return &bpf_xdp_adjust_head_proto;
4de16969
DB		 2697 default:
2698 return sk_filter_func_proto(func_id);
2699 }
6a773a15
BB		 2700}
2701
0e33661d
DM		 2702static const struct bpf_func_proto *
2703cg_skb_func_proto(enum bpf_func_id func_id)
2704{
2705 switch (func_id) {
2706 case BPF_FUNC_skb_load_bytes:
2707 return &bpf_skb_load_bytes_proto;
2708 default:
2709 return sk_filter_func_proto(func_id);
2710 }
2711}
2712
3a0af8fd
TG		 2713static const struct bpf_func_proto *
2714lwt_inout_func_proto(enum bpf_func_id func_id)
2715{
2716 switch (func_id) {
2717 case BPF_FUNC_skb_load_bytes:
2718 return &bpf_skb_load_bytes_proto;
2719 case BPF_FUNC_skb_pull_data:
2720 return &bpf_skb_pull_data_proto;
2721 case BPF_FUNC_csum_diff:
2722 return &bpf_csum_diff_proto;
2723 case BPF_FUNC_get_cgroup_classid:
2724 return &bpf_get_cgroup_classid_proto;
2725 case BPF_FUNC_get_route_realm:
2726 return &bpf_get_route_realm_proto;
2727 case BPF_FUNC_get_hash_recalc:
2728 return &bpf_get_hash_recalc_proto;
2729 case BPF_FUNC_perf_event_output:
2730 return &bpf_skb_event_output_proto;
2731 case BPF_FUNC_get_smp_processor_id:
2732 return &bpf_get_smp_processor_id_proto;
2733 case BPF_FUNC_skb_under_cgroup:
2734 return &bpf_skb_under_cgroup_proto;
2735 default:
2736 return sk_filter_func_proto(func_id);
2737 }
2738}
2739
2740static const struct bpf_func_proto *
2741lwt_xmit_func_proto(enum bpf_func_id func_id)
2742{
2743 switch (func_id) {
2744 case BPF_FUNC_skb_get_tunnel_key:
2745 return &bpf_skb_get_tunnel_key_proto;
2746 case BPF_FUNC_skb_set_tunnel_key:
2747 return bpf_get_skb_set_tunnel_proto(func_id);
2748 case BPF_FUNC_skb_get_tunnel_opt:
2749 return &bpf_skb_get_tunnel_opt_proto;
2750 case BPF_FUNC_skb_set_tunnel_opt:
2751 return bpf_get_skb_set_tunnel_proto(func_id);
2752 case BPF_FUNC_redirect:
2753 return &bpf_redirect_proto;
2754 case BPF_FUNC_clone_redirect:
2755 return &bpf_clone_redirect_proto;
2756 case BPF_FUNC_skb_change_tail:
2757 return &bpf_skb_change_tail_proto;
2758 case BPF_FUNC_skb_change_head:
2759 return &bpf_skb_change_head_proto;
2760 case BPF_FUNC_skb_store_bytes:
2761 return &bpf_skb_store_bytes_proto;
2762 case BPF_FUNC_csum_update:
2763 return &bpf_csum_update_proto;
2764 case BPF_FUNC_l3_csum_replace:
2765 return &bpf_l3_csum_replace_proto;
2766 case BPF_FUNC_l4_csum_replace:
2767 return &bpf_l4_csum_replace_proto;
2768 case BPF_FUNC_set_hash_invalid:
2769 return &bpf_set_hash_invalid_proto;
2770 default:
2771 return lwt_inout_func_proto(func_id);
2772 }
2773}
2774
1afaf661 2775static bool __is_valid_access(int off, int size)
89aa0758 2776{
9bac3d6d
AS		 2777 if (off < 0 || off >= sizeof(struct __sk_buff))
2778 return false;
4936e352 2779 /* The verifier guarantees that size > 0. */
9bac3d6d
AS		 2780 if (off % size != 0)
2781 return false;
4936e352 2782 if (size != sizeof(__u32))
9bac3d6d
AS		 2783 return false;
2784
2785 return true;
2786}
2787
d691f9e8 2788static bool sk_filter_is_valid_access(int off, int size,
19de99f7
AS		 2789 enum bpf_access_type type,
2790 enum bpf_reg_type *reg_type)
d691f9e8 2791{
db58ba45
AS		 2792 switch (off) {
2793 case offsetof(struct __sk_buff, tc_classid):
2794 case offsetof(struct __sk_buff, data):
2795 case offsetof(struct __sk_buff, data_end):
045efa82 2796 return false;
db58ba45 2797 }
045efa82 2798
d691f9e8
AS		 2799 if (type == BPF_WRITE) {
2800 switch (off) {
2801 case offsetof(struct __sk_buff, cb[0]) ...
4936e352 2802 offsetof(struct __sk_buff, cb[4]):
d691f9e8
AS		 2803 break;
2804 default:
2805 return false;
2806 }
2807 }
2808
1afaf661 2809 return __is_valid_access(off, size);
d691f9e8
AS		 2810}
2811
3a0af8fd
TG		 2812static bool lwt_is_valid_access(int off, int size,
2813 enum bpf_access_type type,
2814 enum bpf_reg_type *reg_type)
2815{
2816 switch (off) {
2817 case offsetof(struct __sk_buff, tc_classid):
2818 return false;
2819 }
2820
2821 if (type == BPF_WRITE) {
2822 switch (off) {
2823 case offsetof(struct __sk_buff, mark):
2824 case offsetof(struct __sk_buff, priority):
2825 case offsetof(struct __sk_buff, cb[0]) ...
2826 offsetof(struct __sk_buff, cb[4]):
2827 break;
2828 default:
2829 return false;
2830 }
2831 }
2832
2833 switch (off) {
2834 case offsetof(struct __sk_buff, data):
2835 *reg_type = PTR_TO_PACKET;
2836 break;
2837 case offsetof(struct __sk_buff, data_end):
2838 *reg_type = PTR_TO_PACKET_END;
2839 break;
2840 }
2841
1afaf661 2842 return __is_valid_access(off, size);
3a0af8fd
TG		 2843}
2844
61023658
DA		 2845static bool sock_filter_is_valid_access(int off, int size,
2846 enum bpf_access_type type,
2847 enum bpf_reg_type *reg_type)
2848{
2849 if (type == BPF_WRITE) {
2850 switch (off) {
2851 case offsetof(struct bpf_sock, bound_dev_if):
2852 break;
2853 default:
2854 return false;
2855 }
2856 }
2857
2858 if (off < 0 || off + size > sizeof(struct bpf_sock))
2859 return false;
61023658
DA		 2860 /* The verifier guarantees that size > 0. */
2861 if (off % size != 0)
2862 return false;
61023658
DA		 2863 if (size != sizeof(__u32))
2864 return false;
2865
2866 return true;
2867}
2868
36bbef52
DB		 2869static int tc_cls_act_prologue(struct bpf_insn *insn_buf, bool direct_write,
2870 const struct bpf_prog *prog)
2871{
2872 struct bpf_insn *insn = insn_buf;
2873
2874 if (!direct_write)
2875 return 0;
2876
2877 /* if (!skb->cloned)
2878 * goto start;
2879 *
2880 * (Fast-path, otherwise approximation that we might be
2881 * a clone, do the rest in helper.)
2882 */
2883 *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET());
2884 *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_6, CLONED_MASK);
2885 *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_6, 0, 7);
2886
2887 /* ret = bpf_skb_pull_data(skb, 0); */
2888 *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
2889 *insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_2, BPF_REG_2);
2890 *insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
2891 BPF_FUNC_skb_pull_data);
2892 /* if (!ret)
2893 * goto restore;
2894 * return TC_ACT_SHOT;
2895 */
2896 *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2);
2897 *insn++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, TC_ACT_SHOT);
2898 *insn++ = BPF_EXIT_INSN();
2899
2900 /* restore: */
2901 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
2902 /* start: */
2903 *insn++ = prog->insnsi[0];
2904
2905 return insn - insn_buf;
2906}
2907
d691f9e8 2908static bool tc_cls_act_is_valid_access(int off, int size,
19de99f7
AS		 2909 enum bpf_access_type type,
2910 enum bpf_reg_type *reg_type)
d691f9e8
AS		 2911{
2912 if (type == BPF_WRITE) {
2913 switch (off) {
2914 case offsetof(struct __sk_buff, mark):
2915 case offsetof(struct __sk_buff, tc_index):
754f1e6a 2916 case offsetof(struct __sk_buff, priority):
d691f9e8 2917 case offsetof(struct __sk_buff, cb[0]) ...
09c37a2c
DB		 2918 offsetof(struct __sk_buff, cb[4]):
2919 case offsetof(struct __sk_buff, tc_classid):
d691f9e8
AS		 2920 break;
2921 default:
2922 return false;
2923 }
2924 }
19de99f7
AS		 2925
2926 switch (off) {
2927 case offsetof(struct __sk_buff, data):
2928 *reg_type = PTR_TO_PACKET;
2929 break;
2930 case offsetof(struct __sk_buff, data_end):
2931 *reg_type = PTR_TO_PACKET_END;
2932 break;
2933 }
2934
1afaf661 2935 return __is_valid_access(off, size);
d691f9e8
AS		 2936}
2937
1afaf661 2938static bool __is_valid_xdp_access(int off, int size)
6a773a15
BB		 2939{
2940 if (off < 0 || off >= sizeof(struct xdp_md))
2941 return false;
2942 if (off % size != 0)
2943 return false;
6088b582 2944 if (size != sizeof(__u32))
6a773a15
BB		 2945 return false;
2946
2947 return true;
2948}
2949
2950static bool xdp_is_valid_access(int off, int size,
2951 enum bpf_access_type type,
2952 enum bpf_reg_type *reg_type)
2953{
2954 if (type == BPF_WRITE)
2955 return false;
2956
2957 switch (off) {
2958 case offsetof(struct xdp_md, data):
2959 *reg_type = PTR_TO_PACKET;
2960 break;
2961 case offsetof(struct xdp_md, data_end):
2962 *reg_type = PTR_TO_PACKET_END;
2963 break;
2964 }
2965
1afaf661 2966 return __is_valid_xdp_access(off, size);
6a773a15
BB		 2967}
2968
2969void bpf_warn_invalid_xdp_action(u32 act)
2970{
2971 WARN_ONCE(1, "Illegal XDP return value %u, expect packet loss\n", act);
2972}
2973EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action);
2974
374fb54e
DB		 2975static u32 sk_filter_convert_ctx_access(enum bpf_access_type type, int dst_reg,
2976 int src_reg, int ctx_off,
2977 struct bpf_insn *insn_buf,
2978 struct bpf_prog *prog)
9bac3d6d
AS		 2979{
2980 struct bpf_insn *insn = insn_buf;
2981
2982 switch (ctx_off) {
2983 case offsetof(struct __sk_buff, len):
2984 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
2985
2986 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
2987 offsetof(struct sk_buff, len));
2988 break;
2989
0b8c707d
DB		 2990 case offsetof(struct __sk_buff, protocol):
2991 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
2992
2993 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
2994 offsetof(struct sk_buff, protocol));
2995 break;
2996
27cd5452
MS		 2997 case offsetof(struct __sk_buff, vlan_proto):
2998 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_proto) != 2);
2999
3000 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
3001 offsetof(struct sk_buff, vlan_proto));
3002 break;
3003
bcad5718
DB		 3004 case offsetof(struct __sk_buff, priority):
3005 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, priority) != 4);
3006
754f1e6a
DB		 3007 if (type == BPF_WRITE)
3008 *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg,
3009 offsetof(struct sk_buff, priority));
3010 else
3011 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
3012 offsetof(struct sk_buff, priority));
bcad5718
DB		 3013 break;
3014
37e82c2f
AS		 3015 case offsetof(struct __sk_buff, ingress_ifindex):
3016 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, skb_iif) != 4);
3017
3018 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
3019 offsetof(struct sk_buff, skb_iif));
3020 break;
3021
3022 case offsetof(struct __sk_buff, ifindex):
3023 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
3024
f035a515 3025 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
37e82c2f
AS		 3026 dst_reg, src_reg,
3027 offsetof(struct sk_buff, dev));
3028 *insn++ = BPF_JMP_IMM(BPF_JEQ, dst_reg, 0, 1);
3029 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, dst_reg,
3030 offsetof(struct net_device, ifindex));
3031 break;
3032
ba7591d8
DB		 3033 case offsetof(struct __sk_buff, hash):
3034 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
3035
3036 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
3037 offsetof(struct sk_buff, hash));
3038 break;
3039
9bac3d6d 3040 case offsetof(struct __sk_buff, mark):
d691f9e8
AS		 3041 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
3042
3043 if (type == BPF_WRITE)
3044 *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg,
3045 offsetof(struct sk_buff, mark));
3046 else
3047 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
3048 offsetof(struct sk_buff, mark));
3049 break;
9bac3d6d
AS		 3050
3051 case offsetof(struct __sk_buff, pkt_type):
3052 return convert_skb_access(SKF_AD_PKTTYPE, dst_reg, src_reg, insn);
3053
3054 case offsetof(struct __sk_buff, queue_mapping):
3055 return convert_skb_access(SKF_AD_QUEUE, dst_reg, src_reg, insn);
c2497395 3056
c2497395
AS		 3057 case offsetof(struct __sk_buff, vlan_present):
3058 return convert_skb_access(SKF_AD_VLAN_TAG_PRESENT,
3059 dst_reg, src_reg, insn);
3060
3061 case offsetof(struct __sk_buff, vlan_tci):
3062 return convert_skb_access(SKF_AD_VLAN_TAG,
3063 dst_reg, src_reg, insn);
d691f9e8
AS		 3064
3065 case offsetof(struct __sk_buff, cb[0]) ...
6088b582 3066 offsetof(struct __sk_buff, cb[4]):
d691f9e8
AS		 3067 BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, data) < 20);
3068
ff936a04 3069 prog->cb_access = 1;
d691f9e8
AS		 3070 ctx_off -= offsetof(struct __sk_buff, cb[0]);
3071 ctx_off += offsetof(struct sk_buff, cb);
3072 ctx_off += offsetof(struct qdisc_skb_cb, data);
3073 if (type == BPF_WRITE)
3074 *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, ctx_off);
3075 else
3076 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, ctx_off);
3077 break;
3078
045efa82
DB		 3079 case offsetof(struct __sk_buff, tc_classid):
3080 ctx_off -= offsetof(struct __sk_buff, tc_classid);
3081 ctx_off += offsetof(struct sk_buff, cb);
3082 ctx_off += offsetof(struct qdisc_skb_cb, tc_classid);
09c37a2c
DB		 3083 if (type == BPF_WRITE)
3084 *insn++ = BPF_STX_MEM(BPF_H, dst_reg, src_reg, ctx_off);
3085 else
3086 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, ctx_off);
045efa82
DB		 3087 break;
3088
db58ba45 3089 case offsetof(struct __sk_buff, data):
f035a515 3090 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
db58ba45
AS		 3091 dst_reg, src_reg,
3092 offsetof(struct sk_buff, data));
3093 break;
3094
3095 case offsetof(struct __sk_buff, data_end):
3096 ctx_off -= offsetof(struct __sk_buff, data_end);
3097 ctx_off += offsetof(struct sk_buff, cb);
3098 ctx_off += offsetof(struct bpf_skb_data_end, data_end);
f035a515
DB		 3099 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), dst_reg, src_reg,
3100 ctx_off);
db58ba45
AS		 3101 break;
3102
d691f9e8
AS		 3103 case offsetof(struct __sk_buff, tc_index):
3104#ifdef CONFIG_NET_SCHED
3105 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, tc_index) != 2);
3106
3107 if (type == BPF_WRITE)
3108 *insn++ = BPF_STX_MEM(BPF_H, dst_reg, src_reg,
3109 offsetof(struct sk_buff, tc_index));
3110 else
3111 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
3112 offsetof(struct sk_buff, tc_index));
3113 break;
3114#else
3115 if (type == BPF_WRITE)
3116 *insn++ = BPF_MOV64_REG(dst_reg, dst_reg);
3117 else
3118 *insn++ = BPF_MOV64_IMM(dst_reg, 0);
3119 break;
3120#endif
9bac3d6d
AS		 3121 }
3122
3123 return insn - insn_buf;
89aa0758
AS		 3124}
3125
61023658
DA		 3126static u32 sock_filter_convert_ctx_access(enum bpf_access_type type,
3127 int dst_reg, int src_reg,
3128 int ctx_off,
3129 struct bpf_insn *insn_buf,
3130 struct bpf_prog *prog)
3131{
3132 struct bpf_insn *insn = insn_buf;
3133
3134 switch (ctx_off) {
3135 case offsetof(struct bpf_sock, bound_dev_if):
3136 BUILD_BUG_ON(FIELD_SIZEOF(struct sock, sk_bound_dev_if) != 4);
3137
3138 if (type == BPF_WRITE)
3139 *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg,
3140 offsetof(struct sock, sk_bound_dev_if));
3141 else
3142 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
3143 offsetof(struct sock, sk_bound_dev_if));
3144 break;
aa4c1037
DA		 3145
3146 case offsetof(struct bpf_sock, family):
3147 BUILD_BUG_ON(FIELD_SIZEOF(struct sock, sk_family) != 2);
3148
3149 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
3150 offsetof(struct sock, sk_family));
3151 break;
3152
3153 case offsetof(struct bpf_sock, type):
3154 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
3155 offsetof(struct sock, __sk_flags_offset));
3156 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, SK_FL_TYPE_MASK);
3157 *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, SK_FL_TYPE_SHIFT);
3158 break;
3159
3160 case offsetof(struct bpf_sock, protocol):
3161 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
3162 offsetof(struct sock, __sk_flags_offset));
3163 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, SK_FL_PROTO_MASK);
3164 *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, SK_FL_PROTO_SHIFT);
3165 break;
61023658
DA		 3166 }
3167
3168 return insn - insn_buf;
3169}
3170
374fb54e
DB		 3171static u32 tc_cls_act_convert_ctx_access(enum bpf_access_type type, int dst_reg,
3172 int src_reg, int ctx_off,
3173 struct bpf_insn *insn_buf,
3174 struct bpf_prog *prog)
3175{
3176 struct bpf_insn *insn = insn_buf;
3177
3178 switch (ctx_off) {
3179 case offsetof(struct __sk_buff, ifindex):
3180 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
3181
3182 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
3183 dst_reg, src_reg,
3184 offsetof(struct sk_buff, dev));
3185 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, dst_reg,
3186 offsetof(struct net_device, ifindex));
3187 break;
3188 default:
3189 return sk_filter_convert_ctx_access(type, dst_reg, src_reg,
3190 ctx_off, insn_buf, prog);
3191 }
3192
3193 return insn - insn_buf;
3194}
3195
6a773a15
BB		 3196static u32 xdp_convert_ctx_access(enum bpf_access_type type, int dst_reg,
3197 int src_reg, int ctx_off,
3198 struct bpf_insn *insn_buf,
3199 struct bpf_prog *prog)
3200{
3201 struct bpf_insn *insn = insn_buf;
3202
3203 switch (ctx_off) {
3204 case offsetof(struct xdp_md, data):
f035a515 3205 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data),
6a773a15
BB		 3206 dst_reg, src_reg,
3207 offsetof(struct xdp_buff, data));
3208 break;
3209 case offsetof(struct xdp_md, data_end):
f035a515 3210 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data_end),
6a773a15
BB		 3211 dst_reg, src_reg,
3212 offsetof(struct xdp_buff, data_end));
3213 break;
3214 }
3215
3216 return insn - insn_buf;
3217}
3218
d4052c4a 3219static const struct bpf_verifier_ops sk_filter_ops = {
4936e352
DB		 3220 .get_func_proto = sk_filter_func_proto,
3221 .is_valid_access = sk_filter_is_valid_access,
374fb54e 3222 .convert_ctx_access = sk_filter_convert_ctx_access,
89aa0758
AS		 3223};
3224
608cd71a 3225static const struct bpf_verifier_ops tc_cls_act_ops = {
4936e352
DB		 3226 .get_func_proto = tc_cls_act_func_proto,
3227 .is_valid_access = tc_cls_act_is_valid_access,
374fb54e 3228 .convert_ctx_access = tc_cls_act_convert_ctx_access,
36bbef52 3229 .gen_prologue = tc_cls_act_prologue,
608cd71a
AS		 3230};
3231
6a773a15
BB		 3232static const struct bpf_verifier_ops xdp_ops = {
3233 .get_func_proto = xdp_func_proto,
3234 .is_valid_access = xdp_is_valid_access,
3235 .convert_ctx_access = xdp_convert_ctx_access,
3236};
3237
0e33661d
DM		 3238static const struct bpf_verifier_ops cg_skb_ops = {
3239 .get_func_proto = cg_skb_func_proto,
3240 .is_valid_access = sk_filter_is_valid_access,
3241 .convert_ctx_access = sk_filter_convert_ctx_access,
3242};
3243
3a0af8fd
TG		 3244static const struct bpf_verifier_ops lwt_inout_ops = {
3245 .get_func_proto = lwt_inout_func_proto,
3246 .is_valid_access = lwt_is_valid_access,
3247 .convert_ctx_access = sk_filter_convert_ctx_access,
3248};
3249
3250static const struct bpf_verifier_ops lwt_xmit_ops = {
3251 .get_func_proto = lwt_xmit_func_proto,
3252 .is_valid_access = lwt_is_valid_access,
3253 .convert_ctx_access = sk_filter_convert_ctx_access,
3254 .gen_prologue = tc_cls_act_prologue,
3255};
3256
61023658
DA		 3257static const struct bpf_verifier_ops cg_sock_ops = {
3258 .get_func_proto = sk_filter_func_proto,
3259 .is_valid_access = sock_filter_is_valid_access,
3260 .convert_ctx_access = sock_filter_convert_ctx_access,
3261};
3262
d4052c4a 3263static struct bpf_prog_type_list sk_filter_type __read_mostly = {
4936e352
DB		 3264 .ops = &sk_filter_ops,
3265 .type = BPF_PROG_TYPE_SOCKET_FILTER,
89aa0758
AS		 3266};
3267
96be4325 3268static struct bpf_prog_type_list sched_cls_type __read_mostly = {
4936e352
DB		 3269 .ops = &tc_cls_act_ops,
3270 .type = BPF_PROG_TYPE_SCHED_CLS,
96be4325
DB		 3271};
3272
94caee8c 3273static struct bpf_prog_type_list sched_act_type __read_mostly = {
4936e352
DB		 3274 .ops = &tc_cls_act_ops,
3275 .type = BPF_PROG_TYPE_SCHED_ACT,
94caee8c
DB		 3276};
3277
6a773a15
BB		 3278static struct bpf_prog_type_list xdp_type __read_mostly = {
3279 .ops = &xdp_ops,
3280 .type = BPF_PROG_TYPE_XDP,
3281};
3282
0e33661d
DM		 3283static struct bpf_prog_type_list cg_skb_type __read_mostly = {
3284 .ops = &cg_skb_ops,
3285 .type = BPF_PROG_TYPE_CGROUP_SKB,
3286};
3287
3a0af8fd
TG		 3288static struct bpf_prog_type_list lwt_in_type __read_mostly = {
3289 .ops = &lwt_inout_ops,
3290 .type = BPF_PROG_TYPE_LWT_IN,
3291};
3292
3293static struct bpf_prog_type_list lwt_out_type __read_mostly = {
3294 .ops = &lwt_inout_ops,
3295 .type = BPF_PROG_TYPE_LWT_OUT,
3296};
3297
3298static struct bpf_prog_type_list lwt_xmit_type __read_mostly = {
3299 .ops = &lwt_xmit_ops,
3300 .type = BPF_PROG_TYPE_LWT_XMIT,
3301};
3302
61023658
DA		 3303static struct bpf_prog_type_list cg_sock_type __read_mostly = {
3304 .ops = &cg_sock_ops,
3305 .type = BPF_PROG_TYPE_CGROUP_SOCK
3306};
3307
d4052c4a 3308static int __init register_sk_filter_ops(void)
89aa0758 3309{
d4052c4a 3310 bpf_register_prog_type(&sk_filter_type);
96be4325 3311 bpf_register_prog_type(&sched_cls_type);
94caee8c 3312 bpf_register_prog_type(&sched_act_type);
6a773a15 3313 bpf_register_prog_type(&xdp_type);
0e33661d 3314 bpf_register_prog_type(&cg_skb_type);
61023658 3315 bpf_register_prog_type(&cg_sock_type);
3a0af8fd
TG		 3316 bpf_register_prog_type(&lwt_in_type);
3317 bpf_register_prog_type(&lwt_out_type);
3318 bpf_register_prog_type(&lwt_xmit_type);
96be4325 3319
89aa0758
AS		 3320 return 0;
3321}
d4052c4a
DB		 3322late_initcall(register_sk_filter_ops);
3323
8ced425e 3324int sk_detach_filter(struct sock *sk)
55b33325
PE		 3325{
3326 int ret = -ENOENT;
3327 struct sk_filter *filter;
3328
d59577b6
VB		 3329 if (sock_flag(sk, SOCK_FILTER_LOCKED))
3330 return -EPERM;
3331
8ced425e
HFS		 3332 filter = rcu_dereference_protected(sk->sk_filter,
3333 lockdep_sock_is_held(sk));
55b33325 3334 if (filter) {
a9b3cd7f 3335 RCU_INIT_POINTER(sk->sk_filter, NULL);
46bcf14f 3336 sk_filter_uncharge(sk, filter);
55b33325
PE		 3337 ret = 0;
3338 }
a3ea269b 3339
55b33325
PE		 3340 return ret;
3341}
8ced425e 3342EXPORT_SYMBOL_GPL(sk_detach_filter);
a8fc9277 3343
a3ea269b
DB		 3344int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf,
3345 unsigned int len)
a8fc9277 3346{
a3ea269b 3347 struct sock_fprog_kern *fprog;
a8fc9277 3348 struct sk_filter *filter;
a3ea269b 3349 int ret = 0;
a8fc9277
PE		 3350
3351 lock_sock(sk);
3352 filter = rcu_dereference_protected(sk->sk_filter,
8ced425e 3353 lockdep_sock_is_held(sk));
a8fc9277
PE		 3354 if (!filter)
3355 goto out;
a3ea269b
DB		 3356
3357 /* We're copying the filter that has been originally attached,
93d08b69
DB		 3358 * so no conversion/decode needed anymore. eBPF programs that
3359 * have no original program cannot be dumped through this.
a3ea269b 3360 */
93d08b69 3361 ret = -EACCES;
7ae457c1 3362 fprog = filter->prog->orig_prog;
93d08b69
DB		 3363 if (!fprog)
3364 goto out;
a3ea269b
DB		 3365
3366 ret = fprog->len;
a8fc9277 3367 if (!len)
a3ea269b 3368 /* User space only enquires number of filter blocks. */
a8fc9277 3369 goto out;
a3ea269b 3370
a8fc9277 3371 ret = -EINVAL;
a3ea269b 3372 if (len < fprog->len)
a8fc9277
PE		 3373 goto out;
3374
3375 ret = -EFAULT;
009937e7 3376 if (copy_to_user(ubuf, fprog->filter, bpf_classic_proglen(fprog)))
a3ea269b 3377 goto out;
a8fc9277 3378
a3ea269b
DB		 3379 /* Instead of bytes, the API requests to return the number
3380 * of filter blocks.
3381 */
3382 ret = fprog->len;
a8fc9277
PE		 3383out:
3384 release_sock(sk);
3385 return ret;
3386}
ubuntu-zesty-kernel mirror