2 * Linux Socket Filter - Kernel level socket filtering
4 * Based on the design of the Berkeley Packet Filter. The new
5 * internal format has been designed by PLUMgrid:
7 * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
11 * Jay Schulist <jschlst@samba.org>
12 * Alexei Starovoitov <ast@plumgrid.com>
13 * Daniel Borkmann <dborkman@redhat.com>
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
20 * Andi Kleen - Fix a few bad bugs and races.
21 * Kris Katterjohn - Added many additional checks in bpf_check_classic()
24 #include <linux/filter.h>
25 #include <linux/skbuff.h>
26 #include <linux/vmalloc.h>
27 #include <linux/random.h>
28 #include <linux/moduleloader.h>
29 #include <linux/bpf.h>
30 #include <linux/frame.h>
32 #include <asm/unaligned.h>
35 #define BPF_R0 regs[BPF_REG_0]
36 #define BPF_R1 regs[BPF_REG_1]
37 #define BPF_R2 regs[BPF_REG_2]
38 #define BPF_R3 regs[BPF_REG_3]
39 #define BPF_R4 regs[BPF_REG_4]
40 #define BPF_R5 regs[BPF_REG_5]
41 #define BPF_R6 regs[BPF_REG_6]
42 #define BPF_R7 regs[BPF_REG_7]
43 #define BPF_R8 regs[BPF_REG_8]
44 #define BPF_R9 regs[BPF_REG_9]
45 #define BPF_R10 regs[BPF_REG_10]
48 #define DST regs[insn->dst_reg]
49 #define SRC regs[insn->src_reg]
50 #define FP regs[BPF_REG_FP]
51 #define ARG1 regs[BPF_REG_ARG1]
52 #define CTX regs[BPF_REG_CTX]
55 /* No hurry in this branch
57 * Exported for the bpf jit load helper.
59 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff
*skb
, int k
, unsigned int size
)
64 ptr
= skb_network_header(skb
) + k
- SKF_NET_OFF
;
65 else if (k
>= SKF_LL_OFF
)
66 ptr
= skb_mac_header(skb
) + k
- SKF_LL_OFF
;
68 if (ptr
>= skb
->head
&& ptr
+ size
<= skb_tail_pointer(skb
))
74 struct bpf_prog
*bpf_prog_alloc(unsigned int size
, gfp_t gfp_extra_flags
)
76 gfp_t gfp_flags
= GFP_KERNEL
| __GFP_HIGHMEM
| __GFP_ZERO
|
78 struct bpf_prog_aux
*aux
;
81 size
= round_up(size
, PAGE_SIZE
);
82 fp
= __vmalloc(size
, gfp_flags
, PAGE_KERNEL
);
86 kmemcheck_annotate_bitfield(fp
, meta
);
88 aux
= kzalloc(sizeof(*aux
), GFP_KERNEL
| gfp_extra_flags
);
94 fp
->pages
= size
/ PAGE_SIZE
;
100 EXPORT_SYMBOL_GPL(bpf_prog_alloc
);
102 struct bpf_prog
*bpf_prog_realloc(struct bpf_prog
*fp_old
, unsigned int size
,
103 gfp_t gfp_extra_flags
)
105 gfp_t gfp_flags
= GFP_KERNEL
| __GFP_HIGHMEM
| __GFP_ZERO
|
109 BUG_ON(fp_old
== NULL
);
111 size
= round_up(size
, PAGE_SIZE
);
112 if (size
<= fp_old
->pages
* PAGE_SIZE
)
115 fp
= __vmalloc(size
, gfp_flags
, PAGE_KERNEL
);
117 kmemcheck_annotate_bitfield(fp
, meta
);
119 memcpy(fp
, fp_old
, fp_old
->pages
* PAGE_SIZE
);
120 fp
->pages
= size
/ PAGE_SIZE
;
123 /* We keep fp->aux from fp_old around in the new
124 * reallocated structure.
127 __bpf_prog_free(fp_old
);
133 void __bpf_prog_free(struct bpf_prog
*fp
)
139 #define SHA_BPF_RAW_SIZE \
140 round_up(MAX_BPF_SIZE + sizeof(__be64) + 1, SHA_MESSAGE_BYTES)
142 /* Called under verifier mutex. */
143 void bpf_prog_calc_digest(struct bpf_prog
*fp
)
145 const u32 bits_offset
= SHA_MESSAGE_BYTES
- sizeof(__be64
);
146 static u32 ws
[SHA_WORKSPACE_WORDS
];
147 static u8 raw
[SHA_BPF_RAW_SIZE
];
148 struct bpf_insn
*dst
= (void *)raw
;
149 u32 i
, bsize
, psize
, blocks
;
155 sha_init(fp
->digest
);
156 memset(ws
, 0, sizeof(ws
));
158 /* We need to take out the map fd for the digest calculation
159 * since they are unstable from user space side.
161 for (i
= 0, was_ld_map
= false; i
< fp
->len
; i
++) {
162 dst
[i
] = fp
->insnsi
[i
];
164 dst
[i
].code
== (BPF_LD
| BPF_IMM
| BPF_DW
) &&
165 dst
[i
].src_reg
== BPF_PSEUDO_MAP_FD
) {
168 } else if (was_ld_map
&&
170 dst
[i
].dst_reg
== 0 &&
171 dst
[i
].src_reg
== 0 &&
180 psize
= fp
->len
* sizeof(struct bpf_insn
);
181 memset(&raw
[psize
], 0, sizeof(raw
) - psize
);
184 bsize
= round_up(psize
, SHA_MESSAGE_BYTES
);
185 blocks
= bsize
/ SHA_MESSAGE_BYTES
;
186 if (bsize
- psize
>= sizeof(__be64
)) {
187 bits
= (__be64
*)(todo
+ bsize
- sizeof(__be64
));
189 bits
= (__be64
*)(todo
+ bsize
+ bits_offset
);
192 *bits
= cpu_to_be64((psize
- 1) << 3);
195 sha_transform(fp
->digest
, todo
, ws
);
196 todo
+= SHA_MESSAGE_BYTES
;
199 result
= (__force __be32
*)fp
->digest
;
200 for (i
= 0; i
< SHA_DIGEST_WORDS
; i
++)
201 result
[i
] = cpu_to_be32(fp
->digest
[i
]);
204 static bool bpf_is_jmp_and_has_target(const struct bpf_insn
*insn
)
206 return BPF_CLASS(insn
->code
) == BPF_JMP
&&
207 /* Call and Exit are both special jumps with no
208 * target inside the BPF instruction image.
210 BPF_OP(insn
->code
) != BPF_CALL
&&
211 BPF_OP(insn
->code
) != BPF_EXIT
;
214 static void bpf_adj_branches(struct bpf_prog
*prog
, u32 pos
, u32 delta
)
216 struct bpf_insn
*insn
= prog
->insnsi
;
217 u32 i
, insn_cnt
= prog
->len
;
219 for (i
= 0; i
< insn_cnt
; i
++, insn
++) {
220 if (!bpf_is_jmp_and_has_target(insn
))
223 /* Adjust offset of jmps if we cross boundaries. */
224 if (i
< pos
&& i
+ insn
->off
+ 1 > pos
)
226 else if (i
> pos
+ delta
&& i
+ insn
->off
+ 1 <= pos
+ delta
)
231 struct bpf_prog
*bpf_patch_insn_single(struct bpf_prog
*prog
, u32 off
,
232 const struct bpf_insn
*patch
, u32 len
)
234 u32 insn_adj_cnt
, insn_rest
, insn_delta
= len
- 1;
235 struct bpf_prog
*prog_adj
;
237 /* Since our patchlet doesn't expand the image, we're done. */
238 if (insn_delta
== 0) {
239 memcpy(prog
->insnsi
+ off
, patch
, sizeof(*patch
));
243 insn_adj_cnt
= prog
->len
+ insn_delta
;
245 /* Several new instructions need to be inserted. Make room
246 * for them. Likely, there's no need for a new allocation as
247 * last page could have large enough tailroom.
249 prog_adj
= bpf_prog_realloc(prog
, bpf_prog_size(insn_adj_cnt
),
254 prog_adj
->len
= insn_adj_cnt
;
256 /* Patching happens in 3 steps:
258 * 1) Move over tail of insnsi from next instruction onwards,
259 * so we can patch the single target insn with one or more
260 * new ones (patching is always from 1 to n insns, n > 0).
261 * 2) Inject new instructions at the target location.
262 * 3) Adjust branch offsets if necessary.
264 insn_rest
= insn_adj_cnt
- off
- len
;
266 memmove(prog_adj
->insnsi
+ off
+ len
, prog_adj
->insnsi
+ off
+ 1,
267 sizeof(*patch
) * insn_rest
);
268 memcpy(prog_adj
->insnsi
+ off
, patch
, sizeof(*patch
) * len
);
270 bpf_adj_branches(prog_adj
, off
, insn_delta
);
275 #ifdef CONFIG_BPF_JIT
276 struct bpf_binary_header
*
277 bpf_jit_binary_alloc(unsigned int proglen
, u8
**image_ptr
,
278 unsigned int alignment
,
279 bpf_jit_fill_hole_t bpf_fill_ill_insns
)
281 struct bpf_binary_header
*hdr
;
282 unsigned int size
, hole
, start
;
284 /* Most of BPF filters are really small, but if some of them
285 * fill a page, allow at least 128 extra bytes to insert a
286 * random section of illegal instructions.
288 size
= round_up(proglen
+ sizeof(*hdr
) + 128, PAGE_SIZE
);
289 hdr
= module_alloc(size
);
293 /* Fill space with illegal/arch-dep instructions. */
294 bpf_fill_ill_insns(hdr
, size
);
296 hdr
->pages
= size
/ PAGE_SIZE
;
297 hole
= min_t(unsigned int, size
- (proglen
+ sizeof(*hdr
)),
298 PAGE_SIZE
- sizeof(*hdr
));
299 start
= (get_random_int() % hole
) & ~(alignment
- 1);
301 /* Leave a random number of instructions before BPF code. */
302 *image_ptr
= &hdr
->image
[start
];
307 void bpf_jit_binary_free(struct bpf_binary_header
*hdr
)
312 int bpf_jit_harden __read_mostly
;
314 static int bpf_jit_blind_insn(const struct bpf_insn
*from
,
315 const struct bpf_insn
*aux
,
316 struct bpf_insn
*to_buff
)
318 struct bpf_insn
*to
= to_buff
;
319 u32 imm_rnd
= get_random_int();
322 BUILD_BUG_ON(BPF_REG_AX
+ 1 != MAX_BPF_JIT_REG
);
323 BUILD_BUG_ON(MAX_BPF_REG
+ 1 != MAX_BPF_JIT_REG
);
325 if (from
->imm
== 0 &&
326 (from
->code
== (BPF_ALU
| BPF_MOV
| BPF_K
) ||
327 from
->code
== (BPF_ALU64
| BPF_MOV
| BPF_K
))) {
328 *to
++ = BPF_ALU64_REG(BPF_XOR
, from
->dst_reg
, from
->dst_reg
);
332 switch (from
->code
) {
333 case BPF_ALU
| BPF_ADD
| BPF_K
:
334 case BPF_ALU
| BPF_SUB
| BPF_K
:
335 case BPF_ALU
| BPF_AND
| BPF_K
:
336 case BPF_ALU
| BPF_OR
| BPF_K
:
337 case BPF_ALU
| BPF_XOR
| BPF_K
:
338 case BPF_ALU
| BPF_MUL
| BPF_K
:
339 case BPF_ALU
| BPF_MOV
| BPF_K
:
340 case BPF_ALU
| BPF_DIV
| BPF_K
:
341 case BPF_ALU
| BPF_MOD
| BPF_K
:
342 *to
++ = BPF_ALU32_IMM(BPF_MOV
, BPF_REG_AX
, imm_rnd
^ from
->imm
);
343 *to
++ = BPF_ALU32_IMM(BPF_XOR
, BPF_REG_AX
, imm_rnd
);
344 *to
++ = BPF_ALU32_REG(from
->code
, from
->dst_reg
, BPF_REG_AX
);
347 case BPF_ALU64
| BPF_ADD
| BPF_K
:
348 case BPF_ALU64
| BPF_SUB
| BPF_K
:
349 case BPF_ALU64
| BPF_AND
| BPF_K
:
350 case BPF_ALU64
| BPF_OR
| BPF_K
:
351 case BPF_ALU64
| BPF_XOR
| BPF_K
:
352 case BPF_ALU64
| BPF_MUL
| BPF_K
:
353 case BPF_ALU64
| BPF_MOV
| BPF_K
:
354 case BPF_ALU64
| BPF_DIV
| BPF_K
:
355 case BPF_ALU64
| BPF_MOD
| BPF_K
:
356 *to
++ = BPF_ALU64_IMM(BPF_MOV
, BPF_REG_AX
, imm_rnd
^ from
->imm
);
357 *to
++ = BPF_ALU64_IMM(BPF_XOR
, BPF_REG_AX
, imm_rnd
);
358 *to
++ = BPF_ALU64_REG(from
->code
, from
->dst_reg
, BPF_REG_AX
);
361 case BPF_JMP
| BPF_JEQ
| BPF_K
:
362 case BPF_JMP
| BPF_JNE
| BPF_K
:
363 case BPF_JMP
| BPF_JGT
| BPF_K
:
364 case BPF_JMP
| BPF_JGE
| BPF_K
:
365 case BPF_JMP
| BPF_JSGT
| BPF_K
:
366 case BPF_JMP
| BPF_JSGE
| BPF_K
:
367 case BPF_JMP
| BPF_JSET
| BPF_K
:
368 /* Accommodate for extra offset in case of a backjump. */
372 *to
++ = BPF_ALU64_IMM(BPF_MOV
, BPF_REG_AX
, imm_rnd
^ from
->imm
);
373 *to
++ = BPF_ALU64_IMM(BPF_XOR
, BPF_REG_AX
, imm_rnd
);
374 *to
++ = BPF_JMP_REG(from
->code
, from
->dst_reg
, BPF_REG_AX
, off
);
377 case BPF_LD
| BPF_ABS
| BPF_W
:
378 case BPF_LD
| BPF_ABS
| BPF_H
:
379 case BPF_LD
| BPF_ABS
| BPF_B
:
380 *to
++ = BPF_ALU64_IMM(BPF_MOV
, BPF_REG_AX
, imm_rnd
^ from
->imm
);
381 *to
++ = BPF_ALU64_IMM(BPF_XOR
, BPF_REG_AX
, imm_rnd
);
382 *to
++ = BPF_LD_IND(from
->code
, BPF_REG_AX
, 0);
385 case BPF_LD
| BPF_IND
| BPF_W
:
386 case BPF_LD
| BPF_IND
| BPF_H
:
387 case BPF_LD
| BPF_IND
| BPF_B
:
388 *to
++ = BPF_ALU64_IMM(BPF_MOV
, BPF_REG_AX
, imm_rnd
^ from
->imm
);
389 *to
++ = BPF_ALU64_IMM(BPF_XOR
, BPF_REG_AX
, imm_rnd
);
390 *to
++ = BPF_ALU32_REG(BPF_ADD
, BPF_REG_AX
, from
->src_reg
);
391 *to
++ = BPF_LD_IND(from
->code
, BPF_REG_AX
, 0);
394 case BPF_LD
| BPF_IMM
| BPF_DW
:
395 *to
++ = BPF_ALU64_IMM(BPF_MOV
, BPF_REG_AX
, imm_rnd
^ aux
[1].imm
);
396 *to
++ = BPF_ALU64_IMM(BPF_XOR
, BPF_REG_AX
, imm_rnd
);
397 *to
++ = BPF_ALU64_IMM(BPF_LSH
, BPF_REG_AX
, 32);
398 *to
++ = BPF_ALU64_REG(BPF_MOV
, aux
[0].dst_reg
, BPF_REG_AX
);
400 case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */
401 *to
++ = BPF_ALU32_IMM(BPF_MOV
, BPF_REG_AX
, imm_rnd
^ aux
[0].imm
);
402 *to
++ = BPF_ALU32_IMM(BPF_XOR
, BPF_REG_AX
, imm_rnd
);
403 *to
++ = BPF_ALU64_REG(BPF_OR
, aux
[0].dst_reg
, BPF_REG_AX
);
406 case BPF_ST
| BPF_MEM
| BPF_DW
:
407 case BPF_ST
| BPF_MEM
| BPF_W
:
408 case BPF_ST
| BPF_MEM
| BPF_H
:
409 case BPF_ST
| BPF_MEM
| BPF_B
:
410 *to
++ = BPF_ALU64_IMM(BPF_MOV
, BPF_REG_AX
, imm_rnd
^ from
->imm
);
411 *to
++ = BPF_ALU64_IMM(BPF_XOR
, BPF_REG_AX
, imm_rnd
);
412 *to
++ = BPF_STX_MEM(from
->code
, from
->dst_reg
, BPF_REG_AX
, from
->off
);
419 static struct bpf_prog
*bpf_prog_clone_create(struct bpf_prog
*fp_other
,
420 gfp_t gfp_extra_flags
)
422 gfp_t gfp_flags
= GFP_KERNEL
| __GFP_HIGHMEM
| __GFP_ZERO
|
426 fp
= __vmalloc(fp_other
->pages
* PAGE_SIZE
, gfp_flags
, PAGE_KERNEL
);
428 kmemcheck_annotate_bitfield(fp
, meta
);
430 /* aux->prog still points to the fp_other one, so
431 * when promoting the clone to the real program,
432 * this still needs to be adapted.
434 memcpy(fp
, fp_other
, fp_other
->pages
* PAGE_SIZE
);
440 static void bpf_prog_clone_free(struct bpf_prog
*fp
)
442 /* aux was stolen by the other clone, so we cannot free
443 * it from this path! It will be freed eventually by the
444 * other program on release.
446 * At this point, we don't need a deferred release since
447 * clone is guaranteed to not be locked.
453 void bpf_jit_prog_release_other(struct bpf_prog
*fp
, struct bpf_prog
*fp_other
)
455 /* We have to repoint aux->prog to self, as we don't
456 * know whether fp here is the clone or the original.
459 bpf_prog_clone_free(fp_other
);
462 struct bpf_prog
*bpf_jit_blind_constants(struct bpf_prog
*prog
)
464 struct bpf_insn insn_buff
[16], aux
[2];
465 struct bpf_prog
*clone
, *tmp
;
466 int insn_delta
, insn_cnt
;
467 struct bpf_insn
*insn
;
470 if (!bpf_jit_blinding_enabled())
473 clone
= bpf_prog_clone_create(prog
, GFP_USER
);
475 return ERR_PTR(-ENOMEM
);
477 insn_cnt
= clone
->len
;
478 insn
= clone
->insnsi
;
480 for (i
= 0; i
< insn_cnt
; i
++, insn
++) {
481 /* We temporarily need to hold the original ld64 insn
482 * so that we can still access the first part in the
483 * second blinding run.
485 if (insn
[0].code
== (BPF_LD
| BPF_IMM
| BPF_DW
) &&
487 memcpy(aux
, insn
, sizeof(aux
));
489 rewritten
= bpf_jit_blind_insn(insn
, aux
, insn_buff
);
493 tmp
= bpf_patch_insn_single(clone
, i
, insn_buff
, rewritten
);
495 /* Patching may have repointed aux->prog during
496 * realloc from the original one, so we need to
497 * fix it up here on error.
499 bpf_jit_prog_release_other(prog
, clone
);
500 return ERR_PTR(-ENOMEM
);
504 insn_delta
= rewritten
- 1;
506 /* Walk new program and skip insns we just inserted. */
507 insn
= clone
->insnsi
+ i
+ insn_delta
;
508 insn_cnt
+= insn_delta
;
514 #endif /* CONFIG_BPF_JIT */
516 /* Base function for offset calculation. Needs to go into .text section,
517 * therefore keeping it non-static as well; will also be used by JITs
518 * anyway later on, so do not let the compiler omit it.
520 noinline u64
__bpf_call_base(u64 r1
, u64 r2
, u64 r3
, u64 r4
, u64 r5
)
524 EXPORT_SYMBOL_GPL(__bpf_call_base
);
527 * __bpf_prog_run - run eBPF program on a given context
528 * @ctx: is the data we are operating on
529 * @insn: is the array of eBPF instructions
531 * Decode and execute eBPF instructions.
533 static unsigned int __bpf_prog_run(void *ctx
, const struct bpf_insn
*insn
)
535 u64 stack
[MAX_BPF_STACK
/ sizeof(u64
)];
536 u64 regs
[MAX_BPF_REG
], tmp
;
537 static const void *jumptable
[256] = {
538 [0 ... 255] = &&default_label
,
539 /* Now overwrite non-defaults ... */
540 /* 32 bit ALU operations */
541 [BPF_ALU
| BPF_ADD
| BPF_X
] = &&ALU_ADD_X
,
542 [BPF_ALU
| BPF_ADD
| BPF_K
] = &&ALU_ADD_K
,
543 [BPF_ALU
| BPF_SUB
| BPF_X
] = &&ALU_SUB_X
,
544 [BPF_ALU
| BPF_SUB
| BPF_K
] = &&ALU_SUB_K
,
545 [BPF_ALU
| BPF_AND
| BPF_X
] = &&ALU_AND_X
,
546 [BPF_ALU
| BPF_AND
| BPF_K
] = &&ALU_AND_K
,
547 [BPF_ALU
| BPF_OR
| BPF_X
] = &&ALU_OR_X
,
548 [BPF_ALU
| BPF_OR
| BPF_K
] = &&ALU_OR_K
,
549 [BPF_ALU
| BPF_LSH
| BPF_X
] = &&ALU_LSH_X
,
550 [BPF_ALU
| BPF_LSH
| BPF_K
] = &&ALU_LSH_K
,
551 [BPF_ALU
| BPF_RSH
| BPF_X
] = &&ALU_RSH_X
,
552 [BPF_ALU
| BPF_RSH
| BPF_K
] = &&ALU_RSH_K
,
553 [BPF_ALU
| BPF_XOR
| BPF_X
] = &&ALU_XOR_X
,
554 [BPF_ALU
| BPF_XOR
| BPF_K
] = &&ALU_XOR_K
,
555 [BPF_ALU
| BPF_MUL
| BPF_X
] = &&ALU_MUL_X
,
556 [BPF_ALU
| BPF_MUL
| BPF_K
] = &&ALU_MUL_K
,
557 [BPF_ALU
| BPF_MOV
| BPF_X
] = &&ALU_MOV_X
,
558 [BPF_ALU
| BPF_MOV
| BPF_K
] = &&ALU_MOV_K
,
559 [BPF_ALU
| BPF_DIV
| BPF_X
] = &&ALU_DIV_X
,
560 [BPF_ALU
| BPF_DIV
| BPF_K
] = &&ALU_DIV_K
,
561 [BPF_ALU
| BPF_MOD
| BPF_X
] = &&ALU_MOD_X
,
562 [BPF_ALU
| BPF_MOD
| BPF_K
] = &&ALU_MOD_K
,
563 [BPF_ALU
| BPF_NEG
] = &&ALU_NEG
,
564 [BPF_ALU
| BPF_END
| BPF_TO_BE
] = &&ALU_END_TO_BE
,
565 [BPF_ALU
| BPF_END
| BPF_TO_LE
] = &&ALU_END_TO_LE
,
566 /* 64 bit ALU operations */
567 [BPF_ALU64
| BPF_ADD
| BPF_X
] = &&ALU64_ADD_X
,
568 [BPF_ALU64
| BPF_ADD
| BPF_K
] = &&ALU64_ADD_K
,
569 [BPF_ALU64
| BPF_SUB
| BPF_X
] = &&ALU64_SUB_X
,
570 [BPF_ALU64
| BPF_SUB
| BPF_K
] = &&ALU64_SUB_K
,
571 [BPF_ALU64
| BPF_AND
| BPF_X
] = &&ALU64_AND_X
,
572 [BPF_ALU64
| BPF_AND
| BPF_K
] = &&ALU64_AND_K
,
573 [BPF_ALU64
| BPF_OR
| BPF_X
] = &&ALU64_OR_X
,
574 [BPF_ALU64
| BPF_OR
| BPF_K
] = &&ALU64_OR_K
,
575 [BPF_ALU64
| BPF_LSH
| BPF_X
] = &&ALU64_LSH_X
,
576 [BPF_ALU64
| BPF_LSH
| BPF_K
] = &&ALU64_LSH_K
,
577 [BPF_ALU64
| BPF_RSH
| BPF_X
] = &&ALU64_RSH_X
,
578 [BPF_ALU64
| BPF_RSH
| BPF_K
] = &&ALU64_RSH_K
,
579 [BPF_ALU64
| BPF_XOR
| BPF_X
] = &&ALU64_XOR_X
,
580 [BPF_ALU64
| BPF_XOR
| BPF_K
] = &&ALU64_XOR_K
,
581 [BPF_ALU64
| BPF_MUL
| BPF_X
] = &&ALU64_MUL_X
,
582 [BPF_ALU64
| BPF_MUL
| BPF_K
] = &&ALU64_MUL_K
,
583 [BPF_ALU64
| BPF_MOV
| BPF_X
] = &&ALU64_MOV_X
,
584 [BPF_ALU64
| BPF_MOV
| BPF_K
] = &&ALU64_MOV_K
,
585 [BPF_ALU64
| BPF_ARSH
| BPF_X
] = &&ALU64_ARSH_X
,
586 [BPF_ALU64
| BPF_ARSH
| BPF_K
] = &&ALU64_ARSH_K
,
587 [BPF_ALU64
| BPF_DIV
| BPF_X
] = &&ALU64_DIV_X
,
588 [BPF_ALU64
| BPF_DIV
| BPF_K
] = &&ALU64_DIV_K
,
589 [BPF_ALU64
| BPF_MOD
| BPF_X
] = &&ALU64_MOD_X
,
590 [BPF_ALU64
| BPF_MOD
| BPF_K
] = &&ALU64_MOD_K
,
591 [BPF_ALU64
| BPF_NEG
] = &&ALU64_NEG
,
592 /* Call instruction */
593 [BPF_JMP
| BPF_CALL
] = &&JMP_CALL
,
594 [BPF_JMP
| BPF_CALL
| BPF_X
] = &&JMP_TAIL_CALL
,
596 [BPF_JMP
| BPF_JA
] = &&JMP_JA
,
597 [BPF_JMP
| BPF_JEQ
| BPF_X
] = &&JMP_JEQ_X
,
598 [BPF_JMP
| BPF_JEQ
| BPF_K
] = &&JMP_JEQ_K
,
599 [BPF_JMP
| BPF_JNE
| BPF_X
] = &&JMP_JNE_X
,
600 [BPF_JMP
| BPF_JNE
| BPF_K
] = &&JMP_JNE_K
,
601 [BPF_JMP
| BPF_JGT
| BPF_X
] = &&JMP_JGT_X
,
602 [BPF_JMP
| BPF_JGT
| BPF_K
] = &&JMP_JGT_K
,
603 [BPF_JMP
| BPF_JGE
| BPF_X
] = &&JMP_JGE_X
,
604 [BPF_JMP
| BPF_JGE
| BPF_K
] = &&JMP_JGE_K
,
605 [BPF_JMP
| BPF_JSGT
| BPF_X
] = &&JMP_JSGT_X
,
606 [BPF_JMP
| BPF_JSGT
| BPF_K
] = &&JMP_JSGT_K
,
607 [BPF_JMP
| BPF_JSGE
| BPF_X
] = &&JMP_JSGE_X
,
608 [BPF_JMP
| BPF_JSGE
| BPF_K
] = &&JMP_JSGE_K
,
609 [BPF_JMP
| BPF_JSET
| BPF_X
] = &&JMP_JSET_X
,
610 [BPF_JMP
| BPF_JSET
| BPF_K
] = &&JMP_JSET_K
,
612 [BPF_JMP
| BPF_EXIT
] = &&JMP_EXIT
,
613 /* Store instructions */
614 [BPF_STX
| BPF_MEM
| BPF_B
] = &&STX_MEM_B
,
615 [BPF_STX
| BPF_MEM
| BPF_H
] = &&STX_MEM_H
,
616 [BPF_STX
| BPF_MEM
| BPF_W
] = &&STX_MEM_W
,
617 [BPF_STX
| BPF_MEM
| BPF_DW
] = &&STX_MEM_DW
,
618 [BPF_STX
| BPF_XADD
| BPF_W
] = &&STX_XADD_W
,
619 [BPF_STX
| BPF_XADD
| BPF_DW
] = &&STX_XADD_DW
,
620 [BPF_ST
| BPF_MEM
| BPF_B
] = &&ST_MEM_B
,
621 [BPF_ST
| BPF_MEM
| BPF_H
] = &&ST_MEM_H
,
622 [BPF_ST
| BPF_MEM
| BPF_W
] = &&ST_MEM_W
,
623 [BPF_ST
| BPF_MEM
| BPF_DW
] = &&ST_MEM_DW
,
624 /* Load instructions */
625 [BPF_LDX
| BPF_MEM
| BPF_B
] = &&LDX_MEM_B
,
626 [BPF_LDX
| BPF_MEM
| BPF_H
] = &&LDX_MEM_H
,
627 [BPF_LDX
| BPF_MEM
| BPF_W
] = &&LDX_MEM_W
,
628 [BPF_LDX
| BPF_MEM
| BPF_DW
] = &&LDX_MEM_DW
,
629 [BPF_LD
| BPF_ABS
| BPF_W
] = &&LD_ABS_W
,
630 [BPF_LD
| BPF_ABS
| BPF_H
] = &&LD_ABS_H
,
631 [BPF_LD
| BPF_ABS
| BPF_B
] = &&LD_ABS_B
,
632 [BPF_LD
| BPF_IND
| BPF_W
] = &&LD_IND_W
,
633 [BPF_LD
| BPF_IND
| BPF_H
] = &&LD_IND_H
,
634 [BPF_LD
| BPF_IND
| BPF_B
] = &&LD_IND_B
,
635 [BPF_LD
| BPF_IMM
| BPF_DW
] = &&LD_IMM_DW
,
637 u32 tail_call_cnt
= 0;
641 #define CONT ({ insn++; goto select_insn; })
642 #define CONT_JMP ({ insn++; goto select_insn; })
644 FP
= (u64
) (unsigned long) &stack
[ARRAY_SIZE(stack
)];
645 ARG1
= (u64
) (unsigned long) ctx
;
648 goto *jumptable
[insn
->code
];
651 #define ALU(OPCODE, OP) \
652 ALU64_##OPCODE##_X: \
656 DST = (u32) DST OP (u32) SRC; \
658 ALU64_##OPCODE##_K: \
662 DST = (u32) DST OP (u32) IMM; \
693 DST
= (u64
) (u32
) insn
[0].imm
| ((u64
) (u32
) insn
[1].imm
) << 32;
697 (*(s64
*) &DST
) >>= SRC
;
700 (*(s64
*) &DST
) >>= IMM
;
703 if (unlikely(SRC
== 0))
705 div64_u64_rem(DST
, SRC
, &tmp
);
709 if (unlikely(SRC
== 0))
712 DST
= do_div(tmp
, (u32
) SRC
);
715 div64_u64_rem(DST
, IMM
, &tmp
);
720 DST
= do_div(tmp
, (u32
) IMM
);
723 if (unlikely(SRC
== 0))
725 DST
= div64_u64(DST
, SRC
);
728 if (unlikely(SRC
== 0))
731 do_div(tmp
, (u32
) SRC
);
735 DST
= div64_u64(DST
, IMM
);
739 do_div(tmp
, (u32
) IMM
);
745 DST
= (__force u16
) cpu_to_be16(DST
);
748 DST
= (__force u32
) cpu_to_be32(DST
);
751 DST
= (__force u64
) cpu_to_be64(DST
);
758 DST
= (__force u16
) cpu_to_le16(DST
);
761 DST
= (__force u32
) cpu_to_le32(DST
);
764 DST
= (__force u64
) cpu_to_le64(DST
);
771 /* Function call scratches BPF_R1-BPF_R5 registers,
772 * preserves BPF_R6-BPF_R9, and stores return value
775 BPF_R0
= (__bpf_call_base
+ insn
->imm
)(BPF_R1
, BPF_R2
, BPF_R3
,
780 struct bpf_map
*map
= (struct bpf_map
*) (unsigned long) BPF_R2
;
781 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
782 struct bpf_prog
*prog
;
785 if (unlikely(index
>= array
->map
.max_entries
))
787 if (unlikely(tail_call_cnt
> MAX_TAIL_CALL_CNT
))
792 prog
= READ_ONCE(array
->ptrs
[index
]);
796 /* ARG1 at this point is guaranteed to point to CTX from
797 * the verifier side due to the fact that the tail call is
798 * handeled like a helper, that is, bpf_tail_call_proto,
799 * where arg1_type is ARG_PTR_TO_CTX.
859 if (((s64
) DST
) > ((s64
) SRC
)) {
865 if (((s64
) DST
) > ((s64
) IMM
)) {
871 if (((s64
) DST
) >= ((s64
) SRC
)) {
877 if (((s64
) DST
) >= ((s64
) IMM
)) {
897 /* STX and ST and LDX*/
898 #define LDST(SIZEOP, SIZE) \
900 *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \
903 *(SIZE *)(unsigned long) (DST + insn->off) = IMM; \
906 DST = *(SIZE *)(unsigned long) (SRC + insn->off); \
914 STX_XADD_W
: /* lock xadd *(u32 *)(dst_reg + off16) += src_reg */
915 atomic_add((u32
) SRC
, (atomic_t
*)(unsigned long)
918 STX_XADD_DW
: /* lock xadd *(u64 *)(dst_reg + off16) += src_reg */
919 atomic64_add((u64
) SRC
, (atomic64_t
*)(unsigned long)
922 LD_ABS_W
: /* BPF_R0 = ntohl(*(u32 *) (skb->data + imm32)) */
925 /* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are
926 * only appearing in the programs where ctx ==
927 * skb. All programs keep 'ctx' in regs[BPF_REG_CTX]
928 * == BPF_R6, bpf_convert_filter() saves it in BPF_R6,
929 * internal BPF verifier will check that BPF_R6 ==
932 * BPF_ABS and BPF_IND are wrappers of function calls,
933 * so they scratch BPF_R1-BPF_R5 registers, preserve
934 * BPF_R6-BPF_R9, and store return value into BPF_R0.
937 * ctx == skb == BPF_R6 == CTX
940 * SRC == any register
941 * IMM == 32-bit immediate
944 * BPF_R0 - 8/16/32-bit skb data converted to cpu endianness
947 ptr
= bpf_load_pointer((struct sk_buff
*) (unsigned long) CTX
, off
, 4, &tmp
);
948 if (likely(ptr
!= NULL
)) {
949 BPF_R0
= get_unaligned_be32(ptr
);
954 LD_ABS_H
: /* BPF_R0 = ntohs(*(u16 *) (skb->data + imm32)) */
957 ptr
= bpf_load_pointer((struct sk_buff
*) (unsigned long) CTX
, off
, 2, &tmp
);
958 if (likely(ptr
!= NULL
)) {
959 BPF_R0
= get_unaligned_be16(ptr
);
964 LD_ABS_B
: /* BPF_R0 = *(u8 *) (skb->data + imm32) */
967 ptr
= bpf_load_pointer((struct sk_buff
*) (unsigned long) CTX
, off
, 1, &tmp
);
968 if (likely(ptr
!= NULL
)) {
974 LD_IND_W
: /* BPF_R0 = ntohl(*(u32 *) (skb->data + src_reg + imm32)) */
977 LD_IND_H
: /* BPF_R0 = ntohs(*(u16 *) (skb->data + src_reg + imm32)) */
980 LD_IND_B
: /* BPF_R0 = *(u8 *) (skb->data + src_reg + imm32) */
985 /* If we ever reach this, we have a bug somewhere. */
986 WARN_RATELIMIT(1, "unknown opcode %02x\n", insn
->code
);
989 STACK_FRAME_NON_STANDARD(__bpf_prog_run
); /* jump table */
991 bool bpf_prog_array_compatible(struct bpf_array
*array
,
992 const struct bpf_prog
*fp
)
994 if (!array
->owner_prog_type
) {
995 /* There's no owner yet where we could check for
998 array
->owner_prog_type
= fp
->type
;
999 array
->owner_jited
= fp
->jited
;
1004 return array
->owner_prog_type
== fp
->type
&&
1005 array
->owner_jited
== fp
->jited
;
1008 static int bpf_check_tail_call(const struct bpf_prog
*fp
)
1010 struct bpf_prog_aux
*aux
= fp
->aux
;
1013 for (i
= 0; i
< aux
->used_map_cnt
; i
++) {
1014 struct bpf_map
*map
= aux
->used_maps
[i
];
1015 struct bpf_array
*array
;
1017 if (map
->map_type
!= BPF_MAP_TYPE_PROG_ARRAY
)
1020 array
= container_of(map
, struct bpf_array
, map
);
1021 if (!bpf_prog_array_compatible(array
, fp
))
1029 * bpf_prog_select_runtime - select exec runtime for BPF program
1030 * @fp: bpf_prog populated with internal BPF program
1031 * @err: pointer to error variable
1033 * Try to JIT eBPF program, if JIT is not available, use interpreter.
1034 * The BPF program will be executed via BPF_PROG_RUN() macro.
1036 struct bpf_prog
*bpf_prog_select_runtime(struct bpf_prog
*fp
, int *err
)
1038 fp
->bpf_func
= (void *) __bpf_prog_run
;
1040 /* eBPF JITs can rewrite the program in case constant
1041 * blinding is active. However, in case of error during
1042 * blinding, bpf_int_jit_compile() must always return a
1043 * valid program, which in this case would simply not
1044 * be JITed, but falls back to the interpreter.
1046 fp
= bpf_int_jit_compile(fp
);
1047 bpf_prog_lock_ro(fp
);
1049 /* The tail call compatibility check can only be done at
1050 * this late stage as we need to determine, if we deal
1051 * with JITed or non JITed program concatenations and not
1052 * all eBPF JITs might immediately support all features.
1054 *err
= bpf_check_tail_call(fp
);
1058 EXPORT_SYMBOL_GPL(bpf_prog_select_runtime
);
1060 static void bpf_prog_free_deferred(struct work_struct
*work
)
1062 struct bpf_prog_aux
*aux
;
1064 aux
= container_of(work
, struct bpf_prog_aux
, work
);
1065 bpf_jit_free(aux
->prog
);
1068 /* Free internal BPF program */
1069 void bpf_prog_free(struct bpf_prog
*fp
)
1071 struct bpf_prog_aux
*aux
= fp
->aux
;
1073 INIT_WORK(&aux
->work
, bpf_prog_free_deferred
);
1074 schedule_work(&aux
->work
);
1076 EXPORT_SYMBOL_GPL(bpf_prog_free
);
1078 /* RNG for unpriviledged user space with separated state from prandom_u32(). */
1079 static DEFINE_PER_CPU(struct rnd_state
, bpf_user_rnd_state
);
1081 void bpf_user_rnd_init_once(void)
1083 prandom_init_once(&bpf_user_rnd_state
);
1086 BPF_CALL_0(bpf_user_rnd_u32
)
1088 /* Should someone ever have the rather unwise idea to use some
1089 * of the registers passed into this function, then note that
1090 * this function is called from native eBPF and classic-to-eBPF
1091 * transformations. Register assignments from both sides are
1092 * different, f.e. classic always sets fn(ctx, A, X) here.
1094 struct rnd_state
*state
;
1097 state
= &get_cpu_var(bpf_user_rnd_state
);
1098 res
= prandom_u32_state(state
);
1099 put_cpu_var(bpf_user_rnd_state
);
1104 /* Weak definitions of helper functions in case we don't have bpf syscall. */
1105 const struct bpf_func_proto bpf_map_lookup_elem_proto __weak
;
1106 const struct bpf_func_proto bpf_map_update_elem_proto __weak
;
1107 const struct bpf_func_proto bpf_map_delete_elem_proto __weak
;
1109 const struct bpf_func_proto bpf_get_prandom_u32_proto __weak
;
1110 const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak
;
1111 const struct bpf_func_proto bpf_get_numa_node_id_proto __weak
;
1112 const struct bpf_func_proto bpf_ktime_get_ns_proto __weak
;
1114 const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak
;
1115 const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak
;
1116 const struct bpf_func_proto bpf_get_current_comm_proto __weak
;
1118 const struct bpf_func_proto
* __weak
bpf_get_trace_printk_proto(void)
1124 bpf_event_output(struct bpf_map
*map
, u64 flags
, void *meta
, u64 meta_size
,
1125 void *ctx
, u64 ctx_size
, bpf_ctx_copy_t ctx_copy
)
1130 /* Always built-in helper functions. */
1131 const struct bpf_func_proto bpf_tail_call_proto
= {
1134 .ret_type
= RET_VOID
,
1135 .arg1_type
= ARG_PTR_TO_CTX
,
1136 .arg2_type
= ARG_CONST_MAP_PTR
,
1137 .arg3_type
= ARG_ANYTHING
,
1140 /* For classic BPF JITs that don't implement bpf_int_jit_compile(). */
1141 struct bpf_prog
* __weak
bpf_int_jit_compile(struct bpf_prog
*prog
)
1146 bool __weak
bpf_helper_changes_skb_data(void *func
)
1151 /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
1152 * skb_copy_bits(), so provide a weak definition of it for NET-less config.
1154 int __weak
skb_copy_bits(const struct sk_buff
*skb
, int offset
, void *to
,