2 * BPF JIT compiler for ARM64
4 * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 #define pr_fmt(fmt) "bpf_jit: " fmt
21 #include <linux/filter.h>
22 #include <linux/printk.h>
23 #include <linux/skbuff.h>
24 #include <linux/slab.h>
26 #include <asm/byteorder.h>
27 #include <asm/cacheflush.h>
28 #include <asm/debug-monitors.h>
32 int bpf_jit_enable __read_mostly
;
34 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
35 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
37 /* Map BPF registers to A64 registers */
38 static const int bpf2a64
[] = {
39 /* return value from in-kernel function, and exit value from eBPF */
40 [BPF_REG_0
] = A64_R(7),
41 /* arguments from eBPF program to in-kernel function */
42 [BPF_REG_1
] = A64_R(0),
43 [BPF_REG_2
] = A64_R(1),
44 [BPF_REG_3
] = A64_R(2),
45 [BPF_REG_4
] = A64_R(3),
46 [BPF_REG_5
] = A64_R(4),
47 /* callee saved registers that in-kernel function will preserve */
48 [BPF_REG_6
] = A64_R(19),
49 [BPF_REG_7
] = A64_R(20),
50 [BPF_REG_8
] = A64_R(21),
51 [BPF_REG_9
] = A64_R(22),
52 /* read-only frame pointer to access stack */
53 [BPF_REG_FP
] = A64_R(25),
54 /* temporary register for internal BPF JIT */
55 [TMP_REG_1
] = A64_R(23),
56 [TMP_REG_2
] = A64_R(24),
57 /* temporary register for blinding constants */
58 [BPF_REG_AX
] = A64_R(9),
62 const struct bpf_prog
*prog
;
70 static inline void emit(const u32 insn
, struct jit_ctx
*ctx
)
72 if (ctx
->image
!= NULL
)
73 ctx
->image
[ctx
->idx
] = cpu_to_le32(insn
);
78 static inline void emit_a64_mov_i64(const int reg
, const u64 val
,
84 emit(A64_MOVZ(1, reg
, tmp
& 0xffff, shift
), ctx
);
89 emit(A64_MOVK(1, reg
, tmp
& 0xffff, shift
), ctx
);
95 static inline void emit_a64_mov_i(const int is64
, const int reg
,
96 const s32 val
, struct jit_ctx
*ctx
)
99 u16 lo
= val
& 0xffff;
103 emit(A64_MOVN(is64
, reg
, (u16
)~lo
, 0), ctx
);
105 emit(A64_MOVN(is64
, reg
, (u16
)~hi
, 16), ctx
);
106 emit(A64_MOVK(is64
, reg
, lo
, 0), ctx
);
109 emit(A64_MOVZ(is64
, reg
, lo
, 0), ctx
);
111 emit(A64_MOVK(is64
, reg
, hi
, 16), ctx
);
115 static inline int bpf2a64_offset(int bpf_to
, int bpf_from
,
116 const struct jit_ctx
*ctx
)
118 int to
= ctx
->offset
[bpf_to
];
119 /* -1 to account for the Branch instruction */
120 int from
= ctx
->offset
[bpf_from
] - 1;
125 static void jit_fill_hole(void *area
, unsigned int size
)
128 /* We are guaranteed to have aligned memory. */
129 for (ptr
= area
; size
>= sizeof(u32
); size
-= sizeof(u32
))
130 *ptr
++ = cpu_to_le32(AARCH64_BREAK_FAULT
);
133 static inline int epilogue_offset(const struct jit_ctx
*ctx
)
135 int to
= ctx
->epilogue_offset
;
141 /* Stack must be multiples of 16B */
142 #define STACK_ALIGN(sz) (((sz) + 15) & ~15)
144 #define _STACK_SIZE \
146 + 4 /* extra for skb_copy_bits buffer */)
148 #define STACK_SIZE STACK_ALIGN(_STACK_SIZE)
150 static void build_prologue(struct jit_ctx
*ctx
)
152 const u8 r6
= bpf2a64
[BPF_REG_6
];
153 const u8 r7
= bpf2a64
[BPF_REG_7
];
154 const u8 r8
= bpf2a64
[BPF_REG_8
];
155 const u8 r9
= bpf2a64
[BPF_REG_9
];
156 const u8 fp
= bpf2a64
[BPF_REG_FP
];
157 const u8 tmp1
= bpf2a64
[TMP_REG_1
];
158 const u8 tmp2
= bpf2a64
[TMP_REG_2
];
161 * BPF prog stack layout
164 * original A64_SP => 0:+-----+ BPF prologue
166 * current A64_FP => -16:+-----+
167 * | ... | callee saved registers
170 * BPF fp register => -80:+-----+ <= (BPF_FP)
172 * | ... | BPF prog stack
174 * +-----+ <= (BPF_FP - MAX_BPF_STACK)
175 * |RSVD | JIT scratchpad
176 * current A64_SP => +-----+ <= (BPF_FP - STACK_SIZE)
178 * | ... | Function call stack
185 /* Save FP and LR registers to stay align with ARM64 AAPCS */
186 emit(A64_PUSH(A64_FP
, A64_LR
, A64_SP
), ctx
);
187 emit(A64_MOV(1, A64_FP
, A64_SP
), ctx
);
189 /* Save callee-saved register */
190 emit(A64_PUSH(r6
, r7
, A64_SP
), ctx
);
191 emit(A64_PUSH(r8
, r9
, A64_SP
), ctx
);
193 emit(A64_PUSH(tmp1
, tmp2
, A64_SP
), ctx
);
195 /* Save fp (x25) and x26. SP requires 16 bytes alignment */
196 emit(A64_PUSH(fp
, A64_R(26), A64_SP
), ctx
);
198 /* Set up BPF prog stack base register (x25) */
199 emit(A64_MOV(1, fp
, A64_SP
), ctx
);
201 /* Set up function call stack */
202 emit(A64_SUB_I(1, A64_SP
, A64_SP
, STACK_SIZE
), ctx
);
205 static void build_epilogue(struct jit_ctx
*ctx
)
207 const u8 r0
= bpf2a64
[BPF_REG_0
];
208 const u8 r6
= bpf2a64
[BPF_REG_6
];
209 const u8 r7
= bpf2a64
[BPF_REG_7
];
210 const u8 r8
= bpf2a64
[BPF_REG_8
];
211 const u8 r9
= bpf2a64
[BPF_REG_9
];
212 const u8 fp
= bpf2a64
[BPF_REG_FP
];
213 const u8 tmp1
= bpf2a64
[TMP_REG_1
];
214 const u8 tmp2
= bpf2a64
[TMP_REG_2
];
216 /* We're done with BPF stack */
217 emit(A64_ADD_I(1, A64_SP
, A64_SP
, STACK_SIZE
), ctx
);
219 /* Restore fs (x25) and x26 */
220 emit(A64_POP(fp
, A64_R(26), A64_SP
), ctx
);
222 /* Restore callee-saved register */
224 emit(A64_POP(tmp1
, tmp2
, A64_SP
), ctx
);
225 emit(A64_POP(r8
, r9
, A64_SP
), ctx
);
226 emit(A64_POP(r6
, r7
, A64_SP
), ctx
);
228 /* Restore FP/LR registers */
229 emit(A64_POP(A64_FP
, A64_LR
, A64_SP
), ctx
);
231 /* Set return value */
232 emit(A64_MOV(1, A64_R(0), r0
), ctx
);
234 emit(A64_RET(A64_LR
), ctx
);
237 /* JITs an eBPF instruction.
239 * 0 - successfully JITed an 8-byte eBPF instruction.
240 * >0 - successfully JITed a 16-byte eBPF instruction.
241 * <0 - failed to JIT.
243 static int build_insn(const struct bpf_insn
*insn
, struct jit_ctx
*ctx
)
245 const u8 code
= insn
->code
;
246 const u8 dst
= bpf2a64
[insn
->dst_reg
];
247 const u8 src
= bpf2a64
[insn
->src_reg
];
248 const u8 tmp
= bpf2a64
[TMP_REG_1
];
249 const u8 tmp2
= bpf2a64
[TMP_REG_2
];
250 const s16 off
= insn
->off
;
251 const s32 imm
= insn
->imm
;
252 const int i
= insn
- ctx
->prog
->insnsi
;
253 const bool is64
= BPF_CLASS(code
) == BPF_ALU64
;
257 #define check_imm(bits, imm) do { \
258 if ((((imm) > 0) && ((imm) >> (bits))) || \
259 (((imm) < 0) && (~(imm) >> (bits)))) { \
260 pr_info("[%2d] imm=%d(0x%x) out of range\n", \
265 #define check_imm19(imm) check_imm(19, imm)
266 #define check_imm26(imm) check_imm(26, imm)
270 case BPF_ALU
| BPF_MOV
| BPF_X
:
271 case BPF_ALU64
| BPF_MOV
| BPF_X
:
272 emit(A64_MOV(is64
, dst
, src
), ctx
);
274 /* dst = dst OP src */
275 case BPF_ALU
| BPF_ADD
| BPF_X
:
276 case BPF_ALU64
| BPF_ADD
| BPF_X
:
277 emit(A64_ADD(is64
, dst
, dst
, src
), ctx
);
279 case BPF_ALU
| BPF_SUB
| BPF_X
:
280 case BPF_ALU64
| BPF_SUB
| BPF_X
:
281 emit(A64_SUB(is64
, dst
, dst
, src
), ctx
);
283 case BPF_ALU
| BPF_AND
| BPF_X
:
284 case BPF_ALU64
| BPF_AND
| BPF_X
:
285 emit(A64_AND(is64
, dst
, dst
, src
), ctx
);
287 case BPF_ALU
| BPF_OR
| BPF_X
:
288 case BPF_ALU64
| BPF_OR
| BPF_X
:
289 emit(A64_ORR(is64
, dst
, dst
, src
), ctx
);
291 case BPF_ALU
| BPF_XOR
| BPF_X
:
292 case BPF_ALU64
| BPF_XOR
| BPF_X
:
293 emit(A64_EOR(is64
, dst
, dst
, src
), ctx
);
295 case BPF_ALU
| BPF_MUL
| BPF_X
:
296 case BPF_ALU64
| BPF_MUL
| BPF_X
:
297 emit(A64_MUL(is64
, dst
, dst
, src
), ctx
);
299 case BPF_ALU
| BPF_DIV
| BPF_X
:
300 case BPF_ALU64
| BPF_DIV
| BPF_X
:
301 case BPF_ALU
| BPF_MOD
| BPF_X
:
302 case BPF_ALU64
| BPF_MOD
| BPF_X
:
304 const u8 r0
= bpf2a64
[BPF_REG_0
];
306 /* if (src == 0) return 0 */
307 jmp_offset
= 3; /* skip ahead to else path */
308 check_imm19(jmp_offset
);
309 emit(A64_CBNZ(is64
, src
, jmp_offset
), ctx
);
310 emit(A64_MOVZ(1, r0
, 0, 0), ctx
);
311 jmp_offset
= epilogue_offset(ctx
);
312 check_imm26(jmp_offset
);
313 emit(A64_B(jmp_offset
), ctx
);
315 switch (BPF_OP(code
)) {
317 emit(A64_UDIV(is64
, dst
, dst
, src
), ctx
);
321 emit(A64_UDIV(is64
, tmp
, dst
, src
), ctx
);
322 emit(A64_MUL(is64
, tmp
, tmp
, src
), ctx
);
323 emit(A64_SUB(is64
, dst
, dst
, tmp
), ctx
);
328 case BPF_ALU
| BPF_LSH
| BPF_X
:
329 case BPF_ALU64
| BPF_LSH
| BPF_X
:
330 emit(A64_LSLV(is64
, dst
, dst
, src
), ctx
);
332 case BPF_ALU
| BPF_RSH
| BPF_X
:
333 case BPF_ALU64
| BPF_RSH
| BPF_X
:
334 emit(A64_LSRV(is64
, dst
, dst
, src
), ctx
);
336 case BPF_ALU
| BPF_ARSH
| BPF_X
:
337 case BPF_ALU64
| BPF_ARSH
| BPF_X
:
338 emit(A64_ASRV(is64
, dst
, dst
, src
), ctx
);
341 case BPF_ALU
| BPF_NEG
:
342 case BPF_ALU64
| BPF_NEG
:
343 emit(A64_NEG(is64
, dst
, dst
), ctx
);
345 /* dst = BSWAP##imm(dst) */
346 case BPF_ALU
| BPF_END
| BPF_FROM_LE
:
347 case BPF_ALU
| BPF_END
| BPF_FROM_BE
:
348 #ifdef CONFIG_CPU_BIG_ENDIAN
349 if (BPF_SRC(code
) == BPF_FROM_BE
)
351 #else /* !CONFIG_CPU_BIG_ENDIAN */
352 if (BPF_SRC(code
) == BPF_FROM_LE
)
357 emit(A64_REV16(is64
, dst
, dst
), ctx
);
358 /* zero-extend 16 bits into 64 bits */
359 emit(A64_UXTH(is64
, dst
, dst
), ctx
);
362 emit(A64_REV32(is64
, dst
, dst
), ctx
);
363 /* upper 32 bits already cleared */
366 emit(A64_REV64(dst
, dst
), ctx
);
373 /* zero-extend 16 bits into 64 bits */
374 emit(A64_UXTH(is64
, dst
, dst
), ctx
);
377 /* zero-extend 32 bits into 64 bits */
378 emit(A64_UXTW(is64
, dst
, dst
), ctx
);
386 case BPF_ALU
| BPF_MOV
| BPF_K
:
387 case BPF_ALU64
| BPF_MOV
| BPF_K
:
388 emit_a64_mov_i(is64
, dst
, imm
, ctx
);
390 /* dst = dst OP imm */
391 case BPF_ALU
| BPF_ADD
| BPF_K
:
392 case BPF_ALU64
| BPF_ADD
| BPF_K
:
394 emit_a64_mov_i(is64
, tmp
, imm
, ctx
);
395 emit(A64_ADD(is64
, dst
, dst
, tmp
), ctx
);
397 case BPF_ALU
| BPF_SUB
| BPF_K
:
398 case BPF_ALU64
| BPF_SUB
| BPF_K
:
400 emit_a64_mov_i(is64
, tmp
, imm
, ctx
);
401 emit(A64_SUB(is64
, dst
, dst
, tmp
), ctx
);
403 case BPF_ALU
| BPF_AND
| BPF_K
:
404 case BPF_ALU64
| BPF_AND
| BPF_K
:
406 emit_a64_mov_i(is64
, tmp
, imm
, ctx
);
407 emit(A64_AND(is64
, dst
, dst
, tmp
), ctx
);
409 case BPF_ALU
| BPF_OR
| BPF_K
:
410 case BPF_ALU64
| BPF_OR
| BPF_K
:
412 emit_a64_mov_i(is64
, tmp
, imm
, ctx
);
413 emit(A64_ORR(is64
, dst
, dst
, tmp
), ctx
);
415 case BPF_ALU
| BPF_XOR
| BPF_K
:
416 case BPF_ALU64
| BPF_XOR
| BPF_K
:
418 emit_a64_mov_i(is64
, tmp
, imm
, ctx
);
419 emit(A64_EOR(is64
, dst
, dst
, tmp
), ctx
);
421 case BPF_ALU
| BPF_MUL
| BPF_K
:
422 case BPF_ALU64
| BPF_MUL
| BPF_K
:
424 emit_a64_mov_i(is64
, tmp
, imm
, ctx
);
425 emit(A64_MUL(is64
, dst
, dst
, tmp
), ctx
);
427 case BPF_ALU
| BPF_DIV
| BPF_K
:
428 case BPF_ALU64
| BPF_DIV
| BPF_K
:
430 emit_a64_mov_i(is64
, tmp
, imm
, ctx
);
431 emit(A64_UDIV(is64
, dst
, dst
, tmp
), ctx
);
433 case BPF_ALU
| BPF_MOD
| BPF_K
:
434 case BPF_ALU64
| BPF_MOD
| BPF_K
:
436 emit_a64_mov_i(is64
, tmp2
, imm
, ctx
);
437 emit(A64_UDIV(is64
, tmp
, dst
, tmp2
), ctx
);
438 emit(A64_MUL(is64
, tmp
, tmp
, tmp2
), ctx
);
439 emit(A64_SUB(is64
, dst
, dst
, tmp
), ctx
);
441 case BPF_ALU
| BPF_LSH
| BPF_K
:
442 case BPF_ALU64
| BPF_LSH
| BPF_K
:
443 emit(A64_LSL(is64
, dst
, dst
, imm
), ctx
);
445 case BPF_ALU
| BPF_RSH
| BPF_K
:
446 case BPF_ALU64
| BPF_RSH
| BPF_K
:
447 emit(A64_LSR(is64
, dst
, dst
, imm
), ctx
);
449 case BPF_ALU
| BPF_ARSH
| BPF_K
:
450 case BPF_ALU64
| BPF_ARSH
| BPF_K
:
451 emit(A64_ASR(is64
, dst
, dst
, imm
), ctx
);
455 case BPF_JMP
| BPF_JA
:
456 jmp_offset
= bpf2a64_offset(i
+ off
, i
, ctx
);
457 check_imm26(jmp_offset
);
458 emit(A64_B(jmp_offset
), ctx
);
460 /* IF (dst COND src) JUMP off */
461 case BPF_JMP
| BPF_JEQ
| BPF_X
:
462 case BPF_JMP
| BPF_JGT
| BPF_X
:
463 case BPF_JMP
| BPF_JGE
| BPF_X
:
464 case BPF_JMP
| BPF_JNE
| BPF_X
:
465 case BPF_JMP
| BPF_JSGT
| BPF_X
:
466 case BPF_JMP
| BPF_JSGE
| BPF_X
:
467 emit(A64_CMP(1, dst
, src
), ctx
);
469 jmp_offset
= bpf2a64_offset(i
+ off
, i
, ctx
);
470 check_imm19(jmp_offset
);
471 switch (BPF_OP(code
)) {
473 jmp_cond
= A64_COND_EQ
;
476 jmp_cond
= A64_COND_HI
;
479 jmp_cond
= A64_COND_CS
;
483 jmp_cond
= A64_COND_NE
;
486 jmp_cond
= A64_COND_GT
;
489 jmp_cond
= A64_COND_GE
;
494 emit(A64_B_(jmp_cond
, jmp_offset
), ctx
);
496 case BPF_JMP
| BPF_JSET
| BPF_X
:
497 emit(A64_TST(1, dst
, src
), ctx
);
499 /* IF (dst COND imm) JUMP off */
500 case BPF_JMP
| BPF_JEQ
| BPF_K
:
501 case BPF_JMP
| BPF_JGT
| BPF_K
:
502 case BPF_JMP
| BPF_JGE
| BPF_K
:
503 case BPF_JMP
| BPF_JNE
| BPF_K
:
504 case BPF_JMP
| BPF_JSGT
| BPF_K
:
505 case BPF_JMP
| BPF_JSGE
| BPF_K
:
507 emit_a64_mov_i(1, tmp
, imm
, ctx
);
508 emit(A64_CMP(1, dst
, tmp
), ctx
);
510 case BPF_JMP
| BPF_JSET
| BPF_K
:
512 emit_a64_mov_i(1, tmp
, imm
, ctx
);
513 emit(A64_TST(1, dst
, tmp
), ctx
);
516 case BPF_JMP
| BPF_CALL
:
518 const u8 r0
= bpf2a64
[BPF_REG_0
];
519 const u64 func
= (u64
)__bpf_call_base
+ imm
;
522 emit_a64_mov_i64(tmp
, func
, ctx
);
523 emit(A64_PUSH(A64_FP
, A64_LR
, A64_SP
), ctx
);
524 emit(A64_MOV(1, A64_FP
, A64_SP
), ctx
);
525 emit(A64_BLR(tmp
), ctx
);
526 emit(A64_MOV(1, r0
, A64_R(0)), ctx
);
527 emit(A64_POP(A64_FP
, A64_LR
, A64_SP
), ctx
);
530 /* function return */
531 case BPF_JMP
| BPF_EXIT
:
532 /* Optimization: when last instruction is EXIT,
533 simply fallthrough to epilogue. */
534 if (i
== ctx
->prog
->len
- 1)
536 jmp_offset
= epilogue_offset(ctx
);
537 check_imm26(jmp_offset
);
538 emit(A64_B(jmp_offset
), ctx
);
542 case BPF_LD
| BPF_IMM
| BPF_DW
:
544 const struct bpf_insn insn1
= insn
[1];
547 if (insn1
.code
!= 0 || insn1
.src_reg
!= 0 ||
548 insn1
.dst_reg
!= 0 || insn1
.off
!= 0) {
549 /* Note: verifier in BPF core must catch invalid
552 pr_err_once("Invalid BPF_LD_IMM64 instruction\n");
556 imm64
= (u64
)insn1
.imm
<< 32 | (u32
)imm
;
557 emit_a64_mov_i64(dst
, imm64
, ctx
);
562 /* LDX: dst = *(size *)(src + off) */
563 case BPF_LDX
| BPF_MEM
| BPF_W
:
564 case BPF_LDX
| BPF_MEM
| BPF_H
:
565 case BPF_LDX
| BPF_MEM
| BPF_B
:
566 case BPF_LDX
| BPF_MEM
| BPF_DW
:
568 emit_a64_mov_i(1, tmp
, off
, ctx
);
569 switch (BPF_SIZE(code
)) {
571 emit(A64_LDR32(dst
, src
, tmp
), ctx
);
574 emit(A64_LDRH(dst
, src
, tmp
), ctx
);
577 emit(A64_LDRB(dst
, src
, tmp
), ctx
);
580 emit(A64_LDR64(dst
, src
, tmp
), ctx
);
585 /* ST: *(size *)(dst + off) = imm */
586 case BPF_ST
| BPF_MEM
| BPF_W
:
587 case BPF_ST
| BPF_MEM
| BPF_H
:
588 case BPF_ST
| BPF_MEM
| BPF_B
:
589 case BPF_ST
| BPF_MEM
| BPF_DW
:
590 /* Load imm to a register then store it */
592 emit_a64_mov_i(1, tmp2
, off
, ctx
);
593 emit_a64_mov_i(1, tmp
, imm
, ctx
);
594 switch (BPF_SIZE(code
)) {
596 emit(A64_STR32(tmp
, dst
, tmp2
), ctx
);
599 emit(A64_STRH(tmp
, dst
, tmp2
), ctx
);
602 emit(A64_STRB(tmp
, dst
, tmp2
), ctx
);
605 emit(A64_STR64(tmp
, dst
, tmp2
), ctx
);
610 /* STX: *(size *)(dst + off) = src */
611 case BPF_STX
| BPF_MEM
| BPF_W
:
612 case BPF_STX
| BPF_MEM
| BPF_H
:
613 case BPF_STX
| BPF_MEM
| BPF_B
:
614 case BPF_STX
| BPF_MEM
| BPF_DW
:
616 emit_a64_mov_i(1, tmp
, off
, ctx
);
617 switch (BPF_SIZE(code
)) {
619 emit(A64_STR32(src
, dst
, tmp
), ctx
);
622 emit(A64_STRH(src
, dst
, tmp
), ctx
);
625 emit(A64_STRB(src
, dst
, tmp
), ctx
);
628 emit(A64_STR64(src
, dst
, tmp
), ctx
);
632 /* STX XADD: lock *(u32 *)(dst + off) += src */
633 case BPF_STX
| BPF_XADD
| BPF_W
:
634 /* STX XADD: lock *(u64 *)(dst + off) += src */
635 case BPF_STX
| BPF_XADD
| BPF_DW
:
638 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
639 case BPF_LD
| BPF_ABS
| BPF_W
:
640 case BPF_LD
| BPF_ABS
| BPF_H
:
641 case BPF_LD
| BPF_ABS
| BPF_B
:
642 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
643 case BPF_LD
| BPF_IND
| BPF_W
:
644 case BPF_LD
| BPF_IND
| BPF_H
:
645 case BPF_LD
| BPF_IND
| BPF_B
:
647 const u8 r0
= bpf2a64
[BPF_REG_0
]; /* r0 = return value */
648 const u8 r6
= bpf2a64
[BPF_REG_6
]; /* r6 = pointer to sk_buff */
649 const u8 fp
= bpf2a64
[BPF_REG_FP
];
650 const u8 r1
= bpf2a64
[BPF_REG_1
]; /* r1: struct sk_buff *skb */
651 const u8 r2
= bpf2a64
[BPF_REG_2
]; /* r2: int k */
652 const u8 r3
= bpf2a64
[BPF_REG_3
]; /* r3: unsigned int size */
653 const u8 r4
= bpf2a64
[BPF_REG_4
]; /* r4: void *buffer */
654 const u8 r5
= bpf2a64
[BPF_REG_5
]; /* r5: void *(*func)(...) */
657 emit(A64_MOV(1, r1
, r6
), ctx
);
658 emit_a64_mov_i(0, r2
, imm
, ctx
);
659 if (BPF_MODE(code
) == BPF_IND
)
660 emit(A64_ADD(0, r2
, r2
, src
), ctx
);
661 switch (BPF_SIZE(code
)) {
674 emit_a64_mov_i64(r3
, size
, ctx
);
675 emit(A64_SUB_I(1, r4
, fp
, STACK_SIZE
), ctx
);
676 emit_a64_mov_i64(r5
, (unsigned long)bpf_load_pointer
, ctx
);
677 emit(A64_PUSH(A64_FP
, A64_LR
, A64_SP
), ctx
);
678 emit(A64_MOV(1, A64_FP
, A64_SP
), ctx
);
679 emit(A64_BLR(r5
), ctx
);
680 emit(A64_MOV(1, r0
, A64_R(0)), ctx
);
681 emit(A64_POP(A64_FP
, A64_LR
, A64_SP
), ctx
);
683 jmp_offset
= epilogue_offset(ctx
);
684 check_imm19(jmp_offset
);
685 emit(A64_CBZ(1, r0
, jmp_offset
), ctx
);
686 emit(A64_MOV(1, r5
, r0
), ctx
);
687 switch (BPF_SIZE(code
)) {
689 emit(A64_LDR32(r0
, r5
, A64_ZR
), ctx
);
690 #ifndef CONFIG_CPU_BIG_ENDIAN
691 emit(A64_REV32(0, r0
, r0
), ctx
);
695 emit(A64_LDRH(r0
, r5
, A64_ZR
), ctx
);
696 #ifndef CONFIG_CPU_BIG_ENDIAN
697 emit(A64_REV16(0, r0
, r0
), ctx
);
701 emit(A64_LDRB(r0
, r5
, A64_ZR
), ctx
);
707 pr_info_once("*** NOT YET: opcode %02x ***\n", code
);
711 pr_err_once("unknown opcode %02x\n", code
);
718 static int build_body(struct jit_ctx
*ctx
)
720 const struct bpf_prog
*prog
= ctx
->prog
;
723 for (i
= 0; i
< prog
->len
; i
++) {
724 const struct bpf_insn
*insn
= &prog
->insnsi
[i
];
727 ret
= build_insn(insn
, ctx
);
729 if (ctx
->image
== NULL
)
730 ctx
->offset
[i
] = ctx
->idx
;
743 static int validate_code(struct jit_ctx
*ctx
)
747 for (i
= 0; i
< ctx
->idx
; i
++) {
748 u32 a64_insn
= le32_to_cpu(ctx
->image
[i
]);
750 if (a64_insn
== AARCH64_BREAK_FAULT
)
757 static inline void bpf_flush_icache(void *start
, void *end
)
759 flush_icache_range((unsigned long)start
, (unsigned long)end
);
762 void bpf_jit_compile(struct bpf_prog
*prog
)
764 /* Nothing to do here. We support Internal BPF. */
767 struct bpf_prog
*bpf_int_jit_compile(struct bpf_prog
*prog
)
769 struct bpf_prog
*tmp
, *orig_prog
= prog
;
770 struct bpf_binary_header
*header
;
771 bool tmp_blinded
= false;
779 tmp
= bpf_jit_blind_constants(prog
);
780 /* If blinding was requested and we failed during blinding,
781 * we must fall back to the interpreter.
790 memset(&ctx
, 0, sizeof(ctx
));
793 ctx
.offset
= kcalloc(prog
->len
, sizeof(int), GFP_KERNEL
);
794 if (ctx
.offset
== NULL
) {
799 /* 1. Initial fake pass to compute ctx->idx. */
801 /* Fake pass to fill in ctx->offset and ctx->tmp_used. */
802 if (build_body(&ctx
)) {
807 build_prologue(&ctx
);
809 ctx
.epilogue_offset
= ctx
.idx
;
810 build_epilogue(&ctx
);
812 /* Now we know the actual image size. */
813 image_size
= sizeof(u32
) * ctx
.idx
;
814 header
= bpf_jit_binary_alloc(image_size
, &image_ptr
,
815 sizeof(u32
), jit_fill_hole
);
816 if (header
== NULL
) {
821 /* 2. Now, the actual pass. */
823 ctx
.image
= (u32
*)image_ptr
;
826 build_prologue(&ctx
);
828 if (build_body(&ctx
)) {
829 bpf_jit_binary_free(header
);
834 build_epilogue(&ctx
);
836 /* 3. Extra pass to validate JITed code. */
837 if (validate_code(&ctx
)) {
838 bpf_jit_binary_free(header
);
843 /* And we're done. */
844 if (bpf_jit_enable
> 1)
845 bpf_jit_dump(prog
->len
, image_size
, 2, ctx
.image
);
847 bpf_flush_icache(header
, ctx
.image
+ ctx
.idx
);
849 set_memory_ro((unsigned long)header
, header
->pages
);
850 prog
->bpf_func
= (void *)ctx
.image
;
857 bpf_jit_prog_release_other(prog
, prog
== orig_prog
?
862 void bpf_jit_free(struct bpf_prog
*prog
)
864 unsigned long addr
= (unsigned long)prog
->bpf_func
& PAGE_MASK
;
865 struct bpf_binary_header
*header
= (void *)addr
;
870 set_memory_rw(addr
, header
->pages
);
871 bpf_jit_binary_free(header
);
874 bpf_prog_unlock_free(prog
);