]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - arch/powerpc/net/bpf_jit_comp64.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
powerpc64/bpf: Limit 'ldbrx' to processors compliant with ISA v2.06
[mirror_ubuntu-jammy-kernel.git] / arch / powerpc / net / bpf_jit_comp64.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * bpf_jit_comp64.c: eBPF JIT compiler
4 *
5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
6 * IBM Corporation
7 *
8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
9 */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <asm/asm-compat.h>
13 #include <linux/netdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_vlan.h>
16 #include <asm/kprobes.h>
17 #include <linux/bpf.h>
18 #include <asm/security_features.h>
19
20 #include "bpf_jit64.h"
21
22 static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
23 {
24 /*
25 * We only need a stack frame if:
26 * - we call other functions (kernel helpers), or
27 * - the bpf program uses its stack area
28 * The latter condition is deduced from the usage of BPF_REG_FP
29 */
30 return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, b2p[BPF_REG_FP]);
31 }
32
33 /*
34 * When not setting up our own stackframe, the redzone usage is:
35 *
36 * [ prev sp ] <-------------
37 * [ ... ] |
38 * sp (r1) ---> [ stack pointer ] --------------
39 * [ nv gpr save area ] 5*8
40 * [ tail_call_cnt ] 8
41 * [ local_tmp_var ] 16
42 * [ unused red zone ] 208 bytes protected
43 */
44 static int bpf_jit_stack_local(struct codegen_context *ctx)
45 {
46 if (bpf_has_stack_frame(ctx))
47 return STACK_FRAME_MIN_SIZE + ctx->stack_size;
48 else
49 return -(BPF_PPC_STACK_SAVE + 24);
50 }
51
52 static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx)
53 {
54 return bpf_jit_stack_local(ctx) + 16;
55 }
56
57 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
58 {
59 if (reg >= BPF_PPC_NVR_MIN && reg < 32)
60 return (bpf_has_stack_frame(ctx) ?
61 (BPF_PPC_STACKFRAME + ctx->stack_size) : 0)
62 - (8 * (32 - reg));
63
64 pr_err("BPF JIT is asking about unknown registers");
65 BUG();
66 }
67
68 void bpf_jit_realloc_regs(struct codegen_context *ctx)
69 {
70 }
71
72 void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
73 {
74 int i;
75
76 /*
77 * Initialize tail_call_cnt if we do tail calls.
78 * Otherwise, put in NOPs so that it can be skipped when we are
79 * invoked through a tail call.
80 */
81 if (ctx->seen & SEEN_TAILCALL) {
82 EMIT(PPC_RAW_LI(b2p[TMP_REG_1], 0));
83 /* this goes in the redzone */
84 PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8));
85 } else {
86 EMIT(PPC_RAW_NOP());
87 EMIT(PPC_RAW_NOP());
88 }
89
90 #define BPF_TAILCALL_PROLOGUE_SIZE 8
91
92 if (bpf_has_stack_frame(ctx)) {
93 /*
94 * We need a stack frame, but we don't necessarily need to
95 * save/restore LR unless we call other functions
96 */
97 if (ctx->seen & SEEN_FUNC) {
98 EMIT(PPC_RAW_MFLR(_R0));
99 PPC_BPF_STL(0, 1, PPC_LR_STKOFF);
100 }
101
102 PPC_BPF_STLU(1, 1, -(BPF_PPC_STACKFRAME + ctx->stack_size));
103 }
104
105 /*
106 * Back up non-volatile regs -- BPF registers 6-10
107 * If we haven't created our own stack frame, we save these
108 * in the protected zone below the previous stack frame
109 */
110 for (i = BPF_REG_6; i <= BPF_REG_10; i++)
111 if (bpf_is_seen_register(ctx, b2p[i]))
112 PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
113
114 /* Setup frame pointer to point to the bpf stack area */
115 if (bpf_is_seen_register(ctx, b2p[BPF_REG_FP]))
116 EMIT(PPC_RAW_ADDI(b2p[BPF_REG_FP], 1,
117 STACK_FRAME_MIN_SIZE + ctx->stack_size));
118 }
119
120 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
121 {
122 int i;
123
124 /* Restore NVRs */
125 for (i = BPF_REG_6; i <= BPF_REG_10; i++)
126 if (bpf_is_seen_register(ctx, b2p[i]))
127 PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
128
129 /* Tear down our stack frame */
130 if (bpf_has_stack_frame(ctx)) {
131 EMIT(PPC_RAW_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size));
132 if (ctx->seen & SEEN_FUNC) {
133 PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
134 EMIT(PPC_RAW_MTLR(0));
135 }
136 }
137 }
138
139 void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
140 {
141 bpf_jit_emit_common_epilogue(image, ctx);
142
143 /* Move result to r3 */
144 EMIT(PPC_RAW_MR(3, b2p[BPF_REG_0]));
145
146 EMIT(PPC_RAW_BLR());
147 }
148
149 static void bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx,
150 u64 func)
151 {
152 #ifdef PPC64_ELF_ABI_v1
153 /* func points to the function descriptor */
154 PPC_LI64(b2p[TMP_REG_2], func);
155 /* Load actual entry point from function descriptor */
156 PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
157 /* ... and move it to CTR */
158 EMIT(PPC_RAW_MTCTR(b2p[TMP_REG_1]));
159 /*
160 * Load TOC from function descriptor at offset 8.
161 * We can clobber r2 since we get called through a
162 * function pointer (so caller will save/restore r2)
163 * and since we don't use a TOC ourself.
164 */
165 PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
166 #else
167 /* We can clobber r12 */
168 PPC_FUNC_ADDR(12, func);
169 EMIT(PPC_RAW_MTCTR(12));
170 #endif
171 EMIT(PPC_RAW_BCTRL());
172 }
173
174 void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
175 {
176 unsigned int i, ctx_idx = ctx->idx;
177
178 /* Load function address into r12 */
179 PPC_LI64(12, func);
180
181 /* For bpf-to-bpf function calls, the callee's address is unknown
182 * until the last extra pass. As seen above, we use PPC_LI64() to
183 * load the callee's address, but this may optimize the number of
184 * instructions required based on the nature of the address.
185 *
186 * Since we don't want the number of instructions emitted to change,
187 * we pad the optimized PPC_LI64() call with NOPs to guarantee that
188 * we always have a five-instruction sequence, which is the maximum
189 * that PPC_LI64() can emit.
190 */
191 for (i = ctx->idx - ctx_idx; i < 5; i++)
192 EMIT(PPC_RAW_NOP());
193
194 #ifdef PPC64_ELF_ABI_v1
195 /*
196 * Load TOC from function descriptor at offset 8.
197 * We can clobber r2 since we get called through a
198 * function pointer (so caller will save/restore r2)
199 * and since we don't use a TOC ourself.
200 */
201 PPC_BPF_LL(2, 12, 8);
202 /* Load actual entry point from function descriptor */
203 PPC_BPF_LL(12, 12, 0);
204 #endif
205
206 EMIT(PPC_RAW_MTCTR(12));
207 EMIT(PPC_RAW_BCTRL());
208 }
209
210 static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
211 {
212 /*
213 * By now, the eBPF program has already setup parameters in r3, r4 and r5
214 * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
215 * r4/BPF_REG_2 - pointer to bpf_array
216 * r5/BPF_REG_3 - index in bpf_array
217 */
218 int b2p_bpf_array = b2p[BPF_REG_2];
219 int b2p_index = b2p[BPF_REG_3];
220
221 /*
222 * if (index >= array->map.max_entries)
223 * goto out;
224 */
225 EMIT(PPC_RAW_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
226 EMIT(PPC_RAW_RLWINM(b2p_index, b2p_index, 0, 0, 31));
227 EMIT(PPC_RAW_CMPLW(b2p_index, b2p[TMP_REG_1]));
228 PPC_BCC(COND_GE, out);
229
230 /*
231 * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
232 * goto out;
233 */
234 PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
235 EMIT(PPC_RAW_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT));
236 PPC_BCC(COND_GT, out);
237
238 /*
239 * tail_call_cnt++;
240 */
241 EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1));
242 PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
243
244 /* prog = array->ptrs[index]; */
245 EMIT(PPC_RAW_MULI(b2p[TMP_REG_1], b2p_index, 8));
246 EMIT(PPC_RAW_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array));
247 PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
248
249 /*
250 * if (prog == NULL)
251 * goto out;
252 */
253 EMIT(PPC_RAW_CMPLDI(b2p[TMP_REG_1], 0));
254 PPC_BCC(COND_EQ, out);
255
256 /* goto *(prog->bpf_func + prologue_size); */
257 PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
258 #ifdef PPC64_ELF_ABI_v1
259 /* skip past the function descriptor */
260 EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
261 FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE));
262 #else
263 EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE));
264 #endif
265 EMIT(PPC_RAW_MTCTR(b2p[TMP_REG_1]));
266
267 /* tear down stack, restore NVRs, ... */
268 bpf_jit_emit_common_epilogue(image, ctx);
269
270 EMIT(PPC_RAW_BCTR());
271
272 /* out: */
273 return 0;
274 }
275
276 /*
277 * We spill into the redzone always, even if the bpf program has its own stackframe.
278 * Offsets hardcoded based on BPF_PPC_STACK_SAVE -- see bpf_jit_stack_local()
279 */
280 void bpf_stf_barrier(void);
281
282 asm (
283 " .global bpf_stf_barrier ;"
284 " bpf_stf_barrier: ;"
285 " std 21,-64(1) ;"
286 " std 22,-56(1) ;"
287 " sync ;"
288 " ld 21,-64(1) ;"
289 " ld 22,-56(1) ;"
290 " ori 31,31,0 ;"
291 " .rept 14 ;"
292 " b 1f ;"
293 " 1: ;"
294 " .endr ;"
295 " blr ;"
296 );
297
298 /* Assemble the body code between the prologue & epilogue */
299 int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
300 u32 *addrs, bool extra_pass)
301 {
302 enum stf_barrier_type stf_barrier = stf_barrier_type_get();
303 const struct bpf_insn *insn = fp->insnsi;
304 int flen = fp->len;
305 int i, ret;
306
307 /* Start of epilogue code - will only be valid 2nd pass onwards */
308 u32 exit_addr = addrs[flen];
309
310 for (i = 0; i < flen; i++) {
311 u32 code = insn[i].code;
312 u32 dst_reg = b2p[insn[i].dst_reg];
313 u32 src_reg = b2p[insn[i].src_reg];
314 s16 off = insn[i].off;
315 s32 imm = insn[i].imm;
316 bool func_addr_fixed;
317 u64 func_addr;
318 u64 imm64;
319 u32 true_cond;
320 u32 tmp_idx;
321 int j;
322
323 /*
324 * addrs[] maps a BPF bytecode address into a real offset from
325 * the start of the body code.
326 */
327 addrs[i] = ctx->idx * 4;
328
329 /*
330 * As an optimization, we note down which non-volatile registers
331 * are used so that we can only save/restore those in our
332 * prologue and epilogue. We do this here regardless of whether
333 * the actual BPF instruction uses src/dst registers or not
334 * (for instance, BPF_CALL does not use them). The expectation
335 * is that those instructions will have src_reg/dst_reg set to
336 * 0. Even otherwise, we just lose some prologue/epilogue
337 * optimization but everything else should work without
338 * any issues.
339 */
340 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
341 bpf_set_seen_register(ctx, dst_reg);
342 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
343 bpf_set_seen_register(ctx, src_reg);
344
345 switch (code) {
346 /*
347 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
348 */
349 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
350 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
351 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
352 goto bpf_alu32_trunc;
353 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
354 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
355 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
356 goto bpf_alu32_trunc;
357 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
358 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
359 if (!imm) {
360 goto bpf_alu32_trunc;
361 } else if (imm >= -32768 && imm < 32768) {
362 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
363 } else {
364 PPC_LI32(b2p[TMP_REG_1], imm);
365 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]));
366 }
367 goto bpf_alu32_trunc;
368 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
369 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
370 if (!imm) {
371 goto bpf_alu32_trunc;
372 } else if (imm > -32768 && imm <= 32768) {
373 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(-imm)));
374 } else {
375 PPC_LI32(b2p[TMP_REG_1], imm);
376 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
377 }
378 goto bpf_alu32_trunc;
379 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
380 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
381 if (BPF_CLASS(code) == BPF_ALU)
382 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
383 else
384 EMIT(PPC_RAW_MULD(dst_reg, dst_reg, src_reg));
385 goto bpf_alu32_trunc;
386 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
387 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
388 if (imm >= -32768 && imm < 32768)
389 EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm)));
390 else {
391 PPC_LI32(b2p[TMP_REG_1], imm);
392 if (BPF_CLASS(code) == BPF_ALU)
393 EMIT(PPC_RAW_MULW(dst_reg, dst_reg,
394 b2p[TMP_REG_1]));
395 else
396 EMIT(PPC_RAW_MULD(dst_reg, dst_reg,
397 b2p[TMP_REG_1]));
398 }
399 goto bpf_alu32_trunc;
400 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
401 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
402 if (BPF_OP(code) == BPF_MOD) {
403 EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg));
404 EMIT(PPC_RAW_MULW(b2p[TMP_REG_1], src_reg,
405 b2p[TMP_REG_1]));
406 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
407 } else
408 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
409 goto bpf_alu32_trunc;
410 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
411 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
412 if (BPF_OP(code) == BPF_MOD) {
413 EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_1], dst_reg, src_reg));
414 EMIT(PPC_RAW_MULD(b2p[TMP_REG_1], src_reg,
415 b2p[TMP_REG_1]));
416 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
417 } else
418 EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg));
419 break;
420 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
421 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
422 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
423 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
424 if (imm == 0)
425 return -EINVAL;
426 if (imm == 1) {
427 if (BPF_OP(code) == BPF_DIV) {
428 goto bpf_alu32_trunc;
429 } else {
430 EMIT(PPC_RAW_LI(dst_reg, 0));
431 break;
432 }
433 }
434
435 PPC_LI32(b2p[TMP_REG_1], imm);
436 switch (BPF_CLASS(code)) {
437 case BPF_ALU:
438 if (BPF_OP(code) == BPF_MOD) {
439 EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_2],
440 dst_reg,
441 b2p[TMP_REG_1]));
442 EMIT(PPC_RAW_MULW(b2p[TMP_REG_1],
443 b2p[TMP_REG_1],
444 b2p[TMP_REG_2]));
445 EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
446 b2p[TMP_REG_1]));
447 } else
448 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg,
449 b2p[TMP_REG_1]));
450 break;
451 case BPF_ALU64:
452 if (BPF_OP(code) == BPF_MOD) {
453 EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_2],
454 dst_reg,
455 b2p[TMP_REG_1]));
456 EMIT(PPC_RAW_MULD(b2p[TMP_REG_1],
457 b2p[TMP_REG_1],
458 b2p[TMP_REG_2]));
459 EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
460 b2p[TMP_REG_1]));
461 } else
462 EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg,
463 b2p[TMP_REG_1]));
464 break;
465 }
466 goto bpf_alu32_trunc;
467 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
468 case BPF_ALU64 | BPF_NEG: /* dst = -dst */
469 EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
470 goto bpf_alu32_trunc;
471
472 /*
473 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
474 */
475 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
476 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
477 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
478 goto bpf_alu32_trunc;
479 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
480 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
481 if (!IMM_H(imm))
482 EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
483 else {
484 /* Sign-extended */
485 PPC_LI32(b2p[TMP_REG_1], imm);
486 EMIT(PPC_RAW_AND(dst_reg, dst_reg, b2p[TMP_REG_1]));
487 }
488 goto bpf_alu32_trunc;
489 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
490 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
491 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
492 goto bpf_alu32_trunc;
493 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
494 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
495 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
496 /* Sign-extended */
497 PPC_LI32(b2p[TMP_REG_1], imm);
498 EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_1]));
499 } else {
500 if (IMM_L(imm))
501 EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
502 if (IMM_H(imm))
503 EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
504 }
505 goto bpf_alu32_trunc;
506 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
507 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
508 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
509 goto bpf_alu32_trunc;
510 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
511 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
512 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
513 /* Sign-extended */
514 PPC_LI32(b2p[TMP_REG_1], imm);
515 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]));
516 } else {
517 if (IMM_L(imm))
518 EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
519 if (IMM_H(imm))
520 EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
521 }
522 goto bpf_alu32_trunc;
523 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
524 /* slw clears top 32 bits */
525 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
526 /* skip zero extension move, but set address map. */
527 if (insn_is_zext(&insn[i + 1]))
528 addrs[++i] = ctx->idx * 4;
529 break;
530 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
531 EMIT(PPC_RAW_SLD(dst_reg, dst_reg, src_reg));
532 break;
533 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
534 /* with imm 0, we still need to clear top 32 bits */
535 EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
536 if (insn_is_zext(&insn[i + 1]))
537 addrs[++i] = ctx->idx * 4;
538 break;
539 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
540 if (imm != 0)
541 EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, imm));
542 break;
543 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
544 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
545 if (insn_is_zext(&insn[i + 1]))
546 addrs[++i] = ctx->idx * 4;
547 break;
548 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
549 EMIT(PPC_RAW_SRD(dst_reg, dst_reg, src_reg));
550 break;
551 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
552 EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
553 if (insn_is_zext(&insn[i + 1]))
554 addrs[++i] = ctx->idx * 4;
555 break;
556 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
557 if (imm != 0)
558 EMIT(PPC_RAW_SRDI(dst_reg, dst_reg, imm));
559 break;
560 case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
561 EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
562 goto bpf_alu32_trunc;
563 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
564 EMIT(PPC_RAW_SRAD(dst_reg, dst_reg, src_reg));
565 break;
566 case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
567 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
568 goto bpf_alu32_trunc;
569 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
570 if (imm != 0)
571 EMIT(PPC_RAW_SRADI(dst_reg, dst_reg, imm));
572 break;
573
574 /*
575 * MOV
576 */
577 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
578 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
579 if (imm == 1) {
580 /* special mov32 for zext */
581 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
582 break;
583 }
584 EMIT(PPC_RAW_MR(dst_reg, src_reg));
585 goto bpf_alu32_trunc;
586 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
587 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
588 PPC_LI32(dst_reg, imm);
589 if (imm < 0)
590 goto bpf_alu32_trunc;
591 else if (insn_is_zext(&insn[i + 1]))
592 addrs[++i] = ctx->idx * 4;
593 break;
594
595 bpf_alu32_trunc:
596 /* Truncate to 32-bits */
597 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext)
598 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
599 break;
600
601 /*
602 * BPF_FROM_BE/LE
603 */
604 case BPF_ALU | BPF_END | BPF_FROM_LE:
605 case BPF_ALU | BPF_END | BPF_FROM_BE:
606 #ifdef __BIG_ENDIAN__
607 if (BPF_SRC(code) == BPF_FROM_BE)
608 goto emit_clear;
609 #else /* !__BIG_ENDIAN__ */
610 if (BPF_SRC(code) == BPF_FROM_LE)
611 goto emit_clear;
612 #endif
613 switch (imm) {
614 case 16:
615 /* Rotate 8 bits left & mask with 0x0000ff00 */
616 EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23));
617 /* Rotate 8 bits right & insert LSB to reg */
618 EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31));
619 /* Move result back to dst_reg */
620 EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
621 break;
622 case 32:
623 /*
624 * Rotate word left by 8 bits:
625 * 2 bytes are already in their final position
626 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
627 */
628 EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31));
629 /* Rotate 24 bits and insert byte 1 */
630 EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7));
631 /* Rotate 24 bits and insert byte 3 */
632 EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23));
633 EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
634 break;
635 case 64:
636 /* Store the value to stack and then use byte-reverse loads */
637 PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx));
638 EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)));
639 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
640 EMIT(PPC_RAW_LDBRX(dst_reg, 0, b2p[TMP_REG_1]));
641 } else {
642 EMIT(PPC_RAW_LWBRX(dst_reg, 0, b2p[TMP_REG_1]));
643 if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
644 EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, 32));
645 EMIT(PPC_RAW_LI(b2p[TMP_REG_2], 4));
646 EMIT(PPC_RAW_LWBRX(b2p[TMP_REG_2], b2p[TMP_REG_2], b2p[TMP_REG_1]));
647 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
648 EMIT(PPC_RAW_SLDI(b2p[TMP_REG_2], b2p[TMP_REG_2], 32));
649 EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_2]));
650 }
651 break;
652 }
653 break;
654
655 emit_clear:
656 switch (imm) {
657 case 16:
658 /* zero-extend 16 bits into 64 bits */
659 EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 48));
660 if (insn_is_zext(&insn[i + 1]))
661 addrs[++i] = ctx->idx * 4;
662 break;
663 case 32:
664 if (!fp->aux->verifier_zext)
665 /* zero-extend 32 bits into 64 bits */
666 EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 32));
667 break;
668 case 64:
669 /* nop */
670 break;
671 }
672 break;
673
674 /*
675 * BPF_ST NOSPEC (speculation barrier)
676 */
677 case BPF_ST | BPF_NOSPEC:
678 if (!security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) ||
679 !security_ftr_enabled(SEC_FTR_STF_BARRIER))
680 break;
681
682 switch (stf_barrier) {
683 case STF_BARRIER_EIEIO:
684 EMIT(PPC_RAW_EIEIO() | 0x02000000);
685 break;
686 case STF_BARRIER_SYNC_ORI:
687 EMIT(PPC_RAW_SYNC());
688 EMIT(PPC_RAW_LD(b2p[TMP_REG_1], _R13, 0));
689 EMIT(PPC_RAW_ORI(_R31, _R31, 0));
690 break;
691 case STF_BARRIER_FALLBACK:
692 EMIT(PPC_RAW_MFLR(b2p[TMP_REG_1]));
693 PPC_LI64(12, dereference_kernel_function_descriptor(bpf_stf_barrier));
694 EMIT(PPC_RAW_MTCTR(12));
695 EMIT(PPC_RAW_BCTRL());
696 EMIT(PPC_RAW_MTLR(b2p[TMP_REG_1]));
697 break;
698 case STF_BARRIER_NONE:
699 break;
700 }
701 break;
702
703 /*
704 * BPF_ST(X)
705 */
706 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
707 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
708 if (BPF_CLASS(code) == BPF_ST) {
709 EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
710 src_reg = b2p[TMP_REG_1];
711 }
712 EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
713 break;
714 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
715 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
716 if (BPF_CLASS(code) == BPF_ST) {
717 EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
718 src_reg = b2p[TMP_REG_1];
719 }
720 EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
721 break;
722 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
723 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
724 if (BPF_CLASS(code) == BPF_ST) {
725 PPC_LI32(b2p[TMP_REG_1], imm);
726 src_reg = b2p[TMP_REG_1];
727 }
728 EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
729 break;
730 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
731 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
732 if (BPF_CLASS(code) == BPF_ST) {
733 PPC_LI32(b2p[TMP_REG_1], imm);
734 src_reg = b2p[TMP_REG_1];
735 }
736 PPC_BPF_STL(src_reg, dst_reg, off);
737 break;
738
739 /*
740 * BPF_STX ATOMIC (atomic ops)
741 */
742 case BPF_STX | BPF_ATOMIC | BPF_W:
743 if (imm != BPF_ADD) {
744 pr_err_ratelimited(
745 "eBPF filter atomic op code %02x (@%d) unsupported\n",
746 code, i);
747 return -ENOTSUPP;
748 }
749
750 /* *(u32 *)(dst + off) += src */
751
752 /* Get EA into TMP_REG_1 */
753 EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
754 tmp_idx = ctx->idx * 4;
755 /* load value from memory into TMP_REG_2 */
756 EMIT(PPC_RAW_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
757 /* add value from src_reg into this */
758 EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
759 /* store result back */
760 EMIT(PPC_RAW_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
761 /* we're done if this succeeded */
762 PPC_BCC_SHORT(COND_NE, tmp_idx);
763 break;
764 case BPF_STX | BPF_ATOMIC | BPF_DW:
765 if (imm != BPF_ADD) {
766 pr_err_ratelimited(
767 "eBPF filter atomic op code %02x (@%d) unsupported\n",
768 code, i);
769 return -ENOTSUPP;
770 }
771 /* *(u64 *)(dst + off) += src */
772
773 EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
774 tmp_idx = ctx->idx * 4;
775 EMIT(PPC_RAW_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
776 EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
777 EMIT(PPC_RAW_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
778 PPC_BCC_SHORT(COND_NE, tmp_idx);
779 break;
780
781 /*
782 * BPF_LDX
783 */
784 /* dst = *(u8 *)(ul) (src + off) */
785 case BPF_LDX | BPF_MEM | BPF_B:
786 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
787 if (insn_is_zext(&insn[i + 1]))
788 addrs[++i] = ctx->idx * 4;
789 break;
790 /* dst = *(u16 *)(ul) (src + off) */
791 case BPF_LDX | BPF_MEM | BPF_H:
792 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
793 if (insn_is_zext(&insn[i + 1]))
794 addrs[++i] = ctx->idx * 4;
795 break;
796 /* dst = *(u32 *)(ul) (src + off) */
797 case BPF_LDX | BPF_MEM | BPF_W:
798 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
799 if (insn_is_zext(&insn[i + 1]))
800 addrs[++i] = ctx->idx * 4;
801 break;
802 /* dst = *(u64 *)(ul) (src + off) */
803 case BPF_LDX | BPF_MEM | BPF_DW:
804 PPC_BPF_LL(dst_reg, src_reg, off);
805 break;
806
807 /*
808 * Doubleword load
809 * 16 byte instruction that uses two 'struct bpf_insn'
810 */
811 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
812 imm64 = ((u64)(u32) insn[i].imm) |
813 (((u64)(u32) insn[i+1].imm) << 32);
814 tmp_idx = ctx->idx;
815 PPC_LI64(dst_reg, imm64);
816 /* padding to allow full 5 instructions for later patching */
817 for (j = ctx->idx - tmp_idx; j < 5; j++)
818 EMIT(PPC_RAW_NOP());
819 /* Adjust for two bpf instructions */
820 addrs[++i] = ctx->idx * 4;
821 break;
822
823 /*
824 * Return/Exit
825 */
826 case BPF_JMP | BPF_EXIT:
827 /*
828 * If this isn't the very last instruction, branch to
829 * the epilogue. If we _are_ the last instruction,
830 * we'll just fall through to the epilogue.
831 */
832 if (i != flen - 1)
833 PPC_JMP(exit_addr);
834 /* else fall through to the epilogue */
835 break;
836
837 /*
838 * Call kernel helper or bpf function
839 */
840 case BPF_JMP | BPF_CALL:
841 ctx->seen |= SEEN_FUNC;
842
843 ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass,
844 &func_addr, &func_addr_fixed);
845 if (ret < 0)
846 return ret;
847
848 if (func_addr_fixed)
849 bpf_jit_emit_func_call_hlp(image, ctx, func_addr);
850 else
851 bpf_jit_emit_func_call_rel(image, ctx, func_addr);
852 /* move return value from r3 to BPF_REG_0 */
853 EMIT(PPC_RAW_MR(b2p[BPF_REG_0], 3));
854 break;
855
856 /*
857 * Jumps and branches
858 */
859 case BPF_JMP | BPF_JA:
860 PPC_JMP(addrs[i + 1 + off]);
861 break;
862
863 case BPF_JMP | BPF_JGT | BPF_K:
864 case BPF_JMP | BPF_JGT | BPF_X:
865 case BPF_JMP | BPF_JSGT | BPF_K:
866 case BPF_JMP | BPF_JSGT | BPF_X:
867 case BPF_JMP32 | BPF_JGT | BPF_K:
868 case BPF_JMP32 | BPF_JGT | BPF_X:
869 case BPF_JMP32 | BPF_JSGT | BPF_K:
870 case BPF_JMP32 | BPF_JSGT | BPF_X:
871 true_cond = COND_GT;
872 goto cond_branch;
873 case BPF_JMP | BPF_JLT | BPF_K:
874 case BPF_JMP | BPF_JLT | BPF_X:
875 case BPF_JMP | BPF_JSLT | BPF_K:
876 case BPF_JMP | BPF_JSLT | BPF_X:
877 case BPF_JMP32 | BPF_JLT | BPF_K:
878 case BPF_JMP32 | BPF_JLT | BPF_X:
879 case BPF_JMP32 | BPF_JSLT | BPF_K:
880 case BPF_JMP32 | BPF_JSLT | BPF_X:
881 true_cond = COND_LT;
882 goto cond_branch;
883 case BPF_JMP | BPF_JGE | BPF_K:
884 case BPF_JMP | BPF_JGE | BPF_X:
885 case BPF_JMP | BPF_JSGE | BPF_K:
886 case BPF_JMP | BPF_JSGE | BPF_X:
887 case BPF_JMP32 | BPF_JGE | BPF_K:
888 case BPF_JMP32 | BPF_JGE | BPF_X:
889 case BPF_JMP32 | BPF_JSGE | BPF_K:
890 case BPF_JMP32 | BPF_JSGE | BPF_X:
891 true_cond = COND_GE;
892 goto cond_branch;
893 case BPF_JMP | BPF_JLE | BPF_K:
894 case BPF_JMP | BPF_JLE | BPF_X:
895 case BPF_JMP | BPF_JSLE | BPF_K:
896 case BPF_JMP | BPF_JSLE | BPF_X:
897 case BPF_JMP32 | BPF_JLE | BPF_K:
898 case BPF_JMP32 | BPF_JLE | BPF_X:
899 case BPF_JMP32 | BPF_JSLE | BPF_K:
900 case BPF_JMP32 | BPF_JSLE | BPF_X:
901 true_cond = COND_LE;
902 goto cond_branch;
903 case BPF_JMP | BPF_JEQ | BPF_K:
904 case BPF_JMP | BPF_JEQ | BPF_X:
905 case BPF_JMP32 | BPF_JEQ | BPF_K:
906 case BPF_JMP32 | BPF_JEQ | BPF_X:
907 true_cond = COND_EQ;
908 goto cond_branch;
909 case BPF_JMP | BPF_JNE | BPF_K:
910 case BPF_JMP | BPF_JNE | BPF_X:
911 case BPF_JMP32 | BPF_JNE | BPF_K:
912 case BPF_JMP32 | BPF_JNE | BPF_X:
913 true_cond = COND_NE;
914 goto cond_branch;
915 case BPF_JMP | BPF_JSET | BPF_K:
916 case BPF_JMP | BPF_JSET | BPF_X:
917 case BPF_JMP32 | BPF_JSET | BPF_K:
918 case BPF_JMP32 | BPF_JSET | BPF_X:
919 true_cond = COND_NE;
920 /* Fall through */
921
922 cond_branch:
923 switch (code) {
924 case BPF_JMP | BPF_JGT | BPF_X:
925 case BPF_JMP | BPF_JLT | BPF_X:
926 case BPF_JMP | BPF_JGE | BPF_X:
927 case BPF_JMP | BPF_JLE | BPF_X:
928 case BPF_JMP | BPF_JEQ | BPF_X:
929 case BPF_JMP | BPF_JNE | BPF_X:
930 case BPF_JMP32 | BPF_JGT | BPF_X:
931 case BPF_JMP32 | BPF_JLT | BPF_X:
932 case BPF_JMP32 | BPF_JGE | BPF_X:
933 case BPF_JMP32 | BPF_JLE | BPF_X:
934 case BPF_JMP32 | BPF_JEQ | BPF_X:
935 case BPF_JMP32 | BPF_JNE | BPF_X:
936 /* unsigned comparison */
937 if (BPF_CLASS(code) == BPF_JMP32)
938 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
939 else
940 EMIT(PPC_RAW_CMPLD(dst_reg, src_reg));
941 break;
942 case BPF_JMP | BPF_JSGT | BPF_X:
943 case BPF_JMP | BPF_JSLT | BPF_X:
944 case BPF_JMP | BPF_JSGE | BPF_X:
945 case BPF_JMP | BPF_JSLE | BPF_X:
946 case BPF_JMP32 | BPF_JSGT | BPF_X:
947 case BPF_JMP32 | BPF_JSLT | BPF_X:
948 case BPF_JMP32 | BPF_JSGE | BPF_X:
949 case BPF_JMP32 | BPF_JSLE | BPF_X:
950 /* signed comparison */
951 if (BPF_CLASS(code) == BPF_JMP32)
952 EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
953 else
954 EMIT(PPC_RAW_CMPD(dst_reg, src_reg));
955 break;
956 case BPF_JMP | BPF_JSET | BPF_X:
957 case BPF_JMP32 | BPF_JSET | BPF_X:
958 if (BPF_CLASS(code) == BPF_JMP) {
959 EMIT(PPC_RAW_AND_DOT(b2p[TMP_REG_1], dst_reg,
960 src_reg));
961 } else {
962 int tmp_reg = b2p[TMP_REG_1];
963
964 EMIT(PPC_RAW_AND(tmp_reg, dst_reg, src_reg));
965 EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0,
966 31));
967 }
968 break;
969 case BPF_JMP | BPF_JNE | BPF_K:
970 case BPF_JMP | BPF_JEQ | BPF_K:
971 case BPF_JMP | BPF_JGT | BPF_K:
972 case BPF_JMP | BPF_JLT | BPF_K:
973 case BPF_JMP | BPF_JGE | BPF_K:
974 case BPF_JMP | BPF_JLE | BPF_K:
975 case BPF_JMP32 | BPF_JNE | BPF_K:
976 case BPF_JMP32 | BPF_JEQ | BPF_K:
977 case BPF_JMP32 | BPF_JGT | BPF_K:
978 case BPF_JMP32 | BPF_JLT | BPF_K:
979 case BPF_JMP32 | BPF_JGE | BPF_K:
980 case BPF_JMP32 | BPF_JLE | BPF_K:
981 {
982 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
983
984 /*
985 * Need sign-extended load, so only positive
986 * values can be used as imm in cmpldi
987 */
988 if (imm >= 0 && imm < 32768) {
989 if (is_jmp32)
990 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
991 else
992 EMIT(PPC_RAW_CMPLDI(dst_reg, imm));
993 } else {
994 /* sign-extending load */
995 PPC_LI32(b2p[TMP_REG_1], imm);
996 /* ... but unsigned comparison */
997 if (is_jmp32)
998 EMIT(PPC_RAW_CMPLW(dst_reg,
999 b2p[TMP_REG_1]));
1000 else
1001 EMIT(PPC_RAW_CMPLD(dst_reg,
1002 b2p[TMP_REG_1]));
1003 }
1004 break;
1005 }
1006 case BPF_JMP | BPF_JSGT | BPF_K:
1007 case BPF_JMP | BPF_JSLT | BPF_K:
1008 case BPF_JMP | BPF_JSGE | BPF_K:
1009 case BPF_JMP | BPF_JSLE | BPF_K:
1010 case BPF_JMP32 | BPF_JSGT | BPF_K:
1011 case BPF_JMP32 | BPF_JSLT | BPF_K:
1012 case BPF_JMP32 | BPF_JSGE | BPF_K:
1013 case BPF_JMP32 | BPF_JSLE | BPF_K:
1014 {
1015 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
1016
1017 /*
1018 * signed comparison, so any 16-bit value
1019 * can be used in cmpdi
1020 */
1021 if (imm >= -32768 && imm < 32768) {
1022 if (is_jmp32)
1023 EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1024 else
1025 EMIT(PPC_RAW_CMPDI(dst_reg, imm));
1026 } else {
1027 PPC_LI32(b2p[TMP_REG_1], imm);
1028 if (is_jmp32)
1029 EMIT(PPC_RAW_CMPW(dst_reg,
1030 b2p[TMP_REG_1]));
1031 else
1032 EMIT(PPC_RAW_CMPD(dst_reg,
1033 b2p[TMP_REG_1]));
1034 }
1035 break;
1036 }
1037 case BPF_JMP | BPF_JSET | BPF_K:
1038 case BPF_JMP32 | BPF_JSET | BPF_K:
1039 /* andi does not sign-extend the immediate */
1040 if (imm >= 0 && imm < 32768)
1041 /* PPC_ANDI is _only/always_ dot-form */
1042 EMIT(PPC_RAW_ANDI(b2p[TMP_REG_1], dst_reg, imm));
1043 else {
1044 int tmp_reg = b2p[TMP_REG_1];
1045
1046 PPC_LI32(tmp_reg, imm);
1047 if (BPF_CLASS(code) == BPF_JMP) {
1048 EMIT(PPC_RAW_AND_DOT(tmp_reg, dst_reg,
1049 tmp_reg));
1050 } else {
1051 EMIT(PPC_RAW_AND(tmp_reg, dst_reg,
1052 tmp_reg));
1053 EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg,
1054 0, 0, 31));
1055 }
1056 }
1057 break;
1058 }
1059 PPC_BCC(true_cond, addrs[i + 1 + off]);
1060 break;
1061
1062 /*
1063 * Tail call
1064 */
1065 case BPF_JMP | BPF_TAIL_CALL:
1066 ctx->seen |= SEEN_TAILCALL;
1067 ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1068 if (ret < 0)
1069 return ret;
1070 break;
1071
1072 default:
1073 /*
1074 * The filter contains something cruel & unusual.
1075 * We don't handle it, but also there shouldn't be
1076 * anything missing from our list.
1077 */
1078 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n",
1079 code, i);
1080 return -ENOTSUPP;
1081 }
1082 }
1083
1084 /* Set end-of-body-code address for exit. */
1085 addrs[i] = ctx->idx * 4;
1086
1087 return 0;
1088 }
Ubuntu Jammy Linux kernel mirror