]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blob - arch/sparc/net/bpf_jit_comp_64.c
projects / mirror_ubuntu-focal-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Input: wm97xx: add new AC97 bus support
[mirror_ubuntu-focal-kernel.git] / arch / sparc / net / bpf_jit_comp_64.c
1 #include <linux/moduleloader.h>
2 #include <linux/workqueue.h>
3 #include <linux/netdevice.h>
4 #include <linux/filter.h>
5 #include <linux/bpf.h>
6 #include <linux/cache.h>
7 #include <linux/if_vlan.h>
8
9 #include <asm/cacheflush.h>
10 #include <asm/ptrace.h>
11
12 #include "bpf_jit_64.h"
13
14 int bpf_jit_enable __read_mostly;
15
16 static inline bool is_simm13(unsigned int value)
17 {
18 return value + 0x1000 < 0x2000;
19 }
20
21 static inline bool is_simm10(unsigned int value)
22 {
23 return value + 0x200 < 0x400;
24 }
25
26 static inline bool is_simm5(unsigned int value)
27 {
28 return value + 0x10 < 0x20;
29 }
30
31 static inline bool is_sethi(unsigned int value)
32 {
33 return (value & ~0x3fffff) == 0;
34 }
35
36 static void bpf_flush_icache(void *start_, void *end_)
37 {
38 /* Cheetah's I-cache is fully coherent. */
39 if (tlb_type == spitfire) {
40 unsigned long start = (unsigned long) start_;
41 unsigned long end = (unsigned long) end_;
42
43 start &= ~7UL;
44 end = (end + 7UL) & ~7UL;
45 while (start < end) {
46 flushi(start);
47 start += 32;
48 }
49 }
50 }
51
52 #define SEEN_DATAREF 1 /* might call external helpers */
53 #define SEEN_XREG 2 /* ebx is used */
54 #define SEEN_MEM 4 /* use mem[] for temporary storage */
55
56 #define S13(X) ((X) & 0x1fff)
57 #define S5(X) ((X) & 0x1f)
58 #define IMMED 0x00002000
59 #define RD(X) ((X) << 25)
60 #define RS1(X) ((X) << 14)
61 #define RS2(X) ((X))
62 #define OP(X) ((X) << 30)
63 #define OP2(X) ((X) << 22)
64 #define OP3(X) ((X) << 19)
65 #define COND(X) (((X) & 0xf) << 25)
66 #define CBCOND(X) (((X) & 0x1f) << 25)
67 #define F1(X) OP(X)
68 #define F2(X, Y) (OP(X) | OP2(Y))
69 #define F3(X, Y) (OP(X) | OP3(Y))
70 #define ASI(X) (((X) & 0xff) << 5)
71
72 #define CONDN COND(0x0)
73 #define CONDE COND(0x1)
74 #define CONDLE COND(0x2)
75 #define CONDL COND(0x3)
76 #define CONDLEU COND(0x4)
77 #define CONDCS COND(0x5)
78 #define CONDNEG COND(0x6)
79 #define CONDVC COND(0x7)
80 #define CONDA COND(0x8)
81 #define CONDNE COND(0x9)
82 #define CONDG COND(0xa)
83 #define CONDGE COND(0xb)
84 #define CONDGU COND(0xc)
85 #define CONDCC COND(0xd)
86 #define CONDPOS COND(0xe)
87 #define CONDVS COND(0xf)
88
89 #define CONDGEU CONDCC
90 #define CONDLU CONDCS
91
92 #define WDISP22(X) (((X) >> 2) & 0x3fffff)
93 #define WDISP19(X) (((X) >> 2) & 0x7ffff)
94
95 /* The 10-bit branch displacement for CBCOND is split into two fields */
96 static u32 WDISP10(u32 off)
97 {
98 u32 ret = ((off >> 2) & 0xff) << 5;
99
100 ret |= ((off >> (2 + 8)) & 0x03) << 19;
101
102 return ret;
103 }
104
105 #define CBCONDE CBCOND(0x09)
106 #define CBCONDLE CBCOND(0x0a)
107 #define CBCONDL CBCOND(0x0b)
108 #define CBCONDLEU CBCOND(0x0c)
109 #define CBCONDCS CBCOND(0x0d)
110 #define CBCONDN CBCOND(0x0e)
111 #define CBCONDVS CBCOND(0x0f)
112 #define CBCONDNE CBCOND(0x19)
113 #define CBCONDG CBCOND(0x1a)
114 #define CBCONDGE CBCOND(0x1b)
115 #define CBCONDGU CBCOND(0x1c)
116 #define CBCONDCC CBCOND(0x1d)
117 #define CBCONDPOS CBCOND(0x1e)
118 #define CBCONDVC CBCOND(0x1f)
119
120 #define CBCONDGEU CBCONDCC
121 #define CBCONDLU CBCONDCS
122
123 #define ANNUL (1 << 29)
124 #define XCC (1 << 21)
125
126 #define BRANCH (F2(0, 1) | XCC)
127 #define CBCOND_OP (F2(0, 3) | XCC)
128
129 #define BA (BRANCH | CONDA)
130 #define BG (BRANCH | CONDG)
131 #define BL (BRANCH | CONDL)
132 #define BLE (BRANCH | CONDLE)
133 #define BGU (BRANCH | CONDGU)
134 #define BLEU (BRANCH | CONDLEU)
135 #define BGE (BRANCH | CONDGE)
136 #define BGEU (BRANCH | CONDGEU)
137 #define BLU (BRANCH | CONDLU)
138 #define BE (BRANCH | CONDE)
139 #define BNE (BRANCH | CONDNE)
140
141 #define SETHI(K, REG) \
142 (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
143 #define OR_LO(K, REG) \
144 (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
145
146 #define ADD F3(2, 0x00)
147 #define AND F3(2, 0x01)
148 #define ANDCC F3(2, 0x11)
149 #define OR F3(2, 0x02)
150 #define XOR F3(2, 0x03)
151 #define SUB F3(2, 0x04)
152 #define SUBCC F3(2, 0x14)
153 #define MUL F3(2, 0x0a)
154 #define MULX F3(2, 0x09)
155 #define UDIVX F3(2, 0x0d)
156 #define DIV F3(2, 0x0e)
157 #define SLL F3(2, 0x25)
158 #define SLLX (F3(2, 0x25)|(1<<12))
159 #define SRA F3(2, 0x27)
160 #define SRAX (F3(2, 0x27)|(1<<12))
161 #define SRL F3(2, 0x26)
162 #define SRLX (F3(2, 0x26)|(1<<12))
163 #define JMPL F3(2, 0x38)
164 #define SAVE F3(2, 0x3c)
165 #define RESTORE F3(2, 0x3d)
166 #define CALL F1(1)
167 #define BR F2(0, 0x01)
168 #define RD_Y F3(2, 0x28)
169 #define WR_Y F3(2, 0x30)
170
171 #define LD32 F3(3, 0x00)
172 #define LD8 F3(3, 0x01)
173 #define LD16 F3(3, 0x02)
174 #define LD64 F3(3, 0x0b)
175 #define LD64A F3(3, 0x1b)
176 #define ST8 F3(3, 0x05)
177 #define ST16 F3(3, 0x06)
178 #define ST32 F3(3, 0x04)
179 #define ST64 F3(3, 0x0e)
180
181 #define CAS F3(3, 0x3c)
182 #define CASX F3(3, 0x3e)
183
184 #define LDPTR LD64
185 #define BASE_STACKFRAME 176
186
187 #define LD32I (LD32 | IMMED)
188 #define LD8I (LD8 | IMMED)
189 #define LD16I (LD16 | IMMED)
190 #define LD64I (LD64 | IMMED)
191 #define LDPTRI (LDPTR | IMMED)
192 #define ST32I (ST32 | IMMED)
193
194 struct jit_ctx {
195 struct bpf_prog *prog;
196 unsigned int *offset;
197 int idx;
198 int epilogue_offset;
199 bool tmp_1_used;
200 bool tmp_2_used;
201 bool tmp_3_used;
202 bool saw_ld_abs_ind;
203 bool saw_frame_pointer;
204 bool saw_call;
205 bool saw_tail_call;
206 u32 *image;
207 };
208
209 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
210 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
211 #define SKB_HLEN_REG (MAX_BPF_JIT_REG + 2)
212 #define SKB_DATA_REG (MAX_BPF_JIT_REG + 3)
213 #define TMP_REG_3 (MAX_BPF_JIT_REG + 4)
214
215 /* Map BPF registers to SPARC registers */
216 static const int bpf2sparc[] = {
217 /* return value from in-kernel function, and exit value from eBPF */
218 [BPF_REG_0] = O5,
219
220 /* arguments from eBPF program to in-kernel function */
221 [BPF_REG_1] = O0,
222 [BPF_REG_2] = O1,
223 [BPF_REG_3] = O2,
224 [BPF_REG_4] = O3,
225 [BPF_REG_5] = O4,
226
227 /* callee saved registers that in-kernel function will preserve */
228 [BPF_REG_6] = L0,
229 [BPF_REG_7] = L1,
230 [BPF_REG_8] = L2,
231 [BPF_REG_9] = L3,
232
233 /* read-only frame pointer to access stack */
234 [BPF_REG_FP] = L6,
235
236 [BPF_REG_AX] = G7,
237
238 /* temporary register for internal BPF JIT */
239 [TMP_REG_1] = G1,
240 [TMP_REG_2] = G2,
241 [TMP_REG_3] = G3,
242
243 [SKB_HLEN_REG] = L4,
244 [SKB_DATA_REG] = L5,
245 };
246
247 static void emit(const u32 insn, struct jit_ctx *ctx)
248 {
249 if (ctx->image != NULL)
250 ctx->image[ctx->idx] = insn;
251
252 ctx->idx++;
253 }
254
255 static void emit_call(u32 *func, struct jit_ctx *ctx)
256 {
257 if (ctx->image != NULL) {
258 void *here = &ctx->image[ctx->idx];
259 unsigned int off;
260
261 off = (void *)func - here;
262 ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
263 }
264 ctx->idx++;
265 }
266
267 static void emit_nop(struct jit_ctx *ctx)
268 {
269 emit(SETHI(0, G0), ctx);
270 }
271
272 static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
273 {
274 emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
275 }
276
277 /* Emit 32-bit constant, zero extended. */
278 static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
279 {
280 emit(SETHI(K, reg), ctx);
281 emit(OR_LO(K, reg), ctx);
282 }
283
284 /* Emit 32-bit constant, sign extended. */
285 static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
286 {
287 if (K >= 0) {
288 emit(SETHI(K, reg), ctx);
289 emit(OR_LO(K, reg), ctx);
290 } else {
291 u32 hbits = ~(u32) K;
292 u32 lbits = -0x400 | (u32) K;
293
294 emit(SETHI(hbits, reg), ctx);
295 emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
296 }
297 }
298
299 static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
300 {
301 emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
302 }
303
304 static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
305 {
306 emit(opcode | RS1(a) | RS2(b) | RD(c), ctx);
307 }
308
309 static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
310 struct jit_ctx *ctx)
311 {
312 bool small_immed = is_simm13(imm);
313 unsigned int insn = opcode;
314
315 insn |= RS1(dst) | RD(dst);
316 if (small_immed) {
317 emit(insn | IMMED | S13(imm), ctx);
318 } else {
319 unsigned int tmp = bpf2sparc[TMP_REG_1];
320
321 ctx->tmp_1_used = true;
322
323 emit_set_const_sext(imm, tmp, ctx);
324 emit(insn | RS2(tmp), ctx);
325 }
326 }
327
328 static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
329 unsigned int dst, struct jit_ctx *ctx)
330 {
331 bool small_immed = is_simm13(imm);
332 unsigned int insn = opcode;
333
334 insn |= RS1(src) | RD(dst);
335 if (small_immed) {
336 emit(insn | IMMED | S13(imm), ctx);
337 } else {
338 unsigned int tmp = bpf2sparc[TMP_REG_1];
339
340 ctx->tmp_1_used = true;
341
342 emit_set_const_sext(imm, tmp, ctx);
343 emit(insn | RS2(tmp), ctx);
344 }
345 }
346
347 static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
348 {
349 if (K >= 0 && is_simm13(K)) {
350 /* or %g0, K, DEST */
351 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
352 } else {
353 emit_set_const(K, dest, ctx);
354 }
355 }
356
357 static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
358 {
359 if (is_simm13(K)) {
360 /* or %g0, K, DEST */
361 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
362 } else {
363 emit_set_const(K, dest, ctx);
364 }
365 }
366
367 static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
368 {
369 if (is_simm13(K)) {
370 /* or %g0, K, DEST */
371 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
372 } else {
373 emit_set_const_sext(K, dest, ctx);
374 }
375 }
376
377 static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
378 int *hbsp, int *lbsp, int *abbasp)
379 {
380 int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
381 int i;
382
383 lowest_bit_set = highest_bit_set = -1;
384 i = 0;
385 do {
386 if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
387 lowest_bit_set = i;
388 if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
389 highest_bit_set = (64 - i - 1);
390 } while (++i < 32 && (highest_bit_set == -1 ||
391 lowest_bit_set == -1));
392 if (i == 32) {
393 i = 0;
394 do {
395 if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
396 lowest_bit_set = i + 32;
397 if (highest_bit_set == -1 &&
398 ((low_bits >> (32 - i - 1)) & 1))
399 highest_bit_set = 32 - i - 1;
400 } while (++i < 32 && (highest_bit_set == -1 ||
401 lowest_bit_set == -1));
402 }
403
404 all_bits_between_are_set = 1;
405 for (i = lowest_bit_set; i <= highest_bit_set; i++) {
406 if (i < 32) {
407 if ((low_bits & (1 << i)) != 0)
408 continue;
409 } else {
410 if ((high_bits & (1 << (i - 32))) != 0)
411 continue;
412 }
413 all_bits_between_are_set = 0;
414 break;
415 }
416 *hbsp = highest_bit_set;
417 *lbsp = lowest_bit_set;
418 *abbasp = all_bits_between_are_set;
419 }
420
421 static unsigned long create_simple_focus_bits(unsigned long high_bits,
422 unsigned long low_bits,
423 int lowest_bit_set, int shift)
424 {
425 long hi, lo;
426
427 if (lowest_bit_set < 32) {
428 lo = (low_bits >> lowest_bit_set) << shift;
429 hi = ((high_bits << (32 - lowest_bit_set)) << shift);
430 } else {
431 lo = 0;
432 hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
433 }
434 return hi | lo;
435 }
436
437 static bool const64_is_2insns(unsigned long high_bits,
438 unsigned long low_bits)
439 {
440 int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
441
442 if (high_bits == 0 || high_bits == 0xffffffff)
443 return true;
444
445 analyze_64bit_constant(high_bits, low_bits,
446 &highest_bit_set, &lowest_bit_set,
447 &all_bits_between_are_set);
448
449 if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
450 all_bits_between_are_set != 0)
451 return true;
452
453 if (highest_bit_set - lowest_bit_set < 21)
454 return true;
455
456 return false;
457 }
458
459 static void sparc_emit_set_const64_quick2(unsigned long high_bits,
460 unsigned long low_imm,
461 unsigned int dest,
462 int shift_count, struct jit_ctx *ctx)
463 {
464 emit_loadimm32(high_bits, dest, ctx);
465
466 /* Now shift it up into place. */
467 emit_alu_K(SLLX, dest, shift_count, ctx);
468
469 /* If there is a low immediate part piece, finish up by
470 * putting that in as well.
471 */
472 if (low_imm != 0)
473 emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
474 }
475
476 static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
477 {
478 int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
479 unsigned int tmp = bpf2sparc[TMP_REG_1];
480 u32 low_bits = (K & 0xffffffff);
481 u32 high_bits = (K >> 32);
482
483 /* These two tests also take care of all of the one
484 * instruction cases.
485 */
486 if (high_bits == 0xffffffff && (low_bits & 0x80000000))
487 return emit_loadimm_sext(K, dest, ctx);
488 if (high_bits == 0x00000000)
489 return emit_loadimm32(K, dest, ctx);
490
491 analyze_64bit_constant(high_bits, low_bits, &highest_bit_set,
492 &lowest_bit_set, &all_bits_between_are_set);
493
494 /* 1) mov -1, %reg
495 * sllx %reg, shift, %reg
496 * 2) mov -1, %reg
497 * srlx %reg, shift, %reg
498 * 3) mov some_small_const, %reg
499 * sllx %reg, shift, %reg
500 */
501 if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
502 all_bits_between_are_set != 0) ||
503 ((highest_bit_set - lowest_bit_set) < 12)) {
504 int shift = lowest_bit_set;
505 long the_const = -1;
506
507 if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
508 all_bits_between_are_set == 0) {
509 the_const =
510 create_simple_focus_bits(high_bits, low_bits,
511 lowest_bit_set, 0);
512 } else if (lowest_bit_set == 0)
513 shift = -(63 - highest_bit_set);
514
515 emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
516 if (shift > 0)
517 emit_alu_K(SLLX, dest, shift, ctx);
518 else if (shift < 0)
519 emit_alu_K(SRLX, dest, -shift, ctx);
520
521 return;
522 }
523
524 /* Now a range of 22 or less bits set somewhere.
525 * 1) sethi %hi(focus_bits), %reg
526 * sllx %reg, shift, %reg
527 * 2) sethi %hi(focus_bits), %reg
528 * srlx %reg, shift, %reg
529 */
530 if ((highest_bit_set - lowest_bit_set) < 21) {
531 unsigned long focus_bits =
532 create_simple_focus_bits(high_bits, low_bits,
533 lowest_bit_set, 10);
534
535 emit(SETHI(focus_bits, dest), ctx);
536
537 /* If lowest_bit_set == 10 then a sethi alone could
538 * have done it.
539 */
540 if (lowest_bit_set < 10)
541 emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx);
542 else if (lowest_bit_set > 10)
543 emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx);
544 return;
545 }
546
547 /* Ok, now 3 instruction sequences. */
548 if (low_bits == 0) {
549 emit_loadimm32(high_bits, dest, ctx);
550 emit_alu_K(SLLX, dest, 32, ctx);
551 return;
552 }
553
554 /* We may be able to do something quick
555 * when the constant is negated, so try that.
556 */
557 if (const64_is_2insns((~high_bits) & 0xffffffff,
558 (~low_bits) & 0xfffffc00)) {
559 /* NOTE: The trailing bits get XOR'd so we need the
560 * non-negated bits, not the negated ones.
561 */
562 unsigned long trailing_bits = low_bits & 0x3ff;
563
564 if ((((~high_bits) & 0xffffffff) == 0 &&
565 ((~low_bits) & 0x80000000) == 0) ||
566 (((~high_bits) & 0xffffffff) == 0xffffffff &&
567 ((~low_bits) & 0x80000000) != 0)) {
568 unsigned long fast_int = (~low_bits & 0xffffffff);
569
570 if ((is_sethi(fast_int) &&
571 (~high_bits & 0xffffffff) == 0)) {
572 emit(SETHI(fast_int, dest), ctx);
573 } else if (is_simm13(fast_int)) {
574 emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
575 } else {
576 emit_loadimm64(fast_int, dest, ctx);
577 }
578 } else {
579 u64 n = ((~low_bits) & 0xfffffc00) |
580 (((unsigned long)((~high_bits) & 0xffffffff))<<32);
581 emit_loadimm64(n, dest, ctx);
582 }
583
584 low_bits = -0x400 | trailing_bits;
585
586 emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
587 return;
588 }
589
590 /* 1) sethi %hi(xxx), %reg
591 * or %reg, %lo(xxx), %reg
592 * sllx %reg, yyy, %reg
593 */
594 if ((highest_bit_set - lowest_bit_set) < 32) {
595 unsigned long focus_bits =
596 create_simple_focus_bits(high_bits, low_bits,
597 lowest_bit_set, 0);
598
599 /* So what we know is that the set bits straddle the
600 * middle of the 64-bit word.
601 */
602 sparc_emit_set_const64_quick2(focus_bits, 0, dest,
603 lowest_bit_set, ctx);
604 return;
605 }
606
607 /* 1) sethi %hi(high_bits), %reg
608 * or %reg, %lo(high_bits), %reg
609 * sllx %reg, 32, %reg
610 * or %reg, low_bits, %reg
611 */
612 if (is_simm13(low_bits) && ((int)low_bits > 0)) {
613 sparc_emit_set_const64_quick2(high_bits, low_bits,
614 dest, 32, ctx);
615 return;
616 }
617
618 /* Oh well, we tried... Do a full 64-bit decomposition. */
619 ctx->tmp_1_used = true;
620
621 emit_loadimm32(high_bits, tmp, ctx);
622 emit_loadimm32(low_bits, dest, ctx);
623 emit_alu_K(SLLX, tmp, 32, ctx);
624 emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
625 }
626
627 static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
628 struct jit_ctx *ctx)
629 {
630 unsigned int off = to_idx - from_idx;
631
632 if (br_opc & XCC)
633 emit(br_opc | WDISP19(off << 2), ctx);
634 else
635 emit(br_opc | WDISP22(off << 2), ctx);
636 }
637
638 static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
639 const u8 dst, const u8 src, struct jit_ctx *ctx)
640 {
641 unsigned int off = to_idx - from_idx;
642
643 emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx);
644 }
645
646 static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
647 const u8 dst, s32 imm, struct jit_ctx *ctx)
648 {
649 unsigned int off = to_idx - from_idx;
650
651 emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx);
652 }
653
654 #define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX)
655 #define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
656
657 #define emit_cmp(R1, R2, CTX) \
658 emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
659
660 #define emit_cmpi(R1, IMM, CTX) \
661 emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
662
663 #define emit_btst(R1, R2, CTX) \
664 emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
665
666 #define emit_btsti(R1, IMM, CTX) \
667 emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
668
669 static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
670 const s32 imm, bool is_imm, int branch_dst,
671 struct jit_ctx *ctx)
672 {
673 bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
674 const u8 tmp = bpf2sparc[TMP_REG_1];
675
676 branch_dst = ctx->offset[branch_dst];
677
678 if (!is_simm10(branch_dst - ctx->idx) ||
679 BPF_OP(code) == BPF_JSET)
680 use_cbcond = false;
681
682 if (is_imm) {
683 bool fits = true;
684
685 if (use_cbcond) {
686 if (!is_simm5(imm))
687 fits = false;
688 } else if (!is_simm13(imm)) {
689 fits = false;
690 }
691 if (!fits) {
692 ctx->tmp_1_used = true;
693 emit_loadimm_sext(imm, tmp, ctx);
694 src = tmp;
695 is_imm = false;
696 }
697 }
698
699 if (!use_cbcond) {
700 u32 br_opcode;
701
702 if (BPF_OP(code) == BPF_JSET) {
703 if (is_imm)
704 emit_btsti(dst, imm, ctx);
705 else
706 emit_btst(dst, src, ctx);
707 } else {
708 if (is_imm)
709 emit_cmpi(dst, imm, ctx);
710 else
711 emit_cmp(dst, src, ctx);
712 }
713 switch (BPF_OP(code)) {
714 case BPF_JEQ:
715 br_opcode = BE;
716 break;
717 case BPF_JGT:
718 br_opcode = BGU;
719 break;
720 case BPF_JLT:
721 br_opcode = BLU;
722 break;
723 case BPF_JGE:
724 br_opcode = BGEU;
725 break;
726 case BPF_JLE:
727 br_opcode = BLEU;
728 break;
729 case BPF_JSET:
730 case BPF_JNE:
731 br_opcode = BNE;
732 break;
733 case BPF_JSGT:
734 br_opcode = BG;
735 break;
736 case BPF_JSLT:
737 br_opcode = BL;
738 break;
739 case BPF_JSGE:
740 br_opcode = BGE;
741 break;
742 case BPF_JSLE:
743 br_opcode = BLE;
744 break;
745 default:
746 /* Make sure we dont leak kernel information to the
747 * user.
748 */
749 return -EFAULT;
750 }
751 emit_branch(br_opcode, ctx->idx, branch_dst, ctx);
752 emit_nop(ctx);
753 } else {
754 u32 cbcond_opcode;
755
756 switch (BPF_OP(code)) {
757 case BPF_JEQ:
758 cbcond_opcode = CBCONDE;
759 break;
760 case BPF_JGT:
761 cbcond_opcode = CBCONDGU;
762 break;
763 case BPF_JLT:
764 cbcond_opcode = CBCONDLU;
765 break;
766 case BPF_JGE:
767 cbcond_opcode = CBCONDGEU;
768 break;
769 case BPF_JLE:
770 cbcond_opcode = CBCONDLEU;
771 break;
772 case BPF_JNE:
773 cbcond_opcode = CBCONDNE;
774 break;
775 case BPF_JSGT:
776 cbcond_opcode = CBCONDG;
777 break;
778 case BPF_JSLT:
779 cbcond_opcode = CBCONDL;
780 break;
781 case BPF_JSGE:
782 cbcond_opcode = CBCONDGE;
783 break;
784 case BPF_JSLE:
785 cbcond_opcode = CBCONDLE;
786 break;
787 default:
788 /* Make sure we dont leak kernel information to the
789 * user.
790 */
791 return -EFAULT;
792 }
793 cbcond_opcode |= CBCOND_OP;
794 if (is_imm)
795 emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst,
796 dst, imm, ctx);
797 else
798 emit_cbcond(cbcond_opcode, ctx->idx, branch_dst,
799 dst, src, ctx);
800 }
801 return 0;
802 }
803
804 static void load_skb_regs(struct jit_ctx *ctx, u8 r_skb)
805 {
806 const u8 r_headlen = bpf2sparc[SKB_HLEN_REG];
807 const u8 r_data = bpf2sparc[SKB_DATA_REG];
808 const u8 r_tmp = bpf2sparc[TMP_REG_1];
809 unsigned int off;
810
811 off = offsetof(struct sk_buff, len);
812 emit(LD32I | RS1(r_skb) | S13(off) | RD(r_headlen), ctx);
813
814 off = offsetof(struct sk_buff, data_len);
815 emit(LD32I | RS1(r_skb) | S13(off) | RD(r_tmp), ctx);
816
817 emit(SUB | RS1(r_headlen) | RS2(r_tmp) | RD(r_headlen), ctx);
818
819 off = offsetof(struct sk_buff, data);
820 emit(LDPTRI | RS1(r_skb) | S13(off) | RD(r_data), ctx);
821 }
822
823 /* Just skip the save instruction and the ctx register move. */
824 #define BPF_TAILCALL_PROLOGUE_SKIP 16
825 #define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128)
826
827 static void build_prologue(struct jit_ctx *ctx)
828 {
829 s32 stack_needed = BASE_STACKFRAME;
830
831 if (ctx->saw_frame_pointer || ctx->saw_tail_call) {
832 struct bpf_prog *prog = ctx->prog;
833 u32 stack_depth;
834
835 stack_depth = prog->aux->stack_depth;
836 stack_needed += round_up(stack_depth, 16);
837 }
838
839 if (ctx->saw_tail_call)
840 stack_needed += 8;
841
842 /* save %sp, -176, %sp */
843 emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
844
845 /* tail_call_cnt = 0 */
846 if (ctx->saw_tail_call) {
847 u32 off = BPF_TAILCALL_CNT_SP_OFF;
848
849 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
850 } else {
851 emit_nop(ctx);
852 }
853 if (ctx->saw_frame_pointer) {
854 const u8 vfp = bpf2sparc[BPF_REG_FP];
855
856 emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
857 }
858
859 emit_reg_move(I0, O0, ctx);
860 /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
861
862 if (ctx->saw_ld_abs_ind)
863 load_skb_regs(ctx, bpf2sparc[BPF_REG_1]);
864 }
865
866 static void build_epilogue(struct jit_ctx *ctx)
867 {
868 ctx->epilogue_offset = ctx->idx;
869
870 /* ret (jmpl %i7 + 8, %g0) */
871 emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
872
873 /* restore %i5, %g0, %o0 */
874 emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
875 }
876
877 static void emit_tail_call(struct jit_ctx *ctx)
878 {
879 const u8 bpf_array = bpf2sparc[BPF_REG_2];
880 const u8 bpf_index = bpf2sparc[BPF_REG_3];
881 const u8 tmp = bpf2sparc[TMP_REG_1];
882 u32 off;
883
884 ctx->saw_tail_call = true;
885
886 off = offsetof(struct bpf_array, map.max_entries);
887 emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
888 emit_cmp(bpf_index, tmp, ctx);
889 #define OFFSET1 17
890 emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx);
891 emit_nop(ctx);
892
893 off = BPF_TAILCALL_CNT_SP_OFF;
894 emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
895 emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
896 #define OFFSET2 13
897 emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx);
898 emit_nop(ctx);
899
900 emit_alu_K(ADD, tmp, 1, ctx);
901 off = BPF_TAILCALL_CNT_SP_OFF;
902 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
903
904 emit_alu3_K(SLL, bpf_index, 3, tmp, ctx);
905 emit_alu(ADD, bpf_array, tmp, ctx);
906 off = offsetof(struct bpf_array, ptrs);
907 emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
908
909 emit_cmpi(tmp, 0, ctx);
910 #define OFFSET3 5
911 emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx);
912 emit_nop(ctx);
913
914 off = offsetof(struct bpf_prog, bpf_func);
915 emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
916
917 off = BPF_TAILCALL_PROLOGUE_SKIP;
918 emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
919 emit_nop(ctx);
920 }
921
922 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
923 {
924 const u8 code = insn->code;
925 const u8 dst = bpf2sparc[insn->dst_reg];
926 const u8 src = bpf2sparc[insn->src_reg];
927 const int i = insn - ctx->prog->insnsi;
928 const s16 off = insn->off;
929 const s32 imm = insn->imm;
930 u32 *func;
931
932 if (insn->src_reg == BPF_REG_FP)
933 ctx->saw_frame_pointer = true;
934
935 switch (code) {
936 /* dst = src */
937 case BPF_ALU | BPF_MOV | BPF_X:
938 emit_alu3_K(SRL, src, 0, dst, ctx);
939 break;
940 case BPF_ALU64 | BPF_MOV | BPF_X:
941 emit_reg_move(src, dst, ctx);
942 break;
943 /* dst = dst OP src */
944 case BPF_ALU | BPF_ADD | BPF_X:
945 case BPF_ALU64 | BPF_ADD | BPF_X:
946 emit_alu(ADD, src, dst, ctx);
947 goto do_alu32_trunc;
948 case BPF_ALU | BPF_SUB | BPF_X:
949 case BPF_ALU64 | BPF_SUB | BPF_X:
950 emit_alu(SUB, src, dst, ctx);
951 goto do_alu32_trunc;
952 case BPF_ALU | BPF_AND | BPF_X:
953 case BPF_ALU64 | BPF_AND | BPF_X:
954 emit_alu(AND, src, dst, ctx);
955 goto do_alu32_trunc;
956 case BPF_ALU | BPF_OR | BPF_X:
957 case BPF_ALU64 | BPF_OR | BPF_X:
958 emit_alu(OR, src, dst, ctx);
959 goto do_alu32_trunc;
960 case BPF_ALU | BPF_XOR | BPF_X:
961 case BPF_ALU64 | BPF_XOR | BPF_X:
962 emit_alu(XOR, src, dst, ctx);
963 goto do_alu32_trunc;
964 case BPF_ALU | BPF_MUL | BPF_X:
965 emit_alu(MUL, src, dst, ctx);
966 goto do_alu32_trunc;
967 case BPF_ALU64 | BPF_MUL | BPF_X:
968 emit_alu(MULX, src, dst, ctx);
969 break;
970 case BPF_ALU | BPF_DIV | BPF_X:
971 emit_cmp(src, G0, ctx);
972 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
973 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
974
975 emit_write_y(G0, ctx);
976 emit_alu(DIV, src, dst, ctx);
977 break;
978
979 case BPF_ALU64 | BPF_DIV | BPF_X:
980 emit_cmp(src, G0, ctx);
981 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
982 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
983
984 emit_alu(UDIVX, src, dst, ctx);
985 break;
986
987 case BPF_ALU | BPF_MOD | BPF_X: {
988 const u8 tmp = bpf2sparc[TMP_REG_1];
989
990 ctx->tmp_1_used = true;
991
992 emit_cmp(src, G0, ctx);
993 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
994 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
995
996 emit_write_y(G0, ctx);
997 emit_alu3(DIV, dst, src, tmp, ctx);
998 emit_alu3(MULX, tmp, src, tmp, ctx);
999 emit_alu3(SUB, dst, tmp, dst, ctx);
1000 goto do_alu32_trunc;
1001 }
1002 case BPF_ALU64 | BPF_MOD | BPF_X: {
1003 const u8 tmp = bpf2sparc[TMP_REG_1];
1004
1005 ctx->tmp_1_used = true;
1006
1007 emit_cmp(src, G0, ctx);
1008 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
1009 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
1010
1011 emit_alu3(UDIVX, dst, src, tmp, ctx);
1012 emit_alu3(MULX, tmp, src, tmp, ctx);
1013 emit_alu3(SUB, dst, tmp, dst, ctx);
1014 break;
1015 }
1016 case BPF_ALU | BPF_LSH | BPF_X:
1017 emit_alu(SLL, src, dst, ctx);
1018 goto do_alu32_trunc;
1019 case BPF_ALU64 | BPF_LSH | BPF_X:
1020 emit_alu(SLLX, src, dst, ctx);
1021 break;
1022 case BPF_ALU | BPF_RSH | BPF_X:
1023 emit_alu(SRL, src, dst, ctx);
1024 break;
1025 case BPF_ALU64 | BPF_RSH | BPF_X:
1026 emit_alu(SRLX, src, dst, ctx);
1027 break;
1028 case BPF_ALU | BPF_ARSH | BPF_X:
1029 emit_alu(SRA, src, dst, ctx);
1030 goto do_alu32_trunc;
1031 case BPF_ALU64 | BPF_ARSH | BPF_X:
1032 emit_alu(SRAX, src, dst, ctx);
1033 break;
1034
1035 /* dst = -dst */
1036 case BPF_ALU | BPF_NEG:
1037 case BPF_ALU64 | BPF_NEG:
1038 emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
1039 goto do_alu32_trunc;
1040
1041 case BPF_ALU | BPF_END | BPF_FROM_BE:
1042 switch (imm) {
1043 case 16:
1044 emit_alu_K(SLL, dst, 16, ctx);
1045 emit_alu_K(SRL, dst, 16, ctx);
1046 break;
1047 case 32:
1048 emit_alu_K(SRL, dst, 0, ctx);
1049 break;
1050 case 64:
1051 /* nop */
1052 break;
1053
1054 }
1055 break;
1056
1057 /* dst = BSWAP##imm(dst) */
1058 case BPF_ALU | BPF_END | BPF_FROM_LE: {
1059 const u8 tmp = bpf2sparc[TMP_REG_1];
1060 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1061
1062 ctx->tmp_1_used = true;
1063 switch (imm) {
1064 case 16:
1065 emit_alu3_K(AND, dst, 0xff, tmp, ctx);
1066 emit_alu3_K(SRL, dst, 8, dst, ctx);
1067 emit_alu3_K(AND, dst, 0xff, dst, ctx);
1068 emit_alu3_K(SLL, tmp, 8, tmp, ctx);
1069 emit_alu(OR, tmp, dst, ctx);
1070 break;
1071
1072 case 32:
1073 ctx->tmp_2_used = true;
1074 emit_alu3_K(SRL, dst, 24, tmp, ctx); /* tmp = dst >> 24 */
1075 emit_alu3_K(SRL, dst, 16, tmp2, ctx); /* tmp2 = dst >> 16 */
1076 emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1077 emit_alu3_K(SLL, tmp2, 8, tmp2, ctx); /* tmp2 = tmp2 << 8 */
1078 emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */
1079 emit_alu3_K(SRL, dst, 8, tmp2, ctx); /* tmp2 = dst >> 8 */
1080 emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1081 emit_alu3_K(SLL, tmp2, 16, tmp2, ctx); /* tmp2 = tmp2 << 16 */
1082 emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */
1083 emit_alu3_K(AND, dst, 0xff, dst, ctx); /* dst = dst & 0xff */
1084 emit_alu3_K(SLL, dst, 24, dst, ctx); /* dst = dst << 24 */
1085 emit_alu(OR, tmp, dst, ctx); /* dst = dst | tmp */
1086 break;
1087
1088 case 64:
1089 emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx);
1090 emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1091 emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1092 break;
1093 }
1094 break;
1095 }
1096 /* dst = imm */
1097 case BPF_ALU | BPF_MOV | BPF_K:
1098 emit_loadimm32(imm, dst, ctx);
1099 break;
1100 case BPF_ALU64 | BPF_MOV | BPF_K:
1101 emit_loadimm_sext(imm, dst, ctx);
1102 break;
1103 /* dst = dst OP imm */
1104 case BPF_ALU | BPF_ADD | BPF_K:
1105 case BPF_ALU64 | BPF_ADD | BPF_K:
1106 emit_alu_K(ADD, dst, imm, ctx);
1107 goto do_alu32_trunc;
1108 case BPF_ALU | BPF_SUB | BPF_K:
1109 case BPF_ALU64 | BPF_SUB | BPF_K:
1110 emit_alu_K(SUB, dst, imm, ctx);
1111 goto do_alu32_trunc;
1112 case BPF_ALU | BPF_AND | BPF_K:
1113 case BPF_ALU64 | BPF_AND | BPF_K:
1114 emit_alu_K(AND, dst, imm, ctx);
1115 goto do_alu32_trunc;
1116 case BPF_ALU | BPF_OR | BPF_K:
1117 case BPF_ALU64 | BPF_OR | BPF_K:
1118 emit_alu_K(OR, dst, imm, ctx);
1119 goto do_alu32_trunc;
1120 case BPF_ALU | BPF_XOR | BPF_K:
1121 case BPF_ALU64 | BPF_XOR | BPF_K:
1122 emit_alu_K(XOR, dst, imm, ctx);
1123 goto do_alu32_trunc;
1124 case BPF_ALU | BPF_MUL | BPF_K:
1125 emit_alu_K(MUL, dst, imm, ctx);
1126 goto do_alu32_trunc;
1127 case BPF_ALU64 | BPF_MUL | BPF_K:
1128 emit_alu_K(MULX, dst, imm, ctx);
1129 break;
1130 case BPF_ALU | BPF_DIV | BPF_K:
1131 if (imm == 0)
1132 return -EINVAL;
1133
1134 emit_write_y(G0, ctx);
1135 emit_alu_K(DIV, dst, imm, ctx);
1136 goto do_alu32_trunc;
1137 case BPF_ALU64 | BPF_DIV | BPF_K:
1138 if (imm == 0)
1139 return -EINVAL;
1140
1141 emit_alu_K(UDIVX, dst, imm, ctx);
1142 break;
1143 case BPF_ALU64 | BPF_MOD | BPF_K:
1144 case BPF_ALU | BPF_MOD | BPF_K: {
1145 const u8 tmp = bpf2sparc[TMP_REG_2];
1146 unsigned int div;
1147
1148 if (imm == 0)
1149 return -EINVAL;
1150
1151 div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
1152
1153 ctx->tmp_2_used = true;
1154
1155 if (BPF_CLASS(code) != BPF_ALU64)
1156 emit_write_y(G0, ctx);
1157 if (is_simm13(imm)) {
1158 emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
1159 emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
1160 emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1161 } else {
1162 const u8 tmp1 = bpf2sparc[TMP_REG_1];
1163
1164 ctx->tmp_1_used = true;
1165
1166 emit_set_const_sext(imm, tmp1, ctx);
1167 emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
1168 emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
1169 emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1170 }
1171 goto do_alu32_trunc;
1172 }
1173 case BPF_ALU | BPF_LSH | BPF_K:
1174 emit_alu_K(SLL, dst, imm, ctx);
1175 goto do_alu32_trunc;
1176 case BPF_ALU64 | BPF_LSH | BPF_K:
1177 emit_alu_K(SLLX, dst, imm, ctx);
1178 break;
1179 case BPF_ALU | BPF_RSH | BPF_K:
1180 emit_alu_K(SRL, dst, imm, ctx);
1181 break;
1182 case BPF_ALU64 | BPF_RSH | BPF_K:
1183 emit_alu_K(SRLX, dst, imm, ctx);
1184 break;
1185 case BPF_ALU | BPF_ARSH | BPF_K:
1186 emit_alu_K(SRA, dst, imm, ctx);
1187 goto do_alu32_trunc;
1188 case BPF_ALU64 | BPF_ARSH | BPF_K:
1189 emit_alu_K(SRAX, dst, imm, ctx);
1190 break;
1191
1192 do_alu32_trunc:
1193 if (BPF_CLASS(code) == BPF_ALU)
1194 emit_alu_K(SRL, dst, 0, ctx);
1195 break;
1196
1197 /* JUMP off */
1198 case BPF_JMP | BPF_JA:
1199 emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
1200 emit_nop(ctx);
1201 break;
1202 /* IF (dst COND src) JUMP off */
1203 case BPF_JMP | BPF_JEQ | BPF_X:
1204 case BPF_JMP | BPF_JGT | BPF_X:
1205 case BPF_JMP | BPF_JLT | BPF_X:
1206 case BPF_JMP | BPF_JGE | BPF_X:
1207 case BPF_JMP | BPF_JLE | BPF_X:
1208 case BPF_JMP | BPF_JNE | BPF_X:
1209 case BPF_JMP | BPF_JSGT | BPF_X:
1210 case BPF_JMP | BPF_JSLT | BPF_X:
1211 case BPF_JMP | BPF_JSGE | BPF_X:
1212 case BPF_JMP | BPF_JSLE | BPF_X:
1213 case BPF_JMP | BPF_JSET | BPF_X: {
1214 int err;
1215
1216 err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
1217 if (err)
1218 return err;
1219 break;
1220 }
1221 /* IF (dst COND imm) JUMP off */
1222 case BPF_JMP | BPF_JEQ | BPF_K:
1223 case BPF_JMP | BPF_JGT | BPF_K:
1224 case BPF_JMP | BPF_JLT | BPF_K:
1225 case BPF_JMP | BPF_JGE | BPF_K:
1226 case BPF_JMP | BPF_JLE | BPF_K:
1227 case BPF_JMP | BPF_JNE | BPF_K:
1228 case BPF_JMP | BPF_JSGT | BPF_K:
1229 case BPF_JMP | BPF_JSLT | BPF_K:
1230 case BPF_JMP | BPF_JSGE | BPF_K:
1231 case BPF_JMP | BPF_JSLE | BPF_K:
1232 case BPF_JMP | BPF_JSET | BPF_K: {
1233 int err;
1234
1235 err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
1236 if (err)
1237 return err;
1238 break;
1239 }
1240
1241 /* function call */
1242 case BPF_JMP | BPF_CALL:
1243 {
1244 u8 *func = ((u8 *)__bpf_call_base) + imm;
1245
1246 ctx->saw_call = true;
1247
1248 emit_call((u32 *)func, ctx);
1249 emit_nop(ctx);
1250
1251 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1252
1253 if (bpf_helper_changes_pkt_data(func) && ctx->saw_ld_abs_ind)
1254 load_skb_regs(ctx, bpf2sparc[BPF_REG_6]);
1255 break;
1256 }
1257
1258 /* tail call */
1259 case BPF_JMP | BPF_TAIL_CALL:
1260 emit_tail_call(ctx);
1261 break;
1262
1263 /* function return */
1264 case BPF_JMP | BPF_EXIT:
1265 /* Optimization: when last instruction is EXIT,
1266 simply fallthrough to epilogue. */
1267 if (i == ctx->prog->len - 1)
1268 break;
1269 emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx);
1270 emit_nop(ctx);
1271 break;
1272
1273 /* dst = imm64 */
1274 case BPF_LD | BPF_IMM | BPF_DW:
1275 {
1276 const struct bpf_insn insn1 = insn[1];
1277 u64 imm64;
1278
1279 imm64 = (u64)insn1.imm << 32 | (u32)imm;
1280 emit_loadimm64(imm64, dst, ctx);
1281
1282 return 1;
1283 }
1284
1285 /* LDX: dst = *(size *)(src + off) */
1286 case BPF_LDX | BPF_MEM | BPF_W:
1287 case BPF_LDX | BPF_MEM | BPF_H:
1288 case BPF_LDX | BPF_MEM | BPF_B:
1289 case BPF_LDX | BPF_MEM | BPF_DW: {
1290 const u8 tmp = bpf2sparc[TMP_REG_1];
1291 u32 opcode = 0, rs2;
1292
1293 ctx->tmp_1_used = true;
1294 switch (BPF_SIZE(code)) {
1295 case BPF_W:
1296 opcode = LD32;
1297 break;
1298 case BPF_H:
1299 opcode = LD16;
1300 break;
1301 case BPF_B:
1302 opcode = LD8;
1303 break;
1304 case BPF_DW:
1305 opcode = LD64;
1306 break;
1307 }
1308
1309 if (is_simm13(off)) {
1310 opcode |= IMMED;
1311 rs2 = S13(off);
1312 } else {
1313 emit_loadimm(off, tmp, ctx);
1314 rs2 = RS2(tmp);
1315 }
1316 emit(opcode | RS1(src) | rs2 | RD(dst), ctx);
1317 break;
1318 }
1319 /* ST: *(size *)(dst + off) = imm */
1320 case BPF_ST | BPF_MEM | BPF_W:
1321 case BPF_ST | BPF_MEM | BPF_H:
1322 case BPF_ST | BPF_MEM | BPF_B:
1323 case BPF_ST | BPF_MEM | BPF_DW: {
1324 const u8 tmp = bpf2sparc[TMP_REG_1];
1325 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1326 u32 opcode = 0, rs2;
1327
1328 ctx->tmp_2_used = true;
1329 emit_loadimm(imm, tmp2, ctx);
1330
1331 switch (BPF_SIZE(code)) {
1332 case BPF_W:
1333 opcode = ST32;
1334 break;
1335 case BPF_H:
1336 opcode = ST16;
1337 break;
1338 case BPF_B:
1339 opcode = ST8;
1340 break;
1341 case BPF_DW:
1342 opcode = ST64;
1343 break;
1344 }
1345
1346 if (is_simm13(off)) {
1347 opcode |= IMMED;
1348 rs2 = S13(off);
1349 } else {
1350 ctx->tmp_1_used = true;
1351 emit_loadimm(off, tmp, ctx);
1352 rs2 = RS2(tmp);
1353 }
1354 emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
1355 break;
1356 }
1357
1358 /* STX: *(size *)(dst + off) = src */
1359 case BPF_STX | BPF_MEM | BPF_W:
1360 case BPF_STX | BPF_MEM | BPF_H:
1361 case BPF_STX | BPF_MEM | BPF_B:
1362 case BPF_STX | BPF_MEM | BPF_DW: {
1363 const u8 tmp = bpf2sparc[TMP_REG_1];
1364 u32 opcode = 0, rs2;
1365
1366 switch (BPF_SIZE(code)) {
1367 case BPF_W:
1368 opcode = ST32;
1369 break;
1370 case BPF_H:
1371 opcode = ST16;
1372 break;
1373 case BPF_B:
1374 opcode = ST8;
1375 break;
1376 case BPF_DW:
1377 opcode = ST64;
1378 break;
1379 }
1380 if (is_simm13(off)) {
1381 opcode |= IMMED;
1382 rs2 = S13(off);
1383 } else {
1384 ctx->tmp_1_used = true;
1385 emit_loadimm(off, tmp, ctx);
1386 rs2 = RS2(tmp);
1387 }
1388 emit(opcode | RS1(dst) | rs2 | RD(src), ctx);
1389 break;
1390 }
1391
1392 /* STX XADD: lock *(u32 *)(dst + off) += src */
1393 case BPF_STX | BPF_XADD | BPF_W: {
1394 const u8 tmp = bpf2sparc[TMP_REG_1];
1395 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1396 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1397
1398 ctx->tmp_1_used = true;
1399 ctx->tmp_2_used = true;
1400 ctx->tmp_3_used = true;
1401 emit_loadimm(off, tmp, ctx);
1402 emit_alu3(ADD, dst, tmp, tmp, ctx);
1403
1404 emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1405 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1406 emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1407 emit_cmp(tmp2, tmp3, ctx);
1408 emit_branch(BNE, 4, 0, ctx);
1409 emit_nop(ctx);
1410 break;
1411 }
1412 /* STX XADD: lock *(u64 *)(dst + off) += src */
1413 case BPF_STX | BPF_XADD | BPF_DW: {
1414 const u8 tmp = bpf2sparc[TMP_REG_1];
1415 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1416 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1417
1418 ctx->tmp_1_used = true;
1419 ctx->tmp_2_used = true;
1420 ctx->tmp_3_used = true;
1421 emit_loadimm(off, tmp, ctx);
1422 emit_alu3(ADD, dst, tmp, tmp, ctx);
1423
1424 emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1425 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1426 emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1427 emit_cmp(tmp2, tmp3, ctx);
1428 emit_branch(BNE, 4, 0, ctx);
1429 emit_nop(ctx);
1430 break;
1431 }
1432 #define CHOOSE_LOAD_FUNC(K, func) \
1433 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
1434
1435 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
1436 case BPF_LD | BPF_ABS | BPF_W:
1437 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_word);
1438 goto common_load;
1439 case BPF_LD | BPF_ABS | BPF_H:
1440 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_half);
1441 goto common_load;
1442 case BPF_LD | BPF_ABS | BPF_B:
1443 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_byte);
1444 goto common_load;
1445 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
1446 case BPF_LD | BPF_IND | BPF_W:
1447 func = bpf_jit_load_word;
1448 goto common_load;
1449 case BPF_LD | BPF_IND | BPF_H:
1450 func = bpf_jit_load_half;
1451 goto common_load;
1452
1453 case BPF_LD | BPF_IND | BPF_B:
1454 func = bpf_jit_load_byte;
1455 common_load:
1456 ctx->saw_ld_abs_ind = true;
1457
1458 emit_reg_move(bpf2sparc[BPF_REG_6], O0, ctx);
1459 emit_loadimm(imm, O1, ctx);
1460
1461 if (BPF_MODE(code) == BPF_IND)
1462 emit_alu(ADD, src, O1, ctx);
1463
1464 emit_call(func, ctx);
1465 emit_alu_K(SRA, O1, 0, ctx);
1466
1467 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1468 break;
1469
1470 default:
1471 pr_err_once("unknown opcode %02x\n", code);
1472 return -EINVAL;
1473 }
1474
1475 return 0;
1476 }
1477
1478 static int build_body(struct jit_ctx *ctx)
1479 {
1480 const struct bpf_prog *prog = ctx->prog;
1481 int i;
1482
1483 for (i = 0; i < prog->len; i++) {
1484 const struct bpf_insn *insn = &prog->insnsi[i];
1485 int ret;
1486
1487 ret = build_insn(insn, ctx);
1488
1489 if (ret > 0) {
1490 i++;
1491 ctx->offset[i] = ctx->idx;
1492 continue;
1493 }
1494 ctx->offset[i] = ctx->idx;
1495 if (ret)
1496 return ret;
1497 }
1498 return 0;
1499 }
1500
1501 static void jit_fill_hole(void *area, unsigned int size)
1502 {
1503 u32 *ptr;
1504 /* We are guaranteed to have aligned memory. */
1505 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1506 *ptr++ = 0x91d02005; /* ta 5 */
1507 }
1508
1509 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1510 {
1511 struct bpf_prog *tmp, *orig_prog = prog;
1512 struct bpf_binary_header *header;
1513 bool tmp_blinded = false;
1514 struct jit_ctx ctx;
1515 u32 image_size;
1516 u8 *image_ptr;
1517 int pass;
1518
1519 if (!bpf_jit_enable)
1520 return orig_prog;
1521
1522 tmp = bpf_jit_blind_constants(prog);
1523 /* If blinding was requested and we failed during blinding,
1524 * we must fall back to the interpreter.
1525 */
1526 if (IS_ERR(tmp))
1527 return orig_prog;
1528 if (tmp != prog) {
1529 tmp_blinded = true;
1530 prog = tmp;
1531 }
1532
1533 memset(&ctx, 0, sizeof(ctx));
1534 ctx.prog = prog;
1535
1536 ctx.offset = kcalloc(prog->len, sizeof(unsigned int), GFP_KERNEL);
1537 if (ctx.offset == NULL) {
1538 prog = orig_prog;
1539 goto out;
1540 }
1541
1542 /* Fake pass to detect features used, and get an accurate assessment
1543 * of what the final image size will be.
1544 */
1545 if (build_body(&ctx)) {
1546 prog = orig_prog;
1547 goto out_off;
1548 }
1549 build_prologue(&ctx);
1550 build_epilogue(&ctx);
1551
1552 /* Now we know the actual image size. */
1553 image_size = sizeof(u32) * ctx.idx;
1554 header = bpf_jit_binary_alloc(image_size, &image_ptr,
1555 sizeof(u32), jit_fill_hole);
1556 if (header == NULL) {
1557 prog = orig_prog;
1558 goto out_off;
1559 }
1560
1561 ctx.image = (u32 *)image_ptr;
1562
1563 for (pass = 1; pass < 3; pass++) {
1564 ctx.idx = 0;
1565
1566 build_prologue(&ctx);
1567
1568 if (build_body(&ctx)) {
1569 bpf_jit_binary_free(header);
1570 prog = orig_prog;
1571 goto out_off;
1572 }
1573
1574 build_epilogue(&ctx);
1575
1576 if (bpf_jit_enable > 1)
1577 pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c%c]\n", pass,
1578 image_size - (ctx.idx * 4),
1579 ctx.tmp_1_used ? '1' : ' ',
1580 ctx.tmp_2_used ? '2' : ' ',
1581 ctx.tmp_3_used ? '3' : ' ',
1582 ctx.saw_ld_abs_ind ? 'L' : ' ',
1583 ctx.saw_frame_pointer ? 'F' : ' ',
1584 ctx.saw_call ? 'C' : ' ',
1585 ctx.saw_tail_call ? 'T' : ' ');
1586 }
1587
1588 if (bpf_jit_enable > 1)
1589 bpf_jit_dump(prog->len, image_size, pass, ctx.image);
1590
1591 bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE));
1592
1593 bpf_jit_binary_lock_ro(header);
1594
1595 prog->bpf_func = (void *)ctx.image;
1596 prog->jited = 1;
1597 prog->jited_len = image_size;
1598
1599 out_off:
1600 kfree(ctx.offset);
1601 out:
1602 if (tmp_blinded)
1603 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1604 tmp : orig_prog);
1605 return prog;
1606 }
Ubuntu Focal Linux kernel mirror