]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/lib/librte_bpf/bpf_jit_x86.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
import 15.2.0 Octopus source
[ceph.git] / ceph / src / spdk / dpdk / lib / librte_bpf / bpf_jit_x86.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4
5 #include <stdarg.h>
6 #include <errno.h>
7 #include <stdint.h>
8 #include <inttypes.h>
9
10 #include <rte_common.h>
11 #include <rte_log.h>
12 #include <rte_debug.h>
13 #include <rte_memory.h>
14 #include <rte_eal.h>
15 #include <rte_byteorder.h>
16
17 #include "bpf_impl.h"
18
19 #define GET_BPF_OP(op) (BPF_OP(op) >> 4)
20
21 enum {
22 RAX = 0, /* scratch, return value */
23 RCX = 1, /* scratch, 4th arg */
24 RDX = 2, /* scratch, 3rd arg */
25 RBX = 3, /* callee saved */
26 RSP = 4, /* stack pointer */
27 RBP = 5, /* frame pointer, callee saved */
28 RSI = 6, /* scratch, 2nd arg */
29 RDI = 7, /* scratch, 1st arg */
30 R8 = 8, /* scratch, 5th arg */
31 R9 = 9, /* scratch, 6th arg */
32 R10 = 10, /* scratch */
33 R11 = 11, /* scratch */
34 R12 = 12, /* callee saved */
35 R13 = 13, /* callee saved */
36 R14 = 14, /* callee saved */
37 R15 = 15, /* callee saved */
38 };
39
40 #define IS_EXT_REG(r) ((r) >= R8)
41
42 enum {
43 REX_PREFIX = 0x40, /* fixed value 0100 */
44 REX_W = 0x8, /* 64bit operand size */
45 REX_R = 0x4, /* extension of the ModRM.reg field */
46 REX_X = 0x2, /* extension of the SIB.index field */
47 REX_B = 0x1, /* extension of the ModRM.rm field */
48 };
49
50 enum {
51 MOD_INDIRECT = 0,
52 MOD_IDISP8 = 1,
53 MOD_IDISP32 = 2,
54 MOD_DIRECT = 3,
55 };
56
57 enum {
58 SIB_SCALE_1 = 0,
59 SIB_SCALE_2 = 1,
60 SIB_SCALE_4 = 2,
61 SIB_SCALE_8 = 3,
62 };
63
64 /*
65 * eBPF to x86_64 register mappings.
66 */
67 static const uint32_t ebpf2x86[] = {
68 [EBPF_REG_0] = RAX,
69 [EBPF_REG_1] = RDI,
70 [EBPF_REG_2] = RSI,
71 [EBPF_REG_3] = RDX,
72 [EBPF_REG_4] = RCX,
73 [EBPF_REG_5] = R8,
74 [EBPF_REG_6] = RBX,
75 [EBPF_REG_7] = R13,
76 [EBPF_REG_8] = R14,
77 [EBPF_REG_9] = R15,
78 [EBPF_REG_10] = RBP,
79 };
80
81 /*
82 * r10 and r11 are used as a scratch temporary registers.
83 */
84 enum {
85 REG_DIV_IMM = R9,
86 REG_TMP0 = R11,
87 REG_TMP1 = R10,
88 };
89
90 /*
91 * callee saved registers list.
92 * keep RBP as the last one.
93 */
94 static const uint32_t save_regs[] = {RBX, R12, R13, R14, R15, RBP};
95
96 struct bpf_jit_state {
97 uint32_t idx;
98 size_t sz;
99 struct {
100 uint32_t num;
101 int32_t off;
102 } exit;
103 uint32_t reguse;
104 int32_t *off;
105 uint8_t *ins;
106 };
107
108 #define INUSE(v, r) (((v) >> (r)) & 1)
109 #define USED(v, r) ((v) |= 1 << (r))
110
111 union bpf_jit_imm {
112 uint32_t u32;
113 uint8_t u8[4];
114 };
115
116 /*
117 * In many cases for imm8 we can produce shorter code.
118 */
119 static size_t
120 imm_size(int32_t v)
121 {
122 if (v == (int8_t)v)
123 return sizeof(int8_t);
124 return sizeof(int32_t);
125 }
126
127 static void
128 emit_bytes(struct bpf_jit_state *st, const uint8_t ins[], uint32_t sz)
129 {
130 uint32_t i;
131
132 if (st->ins != NULL) {
133 for (i = 0; i != sz; i++)
134 st->ins[st->sz + i] = ins[i];
135 }
136 st->sz += sz;
137 }
138
139 static void
140 emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz)
141 {
142 union bpf_jit_imm v;
143
144 v.u32 = imm;
145 emit_bytes(st, v.u8, sz);
146 }
147
148 /*
149 * emit REX byte
150 */
151 static void
152 emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm)
153 {
154 uint8_t rex;
155
156 /* mark operand registers as used*/
157 USED(st->reguse, reg);
158 USED(st->reguse, rm);
159
160 rex = 0;
161 if (BPF_CLASS(op) == EBPF_ALU64 ||
162 op == (BPF_ST | BPF_MEM | EBPF_DW) ||
163 op == (BPF_STX | BPF_MEM | EBPF_DW) ||
164 op == (BPF_STX | EBPF_XADD | EBPF_DW) ||
165 op == (BPF_LD | BPF_IMM | EBPF_DW) ||
166 (BPF_CLASS(op) == BPF_LDX &&
167 BPF_MODE(op) == BPF_MEM &&
168 BPF_SIZE(op) != BPF_W))
169 rex |= REX_W;
170
171 if (IS_EXT_REG(reg))
172 rex |= REX_R;
173
174 if (IS_EXT_REG(rm))
175 rex |= REX_B;
176
177 /* store using SIL, DIL */
178 if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI))
179 rex |= REX_PREFIX;
180
181 if (rex != 0) {
182 rex |= REX_PREFIX;
183 emit_bytes(st, &rex, sizeof(rex));
184 }
185 }
186
187 /*
188 * emit MODRegRM byte
189 */
190 static void
191 emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm)
192 {
193 uint8_t v;
194
195 v = mod << 6 | (reg & 7) << 3 | (rm & 7);
196 emit_bytes(st, &v, sizeof(v));
197 }
198
199 /*
200 * emit SIB byte
201 */
202 static void
203 emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base)
204 {
205 uint8_t v;
206
207 v = scale << 6 | (idx & 7) << 3 | (base & 7);
208 emit_bytes(st, &v, sizeof(v));
209 }
210
211 /*
212 * emit OPCODE+REGIDX byte
213 */
214 static void
215 emit_opcode(struct bpf_jit_state *st, uint8_t ops, uint32_t reg)
216 {
217 uint8_t v;
218
219 v = ops | (reg & 7);
220 emit_bytes(st, &v, sizeof(v));
221 }
222
223
224 /*
225 * emit xchg %<sreg>, %<dreg>
226 */
227 static void
228 emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
229 {
230 const uint8_t ops = 0x87;
231
232 emit_rex(st, EBPF_ALU64, sreg, dreg);
233 emit_bytes(st, &ops, sizeof(ops));
234 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
235 }
236
237 /*
238 * emit neg %<dreg>
239 */
240 static void
241 emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
242 {
243 const uint8_t ops = 0xF7;
244 const uint8_t mods = 3;
245
246 emit_rex(st, op, 0, dreg);
247 emit_bytes(st, &ops, sizeof(ops));
248 emit_modregrm(st, MOD_DIRECT, mods, dreg);
249 }
250
251 /*
252 * emit mov %<sreg>, %<dreg>
253 */
254 static void
255 emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
256 uint32_t dreg)
257 {
258 const uint8_t ops = 0x89;
259
260 /* if operands are 32-bit, then it can be used to clear upper 32-bit */
261 if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) {
262 emit_rex(st, op, sreg, dreg);
263 emit_bytes(st, &ops, sizeof(ops));
264 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
265 }
266 }
267
268 /*
269 * emit movzwl %<sreg>, %<dreg>
270 */
271 static void
272 emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
273 {
274 static const uint8_t ops[] = {0x0F, 0xB7};
275
276 emit_rex(st, BPF_ALU, sreg, dreg);
277 emit_bytes(st, ops, sizeof(ops));
278 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
279 }
280
281 /*
282 * emit ror <imm8>, %<dreg>
283 */
284 static void
285 emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
286 {
287 const uint8_t prfx = 0x66;
288 const uint8_t ops = 0xC1;
289 const uint8_t mods = 1;
290
291 emit_bytes(st, &prfx, sizeof(prfx));
292 emit_rex(st, BPF_ALU, 0, dreg);
293 emit_bytes(st, &ops, sizeof(ops));
294 emit_modregrm(st, MOD_DIRECT, mods, dreg);
295 emit_imm(st, imm, imm_size(imm));
296 }
297
298 /*
299 * emit bswap %<dreg>
300 */
301 static void
302 emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
303 {
304 uint32_t rop;
305
306 const uint8_t ops = 0x0F;
307 const uint8_t mods = 1;
308
309 rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU;
310 emit_rex(st, rop, 0, dreg);
311 emit_bytes(st, &ops, sizeof(ops));
312 emit_modregrm(st, MOD_DIRECT, mods, dreg);
313 }
314
315 static void
316 emit_be2le(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
317 {
318 if (imm == 16) {
319 emit_ror_imm(st, dreg, 8);
320 emit_movzwl(st, dreg, dreg);
321 } else
322 emit_be2le_48(st, dreg, imm);
323 }
324
325 /*
326 * In general it is NOP for x86.
327 * Just clear the upper bits.
328 */
329 static void
330 emit_le2be(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
331 {
332 if (imm == 16)
333 emit_movzwl(st, dreg, dreg);
334 else if (imm == 32)
335 emit_mov_reg(st, BPF_ALU | EBPF_MOV | BPF_X, dreg, dreg);
336 }
337
338 /*
339 * emit one of:
340 * add <imm>, %<dreg>
341 * and <imm>, %<dreg>
342 * or <imm>, %<dreg>
343 * sub <imm>, %<dreg>
344 * xor <imm>, %<dreg>
345 */
346 static void
347 emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
348 {
349 uint8_t mod, opcode;
350 uint32_t bop, imsz;
351
352 const uint8_t op8 = 0x83;
353 const uint8_t op32 = 0x81;
354 static const uint8_t mods[] = {
355 [GET_BPF_OP(BPF_ADD)] = 0,
356 [GET_BPF_OP(BPF_AND)] = 4,
357 [GET_BPF_OP(BPF_OR)] = 1,
358 [GET_BPF_OP(BPF_SUB)] = 5,
359 [GET_BPF_OP(BPF_XOR)] = 6,
360 };
361
362 bop = GET_BPF_OP(op);
363 mod = mods[bop];
364
365 imsz = imm_size(imm);
366 opcode = (imsz == 1) ? op8 : op32;
367
368 emit_rex(st, op, 0, dreg);
369 emit_bytes(st, &opcode, sizeof(opcode));
370 emit_modregrm(st, MOD_DIRECT, mod, dreg);
371 emit_imm(st, imm, imsz);
372 }
373
374 /*
375 * emit one of:
376 * add %<sreg>, %<dreg>
377 * and %<sreg>, %<dreg>
378 * or %<sreg>, %<dreg>
379 * sub %<sreg>, %<dreg>
380 * xor %<sreg>, %<dreg>
381 */
382 static void
383 emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
384 uint32_t dreg)
385 {
386 uint32_t bop;
387
388 static const uint8_t ops[] = {
389 [GET_BPF_OP(BPF_ADD)] = 0x01,
390 [GET_BPF_OP(BPF_AND)] = 0x21,
391 [GET_BPF_OP(BPF_OR)] = 0x09,
392 [GET_BPF_OP(BPF_SUB)] = 0x29,
393 [GET_BPF_OP(BPF_XOR)] = 0x31,
394 };
395
396 bop = GET_BPF_OP(op);
397
398 emit_rex(st, op, sreg, dreg);
399 emit_bytes(st, &ops[bop], sizeof(ops[bop]));
400 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
401 }
402
403 static void
404 emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
405 {
406 uint8_t mod;
407 uint32_t bop, opx;
408
409 static const uint8_t ops[] = {0xC1, 0xD3};
410 static const uint8_t mods[] = {
411 [GET_BPF_OP(BPF_LSH)] = 4,
412 [GET_BPF_OP(BPF_RSH)] = 5,
413 [GET_BPF_OP(EBPF_ARSH)] = 7,
414 };
415
416 bop = GET_BPF_OP(op);
417 mod = mods[bop];
418 opx = (BPF_SRC(op) == BPF_X);
419
420 emit_rex(st, op, 0, dreg);
421 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
422 emit_modregrm(st, MOD_DIRECT, mod, dreg);
423 }
424
425 /*
426 * emit one of:
427 * shl <imm>, %<dreg>
428 * shr <imm>, %<dreg>
429 * sar <imm>, %<dreg>
430 */
431 static void
432 emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
433 uint32_t imm)
434 {
435 emit_shift(st, op, dreg);
436 emit_imm(st, imm, imm_size(imm));
437 }
438
439 /*
440 * emit one of:
441 * shl %<dreg>
442 * shr %<dreg>
443 * sar %<dreg>
444 * note that rcx is implicitly used as a source register, so few extra
445 * instructions for register spillage might be necessary.
446 */
447 static void
448 emit_shift_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
449 uint32_t dreg)
450 {
451 if (sreg != RCX)
452 emit_xchg_reg(st, RCX, sreg);
453
454 emit_shift(st, op, (dreg == RCX) ? sreg : dreg);
455
456 if (sreg != RCX)
457 emit_xchg_reg(st, RCX, sreg);
458 }
459
460 /*
461 * emit mov <imm>, %<dreg>
462 */
463 static void
464 emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
465 {
466 const uint8_t ops = 0xC7;
467
468 if (imm == 0) {
469 /* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */
470 op = BPF_CLASS(op) | BPF_XOR | BPF_X;
471 emit_alu_reg(st, op, dreg, dreg);
472 return;
473 }
474
475 emit_rex(st, op, 0, dreg);
476 emit_bytes(st, &ops, sizeof(ops));
477 emit_modregrm(st, MOD_DIRECT, 0, dreg);
478 emit_imm(st, imm, sizeof(imm));
479 }
480
481 /*
482 * emit mov <imm64>, %<dreg>
483 */
484 static void
485 emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
486 uint32_t imm1)
487 {
488 uint32_t op;
489
490 const uint8_t ops = 0xB8;
491
492 op = (imm1 == 0) ? BPF_ALU : EBPF_ALU64;
493
494 emit_rex(st, op, 0, dreg);
495 emit_opcode(st, ops, dreg);
496
497 emit_imm(st, imm0, sizeof(imm0));
498 if (imm1 != 0)
499 emit_imm(st, imm1, sizeof(imm1));
500 }
501
502 /*
503 * note that rax:rdx are implicitly used as source/destination registers,
504 * so some reg spillage is necessary.
505 * emit:
506 * mov %rax, %r11
507 * mov %rdx, %r10
508 * mov %<dreg>, %rax
509 * either:
510 * mov %<sreg>, %rdx
511 * OR
512 * mov <imm>, %rdx
513 * mul %rdx
514 * mov %r10, %rdx
515 * mov %rax, %<dreg>
516 * mov %r11, %rax
517 */
518 static void
519 emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
520 uint32_t imm)
521 {
522 const uint8_t ops = 0xF7;
523 const uint8_t mods = 4;
524
525 /* save rax & rdx */
526 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
527 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
528
529 /* rax = dreg */
530 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
531
532 if (BPF_SRC(op) == BPF_X)
533 /* rdx = sreg */
534 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
535 sreg == RAX ? REG_TMP0 : sreg, RDX);
536 else
537 /* rdx = imm */
538 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm);
539
540 emit_rex(st, op, RAX, RDX);
541 emit_bytes(st, &ops, sizeof(ops));
542 emit_modregrm(st, MOD_DIRECT, mods, RDX);
543
544 if (dreg != RDX)
545 /* restore rdx */
546 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
547
548 if (dreg != RAX) {
549 /* dreg = rax */
550 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
551 /* restore rax */
552 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
553 }
554 }
555
556 /*
557 * emit mov <ofs>(%<sreg>), %<dreg>
558 * note that for non 64-bit ops, higher bits have to be cleared.
559 */
560 static void
561 emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
562 int32_t ofs)
563 {
564 uint32_t mods, opsz;
565 const uint8_t op32 = 0x8B;
566 const uint8_t op16[] = {0x0F, 0xB7};
567 const uint8_t op8[] = {0x0F, 0xB6};
568
569 emit_rex(st, op, dreg, sreg);
570
571 opsz = BPF_SIZE(op);
572 if (opsz == BPF_B)
573 emit_bytes(st, op8, sizeof(op8));
574 else if (opsz == BPF_H)
575 emit_bytes(st, op16, sizeof(op16));
576 else
577 emit_bytes(st, &op32, sizeof(op32));
578
579 mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32;
580
581 emit_modregrm(st, mods, dreg, sreg);
582 if (sreg == RSP || sreg == R12)
583 emit_sib(st, SIB_SCALE_1, sreg, sreg);
584 emit_imm(st, ofs, imm_size(ofs));
585 }
586
587 /*
588 * emit one of:
589 * mov %<sreg>, <ofs>(%<dreg>)
590 * mov <imm>, <ofs>(%<dreg>)
591 */
592 static void
593 emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
594 uint32_t dreg, uint32_t imm, int32_t ofs)
595 {
596 uint32_t mods, imsz, opsz, opx;
597 const uint8_t prfx16 = 0x66;
598
599 /* 8 bit instruction opcodes */
600 static const uint8_t op8[] = {0xC6, 0x88};
601
602 /* 16/32/64 bit instruction opcodes */
603 static const uint8_t ops[] = {0xC7, 0x89};
604
605 /* is the instruction has immediate value or src reg? */
606 opx = (BPF_CLASS(op) == BPF_STX);
607
608 opsz = BPF_SIZE(op);
609 if (opsz == BPF_H)
610 emit_bytes(st, &prfx16, sizeof(prfx16));
611
612 emit_rex(st, op, sreg, dreg);
613
614 if (opsz == BPF_B)
615 emit_bytes(st, &op8[opx], sizeof(op8[opx]));
616 else
617 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
618
619 imsz = imm_size(ofs);
620 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
621
622 emit_modregrm(st, mods, sreg, dreg);
623
624 if (dreg == RSP || dreg == R12)
625 emit_sib(st, SIB_SCALE_1, dreg, dreg);
626
627 emit_imm(st, ofs, imsz);
628
629 if (opx == 0) {
630 imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
631 emit_imm(st, imm, imsz);
632 }
633 }
634
635 static void
636 emit_st_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm,
637 int32_t ofs)
638 {
639 emit_st_common(st, op, 0, dreg, imm, ofs);
640 }
641
642 static void
643 emit_st_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
644 int32_t ofs)
645 {
646 emit_st_common(st, op, sreg, dreg, 0, ofs);
647 }
648
649 /*
650 * emit lock add %<sreg>, <ofs>(%<dreg>)
651 */
652 static void
653 emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
654 uint32_t dreg, int32_t ofs)
655 {
656 uint32_t imsz, mods;
657
658 const uint8_t lck = 0xF0; /* lock prefix */
659 const uint8_t ops = 0x01; /* add opcode */
660
661 imsz = imm_size(ofs);
662 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
663
664 emit_bytes(st, &lck, sizeof(lck));
665 emit_rex(st, op, sreg, dreg);
666 emit_bytes(st, &ops, sizeof(ops));
667 emit_modregrm(st, mods, sreg, dreg);
668 emit_imm(st, ofs, imsz);
669 }
670
671 /*
672 * emit:
673 * mov <imm64>, (%rax)
674 * call *%rax
675 */
676 static void
677 emit_call(struct bpf_jit_state *st, uintptr_t trg)
678 {
679 const uint8_t ops = 0xFF;
680 const uint8_t mods = 2;
681
682 emit_ld_imm64(st, RAX, trg, trg >> 32);
683 emit_bytes(st, &ops, sizeof(ops));
684 emit_modregrm(st, MOD_DIRECT, mods, RAX);
685 }
686
687 /*
688 * emit jmp <ofs>
689 * where 'ofs' is the target offset for the native code.
690 */
691 static void
692 emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs)
693 {
694 int32_t joff;
695 uint32_t imsz;
696
697 const uint8_t op8 = 0xEB;
698 const uint8_t op32 = 0xE9;
699
700 const int32_t sz8 = sizeof(op8) + sizeof(uint8_t);
701 const int32_t sz32 = sizeof(op32) + sizeof(uint32_t);
702
703 /* max possible jmp instruction size */
704 const int32_t iszm = RTE_MAX(sz8, sz32);
705
706 joff = ofs - st->sz;
707 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
708
709 if (imsz == 1) {
710 emit_bytes(st, &op8, sizeof(op8));
711 joff -= sz8;
712 } else {
713 emit_bytes(st, &op32, sizeof(op32));
714 joff -= sz32;
715 }
716
717 emit_imm(st, joff, imsz);
718 }
719
720 /*
721 * emit jmp <ofs>
722 * where 'ofs' is the target offset for the BPF bytecode.
723 */
724 static void
725 emit_jmp(struct bpf_jit_state *st, int32_t ofs)
726 {
727 emit_abs_jmp(st, st->off[st->idx + ofs]);
728 }
729
730 /*
731 * emit one of:
732 * cmovz %<sreg>, <%dreg>
733 * cmovne %<sreg>, <%dreg>
734 * cmova %<sreg>, <%dreg>
735 * cmovb %<sreg>, <%dreg>
736 * cmovae %<sreg>, <%dreg>
737 * cmovbe %<sreg>, <%dreg>
738 * cmovg %<sreg>, <%dreg>
739 * cmovl %<sreg>, <%dreg>
740 * cmovge %<sreg>, <%dreg>
741 * cmovle %<sreg>, <%dreg>
742 */
743 static void
744 emit_movcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
745 uint32_t dreg)
746 {
747 uint32_t bop;
748
749 static const uint8_t ops[][2] = {
750 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x44}, /* CMOVZ */
751 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x45}, /* CMOVNE */
752 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x47}, /* CMOVA */
753 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x42}, /* CMOVB */
754 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x43}, /* CMOVAE */
755 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x46}, /* CMOVBE */
756 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x4F}, /* CMOVG */
757 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x4C}, /* CMOVL */
758 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x4D}, /* CMOVGE */
759 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x4E}, /* CMOVLE */
760 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x45}, /* CMOVNE */
761 };
762
763 bop = GET_BPF_OP(op);
764
765 emit_rex(st, op, dreg, sreg);
766 emit_bytes(st, ops[bop], sizeof(ops[bop]));
767 emit_modregrm(st, MOD_DIRECT, dreg, sreg);
768 }
769
770 /*
771 * emit one of:
772 * je <ofs>
773 * jne <ofs>
774 * ja <ofs>
775 * jb <ofs>
776 * jae <ofs>
777 * jbe <ofs>
778 * jg <ofs>
779 * jl <ofs>
780 * jge <ofs>
781 * jle <ofs>
782 * where 'ofs' is the target offset for the native code.
783 */
784 static void
785 emit_abs_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
786 {
787 uint32_t bop, imsz;
788 int32_t joff;
789
790 static const uint8_t op8[] = {
791 [GET_BPF_OP(BPF_JEQ)] = 0x74, /* JE */
792 [GET_BPF_OP(EBPF_JNE)] = 0x75, /* JNE */
793 [GET_BPF_OP(BPF_JGT)] = 0x77, /* JA */
794 [GET_BPF_OP(EBPF_JLT)] = 0x72, /* JB */
795 [GET_BPF_OP(BPF_JGE)] = 0x73, /* JAE */
796 [GET_BPF_OP(EBPF_JLE)] = 0x76, /* JBE */
797 [GET_BPF_OP(EBPF_JSGT)] = 0x7F, /* JG */
798 [GET_BPF_OP(EBPF_JSLT)] = 0x7C, /* JL */
799 [GET_BPF_OP(EBPF_JSGE)] = 0x7D, /*JGE */
800 [GET_BPF_OP(EBPF_JSLE)] = 0x7E, /* JLE */
801 [GET_BPF_OP(BPF_JSET)] = 0x75, /*JNE */
802 };
803
804 static const uint8_t op32[][2] = {
805 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x84}, /* JE */
806 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x85}, /* JNE */
807 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x87}, /* JA */
808 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x82}, /* JB */
809 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x83}, /* JAE */
810 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x86}, /* JBE */
811 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x8F}, /* JG */
812 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x8C}, /* JL */
813 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x8D}, /*JGE */
814 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x8E}, /* JLE */
815 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x85}, /*JNE */
816 };
817
818 const int32_t sz8 = sizeof(op8[0]) + sizeof(uint8_t);
819 const int32_t sz32 = sizeof(op32[0]) + sizeof(uint32_t);
820
821 /* max possible jcc instruction size */
822 const int32_t iszm = RTE_MAX(sz8, sz32);
823
824 joff = ofs - st->sz;
825 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
826
827 bop = GET_BPF_OP(op);
828
829 if (imsz == 1) {
830 emit_bytes(st, &op8[bop], sizeof(op8[bop]));
831 joff -= sz8;
832 } else {
833 emit_bytes(st, op32[bop], sizeof(op32[bop]));
834 joff -= sz32;
835 }
836
837 emit_imm(st, joff, imsz);
838 }
839
840 /*
841 * emit one of:
842 * je <ofs>
843 * jne <ofs>
844 * ja <ofs>
845 * jb <ofs>
846 * jae <ofs>
847 * jbe <ofs>
848 * jg <ofs>
849 * jl <ofs>
850 * jge <ofs>
851 * jle <ofs>
852 * where 'ofs' is the target offset for the BPF bytecode.
853 */
854 static void
855 emit_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
856 {
857 emit_abs_jcc(st, op, st->off[st->idx + ofs]);
858 }
859
860
861 /*
862 * emit cmp <imm>, %<dreg>
863 */
864 static void
865 emit_cmp_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
866 {
867 uint8_t ops;
868 uint32_t imsz;
869
870 const uint8_t op8 = 0x83;
871 const uint8_t op32 = 0x81;
872 const uint8_t mods = 7;
873
874 imsz = imm_size(imm);
875 ops = (imsz == 1) ? op8 : op32;
876
877 emit_rex(st, op, 0, dreg);
878 emit_bytes(st, &ops, sizeof(ops));
879 emit_modregrm(st, MOD_DIRECT, mods, dreg);
880 emit_imm(st, imm, imsz);
881 }
882
883 /*
884 * emit test <imm>, %<dreg>
885 */
886 static void
887 emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
888 {
889 const uint8_t ops = 0xF7;
890 const uint8_t mods = 0;
891
892 emit_rex(st, op, 0, dreg);
893 emit_bytes(st, &ops, sizeof(ops));
894 emit_modregrm(st, MOD_DIRECT, mods, dreg);
895 emit_imm(st, imm, imm_size(imm));
896 }
897
898 static void
899 emit_jcc_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
900 uint32_t imm, int32_t ofs)
901 {
902 if (BPF_OP(op) == BPF_JSET)
903 emit_tst_imm(st, EBPF_ALU64, dreg, imm);
904 else
905 emit_cmp_imm(st, EBPF_ALU64, dreg, imm);
906
907 emit_jcc(st, op, ofs);
908 }
909
910 /*
911 * emit test %<sreg>, %<dreg>
912 */
913 static void
914 emit_tst_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
915 uint32_t dreg)
916 {
917 const uint8_t ops = 0x85;
918
919 emit_rex(st, op, sreg, dreg);
920 emit_bytes(st, &ops, sizeof(ops));
921 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
922 }
923
924 /*
925 * emit cmp %<sreg>, %<dreg>
926 */
927 static void
928 emit_cmp_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
929 uint32_t dreg)
930 {
931 const uint8_t ops = 0x39;
932
933 emit_rex(st, op, sreg, dreg);
934 emit_bytes(st, &ops, sizeof(ops));
935 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
936
937 }
938
939 static void
940 emit_jcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
941 uint32_t dreg, int32_t ofs)
942 {
943 if (BPF_OP(op) == BPF_JSET)
944 emit_tst_reg(st, EBPF_ALU64, sreg, dreg);
945 else
946 emit_cmp_reg(st, EBPF_ALU64, sreg, dreg);
947
948 emit_jcc(st, op, ofs);
949 }
950
951 /*
952 * note that rax:rdx are implicitly used as source/destination registers,
953 * so some reg spillage is necessary.
954 * emit:
955 * mov %rax, %r11
956 * mov %rdx, %r10
957 * mov %<dreg>, %rax
958 * xor %rdx, %rdx
959 * for divisor as immediate value:
960 * mov <imm>, %r9
961 * div %<divisor_reg>
962 * mov %r10, %rdx
963 * mov %rax, %<dreg>
964 * mov %r11, %rax
965 * either:
966 * mov %rax, %<dreg>
967 * OR
968 * mov %rdx, %<dreg>
969 * mov %r11, %rax
970 * mov %r10, %rdx
971 */
972 static void
973 emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
974 uint32_t imm)
975 {
976 uint32_t sr;
977
978 const uint8_t ops = 0xF7;
979 const uint8_t mods = 6;
980
981 if (BPF_SRC(op) == BPF_X) {
982
983 /* check that src divisor is not zero */
984 emit_tst_reg(st, BPF_CLASS(op), sreg, sreg);
985
986 /* exit with return value zero */
987 emit_movcc_reg(st, BPF_CLASS(op) | BPF_JEQ | BPF_X, sreg, RAX);
988 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
989 }
990
991 /* save rax & rdx */
992 if (dreg != RAX)
993 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
994 if (dreg != RDX)
995 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
996
997 /* fill rax & rdx */
998 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
999 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, 0);
1000
1001 if (BPF_SRC(op) == BPF_X) {
1002 sr = sreg;
1003 if (sr == RAX)
1004 sr = REG_TMP0;
1005 else if (sr == RDX)
1006 sr = REG_TMP1;
1007 } else {
1008 sr = REG_DIV_IMM;
1009 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, sr, imm);
1010 }
1011
1012 emit_rex(st, op, 0, sr);
1013 emit_bytes(st, &ops, sizeof(ops));
1014 emit_modregrm(st, MOD_DIRECT, mods, sr);
1015
1016 if (BPF_OP(op) == BPF_DIV)
1017 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
1018 else
1019 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, dreg);
1020
1021 if (dreg != RAX)
1022 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
1023 if (dreg != RDX)
1024 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
1025 }
1026
1027 static void
1028 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
1029 {
1030 uint32_t i;
1031 int32_t spil, ofs;
1032
1033 spil = 0;
1034 for (i = 0; i != RTE_DIM(save_regs); i++)
1035 spil += INUSE(st->reguse, save_regs[i]);
1036
1037 /* we can avoid touching the stack at all */
1038 if (spil == 0)
1039 return;
1040
1041
1042 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP,
1043 spil * sizeof(uint64_t));
1044
1045 ofs = 0;
1046 for (i = 0; i != RTE_DIM(save_regs); i++) {
1047 if (INUSE(st->reguse, save_regs[i]) != 0) {
1048 emit_st_reg(st, BPF_STX | BPF_MEM | EBPF_DW,
1049 save_regs[i], RSP, ofs);
1050 ofs += sizeof(uint64_t);
1051 }
1052 }
1053
1054 if (INUSE(st->reguse, RBP) != 0) {
1055 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RSP, RBP);
1056 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, stack_size);
1057 }
1058 }
1059
1060 /*
1061 * emit ret
1062 */
1063 static void
1064 emit_ret(struct bpf_jit_state *st)
1065 {
1066 const uint8_t ops = 0xC3;
1067
1068 emit_bytes(st, &ops, sizeof(ops));
1069 }
1070
1071 static void
1072 emit_epilog(struct bpf_jit_state *st)
1073 {
1074 uint32_t i;
1075 int32_t spil, ofs;
1076
1077 /* if we allready have an epilog generate a jump to it */
1078 if (st->exit.num++ != 0) {
1079 emit_abs_jmp(st, st->exit.off);
1080 return;
1081 }
1082
1083 /* store offset of epilog block */
1084 st->exit.off = st->sz;
1085
1086 spil = 0;
1087 for (i = 0; i != RTE_DIM(save_regs); i++)
1088 spil += INUSE(st->reguse, save_regs[i]);
1089
1090 if (spil != 0) {
1091
1092 if (INUSE(st->reguse, RBP) != 0)
1093 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
1094 RBP, RSP);
1095
1096 ofs = 0;
1097 for (i = 0; i != RTE_DIM(save_regs); i++) {
1098 if (INUSE(st->reguse, save_regs[i]) != 0) {
1099 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
1100 RSP, save_regs[i], ofs);
1101 ofs += sizeof(uint64_t);
1102 }
1103 }
1104
1105 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, RSP,
1106 spil * sizeof(uint64_t));
1107 }
1108
1109 emit_ret(st);
1110 }
1111
1112 /*
1113 * walk through bpf code and translate them x86_64 one.
1114 */
1115 static int
1116 emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
1117 {
1118 uint32_t i, dr, op, sr;
1119 const struct ebpf_insn *ins;
1120
1121 /* reset state fields */
1122 st->sz = 0;
1123 st->exit.num = 0;
1124
1125 emit_prolog(st, bpf->stack_sz);
1126
1127 for (i = 0; i != bpf->prm.nb_ins; i++) {
1128
1129 st->idx = i;
1130 st->off[i] = st->sz;
1131
1132 ins = bpf->prm.ins + i;
1133
1134 dr = ebpf2x86[ins->dst_reg];
1135 sr = ebpf2x86[ins->src_reg];
1136 op = ins->code;
1137
1138 switch (op) {
1139 /* 32 bit ALU IMM operations */
1140 case (BPF_ALU | BPF_ADD | BPF_K):
1141 case (BPF_ALU | BPF_SUB | BPF_K):
1142 case (BPF_ALU | BPF_AND | BPF_K):
1143 case (BPF_ALU | BPF_OR | BPF_K):
1144 case (BPF_ALU | BPF_XOR | BPF_K):
1145 emit_alu_imm(st, op, dr, ins->imm);
1146 break;
1147 case (BPF_ALU | BPF_LSH | BPF_K):
1148 case (BPF_ALU | BPF_RSH | BPF_K):
1149 emit_shift_imm(st, op, dr, ins->imm);
1150 break;
1151 case (BPF_ALU | EBPF_MOV | BPF_K):
1152 emit_mov_imm(st, op, dr, ins->imm);
1153 break;
1154 /* 32 bit ALU REG operations */
1155 case (BPF_ALU | BPF_ADD | BPF_X):
1156 case (BPF_ALU | BPF_SUB | BPF_X):
1157 case (BPF_ALU | BPF_AND | BPF_X):
1158 case (BPF_ALU | BPF_OR | BPF_X):
1159 case (BPF_ALU | BPF_XOR | BPF_X):
1160 emit_alu_reg(st, op, sr, dr);
1161 break;
1162 case (BPF_ALU | BPF_LSH | BPF_X):
1163 case (BPF_ALU | BPF_RSH | BPF_X):
1164 emit_shift_reg(st, op, sr, dr);
1165 break;
1166 case (BPF_ALU | EBPF_MOV | BPF_X):
1167 emit_mov_reg(st, op, sr, dr);
1168 break;
1169 case (BPF_ALU | BPF_NEG):
1170 emit_neg(st, op, dr);
1171 break;
1172 case (BPF_ALU | EBPF_END | EBPF_TO_BE):
1173 emit_be2le(st, dr, ins->imm);
1174 break;
1175 case (BPF_ALU | EBPF_END | EBPF_TO_LE):
1176 emit_le2be(st, dr, ins->imm);
1177 break;
1178 /* 64 bit ALU IMM operations */
1179 case (EBPF_ALU64 | BPF_ADD | BPF_K):
1180 case (EBPF_ALU64 | BPF_SUB | BPF_K):
1181 case (EBPF_ALU64 | BPF_AND | BPF_K):
1182 case (EBPF_ALU64 | BPF_OR | BPF_K):
1183 case (EBPF_ALU64 | BPF_XOR | BPF_K):
1184 emit_alu_imm(st, op, dr, ins->imm);
1185 break;
1186 case (EBPF_ALU64 | BPF_LSH | BPF_K):
1187 case (EBPF_ALU64 | BPF_RSH | BPF_K):
1188 case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
1189 emit_shift_imm(st, op, dr, ins->imm);
1190 break;
1191 case (EBPF_ALU64 | EBPF_MOV | BPF_K):
1192 emit_mov_imm(st, op, dr, ins->imm);
1193 break;
1194 /* 64 bit ALU REG operations */
1195 case (EBPF_ALU64 | BPF_ADD | BPF_X):
1196 case (EBPF_ALU64 | BPF_SUB | BPF_X):
1197 case (EBPF_ALU64 | BPF_AND | BPF_X):
1198 case (EBPF_ALU64 | BPF_OR | BPF_X):
1199 case (EBPF_ALU64 | BPF_XOR | BPF_X):
1200 emit_alu_reg(st, op, sr, dr);
1201 break;
1202 case (EBPF_ALU64 | BPF_LSH | BPF_X):
1203 case (EBPF_ALU64 | BPF_RSH | BPF_X):
1204 case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
1205 emit_shift_reg(st, op, sr, dr);
1206 break;
1207 case (EBPF_ALU64 | EBPF_MOV | BPF_X):
1208 emit_mov_reg(st, op, sr, dr);
1209 break;
1210 case (EBPF_ALU64 | BPF_NEG):
1211 emit_neg(st, op, dr);
1212 break;
1213 /* multiply instructions */
1214 case (BPF_ALU | BPF_MUL | BPF_K):
1215 case (BPF_ALU | BPF_MUL | BPF_X):
1216 case (EBPF_ALU64 | BPF_MUL | BPF_K):
1217 case (EBPF_ALU64 | BPF_MUL | BPF_X):
1218 emit_mul(st, op, sr, dr, ins->imm);
1219 break;
1220 /* divide instructions */
1221 case (BPF_ALU | BPF_DIV | BPF_K):
1222 case (BPF_ALU | BPF_MOD | BPF_K):
1223 case (BPF_ALU | BPF_DIV | BPF_X):
1224 case (BPF_ALU | BPF_MOD | BPF_X):
1225 case (EBPF_ALU64 | BPF_DIV | BPF_K):
1226 case (EBPF_ALU64 | BPF_MOD | BPF_K):
1227 case (EBPF_ALU64 | BPF_DIV | BPF_X):
1228 case (EBPF_ALU64 | BPF_MOD | BPF_X):
1229 emit_div(st, op, sr, dr, ins->imm);
1230 break;
1231 /* load instructions */
1232 case (BPF_LDX | BPF_MEM | BPF_B):
1233 case (BPF_LDX | BPF_MEM | BPF_H):
1234 case (BPF_LDX | BPF_MEM | BPF_W):
1235 case (BPF_LDX | BPF_MEM | EBPF_DW):
1236 emit_ld_reg(st, op, sr, dr, ins->off);
1237 break;
1238 /* load 64 bit immediate value */
1239 case (BPF_LD | BPF_IMM | EBPF_DW):
1240 emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
1241 i++;
1242 break;
1243 /* store instructions */
1244 case (BPF_STX | BPF_MEM | BPF_B):
1245 case (BPF_STX | BPF_MEM | BPF_H):
1246 case (BPF_STX | BPF_MEM | BPF_W):
1247 case (BPF_STX | BPF_MEM | EBPF_DW):
1248 emit_st_reg(st, op, sr, dr, ins->off);
1249 break;
1250 case (BPF_ST | BPF_MEM | BPF_B):
1251 case (BPF_ST | BPF_MEM | BPF_H):
1252 case (BPF_ST | BPF_MEM | BPF_W):
1253 case (BPF_ST | BPF_MEM | EBPF_DW):
1254 emit_st_imm(st, op, dr, ins->imm, ins->off);
1255 break;
1256 /* atomic add instructions */
1257 case (BPF_STX | EBPF_XADD | BPF_W):
1258 case (BPF_STX | EBPF_XADD | EBPF_DW):
1259 emit_st_xadd(st, op, sr, dr, ins->off);
1260 break;
1261 /* jump instructions */
1262 case (BPF_JMP | BPF_JA):
1263 emit_jmp(st, ins->off + 1);
1264 break;
1265 /* jump IMM instructions */
1266 case (BPF_JMP | BPF_JEQ | BPF_K):
1267 case (BPF_JMP | EBPF_JNE | BPF_K):
1268 case (BPF_JMP | BPF_JGT | BPF_K):
1269 case (BPF_JMP | EBPF_JLT | BPF_K):
1270 case (BPF_JMP | BPF_JGE | BPF_K):
1271 case (BPF_JMP | EBPF_JLE | BPF_K):
1272 case (BPF_JMP | EBPF_JSGT | BPF_K):
1273 case (BPF_JMP | EBPF_JSLT | BPF_K):
1274 case (BPF_JMP | EBPF_JSGE | BPF_K):
1275 case (BPF_JMP | EBPF_JSLE | BPF_K):
1276 case (BPF_JMP | BPF_JSET | BPF_K):
1277 emit_jcc_imm(st, op, dr, ins->imm, ins->off + 1);
1278 break;
1279 /* jump REG instructions */
1280 case (BPF_JMP | BPF_JEQ | BPF_X):
1281 case (BPF_JMP | EBPF_JNE | BPF_X):
1282 case (BPF_JMP | BPF_JGT | BPF_X):
1283 case (BPF_JMP | EBPF_JLT | BPF_X):
1284 case (BPF_JMP | BPF_JGE | BPF_X):
1285 case (BPF_JMP | EBPF_JLE | BPF_X):
1286 case (BPF_JMP | EBPF_JSGT | BPF_X):
1287 case (BPF_JMP | EBPF_JSLT | BPF_X):
1288 case (BPF_JMP | EBPF_JSGE | BPF_X):
1289 case (BPF_JMP | EBPF_JSLE | BPF_X):
1290 case (BPF_JMP | BPF_JSET | BPF_X):
1291 emit_jcc_reg(st, op, sr, dr, ins->off + 1);
1292 break;
1293 /* call instructions */
1294 case (BPF_JMP | EBPF_CALL):
1295 emit_call(st,
1296 (uintptr_t)bpf->prm.xsym[ins->imm].func.val);
1297 break;
1298 /* return instruction */
1299 case (BPF_JMP | EBPF_EXIT):
1300 emit_epilog(st);
1301 break;
1302 default:
1303 RTE_BPF_LOG(ERR,
1304 "%s(%p): invalid opcode %#x at pc: %u;\n",
1305 __func__, bpf, ins->code, i);
1306 return -EINVAL;
1307 }
1308 }
1309
1310 return 0;
1311 }
1312
1313 /*
1314 * produce a native ISA version of the given BPF code.
1315 */
1316 int
1317 bpf_jit_x86(struct rte_bpf *bpf)
1318 {
1319 int32_t rc;
1320 uint32_t i;
1321 size_t sz;
1322 struct bpf_jit_state st;
1323
1324 /* init state */
1325 memset(&st, 0, sizeof(st));
1326 st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0]));
1327 if (st.off == NULL)
1328 return -ENOMEM;
1329
1330 /* fill with fake offsets */
1331 st.exit.off = INT32_MAX;
1332 for (i = 0; i != bpf->prm.nb_ins; i++)
1333 st.off[i] = INT32_MAX;
1334
1335 /*
1336 * dry runs, used to calculate total code size and valid jump offsets.
1337 * stop when we get minimal possible size
1338 */
1339 do {
1340 sz = st.sz;
1341 rc = emit(&st, bpf);
1342 } while (rc == 0 && sz != st.sz);
1343
1344 if (rc == 0) {
1345
1346 /* allocate memory needed */
1347 st.ins = mmap(NULL, st.sz, PROT_READ | PROT_WRITE,
1348 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1349 if (st.ins == MAP_FAILED)
1350 rc = -ENOMEM;
1351 else
1352 /* generate code */
1353 rc = emit(&st, bpf);
1354 }
1355
1356 if (rc == 0 && mprotect(st.ins, st.sz, PROT_READ | PROT_EXEC) != 0)
1357 rc = -ENOMEM;
1358
1359 if (rc != 0)
1360 munmap(st.ins, st.sz);
1361 else {
1362 bpf->jit.func = (void *)st.ins;
1363 bpf->jit.sz = st.sz;
1364 }
1365
1366 free(st.off);
1367 return rc;
1368 }
Ceph