projects / mirror_ubuntu-artful-kernel.git / blame
| Commit | Line | Data |
|---|---|---|
| c10302ef MS |
1 | /* |
| 2 | * BPF Jit compiler for s390. | |
| 3 | * | |
| 4 | * Copyright IBM Corp. 2012 | |
| 5 | * | |
| 6 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> | |
| 7 | */ | |
| 8 | #include <linux/moduleloader.h> | |
| 9 | #include <linux/netdevice.h> | |
| 5303a0fe | 10 | #include <linux/if_vlan.h> |
| c10302ef | 11 | #include <linux/filter.h> |
| aa2d2c73 | 12 | #include <linux/random.h> |
| c9a7afa3 | 13 | #include <linux/init.h> |
| c10302ef | 14 | #include <asm/cacheflush.h> |
| 68d9884d | 15 | #include <asm/facility.h> |
| 0f20822a | 16 | #include <asm/dis.h> |
| c10302ef MS |
17 | |
| 18 | /* | |
| 19 | * Conventions: | |
| 20 | * %r2 = skb pointer | |
| 21 | * %r3 = offset parameter | |
| 22 | * %r4 = scratch register / length parameter | |
| 23 | * %r5 = BPF A accumulator | |
| 24 | * %r8 = return address | |
| 25 | * %r9 = save register for skb pointer | |
| 26 | * %r10 = skb->data | |
| 27 | * %r11 = skb->len - skb->data_len (headlen) | |
| 28 | * %r12 = BPF X accumulator | |
| 29 | * %r13 = literal pool pointer | |
| 30 | * 0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS | |
| 31 | */ | |
| 32 | int bpf_jit_enable __read_mostly; | |
| 33 | ||
| 34 | /* | |
| 35 | * assembly code in arch/x86/net/bpf_jit.S | |
| 36 | */ | |
| 37 | extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; | |
| 38 | extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[]; | |
| 39 | ||
| 40 | struct bpf_jit { | |
| 41 | unsigned int seen; | |
| 42 | u8 *start; | |
| 43 | u8 *prg; | |
| 44 | u8 *mid; | |
| 45 | u8 *lit; | |
| 46 | u8 *end; | |
| 47 | u8 *base_ip; | |
| 48 | u8 *ret0_ip; | |
| 49 | u8 *exit_ip; | |
| 50 | unsigned int off_load_word; | |
| 51 | unsigned int off_load_half; | |
| 52 | unsigned int off_load_byte; | |
| 53 | unsigned int off_load_bmsh; | |
| 54 | unsigned int off_load_iword; | |
| 55 | unsigned int off_load_ihalf; | |
| 56 | unsigned int off_load_ibyte; | |
| 57 | }; | |
| 58 | ||
| 59 | #define BPF_SIZE_MAX 4096 /* Max size for program */ | |
| 60 | ||
| 61 | #define SEEN_DATAREF 1 /* might call external helpers */ | |
| 62 | #define SEEN_XREG 2 /* ebx is used */ | |
| 63 | #define SEEN_MEM 4 /* use mem[] for temporary storage */ | |
| 64 | #define SEEN_RET0 8 /* pc_ret0 points to a valid return 0 */ | |
| 65 | #define SEEN_LITERAL 16 /* code uses literals */ | |
| 66 | #define SEEN_LOAD_WORD 32 /* code uses sk_load_word */ | |
| 67 | #define SEEN_LOAD_HALF 64 /* code uses sk_load_half */ | |
| 68 | #define SEEN_LOAD_BYTE 128 /* code uses sk_load_byte */ | |
| 69 | #define SEEN_LOAD_BMSH 256 /* code uses sk_load_byte_msh */ | |
| 70 | #define SEEN_LOAD_IWORD 512 /* code uses sk_load_word_ind */ | |
| 71 | #define SEEN_LOAD_IHALF 1024 /* code uses sk_load_half_ind */ | |
| 72 | #define SEEN_LOAD_IBYTE 2048 /* code uses sk_load_byte_ind */ | |
| 73 | ||
| 74 | #define EMIT2(op) \ | |
| 75 | ({ \ | |
| 76 | if (jit->prg + 2 <= jit->mid) \ | |
| 77 | *(u16 *) jit->prg = op; \ | |
| 78 | jit->prg += 2; \ | |
| 79 | }) | |
| 80 | ||
| 81 | #define EMIT4(op) \ | |
| 82 | ({ \ | |
| 83 | if (jit->prg + 4 <= jit->mid) \ | |
| 84 | *(u32 *) jit->prg = op; \ | |
| 85 | jit->prg += 4; \ | |
| 86 | }) | |
| 87 | ||
| 88 | #define EMIT4_DISP(op, disp) \ | |
| 89 | ({ \ | |
| 90 | unsigned int __disp = (disp) & 0xfff; \ | |
| 91 | EMIT4(op | __disp); \ | |
| 92 | }) | |
| 93 | ||
| 94 | #define EMIT4_IMM(op, imm) \ | |
| 95 | ({ \ | |
| 96 | unsigned int __imm = (imm) & 0xffff; \ | |
| 97 | EMIT4(op | __imm); \ | |
| 98 | }) | |
| 99 | ||
| 100 | #define EMIT4_PCREL(op, pcrel) \ | |
| 101 | ({ \ | |
| 102 | long __pcrel = ((pcrel) >> 1) & 0xffff; \ | |
| 103 | EMIT4(op | __pcrel); \ | |
| 104 | }) | |
| 105 | ||
| 106 | #define EMIT6(op1, op2) \ | |
| 107 | ({ \ | |
| 108 | if (jit->prg + 6 <= jit->mid) { \ | |
| 109 | *(u32 *) jit->prg = op1; \ | |
| 110 | *(u16 *) (jit->prg + 4) = op2; \ | |
| 111 | } \ | |
| 112 | jit->prg += 6; \ | |
| 113 | }) | |
| 114 | ||
| 115 | #define EMIT6_DISP(op1, op2, disp) \ | |
| 116 | ({ \ | |
| 117 | unsigned int __disp = (disp) & 0xfff; \ | |
| 118 | EMIT6(op1 | __disp, op2); \ | |
| 119 | }) | |
| 120 | ||
| 68d9884d HC |
121 | #define EMIT6_IMM(op, imm) \ |
| 122 | ({ \ | |
| 123 | unsigned int __imm = (imm); \ | |
| 124 | EMIT6(op | (__imm >> 16), __imm & 0xffff); \ | |
| 125 | }) | |
| 126 | ||
| c10302ef MS |
127 | #define EMIT_CONST(val) \ |
| 128 | ({ \ | |
| 129 | unsigned int ret; \ | |
| 130 | ret = (unsigned int) (jit->lit - jit->base_ip); \ | |
| 131 | jit->seen |= SEEN_LITERAL; \ | |
| 132 | if (jit->lit + 4 <= jit->end) \ | |
| 133 | *(u32 *) jit->lit = val; \ | |
| 134 | jit->lit += 4; \ | |
| 135 | ret; \ | |
| 136 | }) | |
| 137 | ||
| 138 | #define EMIT_FN_CONST(bit, fn) \ | |
| 139 | ({ \ | |
| 140 | unsigned int ret; \ | |
| 141 | ret = (unsigned int) (jit->lit - jit->base_ip); \ | |
| 142 | if (jit->seen & bit) { \ | |
| 143 | jit->seen |= SEEN_LITERAL; \ | |
| 144 | if (jit->lit + 8 <= jit->end) \ | |
| 145 | *(void **) jit->lit = fn; \ | |
| 146 | jit->lit += 8; \ | |
| 147 | } \ | |
| 148 | ret; \ | |
| 149 | }) | |
| 150 | ||
| 151 | static void bpf_jit_prologue(struct bpf_jit *jit) | |
| 152 | { | |
| 153 | /* Save registers and create stack frame if necessary */ | |
| 154 | if (jit->seen & SEEN_DATAREF) { | |
| 155 | /* stmg %r8,%r15,88(%r15) */ | |
| 156 | EMIT6(0xeb8ff058, 0x0024); | |
| 157 | /* lgr %r14,%r15 */ | |
| 158 | EMIT4(0xb90400ef); | |
| 6cef3003 MS |
159 | /* aghi %r15,<offset> */ |
| 160 | EMIT4_IMM(0xa7fb0000, (jit->seen & SEEN_MEM) ? -112 : -80); | |
| c10302ef MS |
161 | /* stg %r14,152(%r15) */ |
| 162 | EMIT6(0xe3e0f098, 0x0024); | |
| 163 | } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) | |
| 164 | /* stmg %r12,%r13,120(%r15) */ | |
| 165 | EMIT6(0xebcdf078, 0x0024); | |
| 166 | else if (jit->seen & SEEN_XREG) | |
| 167 | /* stg %r12,120(%r15) */ | |
| 168 | EMIT6(0xe3c0f078, 0x0024); | |
| 169 | else if (jit->seen & SEEN_LITERAL) | |
| 170 | /* stg %r13,128(%r15) */ | |
| 171 | EMIT6(0xe3d0f080, 0x0024); | |
| 172 | ||
| 173 | /* Setup literal pool */ | |
| 174 | if (jit->seen & SEEN_LITERAL) { | |
| 175 | /* basr %r13,0 */ | |
| 176 | EMIT2(0x0dd0); | |
| 177 | jit->base_ip = jit->prg; | |
| 178 | } | |
| 179 | jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word); | |
| 180 | jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half); | |
| 181 | jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte); | |
| 182 | jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh); | |
| 183 | jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind); | |
| 184 | jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind); | |
| 185 | jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind); | |
| 186 | ||
| 187 | /* Filter needs to access skb data */ | |
| 188 | if (jit->seen & SEEN_DATAREF) { | |
| 189 | /* l %r11,<len>(%r2) */ | |
| 190 | EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len)); | |
| 191 | /* s %r11,<data_len>(%r2) */ | |
| 192 | EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len)); | |
| 193 | /* lg %r10,<data>(%r2) */ | |
| 194 | EMIT6_DISP(0xe3a02000, 0x0004, | |
| 195 | offsetof(struct sk_buff, data)); | |
| 196 | } | |
| 197 | } | |
| 198 | ||
| 199 | static void bpf_jit_epilogue(struct bpf_jit *jit) | |
| 200 | { | |
| 201 | /* Return 0 */ | |
| 202 | if (jit->seen & SEEN_RET0) { | |
| 203 | jit->ret0_ip = jit->prg; | |
| 204 | /* lghi %r2,0 */ | |
| 205 | EMIT4(0xa7290000); | |
| 206 | } | |
| 207 | jit->exit_ip = jit->prg; | |
| 208 | /* Restore registers */ | |
| 209 | if (jit->seen & SEEN_DATAREF) | |
| 210 | /* lmg %r8,%r15,<offset>(%r15) */ | |
| 211 | EMIT6_DISP(0xeb8ff000, 0x0004, | |
| 212 | (jit->seen & SEEN_MEM) ? 200 : 168); | |
| 213 | else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) | |
| 214 | /* lmg %r12,%r13,120(%r15) */ | |
| 215 | EMIT6(0xebcdf078, 0x0004); | |
| 216 | else if (jit->seen & SEEN_XREG) | |
| 217 | /* lg %r12,120(%r15) */ | |
| 218 | EMIT6(0xe3c0f078, 0x0004); | |
| 219 | else if (jit->seen & SEEN_LITERAL) | |
| 220 | /* lg %r13,128(%r15) */ | |
| 221 | EMIT6(0xe3d0f080, 0x0004); | |
| 222 | /* br %r14 */ | |
| 223 | EMIT2(0x07fe); | |
| 224 | } | |
| 225 | ||
| c9a7afa3 HC |
226 | /* Helper to find the offset of pkt_type in sk_buff |
| 227 | * Make sure its still a 3bit field starting at the MSBs within a byte. | |
| 228 | */ | |
| 229 | #define PKT_TYPE_MAX 0xe0 | |
| 230 | static int pkt_type_offset; | |
| 231 | ||
| 232 | static int __init bpf_pkt_type_offset_init(void) | |
| 233 | { | |
| 234 | struct sk_buff skb_probe = { | |
| 235 | .pkt_type = ~0, | |
| 236 | }; | |
| 237 | char *ct = (char *)&skb_probe; | |
| 238 | int off; | |
| 239 | ||
| 240 | pkt_type_offset = -1; | |
| 241 | for (off = 0; off < sizeof(struct sk_buff); off++) { | |
| 242 | if (!ct[off]) | |
| 243 | continue; | |
| 244 | if (ct[off] == PKT_TYPE_MAX) | |
| 245 | pkt_type_offset = off; | |
| 246 | else { | |
| 247 | /* Found non matching bit pattern, fix needed. */ | |
| 248 | WARN_ON_ONCE(1); | |
| 249 | pkt_type_offset = -1; | |
| 250 | return -1; | |
| 251 | } | |
| 252 | } | |
| 253 | return 0; | |
| 254 | } | |
| 255 | device_initcall(bpf_pkt_type_offset_init); | |
| 256 | ||
| c10302ef MS |
257 | /* |
| 258 | * make sure we dont leak kernel information to user | |
| 259 | */ | |
| 260 | static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter) | |
| 261 | { | |
| 262 | /* Clear temporary memory if (seen & SEEN_MEM) */ | |
| 263 | if (jit->seen & SEEN_MEM) | |
| 264 | /* xc 0(64,%r15),0(%r15) */ | |
| 265 | EMIT6(0xd73ff000, 0xf000); | |
| 266 | /* Clear X if (seen & SEEN_XREG) */ | |
| 267 | if (jit->seen & SEEN_XREG) | |
| 268 | /* lhi %r12,0 */ | |
| 269 | EMIT4(0xa7c80000); | |
| 270 | /* Clear A if the first register does not set it. */ | |
| 271 | switch (filter[0].code) { | |
| 272 | case BPF_S_LD_W_ABS: | |
| 273 | case BPF_S_LD_H_ABS: | |
| 274 | case BPF_S_LD_B_ABS: | |
| 275 | case BPF_S_LD_W_LEN: | |
| 276 | case BPF_S_LD_W_IND: | |
| 277 | case BPF_S_LD_H_IND: | |
| 278 | case BPF_S_LD_B_IND: | |
| 279 | case BPF_S_LDX_B_MSH: | |
| 280 | case BPF_S_LD_IMM: | |
| 281 | case BPF_S_LD_MEM: | |
| 282 | case BPF_S_MISC_TXA: | |
| 283 | case BPF_S_ANC_PROTOCOL: | |
| 284 | case BPF_S_ANC_PKTTYPE: | |
| 285 | case BPF_S_ANC_IFINDEX: | |
| 286 | case BPF_S_ANC_MARK: | |
| 287 | case BPF_S_ANC_QUEUE: | |
| 288 | case BPF_S_ANC_HATYPE: | |
| 289 | case BPF_S_ANC_RXHASH: | |
| 290 | case BPF_S_ANC_CPU: | |
| 5303a0fe HC |
291 | case BPF_S_ANC_VLAN_TAG: |
| 292 | case BPF_S_ANC_VLAN_TAG_PRESENT: | |
| c10302ef MS |
293 | case BPF_S_RET_K: |
| 294 | /* first instruction sets A register */ | |
| 295 | break; | |
| 296 | default: /* A = 0 */ | |
| 297 | /* lhi %r5,0 */ | |
| 298 | EMIT4(0xa7580000); | |
| 299 | } | |
| 300 | } | |
| 301 | ||
| 302 | static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, | |
| 303 | unsigned int *addrs, int i, int last) | |
| 304 | { | |
| 305 | unsigned int K; | |
| 306 | int offset; | |
| 307 | unsigned int mask; | |
| 308 | ||
| 309 | K = filter->k; | |
| 310 | switch (filter->code) { | |
| 311 | case BPF_S_ALU_ADD_X: /* A += X */ | |
| 312 | jit->seen |= SEEN_XREG; | |
| 313 | /* ar %r5,%r12 */ | |
| 314 | EMIT2(0x1a5c); | |
| 315 | break; | |
| 316 | case BPF_S_ALU_ADD_K: /* A += K */ | |
| 317 | if (!K) | |
| 318 | break; | |
| 319 | if (K <= 16383) | |
| 320 | /* ahi %r5,<K> */ | |
| 321 | EMIT4_IMM(0xa75a0000, K); | |
| 68d9884d HC |
322 | else if (test_facility(21)) |
| 323 | /* alfi %r5,<K> */ | |
| 324 | EMIT6_IMM(0xc25b0000, K); | |
| c10302ef MS |
325 | else |
| 326 | /* a %r5,<d(K)>(%r13) */ | |
| 327 | EMIT4_DISP(0x5a50d000, EMIT_CONST(K)); | |
| 328 | break; | |
| 329 | case BPF_S_ALU_SUB_X: /* A -= X */ | |
| 330 | jit->seen |= SEEN_XREG; | |
| 331 | /* sr %r5,%r12 */ | |
| 332 | EMIT2(0x1b5c); | |
| 333 | break; | |
| 334 | case BPF_S_ALU_SUB_K: /* A -= K */ | |
| 335 | if (!K) | |
| 336 | break; | |
| 337 | if (K <= 16384) | |
| 338 | /* ahi %r5,-K */ | |
| 339 | EMIT4_IMM(0xa75a0000, -K); | |
| 68d9884d HC |
340 | else if (test_facility(21)) |
| 341 | /* alfi %r5,-K */ | |
| 342 | EMIT6_IMM(0xc25b0000, -K); | |
| c10302ef MS |
343 | else |
| 344 | /* s %r5,<d(K)>(%r13) */ | |
| 345 | EMIT4_DISP(0x5b50d000, EMIT_CONST(K)); | |
| 346 | break; | |
| 347 | case BPF_S_ALU_MUL_X: /* A *= X */ | |
| 348 | jit->seen |= SEEN_XREG; | |
| 349 | /* msr %r5,%r12 */ | |
| 350 | EMIT4(0xb252005c); | |
| 351 | break; | |
| 352 | case BPF_S_ALU_MUL_K: /* A *= K */ | |
| 353 | if (K <= 16383) | |
| 354 | /* mhi %r5,K */ | |
| 355 | EMIT4_IMM(0xa75c0000, K); | |
| 68d9884d HC |
356 | else if (test_facility(34)) |
| 357 | /* msfi %r5,<K> */ | |
| 358 | EMIT6_IMM(0xc2510000, K); | |
| c10302ef MS |
359 | else |
| 360 | /* ms %r5,<d(K)>(%r13) */ | |
| 361 | EMIT4_DISP(0x7150d000, EMIT_CONST(K)); | |
| 362 | break; | |
| 363 | case BPF_S_ALU_DIV_X: /* A /= X */ | |
| 364 | jit->seen |= SEEN_XREG | SEEN_RET0; | |
| 365 | /* ltr %r12,%r12 */ | |
| 366 | EMIT2(0x12cc); | |
| 367 | /* jz <ret0> */ | |
| 368 | EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); | |
| 369 | /* lhi %r4,0 */ | |
| 370 | EMIT4(0xa7480000); | |
| 371 | /* dr %r4,%r12 */ | |
| 372 | EMIT2(0x1d4c); | |
| 373 | break; | |
| 374 | case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K) */ | |
| 375 | /* m %r4,<d(K)>(%r13) */ | |
| 376 | EMIT4_DISP(0x5c40d000, EMIT_CONST(K)); | |
| 32472745 HC |
377 | /* lr %r5,%r4 */ |
| 378 | EMIT2(0x1854); | |
| 379 | break; | |
| 380 | case BPF_S_ALU_MOD_X: /* A %= X */ | |
| 381 | jit->seen |= SEEN_XREG | SEEN_RET0; | |
| 382 | /* ltr %r12,%r12 */ | |
| 383 | EMIT2(0x12cc); | |
| 384 | /* jz <ret0> */ | |
| 385 | EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); | |
| 386 | /* lhi %r4,0 */ | |
| 387 | EMIT4(0xa7480000); | |
| 388 | /* dr %r4,%r12 */ | |
| 389 | EMIT2(0x1d4c); | |
| 390 | /* lr %r5,%r4 */ | |
| 391 | EMIT2(0x1854); | |
| 392 | break; | |
| 393 | case BPF_S_ALU_MOD_K: /* A %= K */ | |
| 394 | /* lhi %r4,0 */ | |
| 395 | EMIT4(0xa7480000); | |
| 396 | /* d %r4,<d(K)>(%r13) */ | |
| 397 | EMIT4_DISP(0x5d40d000, EMIT_CONST(K)); | |
| c10302ef MS |
398 | /* lr %r5,%r4 */ |
| 399 | EMIT2(0x1854); | |
| 400 | break; | |
| 401 | case BPF_S_ALU_AND_X: /* A &= X */ | |
| 402 | jit->seen |= SEEN_XREG; | |
| 403 | /* nr %r5,%r12 */ | |
| 404 | EMIT2(0x145c); | |
| 405 | break; | |
| 406 | case BPF_S_ALU_AND_K: /* A &= K */ | |
| 68d9884d HC |
407 | if (test_facility(21)) |
| 408 | /* nilf %r5,<K> */ | |
| 409 | EMIT6_IMM(0xc05b0000, K); | |
| 410 | else | |
| 411 | /* n %r5,<d(K)>(%r13) */ | |
| 412 | EMIT4_DISP(0x5450d000, EMIT_CONST(K)); | |
| c10302ef MS |
413 | break; |
| 414 | case BPF_S_ALU_OR_X: /* A |= X */ | |
| 415 | jit->seen |= SEEN_XREG; | |
| 416 | /* or %r5,%r12 */ | |
| 417 | EMIT2(0x165c); | |
| 418 | break; | |
| 419 | case BPF_S_ALU_OR_K: /* A |= K */ | |
| 68d9884d HC |
420 | if (test_facility(21)) |
| 421 | /* oilf %r5,<K> */ | |
| 422 | EMIT6_IMM(0xc05d0000, K); | |
| 423 | else | |
| 424 | /* o %r5,<d(K)>(%r13) */ | |
| 425 | EMIT4_DISP(0x5650d000, EMIT_CONST(K)); | |
| c10302ef | 426 | break; |
| c59eed11 | 427 | case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */ |
| 916908df | 428 | case BPF_S_ALU_XOR_X: |
| c59eed11 HC |
429 | jit->seen |= SEEN_XREG; |
| 430 | /* xr %r5,%r12 */ | |
| 431 | EMIT2(0x175c); | |
| 432 | break; | |
| 916908df HC |
433 | case BPF_S_ALU_XOR_K: /* A ^= K */ |
| 434 | if (!K) | |
| 435 | break; | |
| 436 | /* x %r5,<d(K)>(%r13) */ | |
| 437 | EMIT4_DISP(0x5750d000, EMIT_CONST(K)); | |
| 438 | break; | |
| c10302ef MS |
439 | case BPF_S_ALU_LSH_X: /* A <<= X; */ |
| 440 | jit->seen |= SEEN_XREG; | |
| 441 | /* sll %r5,0(%r12) */ | |
| 442 | EMIT4(0x8950c000); | |
| 443 | break; | |
| 444 | case BPF_S_ALU_LSH_K: /* A <<= K */ | |
| 445 | if (K == 0) | |
| 446 | break; | |
| 447 | /* sll %r5,K */ | |
| 448 | EMIT4_DISP(0x89500000, K); | |
| 449 | break; | |
| 450 | case BPF_S_ALU_RSH_X: /* A >>= X; */ | |
| 451 | jit->seen |= SEEN_XREG; | |
| 452 | /* srl %r5,0(%r12) */ | |
| 453 | EMIT4(0x8850c000); | |
| 454 | break; | |
| 455 | case BPF_S_ALU_RSH_K: /* A >>= K; */ | |
| 456 | if (K == 0) | |
| 457 | break; | |
| 458 | /* srl %r5,K */ | |
| 459 | EMIT4_DISP(0x88500000, K); | |
| 460 | break; | |
| 461 | case BPF_S_ALU_NEG: /* A = -A */ | |
| 462 | /* lnr %r5,%r5 */ | |
| 463 | EMIT2(0x1155); | |
| 464 | break; | |
| 465 | case BPF_S_JMP_JA: /* ip += K */ | |
| 466 | offset = addrs[i + K] + jit->start - jit->prg; | |
| 467 | EMIT4_PCREL(0xa7f40000, offset); | |
| 468 | break; | |
| 469 | case BPF_S_JMP_JGT_K: /* ip += (A > K) ? jt : jf */ | |
| 470 | mask = 0x200000; /* jh */ | |
| 471 | goto kbranch; | |
| 472 | case BPF_S_JMP_JGE_K: /* ip += (A >= K) ? jt : jf */ | |
| 473 | mask = 0xa00000; /* jhe */ | |
| 474 | goto kbranch; | |
| 475 | case BPF_S_JMP_JEQ_K: /* ip += (A == K) ? jt : jf */ | |
| 476 | mask = 0x800000; /* je */ | |
| 477 | kbranch: /* Emit compare if the branch targets are different */ | |
| 478 | if (filter->jt != filter->jf) { | |
| 479 | if (K <= 16383) | |
| 480 | /* chi %r5,<K> */ | |
| 481 | EMIT4_IMM(0xa75e0000, K); | |
| 68d9884d HC |
482 | else if (test_facility(21)) |
| 483 | /* clfi %r5,<K> */ | |
| 484 | EMIT6_IMM(0xc25f0000, K); | |
| c10302ef MS |
485 | else |
| 486 | /* c %r5,<d(K)>(%r13) */ | |
| 487 | EMIT4_DISP(0x5950d000, EMIT_CONST(K)); | |
| 488 | } | |
| 489 | branch: if (filter->jt == filter->jf) { | |
| 490 | if (filter->jt == 0) | |
| 491 | break; | |
| 492 | /* j <jt> */ | |
| 493 | offset = addrs[i + filter->jt] + jit->start - jit->prg; | |
| 494 | EMIT4_PCREL(0xa7f40000, offset); | |
| 495 | break; | |
| 496 | } | |
| 497 | if (filter->jt != 0) { | |
| 498 | /* brc <mask>,<jt> */ | |
| 499 | offset = addrs[i + filter->jt] + jit->start - jit->prg; | |
| 500 | EMIT4_PCREL(0xa7040000 | mask, offset); | |
| 501 | } | |
| 502 | if (filter->jf != 0) { | |
| 503 | /* brc <mask^15>,<jf> */ | |
| 504 | offset = addrs[i + filter->jf] + jit->start - jit->prg; | |
| 505 | EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset); | |
| 506 | } | |
| 507 | break; | |
| 508 | case BPF_S_JMP_JSET_K: /* ip += (A & K) ? jt : jf */ | |
| 509 | mask = 0x700000; /* jnz */ | |
| 510 | /* Emit test if the branch targets are different */ | |
| 511 | if (filter->jt != filter->jf) { | |
| 512 | if (K > 65535) { | |
| 513 | /* lr %r4,%r5 */ | |
| 514 | EMIT2(0x1845); | |
| 515 | /* n %r4,<d(K)>(%r13) */ | |
| 516 | EMIT4_DISP(0x5440d000, EMIT_CONST(K)); | |
| 517 | } else | |
| 518 | /* tmll %r5,K */ | |
| 519 | EMIT4_IMM(0xa7510000, K); | |
| 520 | } | |
| 521 | goto branch; | |
| 522 | case BPF_S_JMP_JGT_X: /* ip += (A > X) ? jt : jf */ | |
| 523 | mask = 0x200000; /* jh */ | |
| 524 | goto xbranch; | |
| 525 | case BPF_S_JMP_JGE_X: /* ip += (A >= X) ? jt : jf */ | |
| 526 | mask = 0xa00000; /* jhe */ | |
| 527 | goto xbranch; | |
| 528 | case BPF_S_JMP_JEQ_X: /* ip += (A == X) ? jt : jf */ | |
| 529 | mask = 0x800000; /* je */ | |
| 530 | xbranch: /* Emit compare if the branch targets are different */ | |
| 531 | if (filter->jt != filter->jf) { | |
| 532 | jit->seen |= SEEN_XREG; | |
| 533 | /* cr %r5,%r12 */ | |
| 534 | EMIT2(0x195c); | |
| 535 | } | |
| 536 | goto branch; | |
| 537 | case BPF_S_JMP_JSET_X: /* ip += (A & X) ? jt : jf */ | |
| 538 | mask = 0x700000; /* jnz */ | |
| 539 | /* Emit test if the branch targets are different */ | |
| 540 | if (filter->jt != filter->jf) { | |
| 541 | jit->seen |= SEEN_XREG; | |
| 542 | /* lr %r4,%r5 */ | |
| 543 | EMIT2(0x1845); | |
| 544 | /* nr %r4,%r12 */ | |
| 545 | EMIT2(0x144c); | |
| 546 | } | |
| 547 | goto branch; | |
| 548 | case BPF_S_LD_W_ABS: /* A = *(u32 *) (skb->data+K) */ | |
| 549 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD; | |
| 550 | offset = jit->off_load_word; | |
| 551 | goto load_abs; | |
| 552 | case BPF_S_LD_H_ABS: /* A = *(u16 *) (skb->data+K) */ | |
| 553 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF; | |
| 554 | offset = jit->off_load_half; | |
| 555 | goto load_abs; | |
| 556 | case BPF_S_LD_B_ABS: /* A = *(u8 *) (skb->data+K) */ | |
| 557 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE; | |
| 558 | offset = jit->off_load_byte; | |
| 559 | load_abs: if ((int) K < 0) | |
| 560 | goto out; | |
| 561 | call_fn: /* lg %r1,<d(function)>(%r13) */ | |
| 562 | EMIT6_DISP(0xe310d000, 0x0004, offset); | |
| 563 | /* l %r3,<d(K)>(%r13) */ | |
| 564 | EMIT4_DISP(0x5830d000, EMIT_CONST(K)); | |
| 565 | /* basr %r8,%r1 */ | |
| 566 | EMIT2(0x0d81); | |
| 567 | /* jnz <ret0> */ | |
| 568 | EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg)); | |
| 569 | break; | |
| 570 | case BPF_S_LD_W_IND: /* A = *(u32 *) (skb->data+K+X) */ | |
| 571 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD; | |
| 572 | offset = jit->off_load_iword; | |
| 573 | goto call_fn; | |
| 574 | case BPF_S_LD_H_IND: /* A = *(u16 *) (skb->data+K+X) */ | |
| 575 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF; | |
| 576 | offset = jit->off_load_ihalf; | |
| 577 | goto call_fn; | |
| 578 | case BPF_S_LD_B_IND: /* A = *(u8 *) (skb->data+K+X) */ | |
| 579 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE; | |
| 580 | offset = jit->off_load_ibyte; | |
| 581 | goto call_fn; | |
| 582 | case BPF_S_LDX_B_MSH: | |
| 583 | /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ | |
| 584 | jit->seen |= SEEN_RET0; | |
| 585 | if ((int) K < 0) { | |
| 586 | /* j <ret0> */ | |
| 587 | EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg)); | |
| 588 | break; | |
| 589 | } | |
| 590 | jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH; | |
| 591 | offset = jit->off_load_bmsh; | |
| 592 | goto call_fn; | |
| 593 | case BPF_S_LD_W_LEN: /* A = skb->len; */ | |
| 594 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); | |
| 595 | /* l %r5,<d(len)>(%r2) */ | |
| 596 | EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len)); | |
| 597 | break; | |
| 598 | case BPF_S_LDX_W_LEN: /* X = skb->len; */ | |
| 599 | jit->seen |= SEEN_XREG; | |
| 600 | /* l %r12,<d(len)>(%r2) */ | |
| 601 | EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len)); | |
| 602 | break; | |
| 603 | case BPF_S_LD_IMM: /* A = K */ | |
| 604 | if (K <= 16383) | |
| 605 | /* lhi %r5,K */ | |
| 606 | EMIT4_IMM(0xa7580000, K); | |
| 68d9884d HC |
607 | else if (test_facility(21)) |
| 608 | /* llilf %r5,<K> */ | |
| 609 | EMIT6_IMM(0xc05f0000, K); | |
| c10302ef MS |
610 | else |
| 611 | /* l %r5,<d(K)>(%r13) */ | |
| 612 | EMIT4_DISP(0x5850d000, EMIT_CONST(K)); | |
| 613 | break; | |
| 614 | case BPF_S_LDX_IMM: /* X = K */ | |
| 615 | jit->seen |= SEEN_XREG; | |
| 616 | if (K <= 16383) | |
| 617 | /* lhi %r12,<K> */ | |
| 618 | EMIT4_IMM(0xa7c80000, K); | |
| 68d9884d HC |
619 | else if (test_facility(21)) |
| 620 | /* llilf %r12,<K> */ | |
| 621 | EMIT6_IMM(0xc0cf0000, K); | |
| c10302ef MS |
622 | else |
| 623 | /* l %r12,<d(K)>(%r13) */ | |
| 624 | EMIT4_DISP(0x58c0d000, EMIT_CONST(K)); | |
| 625 | break; | |
| 626 | case BPF_S_LD_MEM: /* A = mem[K] */ | |
| 627 | jit->seen |= SEEN_MEM; | |
| 628 | /* l %r5,<K>(%r15) */ | |
| 629 | EMIT4_DISP(0x5850f000, | |
| 630 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | |
| 631 | break; | |
| 632 | case BPF_S_LDX_MEM: /* X = mem[K] */ | |
| 633 | jit->seen |= SEEN_XREG | SEEN_MEM; | |
| 634 | /* l %r12,<K>(%r15) */ | |
| 635 | EMIT4_DISP(0x58c0f000, | |
| 636 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | |
| 637 | break; | |
| 638 | case BPF_S_MISC_TAX: /* X = A */ | |
| 639 | jit->seen |= SEEN_XREG; | |
| 640 | /* lr %r12,%r5 */ | |
| 641 | EMIT2(0x18c5); | |
| 642 | break; | |
| 643 | case BPF_S_MISC_TXA: /* A = X */ | |
| 644 | jit->seen |= SEEN_XREG; | |
| 645 | /* lr %r5,%r12 */ | |
| 646 | EMIT2(0x185c); | |
| 647 | break; | |
| 648 | case BPF_S_RET_K: | |
| 649 | if (K == 0) { | |
| 650 | jit->seen |= SEEN_RET0; | |
| 651 | if (last) | |
| 652 | break; | |
| 653 | /* j <ret0> */ | |
| 654 | EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg); | |
| 655 | } else { | |
| 656 | if (K <= 16383) | |
| 657 | /* lghi %r2,K */ | |
| 658 | EMIT4_IMM(0xa7290000, K); | |
| 659 | else | |
| 660 | /* llgf %r2,<K>(%r13) */ | |
| 661 | EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K)); | |
| 662 | /* j <exit> */ | |
| 663 | if (last && !(jit->seen & SEEN_RET0)) | |
| 664 | break; | |
| 665 | EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); | |
| 666 | } | |
| 667 | break; | |
| 668 | case BPF_S_RET_A: | |
| 669 | /* llgfr %r2,%r5 */ | |
| 670 | EMIT4(0xb9160025); | |
| 671 | /* j <exit> */ | |
| 672 | EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); | |
| 673 | break; | |
| 674 | case BPF_S_ST: /* mem[K] = A */ | |
| 675 | jit->seen |= SEEN_MEM; | |
| 676 | /* st %r5,<K>(%r15) */ | |
| 677 | EMIT4_DISP(0x5050f000, | |
| 678 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | |
| 679 | break; | |
| 680 | case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */ | |
| 681 | jit->seen |= SEEN_XREG | SEEN_MEM; | |
| 682 | /* st %r12,<K>(%r15) */ | |
| 683 | EMIT4_DISP(0x50c0f000, | |
| 684 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | |
| 685 | break; | |
| 686 | case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */ | |
| 687 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); | |
| 688 | /* lhi %r5,0 */ | |
| 689 | EMIT4(0xa7580000); | |
| 690 | /* icm %r5,3,<d(protocol)>(%r2) */ | |
| 691 | EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol)); | |
| 692 | break; | |
| 693 | case BPF_S_ANC_IFINDEX: /* if (!skb->dev) return 0; | |
| 694 | * A = skb->dev->ifindex */ | |
| 695 | BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); | |
| 696 | jit->seen |= SEEN_RET0; | |
| 697 | /* lg %r1,<d(dev)>(%r2) */ | |
| 698 | EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); | |
| 699 | /* ltgr %r1,%r1 */ | |
| 700 | EMIT4(0xb9020011); | |
| 701 | /* jz <ret0> */ | |
| 702 | EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); | |
| 703 | /* l %r5,<d(ifindex)>(%r1) */ | |
| 704 | EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex)); | |
| 705 | break; | |
| 706 | case BPF_S_ANC_MARK: /* A = skb->mark */ | |
| 707 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); | |
| 708 | /* l %r5,<d(mark)>(%r2) */ | |
| 709 | EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark)); | |
| 710 | break; | |
| 711 | case BPF_S_ANC_QUEUE: /* A = skb->queue_mapping */ | |
| 712 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); | |
| 713 | /* lhi %r5,0 */ | |
| 714 | EMIT4(0xa7580000); | |
| 715 | /* icm %r5,3,<d(queue_mapping)>(%r2) */ | |
| 716 | EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping)); | |
| 717 | break; | |
| 718 | case BPF_S_ANC_HATYPE: /* if (!skb->dev) return 0; | |
| 719 | * A = skb->dev->type */ | |
| 720 | BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); | |
| 721 | jit->seen |= SEEN_RET0; | |
| 722 | /* lg %r1,<d(dev)>(%r2) */ | |
| 723 | EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); | |
| 724 | /* ltgr %r1,%r1 */ | |
| 725 | EMIT4(0xb9020011); | |
| 726 | /* jz <ret0> */ | |
| 727 | EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); | |
| 728 | /* lhi %r5,0 */ | |
| 729 | EMIT4(0xa7580000); | |
| 730 | /* icm %r5,3,<d(type)>(%r1) */ | |
| 731 | EMIT4_DISP(0xbf531000, offsetof(struct net_device, type)); | |
| 732 | break; | |
| 733 | case BPF_S_ANC_RXHASH: /* A = skb->rxhash */ | |
| 734 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4); | |
| 735 | /* l %r5,<d(rxhash)>(%r2) */ | |
| 736 | EMIT4_DISP(0x58502000, offsetof(struct sk_buff, rxhash)); | |
| 737 | break; | |
| 5303a0fe HC |
738 | case BPF_S_ANC_VLAN_TAG: |
| 739 | case BPF_S_ANC_VLAN_TAG_PRESENT: | |
| 740 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); | |
| 741 | BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000); | |
| 742 | /* lhi %r5,0 */ | |
| 743 | EMIT4(0xa7580000); | |
| 744 | /* icm %r5,3,<d(vlan_tci)>(%r2) */ | |
| 745 | EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, vlan_tci)); | |
| 746 | if (filter->code == BPF_S_ANC_VLAN_TAG) { | |
| 747 | /* nill %r5,0xefff */ | |
| 748 | EMIT4_IMM(0xa5570000, ~VLAN_TAG_PRESENT); | |
| 749 | } else { | |
| 750 | /* nill %r5,0x1000 */ | |
| 751 | EMIT4_IMM(0xa5570000, VLAN_TAG_PRESENT); | |
| 752 | /* srl %r5,12 */ | |
| 753 | EMIT4_DISP(0x88500000, 12); | |
| 754 | } | |
| 755 | break; | |
| c9a7afa3 HC |
756 | case BPF_S_ANC_PKTTYPE: |
| 757 | if (pkt_type_offset < 0) | |
| 758 | goto out; | |
| 759 | /* lhi %r5,0 */ | |
| 760 | EMIT4(0xa7580000); | |
| 761 | /* ic %r5,<d(pkt_type_offset)>(%r2) */ | |
| 762 | EMIT4_DISP(0x43502000, pkt_type_offset); | |
| 763 | /* srl %r5,5 */ | |
| 764 | EMIT4_DISP(0x88500000, 5); | |
| 765 | break; | |
| c10302ef MS |
766 | case BPF_S_ANC_CPU: /* A = smp_processor_id() */ |
| 767 | #ifdef CONFIG_SMP | |
| 768 | /* l %r5,<d(cpu_nr)> */ | |
| 769 | EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr)); | |
| 770 | #else | |
| 771 | /* lhi %r5,0 */ | |
| 772 | EMIT4(0xa7580000); | |
| 773 | #endif | |
| 774 | break; | |
| 775 | default: /* too complex, give up */ | |
| 776 | goto out; | |
| 777 | } | |
| 778 | addrs[i] = jit->prg - jit->start; | |
| 779 | return 0; | |
| 780 | out: | |
| 781 | return -1; | |
| 782 | } | |
| 783 | ||
| aa2d2c73 HC |
784 | /* |
| 785 | * Note: for security reasons, bpf code will follow a randomly | |
| 786 | * sized amount of illegal instructions. | |
| 787 | */ | |
| 788 | struct bpf_binary_header { | |
| 789 | unsigned int pages; | |
| 790 | u8 image[]; | |
| 791 | }; | |
| 792 | ||
| 793 | static struct bpf_binary_header *bpf_alloc_binary(unsigned int bpfsize, | |
| 794 | u8 **image_ptr) | |
| 795 | { | |
| 796 | struct bpf_binary_header *header; | |
| 797 | unsigned int sz, hole; | |
| 798 | ||
| 799 | /* Most BPF filters are really small, but if some of them fill a page, | |
| 800 | * allow at least 128 extra bytes for illegal instructions. | |
| 801 | */ | |
| 802 | sz = round_up(bpfsize + sizeof(*header) + 128, PAGE_SIZE); | |
| 803 | header = module_alloc(sz); | |
| 804 | if (!header) | |
| 805 | return NULL; | |
| 806 | memset(header, 0, sz); | |
| 807 | header->pages = sz / PAGE_SIZE; | |
| 4784955a | 808 | hole = sz - (bpfsize + sizeof(*header)); |
| aa2d2c73 HC |
809 | /* Insert random number of illegal instructions before BPF code |
| 810 | * and make sure the first instruction starts at an even address. | |
| 811 | */ | |
| 812 | *image_ptr = &header->image[(prandom_u32() % hole) & -2]; | |
| 813 | return header; | |
| 814 | } | |
| 815 | ||
| c10302ef MS |
816 | void bpf_jit_compile(struct sk_filter *fp) |
| 817 | { | |
| aa2d2c73 | 818 | struct bpf_binary_header *header = NULL; |
| c10302ef MS |
819 | unsigned long size, prg_len, lit_len; |
| 820 | struct bpf_jit jit, cjit; | |
| 821 | unsigned int *addrs; | |
| 822 | int pass, i; | |
| 823 | ||
| 824 | if (!bpf_jit_enable) | |
| 825 | return; | |
| 3d04fea5 | 826 | addrs = kcalloc(fp->len, sizeof(*addrs), GFP_KERNEL); |
| c10302ef MS |
827 | if (addrs == NULL) |
| 828 | return; | |
| c10302ef MS |
829 | memset(&jit, 0, sizeof(cjit)); |
| 830 | memset(&cjit, 0, sizeof(cjit)); | |
| 831 | ||
| 832 | for (pass = 0; pass < 10; pass++) { | |
| 833 | jit.prg = jit.start; | |
| 834 | jit.lit = jit.mid; | |
| 835 | ||
| 836 | bpf_jit_prologue(&jit); | |
| 837 | bpf_jit_noleaks(&jit, fp->insns); | |
| 838 | for (i = 0; i < fp->len; i++) { | |
| 839 | if (bpf_jit_insn(&jit, fp->insns + i, addrs, i, | |
| 840 | i == fp->len - 1)) | |
| 841 | goto out; | |
| 842 | } | |
| 843 | bpf_jit_epilogue(&jit); | |
| 844 | if (jit.start) { | |
| 845 | WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit); | |
| 846 | if (memcmp(&jit, &cjit, sizeof(jit)) == 0) | |
| 847 | break; | |
| 848 | } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) { | |
| 849 | prg_len = jit.prg - jit.start; | |
| 850 | lit_len = jit.lit - jit.mid; | |
| 1eeb7478 | 851 | size = prg_len + lit_len; |
| c10302ef MS |
852 | if (size >= BPF_SIZE_MAX) |
| 853 | goto out; | |
| aa2d2c73 HC |
854 | header = bpf_alloc_binary(size, &jit.start); |
| 855 | if (!header) | |
| c10302ef MS |
856 | goto out; |
| 857 | jit.prg = jit.mid = jit.start + prg_len; | |
| 858 | jit.lit = jit.end = jit.start + prg_len + lit_len; | |
| 859 | jit.base_ip += (unsigned long) jit.start; | |
| 860 | jit.exit_ip += (unsigned long) jit.start; | |
| 861 | jit.ret0_ip += (unsigned long) jit.start; | |
| 862 | } | |
| 863 | cjit = jit; | |
| 864 | } | |
| 865 | if (bpf_jit_enable > 1) { | |
| fee1b548 HC |
866 | bpf_jit_dump(fp->len, jit.end - jit.start, pass, jit.start); |
| 867 | if (jit.start) | |
| c10302ef | 868 | print_fn_code(jit.start, jit.mid - jit.start); |
| c10302ef | 869 | } |
| aa2d2c73 HC |
870 | if (jit.start) { |
| 871 | set_memory_ro((unsigned long)header, header->pages); | |
| c10302ef | 872 | fp->bpf_func = (void *) jit.start; |
| aa2d2c73 | 873 | } |
| c10302ef MS |
874 | out: |
| 875 | kfree(addrs); | |
| 876 | } | |
| 877 | ||
| c10302ef MS |
878 | void bpf_jit_free(struct sk_filter *fp) |
| 879 | { | |
| aa2d2c73 HC |
880 | unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; |
| 881 | struct bpf_binary_header *header = (void *)addr; | |
| 882 | ||
| 883 | if (fp->bpf_func == sk_run_filter) | |
| d45ed4a4 | 884 | goto free_filter; |
| aa2d2c73 HC |
885 | set_memory_rw(addr, header->pages); |
| 886 | module_free(NULL, header); | |
| d45ed4a4 AS |
887 | free_filter: |
| 888 | kfree(fp); | |
| c10302ef | 889 | } |