arch/x86/net/bpf_jit.S

   1 /* bpf_jit.S : BPF JIT helper functions
   2  *
   3  * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; version 2
   8  * of the License.
   9  */
  10 #include <linux/linkage.h>
  11 #include <asm/frame.h>
  12
  13 /*
  14  * Calling convention :
  15  * rbx : skb pointer (callee saved)
  16  * esi : offset of byte(s) to fetch in skb (can be scratched)
  17  * r10 : copy of skb->data
  18  * r9d : hlen = skb->len - skb->data_len
  19  */
  20 #define SKBDATA %r10
  21 #define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
  22 #define MAX_BPF_STACK (512 /* from filter.h */ + \
  23         32 /* space for rbx,r13,r14,r15 */ + \
  24         8 /* space for skb_copy_bits */)
  25
  26 #define FUNC(name) \
  27         .globl name; \
  28         .type name, @function; \
  29         name:
  30
  31 FUNC(sk_load_word)
  32         test    %esi,%esi
  33         js      bpf_slow_path_word_neg
  34
  35 FUNC(sk_load_word_positive_offset)
  36         mov     %r9d,%eax               # hlen
  37         sub     %esi,%eax               # hlen - offset
  38         cmp     $3,%eax
  39         jle     bpf_slow_path_word
  40         mov     (SKBDATA,%rsi),%eax
  41         bswap   %eax                    /* ntohl() */
  42         ret
  43
  44 FUNC(sk_load_half)
  45         test    %esi,%esi
  46         js      bpf_slow_path_half_neg
  47
  48 FUNC(sk_load_half_positive_offset)
  49         mov     %r9d,%eax
  50         sub     %esi,%eax               #       hlen - offset
  51         cmp     $1,%eax
  52         jle     bpf_slow_path_half
  53         movzwl  (SKBDATA,%rsi),%eax
  54         rol     $8,%ax                  # ntohs()
  55         ret
  56
  57 FUNC(sk_load_byte)
  58         test    %esi,%esi
  59         js      bpf_slow_path_byte_neg
  60
  61 FUNC(sk_load_byte_positive_offset)
  62         cmp     %esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
  63         jle     bpf_slow_path_byte
  64         movzbl  (SKBDATA,%rsi),%eax
  65         ret
  66
  67 /* rsi contains offset and can be scratched */
  68 #define bpf_slow_path_common(LEN)               \
  69         lea     -MAX_BPF_STACK + 32(%rbp), %rdx;\
  70         FRAME_BEGIN;                            \
  71         mov     %rbx, %rdi; /* arg1 == skb */   \
  72         push    %r9;                            \
  73         push    SKBDATA;                        \
  74 /* rsi already has offset */                    \
  75         mov     $LEN,%ecx;      /* len */       \
  76         call    skb_copy_bits;                  \
  77         test    %eax,%eax;                      \
  78         pop     SKBDATA;                        \
  79         pop     %r9;                            \
  80         FRAME_END
  81
  82
  83 bpf_slow_path_word:
  84         bpf_slow_path_common(4)
  85         js      bpf_error
  86         mov     - MAX_BPF_STACK + 32(%rbp),%eax
  87         bswap   %eax
  88         ret
  89
  90 bpf_slow_path_half:
  91         bpf_slow_path_common(2)
  92         js      bpf_error
  93         mov     - MAX_BPF_STACK + 32(%rbp),%ax
  94         rol     $8,%ax
  95         movzwl  %ax,%eax
  96         ret
  97
  98 bpf_slow_path_byte:
  99         bpf_slow_path_common(1)
 100         js      bpf_error
 101         movzbl  - MAX_BPF_STACK + 32(%rbp),%eax
 102         ret
 103
 104 #define sk_negative_common(SIZE)                                \
 105         FRAME_BEGIN;                                            \
 106         mov     %rbx, %rdi; /* arg1 == skb */                   \
 107         push    %r9;                                            \
 108         push    SKBDATA;                                        \
 109 /* rsi already has offset */                                    \
 110         mov     $SIZE,%edx;     /* size */                      \
 111         call    bpf_internal_load_pointer_neg_helper;           \
 112         test    %rax,%rax;                                      \
 113         pop     SKBDATA;                                        \
 114         pop     %r9;                                            \
 115         FRAME_END;                                              \
 116         jz      bpf_error
 117
 118 bpf_slow_path_word_neg:
 119         cmp     SKF_MAX_NEG_OFF, %esi   /* test range */
 120         jl      bpf_error       /* offset lower -> error  */
 121
 122 FUNC(sk_load_word_negative_offset)
 123         sk_negative_common(4)
 124         mov     (%rax), %eax
 125         bswap   %eax
 126         ret
 127
 128 bpf_slow_path_half_neg:
 129         cmp     SKF_MAX_NEG_OFF, %esi
 130         jl      bpf_error
 131
 132 FUNC(sk_load_half_negative_offset)
 133         sk_negative_common(2)
 134         mov     (%rax),%ax
 135         rol     $8,%ax
 136         movzwl  %ax,%eax
 137         ret
 138
 139 bpf_slow_path_byte_neg:
 140         cmp     SKF_MAX_NEG_OFF, %esi
 141         jl      bpf_error
 142
 143 FUNC(sk_load_byte_negative_offset)
 144         sk_negative_common(1)
 145         movzbl  (%rax), %eax
 146         ret
 147
 148 bpf_error:
 149 # force a return 0 from jit handler
 150         xor     %eax,%eax
 151         mov     - MAX_BPF_STACK(%rbp),%rbx
 152         mov     - MAX_BPF_STACK + 8(%rbp),%r13
 153         mov     - MAX_BPF_STACK + 16(%rbp),%r14
 154         mov     - MAX_BPF_STACK + 24(%rbp),%r15
 155         leaveq
 156         ret