arch/powerpc/net/bpf_jit_asm64.S

   1 /*
   2  * bpf_jit_asm64.S: Packet/header access helper functions
   3  * for PPC64 BPF compiler.
   4  *
   5  * Copyright 2016, Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
   6  *                 IBM Corporation
   7  *
   8  * Based on bpf_jit_asm.S by Matt Evans
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public License
  12  * as published by the Free Software Foundation; version 2
  13  * of the License.
  14  */
  15
  16 #include <asm/ppc_asm.h>
  17 #include <asm/ptrace.h>
  18 #include "bpf_jit64.h"
  19
  20 /*
  21  * All of these routines are called directly from generated code,
  22  * with the below register usage:
  23  * r27          skb pointer (ctx)
  24  * r25          skb header length
  25  * r26          skb->data pointer
  26  * r4           offset
  27  *
  28  * Result is passed back in:
  29  * r8           data read in host endian format (accumulator)
  30  *
  31  * r9 is used as a temporary register
  32  */
  33
  34 #define r_skb   r27
  35 #define r_hlen  r25
  36 #define r_data  r26
  37 #define r_off   r4
  38 #define r_val   r8
  39 #define r_tmp   r9
  40
  41 _GLOBAL_TOC(sk_load_word)
  42         cmpdi   r_off, 0
  43         blt     bpf_slow_path_word_neg
  44         b       sk_load_word_positive_offset
  45
  46 _GLOBAL_TOC(sk_load_word_positive_offset)
  47         /* Are we accessing past headlen? */
  48         subi    r_tmp, r_hlen, 4
  49         cmpd    r_tmp, r_off
  50         blt     bpf_slow_path_word
  51         /* Nope, just hitting the header.  cr0 here is eq or gt! */
  52         LWZX_BE r_val, r_data, r_off
  53         blr     /* Return success, cr0 != LT */
  54
  55 _GLOBAL_TOC(sk_load_half)
  56         cmpdi   r_off, 0
  57         blt     bpf_slow_path_half_neg
  58         b       sk_load_half_positive_offset
  59
  60 _GLOBAL_TOC(sk_load_half_positive_offset)
  61         subi    r_tmp, r_hlen, 2
  62         cmpd    r_tmp, r_off
  63         blt     bpf_slow_path_half
  64         LHZX_BE r_val, r_data, r_off
  65         blr
  66
  67 _GLOBAL_TOC(sk_load_byte)
  68         cmpdi   r_off, 0
  69         blt     bpf_slow_path_byte_neg
  70         b       sk_load_byte_positive_offset
  71
  72 _GLOBAL_TOC(sk_load_byte_positive_offset)
  73         cmpd    r_hlen, r_off
  74         ble     bpf_slow_path_byte
  75         lbzx    r_val, r_data, r_off
  76         blr
  77
  78 /*
  79  * Call out to skb_copy_bits:
  80  * Allocate a new stack frame here to remain ABI-compliant in
  81  * stashing LR.
  82  */
  83 #define bpf_slow_path_common(SIZE)                                      \
  84         mflr    r0;                                                     \
  85         std     r0, PPC_LR_STKOFF(r1);                                  \
  86         stdu    r1, -(STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS)(r1); \
  87         mr      r3, r_skb;                                              \
  88         /* r4 = r_off as passed */                                      \
  89         addi    r5, r1, STACK_FRAME_MIN_SIZE;                           \
  90         li      r6, SIZE;                                               \
  91         bl      skb_copy_bits;                                          \
  92         nop;                                                            \
  93         /* save r5 */                                                   \
  94         addi    r5, r1, STACK_FRAME_MIN_SIZE;                           \
  95         /* r3 = 0 on success */                                         \
  96         addi    r1, r1, STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS;    \
  97         ld      r0, PPC_LR_STKOFF(r1);                                  \
  98         mtlr    r0;                                                     \
  99         cmpdi   r3, 0;                                                  \
 100         blt     bpf_error;      /* cr0 = LT */
 101
 102 bpf_slow_path_word:
 103         bpf_slow_path_common(4)
 104         /* Data value is on stack, and cr0 != LT */
 105         LWZX_BE r_val, 0, r5
 106         blr
 107
 108 bpf_slow_path_half:
 109         bpf_slow_path_common(2)
 110         LHZX_BE r_val, 0, r5
 111         blr
 112
 113 bpf_slow_path_byte:
 114         bpf_slow_path_common(1)
 115         lbzx    r_val, 0, r5
 116         blr
 117
 118 /*
 119  * Call out to bpf_internal_load_pointer_neg_helper
 120  */
 121 #define sk_negative_common(SIZE)                                \
 122         mflr    r0;                                             \
 123         std     r0, PPC_LR_STKOFF(r1);                          \
 124         stdu    r1, -STACK_FRAME_MIN_SIZE(r1);                  \
 125         mr      r3, r_skb;                                      \
 126         /* r4 = r_off, as passed */                             \
 127         li      r5, SIZE;                                       \
 128         bl      bpf_internal_load_pointer_neg_helper;           \
 129         nop;                                                    \
 130         addi    r1, r1, STACK_FRAME_MIN_SIZE;                   \
 131         ld      r0, PPC_LR_STKOFF(r1);                          \
 132         mtlr    r0;                                             \
 133         /* R3 != 0 on success */                                \
 134         cmpldi  r3, 0;                                          \
 135         beq     bpf_error_slow; /* cr0 = EQ */
 136
 137 bpf_slow_path_word_neg:
 138         lis     r_tmp, -32      /* SKF_LL_OFF */
 139         cmpd    r_off, r_tmp    /* addr < SKF_* */
 140         blt     bpf_error       /* cr0 = LT */
 141         b       sk_load_word_negative_offset
 142
 143 _GLOBAL_TOC(sk_load_word_negative_offset)
 144         sk_negative_common(4)
 145         LWZX_BE r_val, 0, r3
 146         blr
 147
 148 bpf_slow_path_half_neg:
 149         lis     r_tmp, -32      /* SKF_LL_OFF */
 150         cmpd    r_off, r_tmp    /* addr < SKF_* */
 151         blt     bpf_error       /* cr0 = LT */
 152         b       sk_load_half_negative_offset
 153
 154 _GLOBAL_TOC(sk_load_half_negative_offset)
 155         sk_negative_common(2)
 156         LHZX_BE r_val, 0, r3
 157         blr
 158
 159 bpf_slow_path_byte_neg:
 160         lis     r_tmp, -32      /* SKF_LL_OFF */
 161         cmpd    r_off, r_tmp    /* addr < SKF_* */
 162         blt     bpf_error       /* cr0 = LT */
 163         b       sk_load_byte_negative_offset
 164
 165 _GLOBAL_TOC(sk_load_byte_negative_offset)
 166         sk_negative_common(1)
 167         lbzx    r_val, 0, r3
 168         blr
 169
 170 bpf_error_slow:
 171         /* fabricate a cr0 = lt */
 172         li      r_tmp, -1
 173         cmpdi   r_tmp, 0
 174 bpf_error:
 175         /*
 176          * Entered with cr0 = lt
 177          * Generated code will 'blt epilogue', returning 0.
 178          */
 179         li      r_val, 0
 180         blr