[mirror_ubuntu-zesty-kernel.git] / arch / x86 / net / bpf_jit.S

/* bpf_jit.S : BPF JIT helper functions
 *
 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */
#include <linux/linkage.h>
#include <asm/dwarf2.h>

/*
 * Calling convention :
 * rdi : skb pointer
 * esi : offset of byte(s) to fetch in skb (can be scratched)
 * r8  : copy of skb->data
 * r9d : hlen = skb->len - skb->data_len
 */
#define SKBDATA	%r8

sk_load_word_ind:
	.globl	sk_load_word_ind

	add	%ebx,%esi	/* offset += X */
#	test    %esi,%esi	/* if (offset < 0) goto bpf_error; */
	js	bpf_error

sk_load_word:
	.globl	sk_load_word

	mov	%r9d,%eax		# hlen
	sub	%esi,%eax		# hlen - offset
	cmp	$3,%eax
	jle	bpf_slow_path_word
	mov     (SKBDATA,%rsi),%eax
	bswap   %eax  			/* ntohl() */
	ret


sk_load_half_ind:
	.globl sk_load_half_ind

	add	%ebx,%esi	/* offset += X */
	js	bpf_error

sk_load_half:
	.globl	sk_load_half

	mov	%r9d,%eax
	sub	%esi,%eax		#	hlen - offset
	cmp	$1,%eax
	jle	bpf_slow_path_half
	movzwl	(SKBDATA,%rsi),%eax
	rol	$8,%ax			# ntohs()
	ret

sk_load_byte_ind:
	.globl sk_load_byte_ind
	add	%ebx,%esi	/* offset += X */
	js	bpf_error

sk_load_byte:
	.globl	sk_load_byte

	cmp	%esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
	jle	bpf_slow_path_byte
	movzbl	(SKBDATA,%rsi),%eax
	ret

/**
 * sk_load_byte_msh - BPF_S_LDX_B_MSH helper
 *
 * Implements BPF_S_LDX_B_MSH : ldxb  4*([offset]&0xf)
 * Must preserve A accumulator (%eax)
 * Inputs : %esi is the offset value, already known positive
 */
ENTRY(sk_load_byte_msh)
	CFI_STARTPROC
	cmp	%esi,%r9d      /* if (offset >= hlen) goto bpf_slow_path_byte_msh */
	jle	bpf_slow_path_byte_msh
	movzbl	(SKBDATA,%rsi),%ebx
	and	$15,%bl
	shl	$2,%bl
	ret
	CFI_ENDPROC
ENDPROC(sk_load_byte_msh)

bpf_error:
# force a return 0 from jit handler
	xor		%eax,%eax
	mov		-8(%rbp),%rbx
	leaveq
	ret

/* rsi contains offset and can be scratched */
#define bpf_slow_path_common(LEN)		\
	push	%rdi;    /* save skb */		\
	push	%r9;				\
	push	SKBDATA;			\
/* rsi already has offset */			\
	mov	$LEN,%ecx;	/* len */	\
	lea	-12(%rbp),%rdx;			\
	call	skb_copy_bits;			\
	test    %eax,%eax;			\
	pop	SKBDATA;			\
	pop	%r9;				\
	pop	%rdi


bpf_slow_path_word:
	bpf_slow_path_common(4)
	js	bpf_error
	mov	-12(%rbp),%eax
	bswap	%eax
	ret

bpf_slow_path_half:
	bpf_slow_path_common(2)
	js	bpf_error
	mov	-12(%rbp),%ax
	rol	$8,%ax
	movzwl	%ax,%eax
	ret

bpf_slow_path_byte:
	bpf_slow_path_common(1)
	js	bpf_error
	movzbl	-12(%rbp),%eax
	ret

bpf_slow_path_byte_msh:
	xchg	%eax,%ebx /* dont lose A , X is about to be scratched */
	bpf_slow_path_common(1)
	js	bpf_error
	movzbl	-12(%rbp),%eax
	and	$15,%al
	shl	$2,%al
	xchg	%eax,%ebx
	ret
Commit	Line	Data
0a14842f ED	1	/* bpf_jit.S : BPF JIT helper functions
	2	*
	3	* Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
	4	*
	5	* This program is free software; you can redistribute it and/or
	6	* modify it under the terms of the GNU General Public License
	7	* as published by the Free Software Foundation; version 2
	8	* of the License.
	9	*/
	10	#include <linux/linkage.h>
	11	#include <asm/dwarf2.h>
	12
	13	/*
	14	* Calling convention :
	15	* rdi : skb pointer
	16	* esi : offset of byte(s) to fetch in skb (can be scratched)
	17	* r8 : copy of skb->data
	18	* r9d : hlen = skb->len - skb->data_len
	19	*/
	20	#define SKBDATA %r8
	21
	22	sk_load_word_ind:
	23	.globl sk_load_word_ind
	24
	25	add %ebx,%esi /* offset += X */
	26	# test %esi,%esi /* if (offset < 0) goto bpf_error; */
	27	js bpf_error
	28
	29	sk_load_word:
	30	.globl sk_load_word
	31
	32	mov %r9d,%eax # hlen
	33	sub %esi,%eax # hlen - offset
	34	cmp $3,%eax
	35	jle bpf_slow_path_word
	36	mov (SKBDATA,%rsi),%eax
	37	bswap %eax /* ntohl() */
	38	ret
	39
	40
	41	sk_load_half_ind:
	42	.globl sk_load_half_ind
	43
	44	add %ebx,%esi /* offset += X */
	45	js bpf_error
	46
	47	sk_load_half:
	48	.globl sk_load_half
	49
	50	mov %r9d,%eax
	51	sub %esi,%eax # hlen - offset
	52	cmp $1,%eax
	53	jle bpf_slow_path_half
	54	movzwl (SKBDATA,%rsi),%eax
	55	rol $8,%ax # ntohs()
	56	ret
	57
	58	sk_load_byte_ind:
	59	.globl sk_load_byte_ind
	60	add %ebx,%esi /* offset += X */
	61	js bpf_error
	62
	63	sk_load_byte:
	64	.globl sk_load_byte
65
66	cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */
67	jle bpf_slow_path_byte
68	movzbl (SKBDATA,%rsi),%eax
69	ret
70
71	/**
72	* sk_load_byte_msh - BPF_S_LDX_B_MSH helper
73	*
74	* Implements BPF_S_LDX_B_MSH : ldxb 4*([offset]&0xf)
75	* Must preserve A accumulator (%eax)
76	* Inputs : %esi is the offset value, already known positive
77	*/
78	ENTRY(sk_load_byte_msh)
79	CFI_STARTPROC
80	cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte_msh */
81	jle bpf_slow_path_byte_msh
82	movzbl (SKBDATA,%rsi),%ebx
83	and $15,%bl
84	shl $2,%bl
85	ret
86	CFI_ENDPROC
87	ENDPROC(sk_load_byte_msh)
88
89	bpf_error:
90	# force a return 0 from jit handler
91	xor %eax,%eax
92	mov -8(%rbp),%rbx
93	leaveq
94	ret
95
96	/* rsi contains offset and can be scratched */
97	#define bpf_slow_path_common(LEN) \
98	push %rdi; /* save skb */ \
99	push %r9; \
100	push SKBDATA; \
101	/* rsi already has offset */ \
102	mov $LEN,%ecx; /* len */ \
103	lea -12(%rbp),%rdx; \
104	call skb_copy_bits; \
105	test %eax,%eax; \
106	pop SKBDATA; \
107	pop %r9; \
108	pop %rdi
109
110
111	bpf_slow_path_word:
112	bpf_slow_path_common(4)
113	js bpf_error
114	mov -12(%rbp),%eax
115	bswap %eax
116	ret
117
118	bpf_slow_path_half:
119	bpf_slow_path_common(2)
120	js bpf_error
121	mov -12(%rbp),%ax
122	rol $8,%ax
123	movzwl %ax,%eax
124	ret
125
126	bpf_slow_path_byte:
127	bpf_slow_path_common(1)
128	js bpf_error
129	movzbl -12(%rbp),%eax
130	ret
131
132	bpf_slow_path_byte_msh:
133	xchg %eax,%ebx /* dont lose A , X is about to be scratched */
134	bpf_slow_path_common(1)
135	js bpf_error
136	movzbl -12(%rbp),%eax
137	and $15,%al
138	shl $2,%al
139	xchg %eax,%ebx
140	ret