]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
7a12b503 DM |
2 | #include <asm/ptrace.h> |
3 | ||
4 | #include "bpf_jit_64.h" | |
5 | ||
6 | #define SAVE_SZ 176 | |
7 | #define SCRATCH_OFF STACK_BIAS + 128 | |
8 | #define BE_PTR(label) be,pn %xcc, label | |
9 | #define SIGN_EXTEND(reg) sra reg, 0, reg | |
10 | ||
11 | #define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ | |
12 | ||
13 | .text | |
14 | .globl bpf_jit_load_word | |
15 | bpf_jit_load_word: | |
16 | cmp r_OFF, 0 | |
17 | bl bpf_slow_path_word_neg | |
18 | nop | |
19 | .globl bpf_jit_load_word_positive_offset | |
20 | bpf_jit_load_word_positive_offset: | |
21 | sub r_HEADLEN, r_OFF, r_TMP | |
22 | cmp r_TMP, 3 | |
23 | ble bpf_slow_path_word | |
24 | add r_SKB_DATA, r_OFF, r_TMP | |
25 | andcc r_TMP, 3, %g0 | |
26 | bne load_word_unaligned | |
27 | nop | |
28 | retl | |
29 | ld [r_TMP], r_RESULT | |
30 | load_word_unaligned: | |
31 | ldub [r_TMP + 0x0], r_OFF | |
32 | ldub [r_TMP + 0x1], r_TMP2 | |
33 | sll r_OFF, 8, r_OFF | |
34 | or r_OFF, r_TMP2, r_OFF | |
35 | ldub [r_TMP + 0x2], r_TMP2 | |
36 | sll r_OFF, 8, r_OFF | |
37 | or r_OFF, r_TMP2, r_OFF | |
38 | ldub [r_TMP + 0x3], r_TMP2 | |
39 | sll r_OFF, 8, r_OFF | |
40 | retl | |
41 | or r_OFF, r_TMP2, r_RESULT | |
42 | ||
43 | .globl bpf_jit_load_half | |
44 | bpf_jit_load_half: | |
45 | cmp r_OFF, 0 | |
46 | bl bpf_slow_path_half_neg | |
47 | nop | |
48 | .globl bpf_jit_load_half_positive_offset | |
49 | bpf_jit_load_half_positive_offset: | |
50 | sub r_HEADLEN, r_OFF, r_TMP | |
51 | cmp r_TMP, 1 | |
52 | ble bpf_slow_path_half | |
53 | add r_SKB_DATA, r_OFF, r_TMP | |
54 | andcc r_TMP, 1, %g0 | |
55 | bne load_half_unaligned | |
56 | nop | |
57 | retl | |
58 | lduh [r_TMP], r_RESULT | |
59 | load_half_unaligned: | |
60 | ldub [r_TMP + 0x0], r_OFF | |
61 | ldub [r_TMP + 0x1], r_TMP2 | |
62 | sll r_OFF, 8, r_OFF | |
63 | retl | |
64 | or r_OFF, r_TMP2, r_RESULT | |
65 | ||
66 | .globl bpf_jit_load_byte | |
67 | bpf_jit_load_byte: | |
68 | cmp r_OFF, 0 | |
69 | bl bpf_slow_path_byte_neg | |
70 | nop | |
71 | .globl bpf_jit_load_byte_positive_offset | |
72 | bpf_jit_load_byte_positive_offset: | |
73 | cmp r_OFF, r_HEADLEN | |
74 | bge bpf_slow_path_byte | |
75 | nop | |
76 | retl | |
77 | ldub [r_SKB_DATA + r_OFF], r_RESULT | |
78 | ||
79 | #define bpf_slow_path_common(LEN) \ | |
80 | save %sp, -SAVE_SZ, %sp; \ | |
81 | mov %i0, %o0; \ | |
82 | mov %i1, %o1; \ | |
83 | add %fp, SCRATCH_OFF, %o2; \ | |
84 | call skb_copy_bits; \ | |
85 | mov (LEN), %o3; \ | |
86 | cmp %o0, 0; \ | |
87 | restore; | |
88 | ||
89 | bpf_slow_path_word: | |
90 | bpf_slow_path_common(4) | |
91 | bl bpf_error | |
92 | ld [%sp + SCRATCH_OFF], r_RESULT | |
93 | retl | |
94 | nop | |
95 | bpf_slow_path_half: | |
96 | bpf_slow_path_common(2) | |
97 | bl bpf_error | |
98 | lduh [%sp + SCRATCH_OFF], r_RESULT | |
99 | retl | |
100 | nop | |
101 | bpf_slow_path_byte: | |
102 | bpf_slow_path_common(1) | |
103 | bl bpf_error | |
104 | ldub [%sp + SCRATCH_OFF], r_RESULT | |
105 | retl | |
106 | nop | |
107 | ||
108 | #define bpf_negative_common(LEN) \ | |
109 | save %sp, -SAVE_SZ, %sp; \ | |
110 | mov %i0, %o0; \ | |
111 | mov %i1, %o1; \ | |
112 | SIGN_EXTEND(%o1); \ | |
113 | call bpf_internal_load_pointer_neg_helper; \ | |
114 | mov (LEN), %o2; \ | |
115 | mov %o0, r_TMP; \ | |
116 | cmp %o0, 0; \ | |
117 | BE_PTR(bpf_error); \ | |
118 | restore; | |
119 | ||
120 | bpf_slow_path_word_neg: | |
121 | sethi %hi(SKF_MAX_NEG_OFF), r_TMP | |
122 | cmp r_OFF, r_TMP | |
123 | bl bpf_error | |
124 | nop | |
125 | .globl bpf_jit_load_word_negative_offset | |
126 | bpf_jit_load_word_negative_offset: | |
127 | bpf_negative_common(4) | |
128 | andcc r_TMP, 3, %g0 | |
129 | bne load_word_unaligned | |
130 | nop | |
131 | retl | |
132 | ld [r_TMP], r_RESULT | |
133 | ||
134 | bpf_slow_path_half_neg: | |
135 | sethi %hi(SKF_MAX_NEG_OFF), r_TMP | |
136 | cmp r_OFF, r_TMP | |
137 | bl bpf_error | |
138 | nop | |
139 | .globl bpf_jit_load_half_negative_offset | |
140 | bpf_jit_load_half_negative_offset: | |
141 | bpf_negative_common(2) | |
142 | andcc r_TMP, 1, %g0 | |
143 | bne load_half_unaligned | |
144 | nop | |
145 | retl | |
146 | lduh [r_TMP], r_RESULT | |
147 | ||
148 | bpf_slow_path_byte_neg: | |
149 | sethi %hi(SKF_MAX_NEG_OFF), r_TMP | |
150 | cmp r_OFF, r_TMP | |
151 | bl bpf_error | |
152 | nop | |
153 | .globl bpf_jit_load_byte_negative_offset | |
154 | bpf_jit_load_byte_negative_offset: | |
155 | bpf_negative_common(1) | |
156 | retl | |
157 | ldub [r_TMP], r_RESULT | |
158 | ||
159 | bpf_error: | |
160 | /* Make the JIT program itself return zero. */ | |
161 | ret | |
162 | restore %g0, %g0, %o0 |