]>
Commit | Line | Data |
---|---|---|
a4397635 AB |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* | |
3 | * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org> | |
4 | */ | |
5 | ||
6 | #ifdef CONFIG_ARM64 | |
7 | #include <asm/neon-intrinsics.h> | |
8 | ||
9 | #define AES_ROUND "aese %0.16b, %1.16b \n\t aesmc %0.16b, %0.16b" | |
10 | #else | |
11 | #include <arm_neon.h> | |
12 | ||
13 | #define AES_ROUND "aese.8 %q0, %q1 \n\t aesmc.8 %q0, %q0" | |
14 | #endif | |
15 | ||
16 | #define AEGIS_BLOCK_SIZE 16 | |
17 | ||
18 | #include <stddef.h> | |
19 | ||
19842963 AB |
20 | extern int aegis128_have_aes_insn; |
21 | ||
a4397635 AB |
22 | void *memcpy(void *dest, const void *src, size_t n); |
23 | void *memset(void *s, int c, size_t n); | |
24 | ||
25 | struct aegis128_state { | |
26 | uint8x16_t v[5]; | |
27 | }; | |
28 | ||
19842963 AB |
29 | extern const uint8x16x4_t crypto_aes_sbox[]; |
30 | ||
a4397635 AB |
31 | static struct aegis128_state aegis128_load_state_neon(const void *state) |
32 | { | |
33 | return (struct aegis128_state){ { | |
34 | vld1q_u8(state), | |
35 | vld1q_u8(state + 16), | |
36 | vld1q_u8(state + 32), | |
37 | vld1q_u8(state + 48), | |
38 | vld1q_u8(state + 64) | |
39 | } }; | |
40 | } | |
41 | ||
42 | static void aegis128_save_state_neon(struct aegis128_state st, void *state) | |
43 | { | |
44 | vst1q_u8(state, st.v[0]); | |
45 | vst1q_u8(state + 16, st.v[1]); | |
46 | vst1q_u8(state + 32, st.v[2]); | |
47 | vst1q_u8(state + 48, st.v[3]); | |
48 | vst1q_u8(state + 64, st.v[4]); | |
49 | } | |
50 | ||
51 | static inline __attribute__((always_inline)) | |
52 | uint8x16_t aegis_aes_round(uint8x16_t w) | |
53 | { | |
54 | uint8x16_t z = {}; | |
55 | ||
19842963 AB |
56 | #ifdef CONFIG_ARM64 |
57 | if (!__builtin_expect(aegis128_have_aes_insn, 1)) { | |
58 | static const uint8x16_t shift_rows = { | |
59 | 0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3, | |
60 | 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb, | |
61 | }; | |
62 | static const uint8x16_t ror32by8 = { | |
63 | 0x1, 0x2, 0x3, 0x0, 0x5, 0x6, 0x7, 0x4, | |
64 | 0x9, 0xa, 0xb, 0x8, 0xd, 0xe, 0xf, 0xc, | |
65 | }; | |
66 | uint8x16_t v; | |
67 | ||
68 | // shift rows | |
69 | w = vqtbl1q_u8(w, shift_rows); | |
70 | ||
71 | // sub bytes | |
72 | if (!IS_ENABLED(CONFIG_CC_IS_GCC)) { | |
73 | v = vqtbl4q_u8(crypto_aes_sbox[0], w); | |
74 | v = vqtbx4q_u8(v, crypto_aes_sbox[1], w - 0x40); | |
75 | v = vqtbx4q_u8(v, crypto_aes_sbox[2], w - 0x80); | |
76 | v = vqtbx4q_u8(v, crypto_aes_sbox[3], w - 0xc0); | |
77 | } else { | |
78 | asm("tbl %0.16b, {v16.16b-v19.16b}, %1.16b" : "=w"(v) : "w"(w)); | |
79 | w -= 0x40; | |
80 | asm("tbx %0.16b, {v20.16b-v23.16b}, %1.16b" : "+w"(v) : "w"(w)); | |
81 | w -= 0x40; | |
82 | asm("tbx %0.16b, {v24.16b-v27.16b}, %1.16b" : "+w"(v) : "w"(w)); | |
83 | w -= 0x40; | |
84 | asm("tbx %0.16b, {v28.16b-v31.16b}, %1.16b" : "+w"(v) : "w"(w)); | |
85 | } | |
86 | ||
87 | // mix columns | |
88 | w = (v << 1) ^ (uint8x16_t)(((int8x16_t)v >> 7) & 0x1b); | |
89 | w ^= (uint8x16_t)vrev32q_u16((uint16x8_t)v); | |
90 | w ^= vqtbl1q_u8(v ^ w, ror32by8); | |
91 | ||
92 | return w; | |
93 | } | |
94 | #endif | |
95 | ||
a4397635 AB |
96 | /* |
97 | * We use inline asm here instead of the vaeseq_u8/vaesmcq_u8 intrinsics | |
98 | * to force the compiler to issue the aese/aesmc instructions in pairs. | |
99 | * This is much faster on many cores, where the instruction pair can | |
100 | * execute in a single cycle. | |
101 | */ | |
102 | asm(AES_ROUND : "+w"(w) : "w"(z)); | |
103 | return w; | |
104 | } | |
105 | ||
106 | static inline __attribute__((always_inline)) | |
107 | struct aegis128_state aegis128_update_neon(struct aegis128_state st, | |
108 | uint8x16_t m) | |
109 | { | |
110 | m ^= aegis_aes_round(st.v[4]); | |
111 | st.v[4] ^= aegis_aes_round(st.v[3]); | |
112 | st.v[3] ^= aegis_aes_round(st.v[2]); | |
113 | st.v[2] ^= aegis_aes_round(st.v[1]); | |
114 | st.v[1] ^= aegis_aes_round(st.v[0]); | |
115 | st.v[0] ^= m; | |
116 | ||
117 | return st; | |
118 | } | |
119 | ||
19842963 AB |
120 | static inline __attribute__((always_inline)) |
121 | void preload_sbox(void) | |
122 | { | |
123 | if (!IS_ENABLED(CONFIG_ARM64) || | |
124 | !IS_ENABLED(CONFIG_CC_IS_GCC) || | |
125 | __builtin_expect(aegis128_have_aes_insn, 1)) | |
126 | return; | |
127 | ||
128 | asm("ld1 {v16.16b-v19.16b}, [%0], #64 \n\t" | |
129 | "ld1 {v20.16b-v23.16b}, [%0], #64 \n\t" | |
130 | "ld1 {v24.16b-v27.16b}, [%0], #64 \n\t" | |
131 | "ld1 {v28.16b-v31.16b}, [%0] \n\t" | |
132 | :: "r"(crypto_aes_sbox)); | |
133 | } | |
134 | ||
a4397635 AB |
135 | void crypto_aegis128_update_neon(void *state, const void *msg) |
136 | { | |
137 | struct aegis128_state st = aegis128_load_state_neon(state); | |
138 | ||
19842963 AB |
139 | preload_sbox(); |
140 | ||
a4397635 AB |
141 | st = aegis128_update_neon(st, vld1q_u8(msg)); |
142 | ||
143 | aegis128_save_state_neon(st, state); | |
144 | } | |
145 | ||
146 | void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, | |
147 | unsigned int size) | |
148 | { | |
149 | struct aegis128_state st = aegis128_load_state_neon(state); | |
150 | uint8x16_t msg; | |
151 | ||
19842963 AB |
152 | preload_sbox(); |
153 | ||
a4397635 AB |
154 | while (size >= AEGIS_BLOCK_SIZE) { |
155 | uint8x16_t s = st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; | |
156 | ||
157 | msg = vld1q_u8(src); | |
158 | st = aegis128_update_neon(st, msg); | |
159 | vst1q_u8(dst, msg ^ s); | |
160 | ||
161 | size -= AEGIS_BLOCK_SIZE; | |
162 | src += AEGIS_BLOCK_SIZE; | |
163 | dst += AEGIS_BLOCK_SIZE; | |
164 | } | |
165 | ||
166 | if (size > 0) { | |
167 | uint8x16_t s = st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; | |
168 | uint8_t buf[AEGIS_BLOCK_SIZE] = {}; | |
169 | ||
170 | memcpy(buf, src, size); | |
171 | msg = vld1q_u8(buf); | |
172 | st = aegis128_update_neon(st, msg); | |
173 | vst1q_u8(buf, msg ^ s); | |
174 | memcpy(dst, buf, size); | |
175 | } | |
176 | ||
177 | aegis128_save_state_neon(st, state); | |
178 | } | |
179 | ||
180 | void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, | |
181 | unsigned int size) | |
182 | { | |
183 | struct aegis128_state st = aegis128_load_state_neon(state); | |
184 | uint8x16_t msg; | |
185 | ||
19842963 AB |
186 | preload_sbox(); |
187 | ||
a4397635 AB |
188 | while (size >= AEGIS_BLOCK_SIZE) { |
189 | msg = vld1q_u8(src) ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; | |
190 | st = aegis128_update_neon(st, msg); | |
191 | vst1q_u8(dst, msg); | |
192 | ||
193 | size -= AEGIS_BLOCK_SIZE; | |
194 | src += AEGIS_BLOCK_SIZE; | |
195 | dst += AEGIS_BLOCK_SIZE; | |
196 | } | |
197 | ||
198 | if (size > 0) { | |
199 | uint8x16_t s = st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; | |
200 | uint8_t buf[AEGIS_BLOCK_SIZE]; | |
201 | ||
202 | vst1q_u8(buf, s); | |
203 | memcpy(buf, src, size); | |
204 | msg = vld1q_u8(buf) ^ s; | |
205 | vst1q_u8(buf, msg); | |
206 | memcpy(dst, buf, size); | |
207 | ||
208 | st = aegis128_update_neon(st, msg); | |
209 | } | |
210 | ||
211 | aegis128_save_state_neon(st, state); | |
212 | } |