]>
Commit | Line | Data |
---|---|---|
317f2f75 AB |
1 | /* |
2 | * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions | |
3 | * | |
4 | * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | */ | |
10 | ||
11 | #include <asm/neon.h> | |
12 | #include <crypto/aes.h> | |
13 | #include <linux/cpufeature.h> | |
14 | #include <linux/crypto.h> | |
15 | #include <linux/module.h> | |
16 | ||
12ac3efe AB |
17 | #include "aes-ce-setkey.h" |
18 | ||
317f2f75 AB |
19 | MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions"); |
20 | MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); | |
21 | MODULE_LICENSE("GPL v2"); | |
22 | ||
23 | struct aes_block { | |
24 | u8 b[AES_BLOCK_SIZE]; | |
25 | }; | |
26 | ||
27 | static int num_rounds(struct crypto_aes_ctx *ctx) | |
28 | { | |
29 | /* | |
30 | * # of rounds specified by AES: | |
31 | * 128 bit key 10 rounds | |
32 | * 192 bit key 12 rounds | |
33 | * 256 bit key 14 rounds | |
34 | * => n byte key => 6 + (n/4) rounds | |
35 | */ | |
36 | return 6 + ctx->key_length / 4; | |
37 | } | |
38 | ||
39 | static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) | |
40 | { | |
41 | struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); | |
42 | struct aes_block *out = (struct aes_block *)dst; | |
43 | struct aes_block const *in = (struct aes_block *)src; | |
44 | void *dummy0; | |
45 | int dummy1; | |
46 | ||
47 | kernel_neon_begin_partial(4); | |
48 | ||
49 | __asm__(" ld1 {v0.16b}, %[in] ;" | |
50 | " ld1 {v1.2d}, [%[key]], #16 ;" | |
51 | " cmp %w[rounds], #10 ;" | |
52 | " bmi 0f ;" | |
53 | " bne 3f ;" | |
54 | " mov v3.16b, v1.16b ;" | |
55 | " b 2f ;" | |
56 | "0: mov v2.16b, v1.16b ;" | |
57 | " ld1 {v3.2d}, [%[key]], #16 ;" | |
58 | "1: aese v0.16b, v2.16b ;" | |
59 | " aesmc v0.16b, v0.16b ;" | |
60 | "2: ld1 {v1.2d}, [%[key]], #16 ;" | |
61 | " aese v0.16b, v3.16b ;" | |
62 | " aesmc v0.16b, v0.16b ;" | |
63 | "3: ld1 {v2.2d}, [%[key]], #16 ;" | |
64 | " subs %w[rounds], %w[rounds], #3 ;" | |
65 | " aese v0.16b, v1.16b ;" | |
66 | " aesmc v0.16b, v0.16b ;" | |
67 | " ld1 {v3.2d}, [%[key]], #16 ;" | |
68 | " bpl 1b ;" | |
69 | " aese v0.16b, v2.16b ;" | |
70 | " eor v0.16b, v0.16b, v3.16b ;" | |
71 | " st1 {v0.16b}, %[out] ;" | |
72 | ||
73 | : [out] "=Q"(*out), | |
74 | [key] "=r"(dummy0), | |
75 | [rounds] "=r"(dummy1) | |
76 | : [in] "Q"(*in), | |
77 | "1"(ctx->key_enc), | |
78 | "2"(num_rounds(ctx) - 2) | |
79 | : "cc"); | |
80 | ||
81 | kernel_neon_end(); | |
82 | } | |
83 | ||
84 | static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) | |
85 | { | |
86 | struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); | |
87 | struct aes_block *out = (struct aes_block *)dst; | |
88 | struct aes_block const *in = (struct aes_block *)src; | |
89 | void *dummy0; | |
90 | int dummy1; | |
91 | ||
92 | kernel_neon_begin_partial(4); | |
93 | ||
94 | __asm__(" ld1 {v0.16b}, %[in] ;" | |
95 | " ld1 {v1.2d}, [%[key]], #16 ;" | |
96 | " cmp %w[rounds], #10 ;" | |
97 | " bmi 0f ;" | |
98 | " bne 3f ;" | |
99 | " mov v3.16b, v1.16b ;" | |
100 | " b 2f ;" | |
101 | "0: mov v2.16b, v1.16b ;" | |
102 | " ld1 {v3.2d}, [%[key]], #16 ;" | |
103 | "1: aesd v0.16b, v2.16b ;" | |
104 | " aesimc v0.16b, v0.16b ;" | |
105 | "2: ld1 {v1.2d}, [%[key]], #16 ;" | |
106 | " aesd v0.16b, v3.16b ;" | |
107 | " aesimc v0.16b, v0.16b ;" | |
108 | "3: ld1 {v2.2d}, [%[key]], #16 ;" | |
109 | " subs %w[rounds], %w[rounds], #3 ;" | |
110 | " aesd v0.16b, v1.16b ;" | |
111 | " aesimc v0.16b, v0.16b ;" | |
112 | " ld1 {v3.2d}, [%[key]], #16 ;" | |
113 | " bpl 1b ;" | |
114 | " aesd v0.16b, v2.16b ;" | |
115 | " eor v0.16b, v0.16b, v3.16b ;" | |
116 | " st1 {v0.16b}, %[out] ;" | |
117 | ||
118 | : [out] "=Q"(*out), | |
119 | [key] "=r"(dummy0), | |
120 | [rounds] "=r"(dummy1) | |
121 | : [in] "Q"(*in), | |
122 | "1"(ctx->key_dec), | |
123 | "2"(num_rounds(ctx) - 2) | |
124 | : "cc"); | |
125 | ||
126 | kernel_neon_end(); | |
127 | } | |
128 | ||
12ac3efe AB |
129 | /* |
130 | * aes_sub() - use the aese instruction to perform the AES sbox substitution | |
131 | * on each byte in 'input' | |
132 | */ | |
133 | static u32 aes_sub(u32 input) | |
134 | { | |
135 | u32 ret; | |
136 | ||
137 | __asm__("dup v1.4s, %w[in] ;" | |
138 | "movi v0.16b, #0 ;" | |
139 | "aese v0.16b, v1.16b ;" | |
140 | "umov %w[out], v0.4s[0] ;" | |
141 | ||
142 | : [out] "=r"(ret) | |
143 | : [in] "r"(input) | |
144 | : "v0","v1"); | |
145 | ||
146 | return ret; | |
147 | } | |
148 | ||
149 | int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, | |
150 | unsigned int key_len) | |
151 | { | |
152 | /* | |
153 | * The AES key schedule round constants | |
154 | */ | |
155 | static u8 const rcon[] = { | |
156 | 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, | |
157 | }; | |
158 | ||
159 | u32 kwords = key_len / sizeof(u32); | |
160 | struct aes_block *key_enc, *key_dec; | |
161 | int i, j; | |
162 | ||
163 | if (key_len != AES_KEYSIZE_128 && | |
164 | key_len != AES_KEYSIZE_192 && | |
165 | key_len != AES_KEYSIZE_256) | |
166 | return -EINVAL; | |
167 | ||
168 | memcpy(ctx->key_enc, in_key, key_len); | |
169 | ctx->key_length = key_len; | |
170 | ||
171 | kernel_neon_begin_partial(2); | |
172 | for (i = 0; i < sizeof(rcon); i++) { | |
173 | u32 *rki = ctx->key_enc + (i * kwords); | |
174 | u32 *rko = rki + kwords; | |
175 | ||
176 | rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0]; | |
177 | rko[1] = rko[0] ^ rki[1]; | |
178 | rko[2] = rko[1] ^ rki[2]; | |
179 | rko[3] = rko[2] ^ rki[3]; | |
180 | ||
181 | if (key_len == AES_KEYSIZE_192) { | |
182 | if (i >= 7) | |
183 | break; | |
184 | rko[4] = rko[3] ^ rki[4]; | |
185 | rko[5] = rko[4] ^ rki[5]; | |
186 | } else if (key_len == AES_KEYSIZE_256) { | |
187 | if (i >= 6) | |
188 | break; | |
189 | rko[4] = aes_sub(rko[3]) ^ rki[4]; | |
190 | rko[5] = rko[4] ^ rki[5]; | |
191 | rko[6] = rko[5] ^ rki[6]; | |
192 | rko[7] = rko[6] ^ rki[7]; | |
193 | } | |
194 | } | |
195 | ||
196 | /* | |
197 | * Generate the decryption keys for the Equivalent Inverse Cipher. | |
198 | * This involves reversing the order of the round keys, and applying | |
199 | * the Inverse Mix Columns transformation on all but the first and | |
200 | * the last one. | |
201 | */ | |
202 | key_enc = (struct aes_block *)ctx->key_enc; | |
203 | key_dec = (struct aes_block *)ctx->key_dec; | |
204 | j = num_rounds(ctx); | |
205 | ||
206 | key_dec[0] = key_enc[j]; | |
207 | for (i = 1, j--; j > 0; i++, j--) | |
208 | __asm__("ld1 {v0.16b}, %[in] ;" | |
209 | "aesimc v1.16b, v0.16b ;" | |
210 | "st1 {v1.16b}, %[out] ;" | |
211 | ||
212 | : [out] "=Q"(key_dec[i]) | |
213 | : [in] "Q"(key_enc[j]) | |
214 | : "v0","v1"); | |
215 | key_dec[i] = key_enc[0]; | |
216 | ||
217 | kernel_neon_end(); | |
218 | return 0; | |
219 | } | |
220 | EXPORT_SYMBOL(ce_aes_expandkey); | |
221 | ||
222 | int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, | |
223 | unsigned int key_len) | |
224 | { | |
225 | struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); | |
226 | int ret; | |
227 | ||
228 | ret = ce_aes_expandkey(ctx, in_key, key_len); | |
229 | if (!ret) | |
230 | return 0; | |
231 | ||
232 | tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; | |
233 | return -EINVAL; | |
234 | } | |
235 | EXPORT_SYMBOL(ce_aes_setkey); | |
236 | ||
317f2f75 AB |
237 | static struct crypto_alg aes_alg = { |
238 | .cra_name = "aes", | |
239 | .cra_driver_name = "aes-ce", | |
08c6781c | 240 | .cra_priority = 250, |
317f2f75 AB |
241 | .cra_flags = CRYPTO_ALG_TYPE_CIPHER, |
242 | .cra_blocksize = AES_BLOCK_SIZE, | |
243 | .cra_ctxsize = sizeof(struct crypto_aes_ctx), | |
244 | .cra_module = THIS_MODULE, | |
245 | .cra_cipher = { | |
246 | .cia_min_keysize = AES_MIN_KEY_SIZE, | |
247 | .cia_max_keysize = AES_MAX_KEY_SIZE, | |
12ac3efe | 248 | .cia_setkey = ce_aes_setkey, |
317f2f75 AB |
249 | .cia_encrypt = aes_cipher_encrypt, |
250 | .cia_decrypt = aes_cipher_decrypt | |
251 | } | |
252 | }; | |
253 | ||
254 | static int __init aes_mod_init(void) | |
255 | { | |
256 | return crypto_register_alg(&aes_alg); | |
257 | } | |
258 | ||
259 | static void __exit aes_mod_exit(void) | |
260 | { | |
261 | crypto_unregister_alg(&aes_alg); | |
262 | } | |
263 | ||
264 | module_cpu_feature_match(AES, aes_mod_init); | |
265 | module_exit(aes_mod_exit); |