]>
Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
49788fe2 AB |
2 | /* |
3 | * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES | |
4 | * | |
4860620d | 5 | * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> |
49788fe2 AB |
6 | */ |
7 | ||
8 | #include <asm/neon.h> | |
9 | #include <asm/hwcap.h> | |
e2115069 | 10 | #include <asm/simd.h> |
49788fe2 | 11 | #include <crypto/aes.h> |
ff6f4115 | 12 | #include <crypto/ctr.h> |
735177ca | 13 | #include <crypto/sha.h> |
4860620d | 14 | #include <crypto/internal/hash.h> |
d0ed0db1 HX |
15 | #include <crypto/internal/simd.h> |
16 | #include <crypto/internal/skcipher.h> | |
dd597fb3 | 17 | #include <crypto/scatterwalk.h> |
49788fe2 AB |
18 | #include <linux/module.h> |
19 | #include <linux/cpufeature.h> | |
49abc0d2 | 20 | #include <crypto/xts.h> |
49788fe2 | 21 | |
12ac3efe AB |
22 | #include "aes-ce-setkey.h" |
23 | ||
49788fe2 AB |
24 | #ifdef USE_V8_CRYPTO_EXTENSIONS |
25 | #define MODE "ce" | |
26 | #define PRIO 300 | |
12ac3efe | 27 | #define aes_expandkey ce_aes_expandkey |
49788fe2 AB |
28 | #define aes_ecb_encrypt ce_aes_ecb_encrypt |
29 | #define aes_ecb_decrypt ce_aes_ecb_decrypt | |
30 | #define aes_cbc_encrypt ce_aes_cbc_encrypt | |
31 | #define aes_cbc_decrypt ce_aes_cbc_decrypt | |
dd597fb3 AB |
32 | #define aes_cbc_cts_encrypt ce_aes_cbc_cts_encrypt |
33 | #define aes_cbc_cts_decrypt ce_aes_cbc_cts_decrypt | |
735177ca AB |
34 | #define aes_essiv_cbc_encrypt ce_aes_essiv_cbc_encrypt |
35 | #define aes_essiv_cbc_decrypt ce_aes_essiv_cbc_decrypt | |
49788fe2 AB |
36 | #define aes_ctr_encrypt ce_aes_ctr_encrypt |
37 | #define aes_xts_encrypt ce_aes_xts_encrypt | |
38 | #define aes_xts_decrypt ce_aes_xts_decrypt | |
4860620d | 39 | #define aes_mac_update ce_aes_mac_update |
49788fe2 AB |
40 | MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); |
41 | #else | |
42 | #define MODE "neon" | |
43 | #define PRIO 200 | |
44 | #define aes_ecb_encrypt neon_aes_ecb_encrypt | |
45 | #define aes_ecb_decrypt neon_aes_ecb_decrypt | |
46 | #define aes_cbc_encrypt neon_aes_cbc_encrypt | |
47 | #define aes_cbc_decrypt neon_aes_cbc_decrypt | |
dd597fb3 AB |
48 | #define aes_cbc_cts_encrypt neon_aes_cbc_cts_encrypt |
49 | #define aes_cbc_cts_decrypt neon_aes_cbc_cts_decrypt | |
735177ca AB |
50 | #define aes_essiv_cbc_encrypt neon_aes_essiv_cbc_encrypt |
51 | #define aes_essiv_cbc_decrypt neon_aes_essiv_cbc_decrypt | |
49788fe2 AB |
52 | #define aes_ctr_encrypt neon_aes_ctr_encrypt |
53 | #define aes_xts_encrypt neon_aes_xts_encrypt | |
54 | #define aes_xts_decrypt neon_aes_xts_decrypt | |
4860620d | 55 | #define aes_mac_update neon_aes_mac_update |
49788fe2 | 56 | MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); |
69b6f2e8 AB |
57 | #endif |
58 | #if defined(USE_V8_CRYPTO_EXTENSIONS) || !defined(CONFIG_CRYPTO_AES_ARM64_BS) | |
5d26a105 KC |
59 | MODULE_ALIAS_CRYPTO("ecb(aes)"); |
60 | MODULE_ALIAS_CRYPTO("cbc(aes)"); | |
61 | MODULE_ALIAS_CRYPTO("ctr(aes)"); | |
62 | MODULE_ALIAS_CRYPTO("xts(aes)"); | |
69b6f2e8 AB |
63 | #endif |
64 | MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); | |
65 | MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)"); | |
4860620d AB |
66 | MODULE_ALIAS_CRYPTO("cmac(aes)"); |
67 | MODULE_ALIAS_CRYPTO("xcbc(aes)"); | |
68 | MODULE_ALIAS_CRYPTO("cbcmac(aes)"); | |
49788fe2 AB |
69 | |
70 | MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); | |
71 | MODULE_LICENSE("GPL v2"); | |
72 | ||
73 | /* defined in aes-modes.S */ | |
557ecb45 | 74 | asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], |
68338174 | 75 | int rounds, int blocks); |
557ecb45 | 76 | asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], |
68338174 | 77 | int rounds, int blocks); |
49788fe2 | 78 | |
557ecb45 | 79 | asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], |
68338174 | 80 | int rounds, int blocks, u8 iv[]); |
557ecb45 | 81 | asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], |
68338174 | 82 | int rounds, int blocks, u8 iv[]); |
49788fe2 | 83 | |
dd597fb3 AB |
84 | asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], |
85 | int rounds, int bytes, u8 const iv[]); | |
86 | asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], | |
87 | int rounds, int bytes, u8 const iv[]); | |
88 | ||
557ecb45 | 89 | asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], |
68338174 | 90 | int rounds, int blocks, u8 ctr[]); |
49788fe2 | 91 | |
557ecb45 AB |
92 | asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], |
93 | int rounds, int blocks, u32 const rk2[], u8 iv[], | |
49788fe2 | 94 | int first); |
557ecb45 AB |
95 | asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], |
96 | int rounds, int blocks, u32 const rk2[], u8 iv[], | |
49788fe2 AB |
97 | int first); |
98 | ||
735177ca AB |
99 | asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[], |
100 | int rounds, int blocks, u8 iv[], | |
101 | u32 const rk2[]); | |
102 | asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], | |
103 | int rounds, int blocks, u8 iv[], | |
104 | u32 const rk2[]); | |
105 | ||
4860620d AB |
106 | asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds, |
107 | int blocks, u8 dg[], int enc_before, | |
108 | int enc_after); | |
109 | ||
49788fe2 AB |
110 | struct crypto_aes_xts_ctx { |
111 | struct crypto_aes_ctx key1; | |
112 | struct crypto_aes_ctx __aligned(8) key2; | |
113 | }; | |
114 | ||
735177ca AB |
115 | struct crypto_aes_essiv_cbc_ctx { |
116 | struct crypto_aes_ctx key1; | |
117 | struct crypto_aes_ctx __aligned(8) key2; | |
118 | struct crypto_shash *hash; | |
119 | }; | |
120 | ||
4860620d AB |
121 | struct mac_tfm_ctx { |
122 | struct crypto_aes_ctx key; | |
123 | u8 __aligned(8) consts[]; | |
124 | }; | |
125 | ||
126 | struct mac_desc_ctx { | |
127 | unsigned int len; | |
128 | u8 dg[AES_BLOCK_SIZE]; | |
129 | }; | |
130 | ||
d0ed0db1 HX |
131 | static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, |
132 | unsigned int key_len) | |
133 | { | |
c1844729 AB |
134 | struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); |
135 | int ret; | |
136 | ||
137 | ret = aes_expandkey(ctx, in_key, key_len); | |
138 | if (ret) | |
139 | crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); | |
140 | ||
141 | return ret; | |
d0ed0db1 HX |
142 | } |
143 | ||
69b6f2e8 AB |
144 | static int __maybe_unused xts_set_key(struct crypto_skcipher *tfm, |
145 | const u8 *in_key, unsigned int key_len) | |
49788fe2 | 146 | { |
d0ed0db1 | 147 | struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
49788fe2 AB |
148 | int ret; |
149 | ||
d0ed0db1 | 150 | ret = xts_verify_key(tfm, in_key, key_len); |
28856a9e SM |
151 | if (ret) |
152 | return ret; | |
153 | ||
12ac3efe | 154 | ret = aes_expandkey(&ctx->key1, in_key, key_len / 2); |
49788fe2 | 155 | if (!ret) |
12ac3efe AB |
156 | ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2], |
157 | key_len / 2); | |
49788fe2 AB |
158 | if (!ret) |
159 | return 0; | |
160 | ||
d0ed0db1 | 161 | crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); |
49788fe2 AB |
162 | return -EINVAL; |
163 | } | |
164 | ||
69b6f2e8 AB |
165 | static int __maybe_unused essiv_cbc_set_key(struct crypto_skcipher *tfm, |
166 | const u8 *in_key, | |
167 | unsigned int key_len) | |
735177ca AB |
168 | { |
169 | struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); | |
170 | SHASH_DESC_ON_STACK(desc, ctx->hash); | |
171 | u8 digest[SHA256_DIGEST_SIZE]; | |
172 | int ret; | |
173 | ||
174 | ret = aes_expandkey(&ctx->key1, in_key, key_len); | |
175 | if (ret) | |
176 | goto out; | |
177 | ||
178 | desc->tfm = ctx->hash; | |
179 | crypto_shash_digest(desc, in_key, key_len, digest); | |
180 | ||
181 | ret = aes_expandkey(&ctx->key2, digest, sizeof(digest)); | |
182 | if (ret) | |
183 | goto out; | |
184 | ||
185 | return 0; | |
186 | out: | |
187 | crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); | |
188 | return -EINVAL; | |
189 | } | |
190 | ||
69b6f2e8 | 191 | static int __maybe_unused ecb_encrypt(struct skcipher_request *req) |
49788fe2 | 192 | { |
d0ed0db1 HX |
193 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
194 | struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); | |
68338174 | 195 | int err, rounds = 6 + ctx->key_length / 4; |
d0ed0db1 | 196 | struct skcipher_walk walk; |
49788fe2 AB |
197 | unsigned int blocks; |
198 | ||
68338174 | 199 | err = skcipher_walk_virt(&walk, req, false); |
49788fe2 | 200 | |
68338174 AB |
201 | while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { |
202 | kernel_neon_begin(); | |
49788fe2 | 203 | aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, |
557ecb45 | 204 | ctx->key_enc, rounds, blocks); |
68338174 | 205 | kernel_neon_end(); |
d0ed0db1 | 206 | err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); |
49788fe2 | 207 | } |
49788fe2 AB |
208 | return err; |
209 | } | |
210 | ||
69b6f2e8 | 211 | static int __maybe_unused ecb_decrypt(struct skcipher_request *req) |
49788fe2 | 212 | { |
d0ed0db1 HX |
213 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
214 | struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); | |
68338174 | 215 | int err, rounds = 6 + ctx->key_length / 4; |
d0ed0db1 | 216 | struct skcipher_walk walk; |
49788fe2 AB |
217 | unsigned int blocks; |
218 | ||
68338174 | 219 | err = skcipher_walk_virt(&walk, req, false); |
49788fe2 | 220 | |
68338174 AB |
221 | while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { |
222 | kernel_neon_begin(); | |
49788fe2 | 223 | aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, |
557ecb45 | 224 | ctx->key_dec, rounds, blocks); |
68338174 | 225 | kernel_neon_end(); |
d0ed0db1 | 226 | err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); |
49788fe2 | 227 | } |
49788fe2 AB |
228 | return err; |
229 | } | |
230 | ||
65d0042b AB |
231 | static int cbc_encrypt_walk(struct skcipher_request *req, |
232 | struct skcipher_walk *walk) | |
49788fe2 | 233 | { |
d0ed0db1 HX |
234 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
235 | struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); | |
65d0042b | 236 | int err = 0, rounds = 6 + ctx->key_length / 4; |
49788fe2 AB |
237 | unsigned int blocks; |
238 | ||
65d0042b | 239 | while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { |
68338174 | 240 | kernel_neon_begin(); |
65d0042b AB |
241 | aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr, |
242 | ctx->key_enc, rounds, blocks, walk->iv); | |
68338174 | 243 | kernel_neon_end(); |
65d0042b | 244 | err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); |
49788fe2 | 245 | } |
49788fe2 AB |
246 | return err; |
247 | } | |
248 | ||
69b6f2e8 | 249 | static int __maybe_unused cbc_encrypt(struct skcipher_request *req) |
49788fe2 | 250 | { |
d0ed0db1 | 251 | struct skcipher_walk walk; |
65d0042b | 252 | int err; |
49788fe2 | 253 | |
68338174 | 254 | err = skcipher_walk_virt(&walk, req, false); |
65d0042b AB |
255 | if (err) |
256 | return err; | |
257 | return cbc_encrypt_walk(req, &walk); | |
258 | } | |
49788fe2 | 259 | |
65d0042b AB |
260 | static int cbc_decrypt_walk(struct skcipher_request *req, |
261 | struct skcipher_walk *walk) | |
262 | { | |
263 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
264 | struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); | |
265 | int err = 0, rounds = 6 + ctx->key_length / 4; | |
266 | unsigned int blocks; | |
267 | ||
268 | while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { | |
68338174 | 269 | kernel_neon_begin(); |
65d0042b AB |
270 | aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr, |
271 | ctx->key_dec, rounds, blocks, walk->iv); | |
68338174 | 272 | kernel_neon_end(); |
65d0042b | 273 | err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); |
49788fe2 | 274 | } |
49788fe2 AB |
275 | return err; |
276 | } | |
277 | ||
69b6f2e8 | 278 | static int __maybe_unused cbc_decrypt(struct skcipher_request *req) |
65d0042b AB |
279 | { |
280 | struct skcipher_walk walk; | |
281 | int err; | |
282 | ||
283 | err = skcipher_walk_virt(&walk, req, false); | |
284 | if (err) | |
285 | return err; | |
286 | return cbc_decrypt_walk(req, &walk); | |
287 | } | |
288 | ||
dd597fb3 AB |
289 | static int cts_cbc_encrypt(struct skcipher_request *req) |
290 | { | |
291 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
292 | struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); | |
dd597fb3 AB |
293 | int err, rounds = 6 + ctx->key_length / 4; |
294 | int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; | |
295 | struct scatterlist *src = req->src, *dst = req->dst; | |
7c9d65c4 AB |
296 | struct scatterlist sg_src[2], sg_dst[2]; |
297 | struct skcipher_request subreq; | |
dd597fb3 AB |
298 | struct skcipher_walk walk; |
299 | ||
7c9d65c4 AB |
300 | skcipher_request_set_tfm(&subreq, tfm); |
301 | skcipher_request_set_callback(&subreq, skcipher_request_flags(req), | |
302 | NULL, NULL); | |
dd597fb3 | 303 | |
7ff9036a EB |
304 | if (req->cryptlen <= AES_BLOCK_SIZE) { |
305 | if (req->cryptlen < AES_BLOCK_SIZE) | |
306 | return -EINVAL; | |
dd597fb3 | 307 | cbc_blocks = 1; |
7ff9036a | 308 | } |
dd597fb3 AB |
309 | |
310 | if (cbc_blocks > 0) { | |
7c9d65c4 | 311 | skcipher_request_set_crypt(&subreq, req->src, req->dst, |
dd597fb3 AB |
312 | cbc_blocks * AES_BLOCK_SIZE, |
313 | req->iv); | |
314 | ||
7c9d65c4 AB |
315 | err = skcipher_walk_virt(&walk, &subreq, false) ?: |
316 | cbc_encrypt_walk(&subreq, &walk); | |
dd597fb3 AB |
317 | if (err) |
318 | return err; | |
319 | ||
320 | if (req->cryptlen == AES_BLOCK_SIZE) | |
321 | return 0; | |
322 | ||
7c9d65c4 | 323 | dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); |
dd597fb3 | 324 | if (req->dst != req->src) |
7c9d65c4 AB |
325 | dst = scatterwalk_ffwd(sg_dst, req->dst, |
326 | subreq.cryptlen); | |
dd597fb3 AB |
327 | } |
328 | ||
329 | /* handle ciphertext stealing */ | |
7c9d65c4 | 330 | skcipher_request_set_crypt(&subreq, src, dst, |
dd597fb3 AB |
331 | req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, |
332 | req->iv); | |
333 | ||
7c9d65c4 | 334 | err = skcipher_walk_virt(&walk, &subreq, false); |
dd597fb3 AB |
335 | if (err) |
336 | return err; | |
337 | ||
338 | kernel_neon_begin(); | |
339 | aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, | |
340 | ctx->key_enc, rounds, walk.nbytes, walk.iv); | |
341 | kernel_neon_end(); | |
342 | ||
343 | return skcipher_walk_done(&walk, 0); | |
344 | } | |
345 | ||
346 | static int cts_cbc_decrypt(struct skcipher_request *req) | |
347 | { | |
348 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
349 | struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); | |
dd597fb3 AB |
350 | int err, rounds = 6 + ctx->key_length / 4; |
351 | int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; | |
352 | struct scatterlist *src = req->src, *dst = req->dst; | |
7c9d65c4 AB |
353 | struct scatterlist sg_src[2], sg_dst[2]; |
354 | struct skcipher_request subreq; | |
dd597fb3 AB |
355 | struct skcipher_walk walk; |
356 | ||
7c9d65c4 AB |
357 | skcipher_request_set_tfm(&subreq, tfm); |
358 | skcipher_request_set_callback(&subreq, skcipher_request_flags(req), | |
359 | NULL, NULL); | |
dd597fb3 | 360 | |
7ff9036a EB |
361 | if (req->cryptlen <= AES_BLOCK_SIZE) { |
362 | if (req->cryptlen < AES_BLOCK_SIZE) | |
363 | return -EINVAL; | |
dd597fb3 | 364 | cbc_blocks = 1; |
7ff9036a | 365 | } |
dd597fb3 AB |
366 | |
367 | if (cbc_blocks > 0) { | |
7c9d65c4 | 368 | skcipher_request_set_crypt(&subreq, req->src, req->dst, |
dd597fb3 AB |
369 | cbc_blocks * AES_BLOCK_SIZE, |
370 | req->iv); | |
371 | ||
7c9d65c4 AB |
372 | err = skcipher_walk_virt(&walk, &subreq, false) ?: |
373 | cbc_decrypt_walk(&subreq, &walk); | |
dd597fb3 AB |
374 | if (err) |
375 | return err; | |
376 | ||
377 | if (req->cryptlen == AES_BLOCK_SIZE) | |
378 | return 0; | |
379 | ||
7c9d65c4 | 380 | dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); |
dd597fb3 | 381 | if (req->dst != req->src) |
7c9d65c4 AB |
382 | dst = scatterwalk_ffwd(sg_dst, req->dst, |
383 | subreq.cryptlen); | |
dd597fb3 AB |
384 | } |
385 | ||
386 | /* handle ciphertext stealing */ | |
7c9d65c4 | 387 | skcipher_request_set_crypt(&subreq, src, dst, |
dd597fb3 AB |
388 | req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, |
389 | req->iv); | |
390 | ||
7c9d65c4 | 391 | err = skcipher_walk_virt(&walk, &subreq, false); |
dd597fb3 AB |
392 | if (err) |
393 | return err; | |
394 | ||
395 | kernel_neon_begin(); | |
396 | aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, | |
397 | ctx->key_dec, rounds, walk.nbytes, walk.iv); | |
398 | kernel_neon_end(); | |
399 | ||
400 | return skcipher_walk_done(&walk, 0); | |
401 | } | |
402 | ||
69b6f2e8 | 403 | static int __maybe_unused essiv_cbc_init_tfm(struct crypto_skcipher *tfm) |
735177ca AB |
404 | { |
405 | struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); | |
406 | ||
407 | ctx->hash = crypto_alloc_shash("sha256", 0, 0); | |
735177ca | 408 | |
7b865ec1 | 409 | return PTR_ERR_OR_ZERO(ctx->hash); |
735177ca AB |
410 | } |
411 | ||
69b6f2e8 | 412 | static void __maybe_unused essiv_cbc_exit_tfm(struct crypto_skcipher *tfm) |
735177ca AB |
413 | { |
414 | struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); | |
415 | ||
416 | crypto_free_shash(ctx->hash); | |
417 | } | |
418 | ||
69b6f2e8 | 419 | static int __maybe_unused essiv_cbc_encrypt(struct skcipher_request *req) |
735177ca AB |
420 | { |
421 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
422 | struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); | |
423 | int err, rounds = 6 + ctx->key1.key_length / 4; | |
424 | struct skcipher_walk walk; | |
425 | unsigned int blocks; | |
426 | ||
427 | err = skcipher_walk_virt(&walk, req, false); | |
428 | ||
429 | blocks = walk.nbytes / AES_BLOCK_SIZE; | |
430 | if (blocks) { | |
431 | kernel_neon_begin(); | |
432 | aes_essiv_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, | |
433 | ctx->key1.key_enc, rounds, blocks, | |
434 | req->iv, ctx->key2.key_enc); | |
435 | kernel_neon_end(); | |
436 | err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); | |
437 | } | |
438 | return err ?: cbc_encrypt_walk(req, &walk); | |
439 | } | |
440 | ||
69b6f2e8 | 441 | static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req) |
735177ca AB |
442 | { |
443 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
444 | struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); | |
445 | int err, rounds = 6 + ctx->key1.key_length / 4; | |
446 | struct skcipher_walk walk; | |
447 | unsigned int blocks; | |
448 | ||
449 | err = skcipher_walk_virt(&walk, req, false); | |
450 | ||
451 | blocks = walk.nbytes / AES_BLOCK_SIZE; | |
452 | if (blocks) { | |
453 | kernel_neon_begin(); | |
454 | aes_essiv_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, | |
455 | ctx->key1.key_dec, rounds, blocks, | |
456 | req->iv, ctx->key2.key_enc); | |
457 | kernel_neon_end(); | |
458 | err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); | |
459 | } | |
460 | return err ?: cbc_decrypt_walk(req, &walk); | |
461 | } | |
462 | ||
d0ed0db1 | 463 | static int ctr_encrypt(struct skcipher_request *req) |
49788fe2 | 464 | { |
d0ed0db1 HX |
465 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
466 | struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); | |
68338174 | 467 | int err, rounds = 6 + ctx->key_length / 4; |
d0ed0db1 | 468 | struct skcipher_walk walk; |
49788fe2 AB |
469 | int blocks; |
470 | ||
68338174 | 471 | err = skcipher_walk_virt(&walk, req, false); |
49788fe2 | 472 | |
49788fe2 | 473 | while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { |
68338174 | 474 | kernel_neon_begin(); |
49788fe2 | 475 | aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, |
557ecb45 | 476 | ctx->key_enc, rounds, blocks, walk.iv); |
68338174 | 477 | kernel_neon_end(); |
6e88f012 | 478 | err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); |
49788fe2 | 479 | } |
d0ed0db1 | 480 | if (walk.nbytes) { |
49788fe2 | 481 | u8 __aligned(8) tail[AES_BLOCK_SIZE]; |
d0ed0db1 HX |
482 | unsigned int nbytes = walk.nbytes; |
483 | u8 *tdst = walk.dst.virt.addr; | |
484 | u8 *tsrc = walk.src.virt.addr; | |
49788fe2 AB |
485 | |
486 | /* | |
ccc5d51e | 487 | * Tell aes_ctr_encrypt() to process a tail block. |
49788fe2 | 488 | */ |
ccc5d51e | 489 | blocks = -1; |
49788fe2 | 490 | |
68338174 | 491 | kernel_neon_begin(); |
557ecb45 | 492 | aes_ctr_encrypt(tail, NULL, ctx->key_enc, rounds, |
68338174 AB |
493 | blocks, walk.iv); |
494 | kernel_neon_end(); | |
45fe93df | 495 | crypto_xor_cpy(tdst, tsrc, tail, nbytes); |
d0ed0db1 | 496 | err = skcipher_walk_done(&walk, 0); |
49788fe2 | 497 | } |
49788fe2 AB |
498 | |
499 | return err; | |
500 | } | |
501 | ||
ff6f4115 | 502 | static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) |
e2115069 | 503 | { |
ff6f4115 AB |
504 | const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); |
505 | unsigned long flags; | |
506 | ||
507 | /* | |
508 | * Temporarily disable interrupts to avoid races where | |
509 | * cachelines are evicted when the CPU is interrupted | |
510 | * to do something else. | |
511 | */ | |
512 | local_irq_save(flags); | |
513 | aes_encrypt(ctx, dst, src); | |
514 | local_irq_restore(flags); | |
515 | } | |
e2115069 | 516 | |
69b6f2e8 | 517 | static int __maybe_unused ctr_encrypt_sync(struct skcipher_request *req) |
ff6f4115 | 518 | { |
e52b7023 | 519 | if (!crypto_simd_usable()) |
ff6f4115 | 520 | return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); |
e2115069 AB |
521 | |
522 | return ctr_encrypt(req); | |
523 | } | |
524 | ||
69b6f2e8 | 525 | static int __maybe_unused xts_encrypt(struct skcipher_request *req) |
49788fe2 | 526 | { |
d0ed0db1 HX |
527 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
528 | struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
49788fe2 | 529 | int err, first, rounds = 6 + ctx->key1.key_length / 4; |
d0ed0db1 | 530 | struct skcipher_walk walk; |
49788fe2 AB |
531 | unsigned int blocks; |
532 | ||
68338174 | 533 | err = skcipher_walk_virt(&walk, req, false); |
49788fe2 | 534 | |
49788fe2 | 535 | for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { |
68338174 | 536 | kernel_neon_begin(); |
49788fe2 | 537 | aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, |
557ecb45 AB |
538 | ctx->key1.key_enc, rounds, blocks, |
539 | ctx->key2.key_enc, walk.iv, first); | |
68338174 | 540 | kernel_neon_end(); |
d0ed0db1 | 541 | err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); |
49788fe2 | 542 | } |
49788fe2 AB |
543 | |
544 | return err; | |
545 | } | |
546 | ||
69b6f2e8 | 547 | static int __maybe_unused xts_decrypt(struct skcipher_request *req) |
49788fe2 | 548 | { |
d0ed0db1 HX |
549 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
550 | struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
49788fe2 | 551 | int err, first, rounds = 6 + ctx->key1.key_length / 4; |
d0ed0db1 | 552 | struct skcipher_walk walk; |
49788fe2 AB |
553 | unsigned int blocks; |
554 | ||
68338174 | 555 | err = skcipher_walk_virt(&walk, req, false); |
49788fe2 | 556 | |
49788fe2 | 557 | for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { |
68338174 | 558 | kernel_neon_begin(); |
49788fe2 | 559 | aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, |
557ecb45 AB |
560 | ctx->key1.key_dec, rounds, blocks, |
561 | ctx->key2.key_enc, walk.iv, first); | |
68338174 | 562 | kernel_neon_end(); |
d0ed0db1 | 563 | err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); |
49788fe2 | 564 | } |
49788fe2 AB |
565 | |
566 | return err; | |
567 | } | |
568 | ||
d0ed0db1 | 569 | static struct skcipher_alg aes_algs[] = { { |
69b6f2e8 | 570 | #if defined(USE_V8_CRYPTO_EXTENSIONS) || !defined(CONFIG_CRYPTO_AES_ARM64_BS) |
d0ed0db1 HX |
571 | .base = { |
572 | .cra_name = "__ecb(aes)", | |
573 | .cra_driver_name = "__ecb-aes-" MODE, | |
574 | .cra_priority = PRIO, | |
575 | .cra_flags = CRYPTO_ALG_INTERNAL, | |
576 | .cra_blocksize = AES_BLOCK_SIZE, | |
577 | .cra_ctxsize = sizeof(struct crypto_aes_ctx), | |
d0ed0db1 | 578 | .cra_module = THIS_MODULE, |
49788fe2 | 579 | }, |
d0ed0db1 HX |
580 | .min_keysize = AES_MIN_KEY_SIZE, |
581 | .max_keysize = AES_MAX_KEY_SIZE, | |
582 | .setkey = skcipher_aes_setkey, | |
583 | .encrypt = ecb_encrypt, | |
584 | .decrypt = ecb_decrypt, | |
49788fe2 | 585 | }, { |
d0ed0db1 HX |
586 | .base = { |
587 | .cra_name = "__cbc(aes)", | |
588 | .cra_driver_name = "__cbc-aes-" MODE, | |
589 | .cra_priority = PRIO, | |
590 | .cra_flags = CRYPTO_ALG_INTERNAL, | |
591 | .cra_blocksize = AES_BLOCK_SIZE, | |
592 | .cra_ctxsize = sizeof(struct crypto_aes_ctx), | |
d0ed0db1 | 593 | .cra_module = THIS_MODULE, |
49788fe2 | 594 | }, |
d0ed0db1 HX |
595 | .min_keysize = AES_MIN_KEY_SIZE, |
596 | .max_keysize = AES_MAX_KEY_SIZE, | |
597 | .ivsize = AES_BLOCK_SIZE, | |
598 | .setkey = skcipher_aes_setkey, | |
599 | .encrypt = cbc_encrypt, | |
600 | .decrypt = cbc_decrypt, | |
49788fe2 | 601 | }, { |
d0ed0db1 HX |
602 | .base = { |
603 | .cra_name = "__ctr(aes)", | |
604 | .cra_driver_name = "__ctr-aes-" MODE, | |
605 | .cra_priority = PRIO, | |
606 | .cra_flags = CRYPTO_ALG_INTERNAL, | |
607 | .cra_blocksize = 1, | |
608 | .cra_ctxsize = sizeof(struct crypto_aes_ctx), | |
d0ed0db1 | 609 | .cra_module = THIS_MODULE, |
49788fe2 | 610 | }, |
d0ed0db1 HX |
611 | .min_keysize = AES_MIN_KEY_SIZE, |
612 | .max_keysize = AES_MAX_KEY_SIZE, | |
613 | .ivsize = AES_BLOCK_SIZE, | |
614 | .chunksize = AES_BLOCK_SIZE, | |
615 | .setkey = skcipher_aes_setkey, | |
616 | .encrypt = ctr_encrypt, | |
617 | .decrypt = ctr_encrypt, | |
293614ce AB |
618 | }, { |
619 | .base = { | |
620 | .cra_name = "ctr(aes)", | |
621 | .cra_driver_name = "ctr-aes-" MODE, | |
622 | .cra_priority = PRIO - 1, | |
623 | .cra_blocksize = 1, | |
624 | .cra_ctxsize = sizeof(struct crypto_aes_ctx), | |
293614ce AB |
625 | .cra_module = THIS_MODULE, |
626 | }, | |
627 | .min_keysize = AES_MIN_KEY_SIZE, | |
628 | .max_keysize = AES_MAX_KEY_SIZE, | |
629 | .ivsize = AES_BLOCK_SIZE, | |
630 | .chunksize = AES_BLOCK_SIZE, | |
631 | .setkey = skcipher_aes_setkey, | |
e2115069 AB |
632 | .encrypt = ctr_encrypt_sync, |
633 | .decrypt = ctr_encrypt_sync, | |
49788fe2 | 634 | }, { |
d0ed0db1 HX |
635 | .base = { |
636 | .cra_name = "__xts(aes)", | |
637 | .cra_driver_name = "__xts-aes-" MODE, | |
638 | .cra_priority = PRIO, | |
639 | .cra_flags = CRYPTO_ALG_INTERNAL, | |
640 | .cra_blocksize = AES_BLOCK_SIZE, | |
641 | .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), | |
d0ed0db1 | 642 | .cra_module = THIS_MODULE, |
49788fe2 | 643 | }, |
d0ed0db1 HX |
644 | .min_keysize = 2 * AES_MIN_KEY_SIZE, |
645 | .max_keysize = 2 * AES_MAX_KEY_SIZE, | |
646 | .ivsize = AES_BLOCK_SIZE, | |
647 | .setkey = xts_set_key, | |
648 | .encrypt = xts_encrypt, | |
649 | .decrypt = xts_decrypt, | |
69b6f2e8 AB |
650 | }, { |
651 | #endif | |
652 | .base = { | |
653 | .cra_name = "__cts(cbc(aes))", | |
654 | .cra_driver_name = "__cts-cbc-aes-" MODE, | |
655 | .cra_priority = PRIO, | |
656 | .cra_flags = CRYPTO_ALG_INTERNAL, | |
657 | .cra_blocksize = AES_BLOCK_SIZE, | |
658 | .cra_ctxsize = sizeof(struct crypto_aes_ctx), | |
659 | .cra_module = THIS_MODULE, | |
660 | }, | |
661 | .min_keysize = AES_MIN_KEY_SIZE, | |
662 | .max_keysize = AES_MAX_KEY_SIZE, | |
663 | .ivsize = AES_BLOCK_SIZE, | |
664 | .walksize = 2 * AES_BLOCK_SIZE, | |
665 | .setkey = skcipher_aes_setkey, | |
666 | .encrypt = cts_cbc_encrypt, | |
667 | .decrypt = cts_cbc_decrypt, | |
69b6f2e8 AB |
668 | }, { |
669 | .base = { | |
670 | .cra_name = "__essiv(cbc(aes),sha256)", | |
671 | .cra_driver_name = "__essiv-cbc-aes-sha256-" MODE, | |
672 | .cra_priority = PRIO + 1, | |
673 | .cra_flags = CRYPTO_ALG_INTERNAL, | |
674 | .cra_blocksize = AES_BLOCK_SIZE, | |
675 | .cra_ctxsize = sizeof(struct crypto_aes_essiv_cbc_ctx), | |
676 | .cra_module = THIS_MODULE, | |
677 | }, | |
678 | .min_keysize = AES_MIN_KEY_SIZE, | |
679 | .max_keysize = AES_MAX_KEY_SIZE, | |
680 | .ivsize = AES_BLOCK_SIZE, | |
681 | .setkey = essiv_cbc_set_key, | |
682 | .encrypt = essiv_cbc_encrypt, | |
683 | .decrypt = essiv_cbc_decrypt, | |
684 | .init = essiv_cbc_init_tfm, | |
685 | .exit = essiv_cbc_exit_tfm, | |
49788fe2 AB |
686 | } }; |
687 | ||
4860620d AB |
688 | static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key, |
689 | unsigned int key_len) | |
690 | { | |
691 | struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); | |
692 | int err; | |
693 | ||
694 | err = aes_expandkey(&ctx->key, in_key, key_len); | |
695 | if (err) | |
696 | crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); | |
697 | ||
698 | return err; | |
699 | } | |
700 | ||
701 | static void cmac_gf128_mul_by_x(be128 *y, const be128 *x) | |
702 | { | |
703 | u64 a = be64_to_cpu(x->a); | |
704 | u64 b = be64_to_cpu(x->b); | |
705 | ||
706 | y->a = cpu_to_be64((a << 1) | (b >> 63)); | |
707 | y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); | |
708 | } | |
709 | ||
710 | static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key, | |
711 | unsigned int key_len) | |
712 | { | |
713 | struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); | |
714 | be128 *consts = (be128 *)ctx->consts; | |
4860620d AB |
715 | int rounds = 6 + key_len / 4; |
716 | int err; | |
717 | ||
718 | err = cbcmac_setkey(tfm, in_key, key_len); | |
719 | if (err) | |
720 | return err; | |
721 | ||
722 | /* encrypt the zero vector */ | |
723 | kernel_neon_begin(); | |
557ecb45 AB |
724 | aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc, |
725 | rounds, 1); | |
4860620d AB |
726 | kernel_neon_end(); |
727 | ||
728 | cmac_gf128_mul_by_x(consts, consts); | |
729 | cmac_gf128_mul_by_x(consts + 1, consts); | |
730 | ||
731 | return 0; | |
732 | } | |
733 | ||
734 | static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key, | |
735 | unsigned int key_len) | |
736 | { | |
737 | static u8 const ks[3][AES_BLOCK_SIZE] = { | |
738 | { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, | |
739 | { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, | |
740 | { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, | |
741 | }; | |
742 | ||
743 | struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); | |
4860620d AB |
744 | int rounds = 6 + key_len / 4; |
745 | u8 key[AES_BLOCK_SIZE]; | |
746 | int err; | |
747 | ||
748 | err = cbcmac_setkey(tfm, in_key, key_len); | |
749 | if (err) | |
750 | return err; | |
751 | ||
752 | kernel_neon_begin(); | |
557ecb45 AB |
753 | aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1); |
754 | aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2); | |
4860620d AB |
755 | kernel_neon_end(); |
756 | ||
757 | return cbcmac_setkey(tfm, key, sizeof(key)); | |
758 | } | |
759 | ||
760 | static int mac_init(struct shash_desc *desc) | |
761 | { | |
762 | struct mac_desc_ctx *ctx = shash_desc_ctx(desc); | |
763 | ||
764 | memset(ctx->dg, 0, AES_BLOCK_SIZE); | |
765 | ctx->len = 0; | |
766 | ||
767 | return 0; | |
768 | } | |
769 | ||
e2115069 AB |
770 | static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, |
771 | u8 dg[], int enc_before, int enc_after) | |
772 | { | |
773 | int rounds = 6 + ctx->key_length / 4; | |
774 | ||
e52b7023 | 775 | if (crypto_simd_usable()) { |
e2115069 AB |
776 | kernel_neon_begin(); |
777 | aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before, | |
778 | enc_after); | |
779 | kernel_neon_end(); | |
780 | } else { | |
781 | if (enc_before) | |
c1844729 | 782 | aes_encrypt(ctx, dg, dg); |
e2115069 AB |
783 | |
784 | while (blocks--) { | |
785 | crypto_xor(dg, in, AES_BLOCK_SIZE); | |
786 | in += AES_BLOCK_SIZE; | |
787 | ||
788 | if (blocks || enc_after) | |
c1844729 | 789 | aes_encrypt(ctx, dg, dg); |
e2115069 AB |
790 | } |
791 | } | |
792 | } | |
793 | ||
4860620d AB |
794 | static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) |
795 | { | |
796 | struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); | |
797 | struct mac_desc_ctx *ctx = shash_desc_ctx(desc); | |
4860620d AB |
798 | |
799 | while (len > 0) { | |
800 | unsigned int l; | |
801 | ||
802 | if ((ctx->len % AES_BLOCK_SIZE) == 0 && | |
803 | (ctx->len + len) > AES_BLOCK_SIZE) { | |
804 | ||
805 | int blocks = len / AES_BLOCK_SIZE; | |
806 | ||
807 | len %= AES_BLOCK_SIZE; | |
808 | ||
e2115069 AB |
809 | mac_do_update(&tctx->key, p, blocks, ctx->dg, |
810 | (ctx->len != 0), (len != 0)); | |
4860620d AB |
811 | |
812 | p += blocks * AES_BLOCK_SIZE; | |
813 | ||
814 | if (!len) { | |
815 | ctx->len = AES_BLOCK_SIZE; | |
816 | break; | |
817 | } | |
818 | ctx->len = 0; | |
819 | } | |
820 | ||
821 | l = min(len, AES_BLOCK_SIZE - ctx->len); | |
822 | ||
823 | if (l <= AES_BLOCK_SIZE) { | |
824 | crypto_xor(ctx->dg + ctx->len, p, l); | |
825 | ctx->len += l; | |
826 | len -= l; | |
827 | p += l; | |
828 | } | |
829 | } | |
830 | ||
831 | return 0; | |
832 | } | |
833 | ||
834 | static int cbcmac_final(struct shash_desc *desc, u8 *out) | |
835 | { | |
836 | struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); | |
837 | struct mac_desc_ctx *ctx = shash_desc_ctx(desc); | |
4860620d | 838 | |
f6e9af87 | 839 | mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0); |
4860620d AB |
840 | |
841 | memcpy(out, ctx->dg, AES_BLOCK_SIZE); | |
842 | ||
843 | return 0; | |
844 | } | |
845 | ||
846 | static int cmac_final(struct shash_desc *desc, u8 *out) | |
847 | { | |
848 | struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); | |
849 | struct mac_desc_ctx *ctx = shash_desc_ctx(desc); | |
4860620d AB |
850 | u8 *consts = tctx->consts; |
851 | ||
852 | if (ctx->len != AES_BLOCK_SIZE) { | |
853 | ctx->dg[ctx->len] ^= 0x80; | |
854 | consts += AES_BLOCK_SIZE; | |
855 | } | |
856 | ||
e2115069 | 857 | mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1); |
4860620d AB |
858 | |
859 | memcpy(out, ctx->dg, AES_BLOCK_SIZE); | |
860 | ||
861 | return 0; | |
862 | } | |
863 | ||
864 | static struct shash_alg mac_algs[] = { { | |
865 | .base.cra_name = "cmac(aes)", | |
866 | .base.cra_driver_name = "cmac-aes-" MODE, | |
867 | .base.cra_priority = PRIO, | |
4860620d AB |
868 | .base.cra_blocksize = AES_BLOCK_SIZE, |
869 | .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + | |
870 | 2 * AES_BLOCK_SIZE, | |
871 | .base.cra_module = THIS_MODULE, | |
872 | ||
873 | .digestsize = AES_BLOCK_SIZE, | |
874 | .init = mac_init, | |
875 | .update = mac_update, | |
876 | .final = cmac_final, | |
877 | .setkey = cmac_setkey, | |
878 | .descsize = sizeof(struct mac_desc_ctx), | |
879 | }, { | |
880 | .base.cra_name = "xcbc(aes)", | |
881 | .base.cra_driver_name = "xcbc-aes-" MODE, | |
882 | .base.cra_priority = PRIO, | |
4860620d AB |
883 | .base.cra_blocksize = AES_BLOCK_SIZE, |
884 | .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + | |
885 | 2 * AES_BLOCK_SIZE, | |
886 | .base.cra_module = THIS_MODULE, | |
887 | ||
888 | .digestsize = AES_BLOCK_SIZE, | |
889 | .init = mac_init, | |
890 | .update = mac_update, | |
891 | .final = cmac_final, | |
892 | .setkey = xcbc_setkey, | |
893 | .descsize = sizeof(struct mac_desc_ctx), | |
894 | }, { | |
895 | .base.cra_name = "cbcmac(aes)", | |
896 | .base.cra_driver_name = "cbcmac-aes-" MODE, | |
897 | .base.cra_priority = PRIO, | |
4860620d AB |
898 | .base.cra_blocksize = 1, |
899 | .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), | |
900 | .base.cra_module = THIS_MODULE, | |
901 | ||
902 | .digestsize = AES_BLOCK_SIZE, | |
903 | .init = mac_init, | |
904 | .update = mac_update, | |
905 | .final = cbcmac_final, | |
906 | .setkey = cbcmac_setkey, | |
907 | .descsize = sizeof(struct mac_desc_ctx), | |
908 | } }; | |
909 | ||
7f329c17 | 910 | static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; |
d0ed0db1 HX |
911 | |
912 | static void aes_exit(void) | |
49788fe2 | 913 | { |
d0ed0db1 HX |
914 | int i; |
915 | ||
293614ce AB |
916 | for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) |
917 | if (aes_simd_algs[i]) | |
918 | simd_skcipher_free(aes_simd_algs[i]); | |
d0ed0db1 | 919 | |
4860620d | 920 | crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); |
d0ed0db1 | 921 | crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); |
49788fe2 AB |
922 | } |
923 | ||
d0ed0db1 | 924 | static int __init aes_init(void) |
49788fe2 | 925 | { |
d0ed0db1 HX |
926 | struct simd_skcipher_alg *simd; |
927 | const char *basename; | |
928 | const char *algname; | |
929 | const char *drvname; | |
930 | int err; | |
931 | int i; | |
932 | ||
933 | err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); | |
934 | if (err) | |
935 | return err; | |
936 | ||
4860620d AB |
937 | err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); |
938 | if (err) | |
939 | goto unregister_ciphers; | |
940 | ||
d0ed0db1 | 941 | for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { |
293614ce AB |
942 | if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) |
943 | continue; | |
944 | ||
d0ed0db1 HX |
945 | algname = aes_algs[i].base.cra_name + 2; |
946 | drvname = aes_algs[i].base.cra_driver_name + 2; | |
947 | basename = aes_algs[i].base.cra_driver_name; | |
948 | simd = simd_skcipher_create_compat(algname, drvname, basename); | |
949 | err = PTR_ERR(simd); | |
950 | if (IS_ERR(simd)) | |
951 | goto unregister_simds; | |
952 | ||
953 | aes_simd_algs[i] = simd; | |
954 | } | |
955 | ||
956 | return 0; | |
957 | ||
958 | unregister_simds: | |
959 | aes_exit(); | |
45223b78 | 960 | return err; |
4860620d AB |
961 | unregister_ciphers: |
962 | crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); | |
d0ed0db1 | 963 | return err; |
49788fe2 AB |
964 | } |
965 | ||
966 | #ifdef USE_V8_CRYPTO_EXTENSIONS | |
967 | module_cpu_feature_match(AES, aes_init); | |
968 | #else | |
969 | module_init(aes_init); | |
4edd7d01 AB |
970 | EXPORT_SYMBOL(neon_aes_ecb_encrypt); |
971 | EXPORT_SYMBOL(neon_aes_cbc_encrypt); | |
49788fe2 AB |
972 | #endif |
973 | module_exit(aes_exit); |