]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - arch/x86/crypto/aesni-intel_glue.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input
[mirror_ubuntu-artful-kernel.git] / arch / x86 / crypto / aesni-intel_glue.c
1 /*
2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
4 *
5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
7 *
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
10 * Authors: Adrian Hoban <adrian.hoban@intel.com>
11 * Gabriele Paoloni <gabriele.paoloni@intel.com>
12 * Tadeusz Struk (tadeusz.struk@intel.com)
13 * Aidan O'Mahony (aidan.o.mahony@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 */
21
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/module.h>
25 #include <linux/err.h>
26 #include <crypto/algapi.h>
27 #include <crypto/aes.h>
28 #include <crypto/cryptd.h>
29 #include <crypto/ctr.h>
30 #include <crypto/b128ops.h>
31 #include <crypto/xts.h>
32 #include <asm/cpu_device_id.h>
33 #include <asm/fpu/api.h>
34 #include <asm/crypto/aes.h>
35 #include <crypto/scatterwalk.h>
36 #include <crypto/internal/aead.h>
37 #include <crypto/internal/simd.h>
38 #include <crypto/internal/skcipher.h>
39 #include <linux/workqueue.h>
40 #include <linux/spinlock.h>
41 #ifdef CONFIG_X86_64
42 #include <asm/crypto/glue_helper.h>
43 #endif
44
45
46 #define AESNI_ALIGN 16
47 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
48 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
49 #define RFC4106_HASH_SUBKEY_SIZE 16
50 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
51 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
52 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
53
54 /* This data is stored at the end of the crypto_tfm struct.
55 * It's a type of per "session" data storage location.
56 * This needs to be 16 byte aligned.
57 */
58 struct aesni_rfc4106_gcm_ctx {
59 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
60 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
61 u8 nonce[4];
62 };
63
64 struct aesni_xts_ctx {
65 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
66 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
67 };
68
69 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
70 unsigned int key_len);
71 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
72 const u8 *in);
73 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
74 const u8 *in);
75 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
76 const u8 *in, unsigned int len);
77 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
78 const u8 *in, unsigned int len);
79 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
80 const u8 *in, unsigned int len, u8 *iv);
81 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
82 const u8 *in, unsigned int len, u8 *iv);
83
84 int crypto_fpu_init(void);
85 void crypto_fpu_exit(void);
86
87 #define AVX_GEN2_OPTSIZE 640
88 #define AVX_GEN4_OPTSIZE 4096
89
90 #ifdef CONFIG_X86_64
91
92 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
93 const u8 *in, unsigned int len, u8 *iv);
94 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
95 const u8 *in, unsigned int len, u8 *iv);
96
97 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
98 const u8 *in, bool enc, u8 *iv);
99
100 /* asmlinkage void aesni_gcm_enc()
101 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
102 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
103 * const u8 *in, Plaintext input
104 * unsigned long plaintext_len, Length of data in bytes for encryption.
105 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
106 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
107 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
108 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
109 * const u8 *aad, Additional Authentication Data (AAD)
110 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
111 * is going to be 8 or 12 bytes
112 * u8 *auth_tag, Authenticated Tag output.
113 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
114 * Valid values are 16 (most likely), 12 or 8.
115 */
116 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
117 const u8 *in, unsigned long plaintext_len, u8 *iv,
118 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
119 u8 *auth_tag, unsigned long auth_tag_len);
120
121 /* asmlinkage void aesni_gcm_dec()
122 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
123 * u8 *out, Plaintext output. Decrypt in-place is allowed.
124 * const u8 *in, Ciphertext input
125 * unsigned long ciphertext_len, Length of data in bytes for decryption.
126 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
127 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
128 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
129 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
130 * const u8 *aad, Additional Authentication Data (AAD)
131 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
132 * to be 8 or 12 bytes
133 * u8 *auth_tag, Authenticated Tag output.
134 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
135 * Valid values are 16 (most likely), 12 or 8.
136 */
137 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
138 const u8 *in, unsigned long ciphertext_len, u8 *iv,
139 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
140 u8 *auth_tag, unsigned long auth_tag_len);
141
142
143 #ifdef CONFIG_AS_AVX
144 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
145 void *keys, u8 *out, unsigned int num_bytes);
146 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
147 void *keys, u8 *out, unsigned int num_bytes);
148 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
149 void *keys, u8 *out, unsigned int num_bytes);
150 /*
151 * asmlinkage void aesni_gcm_precomp_avx_gen2()
152 * gcm_data *my_ctx_data, context data
153 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
154 */
155 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey);
156
157 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
158 const u8 *in, unsigned long plaintext_len, u8 *iv,
159 const u8 *aad, unsigned long aad_len,
160 u8 *auth_tag, unsigned long auth_tag_len);
161
162 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
163 const u8 *in, unsigned long ciphertext_len, u8 *iv,
164 const u8 *aad, unsigned long aad_len,
165 u8 *auth_tag, unsigned long auth_tag_len);
166
167 static void aesni_gcm_enc_avx(void *ctx, u8 *out,
168 const u8 *in, unsigned long plaintext_len, u8 *iv,
169 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
170 u8 *auth_tag, unsigned long auth_tag_len)
171 {
172 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
173 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){
174 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
175 aad_len, auth_tag, auth_tag_len);
176 } else {
177 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
178 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
179 aad_len, auth_tag, auth_tag_len);
180 }
181 }
182
183 static void aesni_gcm_dec_avx(void *ctx, u8 *out,
184 const u8 *in, unsigned long ciphertext_len, u8 *iv,
185 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
186 u8 *auth_tag, unsigned long auth_tag_len)
187 {
188 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
189 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
190 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,
191 aad_len, auth_tag, auth_tag_len);
192 } else {
193 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
194 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
195 aad_len, auth_tag, auth_tag_len);
196 }
197 }
198 #endif
199
200 #ifdef CONFIG_AS_AVX2
201 /*
202 * asmlinkage void aesni_gcm_precomp_avx_gen4()
203 * gcm_data *my_ctx_data, context data
204 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
205 */
206 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey);
207
208 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
209 const u8 *in, unsigned long plaintext_len, u8 *iv,
210 const u8 *aad, unsigned long aad_len,
211 u8 *auth_tag, unsigned long auth_tag_len);
212
213 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
214 const u8 *in, unsigned long ciphertext_len, u8 *iv,
215 const u8 *aad, unsigned long aad_len,
216 u8 *auth_tag, unsigned long auth_tag_len);
217
218 static void aesni_gcm_enc_avx2(void *ctx, u8 *out,
219 const u8 *in, unsigned long plaintext_len, u8 *iv,
220 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
221 u8 *auth_tag, unsigned long auth_tag_len)
222 {
223 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
224 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
225 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
226 aad_len, auth_tag, auth_tag_len);
227 } else if (plaintext_len < AVX_GEN4_OPTSIZE) {
228 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
229 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
230 aad_len, auth_tag, auth_tag_len);
231 } else {
232 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
233 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,
234 aad_len, auth_tag, auth_tag_len);
235 }
236 }
237
238 static void aesni_gcm_dec_avx2(void *ctx, u8 *out,
239 const u8 *in, unsigned long ciphertext_len, u8 *iv,
240 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
241 u8 *auth_tag, unsigned long auth_tag_len)
242 {
243 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
244 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
245 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,
246 aad, aad_len, auth_tag, auth_tag_len);
247 } else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
248 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
249 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
250 aad_len, auth_tag, auth_tag_len);
251 } else {
252 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
253 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,
254 aad_len, auth_tag, auth_tag_len);
255 }
256 }
257 #endif
258
259 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out,
260 const u8 *in, unsigned long plaintext_len, u8 *iv,
261 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
262 u8 *auth_tag, unsigned long auth_tag_len);
263
264 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
265 const u8 *in, unsigned long ciphertext_len, u8 *iv,
266 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
267 u8 *auth_tag, unsigned long auth_tag_len);
268
269 static inline struct
270 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
271 {
272 unsigned long align = AESNI_ALIGN;
273
274 if (align <= crypto_tfm_ctx_alignment())
275 align = 1;
276 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
277 }
278 #endif
279
280 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
281 {
282 unsigned long addr = (unsigned long)raw_ctx;
283 unsigned long align = AESNI_ALIGN;
284
285 if (align <= crypto_tfm_ctx_alignment())
286 align = 1;
287 return (struct crypto_aes_ctx *)ALIGN(addr, align);
288 }
289
290 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
291 const u8 *in_key, unsigned int key_len)
292 {
293 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
294 u32 *flags = &tfm->crt_flags;
295 int err;
296
297 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
298 key_len != AES_KEYSIZE_256) {
299 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
300 return -EINVAL;
301 }
302
303 if (!irq_fpu_usable())
304 err = crypto_aes_expand_key(ctx, in_key, key_len);
305 else {
306 kernel_fpu_begin();
307 err = aesni_set_key(ctx, in_key, key_len);
308 kernel_fpu_end();
309 }
310
311 return err;
312 }
313
314 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
315 unsigned int key_len)
316 {
317 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
318 }
319
320 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
321 {
322 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
323
324 if (!irq_fpu_usable())
325 crypto_aes_encrypt_x86(ctx, dst, src);
326 else {
327 kernel_fpu_begin();
328 aesni_enc(ctx, dst, src);
329 kernel_fpu_end();
330 }
331 }
332
333 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
334 {
335 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
336
337 if (!irq_fpu_usable())
338 crypto_aes_decrypt_x86(ctx, dst, src);
339 else {
340 kernel_fpu_begin();
341 aesni_dec(ctx, dst, src);
342 kernel_fpu_end();
343 }
344 }
345
346 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
347 {
348 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
349
350 aesni_enc(ctx, dst, src);
351 }
352
353 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
354 {
355 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
356
357 aesni_dec(ctx, dst, src);
358 }
359
360 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
361 unsigned int len)
362 {
363 return aes_set_key_common(crypto_skcipher_tfm(tfm),
364 crypto_skcipher_ctx(tfm), key, len);
365 }
366
367 static int ecb_encrypt(struct skcipher_request *req)
368 {
369 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
370 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
371 struct skcipher_walk walk;
372 unsigned int nbytes;
373 int err;
374
375 err = skcipher_walk_virt(&walk, req, true);
376
377 kernel_fpu_begin();
378 while ((nbytes = walk.nbytes)) {
379 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
380 nbytes & AES_BLOCK_MASK);
381 nbytes &= AES_BLOCK_SIZE - 1;
382 err = skcipher_walk_done(&walk, nbytes);
383 }
384 kernel_fpu_end();
385
386 return err;
387 }
388
389 static int ecb_decrypt(struct skcipher_request *req)
390 {
391 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
392 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
393 struct skcipher_walk walk;
394 unsigned int nbytes;
395 int err;
396
397 err = skcipher_walk_virt(&walk, req, true);
398
399 kernel_fpu_begin();
400 while ((nbytes = walk.nbytes)) {
401 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
402 nbytes & AES_BLOCK_MASK);
403 nbytes &= AES_BLOCK_SIZE - 1;
404 err = skcipher_walk_done(&walk, nbytes);
405 }
406 kernel_fpu_end();
407
408 return err;
409 }
410
411 static int cbc_encrypt(struct skcipher_request *req)
412 {
413 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
414 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
415 struct skcipher_walk walk;
416 unsigned int nbytes;
417 int err;
418
419 err = skcipher_walk_virt(&walk, req, true);
420
421 kernel_fpu_begin();
422 while ((nbytes = walk.nbytes)) {
423 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
424 nbytes & AES_BLOCK_MASK, walk.iv);
425 nbytes &= AES_BLOCK_SIZE - 1;
426 err = skcipher_walk_done(&walk, nbytes);
427 }
428 kernel_fpu_end();
429
430 return err;
431 }
432
433 static int cbc_decrypt(struct skcipher_request *req)
434 {
435 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
436 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
437 struct skcipher_walk walk;
438 unsigned int nbytes;
439 int err;
440
441 err = skcipher_walk_virt(&walk, req, true);
442
443 kernel_fpu_begin();
444 while ((nbytes = walk.nbytes)) {
445 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
446 nbytes & AES_BLOCK_MASK, walk.iv);
447 nbytes &= AES_BLOCK_SIZE - 1;
448 err = skcipher_walk_done(&walk, nbytes);
449 }
450 kernel_fpu_end();
451
452 return err;
453 }
454
455 #ifdef CONFIG_X86_64
456 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
457 struct skcipher_walk *walk)
458 {
459 u8 *ctrblk = walk->iv;
460 u8 keystream[AES_BLOCK_SIZE];
461 u8 *src = walk->src.virt.addr;
462 u8 *dst = walk->dst.virt.addr;
463 unsigned int nbytes = walk->nbytes;
464
465 aesni_enc(ctx, keystream, ctrblk);
466 crypto_xor(keystream, src, nbytes);
467 memcpy(dst, keystream, nbytes);
468 crypto_inc(ctrblk, AES_BLOCK_SIZE);
469 }
470
471 #ifdef CONFIG_AS_AVX
472 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
473 const u8 *in, unsigned int len, u8 *iv)
474 {
475 /*
476 * based on key length, override with the by8 version
477 * of ctr mode encryption/decryption for improved performance
478 * aes_set_key_common() ensures that key length is one of
479 * {128,192,256}
480 */
481 if (ctx->key_length == AES_KEYSIZE_128)
482 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
483 else if (ctx->key_length == AES_KEYSIZE_192)
484 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
485 else
486 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
487 }
488 #endif
489
490 static int ctr_crypt(struct skcipher_request *req)
491 {
492 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
493 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
494 struct skcipher_walk walk;
495 unsigned int nbytes;
496 int err;
497
498 err = skcipher_walk_virt(&walk, req, true);
499
500 kernel_fpu_begin();
501 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
502 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
503 nbytes & AES_BLOCK_MASK, walk.iv);
504 nbytes &= AES_BLOCK_SIZE - 1;
505 err = skcipher_walk_done(&walk, nbytes);
506 }
507 if (walk.nbytes) {
508 ctr_crypt_final(ctx, &walk);
509 err = skcipher_walk_done(&walk, 0);
510 }
511 kernel_fpu_end();
512
513 return err;
514 }
515
516 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
517 unsigned int keylen)
518 {
519 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
520 int err;
521
522 err = xts_verify_key(tfm, key, keylen);
523 if (err)
524 return err;
525
526 keylen /= 2;
527
528 /* first half of xts-key is for crypt */
529 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
530 key, keylen);
531 if (err)
532 return err;
533
534 /* second half of xts-key is for tweak */
535 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
536 key + keylen, keylen);
537 }
538
539
540 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
541 {
542 aesni_enc(ctx, out, in);
543 }
544
545 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
546 {
547 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
548 }
549
550 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
551 {
552 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
553 }
554
555 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
556 {
557 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
558 }
559
560 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
561 {
562 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
563 }
564
565 static const struct common_glue_ctx aesni_enc_xts = {
566 .num_funcs = 2,
567 .fpu_blocks_limit = 1,
568
569 .funcs = { {
570 .num_blocks = 8,
571 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
572 }, {
573 .num_blocks = 1,
574 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
575 } }
576 };
577
578 static const struct common_glue_ctx aesni_dec_xts = {
579 .num_funcs = 2,
580 .fpu_blocks_limit = 1,
581
582 .funcs = { {
583 .num_blocks = 8,
584 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
585 }, {
586 .num_blocks = 1,
587 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
588 } }
589 };
590
591 static int xts_encrypt(struct skcipher_request *req)
592 {
593 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
594 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
595
596 return glue_xts_req_128bit(&aesni_enc_xts, req,
597 XTS_TWEAK_CAST(aesni_xts_tweak),
598 aes_ctx(ctx->raw_tweak_ctx),
599 aes_ctx(ctx->raw_crypt_ctx));
600 }
601
602 static int xts_decrypt(struct skcipher_request *req)
603 {
604 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
605 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
606
607 return glue_xts_req_128bit(&aesni_dec_xts, req,
608 XTS_TWEAK_CAST(aesni_xts_tweak),
609 aes_ctx(ctx->raw_tweak_ctx),
610 aes_ctx(ctx->raw_crypt_ctx));
611 }
612
613 static int rfc4106_init(struct crypto_aead *aead)
614 {
615 struct cryptd_aead *cryptd_tfm;
616 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
617
618 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
619 CRYPTO_ALG_INTERNAL,
620 CRYPTO_ALG_INTERNAL);
621 if (IS_ERR(cryptd_tfm))
622 return PTR_ERR(cryptd_tfm);
623
624 *ctx = cryptd_tfm;
625 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
626 return 0;
627 }
628
629 static void rfc4106_exit(struct crypto_aead *aead)
630 {
631 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
632
633 cryptd_free_aead(*ctx);
634 }
635
636 static int
637 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
638 {
639 struct crypto_cipher *tfm;
640 int ret;
641
642 tfm = crypto_alloc_cipher("aes", 0, 0);
643 if (IS_ERR(tfm))
644 return PTR_ERR(tfm);
645
646 ret = crypto_cipher_setkey(tfm, key, key_len);
647 if (ret)
648 goto out_free_cipher;
649
650 /* Clear the data in the hash sub key container to zero.*/
651 /* We want to cipher all zeros to create the hash sub key. */
652 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
653
654 crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
655
656 out_free_cipher:
657 crypto_free_cipher(tfm);
658 return ret;
659 }
660
661 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
662 unsigned int key_len)
663 {
664 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
665
666 if (key_len < 4) {
667 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
668 return -EINVAL;
669 }
670 /*Account for 4 byte nonce at the end.*/
671 key_len -= 4;
672
673 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
674
675 return aes_set_key_common(crypto_aead_tfm(aead),
676 &ctx->aes_key_expanded, key, key_len) ?:
677 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
678 }
679
680 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
681 unsigned int key_len)
682 {
683 struct cryptd_aead **ctx = crypto_aead_ctx(parent);
684 struct cryptd_aead *cryptd_tfm = *ctx;
685
686 return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
687 }
688
689 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
690 unsigned int authsize)
691 {
692 switch (authsize) {
693 case 8:
694 case 12:
695 case 16:
696 break;
697 default:
698 return -EINVAL;
699 }
700
701 return 0;
702 }
703
704 /* This is the Integrity Check Value (aka the authentication tag length and can
705 * be 8, 12 or 16 bytes long. */
706 static int rfc4106_set_authsize(struct crypto_aead *parent,
707 unsigned int authsize)
708 {
709 struct cryptd_aead **ctx = crypto_aead_ctx(parent);
710 struct cryptd_aead *cryptd_tfm = *ctx;
711
712 return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
713 }
714
715 static int helper_rfc4106_encrypt(struct aead_request *req)
716 {
717 u8 one_entry_in_sg = 0;
718 u8 *src, *dst, *assoc;
719 __be32 counter = cpu_to_be32(1);
720 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
721 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
722 void *aes_ctx = &(ctx->aes_key_expanded);
723 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
724 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
725 struct scatter_walk src_sg_walk;
726 struct scatter_walk dst_sg_walk = {};
727 unsigned int i;
728
729 /* Assuming we are supporting rfc4106 64-bit extended */
730 /* sequence numbers We need to have the AAD length equal */
731 /* to 16 or 20 bytes */
732 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
733 return -EINVAL;
734
735 /* IV below built */
736 for (i = 0; i < 4; i++)
737 *(iv+i) = ctx->nonce[i];
738 for (i = 0; i < 8; i++)
739 *(iv+4+i) = req->iv[i];
740 *((__be32 *)(iv+12)) = counter;
741
742 if (sg_is_last(req->src) &&
743 (!PageHighMem(sg_page(req->src)) ||
744 req->src->offset + req->src->length <= PAGE_SIZE) &&
745 sg_is_last(req->dst) &&
746 (!PageHighMem(sg_page(req->dst)) ||
747 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
748 one_entry_in_sg = 1;
749 scatterwalk_start(&src_sg_walk, req->src);
750 assoc = scatterwalk_map(&src_sg_walk);
751 src = assoc + req->assoclen;
752 dst = src;
753 if (unlikely(req->src != req->dst)) {
754 scatterwalk_start(&dst_sg_walk, req->dst);
755 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
756 }
757 } else {
758 /* Allocate memory for src, dst, assoc */
759 assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
760 GFP_ATOMIC);
761 if (unlikely(!assoc))
762 return -ENOMEM;
763 scatterwalk_map_and_copy(assoc, req->src, 0,
764 req->assoclen + req->cryptlen, 0);
765 src = assoc + req->assoclen;
766 dst = src;
767 }
768
769 kernel_fpu_begin();
770 aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,
771 ctx->hash_subkey, assoc, req->assoclen - 8,
772 dst + req->cryptlen, auth_tag_len);
773 kernel_fpu_end();
774
775 /* The authTag (aka the Integrity Check Value) needs to be written
776 * back to the packet. */
777 if (one_entry_in_sg) {
778 if (unlikely(req->src != req->dst)) {
779 scatterwalk_unmap(dst - req->assoclen);
780 scatterwalk_advance(&dst_sg_walk, req->dst->length);
781 scatterwalk_done(&dst_sg_walk, 1, 0);
782 }
783 scatterwalk_unmap(assoc);
784 scatterwalk_advance(&src_sg_walk, req->src->length);
785 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
786 } else {
787 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
788 req->cryptlen + auth_tag_len, 1);
789 kfree(assoc);
790 }
791 return 0;
792 }
793
794 static int helper_rfc4106_decrypt(struct aead_request *req)
795 {
796 u8 one_entry_in_sg = 0;
797 u8 *src, *dst, *assoc;
798 unsigned long tempCipherLen = 0;
799 __be32 counter = cpu_to_be32(1);
800 int retval = 0;
801 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
802 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
803 void *aes_ctx = &(ctx->aes_key_expanded);
804 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
805 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
806 u8 authTag[16];
807 struct scatter_walk src_sg_walk;
808 struct scatter_walk dst_sg_walk = {};
809 unsigned int i;
810
811 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
812 return -EINVAL;
813
814 /* Assuming we are supporting rfc4106 64-bit extended */
815 /* sequence numbers We need to have the AAD length */
816 /* equal to 16 or 20 bytes */
817
818 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
819 /* IV below built */
820 for (i = 0; i < 4; i++)
821 *(iv+i) = ctx->nonce[i];
822 for (i = 0; i < 8; i++)
823 *(iv+4+i) = req->iv[i];
824 *((__be32 *)(iv+12)) = counter;
825
826 if (sg_is_last(req->src) &&
827 (!PageHighMem(sg_page(req->src)) ||
828 req->src->offset + req->src->length <= PAGE_SIZE) &&
829 sg_is_last(req->dst) &&
830 (!PageHighMem(sg_page(req->dst)) ||
831 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
832 one_entry_in_sg = 1;
833 scatterwalk_start(&src_sg_walk, req->src);
834 assoc = scatterwalk_map(&src_sg_walk);
835 src = assoc + req->assoclen;
836 dst = src;
837 if (unlikely(req->src != req->dst)) {
838 scatterwalk_start(&dst_sg_walk, req->dst);
839 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
840 }
841
842 } else {
843 /* Allocate memory for src, dst, assoc */
844 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
845 if (!assoc)
846 return -ENOMEM;
847 scatterwalk_map_and_copy(assoc, req->src, 0,
848 req->assoclen + req->cryptlen, 0);
849 src = assoc + req->assoclen;
850 dst = src;
851 }
852
853 kernel_fpu_begin();
854 aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
855 ctx->hash_subkey, assoc, req->assoclen - 8,
856 authTag, auth_tag_len);
857 kernel_fpu_end();
858
859 /* Compare generated tag with passed in tag. */
860 retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
861 -EBADMSG : 0;
862
863 if (one_entry_in_sg) {
864 if (unlikely(req->src != req->dst)) {
865 scatterwalk_unmap(dst - req->assoclen);
866 scatterwalk_advance(&dst_sg_walk, req->dst->length);
867 scatterwalk_done(&dst_sg_walk, 1, 0);
868 }
869 scatterwalk_unmap(assoc);
870 scatterwalk_advance(&src_sg_walk, req->src->length);
871 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
872 } else {
873 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
874 tempCipherLen, 1);
875 kfree(assoc);
876 }
877 return retval;
878 }
879
880 static int rfc4106_encrypt(struct aead_request *req)
881 {
882 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
883 struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
884 struct cryptd_aead *cryptd_tfm = *ctx;
885
886 tfm = &cryptd_tfm->base;
887 if (irq_fpu_usable() && (!in_atomic() ||
888 !cryptd_aead_queued(cryptd_tfm)))
889 tfm = cryptd_aead_child(cryptd_tfm);
890
891 aead_request_set_tfm(req, tfm);
892
893 return crypto_aead_encrypt(req);
894 }
895
896 static int rfc4106_decrypt(struct aead_request *req)
897 {
898 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
899 struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
900 struct cryptd_aead *cryptd_tfm = *ctx;
901
902 tfm = &cryptd_tfm->base;
903 if (irq_fpu_usable() && (!in_atomic() ||
904 !cryptd_aead_queued(cryptd_tfm)))
905 tfm = cryptd_aead_child(cryptd_tfm);
906
907 aead_request_set_tfm(req, tfm);
908
909 return crypto_aead_decrypt(req);
910 }
911 #endif
912
913 static struct crypto_alg aesni_algs[] = { {
914 .cra_name = "aes",
915 .cra_driver_name = "aes-aesni",
916 .cra_priority = 300,
917 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
918 .cra_blocksize = AES_BLOCK_SIZE,
919 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
920 .cra_module = THIS_MODULE,
921 .cra_u = {
922 .cipher = {
923 .cia_min_keysize = AES_MIN_KEY_SIZE,
924 .cia_max_keysize = AES_MAX_KEY_SIZE,
925 .cia_setkey = aes_set_key,
926 .cia_encrypt = aes_encrypt,
927 .cia_decrypt = aes_decrypt
928 }
929 }
930 }, {
931 .cra_name = "__aes",
932 .cra_driver_name = "__aes-aesni",
933 .cra_priority = 300,
934 .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
935 .cra_blocksize = AES_BLOCK_SIZE,
936 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
937 .cra_module = THIS_MODULE,
938 .cra_u = {
939 .cipher = {
940 .cia_min_keysize = AES_MIN_KEY_SIZE,
941 .cia_max_keysize = AES_MAX_KEY_SIZE,
942 .cia_setkey = aes_set_key,
943 .cia_encrypt = __aes_encrypt,
944 .cia_decrypt = __aes_decrypt
945 }
946 }
947 } };
948
949 static struct skcipher_alg aesni_skciphers[] = {
950 {
951 .base = {
952 .cra_name = "__ecb(aes)",
953 .cra_driver_name = "__ecb-aes-aesni",
954 .cra_priority = 400,
955 .cra_flags = CRYPTO_ALG_INTERNAL,
956 .cra_blocksize = AES_BLOCK_SIZE,
957 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
958 .cra_module = THIS_MODULE,
959 },
960 .min_keysize = AES_MIN_KEY_SIZE,
961 .max_keysize = AES_MAX_KEY_SIZE,
962 .setkey = aesni_skcipher_setkey,
963 .encrypt = ecb_encrypt,
964 .decrypt = ecb_decrypt,
965 }, {
966 .base = {
967 .cra_name = "__cbc(aes)",
968 .cra_driver_name = "__cbc-aes-aesni",
969 .cra_priority = 400,
970 .cra_flags = CRYPTO_ALG_INTERNAL,
971 .cra_blocksize = AES_BLOCK_SIZE,
972 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
973 .cra_module = THIS_MODULE,
974 },
975 .min_keysize = AES_MIN_KEY_SIZE,
976 .max_keysize = AES_MAX_KEY_SIZE,
977 .ivsize = AES_BLOCK_SIZE,
978 .setkey = aesni_skcipher_setkey,
979 .encrypt = cbc_encrypt,
980 .decrypt = cbc_decrypt,
981 #ifdef CONFIG_X86_64
982 }, {
983 .base = {
984 .cra_name = "__ctr(aes)",
985 .cra_driver_name = "__ctr-aes-aesni",
986 .cra_priority = 400,
987 .cra_flags = CRYPTO_ALG_INTERNAL,
988 .cra_blocksize = 1,
989 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
990 .cra_module = THIS_MODULE,
991 },
992 .min_keysize = AES_MIN_KEY_SIZE,
993 .max_keysize = AES_MAX_KEY_SIZE,
994 .ivsize = AES_BLOCK_SIZE,
995 .chunksize = AES_BLOCK_SIZE,
996 .setkey = aesni_skcipher_setkey,
997 .encrypt = ctr_crypt,
998 .decrypt = ctr_crypt,
999 }, {
1000 .base = {
1001 .cra_name = "__xts(aes)",
1002 .cra_driver_name = "__xts-aes-aesni",
1003 .cra_priority = 401,
1004 .cra_flags = CRYPTO_ALG_INTERNAL,
1005 .cra_blocksize = AES_BLOCK_SIZE,
1006 .cra_ctxsize = XTS_AES_CTX_SIZE,
1007 .cra_module = THIS_MODULE,
1008 },
1009 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1010 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1011 .ivsize = AES_BLOCK_SIZE,
1012 .setkey = xts_aesni_setkey,
1013 .encrypt = xts_encrypt,
1014 .decrypt = xts_decrypt,
1015 #endif
1016 }
1017 };
1018
1019 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1020
1021 struct {
1022 const char *algname;
1023 const char *drvname;
1024 const char *basename;
1025 struct simd_skcipher_alg *simd;
1026 } aesni_simd_skciphers2[] = {
1027 #if (defined(MODULE) && IS_ENABLED(CONFIG_CRYPTO_PCBC)) || \
1028 IS_BUILTIN(CONFIG_CRYPTO_PCBC)
1029 {
1030 .algname = "pcbc(aes)",
1031 .drvname = "pcbc-aes-aesni",
1032 .basename = "fpu(pcbc(__aes-aesni))",
1033 },
1034 #endif
1035 };
1036
1037 #ifdef CONFIG_X86_64
1038 static struct aead_alg aesni_aead_algs[] = { {
1039 .setkey = common_rfc4106_set_key,
1040 .setauthsize = common_rfc4106_set_authsize,
1041 .encrypt = helper_rfc4106_encrypt,
1042 .decrypt = helper_rfc4106_decrypt,
1043 .ivsize = 8,
1044 .maxauthsize = 16,
1045 .base = {
1046 .cra_name = "__gcm-aes-aesni",
1047 .cra_driver_name = "__driver-gcm-aes-aesni",
1048 .cra_flags = CRYPTO_ALG_INTERNAL,
1049 .cra_blocksize = 1,
1050 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
1051 .cra_alignmask = AESNI_ALIGN - 1,
1052 .cra_module = THIS_MODULE,
1053 },
1054 }, {
1055 .init = rfc4106_init,
1056 .exit = rfc4106_exit,
1057 .setkey = rfc4106_set_key,
1058 .setauthsize = rfc4106_set_authsize,
1059 .encrypt = rfc4106_encrypt,
1060 .decrypt = rfc4106_decrypt,
1061 .ivsize = 8,
1062 .maxauthsize = 16,
1063 .base = {
1064 .cra_name = "rfc4106(gcm(aes))",
1065 .cra_driver_name = "rfc4106-gcm-aesni",
1066 .cra_priority = 400,
1067 .cra_flags = CRYPTO_ALG_ASYNC,
1068 .cra_blocksize = 1,
1069 .cra_ctxsize = sizeof(struct cryptd_aead *),
1070 .cra_module = THIS_MODULE,
1071 },
1072 } };
1073 #else
1074 static struct aead_alg aesni_aead_algs[0];
1075 #endif
1076
1077
1078 static const struct x86_cpu_id aesni_cpu_id[] = {
1079 X86_FEATURE_MATCH(X86_FEATURE_AES),
1080 {}
1081 };
1082 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1083
1084 static void aesni_free_simds(void)
1085 {
1086 int i;
1087
1088 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
1089 aesni_simd_skciphers[i]; i++)
1090 simd_skcipher_free(aesni_simd_skciphers[i]);
1091
1092 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++)
1093 if (aesni_simd_skciphers2[i].simd)
1094 simd_skcipher_free(aesni_simd_skciphers2[i].simd);
1095 }
1096
1097 static int __init aesni_init(void)
1098 {
1099 struct simd_skcipher_alg *simd;
1100 const char *basename;
1101 const char *algname;
1102 const char *drvname;
1103 int err;
1104 int i;
1105
1106 if (!x86_match_cpu(aesni_cpu_id))
1107 return -ENODEV;
1108 #ifdef CONFIG_X86_64
1109 #ifdef CONFIG_AS_AVX2
1110 if (boot_cpu_has(X86_FEATURE_AVX2)) {
1111 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1112 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2;
1113 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2;
1114 } else
1115 #endif
1116 #ifdef CONFIG_AS_AVX
1117 if (boot_cpu_has(X86_FEATURE_AVX)) {
1118 pr_info("AVX version of gcm_enc/dec engaged.\n");
1119 aesni_gcm_enc_tfm = aesni_gcm_enc_avx;
1120 aesni_gcm_dec_tfm = aesni_gcm_dec_avx;
1121 } else
1122 #endif
1123 {
1124 pr_info("SSE version of gcm_enc/dec engaged.\n");
1125 aesni_gcm_enc_tfm = aesni_gcm_enc;
1126 aesni_gcm_dec_tfm = aesni_gcm_dec;
1127 }
1128 aesni_ctr_enc_tfm = aesni_ctr_enc;
1129 #ifdef CONFIG_AS_AVX
1130 if (boot_cpu_has(X86_FEATURE_AVX)) {
1131 /* optimize performance of ctr mode encryption transform */
1132 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1133 pr_info("AES CTR mode by8 optimization enabled\n");
1134 }
1135 #endif
1136 #endif
1137
1138 err = crypto_fpu_init();
1139 if (err)
1140 return err;
1141
1142 err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1143 if (err)
1144 goto fpu_exit;
1145
1146 err = crypto_register_skciphers(aesni_skciphers,
1147 ARRAY_SIZE(aesni_skciphers));
1148 if (err)
1149 goto unregister_algs;
1150
1151 err = crypto_register_aeads(aesni_aead_algs,
1152 ARRAY_SIZE(aesni_aead_algs));
1153 if (err)
1154 goto unregister_skciphers;
1155
1156 for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
1157 algname = aesni_skciphers[i].base.cra_name + 2;
1158 drvname = aesni_skciphers[i].base.cra_driver_name + 2;
1159 basename = aesni_skciphers[i].base.cra_driver_name;
1160 simd = simd_skcipher_create_compat(algname, drvname, basename);
1161 err = PTR_ERR(simd);
1162 if (IS_ERR(simd))
1163 goto unregister_simds;
1164
1165 aesni_simd_skciphers[i] = simd;
1166 }
1167
1168 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) {
1169 algname = aesni_simd_skciphers2[i].algname;
1170 drvname = aesni_simd_skciphers2[i].drvname;
1171 basename = aesni_simd_skciphers2[i].basename;
1172 simd = simd_skcipher_create_compat(algname, drvname, basename);
1173 err = PTR_ERR(simd);
1174 if (IS_ERR(simd))
1175 continue;
1176
1177 aesni_simd_skciphers2[i].simd = simd;
1178 }
1179
1180 return 0;
1181
1182 unregister_simds:
1183 aesni_free_simds();
1184 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1185 unregister_skciphers:
1186 crypto_unregister_skciphers(aesni_skciphers,
1187 ARRAY_SIZE(aesni_skciphers));
1188 unregister_algs:
1189 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1190 fpu_exit:
1191 crypto_fpu_exit();
1192 return err;
1193 }
1194
1195 static void __exit aesni_exit(void)
1196 {
1197 aesni_free_simds();
1198 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1199 crypto_unregister_skciphers(aesni_skciphers,
1200 ARRAY_SIZE(aesni_skciphers));
1201 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1202
1203 crypto_fpu_exit();
1204 }
1205
1206 late_initcall(aesni_init);
1207 module_exit(aesni_exit);
1208
1209 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1210 MODULE_LICENSE("GPL");
1211 MODULE_ALIAS_CRYPTO("aes");
Ubuntu Artful kernel mirror