]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/x86/crypto/twofish_avx_glue.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: twofish-x86_64-3way - remove duplicated glue code and use shared glue code...
[mirror_ubuntu-jammy-kernel.git] / arch / x86 / crypto / twofish_avx_glue.c
CommitLineData
107778b5
JG		 1/*
2 * Glue Code for AVX assembler version of Twofish Cipher
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * Glue code based on serpent_sse2_glue.c by:
8 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
23 * USA
24 *
25 */
26
27#include <linux/module.h>
28#include <linux/hardirq.h>
29#include <linux/types.h>
30#include <linux/crypto.h>
31#include <linux/err.h>
32#include <crypto/algapi.h>
33#include <crypto/twofish.h>
34#include <crypto/cryptd.h>
35#include <crypto/b128ops.h>
36#include <crypto/ctr.h>
37#include <crypto/lrw.h>
38#include <crypto/xts.h>
39#include <asm/i387.h>
40#include <asm/xcr.h>
41#include <asm/xsave.h>
30a04008 42#include <asm/crypto/ablk_helper.h>
107778b5
JG		 43#include <crypto/scatterwalk.h>
44#include <linux/workqueue.h>
45#include <linux/spinlock.h>
46
47
48#define TWOFISH_PARALLEL_BLOCKS 8
49
50/* regular block cipher functions from twofish_x86_64 module */
51asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
52 const u8 *src);
53asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
54 const u8 *src);
55
56/* 3-way parallel cipher functions from twofish_x86_64-3way module */
57asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
58 const u8 *src, bool xor);
59asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
60 const u8 *src);
61
62static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
63 const u8 *src)
64{
65 __twofish_enc_blk_3way(ctx, dst, src, false);
66}
67
68static inline void twofish_enc_blk_3way_xor(struct twofish_ctx *ctx, u8 *dst,
69 const u8 *src)
70{
71 __twofish_enc_blk_3way(ctx, dst, src, true);
72}
73
74/* 8-way parallel cipher functions */
75asmlinkage void __twofish_enc_blk_8way(struct twofish_ctx *ctx, u8 *dst,
76 const u8 *src, bool xor);
77asmlinkage void twofish_dec_blk_8way(struct twofish_ctx *ctx, u8 *dst,
78 const u8 *src);
79
80static inline void twofish_enc_blk_xway(struct twofish_ctx *ctx, u8 *dst,
81 const u8 *src)
82{
83 __twofish_enc_blk_8way(ctx, dst, src, false);
84}
85
86static inline void twofish_enc_blk_xway_xor(struct twofish_ctx *ctx, u8 *dst,
87 const u8 *src)
88{
89 __twofish_enc_blk_8way(ctx, dst, src, true);
90}
91
92static inline void twofish_dec_blk_xway(struct twofish_ctx *ctx, u8 *dst,
93 const u8 *src)
94{
95 twofish_dec_blk_8way(ctx, dst, src);
96}
97
98
107778b5
JG		 99static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
100{
101 if (fpu_enabled)
102 return true;
103
104 /* AVX is only used when chunk to be processed is large enough, so
105 * do not enable FPU until it is necessary.
106 */
107 if (nbytes < TF_BLOCK_SIZE * TWOFISH_PARALLEL_BLOCKS)
108 return false;
109
110 kernel_fpu_begin();
111 return true;
112}
113
114static inline void twofish_fpu_end(bool fpu_enabled)
115{
116 if (fpu_enabled)
117 kernel_fpu_end();
118}
119
120static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
121 bool enc)
122{
123 bool fpu_enabled = false;
124 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
125 const unsigned int bsize = TF_BLOCK_SIZE;
126 unsigned int nbytes;
127 int err;
128
129 err = blkcipher_walk_virt(desc, walk);
130 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
131
132 while ((nbytes = walk->nbytes)) {
133 u8 *wsrc = walk->src.virt.addr;
134 u8 *wdst = walk->dst.virt.addr;
135
136 fpu_enabled = twofish_fpu_begin(fpu_enabled, nbytes);
137
138 /* Process multi-block batch */
139 if (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS) {
140 do {
141 if (enc)
142 twofish_enc_blk_xway(ctx, wdst, wsrc);
143 else
144 twofish_dec_blk_xway(ctx, wdst, wsrc);
145
146 wsrc += bsize * TWOFISH_PARALLEL_BLOCKS;
147 wdst += bsize * TWOFISH_PARALLEL_BLOCKS;
148 nbytes -= bsize * TWOFISH_PARALLEL_BLOCKS;
149 } while (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS);
150
151 if (nbytes < bsize)
152 goto done;
153 }
154
155 /* Process three block batch */
156 if (nbytes >= bsize * 3) {
157 do {
158 if (enc)
159 twofish_enc_blk_3way(ctx, wdst, wsrc);
160 else
161 twofish_dec_blk_3way(ctx, wdst, wsrc);
162
163 wsrc += bsize * 3;
164 wdst += bsize * 3;
165 nbytes -= bsize * 3;
166 } while (nbytes >= bsize * 3);
167
168 if (nbytes < bsize)
169 goto done;
170 }
171
172 /* Handle leftovers */
173 do {
174 if (enc)
175 twofish_enc_blk(ctx, wdst, wsrc);
176 else
177 twofish_dec_blk(ctx, wdst, wsrc);
178
179 wsrc += bsize;
180 wdst += bsize;
181 nbytes -= bsize;
182 } while (nbytes >= bsize);
183
184done:
185 err = blkcipher_walk_done(desc, walk, nbytes);
186 }
187
188 twofish_fpu_end(fpu_enabled);
189 return err;
190}
191
192static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
193 struct scatterlist *src, unsigned int nbytes)
194{
195 struct blkcipher_walk walk;
196
197 blkcipher_walk_init(&walk, dst, src, nbytes);
198 return ecb_crypt(desc, &walk, true);
199}
200
201static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
202 struct scatterlist *src, unsigned int nbytes)
203{
204 struct blkcipher_walk walk;
205
206 blkcipher_walk_init(&walk, dst, src, nbytes);
207 return ecb_crypt(desc, &walk, false);
208}
209
210static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
211 struct blkcipher_walk *walk)
212{
213 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
214 const unsigned int bsize = TF_BLOCK_SIZE;
215 unsigned int nbytes = walk->nbytes;
216 u128 *src = (u128 *)walk->src.virt.addr;
217 u128 *dst = (u128 *)walk->dst.virt.addr;
218 u128 *iv = (u128 *)walk->iv;
219
220 do {
221 u128_xor(dst, src, iv);
222 twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
223 iv = dst;
224
225 src += 1;
226 dst += 1;
227 nbytes -= bsize;
228 } while (nbytes >= bsize);
229
230 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
231 return nbytes;
232}
233
234static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
235 struct scatterlist *src, unsigned int nbytes)
236{
237 struct blkcipher_walk walk;
238 int err;
239
240 blkcipher_walk_init(&walk, dst, src, nbytes);
241 err = blkcipher_walk_virt(desc, &walk);
242
243 while ((nbytes = walk.nbytes)) {
244 nbytes = __cbc_encrypt(desc, &walk);
245 err = blkcipher_walk_done(desc, &walk, nbytes);
246 }
247
248 return err;
249}
250
251static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
252 struct blkcipher_walk *walk)
253{
254 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
255 const unsigned int bsize = TF_BLOCK_SIZE;
256 unsigned int nbytes = walk->nbytes;
257 u128 *src = (u128 *)walk->src.virt.addr;
258 u128 *dst = (u128 *)walk->dst.virt.addr;
259 u128 ivs[TWOFISH_PARALLEL_BLOCKS - 1];
260 u128 last_iv;
261 int i;
262
263 /* Start of the last block. */
264 src += nbytes / bsize - 1;
265 dst += nbytes / bsize - 1;
266
267 last_iv = *src;
268
269 /* Process multi-block batch */
270 if (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS) {
271 do {
272 nbytes -= bsize * (TWOFISH_PARALLEL_BLOCKS - 1);
273 src -= TWOFISH_PARALLEL_BLOCKS - 1;
274 dst -= TWOFISH_PARALLEL_BLOCKS - 1;
275
276 for (i = 0; i < TWOFISH_PARALLEL_BLOCKS - 1; i++)
277 ivs[i] = src[i];
278
279 twofish_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
280
281 for (i = 0; i < TWOFISH_PARALLEL_BLOCKS - 1; i++)
282 u128_xor(dst + (i + 1), dst + (i + 1), ivs + i);
283
284 nbytes -= bsize;
285 if (nbytes < bsize)
286 goto done;
287
288 u128_xor(dst, dst, src - 1);
289 src -= 1;
290 dst -= 1;
291 } while (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS);
292
293 if (nbytes < bsize)
294 goto done;
295 }
296
297 /* Process three block batch */
298 if (nbytes >= bsize * 3) {
299 do {
300 nbytes -= bsize * (3 - 1);
301 src -= 3 - 1;
302 dst -= 3 - 1;
303
304 ivs[0] = src[0];
305 ivs[1] = src[1];
306
307 twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
308
309 u128_xor(dst + 1, dst + 1, ivs + 0);
310 u128_xor(dst + 2, dst + 2, ivs + 1);
311
312 nbytes -= bsize;
313 if (nbytes < bsize)
314 goto done;
315
316 u128_xor(dst, dst, src - 1);
317 src -= 1;
318 dst -= 1;
319 } while (nbytes >= bsize * 3);
320
321 if (nbytes < bsize)
322 goto done;
323 }
324
325 /* Handle leftovers */
326 for (;;) {
327 twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
328
329 nbytes -= bsize;
330 if (nbytes < bsize)
331 break;
332
333 u128_xor(dst, dst, src - 1);
334 src -= 1;
335 dst -= 1;
336 }
337
338done:
339 u128_xor(dst, dst, (u128 *)walk->iv);
340 *(u128 *)walk->iv = last_iv;
341
342 return nbytes;
343}
344
345static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
346 struct scatterlist *src, unsigned int nbytes)
347{
348 bool fpu_enabled = false;
349 struct blkcipher_walk walk;
350 int err;
351
352 blkcipher_walk_init(&walk, dst, src, nbytes);
353 err = blkcipher_walk_virt(desc, &walk);
354 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
355
356 while ((nbytes = walk.nbytes)) {
357 fpu_enabled = twofish_fpu_begin(fpu_enabled, nbytes);
358 nbytes = __cbc_decrypt(desc, &walk);
359 err = blkcipher_walk_done(desc, &walk, nbytes);
360 }
361
362 twofish_fpu_end(fpu_enabled);
363 return err;
364}
365
366static inline void u128_to_be128(be128 *dst, const u128 *src)
367{
368 dst->a = cpu_to_be64(src->a);
369 dst->b = cpu_to_be64(src->b);
370}
371
372static inline void be128_to_u128(u128 *dst, const be128 *src)
373{
374 dst->a = be64_to_cpu(src->a);
375 dst->b = be64_to_cpu(src->b);
376}
377
378static inline void u128_inc(u128 *i)
379{
380 i->b++;
381 if (!i->b)
382 i->a++;
383}
384
385static void ctr_crypt_final(struct blkcipher_desc *desc,
386 struct blkcipher_walk *walk)
387{
388 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
389 u8 *ctrblk = walk->iv;
390 u8 keystream[TF_BLOCK_SIZE];
391 u8 *src = walk->src.virt.addr;
392 u8 *dst = walk->dst.virt.addr;
393 unsigned int nbytes = walk->nbytes;
394
395 twofish_enc_blk(ctx, keystream, ctrblk);
396 crypto_xor(keystream, src, nbytes);
397 memcpy(dst, keystream, nbytes);
398
399 crypto_inc(ctrblk, TF_BLOCK_SIZE);
400}
401
402static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
403 struct blkcipher_walk *walk)
404{
405 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
406 const unsigned int bsize = TF_BLOCK_SIZE;
407 unsigned int nbytes = walk->nbytes;
408 u128 *src = (u128 *)walk->src.virt.addr;
409 u128 *dst = (u128 *)walk->dst.virt.addr;
410 u128 ctrblk;
411 be128 ctrblocks[TWOFISH_PARALLEL_BLOCKS];
412 int i;
413
414 be128_to_u128(&ctrblk, (be128 *)walk->iv);
415
416 /* Process multi-block batch */
417 if (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS) {
418 do {
419 /* create ctrblks for parallel encrypt */
420 for (i = 0; i < TWOFISH_PARALLEL_BLOCKS; i++) {
421 if (dst != src)
422 dst[i] = src[i];
423
424 u128_to_be128(&ctrblocks[i], &ctrblk);
425 u128_inc(&ctrblk);
426 }
427
428 twofish_enc_blk_xway_xor(ctx, (u8 *)dst,
429 (u8 *)ctrblocks);
430
431 src += TWOFISH_PARALLEL_BLOCKS;
432 dst += TWOFISH_PARALLEL_BLOCKS;
433 nbytes -= bsize * TWOFISH_PARALLEL_BLOCKS;
434 } while (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS);
435
436 if (nbytes < bsize)
437 goto done;
438 }
439
440 /* Process three block batch */
441 if (nbytes >= bsize * 3) {
442 do {
443 if (dst != src) {
444 dst[0] = src[0];
445 dst[1] = src[1];
446 dst[2] = src[2];
447 }
448
449 /* create ctrblks for parallel encrypt */
450 u128_to_be128(&ctrblocks[0], &ctrblk);
451 u128_inc(&ctrblk);
452 u128_to_be128(&ctrblocks[1], &ctrblk);
453 u128_inc(&ctrblk);
454 u128_to_be128(&ctrblocks[2], &ctrblk);
455 u128_inc(&ctrblk);
456
457 twofish_enc_blk_3way_xor(ctx, (u8 *)dst,
458 (u8 *)ctrblocks);
459
460 src += 3;
461 dst += 3;
462 nbytes -= bsize * 3;
463 } while (nbytes >= bsize * 3);
464
465 if (nbytes < bsize)
466 goto done;
467 }
468
469 /* Handle leftovers */
470 do {
471 if (dst != src)
472 *dst = *src;
473
474 u128_to_be128(&ctrblocks[0], &ctrblk);
475 u128_inc(&ctrblk);
476
477 twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
478 u128_xor(dst, dst, (u128 *)ctrblocks);
479
480 src += 1;
481 dst += 1;
482 nbytes -= bsize;
483 } while (nbytes >= bsize);
484
485done:
486 u128_to_be128((be128 *)walk->iv, &ctrblk);
487 return nbytes;
488}
489
490static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
491 struct scatterlist *src, unsigned int nbytes)
492{
493 bool fpu_enabled = false;
494 struct blkcipher_walk walk;
495 int err;
496
497 blkcipher_walk_init(&walk, dst, src, nbytes);
498 err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);
499 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
500
501 while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
502 fpu_enabled = twofish_fpu_begin(fpu_enabled, nbytes);
503 nbytes = __ctr_crypt(desc, &walk);
504 err = blkcipher_walk_done(desc, &walk, nbytes);
505 }
506
507 twofish_fpu_end(fpu_enabled);
508
509 if (walk.nbytes) {
510 ctr_crypt_final(desc, &walk);
511 err = blkcipher_walk_done(desc, &walk, 0);
512 }
513
514 return err;
515}
516
517struct crypt_priv {
518 struct twofish_ctx *ctx;
519 bool fpu_enabled;
520};
521
522static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
523{
524 const unsigned int bsize = TF_BLOCK_SIZE;
525 struct crypt_priv *ctx = priv;
526 int i;
527
528 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
529
530 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
531 twofish_enc_blk_xway(ctx->ctx, srcdst, srcdst);
532 return;
533 }
534
535 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
536 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
537
538 nbytes %= bsize * 3;
539
540 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
541 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
542}
543
544static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
545{
546 const unsigned int bsize = TF_BLOCK_SIZE;
547 struct crypt_priv *ctx = priv;
548 int i;
549
550 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
551
552 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
553 twofish_dec_blk_xway(ctx->ctx, srcdst, srcdst);
554 return;
555 }
556
557 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
558 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
559
560 nbytes %= bsize * 3;
561
562 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
563 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
564}
565
566struct twofish_lrw_ctx {
567 struct lrw_table_ctx lrw_table;
568 struct twofish_ctx twofish_ctx;
569};
570
571static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
572 unsigned int keylen)
573{
574 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
575 int err;
576
577 err = __twofish_setkey(&ctx->twofish_ctx, key,
578 keylen - TF_BLOCK_SIZE, &tfm->crt_flags);
579 if (err)
580 return err;
581
582 return lrw_init_table(&ctx->lrw_table, key + keylen -
583 TF_BLOCK_SIZE);
584}
585
586static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
587 struct scatterlist *src, unsigned int nbytes)
588{
589 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
590 be128 buf[TWOFISH_PARALLEL_BLOCKS];
591 struct crypt_priv crypt_ctx = {
592 .ctx = &ctx->twofish_ctx,
593 .fpu_enabled = false,
594 };
595 struct lrw_crypt_req req = {
596 .tbuf = buf,
597 .tbuflen = sizeof(buf),
598
599 .table_ctx = &ctx->lrw_table,
600 .crypt_ctx = &crypt_ctx,
601 .crypt_fn = encrypt_callback,
602 };
603 int ret;
604
605 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
606 ret = lrw_crypt(desc, dst, src, nbytes, &req);
607 twofish_fpu_end(crypt_ctx.fpu_enabled);
608
609 return ret;
610}
611
612static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
613 struct scatterlist *src, unsigned int nbytes)
614{
615 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
616 be128 buf[TWOFISH_PARALLEL_BLOCKS];
617 struct crypt_priv crypt_ctx = {
618 .ctx = &ctx->twofish_ctx,
619 .fpu_enabled = false,
620 };
621 struct lrw_crypt_req req = {
622 .tbuf = buf,
623 .tbuflen = sizeof(buf),
624
625 .table_ctx = &ctx->lrw_table,
626 .crypt_ctx = &crypt_ctx,
627 .crypt_fn = decrypt_callback,
628 };
629 int ret;
630
631 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
632 ret = lrw_crypt(desc, dst, src, nbytes, &req);
633 twofish_fpu_end(crypt_ctx.fpu_enabled);
634
635 return ret;
636}
637
638static void lrw_exit_tfm(struct crypto_tfm *tfm)
639{
640 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
641
642 lrw_free_table(&ctx->lrw_table);
643}
644
645struct twofish_xts_ctx {
646 struct twofish_ctx tweak_ctx;
647 struct twofish_ctx crypt_ctx;
648};
649
650static int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
651 unsigned int keylen)
652{
653 struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm);
654 u32 *flags = &tfm->crt_flags;
655 int err;
656
657 /* key consists of keys of equal size concatenated, therefore
658 * the length must be even
659 */
660 if (keylen % 2) {
661 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
662 return -EINVAL;
663 }
664
665 /* first half of xts-key is for crypt */
666 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
667 if (err)
668 return err;
669
670 /* second half of xts-key is for tweak */
671 return __twofish_setkey(&ctx->tweak_ctx,
672 key + keylen / 2, keylen / 2, flags);
673}
674
675static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
676 struct scatterlist *src, unsigned int nbytes)
677{
678 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
679 be128 buf[TWOFISH_PARALLEL_BLOCKS];
680 struct crypt_priv crypt_ctx = {
681 .ctx = &ctx->crypt_ctx,
682 .fpu_enabled = false,
683 };
684 struct xts_crypt_req req = {
685 .tbuf = buf,
686 .tbuflen = sizeof(buf),
687
688 .tweak_ctx = &ctx->tweak_ctx,
689 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
690 .crypt_ctx = &crypt_ctx,
691 .crypt_fn = encrypt_callback,
692 };
693 int ret;
694
695 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
696 ret = xts_crypt(desc, dst, src, nbytes, &req);
697 twofish_fpu_end(crypt_ctx.fpu_enabled);
698
699 return ret;
700}
701
702static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
703 struct scatterlist *src, unsigned int nbytes)
704{
705 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
706 be128 buf[TWOFISH_PARALLEL_BLOCKS];
707 struct crypt_priv crypt_ctx = {
708 .ctx = &ctx->crypt_ctx,
709 .fpu_enabled = false,
710 };
711 struct xts_crypt_req req = {
712 .tbuf = buf,
713 .tbuflen = sizeof(buf),
714
715 .tweak_ctx = &ctx->tweak_ctx,
716 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
717 .crypt_ctx = &crypt_ctx,
718 .crypt_fn = decrypt_callback,
719 };
720 int ret;
721
722 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
723 ret = xts_crypt(desc, dst, src, nbytes, &req);
724 twofish_fpu_end(crypt_ctx.fpu_enabled);
725
726 return ret;
727}
728
107778b5
JG		 729static struct crypto_alg twofish_algs[10] = { {
730 .cra_name = "__ecb-twofish-avx",
731 .cra_driver_name = "__driver-ecb-twofish-avx",
732 .cra_priority = 0,
733 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
734 .cra_blocksize = TF_BLOCK_SIZE,
735 .cra_ctxsize = sizeof(struct twofish_ctx),
736 .cra_alignmask = 0,
737 .cra_type = &crypto_blkcipher_type,
738 .cra_module = THIS_MODULE,
739 .cra_list = LIST_HEAD_INIT(twofish_algs[0].cra_list),
740 .cra_u = {
741 .blkcipher = {
742 .min_keysize = TF_MIN_KEY_SIZE,
743 .max_keysize = TF_MAX_KEY_SIZE,
744 .setkey = twofish_setkey,
745 .encrypt = ecb_encrypt,
746 .decrypt = ecb_decrypt,
747 },
748 },
749}, {
750 .cra_name = "__cbc-twofish-avx",
751 .cra_driver_name = "__driver-cbc-twofish-avx",
752 .cra_priority = 0,
753 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
754 .cra_blocksize = TF_BLOCK_SIZE,
755 .cra_ctxsize = sizeof(struct twofish_ctx),
756 .cra_alignmask = 0,
757 .cra_type = &crypto_blkcipher_type,
758 .cra_module = THIS_MODULE,
759 .cra_list = LIST_HEAD_INIT(twofish_algs[1].cra_list),
760 .cra_u = {
761 .blkcipher = {
762 .min_keysize = TF_MIN_KEY_SIZE,
763 .max_keysize = TF_MAX_KEY_SIZE,
764 .setkey = twofish_setkey,
765 .encrypt = cbc_encrypt,
766 .decrypt = cbc_decrypt,
767 },
768 },
769}, {
770 .cra_name = "__ctr-twofish-avx",
771 .cra_driver_name = "__driver-ctr-twofish-avx",
772 .cra_priority = 0,
773 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
774 .cra_blocksize = 1,
775 .cra_ctxsize = sizeof(struct twofish_ctx),
776 .cra_alignmask = 0,
777 .cra_type = &crypto_blkcipher_type,
778 .cra_module = THIS_MODULE,
779 .cra_list = LIST_HEAD_INIT(twofish_algs[2].cra_list),
780 .cra_u = {
781 .blkcipher = {
782 .min_keysize = TF_MIN_KEY_SIZE,
783 .max_keysize = TF_MAX_KEY_SIZE,
784 .ivsize = TF_BLOCK_SIZE,
785 .setkey = twofish_setkey,
786 .encrypt = ctr_crypt,
787 .decrypt = ctr_crypt,
788 },
789 },
790}, {
791 .cra_name = "__lrw-twofish-avx",
792 .cra_driver_name = "__driver-lrw-twofish-avx",
793 .cra_priority = 0,
794 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
795 .cra_blocksize = TF_BLOCK_SIZE,
796 .cra_ctxsize = sizeof(struct twofish_lrw_ctx),
797 .cra_alignmask = 0,
798 .cra_type = &crypto_blkcipher_type,
799 .cra_module = THIS_MODULE,
800 .cra_list = LIST_HEAD_INIT(twofish_algs[3].cra_list),
801 .cra_exit = lrw_exit_tfm,
802 .cra_u = {
803 .blkcipher = {
804 .min_keysize = TF_MIN_KEY_SIZE +
805 TF_BLOCK_SIZE,
806 .max_keysize = TF_MAX_KEY_SIZE +
807 TF_BLOCK_SIZE,
808 .ivsize = TF_BLOCK_SIZE,
809 .setkey = lrw_twofish_setkey,
810 .encrypt = lrw_encrypt,
811 .decrypt = lrw_decrypt,
812 },
813 },
814}, {
815 .cra_name = "__xts-twofish-avx",
816 .cra_driver_name = "__driver-xts-twofish-avx",
817 .cra_priority = 0,
818 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
819 .cra_blocksize = TF_BLOCK_SIZE,
820 .cra_ctxsize = sizeof(struct twofish_xts_ctx),
821 .cra_alignmask = 0,
822 .cra_type = &crypto_blkcipher_type,
823 .cra_module = THIS_MODULE,
824 .cra_list = LIST_HEAD_INIT(twofish_algs[4].cra_list),
825 .cra_u = {
826 .blkcipher = {
827 .min_keysize = TF_MIN_KEY_SIZE * 2,
828 .max_keysize = TF_MAX_KEY_SIZE * 2,
829 .ivsize = TF_BLOCK_SIZE,
830 .setkey = xts_twofish_setkey,
831 .encrypt = xts_encrypt,
832 .decrypt = xts_decrypt,
833 },
834 },
835}, {
836 .cra_name = "ecb(twofish)",
837 .cra_driver_name = "ecb-twofish-avx",
838 .cra_priority = 400,
839 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
840 .cra_blocksize = TF_BLOCK_SIZE,
30a04008 841 .cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5
JG		 842 .cra_alignmask = 0,
843 .cra_type = &crypto_ablkcipher_type,
844 .cra_module = THIS_MODULE,
845 .cra_list = LIST_HEAD_INIT(twofish_algs[5].cra_list),
846 .cra_init = ablk_init,
847 .cra_exit = ablk_exit,
848 .cra_u = {
849 .ablkcipher = {
850 .min_keysize = TF_MIN_KEY_SIZE,
851 .max_keysize = TF_MAX_KEY_SIZE,
852 .setkey = ablk_set_key,
853 .encrypt = ablk_encrypt,
854 .decrypt = ablk_decrypt,
855 },
856 },
857}, {
858 .cra_name = "cbc(twofish)",
859 .cra_driver_name = "cbc-twofish-avx",
860 .cra_priority = 400,
861 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
862 .cra_blocksize = TF_BLOCK_SIZE,
30a04008 863 .cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5
JG		 864 .cra_alignmask = 0,
865 .cra_type = &crypto_ablkcipher_type,
866 .cra_module = THIS_MODULE,
867 .cra_list = LIST_HEAD_INIT(twofish_algs[6].cra_list),
868 .cra_init = ablk_init,
869 .cra_exit = ablk_exit,
870 .cra_u = {
871 .ablkcipher = {
872 .min_keysize = TF_MIN_KEY_SIZE,
873 .max_keysize = TF_MAX_KEY_SIZE,
874 .ivsize = TF_BLOCK_SIZE,
875 .setkey = ablk_set_key,
876 .encrypt = __ablk_encrypt,
877 .decrypt = ablk_decrypt,
878 },
879 },
880}, {
881 .cra_name = "ctr(twofish)",
882 .cra_driver_name = "ctr-twofish-avx",
883 .cra_priority = 400,
884 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
885 .cra_blocksize = 1,
30a04008 886 .cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5
JG		 887 .cra_alignmask = 0,
888 .cra_type = &crypto_ablkcipher_type,
889 .cra_module = THIS_MODULE,
890 .cra_list = LIST_HEAD_INIT(twofish_algs[7].cra_list),
891 .cra_init = ablk_init,
892 .cra_exit = ablk_exit,
893 .cra_u = {
894 .ablkcipher = {
895 .min_keysize = TF_MIN_KEY_SIZE,
896 .max_keysize = TF_MAX_KEY_SIZE,
897 .ivsize = TF_BLOCK_SIZE,
898 .setkey = ablk_set_key,
899 .encrypt = ablk_encrypt,
900 .decrypt = ablk_encrypt,
901 .geniv = "chainiv",
902 },
903 },
904}, {
905 .cra_name = "lrw(twofish)",
906 .cra_driver_name = "lrw-twofish-avx",
907 .cra_priority = 400,
908 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
909 .cra_blocksize = TF_BLOCK_SIZE,
30a04008 910 .cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5
JG		 911 .cra_alignmask = 0,
912 .cra_type = &crypto_ablkcipher_type,
913 .cra_module = THIS_MODULE,
914 .cra_list = LIST_HEAD_INIT(twofish_algs[8].cra_list),
915 .cra_init = ablk_init,
916 .cra_exit = ablk_exit,
917 .cra_u = {
918 .ablkcipher = {
919 .min_keysize = TF_MIN_KEY_SIZE +
920 TF_BLOCK_SIZE,
921 .max_keysize = TF_MAX_KEY_SIZE +
922 TF_BLOCK_SIZE,
923 .ivsize = TF_BLOCK_SIZE,
924 .setkey = ablk_set_key,
925 .encrypt = ablk_encrypt,
926 .decrypt = ablk_decrypt,
927 },
928 },
929}, {
930 .cra_name = "xts(twofish)",
931 .cra_driver_name = "xts-twofish-avx",
932 .cra_priority = 400,
933 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
934 .cra_blocksize = TF_BLOCK_SIZE,
30a04008 935 .cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5
JG		 936 .cra_alignmask = 0,
937 .cra_type = &crypto_ablkcipher_type,
938 .cra_module = THIS_MODULE,
939 .cra_list = LIST_HEAD_INIT(twofish_algs[9].cra_list),
940 .cra_init = ablk_init,
941 .cra_exit = ablk_exit,
942 .cra_u = {
943 .ablkcipher = {
944 .min_keysize = TF_MIN_KEY_SIZE * 2,
945 .max_keysize = TF_MAX_KEY_SIZE * 2,
946 .ivsize = TF_BLOCK_SIZE,
947 .setkey = ablk_set_key,
948 .encrypt = ablk_encrypt,
949 .decrypt = ablk_decrypt,
950 },
951 },
952} };
953
954static int __init twofish_init(void)
955{
956 u64 xcr0;
957
958 if (!cpu_has_avx || !cpu_has_osxsave) {
959 printk(KERN_INFO "AVX instructions are not detected.\n");
960 return -ENODEV;
961 }
962
963 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
964 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
965 printk(KERN_INFO "AVX detected but unusable.\n");
966 return -ENODEV;
967 }
968
969 return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
970}
971
972static void __exit twofish_exit(void)
973{
974 crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
975}
976
977module_init(twofish_init);
978module_exit(twofish_exit);
979
980MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
981MODULE_LICENSE("GPL");
982MODULE_ALIAS("twofish");
Ubuntu Jammy Linux kernel mirror