2 * Glue Code for the AVX assembler implemention of the Cast5 Cipher
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
24 #include <linux/module.h>
25 #include <linux/hardirq.h>
26 #include <linux/types.h>
27 #include <linux/crypto.h>
28 #include <linux/err.h>
29 #include <crypto/ablk_helper.h>
30 #include <crypto/algapi.h>
31 #include <crypto/cast5.h>
32 #include <crypto/cryptd.h>
33 #include <crypto/ctr.h>
35 #include <asm/xsave.h>
36 #include <asm/crypto/glue_helper.h>
38 #define CAST5_PARALLEL_BLOCKS 16
40 asmlinkage
void cast5_ecb_enc_16way(struct cast5_ctx
*ctx
, u8
*dst
,
42 asmlinkage
void cast5_ecb_dec_16way(struct cast5_ctx
*ctx
, u8
*dst
,
44 asmlinkage
void cast5_cbc_dec_16way(struct cast5_ctx
*ctx
, u8
*dst
,
46 asmlinkage
void cast5_ctr_16way(struct cast5_ctx
*ctx
, u8
*dst
, const u8
*src
,
49 static inline bool cast5_fpu_begin(bool fpu_enabled
, unsigned int nbytes
)
51 return glue_fpu_begin(CAST5_BLOCK_SIZE
, CAST5_PARALLEL_BLOCKS
,
52 NULL
, fpu_enabled
, nbytes
);
55 static inline void cast5_fpu_end(bool fpu_enabled
)
57 return glue_fpu_end(fpu_enabled
);
60 static int ecb_crypt(struct blkcipher_desc
*desc
, struct blkcipher_walk
*walk
,
63 bool fpu_enabled
= false;
64 struct cast5_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
65 const unsigned int bsize
= CAST5_BLOCK_SIZE
;
67 void (*fn
)(struct cast5_ctx
*ctx
, u8
*dst
, const u8
*src
);
70 fn
= (enc
) ? cast5_ecb_enc_16way
: cast5_ecb_dec_16way
;
72 err
= blkcipher_walk_virt(desc
, walk
);
73 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
75 while ((nbytes
= walk
->nbytes
)) {
76 u8
*wsrc
= walk
->src
.virt
.addr
;
77 u8
*wdst
= walk
->dst
.virt
.addr
;
79 fpu_enabled
= cast5_fpu_begin(fpu_enabled
, nbytes
);
81 /* Process multi-block batch */
82 if (nbytes
>= bsize
* CAST5_PARALLEL_BLOCKS
) {
86 wsrc
+= bsize
* CAST5_PARALLEL_BLOCKS
;
87 wdst
+= bsize
* CAST5_PARALLEL_BLOCKS
;
88 nbytes
-= bsize
* CAST5_PARALLEL_BLOCKS
;
89 } while (nbytes
>= bsize
* CAST5_PARALLEL_BLOCKS
);
95 fn
= (enc
) ? __cast5_encrypt
: __cast5_decrypt
;
97 /* Handle leftovers */
104 } while (nbytes
>= bsize
);
107 err
= blkcipher_walk_done(desc
, walk
, nbytes
);
110 cast5_fpu_end(fpu_enabled
);
114 static int ecb_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
115 struct scatterlist
*src
, unsigned int nbytes
)
117 struct blkcipher_walk walk
;
119 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
120 return ecb_crypt(desc
, &walk
, true);
123 static int ecb_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
124 struct scatterlist
*src
, unsigned int nbytes
)
126 struct blkcipher_walk walk
;
128 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
129 return ecb_crypt(desc
, &walk
, false);
132 static unsigned int __cbc_encrypt(struct blkcipher_desc
*desc
,
133 struct blkcipher_walk
*walk
)
135 struct cast5_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
136 const unsigned int bsize
= CAST5_BLOCK_SIZE
;
137 unsigned int nbytes
= walk
->nbytes
;
138 u64
*src
= (u64
*)walk
->src
.virt
.addr
;
139 u64
*dst
= (u64
*)walk
->dst
.virt
.addr
;
140 u64
*iv
= (u64
*)walk
->iv
;
144 __cast5_encrypt(ctx
, (u8
*)dst
, (u8
*)dst
);
150 } while (nbytes
>= bsize
);
152 *(u64
*)walk
->iv
= *iv
;
156 static int cbc_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
157 struct scatterlist
*src
, unsigned int nbytes
)
159 struct blkcipher_walk walk
;
162 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
163 err
= blkcipher_walk_virt(desc
, &walk
);
165 while ((nbytes
= walk
.nbytes
)) {
166 nbytes
= __cbc_encrypt(desc
, &walk
);
167 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
173 static unsigned int __cbc_decrypt(struct blkcipher_desc
*desc
,
174 struct blkcipher_walk
*walk
)
176 struct cast5_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
177 const unsigned int bsize
= CAST5_BLOCK_SIZE
;
178 unsigned int nbytes
= walk
->nbytes
;
179 u64
*src
= (u64
*)walk
->src
.virt
.addr
;
180 u64
*dst
= (u64
*)walk
->dst
.virt
.addr
;
183 /* Start of the last block. */
184 src
+= nbytes
/ bsize
- 1;
185 dst
+= nbytes
/ bsize
- 1;
189 /* Process multi-block batch */
190 if (nbytes
>= bsize
* CAST5_PARALLEL_BLOCKS
) {
192 nbytes
-= bsize
* (CAST5_PARALLEL_BLOCKS
- 1);
193 src
-= CAST5_PARALLEL_BLOCKS
- 1;
194 dst
-= CAST5_PARALLEL_BLOCKS
- 1;
196 cast5_cbc_dec_16way(ctx
, (u8
*)dst
, (u8
*)src
);
205 } while (nbytes
>= bsize
* CAST5_PARALLEL_BLOCKS
);
208 /* Handle leftovers */
210 __cast5_decrypt(ctx
, (u8
*)dst
, (u8
*)src
);
222 *dst
^= *(u64
*)walk
->iv
;
223 *(u64
*)walk
->iv
= last_iv
;
228 static int cbc_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
229 struct scatterlist
*src
, unsigned int nbytes
)
231 bool fpu_enabled
= false;
232 struct blkcipher_walk walk
;
235 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
236 err
= blkcipher_walk_virt(desc
, &walk
);
237 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
239 while ((nbytes
= walk
.nbytes
)) {
240 fpu_enabled
= cast5_fpu_begin(fpu_enabled
, nbytes
);
241 nbytes
= __cbc_decrypt(desc
, &walk
);
242 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
245 cast5_fpu_end(fpu_enabled
);
249 static void ctr_crypt_final(struct blkcipher_desc
*desc
,
250 struct blkcipher_walk
*walk
)
252 struct cast5_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
253 u8
*ctrblk
= walk
->iv
;
254 u8 keystream
[CAST5_BLOCK_SIZE
];
255 u8
*src
= walk
->src
.virt
.addr
;
256 u8
*dst
= walk
->dst
.virt
.addr
;
257 unsigned int nbytes
= walk
->nbytes
;
259 __cast5_encrypt(ctx
, keystream
, ctrblk
);
260 crypto_xor(keystream
, src
, nbytes
);
261 memcpy(dst
, keystream
, nbytes
);
263 crypto_inc(ctrblk
, CAST5_BLOCK_SIZE
);
266 static unsigned int __ctr_crypt(struct blkcipher_desc
*desc
,
267 struct blkcipher_walk
*walk
)
269 struct cast5_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
270 const unsigned int bsize
= CAST5_BLOCK_SIZE
;
271 unsigned int nbytes
= walk
->nbytes
;
272 u64
*src
= (u64
*)walk
->src
.virt
.addr
;
273 u64
*dst
= (u64
*)walk
->dst
.virt
.addr
;
275 /* Process multi-block batch */
276 if (nbytes
>= bsize
* CAST5_PARALLEL_BLOCKS
) {
278 cast5_ctr_16way(ctx
, (u8
*)dst
, (u8
*)src
,
281 src
+= CAST5_PARALLEL_BLOCKS
;
282 dst
+= CAST5_PARALLEL_BLOCKS
;
283 nbytes
-= bsize
* CAST5_PARALLEL_BLOCKS
;
284 } while (nbytes
>= bsize
* CAST5_PARALLEL_BLOCKS
);
290 /* Handle leftovers */
297 ctrblk
= *(u64
*)walk
->iv
;
298 be64_add_cpu((__be64
*)walk
->iv
, 1);
300 __cast5_encrypt(ctx
, (u8
*)&ctrblk
, (u8
*)&ctrblk
);
306 } while (nbytes
>= bsize
);
312 static int ctr_crypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
313 struct scatterlist
*src
, unsigned int nbytes
)
315 bool fpu_enabled
= false;
316 struct blkcipher_walk walk
;
319 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
320 err
= blkcipher_walk_virt_block(desc
, &walk
, CAST5_BLOCK_SIZE
);
321 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
323 while ((nbytes
= walk
.nbytes
) >= CAST5_BLOCK_SIZE
) {
324 fpu_enabled
= cast5_fpu_begin(fpu_enabled
, nbytes
);
325 nbytes
= __ctr_crypt(desc
, &walk
);
326 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
329 cast5_fpu_end(fpu_enabled
);
332 ctr_crypt_final(desc
, &walk
);
333 err
= blkcipher_walk_done(desc
, &walk
, 0);
340 static struct crypto_alg cast5_algs
[6] = { {
341 .cra_name
= "__ecb-cast5-avx",
342 .cra_driver_name
= "__driver-ecb-cast5-avx",
344 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
345 .cra_blocksize
= CAST5_BLOCK_SIZE
,
346 .cra_ctxsize
= sizeof(struct cast5_ctx
),
348 .cra_type
= &crypto_blkcipher_type
,
349 .cra_module
= THIS_MODULE
,
352 .min_keysize
= CAST5_MIN_KEY_SIZE
,
353 .max_keysize
= CAST5_MAX_KEY_SIZE
,
354 .setkey
= cast5_setkey
,
355 .encrypt
= ecb_encrypt
,
356 .decrypt
= ecb_decrypt
,
360 .cra_name
= "__cbc-cast5-avx",
361 .cra_driver_name
= "__driver-cbc-cast5-avx",
363 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
364 .cra_blocksize
= CAST5_BLOCK_SIZE
,
365 .cra_ctxsize
= sizeof(struct cast5_ctx
),
367 .cra_type
= &crypto_blkcipher_type
,
368 .cra_module
= THIS_MODULE
,
371 .min_keysize
= CAST5_MIN_KEY_SIZE
,
372 .max_keysize
= CAST5_MAX_KEY_SIZE
,
373 .setkey
= cast5_setkey
,
374 .encrypt
= cbc_encrypt
,
375 .decrypt
= cbc_decrypt
,
379 .cra_name
= "__ctr-cast5-avx",
380 .cra_driver_name
= "__driver-ctr-cast5-avx",
382 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
384 .cra_ctxsize
= sizeof(struct cast5_ctx
),
386 .cra_type
= &crypto_blkcipher_type
,
387 .cra_module
= THIS_MODULE
,
390 .min_keysize
= CAST5_MIN_KEY_SIZE
,
391 .max_keysize
= CAST5_MAX_KEY_SIZE
,
392 .ivsize
= CAST5_BLOCK_SIZE
,
393 .setkey
= cast5_setkey
,
394 .encrypt
= ctr_crypt
,
395 .decrypt
= ctr_crypt
,
399 .cra_name
= "ecb(cast5)",
400 .cra_driver_name
= "ecb-cast5-avx",
402 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
,
403 .cra_blocksize
= CAST5_BLOCK_SIZE
,
404 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
406 .cra_type
= &crypto_ablkcipher_type
,
407 .cra_module
= THIS_MODULE
,
408 .cra_init
= ablk_init
,
409 .cra_exit
= ablk_exit
,
412 .min_keysize
= CAST5_MIN_KEY_SIZE
,
413 .max_keysize
= CAST5_MAX_KEY_SIZE
,
414 .setkey
= ablk_set_key
,
415 .encrypt
= ablk_encrypt
,
416 .decrypt
= ablk_decrypt
,
420 .cra_name
= "cbc(cast5)",
421 .cra_driver_name
= "cbc-cast5-avx",
423 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
,
424 .cra_blocksize
= CAST5_BLOCK_SIZE
,
425 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
427 .cra_type
= &crypto_ablkcipher_type
,
428 .cra_module
= THIS_MODULE
,
429 .cra_init
= ablk_init
,
430 .cra_exit
= ablk_exit
,
433 .min_keysize
= CAST5_MIN_KEY_SIZE
,
434 .max_keysize
= CAST5_MAX_KEY_SIZE
,
435 .ivsize
= CAST5_BLOCK_SIZE
,
436 .setkey
= ablk_set_key
,
437 .encrypt
= __ablk_encrypt
,
438 .decrypt
= ablk_decrypt
,
442 .cra_name
= "ctr(cast5)",
443 .cra_driver_name
= "ctr-cast5-avx",
445 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
,
447 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
449 .cra_type
= &crypto_ablkcipher_type
,
450 .cra_module
= THIS_MODULE
,
451 .cra_init
= ablk_init
,
452 .cra_exit
= ablk_exit
,
455 .min_keysize
= CAST5_MIN_KEY_SIZE
,
456 .max_keysize
= CAST5_MAX_KEY_SIZE
,
457 .ivsize
= CAST5_BLOCK_SIZE
,
458 .setkey
= ablk_set_key
,
459 .encrypt
= ablk_encrypt
,
460 .decrypt
= ablk_encrypt
,
466 static int __init
cast5_init(void)
470 if (!cpu_has_avx
|| !cpu_has_osxsave
) {
471 pr_info("AVX instructions are not detected.\n");
475 xcr0
= xgetbv(XCR_XFEATURE_ENABLED_MASK
);
476 if ((xcr0
& (XSTATE_SSE
| XSTATE_YMM
)) != (XSTATE_SSE
| XSTATE_YMM
)) {
477 pr_info("AVX detected but unusable.\n");
481 return crypto_register_algs(cast5_algs
, ARRAY_SIZE(cast5_algs
));
484 static void __exit
cast5_exit(void)
486 crypto_unregister_algs(cast5_algs
, ARRAY_SIZE(cast5_algs
));
489 module_init(cast5_init
);
490 module_exit(cast5_exit
);
492 MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized");
493 MODULE_LICENSE("GPL");
494 MODULE_ALIAS_CRYPTO("cast5");