1 /* Glue code for DES encryption optimized for sparc64 crypto opcodes.
3 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/crypto.h>
9 #include <linux/init.h>
10 #include <linux/module.h>
12 #include <linux/types.h>
13 #include <crypto/algapi.h>
14 #include <crypto/des.h>
16 #include <asm/fpumacro.h>
17 #include <asm/pstate.h>
22 struct des_sparc64_ctx
{
23 u64 encrypt_expkey
[DES_EXPKEY_WORDS
/ 2];
24 u64 decrypt_expkey
[DES_EXPKEY_WORDS
/ 2];
27 struct des3_ede_sparc64_ctx
{
28 u64 encrypt_expkey
[DES3_EDE_EXPKEY_WORDS
/ 2];
29 u64 decrypt_expkey
[DES3_EDE_EXPKEY_WORDS
/ 2];
32 static void encrypt_to_decrypt(u64
*d
, const u64
*e
)
34 const u64
*s
= e
+ (DES_EXPKEY_WORDS
/ 2) - 1;
37 for (i
= 0; i
< DES_EXPKEY_WORDS
/ 2; i
++)
41 extern void des_sparc64_key_expand(const u32
*input_key
, u64
*key
);
43 static int des_set_key(struct crypto_tfm
*tfm
, const u8
*key
,
46 struct des_sparc64_ctx
*dctx
= crypto_tfm_ctx(tfm
);
47 u32
*flags
= &tfm
->crt_flags
;
48 u32 tmp
[DES_EXPKEY_WORDS
];
51 /* Even though we have special instructions for key expansion,
52 * we call des_ekey() so that we don't have to write our own
53 * weak key detection code.
55 ret
= des_ekey(tmp
, key
);
56 if (unlikely(ret
== 0) && (*flags
& CRYPTO_TFM_REQ_WEAK_KEY
)) {
57 *flags
|= CRYPTO_TFM_RES_WEAK_KEY
;
61 des_sparc64_key_expand((const u32
*) key
, &dctx
->encrypt_expkey
[0]);
62 encrypt_to_decrypt(&dctx
->decrypt_expkey
[0], &dctx
->encrypt_expkey
[0]);
67 extern void des_sparc64_crypt(const u64
*key
, const u64
*input
,
70 static void des_encrypt(struct crypto_tfm
*tfm
, u8
*dst
, const u8
*src
)
72 struct des_sparc64_ctx
*ctx
= crypto_tfm_ctx(tfm
);
73 const u64
*K
= ctx
->encrypt_expkey
;
75 des_sparc64_crypt(K
, (const u64
*) src
, (u64
*) dst
);
78 static void des_decrypt(struct crypto_tfm
*tfm
, u8
*dst
, const u8
*src
)
80 struct des_sparc64_ctx
*ctx
= crypto_tfm_ctx(tfm
);
81 const u64
*K
= ctx
->decrypt_expkey
;
83 des_sparc64_crypt(K
, (const u64
*) src
, (u64
*) dst
);
86 extern void des_sparc64_load_keys(const u64
*key
);
88 extern void des_sparc64_ecb_crypt(const u64
*input
, u64
*output
,
91 #define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1))
93 static int __ecb_crypt(struct blkcipher_desc
*desc
,
94 struct scatterlist
*dst
, struct scatterlist
*src
,
95 unsigned int nbytes
, bool encrypt
)
97 struct des_sparc64_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
98 struct blkcipher_walk walk
;
101 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
102 err
= blkcipher_walk_virt(desc
, &walk
);
105 des_sparc64_load_keys(&ctx
->encrypt_expkey
[0]);
107 des_sparc64_load_keys(&ctx
->decrypt_expkey
[0]);
108 while ((nbytes
= walk
.nbytes
)) {
109 unsigned int block_len
= nbytes
& DES_BLOCK_MASK
;
111 if (likely(block_len
)) {
112 des_sparc64_ecb_crypt((const u64
*)walk
.src
.virt
.addr
,
113 (u64
*) walk
.dst
.virt
.addr
,
116 nbytes
&= DES_BLOCK_SIZE
- 1;
117 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
123 static int ecb_encrypt(struct blkcipher_desc
*desc
,
124 struct scatterlist
*dst
, struct scatterlist
*src
,
127 return __ecb_crypt(desc
, dst
, src
, nbytes
, true);
130 static int ecb_decrypt(struct blkcipher_desc
*desc
,
131 struct scatterlist
*dst
, struct scatterlist
*src
,
134 return __ecb_crypt(desc
, dst
, src
, nbytes
, false);
137 extern void des_sparc64_cbc_encrypt(const u64
*input
, u64
*output
,
138 unsigned int len
, u64
*iv
);
140 static int cbc_encrypt(struct blkcipher_desc
*desc
,
141 struct scatterlist
*dst
, struct scatterlist
*src
,
144 struct des_sparc64_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
145 struct blkcipher_walk walk
;
148 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
149 err
= blkcipher_walk_virt(desc
, &walk
);
151 des_sparc64_load_keys(&ctx
->encrypt_expkey
[0]);
152 while ((nbytes
= walk
.nbytes
)) {
153 unsigned int block_len
= nbytes
& DES_BLOCK_MASK
;
155 if (likely(block_len
)) {
156 des_sparc64_cbc_encrypt((const u64
*)walk
.src
.virt
.addr
,
157 (u64
*) walk
.dst
.virt
.addr
,
158 block_len
, (u64
*) walk
.iv
);
160 nbytes
&= DES_BLOCK_SIZE
- 1;
161 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
167 extern void des_sparc64_cbc_decrypt(const u64
*input
, u64
*output
,
168 unsigned int len
, u64
*iv
);
170 static int cbc_decrypt(struct blkcipher_desc
*desc
,
171 struct scatterlist
*dst
, struct scatterlist
*src
,
174 struct des_sparc64_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
175 struct blkcipher_walk walk
;
178 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
179 err
= blkcipher_walk_virt(desc
, &walk
);
181 des_sparc64_load_keys(&ctx
->decrypt_expkey
[0]);
182 while ((nbytes
= walk
.nbytes
)) {
183 unsigned int block_len
= nbytes
& DES_BLOCK_MASK
;
185 if (likely(block_len
)) {
186 des_sparc64_cbc_decrypt((const u64
*)walk
.src
.virt
.addr
,
187 (u64
*) walk
.dst
.virt
.addr
,
188 block_len
, (u64
*) walk
.iv
);
190 nbytes
&= DES_BLOCK_SIZE
- 1;
191 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
197 static int des3_ede_set_key(struct crypto_tfm
*tfm
, const u8
*key
,
200 struct des3_ede_sparc64_ctx
*dctx
= crypto_tfm_ctx(tfm
);
201 const u32
*K
= (const u32
*)key
;
202 u32
*flags
= &tfm
->crt_flags
;
203 u64 k1
[DES_EXPKEY_WORDS
/ 2];
204 u64 k2
[DES_EXPKEY_WORDS
/ 2];
205 u64 k3
[DES_EXPKEY_WORDS
/ 2];
207 if (unlikely(!((K
[0] ^ K
[2]) | (K
[1] ^ K
[3])) ||
208 !((K
[2] ^ K
[4]) | (K
[3] ^ K
[5]))) &&
209 (*flags
& CRYPTO_TFM_REQ_WEAK_KEY
)) {
210 *flags
|= CRYPTO_TFM_RES_WEAK_KEY
;
214 des_sparc64_key_expand((const u32
*)key
, k1
);
216 des_sparc64_key_expand((const u32
*)key
, k2
);
218 des_sparc64_key_expand((const u32
*)key
, k3
);
220 memcpy(&dctx
->encrypt_expkey
[0], &k1
[0], sizeof(k1
));
221 encrypt_to_decrypt(&dctx
->encrypt_expkey
[DES_EXPKEY_WORDS
/ 2], &k2
[0]);
222 memcpy(&dctx
->encrypt_expkey
[(DES_EXPKEY_WORDS
/ 2) * 2],
225 encrypt_to_decrypt(&dctx
->decrypt_expkey
[0], &k3
[0]);
226 memcpy(&dctx
->decrypt_expkey
[DES_EXPKEY_WORDS
/ 2],
228 encrypt_to_decrypt(&dctx
->decrypt_expkey
[(DES_EXPKEY_WORDS
/ 2) * 2],
234 extern void des3_ede_sparc64_crypt(const u64
*key
, const u64
*input
,
237 static void des3_ede_encrypt(struct crypto_tfm
*tfm
, u8
*dst
, const u8
*src
)
239 struct des3_ede_sparc64_ctx
*ctx
= crypto_tfm_ctx(tfm
);
240 const u64
*K
= ctx
->encrypt_expkey
;
242 des3_ede_sparc64_crypt(K
, (const u64
*) src
, (u64
*) dst
);
245 static void des3_ede_decrypt(struct crypto_tfm
*tfm
, u8
*dst
, const u8
*src
)
247 struct des3_ede_sparc64_ctx
*ctx
= crypto_tfm_ctx(tfm
);
248 const u64
*K
= ctx
->decrypt_expkey
;
250 des3_ede_sparc64_crypt(K
, (const u64
*) src
, (u64
*) dst
);
253 extern void des3_ede_sparc64_load_keys(const u64
*key
);
255 extern void des3_ede_sparc64_ecb_crypt(const u64
*expkey
, const u64
*input
,
256 u64
*output
, unsigned int len
);
258 static int __ecb3_crypt(struct blkcipher_desc
*desc
,
259 struct scatterlist
*dst
, struct scatterlist
*src
,
260 unsigned int nbytes
, bool encrypt
)
262 struct des3_ede_sparc64_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
263 struct blkcipher_walk walk
;
267 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
268 err
= blkcipher_walk_virt(desc
, &walk
);
271 K
= &ctx
->encrypt_expkey
[0];
273 K
= &ctx
->decrypt_expkey
[0];
274 des3_ede_sparc64_load_keys(K
);
275 while ((nbytes
= walk
.nbytes
)) {
276 unsigned int block_len
= nbytes
& DES_BLOCK_MASK
;
278 if (likely(block_len
)) {
279 const u64
*src64
= (const u64
*)walk
.src
.virt
.addr
;
280 des3_ede_sparc64_ecb_crypt(K
, src64
,
281 (u64
*) walk
.dst
.virt
.addr
,
284 nbytes
&= DES_BLOCK_SIZE
- 1;
285 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
291 static int ecb3_encrypt(struct blkcipher_desc
*desc
,
292 struct scatterlist
*dst
, struct scatterlist
*src
,
295 return __ecb3_crypt(desc
, dst
, src
, nbytes
, true);
298 static int ecb3_decrypt(struct blkcipher_desc
*desc
,
299 struct scatterlist
*dst
, struct scatterlist
*src
,
302 return __ecb3_crypt(desc
, dst
, src
, nbytes
, false);
305 extern void des3_ede_sparc64_cbc_encrypt(const u64
*expkey
, const u64
*input
,
306 u64
*output
, unsigned int len
,
309 static int cbc3_encrypt(struct blkcipher_desc
*desc
,
310 struct scatterlist
*dst
, struct scatterlist
*src
,
313 struct des3_ede_sparc64_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
314 struct blkcipher_walk walk
;
318 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
319 err
= blkcipher_walk_virt(desc
, &walk
);
321 K
= &ctx
->encrypt_expkey
[0];
322 des3_ede_sparc64_load_keys(K
);
323 while ((nbytes
= walk
.nbytes
)) {
324 unsigned int block_len
= nbytes
& DES_BLOCK_MASK
;
326 if (likely(block_len
)) {
327 const u64
*src64
= (const u64
*)walk
.src
.virt
.addr
;
328 des3_ede_sparc64_cbc_encrypt(K
, src64
,
329 (u64
*) walk
.dst
.virt
.addr
,
333 nbytes
&= DES_BLOCK_SIZE
- 1;
334 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
340 extern void des3_ede_sparc64_cbc_decrypt(const u64
*expkey
, const u64
*input
,
341 u64
*output
, unsigned int len
,
344 static int cbc3_decrypt(struct blkcipher_desc
*desc
,
345 struct scatterlist
*dst
, struct scatterlist
*src
,
348 struct des3_ede_sparc64_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
349 struct blkcipher_walk walk
;
353 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
354 err
= blkcipher_walk_virt(desc
, &walk
);
356 K
= &ctx
->decrypt_expkey
[0];
357 des3_ede_sparc64_load_keys(K
);
358 while ((nbytes
= walk
.nbytes
)) {
359 unsigned int block_len
= nbytes
& DES_BLOCK_MASK
;
361 if (likely(block_len
)) {
362 const u64
*src64
= (const u64
*)walk
.src
.virt
.addr
;
363 des3_ede_sparc64_cbc_decrypt(K
, src64
,
364 (u64
*) walk
.dst
.virt
.addr
,
368 nbytes
&= DES_BLOCK_SIZE
- 1;
369 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
375 static struct crypto_alg algs
[] = { {
377 .cra_driver_name
= "des-sparc64",
378 .cra_priority
= SPARC_CR_OPCODE_PRIORITY
,
379 .cra_flags
= CRYPTO_ALG_TYPE_CIPHER
,
380 .cra_blocksize
= DES_BLOCK_SIZE
,
381 .cra_ctxsize
= sizeof(struct des_sparc64_ctx
),
383 .cra_module
= THIS_MODULE
,
386 .cia_min_keysize
= DES_KEY_SIZE
,
387 .cia_max_keysize
= DES_KEY_SIZE
,
388 .cia_setkey
= des_set_key
,
389 .cia_encrypt
= des_encrypt
,
390 .cia_decrypt
= des_decrypt
394 .cra_name
= "ecb(des)",
395 .cra_driver_name
= "ecb-des-sparc64",
396 .cra_priority
= SPARC_CR_OPCODE_PRIORITY
,
397 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
398 .cra_blocksize
= DES_BLOCK_SIZE
,
399 .cra_ctxsize
= sizeof(struct des_sparc64_ctx
),
401 .cra_type
= &crypto_blkcipher_type
,
402 .cra_module
= THIS_MODULE
,
405 .min_keysize
= DES_KEY_SIZE
,
406 .max_keysize
= DES_KEY_SIZE
,
407 .setkey
= des_set_key
,
408 .encrypt
= ecb_encrypt
,
409 .decrypt
= ecb_decrypt
,
413 .cra_name
= "cbc(des)",
414 .cra_driver_name
= "cbc-des-sparc64",
415 .cra_priority
= SPARC_CR_OPCODE_PRIORITY
,
416 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
417 .cra_blocksize
= DES_BLOCK_SIZE
,
418 .cra_ctxsize
= sizeof(struct des_sparc64_ctx
),
420 .cra_type
= &crypto_blkcipher_type
,
421 .cra_module
= THIS_MODULE
,
424 .min_keysize
= DES_KEY_SIZE
,
425 .max_keysize
= DES_KEY_SIZE
,
426 .setkey
= des_set_key
,
427 .encrypt
= cbc_encrypt
,
428 .decrypt
= cbc_decrypt
,
432 .cra_name
= "des3_ede",
433 .cra_driver_name
= "des3_ede-sparc64",
434 .cra_priority
= SPARC_CR_OPCODE_PRIORITY
,
435 .cra_flags
= CRYPTO_ALG_TYPE_CIPHER
,
436 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
437 .cra_ctxsize
= sizeof(struct des3_ede_sparc64_ctx
),
439 .cra_module
= THIS_MODULE
,
442 .cia_min_keysize
= DES3_EDE_KEY_SIZE
,
443 .cia_max_keysize
= DES3_EDE_KEY_SIZE
,
444 .cia_setkey
= des3_ede_set_key
,
445 .cia_encrypt
= des3_ede_encrypt
,
446 .cia_decrypt
= des3_ede_decrypt
450 .cra_name
= "ecb(des3_ede)",
451 .cra_driver_name
= "ecb-des3_ede-sparc64",
452 .cra_priority
= SPARC_CR_OPCODE_PRIORITY
,
453 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
454 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
455 .cra_ctxsize
= sizeof(struct des3_ede_sparc64_ctx
),
457 .cra_type
= &crypto_blkcipher_type
,
458 .cra_module
= THIS_MODULE
,
461 .min_keysize
= DES3_EDE_KEY_SIZE
,
462 .max_keysize
= DES3_EDE_KEY_SIZE
,
463 .setkey
= des3_ede_set_key
,
464 .encrypt
= ecb3_encrypt
,
465 .decrypt
= ecb3_decrypt
,
469 .cra_name
= "cbc(des3_ede)",
470 .cra_driver_name
= "cbc-des3_ede-sparc64",
471 .cra_priority
= SPARC_CR_OPCODE_PRIORITY
,
472 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
473 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
474 .cra_ctxsize
= sizeof(struct des3_ede_sparc64_ctx
),
476 .cra_type
= &crypto_blkcipher_type
,
477 .cra_module
= THIS_MODULE
,
480 .min_keysize
= DES3_EDE_KEY_SIZE
,
481 .max_keysize
= DES3_EDE_KEY_SIZE
,
482 .setkey
= des3_ede_set_key
,
483 .encrypt
= cbc3_encrypt
,
484 .decrypt
= cbc3_decrypt
,
489 static bool __init
sparc64_has_des_opcode(void)
493 if (!(sparc64_elf_hwcap
& HWCAP_SPARC_CRYPTO
))
496 __asm__
__volatile__("rd %%asr26, %0" : "=r" (cfr
));
497 if (!(cfr
& CFR_DES
))
503 static int __init
des_sparc64_mod_init(void)
507 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++)
508 INIT_LIST_HEAD(&algs
[i
].cra_list
);
510 if (sparc64_has_des_opcode()) {
511 pr_info("Using sparc64 des opcodes optimized DES implementation\n");
512 return crypto_register_algs(algs
, ARRAY_SIZE(algs
));
514 pr_info("sparc64 des opcodes not available.\n");
518 static void __exit
des_sparc64_mod_fini(void)
520 crypto_unregister_algs(algs
, ARRAY_SIZE(algs
));
523 module_init(des_sparc64_mod_init
);
524 module_exit(des_sparc64_mod_fini
);
526 MODULE_LICENSE("GPL");
527 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
531 #include "crop_devid.c"