]>
git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blob - arch/x86/crypto/chacha20_glue.c
82e46589a189d0ea7fdaef47289582914608ba22
2 * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
4 * Copyright (C) 2015 Martin Willi
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
12 #include <crypto/algapi.h>
13 #include <crypto/chacha20.h>
14 #include <crypto/internal/skcipher.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17 #include <asm/fpu/api.h>
20 #define CHACHA20_STATE_ALIGN 16
22 asmlinkage
void chacha20_block_xor_ssse3(u32
*state
, u8
*dst
, const u8
*src
,
24 asmlinkage
void chacha20_4block_xor_ssse3(u32
*state
, u8
*dst
, const u8
*src
,
27 asmlinkage
void chacha20_2block_xor_avx2(u32
*state
, u8
*dst
, const u8
*src
,
29 asmlinkage
void chacha20_8block_xor_avx2(u32
*state
, u8
*dst
, const u8
*src
,
31 static bool chacha20_use_avx2
;
34 static unsigned int chacha20_advance(unsigned int len
, unsigned int maxblocks
)
36 len
= min(len
, maxblocks
* CHACHA20_BLOCK_SIZE
);
37 return round_up(len
, CHACHA20_BLOCK_SIZE
) / CHACHA20_BLOCK_SIZE
;
40 static void chacha20_dosimd(u32
*state
, u8
*dst
, const u8
*src
,
44 if (chacha20_use_avx2
) {
45 while (bytes
>= CHACHA20_BLOCK_SIZE
* 8) {
46 chacha20_8block_xor_avx2(state
, dst
, src
, bytes
);
47 bytes
-= CHACHA20_BLOCK_SIZE
* 8;
48 src
+= CHACHA20_BLOCK_SIZE
* 8;
49 dst
+= CHACHA20_BLOCK_SIZE
* 8;
52 if (bytes
> CHACHA20_BLOCK_SIZE
* 4) {
53 chacha20_8block_xor_avx2(state
, dst
, src
, bytes
);
54 state
[12] += chacha20_advance(bytes
, 8);
57 if (bytes
> CHACHA20_BLOCK_SIZE
) {
58 chacha20_2block_xor_avx2(state
, dst
, src
, bytes
);
59 state
[12] += chacha20_advance(bytes
, 2);
64 while (bytes
>= CHACHA20_BLOCK_SIZE
* 4) {
65 chacha20_4block_xor_ssse3(state
, dst
, src
, bytes
);
66 bytes
-= CHACHA20_BLOCK_SIZE
* 4;
67 src
+= CHACHA20_BLOCK_SIZE
* 4;
68 dst
+= CHACHA20_BLOCK_SIZE
* 4;
71 if (bytes
> CHACHA20_BLOCK_SIZE
) {
72 chacha20_4block_xor_ssse3(state
, dst
, src
, bytes
);
73 state
[12] += chacha20_advance(bytes
, 4);
77 chacha20_block_xor_ssse3(state
, dst
, src
, bytes
);
82 static int chacha20_simd(struct skcipher_request
*req
)
84 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
85 struct chacha20_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
86 u32
*state
, state_buf
[16 + 2] __aligned(8);
87 struct skcipher_walk walk
;
90 BUILD_BUG_ON(CHACHA20_STATE_ALIGN
!= 16);
91 state
= PTR_ALIGN(state_buf
+ 0, CHACHA20_STATE_ALIGN
);
93 if (req
->cryptlen
<= CHACHA20_BLOCK_SIZE
|| !may_use_simd())
94 return crypto_chacha20_crypt(req
);
96 err
= skcipher_walk_virt(&walk
, req
, true);
98 crypto_chacha20_init(state
, ctx
, walk
.iv
);
102 while (walk
.nbytes
> 0) {
103 unsigned int nbytes
= walk
.nbytes
;
105 if (nbytes
< walk
.total
)
106 nbytes
= round_down(nbytes
, walk
.stride
);
108 chacha20_dosimd(state
, walk
.dst
.virt
.addr
, walk
.src
.virt
.addr
,
111 err
= skcipher_walk_done(&walk
, walk
.nbytes
- nbytes
);
119 static struct skcipher_alg alg
= {
120 .base
.cra_name
= "chacha20",
121 .base
.cra_driver_name
= "chacha20-simd",
122 .base
.cra_priority
= 300,
123 .base
.cra_blocksize
= 1,
124 .base
.cra_ctxsize
= sizeof(struct chacha20_ctx
),
125 .base
.cra_module
= THIS_MODULE
,
127 .min_keysize
= CHACHA20_KEY_SIZE
,
128 .max_keysize
= CHACHA20_KEY_SIZE
,
129 .ivsize
= CHACHA20_IV_SIZE
,
130 .chunksize
= CHACHA20_BLOCK_SIZE
,
131 .setkey
= crypto_chacha20_setkey
,
132 .encrypt
= chacha20_simd
,
133 .decrypt
= chacha20_simd
,
136 static int __init
chacha20_simd_mod_init(void)
138 if (!boot_cpu_has(X86_FEATURE_SSSE3
))
141 #ifdef CONFIG_AS_AVX2
142 chacha20_use_avx2
= boot_cpu_has(X86_FEATURE_AVX
) &&
143 boot_cpu_has(X86_FEATURE_AVX2
) &&
144 cpu_has_xfeatures(XFEATURE_MASK_SSE
| XFEATURE_MASK_YMM
, NULL
);
146 return crypto_register_skcipher(&alg
);
149 static void __exit
chacha20_simd_mod_fini(void)
151 crypto_unregister_skcipher(&alg
);
154 module_init(chacha20_simd_mod_init
);
155 module_exit(chacha20_simd_mod_fini
);
157 MODULE_LICENSE("GPL");
158 MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
159 MODULE_DESCRIPTION("chacha20 cipher algorithm, SIMD accelerated");
160 MODULE_ALIAS_CRYPTO("chacha20");
161 MODULE_ALIAS_CRYPTO("chacha20-simd");