]>
git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - arch/x86/crypto/chacha_glue.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * x64 SIMD accelerated ChaCha and XChaCha stream ciphers,
4 * including ChaCha20 (RFC7539)
6 * Copyright (C) 2015 Martin Willi
9 #include <crypto/algapi.h>
10 #include <crypto/chacha.h>
11 #include <crypto/internal/simd.h>
12 #include <crypto/internal/skcipher.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
17 #define CHACHA_STATE_ALIGN 16
19 asmlinkage
void chacha_block_xor_ssse3(u32
*state
, u8
*dst
, const u8
*src
,
20 unsigned int len
, int nrounds
);
21 asmlinkage
void chacha_4block_xor_ssse3(u32
*state
, u8
*dst
, const u8
*src
,
22 unsigned int len
, int nrounds
);
23 asmlinkage
void hchacha_block_ssse3(const u32
*state
, u32
*out
, int nrounds
);
25 asmlinkage
void chacha_2block_xor_avx2(u32
*state
, u8
*dst
, const u8
*src
,
26 unsigned int len
, int nrounds
);
27 asmlinkage
void chacha_4block_xor_avx2(u32
*state
, u8
*dst
, const u8
*src
,
28 unsigned int len
, int nrounds
);
29 asmlinkage
void chacha_8block_xor_avx2(u32
*state
, u8
*dst
, const u8
*src
,
30 unsigned int len
, int nrounds
);
31 static bool chacha_use_avx2
;
32 #ifdef CONFIG_AS_AVX512
33 asmlinkage
void chacha_2block_xor_avx512vl(u32
*state
, u8
*dst
, const u8
*src
,
34 unsigned int len
, int nrounds
);
35 asmlinkage
void chacha_4block_xor_avx512vl(u32
*state
, u8
*dst
, const u8
*src
,
36 unsigned int len
, int nrounds
);
37 asmlinkage
void chacha_8block_xor_avx512vl(u32
*state
, u8
*dst
, const u8
*src
,
38 unsigned int len
, int nrounds
);
39 static bool chacha_use_avx512vl
;
43 static unsigned int chacha_advance(unsigned int len
, unsigned int maxblocks
)
45 len
= min(len
, maxblocks
* CHACHA_BLOCK_SIZE
);
46 return round_up(len
, CHACHA_BLOCK_SIZE
) / CHACHA_BLOCK_SIZE
;
49 static void chacha_dosimd(u32
*state
, u8
*dst
, const u8
*src
,
50 unsigned int bytes
, int nrounds
)
53 #ifdef CONFIG_AS_AVX512
54 if (chacha_use_avx512vl
) {
55 while (bytes
>= CHACHA_BLOCK_SIZE
* 8) {
56 chacha_8block_xor_avx512vl(state
, dst
, src
, bytes
,
58 bytes
-= CHACHA_BLOCK_SIZE
* 8;
59 src
+= CHACHA_BLOCK_SIZE
* 8;
60 dst
+= CHACHA_BLOCK_SIZE
* 8;
63 if (bytes
> CHACHA_BLOCK_SIZE
* 4) {
64 chacha_8block_xor_avx512vl(state
, dst
, src
, bytes
,
66 state
[12] += chacha_advance(bytes
, 8);
69 if (bytes
> CHACHA_BLOCK_SIZE
* 2) {
70 chacha_4block_xor_avx512vl(state
, dst
, src
, bytes
,
72 state
[12] += chacha_advance(bytes
, 4);
76 chacha_2block_xor_avx512vl(state
, dst
, src
, bytes
,
78 state
[12] += chacha_advance(bytes
, 2);
83 if (chacha_use_avx2
) {
84 while (bytes
>= CHACHA_BLOCK_SIZE
* 8) {
85 chacha_8block_xor_avx2(state
, dst
, src
, bytes
, nrounds
);
86 bytes
-= CHACHA_BLOCK_SIZE
* 8;
87 src
+= CHACHA_BLOCK_SIZE
* 8;
88 dst
+= CHACHA_BLOCK_SIZE
* 8;
91 if (bytes
> CHACHA_BLOCK_SIZE
* 4) {
92 chacha_8block_xor_avx2(state
, dst
, src
, bytes
, nrounds
);
93 state
[12] += chacha_advance(bytes
, 8);
96 if (bytes
> CHACHA_BLOCK_SIZE
* 2) {
97 chacha_4block_xor_avx2(state
, dst
, src
, bytes
, nrounds
);
98 state
[12] += chacha_advance(bytes
, 4);
101 if (bytes
> CHACHA_BLOCK_SIZE
) {
102 chacha_2block_xor_avx2(state
, dst
, src
, bytes
, nrounds
);
103 state
[12] += chacha_advance(bytes
, 2);
108 while (bytes
>= CHACHA_BLOCK_SIZE
* 4) {
109 chacha_4block_xor_ssse3(state
, dst
, src
, bytes
, nrounds
);
110 bytes
-= CHACHA_BLOCK_SIZE
* 4;
111 src
+= CHACHA_BLOCK_SIZE
* 4;
112 dst
+= CHACHA_BLOCK_SIZE
* 4;
115 if (bytes
> CHACHA_BLOCK_SIZE
) {
116 chacha_4block_xor_ssse3(state
, dst
, src
, bytes
, nrounds
);
117 state
[12] += chacha_advance(bytes
, 4);
121 chacha_block_xor_ssse3(state
, dst
, src
, bytes
, nrounds
);
126 static int chacha_simd_stream_xor(struct skcipher_walk
*walk
,
127 struct chacha_ctx
*ctx
, u8
*iv
)
129 u32
*state
, state_buf
[16 + 2] __aligned(8);
130 int next_yield
= 4096; /* bytes until next FPU yield */
133 BUILD_BUG_ON(CHACHA_STATE_ALIGN
!= 16);
134 state
= PTR_ALIGN(state_buf
+ 0, CHACHA_STATE_ALIGN
);
136 crypto_chacha_init(state
, ctx
, iv
);
138 while (walk
->nbytes
> 0) {
139 unsigned int nbytes
= walk
->nbytes
;
141 if (nbytes
< walk
->total
) {
142 nbytes
= round_down(nbytes
, walk
->stride
);
143 next_yield
-= nbytes
;
146 chacha_dosimd(state
, walk
->dst
.virt
.addr
, walk
->src
.virt
.addr
,
147 nbytes
, ctx
->nrounds
);
149 if (next_yield
<= 0) {
150 /* temporarily allow preemption */
156 err
= skcipher_walk_done(walk
, walk
->nbytes
- nbytes
);
162 static int chacha_simd(struct skcipher_request
*req
)
164 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
165 struct chacha_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
166 struct skcipher_walk walk
;
169 if (req
->cryptlen
<= CHACHA_BLOCK_SIZE
|| !crypto_simd_usable())
170 return crypto_chacha_crypt(req
);
172 err
= skcipher_walk_virt(&walk
, req
, true);
177 err
= chacha_simd_stream_xor(&walk
, ctx
, req
->iv
);
182 static int xchacha_simd(struct skcipher_request
*req
)
184 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
185 struct chacha_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
186 struct skcipher_walk walk
;
187 struct chacha_ctx subctx
;
188 u32
*state
, state_buf
[16 + 2] __aligned(8);
192 if (req
->cryptlen
<= CHACHA_BLOCK_SIZE
|| !crypto_simd_usable())
193 return crypto_xchacha_crypt(req
);
195 err
= skcipher_walk_virt(&walk
, req
, true);
199 BUILD_BUG_ON(CHACHA_STATE_ALIGN
!= 16);
200 state
= PTR_ALIGN(state_buf
+ 0, CHACHA_STATE_ALIGN
);
201 crypto_chacha_init(state
, ctx
, req
->iv
);
205 hchacha_block_ssse3(state
, subctx
.key
, ctx
->nrounds
);
206 subctx
.nrounds
= ctx
->nrounds
;
208 memcpy(&real_iv
[0], req
->iv
+ 24, 8);
209 memcpy(&real_iv
[8], req
->iv
+ 16, 8);
210 err
= chacha_simd_stream_xor(&walk
, &subctx
, real_iv
);
217 static struct skcipher_alg algs
[] = {
219 .base
.cra_name
= "chacha20",
220 .base
.cra_driver_name
= "chacha20-simd",
221 .base
.cra_priority
= 300,
222 .base
.cra_blocksize
= 1,
223 .base
.cra_ctxsize
= sizeof(struct chacha_ctx
),
224 .base
.cra_module
= THIS_MODULE
,
226 .min_keysize
= CHACHA_KEY_SIZE
,
227 .max_keysize
= CHACHA_KEY_SIZE
,
228 .ivsize
= CHACHA_IV_SIZE
,
229 .chunksize
= CHACHA_BLOCK_SIZE
,
230 .setkey
= crypto_chacha20_setkey
,
231 .encrypt
= chacha_simd
,
232 .decrypt
= chacha_simd
,
234 .base
.cra_name
= "xchacha20",
235 .base
.cra_driver_name
= "xchacha20-simd",
236 .base
.cra_priority
= 300,
237 .base
.cra_blocksize
= 1,
238 .base
.cra_ctxsize
= sizeof(struct chacha_ctx
),
239 .base
.cra_module
= THIS_MODULE
,
241 .min_keysize
= CHACHA_KEY_SIZE
,
242 .max_keysize
= CHACHA_KEY_SIZE
,
243 .ivsize
= XCHACHA_IV_SIZE
,
244 .chunksize
= CHACHA_BLOCK_SIZE
,
245 .setkey
= crypto_chacha20_setkey
,
246 .encrypt
= xchacha_simd
,
247 .decrypt
= xchacha_simd
,
249 .base
.cra_name
= "xchacha12",
250 .base
.cra_driver_name
= "xchacha12-simd",
251 .base
.cra_priority
= 300,
252 .base
.cra_blocksize
= 1,
253 .base
.cra_ctxsize
= sizeof(struct chacha_ctx
),
254 .base
.cra_module
= THIS_MODULE
,
256 .min_keysize
= CHACHA_KEY_SIZE
,
257 .max_keysize
= CHACHA_KEY_SIZE
,
258 .ivsize
= XCHACHA_IV_SIZE
,
259 .chunksize
= CHACHA_BLOCK_SIZE
,
260 .setkey
= crypto_chacha12_setkey
,
261 .encrypt
= xchacha_simd
,
262 .decrypt
= xchacha_simd
,
266 static int __init
chacha_simd_mod_init(void)
268 if (!boot_cpu_has(X86_FEATURE_SSSE3
))
271 #ifdef CONFIG_AS_AVX2
272 chacha_use_avx2
= boot_cpu_has(X86_FEATURE_AVX
) &&
273 boot_cpu_has(X86_FEATURE_AVX2
) &&
274 cpu_has_xfeatures(XFEATURE_MASK_SSE
| XFEATURE_MASK_YMM
, NULL
);
275 #ifdef CONFIG_AS_AVX512
276 chacha_use_avx512vl
= chacha_use_avx2
&&
277 boot_cpu_has(X86_FEATURE_AVX512VL
) &&
278 boot_cpu_has(X86_FEATURE_AVX512BW
); /* kmovq */
281 return crypto_register_skciphers(algs
, ARRAY_SIZE(algs
));
284 static void __exit
chacha_simd_mod_fini(void)
286 crypto_unregister_skciphers(algs
, ARRAY_SIZE(algs
));
289 module_init(chacha_simd_mod_init
);
290 module_exit(chacha_simd_mod_fini
);
292 MODULE_LICENSE("GPL");
293 MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
294 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (x64 SIMD accelerated)");
295 MODULE_ALIAS_CRYPTO("chacha20");
296 MODULE_ALIAS_CRYPTO("chacha20-simd");
297 MODULE_ALIAS_CRYPTO("xchacha20");
298 MODULE_ALIAS_CRYPTO("xchacha20-simd");
299 MODULE_ALIAS_CRYPTO("xchacha12");
300 MODULE_ALIAS_CRYPTO("xchacha12-simd");