]>
Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
107778b5 JG |
2 | /* |
3 | * Glue Code for AVX assembler version of Twofish Cipher | |
4 | * | |
5 | * Copyright (C) 2012 Johannes Goetzfried | |
6 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> | |
7 | * | |
18be4527 | 8 | * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> |
107778b5 JG |
9 | */ |
10 | ||
11 | #include <linux/module.h> | |
107778b5 JG |
12 | #include <linux/types.h> |
13 | #include <linux/crypto.h> | |
14 | #include <linux/err.h> | |
15 | #include <crypto/algapi.h> | |
0e6ab46d | 16 | #include <crypto/internal/simd.h> |
107778b5 | 17 | #include <crypto/twofish.h> |
107778b5 | 18 | #include <crypto/xts.h> |
a7378d4e | 19 | #include <asm/crypto/glue_helper.h> |
0e6ab46d | 20 | #include <asm/crypto/twofish.h> |
107778b5 | 21 | |
107778b5 JG |
22 | #define TWOFISH_PARALLEL_BLOCKS 8 |
23 | ||
107778b5 | 24 | /* 8-way parallel cipher functions */ |
9c1e8836 KC |
25 | asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src); |
26 | asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src); | |
107778b5 | 27 | |
9c1e8836 KC |
28 | asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src); |
29 | asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src, | |
30 | le128 *iv); | |
107778b5 | 31 | |
9c1e8836 KC |
32 | asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src, |
33 | le128 *iv); | |
34 | asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src, | |
35 | le128 *iv); | |
18be4527 | 36 | |
0e6ab46d EB |
37 | static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, |
38 | const u8 *key, unsigned int keylen) | |
39 | { | |
40 | return twofish_setkey(&tfm->base, key, keylen); | |
41 | } | |
42 | ||
9c1e8836 | 43 | static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src) |
107778b5 | 44 | { |
8435a3c3 | 45 | __twofish_enc_blk_3way(ctx, dst, src, false); |
107778b5 JG |
46 | } |
47 | ||
9c1e8836 | 48 | static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) |
18be4527 | 49 | { |
9c1e8836 | 50 | glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk); |
18be4527 JK |
51 | } |
52 | ||
9c1e8836 | 53 | static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) |
18be4527 | 54 | { |
9c1e8836 | 55 | glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk); |
18be4527 JK |
56 | } |
57 | ||
ebeea983 EB |
58 | struct twofish_xts_ctx { |
59 | struct twofish_ctx tweak_ctx; | |
60 | struct twofish_ctx crypt_ctx; | |
61 | }; | |
62 | ||
0e6ab46d | 63 | static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key, |
ebeea983 EB |
64 | unsigned int keylen) |
65 | { | |
0e6ab46d | 66 | struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
ebeea983 EB |
67 | int err; |
68 | ||
0e6ab46d | 69 | err = xts_verify_key(tfm, key, keylen); |
ebeea983 EB |
70 | if (err) |
71 | return err; | |
72 | ||
73 | /* first half of xts-key is for crypt */ | |
674f368a | 74 | err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2); |
ebeea983 EB |
75 | if (err) |
76 | return err; | |
77 | ||
78 | /* second half of xts-key is for tweak */ | |
674f368a | 79 | return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); |
ebeea983 | 80 | } |
107778b5 | 81 | |
a7378d4e JK |
82 | static const struct common_glue_ctx twofish_enc = { |
83 | .num_funcs = 3, | |
84 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
85 | ||
86 | .funcs = { { | |
87 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
9c1e8836 | 88 | .fn_u = { .ecb = twofish_ecb_enc_8way } |
a7378d4e JK |
89 | }, { |
90 | .num_blocks = 3, | |
9c1e8836 | 91 | .fn_u = { .ecb = twofish_enc_blk_3way } |
a7378d4e JK |
92 | }, { |
93 | .num_blocks = 1, | |
9c1e8836 | 94 | .fn_u = { .ecb = twofish_enc_blk } |
a7378d4e JK |
95 | } } |
96 | }; | |
97 | ||
98 | static const struct common_glue_ctx twofish_ctr = { | |
99 | .num_funcs = 3, | |
100 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
101 | ||
102 | .funcs = { { | |
103 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
9c1e8836 | 104 | .fn_u = { .ctr = twofish_ctr_8way } |
a7378d4e JK |
105 | }, { |
106 | .num_blocks = 3, | |
9c1e8836 | 107 | .fn_u = { .ctr = twofish_enc_blk_ctr_3way } |
a7378d4e JK |
108 | }, { |
109 | .num_blocks = 1, | |
9c1e8836 | 110 | .fn_u = { .ctr = twofish_enc_blk_ctr } |
a7378d4e JK |
111 | } } |
112 | }; | |
113 | ||
18be4527 JK |
114 | static const struct common_glue_ctx twofish_enc_xts = { |
115 | .num_funcs = 2, | |
116 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
117 | ||
118 | .funcs = { { | |
119 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
9c1e8836 | 120 | .fn_u = { .xts = twofish_xts_enc_8way } |
18be4527 JK |
121 | }, { |
122 | .num_blocks = 1, | |
9c1e8836 | 123 | .fn_u = { .xts = twofish_xts_enc } |
18be4527 JK |
124 | } } |
125 | }; | |
126 | ||
a7378d4e JK |
127 | static const struct common_glue_ctx twofish_dec = { |
128 | .num_funcs = 3, | |
129 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
130 | ||
131 | .funcs = { { | |
132 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
9c1e8836 | 133 | .fn_u = { .ecb = twofish_ecb_dec_8way } |
a7378d4e JK |
134 | }, { |
135 | .num_blocks = 3, | |
9c1e8836 | 136 | .fn_u = { .ecb = twofish_dec_blk_3way } |
a7378d4e JK |
137 | }, { |
138 | .num_blocks = 1, | |
9c1e8836 | 139 | .fn_u = { .ecb = twofish_dec_blk } |
a7378d4e JK |
140 | } } |
141 | }; | |
142 | ||
143 | static const struct common_glue_ctx twofish_dec_cbc = { | |
144 | .num_funcs = 3, | |
145 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
146 | ||
147 | .funcs = { { | |
148 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
9c1e8836 | 149 | .fn_u = { .cbc = twofish_cbc_dec_8way } |
a7378d4e JK |
150 | }, { |
151 | .num_blocks = 3, | |
9c1e8836 | 152 | .fn_u = { .cbc = twofish_dec_blk_cbc_3way } |
a7378d4e JK |
153 | }, { |
154 | .num_blocks = 1, | |
9c1e8836 | 155 | .fn_u = { .cbc = twofish_dec_blk } |
a7378d4e JK |
156 | } } |
157 | }; | |
158 | ||
18be4527 JK |
159 | static const struct common_glue_ctx twofish_dec_xts = { |
160 | .num_funcs = 2, | |
161 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
162 | ||
163 | .funcs = { { | |
164 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
9c1e8836 | 165 | .fn_u = { .xts = twofish_xts_dec_8way } |
18be4527 JK |
166 | }, { |
167 | .num_blocks = 1, | |
9c1e8836 | 168 | .fn_u = { .xts = twofish_xts_dec } |
18be4527 JK |
169 | } } |
170 | }; | |
171 | ||
0e6ab46d | 172 | static int ecb_encrypt(struct skcipher_request *req) |
107778b5 | 173 | { |
0e6ab46d | 174 | return glue_ecb_req_128bit(&twofish_enc, req); |
107778b5 JG |
175 | } |
176 | ||
0e6ab46d | 177 | static int ecb_decrypt(struct skcipher_request *req) |
107778b5 | 178 | { |
0e6ab46d | 179 | return glue_ecb_req_128bit(&twofish_dec, req); |
107778b5 JG |
180 | } |
181 | ||
0e6ab46d | 182 | static int cbc_encrypt(struct skcipher_request *req) |
107778b5 | 183 | { |
9c1e8836 | 184 | return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req); |
107778b5 JG |
185 | } |
186 | ||
0e6ab46d | 187 | static int cbc_decrypt(struct skcipher_request *req) |
107778b5 | 188 | { |
0e6ab46d | 189 | return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); |
107778b5 JG |
190 | } |
191 | ||
0e6ab46d | 192 | static int ctr_crypt(struct skcipher_request *req) |
107778b5 | 193 | { |
0e6ab46d | 194 | return glue_ctr_req_128bit(&twofish_ctr, req); |
107778b5 JG |
195 | } |
196 | ||
0e6ab46d | 197 | static int xts_encrypt(struct skcipher_request *req) |
107778b5 | 198 | { |
0e6ab46d EB |
199 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
200 | struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
107778b5 | 201 | |
9c1e8836 | 202 | return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk, |
8ce5fac2 | 203 | &ctx->tweak_ctx, &ctx->crypt_ctx, false); |
107778b5 JG |
204 | } |
205 | ||
0e6ab46d | 206 | static int xts_decrypt(struct skcipher_request *req) |
107778b5 | 207 | { |
0e6ab46d EB |
208 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
209 | struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
107778b5 | 210 | |
9c1e8836 | 211 | return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk, |
8ce5fac2 | 212 | &ctx->tweak_ctx, &ctx->crypt_ctx, true); |
107778b5 JG |
213 | } |
214 | ||
0e6ab46d EB |
215 | static struct skcipher_alg twofish_algs[] = { |
216 | { | |
217 | .base.cra_name = "__ecb(twofish)", | |
218 | .base.cra_driver_name = "__ecb-twofish-avx", | |
219 | .base.cra_priority = 400, | |
220 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
221 | .base.cra_blocksize = TF_BLOCK_SIZE, | |
222 | .base.cra_ctxsize = sizeof(struct twofish_ctx), | |
223 | .base.cra_module = THIS_MODULE, | |
224 | .min_keysize = TF_MIN_KEY_SIZE, | |
225 | .max_keysize = TF_MAX_KEY_SIZE, | |
226 | .setkey = twofish_setkey_skcipher, | |
227 | .encrypt = ecb_encrypt, | |
228 | .decrypt = ecb_decrypt, | |
229 | }, { | |
230 | .base.cra_name = "__cbc(twofish)", | |
231 | .base.cra_driver_name = "__cbc-twofish-avx", | |
232 | .base.cra_priority = 400, | |
233 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
234 | .base.cra_blocksize = TF_BLOCK_SIZE, | |
235 | .base.cra_ctxsize = sizeof(struct twofish_ctx), | |
236 | .base.cra_module = THIS_MODULE, | |
237 | .min_keysize = TF_MIN_KEY_SIZE, | |
238 | .max_keysize = TF_MAX_KEY_SIZE, | |
239 | .ivsize = TF_BLOCK_SIZE, | |
240 | .setkey = twofish_setkey_skcipher, | |
241 | .encrypt = cbc_encrypt, | |
242 | .decrypt = cbc_decrypt, | |
243 | }, { | |
244 | .base.cra_name = "__ctr(twofish)", | |
245 | .base.cra_driver_name = "__ctr-twofish-avx", | |
246 | .base.cra_priority = 400, | |
247 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
248 | .base.cra_blocksize = 1, | |
249 | .base.cra_ctxsize = sizeof(struct twofish_ctx), | |
250 | .base.cra_module = THIS_MODULE, | |
251 | .min_keysize = TF_MIN_KEY_SIZE, | |
252 | .max_keysize = TF_MAX_KEY_SIZE, | |
253 | .ivsize = TF_BLOCK_SIZE, | |
254 | .chunksize = TF_BLOCK_SIZE, | |
255 | .setkey = twofish_setkey_skcipher, | |
256 | .encrypt = ctr_crypt, | |
257 | .decrypt = ctr_crypt, | |
258 | }, { | |
259 | .base.cra_name = "__xts(twofish)", | |
260 | .base.cra_driver_name = "__xts-twofish-avx", | |
261 | .base.cra_priority = 400, | |
262 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
263 | .base.cra_blocksize = TF_BLOCK_SIZE, | |
264 | .base.cra_ctxsize = sizeof(struct twofish_xts_ctx), | |
265 | .base.cra_module = THIS_MODULE, | |
266 | .min_keysize = 2 * TF_MIN_KEY_SIZE, | |
267 | .max_keysize = 2 * TF_MAX_KEY_SIZE, | |
268 | .ivsize = TF_BLOCK_SIZE, | |
269 | .setkey = xts_twofish_setkey, | |
270 | .encrypt = xts_encrypt, | |
271 | .decrypt = xts_decrypt, | |
107778b5 | 272 | }, |
0e6ab46d EB |
273 | }; |
274 | ||
275 | static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)]; | |
107778b5 JG |
276 | |
277 | static int __init twofish_init(void) | |
278 | { | |
4eecd261 | 279 | const char *feature_name; |
107778b5 | 280 | |
158ecc39 | 281 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) { |
4eecd261 | 282 | pr_info("CPU feature '%s' is not supported.\n", feature_name); |
107778b5 JG |
283 | return -ENODEV; |
284 | } | |
285 | ||
0e6ab46d EB |
286 | return simd_register_skciphers_compat(twofish_algs, |
287 | ARRAY_SIZE(twofish_algs), | |
288 | twofish_simd_algs); | |
107778b5 JG |
289 | } |
290 | ||
291 | static void __exit twofish_exit(void) | |
292 | { | |
0e6ab46d EB |
293 | simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs), |
294 | twofish_simd_algs); | |
107778b5 JG |
295 | } |
296 | ||
297 | module_init(twofish_init); | |
298 | module_exit(twofish_exit); | |
299 | ||
300 | MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); | |
301 | MODULE_LICENSE("GPL"); | |
5d26a105 | 302 | MODULE_ALIAS_CRYPTO("twofish"); |