[mirror_ubuntu-jammy-kernel.git] / arch / x86 / crypto / twofish_avx_glue.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Glue Code for AVX assembler version of Twofish Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/twofish.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>
#include <asm/crypto/twofish.h>

#define TWOFISH_PARALLEL_BLOCKS 8

/* 8-way parallel cipher functions */
asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src);
asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src);

asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src);
asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
				 const u8 *src, le128 *iv);

asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src, le128 *iv);
asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src, le128 *iv);

static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
				   const u8 *key, unsigned int keylen)
{
	return twofish_setkey(&tfm->base, key, keylen);
}

static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
					const u8 *src)
{
	__twofish_enc_blk_3way(ctx, dst, src, false);
}

static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
				  GLUE_FUNC_CAST(twofish_enc_blk));
}

static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
				  GLUE_FUNC_CAST(twofish_dec_blk));
}

struct twofish_xts_ctx {
	struct twofish_ctx tweak_ctx;
	struct twofish_ctx crypt_ctx;
};

static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
	u32 *flags = &tfm->base.crt_flags;
	int err;

	err = xts_verify_key(tfm, key, keylen);
	if (err)
		return err;

	/* first half of xts-key is for crypt */
	err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
	if (err)
		return err;

	/* second half of xts-key is for tweak */
	return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
				flags);
}

static const struct common_glue_ctx twofish_enc = {
	.num_funcs = 3,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
	}, {
		.num_blocks = 3,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
	} }
};

static const struct common_glue_ctx twofish_ctr = {
	.num_funcs = 3,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
	}, {
		.num_blocks = 3,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
	} }
};

static const struct common_glue_ctx twofish_enc_xts = {
	.num_funcs = 2,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
	} }
};

static const struct common_glue_ctx twofish_dec = {
	.num_funcs = 3,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
	}, {
		.num_blocks = 3,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
	} }
};

static const struct common_glue_ctx twofish_dec_cbc = {
	.num_funcs = 3,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
	}, {
		.num_blocks = 3,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
	} }
};

static const struct common_glue_ctx twofish_dec_xts = {
	.num_funcs = 2,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
	} }
};

static int ecb_encrypt(struct skcipher_request *req)
{
	return glue_ecb_req_128bit(&twofish_enc, req);
}

static int ecb_decrypt(struct skcipher_request *req)
{
	return glue_ecb_req_128bit(&twofish_dec, req);
}

static int cbc_encrypt(struct skcipher_request *req)
{
	return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk),
					   req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
	return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
}

static int ctr_crypt(struct skcipher_request *req)
{
	return glue_ctr_req_128bit(&twofish_ctr, req);
}

static int xts_encrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

	return glue_xts_req_128bit(&twofish_enc_xts, req,
				   XTS_TWEAK_CAST(twofish_enc_blk),
				   &ctx->tweak_ctx, &ctx->crypt_ctx, false);
}

static int xts_decrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

	return glue_xts_req_128bit(&twofish_dec_xts, req,
				   XTS_TWEAK_CAST(twofish_enc_blk),
				   &ctx->tweak_ctx, &ctx->crypt_ctx, true);
}

static struct skcipher_alg twofish_algs[] = {
	{
		.base.cra_name		= "__ecb(twofish)",
		.base.cra_driver_name	= "__ecb-twofish-avx",
		.base.cra_priority	= 400,
		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
		.base.cra_blocksize	= TF_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct twofish_ctx),
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= TF_MIN_KEY_SIZE,
		.max_keysize		= TF_MAX_KEY_SIZE,
		.setkey			= twofish_setkey_skcipher,
		.encrypt		= ecb_encrypt,
		.decrypt		= ecb_decrypt,
	}, {
		.base.cra_name		= "__cbc(twofish)",
		.base.cra_driver_name	= "__cbc-twofish-avx",
		.base.cra_priority	= 400,
		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
		.base.cra_blocksize	= TF_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct twofish_ctx),
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= TF_MIN_KEY_SIZE,
		.max_keysize		= TF_MAX_KEY_SIZE,
		.ivsize			= TF_BLOCK_SIZE,
		.setkey			= twofish_setkey_skcipher,
		.encrypt		= cbc_encrypt,
		.decrypt		= cbc_decrypt,
	}, {
		.base.cra_name		= "__ctr(twofish)",
		.base.cra_driver_name	= "__ctr-twofish-avx",
		.base.cra_priority	= 400,
		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
		.base.cra_blocksize	= 1,
		.base.cra_ctxsize	= sizeof(struct twofish_ctx),
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= TF_MIN_KEY_SIZE,
		.max_keysize		= TF_MAX_KEY_SIZE,
		.ivsize			= TF_BLOCK_SIZE,
		.chunksize		= TF_BLOCK_SIZE,
		.setkey			= twofish_setkey_skcipher,
		.encrypt		= ctr_crypt,
		.decrypt		= ctr_crypt,
	}, {
		.base.cra_name		= "__xts(twofish)",
		.base.cra_driver_name	= "__xts-twofish-avx",
		.base.cra_priority	= 400,
		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
		.base.cra_blocksize	= TF_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct twofish_xts_ctx),
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= 2 * TF_MIN_KEY_SIZE,
		.max_keysize		= 2 * TF_MAX_KEY_SIZE,
		.ivsize			= TF_BLOCK_SIZE,
		.setkey			= xts_twofish_setkey,
		.encrypt		= xts_encrypt,
		.decrypt		= xts_decrypt,
	},
};

static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];

static int __init twofish_init(void)
{
	const char *feature_name;

	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
		pr_info("CPU feature '%s' is not supported.\n", feature_name);
		return -ENODEV;
	}

	return simd_register_skciphers_compat(twofish_algs,
					      ARRAY_SIZE(twofish_algs),
					      twofish_simd_algs);
}

static void __exit twofish_exit(void)
{
	simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
				  twofish_simd_algs);
}

module_init(twofish_init);
module_exit(twofish_exit);

MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("twofish");
Commit	Line	Data
1a59d1b8	1	// SPDX-License-Identifier: GPL-2.0-or-later
107778b5 JG	2	/*
	3	* Glue Code for AVX assembler version of Twofish Cipher
	4	*
	5	* Copyright (C) 2012 Johannes Goetzfried
	6	* <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
	7	*
18be4527	8	* Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
107778b5 JG	9	*/
	10
	11	#include <linux/module.h>
107778b5 JG	12	#include <linux/types.h>
	13	#include <linux/crypto.h>
	14	#include <linux/err.h>
	15	#include <crypto/algapi.h>
0e6ab46d	16	#include <crypto/internal/simd.h>
107778b5	17	#include <crypto/twofish.h>
107778b5	18	#include <crypto/xts.h>
a7378d4e	19	#include <asm/crypto/glue_helper.h>
0e6ab46d	20	#include <asm/crypto/twofish.h>
107778b5	21
107778b5 JG	22	#define TWOFISH_PARALLEL_BLOCKS 8
107778b5 JG	23
107778b5	24	/* 8-way parallel cipher functions */
8435a3c3 JK	25	asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx ctx, u8 dst,
	26	const u8 *src);
	27	asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx ctx, u8 dst,
107778b5 JG	28	const u8 *src);
107778b5 JG	29
8435a3c3 JK	30	asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx ctx, u8 dst,
	31	const u8 *src);
	32	asmlinkage void twofish_ctr_8way(struct twofish_ctx ctx, u8 dst,
	33	const u8 src, le128 iv);
107778b5	34
18be4527 JK	35	asmlinkage void twofish_xts_enc_8way(struct twofish_ctx ctx, u8 dst,
	36	const u8 src, le128 iv);
	37	asmlinkage void twofish_xts_dec_8way(struct twofish_ctx ctx, u8 dst,
	38	const u8 src, le128 iv);
	39
0e6ab46d EB	40	static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
	41	const u8 *key, unsigned int keylen)
	42	{
	43	return twofish_setkey(&tfm->base, key, keylen);
	44	}
	45
8435a3c3	46	static inline void twofish_enc_blk_3way(struct twofish_ctx ctx, u8 dst,
107778b5 JG	47	const u8 *src)
107778b5 JG	48	{
8435a3c3	49	__twofish_enc_blk_3way(ctx, dst, src, false);
107778b5 JG	50	}
107778b5 JG	51
99f42f93	52	static void twofish_xts_enc(void ctx, u128 dst, const u128 src, le128 iv)
18be4527 JK	53	{
	54	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
	55	GLUE_FUNC_CAST(twofish_enc_blk));
	56	}
	57
99f42f93	58	static void twofish_xts_dec(void ctx, u128 dst, const u128 src, le128 iv)
18be4527 JK	59	{
	60	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
	61	GLUE_FUNC_CAST(twofish_dec_blk));
	62	}
	63
ebeea983 EB	64	struct twofish_xts_ctx {
	65	struct twofish_ctx tweak_ctx;
	66	struct twofish_ctx crypt_ctx;
	67	};
	68
0e6ab46d	69	static int xts_twofish_setkey(struct crypto_skcipher tfm, const u8 key,
ebeea983 EB	70	unsigned int keylen)
ebeea983 EB	71	{
0e6ab46d EB	72	struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
0e6ab46d EB	73	u32 *flags = &tfm->base.crt_flags;
ebeea983 EB	74	int err;
ebeea983 EB	75
0e6ab46d	76	err = xts_verify_key(tfm, key, keylen);
ebeea983 EB	77	if (err)
	78	return err;
	79
	80	/* first half of xts-key is for crypt */
	81	err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
	82	if (err)
	83	return err;
	84
	85	/* second half of xts-key is for tweak */
	86	return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
	87	flags);
	88	}
107778b5	89
a7378d4e JK	90	static const struct common_glue_ctx twofish_enc = {
	91	.num_funcs = 3,
	92	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	93
	94	.funcs = { {
	95	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
8435a3c3	96	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
a7378d4e JK	97	}, {
	98	.num_blocks = 3,
	99	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
	100	}, {
	101	.num_blocks = 1,
	102	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
	103	} }
	104	};
	105
	106	static const struct common_glue_ctx twofish_ctr = {
	107	.num_funcs = 3,
	108	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	109
	110	.funcs = { {
	111	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
8435a3c3	112	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
a7378d4e JK	113	}, {
	114	.num_blocks = 3,
	115	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
	116	}, {
	117	.num_blocks = 1,
	118	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
	119	} }
	120	};
	121
18be4527 JK	122	static const struct common_glue_ctx twofish_enc_xts = {
	123	.num_funcs = 2,
	124	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	125
	126	.funcs = { {
	127	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
	128	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
	129	}, {
	130	.num_blocks = 1,
	131	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
	132	} }
	133	};
	134
a7378d4e JK	135	static const struct common_glue_ctx twofish_dec = {
	136	.num_funcs = 3,
	137	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	138
	139	.funcs = { {
	140	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
8435a3c3	141	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
a7378d4e JK	142	}, {
	143	.num_blocks = 3,
	144	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
	145	}, {
	146	.num_blocks = 1,
	147	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
	148	} }
	149	};
	150
	151	static const struct common_glue_ctx twofish_dec_cbc = {
	152	.num_funcs = 3,
	153	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	154
	155	.funcs = { {
	156	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
8435a3c3	157	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
a7378d4e JK	158	}, {
	159	.num_blocks = 3,
	160	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
	161	}, {
	162	.num_blocks = 1,
	163	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
	164	} }
	165	};
	166
18be4527 JK	167	static const struct common_glue_ctx twofish_dec_xts = {
	168	.num_funcs = 2,
	169	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	170
	171	.funcs = { {
	172	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
	173	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
	174	}, {
	175	.num_blocks = 1,
	176	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
	177	} }
	178	};
	179
0e6ab46d	180	static int ecb_encrypt(struct skcipher_request *req)
107778b5	181	{
0e6ab46d	182	return glue_ecb_req_128bit(&twofish_enc, req);
107778b5 JG	183	}
107778b5 JG	184
0e6ab46d	185	static int ecb_decrypt(struct skcipher_request *req)
107778b5	186	{
0e6ab46d	187	return glue_ecb_req_128bit(&twofish_dec, req);
107778b5 JG	188	}
107778b5 JG	189
0e6ab46d	190	static int cbc_encrypt(struct skcipher_request *req)
107778b5	191	{
0e6ab46d EB	192	return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk),
0e6ab46d EB	193	req);
107778b5 JG	194	}
107778b5 JG	195
0e6ab46d	196	static int cbc_decrypt(struct skcipher_request *req)
107778b5	197	{
0e6ab46d	198	return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
107778b5 JG	199	}
107778b5 JG	200
0e6ab46d	201	static int ctr_crypt(struct skcipher_request *req)
107778b5	202	{
0e6ab46d	203	return glue_ctr_req_128bit(&twofish_ctr, req);
107778b5 JG	204	}
107778b5 JG	205
0e6ab46d	206	static int xts_encrypt(struct skcipher_request *req)
107778b5	207	{
0e6ab46d EB	208	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0e6ab46d EB	209	struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
107778b5	210
0e6ab46d EB	211	return glue_xts_req_128bit(&twofish_enc_xts, req,
0e6ab46d EB	212	XTS_TWEAK_CAST(twofish_enc_blk),
8ce5fac2	213	&ctx->tweak_ctx, &ctx->crypt_ctx, false);
107778b5 JG	214	}
107778b5 JG	215
0e6ab46d	216	static int xts_decrypt(struct skcipher_request *req)
107778b5	217	{
0e6ab46d EB	218	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0e6ab46d EB	219	struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
107778b5	220
0e6ab46d EB	221	return glue_xts_req_128bit(&twofish_dec_xts, req,
0e6ab46d EB	222	XTS_TWEAK_CAST(twofish_enc_blk),
8ce5fac2	223	&ctx->tweak_ctx, &ctx->crypt_ctx, true);
107778b5 JG	224	}
107778b5 JG	225
0e6ab46d EB	226	static struct skcipher_alg twofish_algs[] = {
	227	{
	228	.base.cra_name = "__ecb(twofish)",
	229	.base.cra_driver_name = "__ecb-twofish-avx",
	230	.base.cra_priority = 400,
	231	.base.cra_flags = CRYPTO_ALG_INTERNAL,
	232	.base.cra_blocksize = TF_BLOCK_SIZE,
	233	.base.cra_ctxsize = sizeof(struct twofish_ctx),
	234	.base.cra_module = THIS_MODULE,
	235	.min_keysize = TF_MIN_KEY_SIZE,
	236	.max_keysize = TF_MAX_KEY_SIZE,
	237	.setkey = twofish_setkey_skcipher,
	238	.encrypt = ecb_encrypt,
	239	.decrypt = ecb_decrypt,
	240	}, {
	241	.base.cra_name = "__cbc(twofish)",
	242	.base.cra_driver_name = "__cbc-twofish-avx",
	243	.base.cra_priority = 400,
	244	.base.cra_flags = CRYPTO_ALG_INTERNAL,
	245	.base.cra_blocksize = TF_BLOCK_SIZE,
	246	.base.cra_ctxsize = sizeof(struct twofish_ctx),
	247	.base.cra_module = THIS_MODULE,
	248	.min_keysize = TF_MIN_KEY_SIZE,
	249	.max_keysize = TF_MAX_KEY_SIZE,
	250	.ivsize = TF_BLOCK_SIZE,
	251	.setkey = twofish_setkey_skcipher,
	252	.encrypt = cbc_encrypt,
	253	.decrypt = cbc_decrypt,
	254	}, {
	255	.base.cra_name = "__ctr(twofish)",
	256	.base.cra_driver_name = "__ctr-twofish-avx",
	257	.base.cra_priority = 400,
	258	.base.cra_flags = CRYPTO_ALG_INTERNAL,
	259	.base.cra_blocksize = 1,
	260	.base.cra_ctxsize = sizeof(struct twofish_ctx),
	261	.base.cra_module = THIS_MODULE,
	262	.min_keysize = TF_MIN_KEY_SIZE,
	263	.max_keysize = TF_MAX_KEY_SIZE,
	264	.ivsize = TF_BLOCK_SIZE,
	265	.chunksize = TF_BLOCK_SIZE,
	266	.setkey = twofish_setkey_skcipher,
	267	.encrypt = ctr_crypt,
	268	.decrypt = ctr_crypt,
	269	}, {
	270	.base.cra_name = "__xts(twofish)",
	271	.base.cra_driver_name = "__xts-twofish-avx",
	272	.base.cra_priority = 400,
	273	.base.cra_flags = CRYPTO_ALG_INTERNAL,
	274	.base.cra_blocksize = TF_BLOCK_SIZE,
	275	.base.cra_ctxsize = sizeof(struct twofish_xts_ctx),
	276	.base.cra_module = THIS_MODULE,
	277	.min_keysize = 2 * TF_MIN_KEY_SIZE,
	278	.max_keysize = 2 * TF_MAX_KEY_SIZE,
	279	.ivsize = TF_BLOCK_SIZE,
	280	.setkey = xts_twofish_setkey,
	281	.encrypt = xts_encrypt,
	282	.decrypt = xts_decrypt,
107778b5	283	},
0e6ab46d EB	284	};
	285
	286	static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
107778b5 JG	287
	288	static int __init twofish_init(void)
	289	{
4eecd261	290	const char *feature_name;
107778b5	291
158ecc39	292	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE \| XFEATURE_MASK_YMM, &feature_name)) {
4eecd261	293	pr_info("CPU feature '%s' is not supported.\n", feature_name);
107778b5 JG	294	return -ENODEV;
	295	}
	296
0e6ab46d EB	297	return simd_register_skciphers_compat(twofish_algs,
	298	ARRAY_SIZE(twofish_algs),
	299	twofish_simd_algs);
107778b5 JG	300	}
	301
	302	static void __exit twofish_exit(void)
	303	{
0e6ab46d EB	304	simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
0e6ab46d EB	305	twofish_simd_algs);
107778b5 JG	306	}
	307
	308	module_init(twofish_init);
	309	module_exit(twofish_exit);
	310
	311	MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
	312	MODULE_LICENSE("GPL");
5d26a105	313	MODULE_ALIAS_CRYPTO("twofish");