[mirror_ubuntu-jammy-kernel.git] / arch / x86 / crypto / camellia_aesni_avx2_glue.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 */

#include <asm/crypto/camellia.h>
#include <asm/crypto/glue_helper.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/xts.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/types.h>

#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
#define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32

/* 32-way AVX2/AES-NI parallel cipher functions */
asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src);
asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src);

asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src);
asmlinkage void camellia_ctr_32way(const void *ctx, u8 *dst, const u8 *src,
				   le128 *iv);

asmlinkage void camellia_xts_enc_32way(const void *ctx, u8 *dst, const u8 *src,
				       le128 *iv);
asmlinkage void camellia_xts_dec_32way(const void *ctx, u8 *dst, const u8 *src,
				       le128 *iv);

static const struct common_glue_ctx camellia_enc = {
	.num_funcs = 4,
	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
		.fn_u = { .ecb = camellia_ecb_enc_32way }
	}, {
		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
		.fn_u = { .ecb = camellia_ecb_enc_16way }
	}, {
		.num_blocks = 2,
		.fn_u = { .ecb = camellia_enc_blk_2way }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = camellia_enc_blk }
	} }
};

static const struct common_glue_ctx camellia_ctr = {
	.num_funcs = 4,
	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
		.fn_u = { .ctr = camellia_ctr_32way }
	}, {
		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
		.fn_u = { .ctr = camellia_ctr_16way }
	}, {
		.num_blocks = 2,
		.fn_u = { .ctr = camellia_crypt_ctr_2way }
	}, {
		.num_blocks = 1,
		.fn_u = { .ctr = camellia_crypt_ctr }
	} }
};

static const struct common_glue_ctx camellia_enc_xts = {
	.num_funcs = 3,
	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
		.fn_u = { .xts = camellia_xts_enc_32way }
	}, {
		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
		.fn_u = { .xts = camellia_xts_enc_16way }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = camellia_xts_enc }
	} }
};

static const struct common_glue_ctx camellia_dec = {
	.num_funcs = 4,
	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
		.fn_u = { .ecb = camellia_ecb_dec_32way }
	}, {
		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
		.fn_u = { .ecb = camellia_ecb_dec_16way }
	}, {
		.num_blocks = 2,
		.fn_u = { .ecb = camellia_dec_blk_2way }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = camellia_dec_blk }
	} }
};

static const struct common_glue_ctx camellia_dec_cbc = {
	.num_funcs = 4,
	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
		.fn_u = { .cbc = camellia_cbc_dec_32way }
	}, {
		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
		.fn_u = { .cbc = camellia_cbc_dec_16way }
	}, {
		.num_blocks = 2,
		.fn_u = { .cbc = camellia_decrypt_cbc_2way }
	}, {
		.num_blocks = 1,
		.fn_u = { .cbc = camellia_dec_blk }
	} }
};

static const struct common_glue_ctx camellia_dec_xts = {
	.num_funcs = 3,
	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
		.fn_u = { .xts = camellia_xts_dec_32way }
	}, {
		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
		.fn_u = { .xts = camellia_xts_dec_16way }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = camellia_xts_dec }
	} }
};

static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
			   unsigned int keylen)
{
	return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen);
}

static int ecb_encrypt(struct skcipher_request *req)
{
	return glue_ecb_req_128bit(&camellia_enc, req);
}

static int ecb_decrypt(struct skcipher_request *req)
{
	return glue_ecb_req_128bit(&camellia_dec, req);
}

static int cbc_encrypt(struct skcipher_request *req)
{
	return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
	return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
}

static int ctr_crypt(struct skcipher_request *req)
{
	return glue_ctr_req_128bit(&camellia_ctr, req);
}

static int xts_encrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

	return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk,
				   &ctx->tweak_ctx, &ctx->crypt_ctx, false);
}

static int xts_decrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

	return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk,
				   &ctx->tweak_ctx, &ctx->crypt_ctx, true);
}

static struct skcipher_alg camellia_algs[] = {
	{
		.base.cra_name		= "__ecb(camellia)",
		.base.cra_driver_name	= "__ecb-camellia-aesni-avx2",
		.base.cra_priority	= 500,
		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= CAMELLIA_MIN_KEY_SIZE,
		.max_keysize		= CAMELLIA_MAX_KEY_SIZE,
		.setkey			= camellia_setkey,
		.encrypt		= ecb_encrypt,
		.decrypt		= ecb_decrypt,
	}, {
		.base.cra_name		= "__cbc(camellia)",
		.base.cra_driver_name	= "__cbc-camellia-aesni-avx2",
		.base.cra_priority	= 500,
		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= CAMELLIA_MIN_KEY_SIZE,
		.max_keysize		= CAMELLIA_MAX_KEY_SIZE,
		.ivsize			= CAMELLIA_BLOCK_SIZE,
		.setkey			= camellia_setkey,
		.encrypt		= cbc_encrypt,
		.decrypt		= cbc_decrypt,
	}, {
		.base.cra_name		= "__ctr(camellia)",
		.base.cra_driver_name	= "__ctr-camellia-aesni-avx2",
		.base.cra_priority	= 500,
		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
		.base.cra_blocksize	= 1,
		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= CAMELLIA_MIN_KEY_SIZE,
		.max_keysize		= CAMELLIA_MAX_KEY_SIZE,
		.ivsize			= CAMELLIA_BLOCK_SIZE,
		.chunksize		= CAMELLIA_BLOCK_SIZE,
		.setkey			= camellia_setkey,
		.encrypt		= ctr_crypt,
		.decrypt		= ctr_crypt,
	}, {
		.base.cra_name		= "__xts(camellia)",
		.base.cra_driver_name	= "__xts-camellia-aesni-avx2",
		.base.cra_priority	= 500,
		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct camellia_xts_ctx),
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= 2 * CAMELLIA_MIN_KEY_SIZE,
		.max_keysize		= 2 * CAMELLIA_MAX_KEY_SIZE,
		.ivsize			= CAMELLIA_BLOCK_SIZE,
		.setkey			= xts_camellia_setkey,
		.encrypt		= xts_encrypt,
		.decrypt		= xts_decrypt,
	},
};

static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];

static int __init camellia_aesni_init(void)
{
	const char *feature_name;

	if (!boot_cpu_has(X86_FEATURE_AVX) ||
	    !boot_cpu_has(X86_FEATURE_AVX2) ||
	    !boot_cpu_has(X86_FEATURE_AES) ||
	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
		pr_info("AVX2 or AES-NI instructions are not detected.\n");
		return -ENODEV;
	}

	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
				&feature_name)) {
		pr_info("CPU feature '%s' is not supported.\n", feature_name);
		return -ENODEV;
	}

	return simd_register_skciphers_compat(camellia_algs,
					      ARRAY_SIZE(camellia_algs),
					      camellia_simd_algs);
}

static void __exit camellia_aesni_fini(void)
{
	simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
				  camellia_simd_algs);
}

module_init(camellia_aesni_init);
module_exit(camellia_aesni_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
MODULE_ALIAS_CRYPTO("camellia");
MODULE_ALIAS_CRYPTO("camellia-asm");
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
f3f935a7 JK	2	/*
	3	* Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
	4	*
	5	* Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
f3f935a7 JK	6	*/
f3f935a7 JK	7
f3f935a7	8	#include <asm/crypto/camellia.h>
f3f935a7	9	#include <asm/crypto/glue_helper.h>
44893bc2 EB	10	#include <crypto/algapi.h>
	11	#include <crypto/internal/simd.h>
	12	#include <crypto/xts.h>
	13	#include <linux/crypto.h>
	14	#include <linux/err.h>
	15	#include <linux/module.h>
	16	#include <linux/types.h>
f3f935a7 JK	17
	18	#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
	19	#define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32
	20
	21	/* 32-way AVX2/AES-NI parallel cipher functions */
9c1e8836 KC	22	asmlinkage void camellia_ecb_enc_32way(const void ctx, u8 dst, const u8 *src);
9c1e8836 KC	23	asmlinkage void camellia_ecb_dec_32way(const void ctx, u8 dst, const u8 *src);
f3f935a7	24
9c1e8836 KC	25	asmlinkage void camellia_cbc_dec_32way(const void ctx, u8 dst, const u8 *src);
	26	asmlinkage void camellia_ctr_32way(const void ctx, u8 dst, const u8 *src,
	27	le128 *iv);
f3f935a7	28
9c1e8836 KC	29	asmlinkage void camellia_xts_enc_32way(const void ctx, u8 dst, const u8 *src,
	30	le128 *iv);
	31	asmlinkage void camellia_xts_dec_32way(const void ctx, u8 dst, const u8 *src,
	32	le128 *iv);
f3f935a7 JK	33
	34	static const struct common_glue_ctx camellia_enc = {
	35	.num_funcs = 4,
	36	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
	37
	38	.funcs = { {
	39	.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
9c1e8836	40	.fn_u = { .ecb = camellia_ecb_enc_32way }
f3f935a7 JK	41	}, {
f3f935a7 JK	42	.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
9c1e8836	43	.fn_u = { .ecb = camellia_ecb_enc_16way }
f3f935a7 JK	44	}, {
f3f935a7 JK	45	.num_blocks = 2,
9c1e8836	46	.fn_u = { .ecb = camellia_enc_blk_2way }
f3f935a7 JK	47	}, {
f3f935a7 JK	48	.num_blocks = 1,
9c1e8836	49	.fn_u = { .ecb = camellia_enc_blk }
f3f935a7 JK	50	} }
	51	};
	52
	53	static const struct common_glue_ctx camellia_ctr = {
	54	.num_funcs = 4,
	55	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
	56
	57	.funcs = { {
	58	.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
9c1e8836	59	.fn_u = { .ctr = camellia_ctr_32way }
f3f935a7 JK	60	}, {
f3f935a7 JK	61	.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
9c1e8836	62	.fn_u = { .ctr = camellia_ctr_16way }
f3f935a7 JK	63	}, {
f3f935a7 JK	64	.num_blocks = 2,
9c1e8836	65	.fn_u = { .ctr = camellia_crypt_ctr_2way }
f3f935a7 JK	66	}, {
f3f935a7 JK	67	.num_blocks = 1,
9c1e8836	68	.fn_u = { .ctr = camellia_crypt_ctr }
f3f935a7 JK	69	} }
	70	};
	71
	72	static const struct common_glue_ctx camellia_enc_xts = {
	73	.num_funcs = 3,
	74	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
	75
	76	.funcs = { {
	77	.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
9c1e8836	78	.fn_u = { .xts = camellia_xts_enc_32way }
f3f935a7 JK	79	}, {
f3f935a7 JK	80	.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
9c1e8836	81	.fn_u = { .xts = camellia_xts_enc_16way }
f3f935a7 JK	82	}, {
f3f935a7 JK	83	.num_blocks = 1,
9c1e8836	84	.fn_u = { .xts = camellia_xts_enc }
f3f935a7 JK	85	} }
	86	};
	87
	88	static const struct common_glue_ctx camellia_dec = {
	89	.num_funcs = 4,
	90	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
	91
	92	.funcs = { {
	93	.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
9c1e8836	94	.fn_u = { .ecb = camellia_ecb_dec_32way }
f3f935a7 JK	95	}, {
f3f935a7 JK	96	.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
9c1e8836	97	.fn_u = { .ecb = camellia_ecb_dec_16way }
f3f935a7 JK	98	}, {
f3f935a7 JK	99	.num_blocks = 2,
9c1e8836	100	.fn_u = { .ecb = camellia_dec_blk_2way }
f3f935a7 JK	101	}, {
f3f935a7 JK	102	.num_blocks = 1,
9c1e8836	103	.fn_u = { .ecb = camellia_dec_blk }
f3f935a7 JK	104	} }
	105	};
	106
	107	static const struct common_glue_ctx camellia_dec_cbc = {
	108	.num_funcs = 4,
	109	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
	110
	111	.funcs = { {
	112	.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
9c1e8836	113	.fn_u = { .cbc = camellia_cbc_dec_32way }
f3f935a7 JK	114	}, {
f3f935a7 JK	115	.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
9c1e8836	116	.fn_u = { .cbc = camellia_cbc_dec_16way }
f3f935a7 JK	117	}, {
f3f935a7 JK	118	.num_blocks = 2,
9c1e8836	119	.fn_u = { .cbc = camellia_decrypt_cbc_2way }
f3f935a7 JK	120	}, {
f3f935a7 JK	121	.num_blocks = 1,
9c1e8836	122	.fn_u = { .cbc = camellia_dec_blk }
f3f935a7 JK	123	} }
	124	};
	125
	126	static const struct common_glue_ctx camellia_dec_xts = {
	127	.num_funcs = 3,
	128	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
	129
	130	.funcs = { {
	131	.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
9c1e8836	132	.fn_u = { .xts = camellia_xts_dec_32way }
f3f935a7 JK	133	}, {
f3f935a7 JK	134	.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
9c1e8836	135	.fn_u = { .xts = camellia_xts_dec_16way }
f3f935a7 JK	136	}, {
f3f935a7 JK	137	.num_blocks = 1,
9c1e8836	138	.fn_u = { .xts = camellia_xts_dec }
f3f935a7 JK	139	} }
	140	};
	141
44893bc2 EB	142	static int camellia_setkey(struct crypto_skcipher tfm, const u8 key,
44893bc2 EB	143	unsigned int keylen)
f3f935a7	144	{
674f368a	145	return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen);
f3f935a7 JK	146	}
f3f935a7 JK	147
44893bc2	148	static int ecb_encrypt(struct skcipher_request *req)
f3f935a7	149	{
44893bc2	150	return glue_ecb_req_128bit(&camellia_enc, req);
f3f935a7 JK	151	}
f3f935a7 JK	152
44893bc2	153	static int ecb_decrypt(struct skcipher_request *req)
f3f935a7	154	{
44893bc2	155	return glue_ecb_req_128bit(&camellia_dec, req);
f3f935a7 JK	156	}
f3f935a7 JK	157
44893bc2	158	static int cbc_encrypt(struct skcipher_request *req)
f3f935a7	159	{
9c1e8836	160	return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req);
f3f935a7 JK	161	}
f3f935a7 JK	162
44893bc2	163	static int cbc_decrypt(struct skcipher_request *req)
f3f935a7	164	{
44893bc2	165	return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
f3f935a7 JK	166	}
f3f935a7 JK	167
44893bc2	168	static int ctr_crypt(struct skcipher_request *req)
f3f935a7	169	{
44893bc2	170	return glue_ctr_req_128bit(&camellia_ctr, req);
f3f935a7 JK	171	}
f3f935a7 JK	172
44893bc2	173	static int xts_encrypt(struct skcipher_request *req)
f3f935a7	174	{
44893bc2 EB	175	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
44893bc2 EB	176	struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
f3f935a7	177
9c1e8836	178	return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk,
8ce5fac2	179	&ctx->tweak_ctx, &ctx->crypt_ctx, false);
f3f935a7 JK	180	}
f3f935a7 JK	181
44893bc2	182	static int xts_decrypt(struct skcipher_request *req)
f3f935a7	183	{
44893bc2 EB	184	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
44893bc2 EB	185	struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
f3f935a7	186
9c1e8836	187	return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk,
8ce5fac2	188	&ctx->tweak_ctx, &ctx->crypt_ctx, true);
f3f935a7 JK	189	}
f3f935a7 JK	190
44893bc2 EB	191	static struct skcipher_alg camellia_algs[] = {
	192	{
	193	.base.cra_name = "__ecb(camellia)",
	194	.base.cra_driver_name = "__ecb-camellia-aesni-avx2",
	195	.base.cra_priority = 500,
	196	.base.cra_flags = CRYPTO_ALG_INTERNAL,
	197	.base.cra_blocksize = CAMELLIA_BLOCK_SIZE,
	198	.base.cra_ctxsize = sizeof(struct camellia_ctx),
	199	.base.cra_module = THIS_MODULE,
	200	.min_keysize = CAMELLIA_MIN_KEY_SIZE,
	201	.max_keysize = CAMELLIA_MAX_KEY_SIZE,
	202	.setkey = camellia_setkey,
	203	.encrypt = ecb_encrypt,
	204	.decrypt = ecb_decrypt,
	205	}, {
	206	.base.cra_name = "__cbc(camellia)",
	207	.base.cra_driver_name = "__cbc-camellia-aesni-avx2",
	208	.base.cra_priority = 500,
	209	.base.cra_flags = CRYPTO_ALG_INTERNAL,
	210	.base.cra_blocksize = CAMELLIA_BLOCK_SIZE,
	211	.base.cra_ctxsize = sizeof(struct camellia_ctx),
	212	.base.cra_module = THIS_MODULE,
	213	.min_keysize = CAMELLIA_MIN_KEY_SIZE,
	214	.max_keysize = CAMELLIA_MAX_KEY_SIZE,
	215	.ivsize = CAMELLIA_BLOCK_SIZE,
	216	.setkey = camellia_setkey,
	217	.encrypt = cbc_encrypt,
	218	.decrypt = cbc_decrypt,
	219	}, {
	220	.base.cra_name = "__ctr(camellia)",
	221	.base.cra_driver_name = "__ctr-camellia-aesni-avx2",
	222	.base.cra_priority = 500,
	223	.base.cra_flags = CRYPTO_ALG_INTERNAL,
	224	.base.cra_blocksize = 1,
	225	.base.cra_ctxsize = sizeof(struct camellia_ctx),
	226	.base.cra_module = THIS_MODULE,
	227	.min_keysize = CAMELLIA_MIN_KEY_SIZE,
	228	.max_keysize = CAMELLIA_MAX_KEY_SIZE,
	229	.ivsize = CAMELLIA_BLOCK_SIZE,
	230	.chunksize = CAMELLIA_BLOCK_SIZE,
	231	.setkey = camellia_setkey,
	232	.encrypt = ctr_crypt,
	233	.decrypt = ctr_crypt,
	234	}, {
	235	.base.cra_name = "__xts(camellia)",
	236	.base.cra_driver_name = "__xts-camellia-aesni-avx2",
	237	.base.cra_priority = 500,
	238	.base.cra_flags = CRYPTO_ALG_INTERNAL,
	239	.base.cra_blocksize = CAMELLIA_BLOCK_SIZE,
	240	.base.cra_ctxsize = sizeof(struct camellia_xts_ctx),
	241	.base.cra_module = THIS_MODULE,
	242	.min_keysize = 2 * CAMELLIA_MIN_KEY_SIZE,
	243	.max_keysize = 2 * CAMELLIA_MAX_KEY_SIZE,
	244	.ivsize = CAMELLIA_BLOCK_SIZE,
	245	.setkey = xts_camellia_setkey,
	246	.encrypt = xts_encrypt,
	247	.decrypt = xts_decrypt,
f3f935a7	248	},
44893bc2 EB	249	};
	250
	251	static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
f3f935a7 JK	252
	253	static int __init camellia_aesni_init(void)
	254	{
7bc371fa	255	const char *feature_name;
f3f935a7	256
da154e82 BP	257	if (!boot_cpu_has(X86_FEATURE_AVX) \|\|
da154e82 BP	258	!boot_cpu_has(X86_FEATURE_AVX2) \|\|
1f4dd793	259	!boot_cpu_has(X86_FEATURE_AES) \|\|
ab4a56fa	260	!boot_cpu_has(X86_FEATURE_OSXSAVE)) {
b54b4bbb IM	261	pr_info("AVX2 or AES-NI instructions are not detected.\n");
	262	return -ENODEV;
	263	}
	264
d91cab78 DH	265	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE \| XFEATURE_MASK_YMM,
d91cab78 DH	266	&feature_name)) {
7bc371fa	267	pr_info("CPU feature '%s' is not supported.\n", feature_name);
f3f935a7 JK	268	return -ENODEV;
	269	}
	270
44893bc2 EB	271	return simd_register_skciphers_compat(camellia_algs,
	272	ARRAY_SIZE(camellia_algs),
	273	camellia_simd_algs);
f3f935a7 JK	274	}
	275
	276	static void __exit camellia_aesni_fini(void)
	277	{
44893bc2 EB	278	simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
44893bc2 EB	279	camellia_simd_algs);
f3f935a7 JK	280	}
	281
	282	module_init(camellia_aesni_init);
	283	module_exit(camellia_aesni_fini);
	284
	285	MODULE_LICENSE("GPL");
	286	MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
5d26a105 KC	287	MODULE_ALIAS_CRYPTO("camellia");
5d26a105 KC	288	MODULE_ALIAS_CRYPTO("camellia-asm");