[mirror_ubuntu-artful-kernel.git] / arch / x86 / crypto / sha1_ssse3_glue.c

/*
 * Cryptographic API.
 *
 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
 * Supplemental SSE3 instructions.
 *
 * This file is based on sha1_generic.c
 *
 * Copyright (c) Alan Smithee.
 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
 * Copyright (c) Mathias Krause <minipli@googlemail.com>
 * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha1_base.h>
#include <asm/fpu/api.h>

typedef void (sha1_transform_fn)(u32 *digest, const char *data,
				unsigned int rounds);

static int sha1_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len, sha1_transform_fn *sha1_xform)
{
	struct sha1_state *sctx = shash_desc_ctx(desc);

	if (!irq_fpu_usable() ||
	    (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
		return crypto_sha1_update(desc, data, len);

	/* make sure casting to sha1_block_fn() is safe */
	BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);

	kernel_fpu_begin();
	sha1_base_do_update(desc, data, len,
			    (sha1_block_fn *)sha1_xform);
	kernel_fpu_end();

	return 0;
}

static int sha1_finup(struct shash_desc *desc, const u8 *data,
		      unsigned int len, u8 *out, sha1_transform_fn *sha1_xform)
{
	if (!irq_fpu_usable())
		return crypto_sha1_finup(desc, data, len, out);

	kernel_fpu_begin();
	if (len)
		sha1_base_do_update(desc, data, len,
				    (sha1_block_fn *)sha1_xform);
	sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform);
	kernel_fpu_end();

	return sha1_base_finish(desc, out);
}

asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
				     unsigned int rounds);

static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len)
{
	return sha1_update(desc, data, len,
			(sha1_transform_fn *) sha1_transform_ssse3);
}

static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
			      unsigned int len, u8 *out)
{
	return sha1_finup(desc, data, len, out,
			(sha1_transform_fn *) sha1_transform_ssse3);
}

/* Add padding and return the message digest. */
static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
{
	return sha1_ssse3_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_ssse3_alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_ssse3_update,
	.final		=	sha1_ssse3_final,
	.finup		=	sha1_ssse3_finup,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name	=	"sha1",
		.cra_driver_name =	"sha1-ssse3",
		.cra_priority	=	150,
		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
		.cra_blocksize	=	SHA1_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
};

static int register_sha1_ssse3(void)
{
	if (boot_cpu_has(X86_FEATURE_SSSE3))
		return crypto_register_shash(&sha1_ssse3_alg);
	return 0;
}

static void unregister_sha1_ssse3(void)
{
	if (boot_cpu_has(X86_FEATURE_SSSE3))
		crypto_unregister_shash(&sha1_ssse3_alg);
}

#ifdef CONFIG_AS_AVX
asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
				   unsigned int rounds);

static int sha1_avx_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len)
{
	return sha1_update(desc, data, len,
			(sha1_transform_fn *) sha1_transform_avx);
}

static int sha1_avx_finup(struct shash_desc *desc, const u8 *data,
			      unsigned int len, u8 *out)
{
	return sha1_finup(desc, data, len, out,
			(sha1_transform_fn *) sha1_transform_avx);
}

static int sha1_avx_final(struct shash_desc *desc, u8 *out)
{
	return sha1_avx_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_avx_alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_avx_update,
	.final		=	sha1_avx_final,
	.finup		=	sha1_avx_finup,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name	=	"sha1",
		.cra_driver_name =	"sha1-avx",
		.cra_priority	=	160,
		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
		.cra_blocksize	=	SHA1_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
};

static bool avx_usable(void)
{
	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
		if (boot_cpu_has(X86_FEATURE_AVX))
			pr_info("AVX detected but unusable.\n");
		return false;
	}

	return true;
}

static int register_sha1_avx(void)
{
	if (avx_usable())
		return crypto_register_shash(&sha1_avx_alg);
	return 0;
}

static void unregister_sha1_avx(void)
{
	if (avx_usable())
		crypto_unregister_shash(&sha1_avx_alg);
}

#else  /* CONFIG_AS_AVX */
static inline int register_sha1_avx(void) { return 0; }
static inline void unregister_sha1_avx(void) { }
#endif /* CONFIG_AS_AVX */


#if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
#define SHA1_AVX2_BLOCK_OPTSIZE	4	/* optimal 4*64 bytes of SHA1 blocks */

asmlinkage void sha1_transform_avx2(u32 *digest, const char *data,
				    unsigned int rounds);

static bool avx2_usable(void)
{
	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
		&& boot_cpu_has(X86_FEATURE_BMI1)
		&& boot_cpu_has(X86_FEATURE_BMI2))
		return true;

	return false;
}

static void sha1_apply_transform_avx2(u32 *digest, const char *data,
				unsigned int rounds)
{
	/* Select the optimal transform based on data block size */
	if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE)
		sha1_transform_avx2(digest, data, rounds);
	else
		sha1_transform_avx(digest, data, rounds);
}

static int sha1_avx2_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len)
{
	return sha1_update(desc, data, len,
		(sha1_transform_fn *) sha1_apply_transform_avx2);
}

static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data,
			      unsigned int len, u8 *out)
{
	return sha1_finup(desc, data, len, out,
		(sha1_transform_fn *) sha1_apply_transform_avx2);
}

static int sha1_avx2_final(struct shash_desc *desc, u8 *out)
{
	return sha1_avx2_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_avx2_alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_avx2_update,
	.final		=	sha1_avx2_final,
	.finup		=	sha1_avx2_finup,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name	=	"sha1",
		.cra_driver_name =	"sha1-avx2",
		.cra_priority	=	170,
		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
		.cra_blocksize	=	SHA1_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
};

static int register_sha1_avx2(void)
{
	if (avx2_usable())
		return crypto_register_shash(&sha1_avx2_alg);
	return 0;
}

static void unregister_sha1_avx2(void)
{
	if (avx2_usable())
		crypto_unregister_shash(&sha1_avx2_alg);
}

#else
static inline int register_sha1_avx2(void) { return 0; }
static inline void unregister_sha1_avx2(void) { }
#endif

#ifdef CONFIG_AS_SHA1_NI
asmlinkage void sha1_ni_transform(u32 *digest, const char *data,
				   unsigned int rounds);

static int sha1_ni_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len)
{
	return sha1_update(desc, data, len,
		(sha1_transform_fn *) sha1_ni_transform);
}

static int sha1_ni_finup(struct shash_desc *desc, const u8 *data,
			      unsigned int len, u8 *out)
{
	return sha1_finup(desc, data, len, out,
		(sha1_transform_fn *) sha1_ni_transform);
}

static int sha1_ni_final(struct shash_desc *desc, u8 *out)
{
	return sha1_ni_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_ni_alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_ni_update,
	.final		=	sha1_ni_final,
	.finup		=	sha1_ni_finup,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name	=	"sha1",
		.cra_driver_name =	"sha1-ni",
		.cra_priority	=	250,
		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
		.cra_blocksize	=	SHA1_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
};

static int register_sha1_ni(void)
{
	if (boot_cpu_has(X86_FEATURE_SHA_NI))
		return crypto_register_shash(&sha1_ni_alg);
	return 0;
}

static void unregister_sha1_ni(void)
{
	if (boot_cpu_has(X86_FEATURE_SHA_NI))
		crypto_unregister_shash(&sha1_ni_alg);
}

#else
static inline int register_sha1_ni(void) { return 0; }
static inline void unregister_sha1_ni(void) { }
#endif

static int __init sha1_ssse3_mod_init(void)
{
	if (register_sha1_ssse3())
		goto fail;

	if (register_sha1_avx()) {
		unregister_sha1_ssse3();
		goto fail;
	}

	if (register_sha1_avx2()) {
		unregister_sha1_avx();
		unregister_sha1_ssse3();
		goto fail;
	}

	if (register_sha1_ni()) {
		unregister_sha1_avx2();
		unregister_sha1_avx();
		unregister_sha1_ssse3();
		goto fail;
	}

	return 0;
fail:
	return -ENODEV;
}

static void __exit sha1_ssse3_mod_fini(void)
{
	unregister_sha1_ni();
	unregister_sha1_avx2();
	unregister_sha1_avx();
	unregister_sha1_ssse3();
}

module_init(sha1_ssse3_mod_init);
module_exit(sha1_ssse3_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");

MODULE_ALIAS_CRYPTO("sha1");
MODULE_ALIAS_CRYPTO("sha1-ssse3");
MODULE_ALIAS_CRYPTO("sha1-avx");
MODULE_ALIAS_CRYPTO("sha1-avx2");
#ifdef CONFIG_AS_SHA1_NI
MODULE_ALIAS_CRYPTO("sha1-ni");
#endif
Commit	Line	Data
66be8951 MK	1	/*
	2	* Cryptographic API.
	3	*
	4	* Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
	5	* Supplemental SSE3 instructions.
	6	*
	7	* This file is based on sha1_generic.c
	8	*
	9	* Copyright (c) Alan Smithee.
	10	* Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
	11	* Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
	12	* Copyright (c) Mathias Krause <minipli@googlemail.com>
7c1da8d0	13	* Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
66be8951 MK	14	*
	15	* This program is free software; you can redistribute it and/or modify it
	16	* under the terms of the GNU General Public License as published by the Free
	17	* Software Foundation; either version 2 of the License, or (at your option)
	18	* any later version.
	19	*
	20	*/
	21
	22	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	23
	24	#include <crypto/internal/hash.h>
	25	#include <linux/init.h>
	26	#include <linux/module.h>
	27	#include <linux/mm.h>
	28	#include <linux/cryptohash.h>
	29	#include <linux/types.h>
	30	#include <crypto/sha.h>
824b4376	31	#include <crypto/sha1_base.h>
df6b35f4	32	#include <asm/fpu/api.h>
66be8951	33
85c66ecd	34	typedef void (sha1_transform_fn)(u32 digest, const char data,
85c66ecd	35	unsigned int rounds);
66be8951	36
85c66ecd	37	static int sha1_update(struct shash_desc desc, const u8 data,
85c66ecd	38	unsigned int len, sha1_transform_fn *sha1_xform)
66be8951 MK	39	{
66be8951 MK	40	struct sha1_state *sctx = shash_desc_ctx(desc);
66be8951	41
824b4376 AB	42	if (!irq_fpu_usable() \|\|
	43	(sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
	44	return crypto_sha1_update(desc, data, len);
66be8951	45
824b4376 AB	46	/* make sure casting to sha1_block_fn() is safe */
824b4376 AB	47	BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
66be8951	48
824b4376 AB	49	kernel_fpu_begin();
824b4376 AB	50	sha1_base_do_update(desc, data, len,
85c66ecd	51	(sha1_block_fn *)sha1_xform);
824b4376	52	kernel_fpu_end();
66be8951 MK	53
	54	return 0;
	55	}
	56
85c66ecd	57	static int sha1_finup(struct shash_desc desc, const u8 data,
85c66ecd	58	unsigned int len, u8 out, sha1_transform_fn sha1_xform)
66be8951	59	{
824b4376 AB	60	if (!irq_fpu_usable())
824b4376 AB	61	return crypto_sha1_finup(desc, data, len, out);
66be8951	62
824b4376 AB	63	kernel_fpu_begin();
	64	if (len)
	65	sha1_base_do_update(desc, data, len,
85c66ecd	66	(sha1_block_fn *)sha1_xform);
85c66ecd	67	sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform);
824b4376	68	kernel_fpu_end();
66be8951	69
824b4376	70	return sha1_base_finish(desc, out);
66be8951 MK	71	}
66be8951 MK	72
85c66ecd	73	asmlinkage void sha1_transform_ssse3(u32 digest, const char data,
	74	unsigned int rounds);
	75
	76	static int sha1_ssse3_update(struct shash_desc desc, const u8 data,
	77	unsigned int len)
66be8951	78	{
85c66ecd	79	return sha1_update(desc, data, len,
85c66ecd	80	(sha1_transform_fn *) sha1_transform_ssse3);
66be8951 MK	81	}
66be8951 MK	82
85c66ecd	83	static int sha1_ssse3_finup(struct shash_desc desc, const u8 data,
85c66ecd	84	unsigned int len, u8 *out)
7c1da8d0	85	{
85c66ecd	86	return sha1_finup(desc, data, len, out,
	87	(sha1_transform_fn *) sha1_transform_ssse3);
	88	}
	89
	90	/* Add padding and return the message digest. */
	91	static int sha1_ssse3_final(struct shash_desc desc, u8 out)
	92	{
	93	return sha1_ssse3_finup(desc, NULL, 0, out);
7c1da8d0	94	}
7c1da8d0	95
85c66ecd	96	static struct shash_alg sha1_ssse3_alg = {
66be8951	97	.digestsize = SHA1_DIGEST_SIZE,
824b4376	98	.init = sha1_base_init,
66be8951 MK	99	.update = sha1_ssse3_update,
66be8951 MK	100	.final = sha1_ssse3_final,
824b4376	101	.finup = sha1_ssse3_finup,
66be8951	102	.descsize = sizeof(struct sha1_state),
66be8951 MK	103	.base = {
66be8951 MK	104	.cra_name = "sha1",
85c66ecd	105	.cra_driver_name = "sha1-ssse3",
66be8951 MK	106	.cra_priority = 150,
	107	.cra_flags = CRYPTO_ALG_TYPE_SHASH,
	108	.cra_blocksize = SHA1_BLOCK_SIZE,
	109	.cra_module = THIS_MODULE,
	110	}
	111	};
	112
85c66ecd	113	static int register_sha1_ssse3(void)
	114	{
	115	if (boot_cpu_has(X86_FEATURE_SSSE3))
	116	return crypto_register_shash(&sha1_ssse3_alg);
	117	return 0;
	118	}
	119
	120	static void unregister_sha1_ssse3(void)
	121	{
	122	if (boot_cpu_has(X86_FEATURE_SSSE3))
	123	crypto_unregister_shash(&sha1_ssse3_alg);
	124	}
	125
65df5774	126	#ifdef CONFIG_AS_AVX
85c66ecd	127	asmlinkage void sha1_transform_avx(u32 digest, const char data,
	128	unsigned int rounds);
	129
	130	static int sha1_avx_update(struct shash_desc desc, const u8 data,
	131	unsigned int len)
	132	{
	133	return sha1_update(desc, data, len,
	134	(sha1_transform_fn *) sha1_transform_avx);
	135	}
	136
	137	static int sha1_avx_finup(struct shash_desc desc, const u8 data,
	138	unsigned int len, u8 *out)
	139	{
	140	return sha1_finup(desc, data, len, out,
	141	(sha1_transform_fn *) sha1_transform_avx);
	142	}
	143
	144	static int sha1_avx_final(struct shash_desc desc, u8 out)
	145	{
	146	return sha1_avx_finup(desc, NULL, 0, out);
	147	}
	148
	149	static struct shash_alg sha1_avx_alg = {
	150	.digestsize = SHA1_DIGEST_SIZE,
	151	.init = sha1_base_init,
	152	.update = sha1_avx_update,
	153	.final = sha1_avx_final,
	154	.finup = sha1_avx_finup,
	155	.descsize = sizeof(struct sha1_state),
	156	.base = {
	157	.cra_name = "sha1",
	158	.cra_driver_name = "sha1-avx",
	159	.cra_priority = 160,
	160	.cra_flags = CRYPTO_ALG_TYPE_SHASH,
	161	.cra_blocksize = SHA1_BLOCK_SIZE,
	162	.cra_module = THIS_MODULE,
	163	}
	164	};
	165
	166	static bool avx_usable(void)
66be8951	167	{
d91cab78	168	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE \| XFEATURE_MASK_YMM, NULL)) {
da154e82	169	if (boot_cpu_has(X86_FEATURE_AVX))
d1e50966	170	pr_info("AVX detected but unusable.\n");
66be8951 MK	171	return false;
	172	}
	173
	174	return true;
	175	}
6ca5afb8	176
85c66ecd	177	static int register_sha1_avx(void)
	178	{
	179	if (avx_usable())
	180	return crypto_register_shash(&sha1_avx_alg);
	181	return 0;
	182	}
	183
	184	static void unregister_sha1_avx(void)
6ca5afb8	185	{
85c66ecd	186	if (avx_usable())
	187	crypto_unregister_shash(&sha1_avx_alg);
	188	}
	189
	190	#else /* CONFIG_AS_AVX */
	191	static inline int register_sha1_avx(void) { return 0; }
	192	static inline void unregister_sha1_avx(void) { }
	193	#endif /* CONFIG_AS_AVX */
	194
	195
	196	#if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
	197	#define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 464 bytes of SHA1 blocks /
	198
	199	asmlinkage void sha1_transform_avx2(u32 digest, const char data,
	200	unsigned int rounds);
	201
	202	static bool avx2_usable(void)
6ca5afb8	203	{
8861249c	204	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
85c66ecd	205	&& boot_cpu_has(X86_FEATURE_BMI1)
85c66ecd	206	&& boot_cpu_has(X86_FEATURE_BMI2))
6ca5afb8 MK	207	return true;
	208
	209	return false;
	210	}
66be8951	211
85c66ecd	212	static void sha1_apply_transform_avx2(u32 digest, const char data,
85c66ecd	213	unsigned int rounds)
66be8951	214	{
85c66ecd	215	/* Select the optimal transform based on data block size */
	216	if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE)
	217	sha1_transform_avx2(digest, data, rounds);
	218	else
	219	sha1_transform_avx(digest, data, rounds);
	220	}
6ca5afb8	221
85c66ecd	222	static int sha1_avx2_update(struct shash_desc desc, const u8 data,
	223	unsigned int len)
	224	{
	225	return sha1_update(desc, data, len,
	226	(sha1_transform_fn *) sha1_apply_transform_avx2);
	227	}
66be8951	228
85c66ecd	229	static int sha1_avx2_finup(struct shash_desc desc, const u8 data,
	230	unsigned int len, u8 *out)
	231	{
	232	return sha1_finup(desc, data, len, out,
	233	(sha1_transform_fn *) sha1_apply_transform_avx2);
	234	}
	235
	236	static int sha1_avx2_final(struct shash_desc desc, u8 out)
	237	{
	238	return sha1_avx2_finup(desc, NULL, 0, out);
	239	}
	240
	241	static struct shash_alg sha1_avx2_alg = {
	242	.digestsize = SHA1_DIGEST_SIZE,
	243	.init = sha1_base_init,
	244	.update = sha1_avx2_update,
	245	.final = sha1_avx2_final,
	246	.finup = sha1_avx2_finup,
	247	.descsize = sizeof(struct sha1_state),
	248	.base = {
	249	.cra_name = "sha1",
	250	.cra_driver_name = "sha1-avx2",
	251	.cra_priority = 170,
	252	.cra_flags = CRYPTO_ALG_TYPE_SHASH,
	253	.cra_blocksize = SHA1_BLOCK_SIZE,
	254	.cra_module = THIS_MODULE,
7c1da8d0	255	}
85c66ecd	256	};
	257
	258	static int register_sha1_avx2(void)
	259	{
	260	if (avx2_usable())
	261	return crypto_register_shash(&sha1_avx2_alg);
	262	return 0;
	263	}
	264
	265	static void unregister_sha1_avx2(void)
	266	{
	267	if (avx2_usable())
	268	crypto_unregister_shash(&sha1_avx2_alg);
	269	}
	270
	271	#else
	272	static inline int register_sha1_avx2(void) { return 0; }
	273	static inline void unregister_sha1_avx2(void) { }
6ca5afb8	274	#endif
85c66ecd	275
95fca7df	276	#ifdef CONFIG_AS_SHA1_NI
85c66ecd	277	asmlinkage void sha1_ni_transform(u32 digest, const char data,
	278	unsigned int rounds);
	279
	280	static int sha1_ni_update(struct shash_desc desc, const u8 data,
	281	unsigned int len)
	282	{
	283	return sha1_update(desc, data, len,
	284	(sha1_transform_fn *) sha1_ni_transform);
	285	}
	286
	287	static int sha1_ni_finup(struct shash_desc desc, const u8 data,
	288	unsigned int len, u8 *out)
	289	{
	290	return sha1_finup(desc, data, len, out,
	291	(sha1_transform_fn *) sha1_ni_transform);
	292	}
	293
	294	static int sha1_ni_final(struct shash_desc desc, u8 out)
	295	{
	296	return sha1_ni_finup(desc, NULL, 0, out);
	297	}
	298
	299	static struct shash_alg sha1_ni_alg = {
	300	.digestsize = SHA1_DIGEST_SIZE,
	301	.init = sha1_base_init,
	302	.update = sha1_ni_update,
	303	.final = sha1_ni_final,
	304	.finup = sha1_ni_finup,
	305	.descsize = sizeof(struct sha1_state),
	306	.base = {
	307	.cra_name = "sha1",
	308	.cra_driver_name = "sha1-ni",
	309	.cra_priority = 250,
	310	.cra_flags = CRYPTO_ALG_TYPE_SHASH,
	311	.cra_blocksize = SHA1_BLOCK_SIZE,
	312	.cra_module = THIS_MODULE,
95fca7df	313	}
85c66ecd	314	};
	315
	316	static int register_sha1_ni(void)
	317	{
	318	if (boot_cpu_has(X86_FEATURE_SHA_NI))
	319	return crypto_register_shash(&sha1_ni_alg);
	320	return 0;
	321	}
	322
	323	static void unregister_sha1_ni(void)
	324	{
	325	if (boot_cpu_has(X86_FEATURE_SHA_NI))
	326	crypto_unregister_shash(&sha1_ni_alg);
	327	}
	328
	329	#else
	330	static inline int register_sha1_ni(void) { return 0; }
	331	static inline void unregister_sha1_ni(void) { }
66be8951 MK	332	#endif
	333
	334	static int __init sha1_ssse3_mod_init(void)
	335	{
85c66ecd	336	if (register_sha1_ssse3())
85c66ecd	337	goto fail;
6ca5afb8	338
85c66ecd	339	if (register_sha1_avx()) {
	340	unregister_sha1_ssse3();
	341	goto fail;
7c1da8d0	342	}
66be8951	343
85c66ecd	344	if (register_sha1_avx2()) {
	345	unregister_sha1_avx();
	346	unregister_sha1_ssse3();
	347	goto fail;
7c1da8d0	348	}
66be8951	349
85c66ecd	350	if (register_sha1_ni()) {
	351	unregister_sha1_avx2();
	352	unregister_sha1_avx();
	353	unregister_sha1_ssse3();
	354	goto fail;
66be8951	355	}
66be8951	356
85c66ecd	357	return 0;
85c66ecd	358	fail:
66be8951 MK	359	return -ENODEV;
	360	}
	361
	362	static void __exit sha1_ssse3_mod_fini(void)
	363	{
85c66ecd	364	unregister_sha1_ni();
	365	unregister_sha1_avx2();
	366	unregister_sha1_avx();
	367	unregister_sha1_ssse3();
66be8951 MK	368	}
	369
	370	module_init(sha1_ssse3_mod_init);
	371	module_exit(sha1_ssse3_mod_fini);
	372
	373	MODULE_LICENSE("GPL");
	374	MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
	375
5d26a105	376	MODULE_ALIAS_CRYPTO("sha1");
1a445e8e SM	377	MODULE_ALIAS_CRYPTO("sha1-ssse3");
	378	MODULE_ALIAS_CRYPTO("sha1-avx");
	379	MODULE_ALIAS_CRYPTO("sha1-avx2");
	380	#ifdef CONFIG_AS_SHA1_NI
	381	MODULE_ALIAS_CRYPTO("sha1-ni");
	382	#endif