[mirror_ubuntu-zesty-kernel.git] / arch / arm64 / crypto / sha256-glue.c

/*
 * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64
 *
 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h>
#include <crypto/internal/hash.h>
#include <crypto/sha.h>
#include <crypto/sha256_base.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <linux/string.h>

MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_CRYPTO("sha224");
MODULE_ALIAS_CRYPTO("sha256");

asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
					unsigned int num_blks);

asmlinkage void sha256_block_neon(u32 *digest, const void *data,
				  unsigned int num_blks);

static int sha256_update(struct shash_desc *desc, const u8 *data,
			 unsigned int len)
{
	return sha256_base_do_update(desc, data, len,
				(sha256_block_fn *)sha256_block_data_order);
}

static int sha256_finup(struct shash_desc *desc, const u8 *data,
			unsigned int len, u8 *out)
{
	if (len)
		sha256_base_do_update(desc, data, len,
				(sha256_block_fn *)sha256_block_data_order);
	sha256_base_do_finalize(desc,
				(sha256_block_fn *)sha256_block_data_order);

	return sha256_base_finish(desc, out);
}

static int sha256_final(struct shash_desc *desc, u8 *out)
{
	return sha256_finup(desc, NULL, 0, out);
}

static struct shash_alg algs[] = { {
	.digestsize		= SHA256_DIGEST_SIZE,
	.init			= sha256_base_init,
	.update			= sha256_update,
	.final			= sha256_final,
	.finup			= sha256_finup,
	.descsize		= sizeof(struct sha256_state),
	.base.cra_name		= "sha256",
	.base.cra_driver_name	= "sha256-arm64",
	.base.cra_priority	= 100,
	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
	.base.cra_blocksize	= SHA256_BLOCK_SIZE,
	.base.cra_module	= THIS_MODULE,
}, {
	.digestsize		= SHA224_DIGEST_SIZE,
	.init			= sha224_base_init,
	.update			= sha256_update,
	.final			= sha256_final,
	.finup			= sha256_finup,
	.descsize		= sizeof(struct sha256_state),
	.base.cra_name		= "sha224",
	.base.cra_driver_name	= "sha224-arm64",
	.base.cra_priority	= 100,
	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
	.base.cra_blocksize	= SHA224_BLOCK_SIZE,
	.base.cra_module	= THIS_MODULE,
} };

static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
			      unsigned int len)
{
	/*
	 * Stacking and unstacking a substantial slice of the NEON register
	 * file may significantly affect performance for small updates when
	 * executing in interrupt context, so fall back to the scalar code
	 * in that case.
	 */
	if (!may_use_simd())
		return sha256_base_do_update(desc, data, len,
				(sha256_block_fn *)sha256_block_data_order);

	kernel_neon_begin();
	sha256_base_do_update(desc, data, len,
				(sha256_block_fn *)sha256_block_neon);
	kernel_neon_end();

	return 0;
}

static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
			     unsigned int len, u8 *out)
{
	if (!may_use_simd()) {
		if (len)
			sha256_base_do_update(desc, data, len,
				(sha256_block_fn *)sha256_block_data_order);
		sha256_base_do_finalize(desc,
				(sha256_block_fn *)sha256_block_data_order);
	} else {
		kernel_neon_begin();
		if (len)
			sha256_base_do_update(desc, data, len,
				(sha256_block_fn *)sha256_block_neon);
		sha256_base_do_finalize(desc,
				(sha256_block_fn *)sha256_block_neon);
		kernel_neon_end();
	}
	return sha256_base_finish(desc, out);
}

static int sha256_final_neon(struct shash_desc *desc, u8 *out)
{
	return sha256_finup_neon(desc, NULL, 0, out);
}

static struct shash_alg neon_algs[] = { {
	.digestsize		= SHA256_DIGEST_SIZE,
	.init			= sha256_base_init,
	.update			= sha256_update_neon,
	.final			= sha256_final_neon,
	.finup			= sha256_finup_neon,
	.descsize		= sizeof(struct sha256_state),
	.base.cra_name		= "sha256",
	.base.cra_driver_name	= "sha256-arm64-neon",
	.base.cra_priority	= 150,
	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
	.base.cra_blocksize	= SHA256_BLOCK_SIZE,
	.base.cra_module	= THIS_MODULE,
}, {
	.digestsize		= SHA224_DIGEST_SIZE,
	.init			= sha224_base_init,
	.update			= sha256_update_neon,
	.final			= sha256_final_neon,
	.finup			= sha256_finup_neon,
	.descsize		= sizeof(struct sha256_state),
	.base.cra_name		= "sha224",
	.base.cra_driver_name	= "sha224-arm64-neon",
	.base.cra_priority	= 150,
	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
	.base.cra_blocksize	= SHA224_BLOCK_SIZE,
	.base.cra_module	= THIS_MODULE,
} };

static int __init sha256_mod_init(void)
{
	int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
	if (ret)
		return ret;

	if (elf_hwcap & HWCAP_ASIMD) {
		ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
		if (ret)
			crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
	}
	return ret;
}

static void __exit sha256_mod_fini(void)
{
	if (elf_hwcap & HWCAP_ASIMD)
		crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
}

module_init(sha256_mod_init);
module_exit(sha256_mod_fini);
Commit	Line	Data
7918ecef AB	1	/*
	2	* Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64
	3	*
	4	* Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
	5	*
	6	* This program is free software; you can redistribute it and/or modify it
	7	* under the terms of the GNU General Public License as published by the Free
	8	* Software Foundation; either version 2 of the License, or (at your option)
	9	* any later version.
	10	*
	11	*/
	12
	13	#include <asm/hwcap.h>
	14	#include <asm/neon.h>
	15	#include <asm/simd.h>
	16	#include <crypto/internal/hash.h>
	17	#include <crypto/sha.h>
	18	#include <crypto/sha256_base.h>
	19	#include <linux/cryptohash.h>
	20	#include <linux/types.h>
	21	#include <linux/string.h>
	22
	23	MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
	24	MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
	25	MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
	26	MODULE_LICENSE("GPL v2");
	27	MODULE_ALIAS_CRYPTO("sha224");
	28	MODULE_ALIAS_CRYPTO("sha256");
	29
	30	asmlinkage void sha256_block_data_order(u32 digest, const void data,
	31	unsigned int num_blks);
	32
	33	asmlinkage void sha256_block_neon(u32 digest, const void data,
	34	unsigned int num_blks);
	35
	36	static int sha256_update(struct shash_desc desc, const u8 data,
	37	unsigned int len)
	38	{
	39	return sha256_base_do_update(desc, data, len,
	40	(sha256_block_fn *)sha256_block_data_order);
	41	}
	42
	43	static int sha256_finup(struct shash_desc desc, const u8 data,
	44	unsigned int len, u8 *out)
	45	{
	46	if (len)
	47	sha256_base_do_update(desc, data, len,
	48	(sha256_block_fn *)sha256_block_data_order);
	49	sha256_base_do_finalize(desc,
	50	(sha256_block_fn *)sha256_block_data_order);
	51
	52	return sha256_base_finish(desc, out);
	53	}
	54
	55	static int sha256_final(struct shash_desc desc, u8 out)
	56	{
	57	return sha256_finup(desc, NULL, 0, out);
	58	}
	59
	60	static struct shash_alg algs[] = { {
	61	.digestsize = SHA256_DIGEST_SIZE,
	62	.init = sha256_base_init,
	63	.update = sha256_update,
	64	.final = sha256_final,
65	.finup = sha256_finup,
66	.descsize = sizeof(struct sha256_state),
67	.base.cra_name = "sha256",
68	.base.cra_driver_name = "sha256-arm64",
69	.base.cra_priority = 100,
70	.base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
71	.base.cra_blocksize = SHA256_BLOCK_SIZE,
72	.base.cra_module = THIS_MODULE,
73	}, {
74	.digestsize = SHA224_DIGEST_SIZE,
75	.init = sha224_base_init,
76	.update = sha256_update,
77	.final = sha256_final,
78	.finup = sha256_finup,
79	.descsize = sizeof(struct sha256_state),
80	.base.cra_name = "sha224",
81	.base.cra_driver_name = "sha224-arm64",
82	.base.cra_priority = 100,
83	.base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
84	.base.cra_blocksize = SHA224_BLOCK_SIZE,
85	.base.cra_module = THIS_MODULE,
86	} };
87
88	static int sha256_update_neon(struct shash_desc desc, const u8 data,
89	unsigned int len)
90	{
91	/*
92	* Stacking and unstacking a substantial slice of the NEON register
93	* file may significantly affect performance for small updates when
94	* executing in interrupt context, so fall back to the scalar code
95	* in that case.
96	*/
97	if (!may_use_simd())
98	return sha256_base_do_update(desc, data, len,
99	(sha256_block_fn *)sha256_block_data_order);
100
101	kernel_neon_begin();
102	sha256_base_do_update(desc, data, len,
103	(sha256_block_fn *)sha256_block_neon);
104	kernel_neon_end();
105
106	return 0;
107	}
108
109	static int sha256_finup_neon(struct shash_desc desc, const u8 data,
110	unsigned int len, u8 *out)
111	{
112	if (!may_use_simd()) {
113	if (len)
114	sha256_base_do_update(desc, data, len,
115	(sha256_block_fn *)sha256_block_data_order);
116	sha256_base_do_finalize(desc,
117	(sha256_block_fn *)sha256_block_data_order);
118	} else {
119	kernel_neon_begin();
120	if (len)
121	sha256_base_do_update(desc, data, len,
122	(sha256_block_fn *)sha256_block_neon);
123	sha256_base_do_finalize(desc,
124	(sha256_block_fn *)sha256_block_neon);
125	kernel_neon_end();
126	}
127	return sha256_base_finish(desc, out);
128	}
129
130	static int sha256_final_neon(struct shash_desc desc, u8 out)
131	{
132	return sha256_finup_neon(desc, NULL, 0, out);
133	}
134
135	static struct shash_alg neon_algs[] = { {
136	.digestsize = SHA256_DIGEST_SIZE,
137	.init = sha256_base_init,
138	.update = sha256_update_neon,
139	.final = sha256_final_neon,
140	.finup = sha256_finup_neon,
141	.descsize = sizeof(struct sha256_state),
142	.base.cra_name = "sha256",
143	.base.cra_driver_name = "sha256-arm64-neon",
144	.base.cra_priority = 150,
145	.base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
146	.base.cra_blocksize = SHA256_BLOCK_SIZE,
147	.base.cra_module = THIS_MODULE,
148	}, {
149	.digestsize = SHA224_DIGEST_SIZE,
150	.init = sha224_base_init,
151	.update = sha256_update_neon,
152	.final = sha256_final_neon,
153	.finup = sha256_finup_neon,
154	.descsize = sizeof(struct sha256_state),
155	.base.cra_name = "sha224",
156	.base.cra_driver_name = "sha224-arm64-neon",
157	.base.cra_priority = 150,
158	.base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
159	.base.cra_blocksize = SHA224_BLOCK_SIZE,
160	.base.cra_module = THIS_MODULE,
161	} };
162
163	static int __init sha256_mod_init(void)
164	{
165	int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
166	if (ret)
167	return ret;
168
169	if (elf_hwcap & HWCAP_ASIMD) {
170	ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
171	if (ret)
172	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
173	}
174	return ret;
175	}
176
177	static void __exit sha256_mod_fini(void)
178	{
179	if (elf_hwcap & HWCAP_ASIMD)
180	crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
181	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
182	}
183
184	module_init(sha256_mod_init);
185	module_exit(sha256_mod_fini);