[mirror_ubuntu-bionic-kernel.git] / arch / arm64 / crypto / crc32-ce-glue.c

/*
 * Accelerated CRC32(C) using arm64 NEON and Crypto Extensions instructions
 *
 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/cpufeature.h>
#include <linux/crc32.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>

#include <crypto/internal/hash.h>

#include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h>
#include <asm/unaligned.h>

#define PMULL_MIN_LEN		64L	/* minimum size of buffer
					 * for crc32_pmull_le_16 */
#define SCALE_F			16L	/* size of NEON register */

asmlinkage u32 crc32_pmull_le(const u8 buf[], u64 len, u32 init_crc);
asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], size_t len);

asmlinkage u32 crc32c_pmull_le(const u8 buf[], u64 len, u32 init_crc);
asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], size_t len);

static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], size_t len);
static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], size_t len);

static int crc32_pmull_cra_init(struct crypto_tfm *tfm)
{
	u32 *key = crypto_tfm_ctx(tfm);

	*key = 0;
	return 0;
}

static int crc32c_pmull_cra_init(struct crypto_tfm *tfm)
{
	u32 *key = crypto_tfm_ctx(tfm);

	*key = ~0;
	return 0;
}

static int crc32_pmull_setkey(struct crypto_shash *hash, const u8 *key,
			      unsigned int keylen)
{
	u32 *mctx = crypto_shash_ctx(hash);

	if (keylen != sizeof(u32)) {
		crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
		return -EINVAL;
	}
	*mctx = le32_to_cpup((__le32 *)key);
	return 0;
}

static int crc32_pmull_init(struct shash_desc *desc)
{
	u32 *mctx = crypto_shash_ctx(desc->tfm);
	u32 *crc = shash_desc_ctx(desc);

	*crc = *mctx;
	return 0;
}

static int crc32_update(struct shash_desc *desc, const u8 *data,
			unsigned int length)
{
	u32 *crc = shash_desc_ctx(desc);

	*crc = crc32_armv8_le(*crc, data, length);
	return 0;
}

static int crc32c_update(struct shash_desc *desc, const u8 *data,
			 unsigned int length)
{
	u32 *crc = shash_desc_ctx(desc);

	*crc = crc32c_armv8_le(*crc, data, length);
	return 0;
}

static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
			 unsigned int length)
{
	u32 *crc = shash_desc_ctx(desc);
	unsigned int l;

	if ((u64)data % SCALE_F) {
		l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));

		*crc = fallback_crc32(*crc, data, l);

		data += l;
		length -= l;
	}

	if (length >= PMULL_MIN_LEN && may_use_simd()) {
		l = round_down(length, SCALE_F);

		kernel_neon_begin();
		*crc = crc32_pmull_le(data, l, *crc);
		kernel_neon_end();

		data += l;
		length -= l;
	}

	if (length > 0)
		*crc = fallback_crc32(*crc, data, length);

	return 0;
}

static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
			 unsigned int length)
{
	u32 *crc = shash_desc_ctx(desc);
	unsigned int l;

	if ((u64)data % SCALE_F) {
		l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));

		*crc = fallback_crc32c(*crc, data, l);

		data += l;
		length -= l;
	}

	if (length >= PMULL_MIN_LEN && may_use_simd()) {
		l = round_down(length, SCALE_F);

		kernel_neon_begin();
		*crc = crc32c_pmull_le(data, l, *crc);
		kernel_neon_end();

		data += l;
		length -= l;
	}

	if (length > 0) {
		*crc = fallback_crc32c(*crc, data, length);
	}

	return 0;
}

static int crc32_pmull_final(struct shash_desc *desc, u8 *out)
{
	u32 *crc = shash_desc_ctx(desc);

	put_unaligned_le32(*crc, out);
	return 0;
}

static int crc32c_pmull_final(struct shash_desc *desc, u8 *out)
{
	u32 *crc = shash_desc_ctx(desc);

	put_unaligned_le32(~*crc, out);
	return 0;
}

static struct shash_alg crc32_pmull_algs[] = { {
	.setkey			= crc32_pmull_setkey,
	.init			= crc32_pmull_init,
	.update			= crc32_update,
	.final			= crc32_pmull_final,
	.descsize		= sizeof(u32),
	.digestsize		= sizeof(u32),

	.base.cra_ctxsize	= sizeof(u32),
	.base.cra_init		= crc32_pmull_cra_init,
	.base.cra_name		= "crc32",
	.base.cra_driver_name	= "crc32-arm64-ce",
	.base.cra_priority	= 200,
	.base.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
	.base.cra_blocksize	= 1,
	.base.cra_module	= THIS_MODULE,
}, {
	.setkey			= crc32_pmull_setkey,
	.init			= crc32_pmull_init,
	.update			= crc32c_update,
	.final			= crc32c_pmull_final,
	.descsize		= sizeof(u32),
	.digestsize		= sizeof(u32),

	.base.cra_ctxsize	= sizeof(u32),
	.base.cra_init		= crc32c_pmull_cra_init,
	.base.cra_name		= "crc32c",
	.base.cra_driver_name	= "crc32c-arm64-ce",
	.base.cra_priority	= 200,
	.base.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
	.base.cra_blocksize	= 1,
	.base.cra_module	= THIS_MODULE,
} };

static int __init crc32_pmull_mod_init(void)
{
	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_PMULL)) {
		crc32_pmull_algs[0].update = crc32_pmull_update;
		crc32_pmull_algs[1].update = crc32c_pmull_update;

		if (elf_hwcap & HWCAP_CRC32) {
			fallback_crc32 = crc32_armv8_le;
			fallback_crc32c = crc32c_armv8_le;
		} else {
			fallback_crc32 = crc32_le;
			fallback_crc32c = __crc32c_le;
		}
	} else if (!(elf_hwcap & HWCAP_CRC32)) {
		return -ENODEV;
	}
	return crypto_register_shashes(crc32_pmull_algs,
				       ARRAY_SIZE(crc32_pmull_algs));
}

static void __exit crc32_pmull_mod_exit(void)
{
	crypto_unregister_shashes(crc32_pmull_algs,
				  ARRAY_SIZE(crc32_pmull_algs));
}

static const struct cpu_feature crc32_cpu_feature[] = {
	{ cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
};
MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);

module_init(crc32_pmull_mod_init);
module_exit(crc32_pmull_mod_exit);

MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
8fefde90 AB	1	/*
	2	* Accelerated CRC32(C) using arm64 NEON and Crypto Extensions instructions
	3	*
15c7d8f8	4	* Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
8fefde90 AB	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*/
	10
	11	#include <linux/cpufeature.h>
	12	#include <linux/crc32.h>
	13	#include <linux/init.h>
	14	#include <linux/kernel.h>
	15	#include <linux/module.h>
	16	#include <linux/string.h>
	17
	18	#include <crypto/internal/hash.h>
	19
	20	#include <asm/hwcap.h>
	21	#include <asm/neon.h>
15c7d8f8	22	#include <asm/simd.h>
8fefde90 AB	23	#include <asm/unaligned.h>
	24
	25	#define PMULL_MIN_LEN 64L /* minimum size of buffer
	26	* for crc32_pmull_le_16 */
	27	#define SCALE_F 16L /* size of NEON register */
	28
	29	asmlinkage u32 crc32_pmull_le(const u8 buf[], u64 len, u32 init_crc);
	30	asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], size_t len);
	31
	32	asmlinkage u32 crc32c_pmull_le(const u8 buf[], u64 len, u32 init_crc);
	33	asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], size_t len);
	34
	35	static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], size_t len);
	36	static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], size_t len);
	37
	38	static int crc32_pmull_cra_init(struct crypto_tfm *tfm)
	39	{
	40	u32 *key = crypto_tfm_ctx(tfm);
	41
	42	*key = 0;
	43	return 0;
	44	}
	45
	46	static int crc32c_pmull_cra_init(struct crypto_tfm *tfm)
	47	{
	48	u32 *key = crypto_tfm_ctx(tfm);
	49
	50	*key = ~0;
	51	return 0;
	52	}
	53
	54	static int crc32_pmull_setkey(struct crypto_shash hash, const u8 key,
	55	unsigned int keylen)
	56	{
	57	u32 *mctx = crypto_shash_ctx(hash);
	58
	59	if (keylen != sizeof(u32)) {
	60	crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
	61	return -EINVAL;
	62	}
	63	mctx = le32_to_cpup((__le32 )key);
	64	return 0;
	65	}
	66
	67	static int crc32_pmull_init(struct shash_desc *desc)
	68	{
	69	u32 *mctx = crypto_shash_ctx(desc->tfm);
	70	u32 *crc = shash_desc_ctx(desc);
	71
	72	crc = mctx;
	73	return 0;
	74	}
	75
5d3d9c8b AB	76	static int crc32_update(struct shash_desc desc, const u8 data,
	77	unsigned int length)
	78	{
	79	u32 *crc = shash_desc_ctx(desc);
	80
	81	crc = crc32_armv8_le(crc, data, length);
	82	return 0;
	83	}
	84
	85	static int crc32c_update(struct shash_desc desc, const u8 data,
	86	unsigned int length)
	87	{
	88	u32 *crc = shash_desc_ctx(desc);
	89
	90	crc = crc32c_armv8_le(crc, data, length);
	91	return 0;
	92	}
	93
8fefde90 AB	94	static int crc32_pmull_update(struct shash_desc desc, const u8 data,
	95	unsigned int length)
	96	{
	97	u32 *crc = shash_desc_ctx(desc);
	98	unsigned int l;
	99
	100	if ((u64)data % SCALE_F) {
	101	l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
	102
	103	crc = fallback_crc32(crc, data, l);
	104
	105	data += l;
	106	length -= l;
	107	}
	108
15c7d8f8	109	if (length >= PMULL_MIN_LEN && may_use_simd()) {
8fefde90 AB	110	l = round_down(length, SCALE_F);
8fefde90 AB	111
15c7d8f8	112	kernel_neon_begin();
8fefde90 AB	113	crc = crc32_pmull_le(data, l, crc);
	114	kernel_neon_end();
	115
	116	data += l;
	117	length -= l;
	118	}
	119
	120	if (length > 0)
	121	crc = fallback_crc32(crc, data, length);
	122
	123	return 0;
	124	}
	125
	126	static int crc32c_pmull_update(struct shash_desc desc, const u8 data,
	127	unsigned int length)
	128	{
	129	u32 *crc = shash_desc_ctx(desc);
	130	unsigned int l;
	131
	132	if ((u64)data % SCALE_F) {
	133	l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
	134
	135	crc = fallback_crc32c(crc, data, l);
	136
	137	data += l;
	138	length -= l;
	139	}
	140
15c7d8f8	141	if (length >= PMULL_MIN_LEN && may_use_simd()) {
8fefde90 AB	142	l = round_down(length, SCALE_F);
8fefde90 AB	143
15c7d8f8	144	kernel_neon_begin();
8fefde90 AB	145	crc = crc32c_pmull_le(data, l, crc);
	146	kernel_neon_end();
	147
	148	data += l;
	149	length -= l;
	150	}
	151
	152	if (length > 0) {
	153	crc = fallback_crc32c(crc, data, length);
	154	}
	155
	156	return 0;
	157	}
	158
	159	static int crc32_pmull_final(struct shash_desc desc, u8 out)
	160	{
	161	u32 *crc = shash_desc_ctx(desc);
	162
	163	put_unaligned_le32(*crc, out);
	164	return 0;
	165	}
	166
	167	static int crc32c_pmull_final(struct shash_desc desc, u8 out)
	168	{
	169	u32 *crc = shash_desc_ctx(desc);
	170
	171	put_unaligned_le32(~*crc, out);
	172	return 0;
	173	}
	174
	175	static struct shash_alg crc32_pmull_algs[] = { {
	176	.setkey = crc32_pmull_setkey,
	177	.init = crc32_pmull_init,
5d3d9c8b	178	.update = crc32_update,
8fefde90 AB	179	.final = crc32_pmull_final,
	180	.descsize = sizeof(u32),
	181	.digestsize = sizeof(u32),
	182
	183	.base.cra_ctxsize = sizeof(u32),
	184	.base.cra_init = crc32_pmull_cra_init,
	185	.base.cra_name = "crc32",
	186	.base.cra_driver_name = "crc32-arm64-ce",
	187	.base.cra_priority = 200,
b5c690bf	188	.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
8fefde90 AB	189	.base.cra_blocksize = 1,
	190	.base.cra_module = THIS_MODULE,
	191	}, {
	192	.setkey = crc32_pmull_setkey,
	193	.init = crc32_pmull_init,
5d3d9c8b	194	.update = crc32c_update,
8fefde90 AB	195	.final = crc32c_pmull_final,
	196	.descsize = sizeof(u32),
	197	.digestsize = sizeof(u32),
	198
	199	.base.cra_ctxsize = sizeof(u32),
	200	.base.cra_init = crc32c_pmull_cra_init,
	201	.base.cra_name = "crc32c",
	202	.base.cra_driver_name = "crc32c-arm64-ce",
	203	.base.cra_priority = 200,
b5c690bf	204	.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
8fefde90 AB	205	.base.cra_blocksize = 1,
	206	.base.cra_module = THIS_MODULE,
	207	} };
	208
	209	static int __init crc32_pmull_mod_init(void)
	210	{
5d3d9c8b AB	211	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_PMULL)) {
	212	crc32_pmull_algs[0].update = crc32_pmull_update;
	213	crc32_pmull_algs[1].update = crc32c_pmull_update;
	214
	215	if (elf_hwcap & HWCAP_CRC32) {
	216	fallback_crc32 = crc32_armv8_le;
	217	fallback_crc32c = crc32c_armv8_le;
	218	} else {
	219	fallback_crc32 = crc32_le;
	220	fallback_crc32c = __crc32c_le;
	221	}
	222	} else if (!(elf_hwcap & HWCAP_CRC32)) {
	223	return -ENODEV;
8fefde90	224	}
8fefde90 AB	225	return crypto_register_shashes(crc32_pmull_algs,
	226	ARRAY_SIZE(crc32_pmull_algs));
	227	}
	228
	229	static void __exit crc32_pmull_mod_exit(void)
	230	{
	231	crypto_unregister_shashes(crc32_pmull_algs,
	232	ARRAY_SIZE(crc32_pmull_algs));
	233	}
	234
5d3d9c8b AB	235	static const struct cpu_feature crc32_cpu_feature[] = {
	236	{ cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
	237	};
	238	MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
	239
	240	module_init(crc32_pmull_mod_init);
8fefde90 AB	241	module_exit(crc32_pmull_mod_exit);
	242
	243	MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
	244	MODULE_LICENSE("GPL v2");