[mirror_ubuntu-bionic-kernel.git] / arch / x86 / crypto / poly1305_glue.c

/*
 * Poly1305 authenticator algorithm, RFC7539, SIMD glue code
 *
 * Copyright (C) 2015 Martin Willi
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <crypto/algapi.h>
#include <crypto/internal/hash.h>
#include <crypto/poly1305.h>
#include <linux/crypto.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/simd.h>

struct poly1305_simd_desc_ctx {
	struct poly1305_desc_ctx base;
	/* derived key u set? */
	bool uset;
#ifdef CONFIG_AS_AVX2
	/* derived keys r^3, r^4 set? */
	bool wset;
#endif
	/* derived Poly1305 key r^2 */
	u32 u[5];
	/* ... silently appended r^3 and r^4 when using AVX2 */
};

asmlinkage void poly1305_block_sse2(u32 *h, const u8 *src,
				    const u32 *r, unsigned int blocks);
asmlinkage void poly1305_2block_sse2(u32 *h, const u8 *src, const u32 *r,
				     unsigned int blocks, const u32 *u);
#ifdef CONFIG_AS_AVX2
asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r,
				     unsigned int blocks, const u32 *u);
static bool poly1305_use_avx2;
#endif

static int poly1305_simd_init(struct shash_desc *desc)
{
	struct poly1305_simd_desc_ctx *sctx = shash_desc_ctx(desc);

	sctx->uset = false;
#ifdef CONFIG_AS_AVX2
	sctx->wset = false;
#endif

	return crypto_poly1305_init(desc);
}

static void poly1305_simd_mult(u32 *a, const u32 *b)
{
	u8 m[POLY1305_BLOCK_SIZE];

	memset(m, 0, sizeof(m));
	/* The poly1305 block function adds a hi-bit to the accumulator which
	 * we don't need for key multiplication; compensate for it. */
	a[4] -= 1 << 24;
	poly1305_block_sse2(a, m, b, 1);
}

static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
					 const u8 *src, unsigned int srclen)
{
	struct poly1305_simd_desc_ctx *sctx;
	unsigned int blocks, datalen;

	BUILD_BUG_ON(offsetof(struct poly1305_simd_desc_ctx, base));
	sctx = container_of(dctx, struct poly1305_simd_desc_ctx, base);

	if (unlikely(!dctx->sset)) {
		datalen = crypto_poly1305_setdesckey(dctx, src, srclen);
		src += srclen - datalen;
		srclen = datalen;
	}

#ifdef CONFIG_AS_AVX2
	if (poly1305_use_avx2 && srclen >= POLY1305_BLOCK_SIZE * 4) {
		if (unlikely(!sctx->wset)) {
			if (!sctx->uset) {
				memcpy(sctx->u, dctx->r, sizeof(sctx->u));
				poly1305_simd_mult(sctx->u, dctx->r);
				sctx->uset = true;
			}
			memcpy(sctx->u + 5, sctx->u, sizeof(sctx->u));
			poly1305_simd_mult(sctx->u + 5, dctx->r);
			memcpy(sctx->u + 10, sctx->u + 5, sizeof(sctx->u));
			poly1305_simd_mult(sctx->u + 10, dctx->r);
			sctx->wset = true;
		}
		blocks = srclen / (POLY1305_BLOCK_SIZE * 4);
		poly1305_4block_avx2(dctx->h, src, dctx->r, blocks, sctx->u);
		src += POLY1305_BLOCK_SIZE * 4 * blocks;
		srclen -= POLY1305_BLOCK_SIZE * 4 * blocks;
	}
#endif
	if (likely(srclen >= POLY1305_BLOCK_SIZE * 2)) {
		if (unlikely(!sctx->uset)) {
			memcpy(sctx->u, dctx->r, sizeof(sctx->u));
			poly1305_simd_mult(sctx->u, dctx->r);
			sctx->uset = true;
		}
		blocks = srclen / (POLY1305_BLOCK_SIZE * 2);
		poly1305_2block_sse2(dctx->h, src, dctx->r, blocks, sctx->u);
		src += POLY1305_BLOCK_SIZE * 2 * blocks;
		srclen -= POLY1305_BLOCK_SIZE * 2 * blocks;
	}
	if (srclen >= POLY1305_BLOCK_SIZE) {
		poly1305_block_sse2(dctx->h, src, dctx->r, 1);
		srclen -= POLY1305_BLOCK_SIZE;
	}
	return srclen;
}

static int poly1305_simd_update(struct shash_desc *desc,
				const u8 *src, unsigned int srclen)
{
	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
	unsigned int bytes;

	/* kernel_fpu_begin/end is costly, use fallback for small updates */
	if (srclen <= 288 || !may_use_simd())
		return crypto_poly1305_update(desc, src, srclen);

	kernel_fpu_begin();

	if (unlikely(dctx->buflen)) {
		bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
		memcpy(dctx->buf + dctx->buflen, src, bytes);
		src += bytes;
		srclen -= bytes;
		dctx->buflen += bytes;

		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
			poly1305_simd_blocks(dctx, dctx->buf,
					     POLY1305_BLOCK_SIZE);
			dctx->buflen = 0;
		}
	}

	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
		bytes = poly1305_simd_blocks(dctx, src, srclen);
		src += srclen - bytes;
		srclen = bytes;
	}

	kernel_fpu_end();

	if (unlikely(srclen)) {
		dctx->buflen = srclen;
		memcpy(dctx->buf, src, srclen);
	}

	return 0;
}

static struct shash_alg alg = {
	.digestsize	= POLY1305_DIGEST_SIZE,
	.init		= poly1305_simd_init,
	.update		= poly1305_simd_update,
	.final		= crypto_poly1305_final,
	.setkey		= crypto_poly1305_setkey,
	.descsize	= sizeof(struct poly1305_simd_desc_ctx),
	.base		= {
		.cra_name		= "poly1305",
		.cra_driver_name	= "poly1305-simd",
		.cra_priority		= 300,
		.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
		.cra_alignmask		= sizeof(u32) - 1,
		.cra_blocksize		= POLY1305_BLOCK_SIZE,
		.cra_module		= THIS_MODULE,
	},
};

static int __init poly1305_simd_mod_init(void)
{
	if (!cpu_has_xmm2)
		return -ENODEV;

#ifdef CONFIG_AS_AVX2
	poly1305_use_avx2 = cpu_has_avx && cpu_has_avx2 &&
			    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
	alg.descsize = sizeof(struct poly1305_simd_desc_ctx);
	if (poly1305_use_avx2)
		alg.descsize += 10 * sizeof(u32);
#endif
	return crypto_register_shash(&alg);
}

static void __exit poly1305_simd_mod_exit(void)
{
	crypto_unregister_shash(&alg);
}

module_init(poly1305_simd_mod_init);
module_exit(poly1305_simd_mod_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
MODULE_DESCRIPTION("Poly1305 authenticator");
MODULE_ALIAS_CRYPTO("poly1305");
MODULE_ALIAS_CRYPTO("poly1305-simd");
Commit	Line	Data
c70f4abe MW	1	/*
	2	* Poly1305 authenticator algorithm, RFC7539, SIMD glue code
	3	*
	4	* Copyright (C) 2015 Martin Willi
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License as published by
	8	* the Free Software Foundation; either version 2 of the License, or
	9	* (at your option) any later version.
	10	*/
	11
	12	#include <crypto/algapi.h>
	13	#include <crypto/internal/hash.h>
	14	#include <crypto/poly1305.h>
	15	#include <linux/crypto.h>
	16	#include <linux/kernel.h>
	17	#include <linux/module.h>
	18	#include <asm/fpu/api.h>
	19	#include <asm/simd.h>
	20
da35b22d MW	21	struct poly1305_simd_desc_ctx {
	22	struct poly1305_desc_ctx base;
	23	/* derived key u set? */
	24	bool uset;
b1ccc8f4 MW	25	#ifdef CONFIG_AS_AVX2
	26	/* derived keys r^3, r^4 set? */
	27	bool wset;
	28	#endif
da35b22d MW	29	/* derived Poly1305 key r^2 */
da35b22d MW	30	u32 u[5];
b1ccc8f4	31	/* ... silently appended r^3 and r^4 when using AVX2 */
da35b22d MW	32	};
da35b22d MW	33
c70f4abe MW	34	asmlinkage void poly1305_block_sse2(u32 h, const u8 src,
c70f4abe MW	35	const u32 *r, unsigned int blocks);
da35b22d MW	36	asmlinkage void poly1305_2block_sse2(u32 h, const u8 src, const u32 *r,
da35b22d MW	37	unsigned int blocks, const u32 *u);
b1ccc8f4 MW	38	#ifdef CONFIG_AS_AVX2
	39	asmlinkage void poly1305_4block_avx2(u32 h, const u8 src, const u32 *r,
	40	unsigned int blocks, const u32 *u);
	41	static bool poly1305_use_avx2;
	42	#endif
da35b22d MW	43
	44	static int poly1305_simd_init(struct shash_desc *desc)
	45	{
	46	struct poly1305_simd_desc_ctx *sctx = shash_desc_ctx(desc);
	47
	48	sctx->uset = false;
b1ccc8f4 MW	49	#ifdef CONFIG_AS_AVX2
	50	sctx->wset = false;
	51	#endif
da35b22d MW	52
	53	return crypto_poly1305_init(desc);
	54	}
	55
	56	static void poly1305_simd_mult(u32 a, const u32 b)
	57	{
	58	u8 m[POLY1305_BLOCK_SIZE];
	59
	60	memset(m, 0, sizeof(m));
	61	/* The poly1305 block function adds a hi-bit to the accumulator which
	62	* we don't need for key multiplication; compensate for it. */
	63	a[4] -= 1 << 24;
	64	poly1305_block_sse2(a, m, b, 1);
	65	}
c70f4abe MW	66
	67	static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
	68	const u8 *src, unsigned int srclen)
	69	{
da35b22d	70	struct poly1305_simd_desc_ctx *sctx;
c70f4abe MW	71	unsigned int blocks, datalen;
c70f4abe MW	72
da35b22d MW	73	BUILD_BUG_ON(offsetof(struct poly1305_simd_desc_ctx, base));
	74	sctx = container_of(dctx, struct poly1305_simd_desc_ctx, base);
	75
c70f4abe MW	76	if (unlikely(!dctx->sset)) {
	77	datalen = crypto_poly1305_setdesckey(dctx, src, srclen);
	78	src += srclen - datalen;
	79	srclen = datalen;
	80	}
	81
b1ccc8f4 MW	82	#ifdef CONFIG_AS_AVX2
	83	if (poly1305_use_avx2 && srclen >= POLY1305_BLOCK_SIZE * 4) {
	84	if (unlikely(!sctx->wset)) {
	85	if (!sctx->uset) {
	86	memcpy(sctx->u, dctx->r, sizeof(sctx->u));
	87	poly1305_simd_mult(sctx->u, dctx->r);
	88	sctx->uset = true;
	89	}
	90	memcpy(sctx->u + 5, sctx->u, sizeof(sctx->u));
	91	poly1305_simd_mult(sctx->u + 5, dctx->r);
	92	memcpy(sctx->u + 10, sctx->u + 5, sizeof(sctx->u));
	93	poly1305_simd_mult(sctx->u + 10, dctx->r);
	94	sctx->wset = true;
	95	}
	96	blocks = srclen / (POLY1305_BLOCK_SIZE * 4);
	97	poly1305_4block_avx2(dctx->h, src, dctx->r, blocks, sctx->u);
	98	src += POLY1305_BLOCK_SIZE * 4 * blocks;
	99	srclen -= POLY1305_BLOCK_SIZE * 4 * blocks;
	100	}
	101	#endif
da35b22d MW	102	if (likely(srclen >= POLY1305_BLOCK_SIZE * 2)) {
	103	if (unlikely(!sctx->uset)) {
	104	memcpy(sctx->u, dctx->r, sizeof(sctx->u));
	105	poly1305_simd_mult(sctx->u, dctx->r);
	106	sctx->uset = true;
	107	}
	108	blocks = srclen / (POLY1305_BLOCK_SIZE * 2);
	109	poly1305_2block_sse2(dctx->h, src, dctx->r, blocks, sctx->u);
	110	src += POLY1305_BLOCK_SIZE * 2 * blocks;
	111	srclen -= POLY1305_BLOCK_SIZE * 2 * blocks;
	112	}
c70f4abe	113	if (srclen >= POLY1305_BLOCK_SIZE) {
da35b22d MW	114	poly1305_block_sse2(dctx->h, src, dctx->r, 1);
da35b22d MW	115	srclen -= POLY1305_BLOCK_SIZE;
c70f4abe MW	116	}
	117	return srclen;
	118	}
	119
	120	static int poly1305_simd_update(struct shash_desc *desc,
	121	const u8 *src, unsigned int srclen)
	122	{
	123	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
	124	unsigned int bytes;
	125
	126	/* kernel_fpu_begin/end is costly, use fallback for small updates */
	127	if (srclen <= 288 \|\| !may_use_simd())
	128	return crypto_poly1305_update(desc, src, srclen);
	129
	130	kernel_fpu_begin();
	131
	132	if (unlikely(dctx->buflen)) {
	133	bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
	134	memcpy(dctx->buf + dctx->buflen, src, bytes);
	135	src += bytes;
	136	srclen -= bytes;
	137	dctx->buflen += bytes;
	138
	139	if (dctx->buflen == POLY1305_BLOCK_SIZE) {
	140	poly1305_simd_blocks(dctx, dctx->buf,
	141	POLY1305_BLOCK_SIZE);
	142	dctx->buflen = 0;
	143	}
	144	}
	145
	146	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
	147	bytes = poly1305_simd_blocks(dctx, src, srclen);
	148	src += srclen - bytes;
	149	srclen = bytes;
	150	}
	151
	152	kernel_fpu_end();
	153
	154	if (unlikely(srclen)) {
	155	dctx->buflen = srclen;
	156	memcpy(dctx->buf, src, srclen);
	157	}
	158
	159	return 0;
	160	}
	161
	162	static struct shash_alg alg = {
	163	.digestsize = POLY1305_DIGEST_SIZE,
da35b22d	164	.init = poly1305_simd_init,
c70f4abe MW	165	.update = poly1305_simd_update,
	166	.final = crypto_poly1305_final,
	167	.setkey = crypto_poly1305_setkey,
da35b22d	168	.descsize = sizeof(struct poly1305_simd_desc_ctx),
c70f4abe MW	169	.base = {
	170	.cra_name = "poly1305",
	171	.cra_driver_name = "poly1305-simd",
	172	.cra_priority = 300,
	173	.cra_flags = CRYPTO_ALG_TYPE_SHASH,
	174	.cra_alignmask = sizeof(u32) - 1,
	175	.cra_blocksize = POLY1305_BLOCK_SIZE,
	176	.cra_module = THIS_MODULE,
	177	},
	178	};
	179
	180	static int __init poly1305_simd_mod_init(void)
	181	{
	182	if (!cpu_has_xmm2)
	183	return -ENODEV;
	184
b1ccc8f4 MW	185	#ifdef CONFIG_AS_AVX2
b1ccc8f4 MW	186	poly1305_use_avx2 = cpu_has_avx && cpu_has_avx2 &&
d91cab78	187	cpu_has_xfeatures(XFEATURE_MASK_SSE \| XFEATURE_MASK_YMM, NULL);
b1ccc8f4 MW	188	alg.descsize = sizeof(struct poly1305_simd_desc_ctx);
	189	if (poly1305_use_avx2)
	190	alg.descsize += 10 * sizeof(u32);
	191	#endif
c70f4abe MW	192	return crypto_register_shash(&alg);
	193	}
	194
	195	static void __exit poly1305_simd_mod_exit(void)
	196	{
	197	crypto_unregister_shash(&alg);
	198	}
	199
	200	module_init(poly1305_simd_mod_init);
	201	module_exit(poly1305_simd_mod_exit);
	202
	203	MODULE_LICENSE("GPL");
	204	MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
	205	MODULE_DESCRIPTION("Poly1305 authenticator");
	206	MODULE_ALIAS_CRYPTO("poly1305");
	207	MODULE_ALIAS_CRYPTO("poly1305-simd");