[mirror_ubuntu-hirsute-kernel.git] / arch / x86 / crypto / ghash-clmulni-intel_glue.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
 * instructions. This file contains glue code.
 *
 * Copyright (c) 2009 Intel Corp.
 *   Author: Huang Ying <ying.huang@intel.com>
 */

#include <linux/err.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/cryptd.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <asm/cpu_device_id.h>
#include <asm/simd.h>

#define GHASH_BLOCK_SIZE	16
#define GHASH_DIGEST_SIZE	16

void clmul_ghash_mul(char *dst, const u128 *shash);

void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
			const u128 *shash);

struct ghash_async_ctx {
	struct cryptd_ahash *cryptd_tfm;
};

struct ghash_ctx {
	u128 shash;
};

struct ghash_desc_ctx {
	u8 buffer[GHASH_BLOCK_SIZE];
	u32 bytes;
};

static int ghash_init(struct shash_desc *desc)
{
	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);

	memset(dctx, 0, sizeof(*dctx));

	return 0;
}

static int ghash_setkey(struct crypto_shash *tfm,
			const u8 *key, unsigned int keylen)
{
	struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
	be128 *x = (be128 *)key;
	u64 a, b;

	if (keylen != GHASH_BLOCK_SIZE) {
		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
		return -EINVAL;
	}

	/* perform multiplication by 'x' in GF(2^128) */
	a = be64_to_cpu(x->a);
	b = be64_to_cpu(x->b);

	ctx->shash.a = (b << 1) | (a >> 63);
	ctx->shash.b = (a << 1) | (b >> 63);

	if (a >> 63)
		ctx->shash.b ^= ((u64)0xc2) << 56;

	return 0;
}

static int ghash_update(struct shash_desc *desc,
			 const u8 *src, unsigned int srclen)
{
	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
	u8 *dst = dctx->buffer;

	kernel_fpu_begin();
	if (dctx->bytes) {
		int n = min(srclen, dctx->bytes);
		u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);

		dctx->bytes -= n;
		srclen -= n;

		while (n--)
			*pos++ ^= *src++;

		if (!dctx->bytes)
			clmul_ghash_mul(dst, &ctx->shash);
	}

	clmul_ghash_update(dst, src, srclen, &ctx->shash);
	kernel_fpu_end();

	if (srclen & 0xf) {
		src += srclen - (srclen & 0xf);
		srclen &= 0xf;
		dctx->bytes = GHASH_BLOCK_SIZE - srclen;
		while (srclen--)
			*dst++ ^= *src++;
	}

	return 0;
}

static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx)
{
	u8 *dst = dctx->buffer;

	if (dctx->bytes) {
		u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes);

		while (dctx->bytes--)
			*tmp++ ^= 0;

		kernel_fpu_begin();
		clmul_ghash_mul(dst, &ctx->shash);
		kernel_fpu_end();
	}

	dctx->bytes = 0;
}

static int ghash_final(struct shash_desc *desc, u8 *dst)
{
	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
	u8 *buf = dctx->buffer;

	ghash_flush(ctx, dctx);
	memcpy(dst, buf, GHASH_BLOCK_SIZE);

	return 0;
}

static struct shash_alg ghash_alg = {
	.digestsize	= GHASH_DIGEST_SIZE,
	.init		= ghash_init,
	.update		= ghash_update,
	.final		= ghash_final,
	.setkey		= ghash_setkey,
	.descsize	= sizeof(struct ghash_desc_ctx),
	.base		= {
		.cra_name		= "__ghash",
		.cra_driver_name	= "__ghash-pclmulqdqni",
		.cra_priority		= 0,
		.cra_flags		= CRYPTO_ALG_INTERNAL,
		.cra_blocksize		= GHASH_BLOCK_SIZE,
		.cra_ctxsize		= sizeof(struct ghash_ctx),
		.cra_module		= THIS_MODULE,
	},
};

static int ghash_async_init(struct ahash_request *req)
{
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
	struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);

	desc->tfm = child;
	return crypto_shash_init(desc);
}

static int ghash_async_update(struct ahash_request *req)
{
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

	if (!crypto_simd_usable() ||
	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
		memcpy(cryptd_req, req, sizeof(*req));
		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
		return crypto_ahash_update(cryptd_req);
	} else {
		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
		return shash_ahash_update(req, desc);
	}
}

static int ghash_async_final(struct ahash_request *req)
{
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

	if (!crypto_simd_usable() ||
	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
		memcpy(cryptd_req, req, sizeof(*req));
		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
		return crypto_ahash_final(cryptd_req);
	} else {
		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
		return crypto_shash_final(desc, req->result);
	}
}

static int ghash_async_import(struct ahash_request *req, const void *in)
{
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);

	ghash_async_init(req);
	memcpy(dctx, in, sizeof(*dctx));
	return 0;

}

static int ghash_async_export(struct ahash_request *req, void *out)
{
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);

	memcpy(out, dctx, sizeof(*dctx));
	return 0;

}

static int ghash_async_digest(struct ahash_request *req)
{
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

	if (!crypto_simd_usable() ||
	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
		memcpy(cryptd_req, req, sizeof(*req));
		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
		return crypto_ahash_digest(cryptd_req);
	} else {
		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);

		desc->tfm = child;
		return shash_ahash_digest(req, desc);
	}
}

static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct crypto_ahash *child = &ctx->cryptd_tfm->base;
	int err;

	crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
	crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
			       & CRYPTO_TFM_REQ_MASK);
	err = crypto_ahash_setkey(child, key, keylen);
	crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
			       & CRYPTO_TFM_RES_MASK);

	return err;
}

static int ghash_async_init_tfm(struct crypto_tfm *tfm)
{
	struct cryptd_ahash *cryptd_tfm;
	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);

	cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni",
					CRYPTO_ALG_INTERNAL,
					CRYPTO_ALG_INTERNAL);
	if (IS_ERR(cryptd_tfm))
		return PTR_ERR(cryptd_tfm);
	ctx->cryptd_tfm = cryptd_tfm;
	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
				 sizeof(struct ahash_request) +
				 crypto_ahash_reqsize(&cryptd_tfm->base));

	return 0;
}

static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
{
	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);

	cryptd_free_ahash(ctx->cryptd_tfm);
}

static struct ahash_alg ghash_async_alg = {
	.init		= ghash_async_init,
	.update		= ghash_async_update,
	.final		= ghash_async_final,
	.setkey		= ghash_async_setkey,
	.digest		= ghash_async_digest,
	.export		= ghash_async_export,
	.import		= ghash_async_import,
	.halg = {
		.digestsize	= GHASH_DIGEST_SIZE,
		.statesize = sizeof(struct ghash_desc_ctx),
		.base = {
			.cra_name		= "ghash",
			.cra_driver_name	= "ghash-clmulni",
			.cra_priority		= 400,
			.cra_ctxsize		= sizeof(struct ghash_async_ctx),
			.cra_flags		= CRYPTO_ALG_ASYNC,
			.cra_blocksize		= GHASH_BLOCK_SIZE,
			.cra_module		= THIS_MODULE,
			.cra_init		= ghash_async_init_tfm,
			.cra_exit		= ghash_async_exit_tfm,
		},
	},
};

static const struct x86_cpu_id pcmul_cpu_id[] = {
	X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), /* Pickle-Mickle-Duck */
	{}
};
MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);

static int __init ghash_pclmulqdqni_mod_init(void)
{
	int err;

	if (!x86_match_cpu(pcmul_cpu_id))
		return -ENODEV;

	err = crypto_register_shash(&ghash_alg);
	if (err)
		goto err_out;
	err = crypto_register_ahash(&ghash_async_alg);
	if (err)
		goto err_shash;

	return 0;

err_shash:
	crypto_unregister_shash(&ghash_alg);
err_out:
	return err;
}

static void __exit ghash_pclmulqdqni_mod_exit(void)
{
	crypto_unregister_ahash(&ghash_async_alg);
	crypto_unregister_shash(&ghash_alg);
}

module_init(ghash_pclmulqdqni_mod_init);
module_exit(ghash_pclmulqdqni_mod_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("GHASH Message Digest Algorithm, "
		   "accelerated by PCLMULQDQ-NI");
MODULE_ALIAS_CRYPTO("ghash");
Commit	Line	Data
d2912cb1	1	// SPDX-License-Identifier: GPL-2.0-only
0e1227d3 HY	2	/*
	3	* Accelerated GHASH implementation with Intel PCLMULQDQ-NI
	4	* instructions. This file contains glue code.
	5	*
	6	* Copyright (c) 2009 Intel Corp.
	7	* Author: Huang Ying <ying.huang@intel.com>
0e1227d3 HY	8	*/
0e1227d3 HY	9
52f6c5ad	10	#include <linux/err.h>
0e1227d3 HY	11	#include <linux/module.h>
	12	#include <linux/init.h>
	13	#include <linux/kernel.h>
	14	#include <linux/crypto.h>
	15	#include <crypto/algapi.h>
	16	#include <crypto/cryptd.h>
	17	#include <crypto/gf128mul.h>
	18	#include <crypto/internal/hash.h>
f2abe0d7	19	#include <crypto/internal/simd.h>
3bd391f0	20	#include <asm/cpu_device_id.h>
f2abe0d7	21	#include <asm/simd.h>
0e1227d3 HY	22
	23	#define GHASH_BLOCK_SIZE 16
	24	#define GHASH_DIGEST_SIZE 16
	25
0ea48146	26	void clmul_ghash_mul(char dst, const u128 shash);
0e1227d3 HY	27
0e1227d3 HY	28	void clmul_ghash_update(char dst, const char src, unsigned int srclen,
0ea48146	29	const u128 *shash);
0e1227d3	30
0e1227d3 HY	31	struct ghash_async_ctx {
	32	struct cryptd_ahash *cryptd_tfm;
	33	};
	34
	35	struct ghash_ctx {
0ea48146	36	u128 shash;
0e1227d3 HY	37	};
	38
	39	struct ghash_desc_ctx {
	40	u8 buffer[GHASH_BLOCK_SIZE];
	41	u32 bytes;
	42	};
	43
	44	static int ghash_init(struct shash_desc *desc)
	45	{
	46	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
	47
	48	memset(dctx, 0, sizeof(*dctx));
	49
	50	return 0;
	51	}
	52
	53	static int ghash_setkey(struct crypto_shash *tfm,
	54	const u8 *key, unsigned int keylen)
	55	{
	56	struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
8ceee728 AB	57	be128 x = (be128 )key;
8ceee728 AB	58	u64 a, b;
0e1227d3 HY	59
	60	if (keylen != GHASH_BLOCK_SIZE) {
	61	crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
	62	return -EINVAL;
	63	}
	64
8ceee728 AB	65	/* perform multiplication by 'x' in GF(2^128) */
	66	a = be64_to_cpu(x->a);
	67	b = be64_to_cpu(x->b);
	68
0ea48146 HX	69	ctx->shash.a = (b << 1) \| (a >> 63);
0ea48146 HX	70	ctx->shash.b = (a << 1) \| (b >> 63);
8ceee728 AB	71
8ceee728 AB	72	if (a >> 63)
0ea48146	73	ctx->shash.b ^= ((u64)0xc2) << 56;
0e1227d3 HY	74
	75	return 0;
	76	}
	77
	78	static int ghash_update(struct shash_desc *desc,
	79	const u8 *src, unsigned int srclen)
	80	{
	81	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
	82	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
	83	u8 *dst = dctx->buffer;
	84
	85	kernel_fpu_begin();
	86	if (dctx->bytes) {
	87	int n = min(srclen, dctx->bytes);
	88	u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
	89
	90	dctx->bytes -= n;
	91	srclen -= n;
	92
	93	while (n--)
	94	pos++ ^= src++;
	95
	96	if (!dctx->bytes)
	97	clmul_ghash_mul(dst, &ctx->shash);
	98	}
	99
	100	clmul_ghash_update(dst, src, srclen, &ctx->shash);
	101	kernel_fpu_end();
	102
	103	if (srclen & 0xf) {
	104	src += srclen - (srclen & 0xf);
	105	srclen &= 0xf;
	106	dctx->bytes = GHASH_BLOCK_SIZE - srclen;
	107	while (srclen--)
	108	dst++ ^= src++;
	109	}
	110
	111	return 0;
	112	}
	113
	114	static void ghash_flush(struct ghash_ctx ctx, struct ghash_desc_ctx dctx)
	115	{
	116	u8 *dst = dctx->buffer;
	117
	118	if (dctx->bytes) {
	119	u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
	120
	121	while (dctx->bytes--)
	122	*tmp++ ^= 0;
	123
	124	kernel_fpu_begin();
	125	clmul_ghash_mul(dst, &ctx->shash);
	126	kernel_fpu_end();
	127	}
	128
	129	dctx->bytes = 0;
	130	}
	131
	132	static int ghash_final(struct shash_desc desc, u8 dst)
	133	{
	134	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
	135	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
	136	u8 *buf = dctx->buffer;
	137
138	ghash_flush(ctx, dctx);
139	memcpy(dst, buf, GHASH_BLOCK_SIZE);
140
141	return 0;
142	}
143
144	static struct shash_alg ghash_alg = {
145	.digestsize = GHASH_DIGEST_SIZE,
146	.init = ghash_init,
147	.update = ghash_update,
148	.final = ghash_final,
149	.setkey = ghash_setkey,
150	.descsize = sizeof(struct ghash_desc_ctx),
151	.base = {
152	.cra_name = "__ghash",
153	.cra_driver_name = "__ghash-pclmulqdqni",
154	.cra_priority = 0,
e50944e2	155	.cra_flags = CRYPTO_ALG_INTERNAL,
0e1227d3 HY	156	.cra_blocksize = GHASH_BLOCK_SIZE,
	157	.cra_ctxsize = sizeof(struct ghash_ctx),
	158	.cra_module = THIS_MODULE,
0e1227d3 HY	159	},
	160	};
	161
	162	static int ghash_async_init(struct ahash_request *req)
	163	{
	164	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	165	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	166	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	167	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
7271b33c HX	168	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
7271b33c HX	169	struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
0e1227d3	170
7271b33c	171	desc->tfm = child;
7271b33c	172	return crypto_shash_init(desc);
0e1227d3 HY	173	}
	174
	175	static int ghash_async_update(struct ahash_request *req)
	176	{
	177	struct ahash_request *cryptd_req = ahash_request_ctx(req);
7271b33c HX	178	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	179	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	180	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
0e1227d3	181
f2abe0d7	182	if (!crypto_simd_usable() \|\|
7271b33c	183	(in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
0e1227d3 HY	184	memcpy(cryptd_req, req, sizeof(*req));
	185	ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
	186	return crypto_ahash_update(cryptd_req);
	187	} else {
	188	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
	189	return shash_ahash_update(req, desc);
	190	}
	191	}
	192
	193	static int ghash_async_final(struct ahash_request *req)
	194	{
	195	struct ahash_request *cryptd_req = ahash_request_ctx(req);
7271b33c HX	196	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	197	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	198	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
0e1227d3	199
f2abe0d7	200	if (!crypto_simd_usable() \|\|
7271b33c	201	(in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
0e1227d3 HY	202	memcpy(cryptd_req, req, sizeof(*req));
	203	ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
	204	return crypto_ahash_final(cryptd_req);
	205	} else {
	206	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
	207	return crypto_shash_final(desc, req->result);
	208	}
	209	}
	210
3a020a72 WR	211	static int ghash_async_import(struct ahash_request req, const void in)
	212	{
	213	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	214	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
	215	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
	216
	217	ghash_async_init(req);
	218	memcpy(dctx, in, sizeof(*dctx));
	219	return 0;
	220
	221	}
	222
	223	static int ghash_async_export(struct ahash_request req, void out)
	224	{
	225	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	226	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
	227	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
	228
	229	memcpy(out, dctx, sizeof(*dctx));
	230	return 0;
	231
	232	}
	233
0e1227d3 HY	234	static int ghash_async_digest(struct ahash_request *req)
	235	{
	236	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	237	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	238	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	239	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
	240
f2abe0d7	241	if (!crypto_simd_usable() \|\|
7271b33c	242	(in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
0e1227d3 HY	243	memcpy(cryptd_req, req, sizeof(*req));
	244	ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
	245	return crypto_ahash_digest(cryptd_req);
	246	} else {
	247	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
	248	struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
	249
	250	desc->tfm = child;
0e1227d3 HY	251	return shash_ahash_digest(req, desc);
	252	}
	253	}
	254
	255	static int ghash_async_setkey(struct crypto_ahash tfm, const u8 key,
	256	unsigned int keylen)
	257	{
	258	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	259	struct crypto_ahash *child = &ctx->cryptd_tfm->base;
	260	int err;
	261
	262	crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
	263	crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
	264	& CRYPTO_TFM_REQ_MASK);
	265	err = crypto_ahash_setkey(child, key, keylen);
	266	crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
	267	& CRYPTO_TFM_RES_MASK);
	268
c3e73e76	269	return err;
0e1227d3 HY	270	}
	271
	272	static int ghash_async_init_tfm(struct crypto_tfm *tfm)
	273	{
	274	struct cryptd_ahash *cryptd_tfm;
	275	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
	276
6a9b52b7 SM	277	cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni",
	278	CRYPTO_ALG_INTERNAL,
	279	CRYPTO_ALG_INTERNAL);
0e1227d3 HY	280	if (IS_ERR(cryptd_tfm))
	281	return PTR_ERR(cryptd_tfm);
	282	ctx->cryptd_tfm = cryptd_tfm;
	283	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
	284	sizeof(struct ahash_request) +
	285	crypto_ahash_reqsize(&cryptd_tfm->base));
	286
	287	return 0;
	288	}
	289
	290	static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
	291	{
	292	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
	293
	294	cryptd_free_ahash(ctx->cryptd_tfm);
	295	}
	296
	297	static struct ahash_alg ghash_async_alg = {
	298	.init = ghash_async_init,
	299	.update = ghash_async_update,
	300	.final = ghash_async_final,
	301	.setkey = ghash_async_setkey,
	302	.digest = ghash_async_digest,
3a020a72 WR	303	.export = ghash_async_export,
3a020a72 WR	304	.import = ghash_async_import,
0e1227d3 HY	305	.halg = {
0e1227d3 HY	306	.digestsize = GHASH_DIGEST_SIZE,
3a020a72	307	.statesize = sizeof(struct ghash_desc_ctx),
0e1227d3 HY	308	.base = {
	309	.cra_name = "ghash",
	310	.cra_driver_name = "ghash-clmulni",
	311	.cra_priority = 400,
71c6da84	312	.cra_ctxsize = sizeof(struct ghash_async_ctx),
6a38f622	313	.cra_flags = CRYPTO_ALG_ASYNC,
0e1227d3	314	.cra_blocksize = GHASH_BLOCK_SIZE,
0e1227d3	315	.cra_module = THIS_MODULE,
0e1227d3 HY	316	.cra_init = ghash_async_init_tfm,
	317	.cra_exit = ghash_async_exit_tfm,
	318	},
	319	},
	320	};
	321
3bd391f0 AK	322	static const struct x86_cpu_id pcmul_cpu_id[] = {
	323	X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), /* Pickle-Mickle-Duck */
	324	{}
	325	};
	326	MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
	327
0e1227d3 HY	328	static int __init ghash_pclmulqdqni_mod_init(void)
	329	{
	330	int err;
	331
3bd391f0	332	if (!x86_match_cpu(pcmul_cpu_id))
0e1227d3	333	return -ENODEV;
0e1227d3 HY	334
	335	err = crypto_register_shash(&ghash_alg);
	336	if (err)
	337	goto err_out;
	338	err = crypto_register_ahash(&ghash_async_alg);
	339	if (err)
	340	goto err_shash;
	341
	342	return 0;
	343
	344	err_shash:
	345	crypto_unregister_shash(&ghash_alg);
	346	err_out:
	347	return err;
	348	}
	349
	350	static void __exit ghash_pclmulqdqni_mod_exit(void)
	351	{
	352	crypto_unregister_ahash(&ghash_async_alg);
	353	crypto_unregister_shash(&ghash_alg);
	354	}
	355
	356	module_init(ghash_pclmulqdqni_mod_init);
	357	module_exit(ghash_pclmulqdqni_mod_exit);
	358
	359	MODULE_LICENSE("GPL");
	360	MODULE_DESCRIPTION("GHASH Message Digest Algorithm, "
158b52ff	361	"accelerated by PCLMULQDQ-NI");
5d26a105	362	MODULE_ALIAS_CRYPTO("ghash");