[mirror_ubuntu-artful-kernel.git] / crypto / gcm.c

/*
 * GCM: Galois/Counter Mode.
 *
 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 */

#include <crypto/gf128mul.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/completion.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>

struct gcm_instance_ctx {
	struct crypto_skcipher_spawn ctr;
};

struct crypto_gcm_ctx {
	struct crypto_ablkcipher *ctr;
	struct gf128mul_4k *gf128;
};

struct crypto_gcm_ghash_ctx {
	u32 bytes;
	u32 flags;
	struct gf128mul_4k *gf128;
	u8 buffer[16];
};

struct crypto_gcm_req_priv_ctx {
	u8 auth_tag[16];
	u8 iauth_tag[16];
	struct scatterlist src[2];
	struct scatterlist dst[2];
	struct crypto_gcm_ghash_ctx ghash;
	struct ablkcipher_request abreq;
};

struct crypto_gcm_setkey_result {
	int err;
	struct completion completion;
};

static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
	struct aead_request *req)
{
	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));

	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
}

static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags,
				  struct gf128mul_4k *gf128)
{
	ctx->bytes = 0;
	ctx->flags = flags;
	ctx->gf128 = gf128;
	memset(ctx->buffer, 0, 16);
}

static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx,
				    const u8 *src, unsigned int srclen)
{
	u8 *dst = ctx->buffer;

	if (ctx->bytes) {
		int n = min(srclen, ctx->bytes);
		u8 *pos = dst + (16 - ctx->bytes);

		ctx->bytes -= n;
		srclen -= n;

		while (n--)
			*pos++ ^= *src++;

		if (!ctx->bytes)
			gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	}

	while (srclen >= 16) {
		crypto_xor(dst, src, 16);
		gf128mul_4k_lle((be128 *)dst, ctx->gf128);
		src += 16;
		srclen -= 16;
	}

	if (srclen) {
		ctx->bytes = 16 - srclen;
		while (srclen--)
			*dst++ ^= *src++;
	}
}

static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx,
				       struct scatterlist *sg, int len)
{
	struct scatter_walk walk;
	u8 *src;
	int n;

	if (!len)
		return;

	scatterwalk_start(&walk, sg);

	while (len) {
		n = scatterwalk_clamp(&walk, len);

		if (!n) {
			scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg));
			n = scatterwalk_clamp(&walk, len);
		}

		src = scatterwalk_map(&walk, 0);

		crypto_gcm_ghash_update(ctx, src, n);
		len -= n;

		scatterwalk_unmap(src, 0);
		scatterwalk_advance(&walk, n);
		scatterwalk_done(&walk, 0, len);
		if (len)
			crypto_yield(ctx->flags);
	}
}

static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx)
{
	u8 *dst = ctx->buffer;

	if (ctx->bytes) {
		u8 *tmp = dst + (16 - ctx->bytes);

		while (ctx->bytes--)
			*tmp++ ^= 0;

		gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	}

	ctx->bytes = 0;
}

static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx,
				       unsigned int authlen,
				       unsigned int cryptlen, u8 *dst)
{
	u8 *buf = ctx->buffer;
	u128 lengths;

	lengths.a = cpu_to_be64(authlen * 8);
	lengths.b = cpu_to_be64(cryptlen * 8);

	crypto_gcm_ghash_flush(ctx);
	crypto_xor(buf, (u8 *)&lengths, 16);
	gf128mul_4k_lle((be128 *)buf, ctx->gf128);
	crypto_xor(dst, buf, 16);
}

static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
{
	struct crypto_gcm_setkey_result *result = req->data;

	if (err == -EINPROGRESS)
		return;

	result->err = err;
	complete(&result->completion);
}

static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
			     unsigned int keylen)
{
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	struct crypto_ablkcipher *ctr = ctx->ctr;
	struct {
		be128 hash;
		u8 iv[8];

		struct crypto_gcm_setkey_result result;

		struct scatterlist sg[1];
		struct ablkcipher_request req;
	} *data;
	int err;

	crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
	crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
				   CRYPTO_TFM_REQ_MASK);

	err = crypto_ablkcipher_setkey(ctr, key, keylen);
	if (err)
		return err;

	crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
				       CRYPTO_TFM_RES_MASK);

	data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
		       GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	init_completion(&data->result.completion);
	sg_init_one(data->sg, &data->hash, sizeof(data->hash));
	ablkcipher_request_set_tfm(&data->req, ctr);
	ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
						    CRYPTO_TFM_REQ_MAY_BACKLOG,
					crypto_gcm_setkey_done,
					&data->result);
	ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
				     sizeof(data->hash), data->iv);

	err = crypto_ablkcipher_encrypt(&data->req);
	if (err == -EINPROGRESS || err == -EBUSY) {
		err = wait_for_completion_interruptible(
			&data->result.completion);
		if (!err)
			err = data->result.err;
	}

	if (err)
		goto out;

	if (ctx->gf128 != NULL)
		gf128mul_free_4k(ctx->gf128);

	ctx->gf128 = gf128mul_init_4k_lle(&data->hash);

	if (ctx->gf128 == NULL)
		err = -ENOMEM;

out:
	kfree(data);
	return err;
}

static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
				  struct aead_request *req,
				  unsigned int cryptlen)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	u32 flags = req->base.tfm->crt_flags;
	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
	struct scatterlist *dst;
	__be32 counter = cpu_to_be32(1);

	memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
	memcpy(req->iv + 12, &counter, 4);

	sg_init_table(pctx->src, 2);
	sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
	scatterwalk_sg_chain(pctx->src, 2, req->src);

	dst = pctx->src;
	if (req->src != req->dst) {
		sg_init_table(pctx->dst, 2);
		sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
		scatterwalk_sg_chain(pctx->dst, 2, req->dst);
		dst = pctx->dst;
	}

	ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
	ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
				     cryptlen + sizeof(pctx->auth_tag),
				     req->iv);

	crypto_gcm_ghash_init(ghash, flags, ctx->gf128);

	crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen);
	crypto_gcm_ghash_flush(ghash);
}

static int crypto_gcm_hash(struct aead_request *req)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	u8 *auth_tag = pctx->auth_tag;
	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;

	crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen);
	crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen,
				   auth_tag);

	scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
				 crypto_aead_authsize(aead), 1);
	return 0;
}

static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err)
{
	struct aead_request *req = areq->data;

	if (!err)
		err = crypto_gcm_hash(req);

	aead_request_complete(req, err);
}

static int crypto_gcm_encrypt(struct aead_request *req)
{
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	struct ablkcipher_request *abreq = &pctx->abreq;
	int err;

	crypto_gcm_init_crypt(abreq, req, req->cryptlen);
	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
					crypto_gcm_encrypt_done, req);

	err = crypto_ablkcipher_encrypt(abreq);
	if (err)
		return err;

	return crypto_gcm_hash(req);
}

static int crypto_gcm_verify(struct aead_request *req)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
	u8 *auth_tag = pctx->auth_tag;
	u8 *iauth_tag = pctx->iauth_tag;
	unsigned int authsize = crypto_aead_authsize(aead);
	unsigned int cryptlen = req->cryptlen - authsize;

	crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag);

	authsize = crypto_aead_authsize(aead);
	scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
	return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
}

static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err)
{
	struct aead_request *req = areq->data;

	if (!err)
		err = crypto_gcm_verify(req);

	aead_request_complete(req, err);
}

static int crypto_gcm_decrypt(struct aead_request *req)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	struct ablkcipher_request *abreq = &pctx->abreq;
	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
	unsigned int cryptlen = req->cryptlen;
	unsigned int authsize = crypto_aead_authsize(aead);
	int err;

	if (cryptlen < authsize)
		return -EINVAL;
	cryptlen -= authsize;

	crypto_gcm_init_crypt(abreq, req, cryptlen);
	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
					crypto_gcm_decrypt_done, req);

	crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen);

	err = crypto_ablkcipher_decrypt(abreq);
	if (err)
		return err;

	return crypto_gcm_verify(req);
}

static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
{
	struct crypto_instance *inst = (void *)tfm->__crt_alg;
	struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
	struct crypto_ablkcipher *ctr;
	unsigned long align;
	int err;

	ctr = crypto_spawn_skcipher(&ictx->ctr);
	err = PTR_ERR(ctr);
	if (IS_ERR(ctr))
		return err;

	ctx->ctr = ctr;
	ctx->gf128 = NULL;

	align = crypto_tfm_alg_alignmask(tfm);
	align &= ~(crypto_tfm_ctx_alignment() - 1);
	tfm->crt_aead.reqsize = align +
				sizeof(struct crypto_gcm_req_priv_ctx) +
				crypto_ablkcipher_reqsize(ctr);

	return 0;
}

static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
{
	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);

	if (ctx->gf128 != NULL)
		gf128mul_free_4k(ctx->gf128);

	crypto_free_ablkcipher(ctx->ctr);
}

static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb,
						       const char *full_name,
						       const char *ctr_name)
{
	struct crypto_attr_type *algt;
	struct crypto_instance *inst;
	struct crypto_alg *ctr;
	struct gcm_instance_ctx *ctx;
	int err;

	algt = crypto_get_attr_type(tb);
	err = PTR_ERR(algt);
	if (IS_ERR(algt))
		return ERR_PTR(err);

	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
		return ERR_PTR(-EINVAL);

	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
	if (!inst)
		return ERR_PTR(-ENOMEM);

	ctx = crypto_instance_ctx(inst);
	crypto_set_skcipher_spawn(&ctx->ctr, inst);
	err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
				   crypto_requires_sync(algt->type,
							algt->mask));
	if (err)
		goto err_free_inst;

	ctr = crypto_skcipher_spawn_alg(&ctx->ctr);

	/* We only support 16-byte blocks. */
	if (ctr->cra_ablkcipher.ivsize != 16)
		goto out_put_ctr;

	/* Not a stream cipher? */
	err = -EINVAL;
	if (ctr->cra_blocksize != 1)
		goto out_put_ctr;

	err = -ENAMETOOLONG;
	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
		     "gcm_base(%s)", ctr->cra_driver_name) >=
	    CRYPTO_MAX_ALG_NAME)
		goto out_put_ctr;

	memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME);

	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
	inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC;
	inst->alg.cra_priority = ctr->cra_priority;
	inst->alg.cra_blocksize = 1;
	inst->alg.cra_alignmask = ctr->cra_alignmask | (__alignof__(u64) - 1);
	inst->alg.cra_type = &crypto_aead_type;
	inst->alg.cra_aead.ivsize = 16;
	inst->alg.cra_aead.maxauthsize = 16;
	inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
	inst->alg.cra_init = crypto_gcm_init_tfm;
	inst->alg.cra_exit = crypto_gcm_exit_tfm;
	inst->alg.cra_aead.setkey = crypto_gcm_setkey;
	inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
	inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;

out:
	return inst;

out_put_ctr:
	crypto_drop_skcipher(&ctx->ctr);
err_free_inst:
	kfree(inst);
	inst = ERR_PTR(err);
	goto out;
}

static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb)
{
	int err;
	const char *cipher_name;
	char ctr_name[CRYPTO_MAX_ALG_NAME];
	char full_name[CRYPTO_MAX_ALG_NAME];

	cipher_name = crypto_attr_alg_name(tb[1]);
	err = PTR_ERR(cipher_name);
	if (IS_ERR(cipher_name))
		return ERR_PTR(err);

	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
	    CRYPTO_MAX_ALG_NAME)
		return ERR_PTR(-ENAMETOOLONG);

	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
	    CRYPTO_MAX_ALG_NAME)
		return ERR_PTR(-ENAMETOOLONG);

	return crypto_gcm_alloc_common(tb, full_name, ctr_name);
}

static void crypto_gcm_free(struct crypto_instance *inst)
{
	struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);

	crypto_drop_skcipher(&ctx->ctr);
	kfree(inst);
}

static struct crypto_template crypto_gcm_tmpl = {
	.name = "gcm",
	.alloc = crypto_gcm_alloc,
	.free = crypto_gcm_free,
	.module = THIS_MODULE,
};

static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb)
{
	int err;
	const char *ctr_name;
	char full_name[CRYPTO_MAX_ALG_NAME];

	ctr_name = crypto_attr_alg_name(tb[1]);
	err = PTR_ERR(ctr_name);
	if (IS_ERR(ctr_name))
		return ERR_PTR(err);

	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s)",
		     ctr_name) >= CRYPTO_MAX_ALG_NAME)
		return ERR_PTR(-ENAMETOOLONG);

	return crypto_gcm_alloc_common(tb, full_name, ctr_name);
}

static struct crypto_template crypto_gcm_base_tmpl = {
	.name = "gcm_base",
	.alloc = crypto_gcm_base_alloc,
	.free = crypto_gcm_free,
	.module = THIS_MODULE,
};

static int __init crypto_gcm_module_init(void)
{
	int err;

	err = crypto_register_template(&crypto_gcm_base_tmpl);
	if (err)
		goto out;

	err = crypto_register_template(&crypto_gcm_tmpl);
	if (err)
		goto out_undo_base;

out:
	return err;

out_undo_base:
	crypto_unregister_template(&crypto_gcm_base_tmpl);
	goto out;
}

static void __exit crypto_gcm_module_exit(void)
{
	crypto_unregister_template(&crypto_gcm_tmpl);
	crypto_unregister_template(&crypto_gcm_base_tmpl);
}

module_init(crypto_gcm_module_init);
module_exit(crypto_gcm_module_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Galois/Counter Mode");
MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
MODULE_ALIAS("gcm_base");
Commit	Line	Data
28db8e3e MH	1	/*
	2	* GCM: Galois/Counter Mode.
	3	*
	4	* Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
	5	*
	6	* This program is free software; you can redistribute it and/or modify it
	7	* under the terms of the GNU General Public License version 2 as published
	8	* by the Free Software Foundation.
	9	*/
	10
28db8e3e	11	#include <crypto/gf128mul.h>
1472e5eb	12	#include <crypto/internal/skcipher.h>
42c271c6	13	#include <crypto/scatterwalk.h>
84c91152	14	#include <linux/completion.h>
28db8e3e MH	15	#include <linux/err.h>
	16	#include <linux/init.h>
	17	#include <linux/kernel.h>
	18	#include <linux/module.h>
	19	#include <linux/slab.h>
	20
28db8e3e	21	struct gcm_instance_ctx {
1472e5eb	22	struct crypto_skcipher_spawn ctr;
28db8e3e MH	23	};
	24
	25	struct crypto_gcm_ctx {
	26	struct crypto_ablkcipher *ctr;
	27	struct gf128mul_4k *gf128;
	28	};
	29
	30	struct crypto_gcm_ghash_ctx {
	31	u32 bytes;
	32	u32 flags;
	33	struct gf128mul_4k *gf128;
	34	u8 buffer[16];
	35	};
	36
	37	struct crypto_gcm_req_priv_ctx {
	38	u8 auth_tag[16];
6160b289	39	u8 iauth_tag[16];
84c91152 HX	40	struct scatterlist src[2];
84c91152 HX	41	struct scatterlist dst[2];
28db8e3e	42	struct crypto_gcm_ghash_ctx ghash;
7f681378	43	struct ablkcipher_request abreq;
28db8e3e MH	44	};
28db8e3e MH	45
84c91152 HX	46	struct crypto_gcm_setkey_result {
	47	int err;
	48	struct completion completion;
	49	};
	50
2589469d HX	51	static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
	52	struct aead_request *req)
	53	{
	54	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
	55
	56	return (void )PTR_ALIGN((u8 )aead_request_ctx(req), align + 1);
	57	}
	58
28db8e3e MH	59	static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags,
	60	struct gf128mul_4k *gf128)
	61	{
	62	ctx->bytes = 0;
	63	ctx->flags = flags;
	64	ctx->gf128 = gf128;
	65	memset(ctx->buffer, 0, 16);
	66	}
	67
	68	static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx,
	69	const u8 *src, unsigned int srclen)
	70	{
	71	u8 *dst = ctx->buffer;
	72
	73	if (ctx->bytes) {
	74	int n = min(srclen, ctx->bytes);
	75	u8 *pos = dst + (16 - ctx->bytes);
	76
	77	ctx->bytes -= n;
	78	srclen -= n;
	79
	80	while (n--)
	81	pos++ ^= src++;
	82
	83	if (!ctx->bytes)
	84	gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	85	}
	86
	87	while (srclen >= 16) {
	88	crypto_xor(dst, src, 16);
	89	gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	90	src += 16;
	91	srclen -= 16;
	92	}
	93
	94	if (srclen) {
	95	ctx->bytes = 16 - srclen;
	96	while (srclen--)
	97	dst++ ^= src++;
	98	}
	99	}
	100
	101	static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx,
	102	struct scatterlist *sg, int len)
	103	{
	104	struct scatter_walk walk;
	105	u8 *src;
	106	int n;
	107
6160b289 HX	108	if (!len)
	109	return;
	110
28db8e3e MH	111	scatterwalk_start(&walk, sg);
	112
	113	while (len) {
	114	n = scatterwalk_clamp(&walk, len);
	115
	116	if (!n) {
b2ab4a57	117	scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg));
28db8e3e MH	118	n = scatterwalk_clamp(&walk, len);
	119	}
	120
	121	src = scatterwalk_map(&walk, 0);
	122
	123	crypto_gcm_ghash_update(ctx, src, n);
	124	len -= n;
	125
	126	scatterwalk_unmap(src, 0);
	127	scatterwalk_advance(&walk, n);
	128	scatterwalk_done(&walk, 0, len);
	129	if (len)
	130	crypto_yield(ctx->flags);
	131	}
	132	}
	133
	134	static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx)
	135	{
	136	u8 *dst = ctx->buffer;
	137
	138	if (ctx->bytes) {
	139	u8 *tmp = dst + (16 - ctx->bytes);
	140
	141	while (ctx->bytes--)
	142	*tmp++ ^= 0;
	143
	144	gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	145	}
	146
	147	ctx->bytes = 0;
	148	}
	149
	150	static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx,
	151	unsigned int authlen,
	152	unsigned int cryptlen, u8 *dst)
	153	{
	154	u8 *buf = ctx->buffer;
	155	u128 lengths;
	156
	157	lengths.a = cpu_to_be64(authlen * 8);
	158	lengths.b = cpu_to_be64(cryptlen * 8);
	159
	160	crypto_gcm_ghash_flush(ctx);
	161	crypto_xor(buf, (u8 *)&lengths, 16);
	162	gf128mul_4k_lle((be128 *)buf, ctx->gf128);
	163	crypto_xor(dst, buf, 16);
	164	}
	165
84c91152	166	static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
28db8e3e	167	{
84c91152	168	struct crypto_gcm_setkey_result *result = req->data;
28db8e3e	169
84c91152 HX	170	if (err == -EINPROGRESS)
	171	return;
	172
	173	result->err = err;
	174	complete(&result->completion);
28db8e3e MH	175	}
	176
	177	static int crypto_gcm_setkey(struct crypto_aead aead, const u8 key,
	178	unsigned int keylen)
	179	{
	180	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	181	struct crypto_ablkcipher *ctr = ctx->ctr;
84c91152 HX	182	struct {
	183	be128 hash;
	184	u8 iv[8];
	185
	186	struct crypto_gcm_setkey_result result;
	187
	188	struct scatterlist sg[1];
	189	struct ablkcipher_request req;
	190	} *data;
	191	int err;
28db8e3e MH	192
	193	crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
	194	crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
	195	CRYPTO_TFM_REQ_MASK);
	196
	197	err = crypto_ablkcipher_setkey(ctr, key, keylen);
	198	if (err)
84c91152	199	return err;
28db8e3e MH	200
	201	crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
	202	CRYPTO_TFM_RES_MASK);
	203
84c91152 HX	204	data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
	205	GFP_KERNEL);
	206	if (!data)
	207	return -ENOMEM;
	208
	209	init_completion(&data->result.completion);
	210	sg_init_one(data->sg, &data->hash, sizeof(data->hash));
	211	ablkcipher_request_set_tfm(&data->req, ctr);
	212	ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP \|
	213	CRYPTO_TFM_REQ_MAY_BACKLOG,
	214	crypto_gcm_setkey_done,
	215	&data->result);
	216	ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
	217	sizeof(data->hash), data->iv);
	218
	219	err = crypto_ablkcipher_encrypt(&data->req);
	220	if (err == -EINPROGRESS \|\| err == -EBUSY) {
	221	err = wait_for_completion_interruptible(
	222	&data->result.completion);
	223	if (!err)
	224	err = data->result.err;
	225	}
	226
28db8e3e MH	227	if (err)
	228	goto out;
	229
	230	if (ctx->gf128 != NULL)
	231	gf128mul_free_4k(ctx->gf128);
	232
84c91152	233	ctx->gf128 = gf128mul_init_4k_lle(&data->hash);
28db8e3e MH	234
	235	if (ctx->gf128 == NULL)
	236	err = -ENOMEM;
	237
84c91152 HX	238	out:
84c91152 HX	239	kfree(data);
28db8e3e MH	240	return err;
	241	}
	242
84c91152 HX	243	static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
	244	struct aead_request *req,
	245	unsigned int cryptlen)
28db8e3e MH	246	{
	247	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	248	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
2589469d	249	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
28db8e3e	250	u32 flags = req->base.tfm->crt_flags;
28db8e3e	251	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
84c91152 HX	252	struct scatterlist *dst;
	253	__be32 counter = cpu_to_be32(1);
	254
	255	memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
	256	memcpy(req->iv + 12, &counter, 4);
	257
	258	sg_init_table(pctx->src, 2);
	259	sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
	260	scatterwalk_sg_chain(pctx->src, 2, req->src);
	261
	262	dst = pctx->src;
	263	if (req->src != req->dst) {
	264	sg_init_table(pctx->dst, 2);
	265	sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
	266	scatterwalk_sg_chain(pctx->dst, 2, req->dst);
	267	dst = pctx->dst;
	268	}
28db8e3e MH	269
28db8e3e MH	270	ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
84c91152 HX	271	ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
	272	cryptlen + sizeof(pctx->auth_tag),
	273	req->iv);
28db8e3e MH	274
	275	crypto_gcm_ghash_init(ghash, flags, ctx->gf128);
	276
6160b289 HX	277	crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen);
6160b289 HX	278	crypto_gcm_ghash_flush(ghash);
28db8e3e MH	279	}
28db8e3e MH	280
6160b289	281	static int crypto_gcm_hash(struct aead_request *req)
28db8e3e	282	{
6160b289	283	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2589469d	284	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
28db8e3e MH	285	u8 *auth_tag = pctx->auth_tag;
	286	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
	287
	288	crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen);
	289	crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen,
	290	auth_tag);
	291
6160b289 HX	292	scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
	293	crypto_aead_authsize(aead), 1);
	294	return 0;
	295	}
	296
	297	static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err)
	298	{
	299	struct aead_request *req = areq->data;
	300
	301	if (!err)
	302	err = crypto_gcm_hash(req);
	303
28db8e3e MH	304	aead_request_complete(req, err);
	305	}
	306
	307	static int crypto_gcm_encrypt(struct aead_request *req)
	308	{
2589469d	309	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
7f681378	310	struct ablkcipher_request *abreq = &pctx->abreq;
84c91152 HX	311	int err;
	312
	313	crypto_gcm_init_crypt(abreq, req, req->cryptlen);
	314	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
	315	crypto_gcm_encrypt_done, req);
28db8e3e	316
84c91152	317	err = crypto_ablkcipher_encrypt(abreq);
28db8e3e MH	318	if (err)
	319	return err;
	320
6160b289	321	return crypto_gcm_hash(req);
28db8e3e MH	322	}
28db8e3e MH	323
84c91152 HX	324	static int crypto_gcm_verify(struct aead_request *req)
	325	{
	326	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	327	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	328	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
	329	u8 *auth_tag = pctx->auth_tag;
	330	u8 *iauth_tag = pctx->iauth_tag;
	331	unsigned int authsize = crypto_aead_authsize(aead);
	332	unsigned int cryptlen = req->cryptlen - authsize;
	333
	334	crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag);
	335
	336	authsize = crypto_aead_authsize(aead);
	337	scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
	338	return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
	339	}
	340
28db8e3e MH	341	static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err)
28db8e3e MH	342	{
84c91152 HX	343	struct aead_request *req = areq->data;
	344
	345	if (!err)
	346	err = crypto_gcm_verify(req);
	347
	348	aead_request_complete(req, err);
28db8e3e MH	349	}
	350
	351	static int crypto_gcm_decrypt(struct aead_request *req)
	352	{
6160b289	353	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2589469d	354	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
7f681378	355	struct ablkcipher_request *abreq = &pctx->abreq;
28db8e3e	356	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
6160b289 HX	357	unsigned int cryptlen = req->cryptlen;
6160b289 HX	358	unsigned int authsize = crypto_aead_authsize(aead);
28db8e3e MH	359	int err;
28db8e3e MH	360
6160b289	361	if (cryptlen < authsize)
28db8e3e	362	return -EINVAL;
6160b289	363	cryptlen -= authsize;
28db8e3e	364
84c91152 HX	365	crypto_gcm_init_crypt(abreq, req, cryptlen);
	366	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
	367	crypto_gcm_decrypt_done, req);
28db8e3e	368
6160b289	369	crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen);
28db8e3e	370
84c91152 HX	371	err = crypto_ablkcipher_decrypt(abreq);
	372	if (err)
	373	return err;
28db8e3e	374
84c91152	375	return crypto_gcm_verify(req);
28db8e3e MH	376	}
	377
	378	static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
	379	{
	380	struct crypto_instance inst = (void )tfm->__crt_alg;
	381	struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
	382	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
	383	struct crypto_ablkcipher *ctr;
	384	unsigned long align;
	385	int err;
	386
1472e5eb	387	ctr = crypto_spawn_skcipher(&ictx->ctr);
28db8e3e MH	388	err = PTR_ERR(ctr);
	389	if (IS_ERR(ctr))
	390	return err;
	391
	392	ctx->ctr = ctr;
	393	ctx->gf128 = NULL;
	394
2589469d	395	align = crypto_tfm_alg_alignmask(tfm);
28db8e3e	396	align &= ~(crypto_tfm_ctx_alignment() - 1);
7f681378 HX	397	tfm->crt_aead.reqsize = align +
	398	sizeof(struct crypto_gcm_req_priv_ctx) +
	399	crypto_ablkcipher_reqsize(ctr);
28db8e3e MH	400
	401	return 0;
	402	}
	403
	404	static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
	405	{
	406	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
	407
	408	if (ctx->gf128 != NULL)
	409	gf128mul_free_4k(ctx->gf128);
	410
	411	crypto_free_ablkcipher(ctx->ctr);
	412	}
	413
d00aa19b HX	414	static struct crypto_instance crypto_gcm_alloc_common(struct rtattr *tb,
	415	const char *full_name,
	416	const char *ctr_name)
28db8e3e	417	{
d00aa19b	418	struct crypto_attr_type *algt;
28db8e3e MH	419	struct crypto_instance *inst;
28db8e3e MH	420	struct crypto_alg *ctr;
28db8e3e MH	421	struct gcm_instance_ctx *ctx;
28db8e3e MH	422	int err;
28db8e3e	423
d00aa19b HX	424	algt = crypto_get_attr_type(tb);
	425	err = PTR_ERR(algt);
	426	if (IS_ERR(algt))
28db8e3e MH	427	return ERR_PTR(err);
28db8e3e MH	428
d00aa19b HX	429	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
d00aa19b HX	430	return ERR_PTR(-EINVAL);
28db8e3e	431
1472e5eb HX	432	inst = kzalloc(sizeof(inst) + sizeof(ctx), GFP_KERNEL);
	433	if (!inst)
	434	return ERR_PTR(-ENOMEM);
28db8e3e	435
1472e5eb HX	436	ctx = crypto_instance_ctx(inst);
	437	crypto_set_skcipher_spawn(&ctx->ctr, inst);
	438	err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
	439	crypto_requires_sync(algt->type,
	440	algt->mask));
	441	if (err)
	442	goto err_free_inst;
	443
	444	ctr = crypto_skcipher_spawn_alg(&ctx->ctr);
28db8e3e	445
d00aa19b	446	/* We only support 16-byte blocks. */
1472e5eb	447	if (ctr->cra_ablkcipher.ivsize != 16)
d00aa19b HX	448	goto out_put_ctr;
	449
	450	/* Not a stream cipher? */
	451	err = -EINVAL;
	452	if (ctr->cra_blocksize != 1)
28db8e3e MH	453	goto out_put_ctr;
28db8e3e MH	454
28db8e3e	455	err = -ENAMETOOLONG;
d00aa19b HX	456	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
	457	"gcm_base(%s)", ctr->cra_driver_name) >=
	458	CRYPTO_MAX_ALG_NAME)
1472e5eb	459	goto out_put_ctr;
28db8e3e	460
d00aa19b HX	461	memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
d00aa19b HX	462
1472e5eb HX	463	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
1472e5eb HX	464	inst->alg.cra_flags \|= ctr->cra_flags & CRYPTO_ALG_ASYNC;
28db8e3e	465	inst->alg.cra_priority = ctr->cra_priority;
d00aa19b	466	inst->alg.cra_blocksize = 1;
2589469d	467	inst->alg.cra_alignmask = ctr->cra_alignmask \| (__alignof__(u64) - 1);
28db8e3e	468	inst->alg.cra_type = &crypto_aead_type;
84c91152	469	inst->alg.cra_aead.ivsize = 16;
7ba683a6	470	inst->alg.cra_aead.maxauthsize = 16;
28db8e3e MH	471	inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
	472	inst->alg.cra_init = crypto_gcm_init_tfm;
	473	inst->alg.cra_exit = crypto_gcm_exit_tfm;
	474	inst->alg.cra_aead.setkey = crypto_gcm_setkey;
	475	inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
	476	inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;
	477
	478	out:
28db8e3e	479	return inst;
1472e5eb HX	480
	481	out_put_ctr:
	482	crypto_drop_skcipher(&ctx->ctr);
28db8e3e MH	483	err_free_inst:
28db8e3e MH	484	kfree(inst);
28db8e3e MH	485	inst = ERR_PTR(err);
	486	goto out;
	487	}
	488
d00aa19b HX	489	static struct crypto_instance crypto_gcm_alloc(struct rtattr *tb)
	490	{
	491	int err;
	492	const char *cipher_name;
	493	char ctr_name[CRYPTO_MAX_ALG_NAME];
	494	char full_name[CRYPTO_MAX_ALG_NAME];
	495
	496	cipher_name = crypto_attr_alg_name(tb[1]);
	497	err = PTR_ERR(cipher_name);
	498	if (IS_ERR(cipher_name))
	499	return ERR_PTR(err);
	500
	501	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
	502	CRYPTO_MAX_ALG_NAME)
	503	return ERR_PTR(-ENAMETOOLONG);
	504
	505	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
	506	CRYPTO_MAX_ALG_NAME)
	507	return ERR_PTR(-ENAMETOOLONG);
	508
	509	return crypto_gcm_alloc_common(tb, full_name, ctr_name);
	510	}
	511
28db8e3e MH	512	static void crypto_gcm_free(struct crypto_instance *inst)
	513	{
	514	struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);
	515
1472e5eb	516	crypto_drop_skcipher(&ctx->ctr);
28db8e3e MH	517	kfree(inst);
	518	}
	519
	520	static struct crypto_template crypto_gcm_tmpl = {
	521	.name = "gcm",
	522	.alloc = crypto_gcm_alloc,
	523	.free = crypto_gcm_free,
	524	.module = THIS_MODULE,
	525	};
	526
d00aa19b HX	527	static struct crypto_instance crypto_gcm_base_alloc(struct rtattr *tb)
	528	{
	529	int err;
	530	const char *ctr_name;
	531	char full_name[CRYPTO_MAX_ALG_NAME];
	532
	533	ctr_name = crypto_attr_alg_name(tb[1]);
	534	err = PTR_ERR(ctr_name);
	535	if (IS_ERR(ctr_name))
	536	return ERR_PTR(err);
	537
	538	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s)",
	539	ctr_name) >= CRYPTO_MAX_ALG_NAME)
	540	return ERR_PTR(-ENAMETOOLONG);
	541
	542	return crypto_gcm_alloc_common(tb, full_name, ctr_name);
	543	}
	544
	545	static struct crypto_template crypto_gcm_base_tmpl = {
	546	.name = "gcm_base",
	547	.alloc = crypto_gcm_base_alloc,
	548	.free = crypto_gcm_free,
	549	.module = THIS_MODULE,
	550	};
	551
28db8e3e MH	552	static int __init crypto_gcm_module_init(void)
28db8e3e MH	553	{
d00aa19b HX	554	int err;
	555
	556	err = crypto_register_template(&crypto_gcm_base_tmpl);
	557	if (err)
	558	goto out;
	559
	560	err = crypto_register_template(&crypto_gcm_tmpl);
	561	if (err)
	562	goto out_undo_base;
	563
	564	out:
	565	return err;
	566
	567	out_undo_base:
	568	crypto_unregister_template(&crypto_gcm_base_tmpl);
	569	goto out;
28db8e3e MH	570	}
	571
	572	static void __exit crypto_gcm_module_exit(void)
	573	{
	574	crypto_unregister_template(&crypto_gcm_tmpl);
d00aa19b	575	crypto_unregister_template(&crypto_gcm_base_tmpl);
28db8e3e MH	576	}
	577
	578	module_init(crypto_gcm_module_init);
	579	module_exit(crypto_gcm_module_exit);
	580
	581	MODULE_LICENSE("GPL");
	582	MODULE_DESCRIPTION("Galois/Counter Mode");
	583	MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
d00aa19b	584	MODULE_ALIAS("gcm_base");