[mirror_ubuntu-bionic-kernel.git] / drivers / crypto / omap-aes-gcm.c

/*
 * Cryptographic API.
 *
 * Support for OMAP AES GCM HW acceleration.
 *
 * Copyright (c) 2016 Texas Instruments Incorporated
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 */

#include <linux/errno.h>
#include <linux/scatterlist.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/omap-dma.h>
#include <linux/interrupt.h>
#include <crypto/aes.h>
#include <crypto/gcm.h>
#include <crypto/scatterwalk.h>
#include <crypto/skcipher.h>
#include <crypto/internal/aead.h>

#include "omap-crypto.h"
#include "omap-aes.h"

static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
				     struct aead_request *req);

static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
{
	struct aead_request *req = dd->aead_req;

	dd->flags &= ~FLAGS_BUSY;
	dd->in_sg = NULL;
	dd->out_sg = NULL;

	req->base.complete(&req->base, ret);
}

static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
{
	u8 *tag;
	int alen, clen, i, ret = 0, nsg;
	struct omap_aes_reqctx *rctx;

	alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
	clen = ALIGN(dd->total, AES_BLOCK_SIZE);
	rctx = aead_request_ctx(dd->aead_req);

	nsg = !!(dd->assoc_len && dd->total);

	dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
			       DMA_FROM_DEVICE);
	dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
	dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
	omap_aes_crypt_dma_stop(dd);

	omap_crypto_cleanup(dd->out_sg, dd->orig_out,
			    dd->aead_req->assoclen, dd->total,
			    FLAGS_OUT_DATA_ST_SHIFT, dd->flags);

	if (dd->flags & FLAGS_ENCRYPT)
		scatterwalk_map_and_copy(rctx->auth_tag,
					 dd->aead_req->dst,
					 dd->total + dd->aead_req->assoclen,
					 dd->authsize, 1);

	omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
			    FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);

	omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
			    FLAGS_IN_DATA_ST_SHIFT, dd->flags);

	if (!(dd->flags & FLAGS_ENCRYPT)) {
		tag = (u8 *)rctx->auth_tag;
		for (i = 0; i < dd->authsize; i++) {
			if (tag[i]) {
				dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n");
				ret = -EBADMSG;
			}
		}
	}

	omap_aes_gcm_finish_req(dd, ret);
	omap_aes_gcm_handle_queue(dd, NULL);
}

static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
				     struct aead_request *req)
{
	int alen, clen, cryptlen, assoclen, ret;
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	unsigned int authlen = crypto_aead_authsize(aead);
	struct scatterlist *tmp, sg_arr[2];
	int nsg;
	u16 flags;

	assoclen = req->assoclen;
	cryptlen = req->cryptlen;

	if (dd->flags & FLAGS_RFC4106_GCM)
		assoclen -= 8;

	if (!(dd->flags & FLAGS_ENCRYPT))
		cryptlen -= authlen;

	alen = ALIGN(assoclen, AES_BLOCK_SIZE);
	clen = ALIGN(cryptlen, AES_BLOCK_SIZE);

	nsg = !!(assoclen && cryptlen);

	omap_aes_clear_copy_flags(dd);

	sg_init_table(dd->in_sgl, nsg + 1);
	if (assoclen) {
		tmp = req->src;
		ret = omap_crypto_align_sg(&tmp, assoclen,
					   AES_BLOCK_SIZE, dd->in_sgl,
					   OMAP_CRYPTO_COPY_DATA |
					   OMAP_CRYPTO_ZERO_BUF |
					   OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
					   FLAGS_ASSOC_DATA_ST_SHIFT,
					   &dd->flags);
	}

	if (cryptlen) {
		tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);

		ret = omap_crypto_align_sg(&tmp, cryptlen,
					   AES_BLOCK_SIZE, &dd->in_sgl[nsg],
					   OMAP_CRYPTO_COPY_DATA |
					   OMAP_CRYPTO_ZERO_BUF |
					   OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
					   FLAGS_IN_DATA_ST_SHIFT,
					   &dd->flags);
	}

	dd->in_sg = dd->in_sgl;
	dd->total = cryptlen;
	dd->assoc_len = assoclen;
	dd->authsize = authlen;

	dd->out_sg = req->dst;
	dd->orig_out = req->dst;

	dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen);

	flags = 0;
	if (req->src == req->dst || dd->out_sg == sg_arr)
		flags |= OMAP_CRYPTO_FORCE_COPY;

	ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
				   AES_BLOCK_SIZE, &dd->out_sgl,
				   flags,
				   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
	if (ret)
		return ret;

	dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
	dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);

	return 0;
}

static void omap_aes_gcm_complete(struct crypto_async_request *req, int err)
{
	struct omap_aes_gcm_result *res = req->data;

	if (err == -EINPROGRESS)
		return;

	res->err = err;
	complete(&res->completion);
}

static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
{
	struct scatterlist iv_sg, tag_sg;
	struct skcipher_request *sk_req;
	struct omap_aes_gcm_result result;
	struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	int ret = 0;

	sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL);
	if (!sk_req) {
		pr_err("skcipher: Failed to allocate request\n");
		return -ENOMEM;
	}

	init_completion(&result.completion);

	sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE);
	sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE);
	skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
				      omap_aes_gcm_complete, &result);
	ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen);
	skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE,
				   NULL);
	ret = crypto_skcipher_encrypt(sk_req);
	switch (ret) {
	case 0:
		break;
	case -EINPROGRESS:
	case -EBUSY:
		ret = wait_for_completion_interruptible(&result.completion);
		if (!ret) {
			ret = result.err;
			if (!ret) {
				reinit_completion(&result.completion);
				break;
			}
		}
		/* fall through */
	default:
		pr_err("Encryption of IV failed for GCM mode\n");
		break;
	}

	skcipher_request_free(sk_req);
	return ret;
}

void omap_aes_gcm_dma_out_callback(void *data)
{
	struct omap_aes_dev *dd = data;
	struct omap_aes_reqctx *rctx;
	int i, val;
	u32 *auth_tag, tag[4];

	if (!(dd->flags & FLAGS_ENCRYPT))
		scatterwalk_map_and_copy(tag, dd->aead_req->src,
					 dd->total + dd->aead_req->assoclen,
					 dd->authsize, 0);

	rctx = aead_request_ctx(dd->aead_req);
	auth_tag = (u32 *)rctx->auth_tag;
	for (i = 0; i < 4; i++) {
		val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
		auth_tag[i] = val ^ auth_tag[i];
		if (!(dd->flags & FLAGS_ENCRYPT))
			auth_tag[i] = auth_tag[i] ^ tag[i];
	}

	omap_aes_gcm_done_task(dd);
}

static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
				     struct aead_request *req)
{
	struct omap_aes_ctx *ctx;
	struct aead_request *backlog;
	struct omap_aes_reqctx *rctx;
	unsigned long flags;
	int err, ret = 0;

	spin_lock_irqsave(&dd->lock, flags);
	if (req)
		ret = aead_enqueue_request(&dd->aead_queue, req);
	if (dd->flags & FLAGS_BUSY) {
		spin_unlock_irqrestore(&dd->lock, flags);
		return ret;
	}

	backlog = aead_get_backlog(&dd->aead_queue);
	req = aead_dequeue_request(&dd->aead_queue);
	if (req)
		dd->flags |= FLAGS_BUSY;
	spin_unlock_irqrestore(&dd->lock, flags);

	if (!req)
		return ret;

	if (backlog)
		backlog->base.complete(&backlog->base, -EINPROGRESS);

	ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	rctx = aead_request_ctx(req);

	dd->ctx = ctx;
	rctx->dd = dd;
	dd->aead_req = req;

	rctx->mode &= FLAGS_MODE_MASK;
	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;

	err = omap_aes_gcm_copy_buffers(dd, req);
	if (err)
		return err;

	err = omap_aes_write_ctrl(dd);
	if (!err)
		err = omap_aes_crypt_dma_start(dd);

	if (err) {
		omap_aes_gcm_finish_req(dd, err);
		omap_aes_gcm_handle_queue(dd, NULL);
	}

	return ret;
}

static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
{
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	unsigned int authlen = crypto_aead_authsize(aead);
	struct omap_aes_dev *dd;
	__be32 counter = cpu_to_be32(1);
	int err, assoclen;

	memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
	memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);

	err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
	if (err)
		return err;

	if (mode & FLAGS_RFC4106_GCM)
		assoclen = req->assoclen - 8;
	else
		assoclen = req->assoclen;
	if (assoclen + req->cryptlen == 0) {
		scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
					 1);
		return 0;
	}

	dd = omap_aes_find_dev(rctx);
	if (!dd)
		return -ENODEV;
	rctx->mode = mode;

	return omap_aes_gcm_handle_queue(dd, req);
}

int omap_aes_gcm_encrypt(struct aead_request *req)
{
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);

	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
	return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
}

int omap_aes_gcm_decrypt(struct aead_request *req)
{
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);

	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
	return omap_aes_gcm_crypt(req, FLAGS_GCM);
}

int omap_aes_4106gcm_encrypt(struct aead_request *req)
{
	struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);

	memcpy(rctx->iv, ctx->nonce, 4);
	memcpy(rctx->iv + 4, req->iv, 8);
	return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
				  FLAGS_RFC4106_GCM);
}

int omap_aes_4106gcm_decrypt(struct aead_request *req)
{
	struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);

	memcpy(rctx->iv, ctx->nonce, 4);
	memcpy(rctx->iv + 4, req->iv, 8);
	return omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
}

int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
			unsigned int keylen)
{
	struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);

	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
	    keylen != AES_KEYSIZE_256)
		return -EINVAL;

	memcpy(ctx->key, key, keylen);
	ctx->keylen = keylen;

	return 0;
}

int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
			    unsigned int keylen)
{
	struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);

	if (keylen < 4)
		return -EINVAL;

	keylen -= 4;
	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
	    keylen != AES_KEYSIZE_256)
		return -EINVAL;

	memcpy(ctx->key, key, keylen);
	memcpy(ctx->nonce, key + keylen, 4);
	ctx->keylen = keylen;

	return 0;
}
Commit	Line	Data
ad18cc9d TK	1	/*
	2	* Cryptographic API.
	3	*
	4	* Support for OMAP AES GCM HW acceleration.
	5	*
	6	* Copyright (c) 2016 Texas Instruments Incorporated
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License version 2 as published
	10	* by the Free Software Foundation.
	11	*
	12	*/
	13
	14	#include <linux/errno.h>
	15	#include <linux/scatterlist.h>
	16	#include <linux/dma-mapping.h>
	17	#include <linux/dmaengine.h>
	18	#include <linux/omap-dma.h>
	19	#include <linux/interrupt.h>
	20	#include <crypto/aes.h>
cb3f3817	21	#include <crypto/gcm.h>
ad18cc9d TK	22	#include <crypto/scatterwalk.h>
	23	#include <crypto/skcipher.h>
	24	#include <crypto/internal/aead.h>
	25
	26	#include "omap-crypto.h"
	27	#include "omap-aes.h"
	28
	29	static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
	30	struct aead_request *req);
	31
	32	static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
	33	{
	34	struct aead_request *req = dd->aead_req;
	35
	36	dd->flags &= ~FLAGS_BUSY;
	37	dd->in_sg = NULL;
	38	dd->out_sg = NULL;
	39
	40	req->base.complete(&req->base, ret);
	41	}
	42
	43	static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
	44	{
	45	u8 *tag;
	46	int alen, clen, i, ret = 0, nsg;
	47	struct omap_aes_reqctx *rctx;
	48
	49	alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
	50	clen = ALIGN(dd->total, AES_BLOCK_SIZE);
	51	rctx = aead_request_ctx(dd->aead_req);
	52
	53	nsg = !!(dd->assoc_len && dd->total);
	54
	55	dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
	56	DMA_FROM_DEVICE);
	57	dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
	58	dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
	59	omap_aes_crypt_dma_stop(dd);
	60
	61	omap_crypto_cleanup(dd->out_sg, dd->orig_out,
	62	dd->aead_req->assoclen, dd->total,
	63	FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
	64
	65	if (dd->flags & FLAGS_ENCRYPT)
	66	scatterwalk_map_and_copy(rctx->auth_tag,
	67	dd->aead_req->dst,
	68	dd->total + dd->aead_req->assoclen,
	69	dd->authsize, 1);
	70
	71	omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
	72	FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
	73
	74	omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
	75	FLAGS_IN_DATA_ST_SHIFT, dd->flags);
	76
	77	if (!(dd->flags & FLAGS_ENCRYPT)) {
	78	tag = (u8 *)rctx->auth_tag;
	79	for (i = 0; i < dd->authsize; i++) {
	80	if (tag[i]) {
	81	dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n");
	82	ret = -EBADMSG;
	83	}
	84	}
	85	}
86
87	omap_aes_gcm_finish_req(dd, ret);
88	omap_aes_gcm_handle_queue(dd, NULL);
89	}
90
91	static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
92	struct aead_request *req)
93	{
94	int alen, clen, cryptlen, assoclen, ret;
95	struct crypto_aead *aead = crypto_aead_reqtfm(req);
96	unsigned int authlen = crypto_aead_authsize(aead);
97	struct scatterlist *tmp, sg_arr[2];
98	int nsg;
99	u16 flags;
100
101	assoclen = req->assoclen;
102	cryptlen = req->cryptlen;
103
104	if (dd->flags & FLAGS_RFC4106_GCM)
105	assoclen -= 8;
106
107	if (!(dd->flags & FLAGS_ENCRYPT))
108	cryptlen -= authlen;
109
110	alen = ALIGN(assoclen, AES_BLOCK_SIZE);
111	clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
112
113	nsg = !!(assoclen && cryptlen);
114
115	omap_aes_clear_copy_flags(dd);
116
117	sg_init_table(dd->in_sgl, nsg + 1);
118	if (assoclen) {
119	tmp = req->src;
120	ret = omap_crypto_align_sg(&tmp, assoclen,
121	AES_BLOCK_SIZE, dd->in_sgl,
122	OMAP_CRYPTO_COPY_DATA \|
123	OMAP_CRYPTO_ZERO_BUF \|
124	OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
125	FLAGS_ASSOC_DATA_ST_SHIFT,
126	&dd->flags);
127	}
128
129	if (cryptlen) {
130	tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
131
132	ret = omap_crypto_align_sg(&tmp, cryptlen,
133	AES_BLOCK_SIZE, &dd->in_sgl[nsg],
134	OMAP_CRYPTO_COPY_DATA \|
135	OMAP_CRYPTO_ZERO_BUF \|
136	OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
137	FLAGS_IN_DATA_ST_SHIFT,
138	&dd->flags);
139	}
140
141	dd->in_sg = dd->in_sgl;
142	dd->total = cryptlen;
143	dd->assoc_len = assoclen;
144	dd->authsize = authlen;
145
146	dd->out_sg = req->dst;
147	dd->orig_out = req->dst;
148
149	dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen);
150
151	flags = 0;
152	if (req->src == req->dst \|\| dd->out_sg == sg_arr)
153	flags \|= OMAP_CRYPTO_FORCE_COPY;
154
155	ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
156	AES_BLOCK_SIZE, &dd->out_sgl,
157	flags,
158	FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
159	if (ret)
160	return ret;
161
162	dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
163	dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
164
165	return 0;
166	}
167
168	static void omap_aes_gcm_complete(struct crypto_async_request *req, int err)
169	{
170	struct omap_aes_gcm_result *res = req->data;
171
172	if (err == -EINPROGRESS)
173	return;
174
175	res->err = err;
176	complete(&res->completion);
177	}
178
179	static int do_encrypt_iv(struct aead_request req, u32 tag, u32 *iv)
180	{
181	struct scatterlist iv_sg, tag_sg;
182	struct skcipher_request *sk_req;
183	struct omap_aes_gcm_result result;
184	struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
185	int ret = 0;
186
187	sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL);
188	if (!sk_req) {
189	pr_err("skcipher: Failed to allocate request\n");
2fd23f2b	190	return -ENOMEM;
ad18cc9d TK	191	}
	192
	193	init_completion(&result.completion);
	194
	195	sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE);
	196	sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE);
	197	skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
	198	omap_aes_gcm_complete, &result);
	199	ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen);
	200	skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE,
	201	NULL);
	202	ret = crypto_skcipher_encrypt(sk_req);
	203	switch (ret) {
	204	case 0:
	205	break;
	206	case -EINPROGRESS:
	207	case -EBUSY:
	208	ret = wait_for_completion_interruptible(&result.completion);
	209	if (!ret) {
	210	ret = result.err;
	211	if (!ret) {
	212	reinit_completion(&result.completion);
	213	break;
	214	}
	215	}
	216	/* fall through */
	217	default:
bfbd6254	218	pr_err("Encryption of IV failed for GCM mode\n");
ad18cc9d TK	219	break;
	220	}
	221
	222	skcipher_request_free(sk_req);
	223	return ret;
	224	}
	225
	226	void omap_aes_gcm_dma_out_callback(void *data)
	227	{
	228	struct omap_aes_dev *dd = data;
	229	struct omap_aes_reqctx *rctx;
	230	int i, val;
	231	u32 *auth_tag, tag[4];
	232
	233	if (!(dd->flags & FLAGS_ENCRYPT))
	234	scatterwalk_map_and_copy(tag, dd->aead_req->src,
	235	dd->total + dd->aead_req->assoclen,
	236	dd->authsize, 0);
	237
	238	rctx = aead_request_ctx(dd->aead_req);
	239	auth_tag = (u32 *)rctx->auth_tag;
	240	for (i = 0; i < 4; i++) {
	241	val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
	242	auth_tag[i] = val ^ auth_tag[i];
	243	if (!(dd->flags & FLAGS_ENCRYPT))
	244	auth_tag[i] = auth_tag[i] ^ tag[i];
	245	}
	246
	247	omap_aes_gcm_done_task(dd);
	248	}
	249
	250	static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
	251	struct aead_request *req)
	252	{
	253	struct omap_aes_ctx *ctx;
	254	struct aead_request *backlog;
	255	struct omap_aes_reqctx *rctx;
	256	unsigned long flags;
	257	int err, ret = 0;
	258
	259	spin_lock_irqsave(&dd->lock, flags);
	260	if (req)
	261	ret = aead_enqueue_request(&dd->aead_queue, req);
	262	if (dd->flags & FLAGS_BUSY) {
	263	spin_unlock_irqrestore(&dd->lock, flags);
	264	return ret;
	265	}
	266
	267	backlog = aead_get_backlog(&dd->aead_queue);
	268	req = aead_dequeue_request(&dd->aead_queue);
	269	if (req)
	270	dd->flags \|= FLAGS_BUSY;
	271	spin_unlock_irqrestore(&dd->lock, flags);
	272
	273	if (!req)
	274	return ret;
	275
	276	if (backlog)
	277	backlog->base.complete(&backlog->base, -EINPROGRESS);
	278
	279	ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	280	rctx = aead_request_ctx(req);
	281
	282	dd->ctx = ctx;
283	rctx->dd = dd;
284	dd->aead_req = req;
285
286	rctx->mode &= FLAGS_MODE_MASK;
287	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) \| rctx->mode;
288
289	err = omap_aes_gcm_copy_buffers(dd, req);
290	if (err)
291	return err;
292
293	err = omap_aes_write_ctrl(dd);
294	if (!err)
295	err = omap_aes_crypt_dma_start(dd);
296
297	if (err) {
298	omap_aes_gcm_finish_req(dd, err);
299	omap_aes_gcm_handle_queue(dd, NULL);
300	}
301
302	return ret;
303	}
304
305	static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
306	{
307	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
308	struct crypto_aead *aead = crypto_aead_reqtfm(req);
309	unsigned int authlen = crypto_aead_authsize(aead);
310	struct omap_aes_dev *dd;
311	__be32 counter = cpu_to_be32(1);
312	int err, assoclen;
313
314	memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
cb3f3817	315	memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
ad18cc9d TK	316
	317	err = do_encrypt_iv(req, (u32 )rctx->auth_tag, (u32 )rctx->iv);
	318	if (err)
	319	return err;
	320
	321	if (mode & FLAGS_RFC4106_GCM)
	322	assoclen = req->assoclen - 8;
	323	else
	324	assoclen = req->assoclen;
	325	if (assoclen + req->cryptlen == 0) {
	326	scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
	327	1);
	328	return 0;
	329	}
	330
	331	dd = omap_aes_find_dev(rctx);
	332	if (!dd)
	333	return -ENODEV;
	334	rctx->mode = mode;
	335
	336	return omap_aes_gcm_handle_queue(dd, req);
	337	}
	338
	339	int omap_aes_gcm_encrypt(struct aead_request *req)
	340	{
	341	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	342
cb3f3817	343	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
ad18cc9d TK	344	return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT \| FLAGS_GCM);
	345	}
	346
	347	int omap_aes_gcm_decrypt(struct aead_request *req)
	348	{
	349	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	350
cb3f3817	351	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
ad18cc9d TK	352	return omap_aes_gcm_crypt(req, FLAGS_GCM);
	353	}
	354
	355	int omap_aes_4106gcm_encrypt(struct aead_request *req)
	356	{
	357	struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	358	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	359
	360	memcpy(rctx->iv, ctx->nonce, 4);
	361	memcpy(rctx->iv + 4, req->iv, 8);
	362	return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT \| FLAGS_GCM \|
	363	FLAGS_RFC4106_GCM);
	364	}
	365
	366	int omap_aes_4106gcm_decrypt(struct aead_request *req)
	367	{
	368	struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	369	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	370
	371	memcpy(rctx->iv, ctx->nonce, 4);
	372	memcpy(rctx->iv + 4, req->iv, 8);
	373	return omap_aes_gcm_crypt(req, FLAGS_GCM \| FLAGS_RFC4106_GCM);
	374	}
	375
	376	int omap_aes_gcm_setkey(struct crypto_aead tfm, const u8 key,
	377	unsigned int keylen)
	378	{
	379	struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
	380
	381	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
	382	keylen != AES_KEYSIZE_256)
	383	return -EINVAL;
	384
	385	memcpy(ctx->key, key, keylen);
	386	ctx->keylen = keylen;
	387
	388	return 0;
	389	}
	390
	391	int omap_aes_4106gcm_setkey(struct crypto_aead tfm, const u8 key,
	392	unsigned int keylen)
	393	{
	394	struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
	395
	396	if (keylen < 4)
	397	return -EINVAL;
	398
	399	keylen -= 4;
	400	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
	401	keylen != AES_KEYSIZE_256)
	402	return -EINVAL;
	403
	404	memcpy(ctx->key, key, keylen);
	405	memcpy(ctx->nonce, key + keylen, 4);
	406	ctx->keylen = keylen;
	407
	408	return 0;
	409	}