[mirror_ubuntu-bionic-kernel.git] / crypto / crypto_engine.c

/*
 * Handle async block request by crypto hardware engine.
 *
 * Copyright (C) 2016 Linaro, Inc.
 *
 * Author: Baolin Wang <baolin.wang@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#include <linux/err.h>
#include <linux/delay.h>
#include <crypto/engine.h>
#include <crypto/internal/hash.h>
#include <uapi/linux/sched/types.h>
#include "internal.h"

#define CRYPTO_ENGINE_MAX_QLEN 10

/**
 * crypto_pump_requests - dequeue one request from engine queue to process
 * @engine: the hardware engine
 * @in_kthread: true if we are in the context of the request pump thread
 *
 * This function checks if there is any request in the engine queue that
 * needs processing and if so call out to the driver to initialize hardware
 * and handle each request.
 */
static void crypto_pump_requests(struct crypto_engine *engine,
				 bool in_kthread)
{
	struct crypto_async_request *async_req, *backlog;
	struct ahash_request *hreq;
	struct ablkcipher_request *breq;
	unsigned long flags;
	bool was_busy = false;
	int ret, rtype;

	spin_lock_irqsave(&engine->queue_lock, flags);

	/* Make sure we are not already running a request */
	if (engine->cur_req)
		goto out;

	/* If another context is idling then defer */
	if (engine->idling) {
		kthread_queue_work(engine->kworker, &engine->pump_requests);
		goto out;
	}

	/* Check if the engine queue is idle */
	if (!crypto_queue_len(&engine->queue) || !engine->running) {
		if (!engine->busy)
			goto out;

		/* Only do teardown in the thread */
		if (!in_kthread) {
			kthread_queue_work(engine->kworker,
					   &engine->pump_requests);
			goto out;
		}

		engine->busy = false;
		engine->idling = true;
		spin_unlock_irqrestore(&engine->queue_lock, flags);

		if (engine->unprepare_crypt_hardware &&
		    engine->unprepare_crypt_hardware(engine))
			dev_err(engine->dev, "failed to unprepare crypt hardware\n");

		spin_lock_irqsave(&engine->queue_lock, flags);
		engine->idling = false;
		goto out;
	}

	/* Get the fist request from the engine queue to handle */
	backlog = crypto_get_backlog(&engine->queue);
	async_req = crypto_dequeue_request(&engine->queue);
	if (!async_req)
		goto out;

	engine->cur_req = async_req;
	if (backlog)
		backlog->complete(backlog, -EINPROGRESS);

	if (engine->busy)
		was_busy = true;
	else
		engine->busy = true;

	spin_unlock_irqrestore(&engine->queue_lock, flags);

	rtype = crypto_tfm_alg_type(engine->cur_req->tfm);
	/* Until here we get the request need to be encrypted successfully */
	if (!was_busy && engine->prepare_crypt_hardware) {
		ret = engine->prepare_crypt_hardware(engine);
		if (ret) {
			dev_err(engine->dev, "failed to prepare crypt hardware\n");
			goto req_err;
		}
	}

	switch (rtype) {
	case CRYPTO_ALG_TYPE_AHASH:
		hreq = ahash_request_cast(engine->cur_req);
		if (engine->prepare_hash_request) {
			ret = engine->prepare_hash_request(engine, hreq);
			if (ret) {
				dev_err(engine->dev, "failed to prepare request: %d\n",
					ret);
				goto req_err;
			}
			engine->cur_req_prepared = true;
		}
		ret = engine->hash_one_request(engine, hreq);
		if (ret) {
			dev_err(engine->dev, "failed to hash one request from queue\n");
			goto req_err;
		}
		return;
	case CRYPTO_ALG_TYPE_ABLKCIPHER:
		breq = ablkcipher_request_cast(engine->cur_req);
		if (engine->prepare_cipher_request) {
			ret = engine->prepare_cipher_request(engine, breq);
			if (ret) {
				dev_err(engine->dev, "failed to prepare request: %d\n",
					ret);
				goto req_err;
			}
			engine->cur_req_prepared = true;
		}
		ret = engine->cipher_one_request(engine, breq);
		if (ret) {
			dev_err(engine->dev, "failed to cipher one request from queue\n");
			goto req_err;
		}
		return;
	default:
		dev_err(engine->dev, "failed to prepare request of unknown type\n");
		return;
	}

req_err:
	switch (rtype) {
	case CRYPTO_ALG_TYPE_AHASH:
		hreq = ahash_request_cast(engine->cur_req);
		crypto_finalize_hash_request(engine, hreq, ret);
		break;
	case CRYPTO_ALG_TYPE_ABLKCIPHER:
		breq = ablkcipher_request_cast(engine->cur_req);
		crypto_finalize_cipher_request(engine, breq, ret);
		break;
	}
	return;

out:
	spin_unlock_irqrestore(&engine->queue_lock, flags);
}

static void crypto_pump_work(struct kthread_work *work)
{
	struct crypto_engine *engine =
		container_of(work, struct crypto_engine, pump_requests);

	crypto_pump_requests(engine, true);
}

/**
 * crypto_transfer_cipher_request - transfer the new request into the
 * enginequeue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_cipher_request(struct crypto_engine *engine,
				   struct ablkcipher_request *req,
				   bool need_pump)
{
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&engine->queue_lock, flags);

	if (!engine->running) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		return -ESHUTDOWN;
	}

	ret = ablkcipher_enqueue_request(&engine->queue, req);

	if (!engine->busy && need_pump)
		kthread_queue_work(engine->kworker, &engine->pump_requests);

	spin_unlock_irqrestore(&engine->queue_lock, flags);
	return ret;
}
EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request);

/**
 * crypto_transfer_cipher_request_to_engine - transfer one request to list
 * into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_cipher_request_to_engine(struct crypto_engine *engine,
					     struct ablkcipher_request *req)
{
	return crypto_transfer_cipher_request(engine, req, true);
}
EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request_to_engine);

/**
 * crypto_transfer_hash_request - transfer the new request into the
 * enginequeue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_hash_request(struct crypto_engine *engine,
				 struct ahash_request *req, bool need_pump)
{
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&engine->queue_lock, flags);

	if (!engine->running) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		return -ESHUTDOWN;
	}

	ret = ahash_enqueue_request(&engine->queue, req);

	if (!engine->busy && need_pump)
		kthread_queue_work(engine->kworker, &engine->pump_requests);

	spin_unlock_irqrestore(&engine->queue_lock, flags);
	return ret;
}
EXPORT_SYMBOL_GPL(crypto_transfer_hash_request);

/**
 * crypto_transfer_hash_request_to_engine - transfer one request to list
 * into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
					   struct ahash_request *req)
{
	return crypto_transfer_hash_request(engine, req, true);
}
EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);

/**
 * crypto_finalize_cipher_request - finalize one request if the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_cipher_request(struct crypto_engine *engine,
				    struct ablkcipher_request *req, int err)
{
	unsigned long flags;
	bool finalize_cur_req = false;
	int ret;

	spin_lock_irqsave(&engine->queue_lock, flags);
	if (engine->cur_req == &req->base)
		finalize_cur_req = true;
	spin_unlock_irqrestore(&engine->queue_lock, flags);

	if (finalize_cur_req) {
		if (engine->cur_req_prepared &&
		    engine->unprepare_cipher_request) {
			ret = engine->unprepare_cipher_request(engine, req);
			if (ret)
				dev_err(engine->dev, "failed to unprepare request\n");
		}
		spin_lock_irqsave(&engine->queue_lock, flags);
		engine->cur_req = NULL;
		engine->cur_req_prepared = false;
		spin_unlock_irqrestore(&engine->queue_lock, flags);
	}

	req->base.complete(&req->base, err);

	kthread_queue_work(engine->kworker, &engine->pump_requests);
}
EXPORT_SYMBOL_GPL(crypto_finalize_cipher_request);

/**
 * crypto_finalize_hash_request - finalize one request if the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_hash_request(struct crypto_engine *engine,
				  struct ahash_request *req, int err)
{
	unsigned long flags;
	bool finalize_cur_req = false;
	int ret;

	spin_lock_irqsave(&engine->queue_lock, flags);
	if (engine->cur_req == &req->base)
		finalize_cur_req = true;
	spin_unlock_irqrestore(&engine->queue_lock, flags);

	if (finalize_cur_req) {
		if (engine->cur_req_prepared &&
		    engine->unprepare_hash_request) {
			ret = engine->unprepare_hash_request(engine, req);
			if (ret)
				dev_err(engine->dev, "failed to unprepare request\n");
		}
		spin_lock_irqsave(&engine->queue_lock, flags);
		engine->cur_req = NULL;
		engine->cur_req_prepared = false;
		spin_unlock_irqrestore(&engine->queue_lock, flags);
	}

	req->base.complete(&req->base, err);

	kthread_queue_work(engine->kworker, &engine->pump_requests);
}
EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);

/**
 * crypto_engine_start - start the hardware engine
 * @engine: the hardware engine need to be started
 *
 * Return 0 on success, else on fail.
 */
int crypto_engine_start(struct crypto_engine *engine)
{
	unsigned long flags;

	spin_lock_irqsave(&engine->queue_lock, flags);

	if (engine->running || engine->busy) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		return -EBUSY;
	}

	engine->running = true;
	spin_unlock_irqrestore(&engine->queue_lock, flags);

	kthread_queue_work(engine->kworker, &engine->pump_requests);

	return 0;
}
EXPORT_SYMBOL_GPL(crypto_engine_start);

/**
 * crypto_engine_stop - stop the hardware engine
 * @engine: the hardware engine need to be stopped
 *
 * Return 0 on success, else on fail.
 */
int crypto_engine_stop(struct crypto_engine *engine)
{
	unsigned long flags;
	unsigned int limit = 500;
	int ret = 0;

	spin_lock_irqsave(&engine->queue_lock, flags);

	/*
	 * If the engine queue is not empty or the engine is on busy state,
	 * we need to wait for a while to pump the requests of engine queue.
	 */
	while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		msleep(20);
		spin_lock_irqsave(&engine->queue_lock, flags);
	}

	if (crypto_queue_len(&engine->queue) || engine->busy)
		ret = -EBUSY;
	else
		engine->running = false;

	spin_unlock_irqrestore(&engine->queue_lock, flags);

	if (ret)
		dev_warn(engine->dev, "could not stop engine\n");

	return ret;
}
EXPORT_SYMBOL_GPL(crypto_engine_stop);

/**
 * crypto_engine_alloc_init - allocate crypto hardware engine structure and
 * initialize it.
 * @dev: the device attached with one hardware engine
 * @rt: whether this queue is set to run as a realtime task
 *
 * This must be called from context that can sleep.
 * Return: the crypto engine structure on success, else NULL.
 */
struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
{
	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
	struct crypto_engine *engine;

	if (!dev)
		return NULL;

	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
	if (!engine)
		return NULL;

	engine->dev = dev;
	engine->rt = rt;
	engine->running = false;
	engine->busy = false;
	engine->idling = false;
	engine->cur_req_prepared = false;
	engine->priv_data = dev;
	snprintf(engine->name, sizeof(engine->name),
		 "%s-engine", dev_name(dev));

	crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
	spin_lock_init(&engine->queue_lock);

	engine->kworker = kthread_create_worker(0, "%s", engine->name);
	if (IS_ERR(engine->kworker)) {
		dev_err(dev, "failed to create crypto request pump task\n");
		return NULL;
	}
	kthread_init_work(&engine->pump_requests, crypto_pump_work);

	if (engine->rt) {
		dev_info(dev, "will run requests pump with realtime priority\n");
		sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
	}

	return engine;
}
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);

/**
 * crypto_engine_exit - free the resources of hardware engine when exit
 * @engine: the hardware engine need to be freed
 *
 * Return 0 for success.
 */
int crypto_engine_exit(struct crypto_engine *engine)
{
	int ret;

	ret = crypto_engine_stop(engine);
	if (ret)
		return ret;

	kthread_destroy_worker(engine->kworker);

	return 0;
}
EXPORT_SYMBOL_GPL(crypto_engine_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Crypto hardware engine framework");
Commit	Line	Data
735d37b5 BW	1	/*
	2	* Handle async block request by crypto hardware engine.
	3	*
	4	* Copyright (C) 2016 Linaro, Inc.
	5	*
	6	* Author: Baolin Wang <baolin.wang@linaro.org>
	7	*
	8	* This program is free software; you can redistribute it and/or modify it
	9	* under the terms of the GNU General Public License as published by the Free
	10	* Software Foundation; either version 2 of the License, or (at your option)
	11	* any later version.
	12	*
	13	*/
	14
	15	#include <linux/err.h>
	16	#include <linux/delay.h>
2589ad84	17	#include <crypto/engine.h>
4cba7cf0	18	#include <crypto/internal/hash.h>
ae7e81c0	19	#include <uapi/linux/sched/types.h>
735d37b5 BW	20	#include "internal.h"
	21
	22	#define CRYPTO_ENGINE_MAX_QLEN 10
	23
735d37b5 BW	24	/**
	25	* crypto_pump_requests - dequeue one request from engine queue to process
	26	* @engine: the hardware engine
	27	* @in_kthread: true if we are in the context of the request pump thread
	28	*
	29	* This function checks if there is any request in the engine queue that
	30	* needs processing and if so call out to the driver to initialize hardware
	31	* and handle each request.
	32	*/
	33	static void crypto_pump_requests(struct crypto_engine *engine,
	34	bool in_kthread)
	35	{
	36	struct crypto_async_request async_req, backlog;
4cba7cf0 CL	37	struct ahash_request *hreq;
4cba7cf0 CL	38	struct ablkcipher_request *breq;
735d37b5 BW	39	unsigned long flags;
735d37b5 BW	40	bool was_busy = false;
4cba7cf0	41	int ret, rtype;
735d37b5 BW	42
	43	spin_lock_irqsave(&engine->queue_lock, flags);
	44
	45	/* Make sure we are not already running a request */
	46	if (engine->cur_req)
	47	goto out;
	48
	49	/* If another context is idling then defer */
	50	if (engine->idling) {
c4ca2b0b	51	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	52	goto out;
	53	}
	54
	55	/* Check if the engine queue is idle */
	56	if (!crypto_queue_len(&engine->queue) \|\| !engine->running) {
	57	if (!engine->busy)
	58	goto out;
	59
	60	/* Only do teardown in the thread */
	61	if (!in_kthread) {
c4ca2b0b	62	kthread_queue_work(engine->kworker,
735d37b5 BW	63	&engine->pump_requests);
	64	goto out;
	65	}
	66
	67	engine->busy = false;
	68	engine->idling = true;
	69	spin_unlock_irqrestore(&engine->queue_lock, flags);
	70
	71	if (engine->unprepare_crypt_hardware &&
	72	engine->unprepare_crypt_hardware(engine))
88d58ef8	73	dev_err(engine->dev, "failed to unprepare crypt hardware\n");
735d37b5 BW	74
	75	spin_lock_irqsave(&engine->queue_lock, flags);
	76	engine->idling = false;
	77	goto out;
	78	}
	79
	80	/* Get the fist request from the engine queue to handle */
	81	backlog = crypto_get_backlog(&engine->queue);
	82	async_req = crypto_dequeue_request(&engine->queue);
	83	if (!async_req)
	84	goto out;
	85
4cba7cf0	86	engine->cur_req = async_req;
735d37b5 BW	87	if (backlog)
	88	backlog->complete(backlog, -EINPROGRESS);
	89
	90	if (engine->busy)
	91	was_busy = true;
	92	else
	93	engine->busy = true;
	94
	95	spin_unlock_irqrestore(&engine->queue_lock, flags);
	96
4cba7cf0	97	rtype = crypto_tfm_alg_type(engine->cur_req->tfm);
735d37b5 BW	98	/* Until here we get the request need to be encrypted successfully */
	99	if (!was_busy && engine->prepare_crypt_hardware) {
	100	ret = engine->prepare_crypt_hardware(engine);
	101	if (ret) {
88d58ef8	102	dev_err(engine->dev, "failed to prepare crypt hardware\n");
735d37b5 BW	103	goto req_err;
	104	}
	105	}
	106
4cba7cf0 CL	107	switch (rtype) {
	108	case CRYPTO_ALG_TYPE_AHASH:
	109	hreq = ahash_request_cast(engine->cur_req);
	110	if (engine->prepare_hash_request) {
	111	ret = engine->prepare_hash_request(engine, hreq);
	112	if (ret) {
88d58ef8 CL	113	dev_err(engine->dev, "failed to prepare request: %d\n",
88d58ef8 CL	114	ret);
4cba7cf0 CL	115	goto req_err;
	116	}
	117	engine->cur_req_prepared = true;
	118	}
	119	ret = engine->hash_one_request(engine, hreq);
735d37b5	120	if (ret) {
88d58ef8	121	dev_err(engine->dev, "failed to hash one request from queue\n");
735d37b5 BW	122	goto req_err;
735d37b5 BW	123	}
4cba7cf0 CL	124	return;
	125	case CRYPTO_ALG_TYPE_ABLKCIPHER:
	126	breq = ablkcipher_request_cast(engine->cur_req);
	127	if (engine->prepare_cipher_request) {
	128	ret = engine->prepare_cipher_request(engine, breq);
	129	if (ret) {
88d58ef8 CL	130	dev_err(engine->dev, "failed to prepare request: %d\n",
88d58ef8 CL	131	ret);
4cba7cf0 CL	132	goto req_err;
	133	}
	134	engine->cur_req_prepared = true;
	135	}
	136	ret = engine->cipher_one_request(engine, breq);
	137	if (ret) {
88d58ef8	138	dev_err(engine->dev, "failed to cipher one request from queue\n");
4cba7cf0 CL	139	goto req_err;
	140	}
	141	return;
	142	default:
88d58ef8	143	dev_err(engine->dev, "failed to prepare request of unknown type\n");
4cba7cf0	144	return;
735d37b5	145	}
735d37b5 BW	146
735d37b5 BW	147	req_err:
4cba7cf0 CL	148	switch (rtype) {
	149	case CRYPTO_ALG_TYPE_AHASH:
	150	hreq = ahash_request_cast(engine->cur_req);
	151	crypto_finalize_hash_request(engine, hreq, ret);
	152	break;
	153	case CRYPTO_ALG_TYPE_ABLKCIPHER:
	154	breq = ablkcipher_request_cast(engine->cur_req);
	155	crypto_finalize_cipher_request(engine, breq, ret);
	156	break;
	157	}
735d37b5 BW	158	return;
	159
	160	out:
	161	spin_unlock_irqrestore(&engine->queue_lock, flags);
	162	}
	163
	164	static void crypto_pump_work(struct kthread_work *work)
	165	{
	166	struct crypto_engine *engine =
	167	container_of(work, struct crypto_engine, pump_requests);
	168
	169	crypto_pump_requests(engine, true);
	170	}
	171
	172	/**
4cba7cf0 CL	173	* crypto_transfer_cipher_request - transfer the new request into the
4cba7cf0 CL	174	* enginequeue
735d37b5 BW	175	* @engine: the hardware engine
	176	* @req: the request need to be listed into the engine queue
	177	*/
4cba7cf0 CL	178	int crypto_transfer_cipher_request(struct crypto_engine *engine,
	179	struct ablkcipher_request *req,
	180	bool need_pump)
735d37b5 BW	181	{
	182	unsigned long flags;
	183	int ret;
	184
	185	spin_lock_irqsave(&engine->queue_lock, flags);
	186
	187	if (!engine->running) {
	188	spin_unlock_irqrestore(&engine->queue_lock, flags);
	189	return -ESHUTDOWN;
	190	}
	191
	192	ret = ablkcipher_enqueue_request(&engine->queue, req);
	193
	194	if (!engine->busy && need_pump)
c4ca2b0b	195	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	196
	197	spin_unlock_irqrestore(&engine->queue_lock, flags);
	198	return ret;
	199	}
4cba7cf0 CL	200	EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request);
	201
	202	/**
	203	* crypto_transfer_cipher_request_to_engine - transfer one request to list
	204	* into the engine queue
	205	* @engine: the hardware engine
	206	* @req: the request need to be listed into the engine queue
	207	*/
	208	int crypto_transfer_cipher_request_to_engine(struct crypto_engine *engine,
	209	struct ablkcipher_request *req)
	210	{
	211	return crypto_transfer_cipher_request(engine, req, true);
	212	}
	213	EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request_to_engine);
	214
	215	/**
	216	* crypto_transfer_hash_request - transfer the new request into the
	217	* enginequeue
	218	* @engine: the hardware engine
	219	* @req: the request need to be listed into the engine queue
	220	*/
	221	int crypto_transfer_hash_request(struct crypto_engine *engine,
	222	struct ahash_request *req, bool need_pump)
	223	{
	224	unsigned long flags;
	225	int ret;
	226
	227	spin_lock_irqsave(&engine->queue_lock, flags);
	228
	229	if (!engine->running) {
	230	spin_unlock_irqrestore(&engine->queue_lock, flags);
	231	return -ESHUTDOWN;
	232	}
	233
	234	ret = ahash_enqueue_request(&engine->queue, req);
	235
	236	if (!engine->busy && need_pump)
c4ca2b0b	237	kthread_queue_work(engine->kworker, &engine->pump_requests);
4cba7cf0 CL	238
	239	spin_unlock_irqrestore(&engine->queue_lock, flags);
	240	return ret;
	241	}
	242	EXPORT_SYMBOL_GPL(crypto_transfer_hash_request);
735d37b5 BW	243
735d37b5 BW	244	/**
4cba7cf0 CL	245	* crypto_transfer_hash_request_to_engine - transfer one request to list
4cba7cf0 CL	246	* into the engine queue
735d37b5 BW	247	* @engine: the hardware engine
	248	* @req: the request need to be listed into the engine queue
	249	*/
4cba7cf0 CL	250	int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
4cba7cf0 CL	251	struct ahash_request *req)
735d37b5	252	{
4cba7cf0	253	return crypto_transfer_hash_request(engine, req, true);
735d37b5	254	}
4cba7cf0	255	EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
735d37b5 BW	256
735d37b5 BW	257	/**
4cba7cf0	258	* crypto_finalize_cipher_request - finalize one request if the request is done
735d37b5 BW	259	* @engine: the hardware engine
	260	* @req: the request need to be finalized
	261	* @err: error number
	262	*/
4cba7cf0 CL	263	void crypto_finalize_cipher_request(struct crypto_engine *engine,
4cba7cf0 CL	264	struct ablkcipher_request *req, int err)
735d37b5 BW	265	{
	266	unsigned long flags;
	267	bool finalize_cur_req = false;
	268	int ret;
	269
	270	spin_lock_irqsave(&engine->queue_lock, flags);
4cba7cf0	271	if (engine->cur_req == &req->base)
735d37b5 BW	272	finalize_cur_req = true;
	273	spin_unlock_irqrestore(&engine->queue_lock, flags);
	274
	275	if (finalize_cur_req) {
4cba7cf0 CL	276	if (engine->cur_req_prepared &&
	277	engine->unprepare_cipher_request) {
	278	ret = engine->unprepare_cipher_request(engine, req);
735d37b5	279	if (ret)
88d58ef8	280	dev_err(engine->dev, "failed to unprepare request\n");
735d37b5	281	}
4cba7cf0 CL	282	spin_lock_irqsave(&engine->queue_lock, flags);
	283	engine->cur_req = NULL;
	284	engine->cur_req_prepared = false;
	285	spin_unlock_irqrestore(&engine->queue_lock, flags);
	286	}
	287
	288	req->base.complete(&req->base, err);
735d37b5	289
c4ca2b0b	290	kthread_queue_work(engine->kworker, &engine->pump_requests);
4cba7cf0 CL	291	}
	292	EXPORT_SYMBOL_GPL(crypto_finalize_cipher_request);
	293
	294	/**
	295	* crypto_finalize_hash_request - finalize one request if the request is done
	296	* @engine: the hardware engine
	297	* @req: the request need to be finalized
	298	* @err: error number
	299	*/
	300	void crypto_finalize_hash_request(struct crypto_engine *engine,
	301	struct ahash_request *req, int err)
	302	{
	303	unsigned long flags;
	304	bool finalize_cur_req = false;
	305	int ret;
	306
	307	spin_lock_irqsave(&engine->queue_lock, flags);
	308	if (engine->cur_req == &req->base)
	309	finalize_cur_req = true;
	310	spin_unlock_irqrestore(&engine->queue_lock, flags);
	311
	312	if (finalize_cur_req) {
	313	if (engine->cur_req_prepared &&
	314	engine->unprepare_hash_request) {
	315	ret = engine->unprepare_hash_request(engine, req);
	316	if (ret)
88d58ef8	317	dev_err(engine->dev, "failed to unprepare request\n");
4cba7cf0	318	}
735d37b5 BW	319	spin_lock_irqsave(&engine->queue_lock, flags);
	320	engine->cur_req = NULL;
	321	engine->cur_req_prepared = false;
	322	spin_unlock_irqrestore(&engine->queue_lock, flags);
	323	}
	324
	325	req->base.complete(&req->base, err);
	326
c4ca2b0b	327	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5	328	}
4cba7cf0	329	EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
735d37b5 BW	330
	331	/**
	332	* crypto_engine_start - start the hardware engine
	333	* @engine: the hardware engine need to be started
	334	*
	335	* Return 0 on success, else on fail.
	336	*/
	337	int crypto_engine_start(struct crypto_engine *engine)
	338	{
	339	unsigned long flags;
	340
	341	spin_lock_irqsave(&engine->queue_lock, flags);
	342
	343	if (engine->running \|\| engine->busy) {
	344	spin_unlock_irqrestore(&engine->queue_lock, flags);
	345	return -EBUSY;
	346	}
	347
	348	engine->running = true;
	349	spin_unlock_irqrestore(&engine->queue_lock, flags);
	350
c4ca2b0b	351	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	352
	353	return 0;
	354	}
	355	EXPORT_SYMBOL_GPL(crypto_engine_start);
	356
	357	/**
	358	* crypto_engine_stop - stop the hardware engine
	359	* @engine: the hardware engine need to be stopped
	360	*
	361	* Return 0 on success, else on fail.
	362	*/
	363	int crypto_engine_stop(struct crypto_engine *engine)
	364	{
	365	unsigned long flags;
4cba7cf0	366	unsigned int limit = 500;
735d37b5 BW	367	int ret = 0;
	368
	369	spin_lock_irqsave(&engine->queue_lock, flags);
	370
	371	/*
	372	* If the engine queue is not empty or the engine is on busy state,
	373	* we need to wait for a while to pump the requests of engine queue.
	374	*/
	375	while ((crypto_queue_len(&engine->queue) \|\| engine->busy) && limit--) {
	376	spin_unlock_irqrestore(&engine->queue_lock, flags);
	377	msleep(20);
	378	spin_lock_irqsave(&engine->queue_lock, flags);
	379	}
	380
	381	if (crypto_queue_len(&engine->queue) \|\| engine->busy)
	382	ret = -EBUSY;
	383	else
	384	engine->running = false;
	385
	386	spin_unlock_irqrestore(&engine->queue_lock, flags);
	387
	388	if (ret)
88d58ef8	389	dev_warn(engine->dev, "could not stop engine\n");
735d37b5 BW	390
	391	return ret;
	392	}
	393	EXPORT_SYMBOL_GPL(crypto_engine_stop);
	394
	395	/**
	396	* crypto_engine_alloc_init - allocate crypto hardware engine structure and
	397	* initialize it.
	398	* @dev: the device attached with one hardware engine
	399	* @rt: whether this queue is set to run as a realtime task
	400	*
	401	* This must be called from context that can sleep.
	402	* Return: the crypto engine structure on success, else NULL.
	403	*/
	404	struct crypto_engine crypto_engine_alloc_init(struct device dev, bool rt)
	405	{
	406	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
	407	struct crypto_engine *engine;
	408
	409	if (!dev)
	410	return NULL;
	411
	412	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
	413	if (!engine)
	414	return NULL;
	415
88d58ef8	416	engine->dev = dev;
735d37b5 BW	417	engine->rt = rt;
	418	engine->running = false;
	419	engine->busy = false;
	420	engine->idling = false;
	421	engine->cur_req_prepared = false;
	422	engine->priv_data = dev;
	423	snprintf(engine->name, sizeof(engine->name),
	424	"%s-engine", dev_name(dev));
	425
	426	crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
	427	spin_lock_init(&engine->queue_lock);
	428
c4ca2b0b PM	429	engine->kworker = kthread_create_worker(0, "%s", engine->name);
c4ca2b0b PM	430	if (IS_ERR(engine->kworker)) {
735d37b5 BW	431	dev_err(dev, "failed to create crypto request pump task\n");
	432	return NULL;
	433	}
3989144f	434	kthread_init_work(&engine->pump_requests, crypto_pump_work);
735d37b5 BW	435
	436	if (engine->rt) {
	437	dev_info(dev, "will run requests pump with realtime priority\n");
c4ca2b0b	438	sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
735d37b5 BW	439	}
	440
	441	return engine;
	442	}
	443	EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
	444
	445	/**
	446	* crypto_engine_exit - free the resources of hardware engine when exit
	447	* @engine: the hardware engine need to be freed
	448	*
	449	* Return 0 for success.
	450	*/
	451	int crypto_engine_exit(struct crypto_engine *engine)
	452	{
	453	int ret;
	454
	455	ret = crypto_engine_stop(engine);
	456	if (ret)
	457	return ret;
	458
c4ca2b0b	459	kthread_destroy_worker(engine->kworker);
735d37b5 BW	460
	461	return 0;
	462	}
	463	EXPORT_SYMBOL_GPL(crypto_engine_exit);
	464
	465	MODULE_LICENSE("GPL");
	466	MODULE_DESCRIPTION("Crypto hardware engine framework");