[mirror_ubuntu-hirsute-kernel.git] / drivers / crypto / ccp / ccp-dev-v3.c

/*
 * AMD Cryptographic Coprocessor (CCP) driver
 *
 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
 *
 * Author: Tom Lendacky <thomas.lendacky@amd.com>
 * Author: Gary R Hook <gary.hook@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/kthread.h>
#include <linux/interrupt.h>
#include <linux/ccp.h>

#include "ccp-dev.h"

static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
{
	struct ccp_cmd_queue *cmd_q = op->cmd_q;
	struct ccp_device *ccp = cmd_q->ccp;
	void __iomem *cr_addr;
	u32 cr0, cmd;
	unsigned int i;
	int ret = 0;

	/* We could read a status register to see how many free slots
	 * are actually available, but reading that register resets it
	 * and you could lose some error information.
	 */
	cmd_q->free_slots--;

	cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
	      | (op->jobid << REQ0_JOBID_SHIFT)
	      | REQ0_WAIT_FOR_WRITE;

	if (op->soc)
		cr0 |= REQ0_STOP_ON_COMPLETE
		       | REQ0_INT_ON_COMPLETE;

	if (op->ioc || !cmd_q->free_slots)
		cr0 |= REQ0_INT_ON_COMPLETE;

	/* Start at CMD_REQ1 */
	cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;

	mutex_lock(&ccp->req_mutex);

	/* Write CMD_REQ1 through CMD_REQx first */
	for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
		iowrite32(*(cr + i), cr_addr);

	/* Tell the CCP to start */
	wmb();
	iowrite32(cr0, ccp->io_regs + CMD_REQ0);

	mutex_unlock(&ccp->req_mutex);

	if (cr0 & REQ0_INT_ON_COMPLETE) {
		/* Wait for the job to complete */
		ret = wait_event_interruptible(cmd_q->int_queue,
					       cmd_q->int_rcvd);
		if (ret || cmd_q->cmd_error) {
			/* On error delete all related jobs from the queue */
			cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
			      | op->jobid;

			iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);

			if (!ret)
				ret = -EIO;
		} else if (op->soc) {
			/* Delete just head job from the queue on SoC */
			cmd = DEL_Q_ACTIVE
			      | (cmd_q->id << DEL_Q_ID_SHIFT)
			      | op->jobid;

			iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
		}

		cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);

		cmd_q->int_rcvd = 0;
	}

	return ret;
}

static int ccp_perform_aes(struct ccp_op *op)
{
	u32 cr[6];

	/* Fill out the register contents for REQ1 through REQ6 */
	cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
		| (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
		| (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
		| (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
		| (op->ksb_key << REQ1_KEY_KSB_SHIFT);
	cr[1] = op->src.u.dma.length - 1;
	cr[2] = ccp_addr_lo(&op->src.u.dma);
	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
		| ccp_addr_hi(&op->src.u.dma);
	cr[4] = ccp_addr_lo(&op->dst.u.dma);
	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
		| ccp_addr_hi(&op->dst.u.dma);

	if (op->u.aes.mode == CCP_AES_MODE_CFB)
		cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);

	if (op->eom)
		cr[0] |= REQ1_EOM;

	if (op->init)
		cr[0] |= REQ1_INIT;

	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
}

static int ccp_perform_xts_aes(struct ccp_op *op)
{
	u32 cr[6];

	/* Fill out the register contents for REQ1 through REQ6 */
	cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
		| (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
		| (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
		| (op->ksb_key << REQ1_KEY_KSB_SHIFT);
	cr[1] = op->src.u.dma.length - 1;
	cr[2] = ccp_addr_lo(&op->src.u.dma);
	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
		| ccp_addr_hi(&op->src.u.dma);
	cr[4] = ccp_addr_lo(&op->dst.u.dma);
	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
		| ccp_addr_hi(&op->dst.u.dma);

	if (op->eom)
		cr[0] |= REQ1_EOM;

	if (op->init)
		cr[0] |= REQ1_INIT;

	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
}

static int ccp_perform_sha(struct ccp_op *op)
{
	u32 cr[6];

	/* Fill out the register contents for REQ1 through REQ6 */
	cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
		| (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
		| REQ1_INIT;
	cr[1] = op->src.u.dma.length - 1;
	cr[2] = ccp_addr_lo(&op->src.u.dma);
	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
		| ccp_addr_hi(&op->src.u.dma);

	if (op->eom) {
		cr[0] |= REQ1_EOM;
		cr[4] = lower_32_bits(op->u.sha.msg_bits);
		cr[5] = upper_32_bits(op->u.sha.msg_bits);
	} else {
		cr[4] = 0;
		cr[5] = 0;
	}

	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
}

static int ccp_perform_rsa(struct ccp_op *op)
{
	u32 cr[6];

	/* Fill out the register contents for REQ1 through REQ6 */
	cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
		| (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
		| (op->ksb_key << REQ1_KEY_KSB_SHIFT)
		| REQ1_EOM;
	cr[1] = op->u.rsa.input_len - 1;
	cr[2] = ccp_addr_lo(&op->src.u.dma);
	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
		| ccp_addr_hi(&op->src.u.dma);
	cr[4] = ccp_addr_lo(&op->dst.u.dma);
	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
		| ccp_addr_hi(&op->dst.u.dma);

	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
}

static int ccp_perform_passthru(struct ccp_op *op)
{
	u32 cr[6];

	/* Fill out the register contents for REQ1 through REQ6 */
	cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
		| (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
		| (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);

	if (op->src.type == CCP_MEMTYPE_SYSTEM)
		cr[1] = op->src.u.dma.length - 1;
	else
		cr[1] = op->dst.u.dma.length - 1;

	if (op->src.type == CCP_MEMTYPE_SYSTEM) {
		cr[2] = ccp_addr_lo(&op->src.u.dma);
		cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
			| ccp_addr_hi(&op->src.u.dma);

		if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
			cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT);
	} else {
		cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
		cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
	}

	if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
		cr[4] = ccp_addr_lo(&op->dst.u.dma);
		cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
			| ccp_addr_hi(&op->dst.u.dma);
	} else {
		cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
		cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
	}

	if (op->eom)
		cr[0] |= REQ1_EOM;

	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
}

static int ccp_perform_ecc(struct ccp_op *op)
{
	u32 cr[6];

	/* Fill out the register contents for REQ1 through REQ6 */
	cr[0] = REQ1_ECC_AFFINE_CONVERT
		| (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
		| (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
		| REQ1_EOM;
	cr[1] = op->src.u.dma.length - 1;
	cr[2] = ccp_addr_lo(&op->src.u.dma);
	cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
		| ccp_addr_hi(&op->src.u.dma);
	cr[4] = ccp_addr_lo(&op->dst.u.dma);
	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
		| ccp_addr_hi(&op->dst.u.dma);

	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
}

static int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
{
	struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng);
	u32 trng_value;
	int len = min_t(int, sizeof(trng_value), max);

	/*
	 * Locking is provided by the caller so we can update device
	 * hwrng-related fields safely
	 */
	trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG);
	if (!trng_value) {
		/* Zero is returned if not data is available or if a
		 * bad-entropy error is present. Assume an error if
		 * we exceed TRNG_RETRIES reads of zero.
		 */
		if (ccp->hwrng_retries++ > TRNG_RETRIES)
			return -EIO;

		return 0;
	}

	/* Reset the counter and save the rng value */
	ccp->hwrng_retries = 0;
	memcpy(data, &trng_value, len);

	return len;
}

static int ccp_init(struct ccp_device *ccp)
{
	struct device *dev = ccp->dev;
	struct ccp_cmd_queue *cmd_q;
	struct dma_pool *dma_pool;
	char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
	unsigned int qmr, qim, i;
	int ret;

	/* Find available queues */
	qim = 0;
	qmr = ioread32(ccp->io_regs + Q_MASK_REG);
	for (i = 0; i < MAX_HW_QUEUES; i++) {
		if (!(qmr & (1 << i)))
			continue;

		/* Allocate a dma pool for this queue */
		snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
			 ccp->name, i);
		dma_pool = dma_pool_create(dma_pool_name, dev,
					   CCP_DMAPOOL_MAX_SIZE,
					   CCP_DMAPOOL_ALIGN, 0);
		if (!dma_pool) {
			dev_err(dev, "unable to allocate dma pool\n");
			ret = -ENOMEM;
			goto e_pool;
		}

		cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
		ccp->cmd_q_count++;

		cmd_q->ccp = ccp;
		cmd_q->id = i;
		cmd_q->dma_pool = dma_pool;

		/* Reserve 2 KSB regions for the queue */
		cmd_q->ksb_key = KSB_START + ccp->ksb_start++;
		cmd_q->ksb_ctx = KSB_START + ccp->ksb_start++;
		ccp->ksb_count -= 2;

		/* Preset some register values and masks that are queue
		 * number dependent
		 */
		cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE +
				    (CMD_Q_STATUS_INCR * i);
		cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +
					(CMD_Q_STATUS_INCR * i);
		cmd_q->int_ok = 1 << (i * 2);
		cmd_q->int_err = 1 << ((i * 2) + 1);

		cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));

		init_waitqueue_head(&cmd_q->int_queue);

		/* Build queue interrupt mask (two interrupts per queue) */
		qim |= cmd_q->int_ok | cmd_q->int_err;

#ifdef CONFIG_ARM64
		/* For arm64 set the recommended queue cache settings */
		iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE +
			  (CMD_Q_CACHE_INC * i));
#endif

		dev_dbg(dev, "queue #%u available\n", i);
	}
	if (ccp->cmd_q_count == 0) {
		dev_notice(dev, "no command queues available\n");
		ret = -EIO;
		goto e_pool;
	}
	dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);

	/* Disable and clear interrupts until ready */
	iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
	for (i = 0; i < ccp->cmd_q_count; i++) {
		cmd_q = &ccp->cmd_q[i];

		ioread32(cmd_q->reg_int_status);
		ioread32(cmd_q->reg_status);
	}
	iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG);

	/* Request an irq */
	ret = ccp->get_irq(ccp);
	if (ret) {
		dev_err(dev, "unable to allocate an IRQ\n");
		goto e_pool;
	}

	/* Initialize the queues used to wait for KSB space and suspend */
	init_waitqueue_head(&ccp->ksb_queue);
	init_waitqueue_head(&ccp->suspend_queue);

	/* Create a kthread for each queue */
	for (i = 0; i < ccp->cmd_q_count; i++) {
		struct task_struct *kthread;

		cmd_q = &ccp->cmd_q[i];

		kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
					 "%s-q%u", ccp->name, cmd_q->id);
		if (IS_ERR(kthread)) {
			dev_err(dev, "error creating queue thread (%ld)\n",
				PTR_ERR(kthread));
			ret = PTR_ERR(kthread);
			goto e_kthread;
		}

		cmd_q->kthread = kthread;
		wake_up_process(kthread);
	}

	/* Register the RNG */
	ccp->hwrng.name = ccp->rngname;
	ccp->hwrng.read = ccp_trng_read;
	ret = hwrng_register(&ccp->hwrng);
	if (ret) {
		dev_err(dev, "error registering hwrng (%d)\n", ret);
		goto e_kthread;
	}

	/* Register the DMA engine support */
	ret = ccp_dmaengine_register(ccp);
	if (ret)
		goto e_hwrng;

	ccp_add_device(ccp);

	/* Enable interrupts */
	iowrite32(qim, ccp->io_regs + IRQ_MASK_REG);

	return 0;

e_hwrng:
	hwrng_unregister(&ccp->hwrng);

e_kthread:
	for (i = 0; i < ccp->cmd_q_count; i++)
		if (ccp->cmd_q[i].kthread)
			kthread_stop(ccp->cmd_q[i].kthread);

	ccp->free_irq(ccp);

e_pool:
	for (i = 0; i < ccp->cmd_q_count; i++)
		dma_pool_destroy(ccp->cmd_q[i].dma_pool);

	return ret;
}

static void ccp_destroy(struct ccp_device *ccp)
{
	struct ccp_cmd_queue *cmd_q;
	struct ccp_cmd *cmd;
	unsigned int qim, i;

	/* Remove this device from the list of available units first */
	ccp_del_device(ccp);

	/* Unregister the DMA engine */
	ccp_dmaengine_unregister(ccp);

	/* Unregister the RNG */
	hwrng_unregister(&ccp->hwrng);

	/* Stop the queue kthreads */
	for (i = 0; i < ccp->cmd_q_count; i++)
		if (ccp->cmd_q[i].kthread)
			kthread_stop(ccp->cmd_q[i].kthread);

	/* Build queue interrupt mask (two interrupt masks per queue) */
	qim = 0;
	for (i = 0; i < ccp->cmd_q_count; i++) {
		cmd_q = &ccp->cmd_q[i];
		qim |= cmd_q->int_ok | cmd_q->int_err;
	}

	/* Disable and clear interrupts */
	iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
	for (i = 0; i < ccp->cmd_q_count; i++) {
		cmd_q = &ccp->cmd_q[i];

		ioread32(cmd_q->reg_int_status);
		ioread32(cmd_q->reg_status);
	}
	iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG);

	ccp->free_irq(ccp);

	for (i = 0; i < ccp->cmd_q_count; i++)
		dma_pool_destroy(ccp->cmd_q[i].dma_pool);

	/* Flush the cmd and backlog queue */
	while (!list_empty(&ccp->cmd)) {
		/* Invoke the callback directly with an error code */
		cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
		list_del(&cmd->entry);
		cmd->callback(cmd->data, -ENODEV);
	}
	while (!list_empty(&ccp->backlog)) {
		/* Invoke the callback directly with an error code */
		cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
		list_del(&cmd->entry);
		cmd->callback(cmd->data, -ENODEV);
	}
}

static irqreturn_t ccp_irq_handler(int irq, void *data)
{
	struct device *dev = data;
	struct ccp_device *ccp = dev_get_drvdata(dev);
	struct ccp_cmd_queue *cmd_q;
	u32 q_int, status;
	unsigned int i;

	status = ioread32(ccp->io_regs + IRQ_STATUS_REG);

	for (i = 0; i < ccp->cmd_q_count; i++) {
		cmd_q = &ccp->cmd_q[i];

		q_int = status & (cmd_q->int_ok | cmd_q->int_err);
		if (q_int) {
			cmd_q->int_status = status;
			cmd_q->q_status = ioread32(cmd_q->reg_status);
			cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);

			/* On error, only save the first error value */
			if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error)
				cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);

			cmd_q->int_rcvd = 1;

			/* Acknowledge the interrupt and wake the kthread */
			iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);
			wake_up_interruptible(&cmd_q->int_queue);
		}
	}

	return IRQ_HANDLED;
}

static const struct ccp_actions ccp3_actions = {
	.perform_aes = ccp_perform_aes,
	.perform_xts_aes = ccp_perform_xts_aes,
	.perform_sha = ccp_perform_sha,
	.perform_rsa = ccp_perform_rsa,
	.perform_passthru = ccp_perform_passthru,
	.perform_ecc = ccp_perform_ecc,
	.init = ccp_init,
	.destroy = ccp_destroy,
	.irqhandler = ccp_irq_handler,
};

struct ccp_vdata ccpv3 = {
	.version = CCP_VERSION(3, 0),
	.perform = &ccp3_actions,
	.bar = 2,
	.offset = 0x20000,
};
Commit	Line	Data
ea0375af GH	1	/*
	2	* AMD Cryptographic Coprocessor (CCP) driver
	3	*
	4	* Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
	5	*
	6	* Author: Tom Lendacky <thomas.lendacky@amd.com>
fba8855c	7	* Author: Gary R Hook <gary.hook@amd.com>
ea0375af GH	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License version 2 as
	11	* published by the Free Software Foundation.
	12	*/
	13
	14	#include <linux/module.h>
	15	#include <linux/kernel.h>
	16	#include <linux/pci.h>
	17	#include <linux/kthread.h>
	18	#include <linux/interrupt.h>
	19	#include <linux/ccp.h>
	20
	21	#include "ccp-dev.h"
	22
	23	static int ccp_do_cmd(struct ccp_op op, u32 cr, unsigned int cr_count)
	24	{
	25	struct ccp_cmd_queue *cmd_q = op->cmd_q;
	26	struct ccp_device *ccp = cmd_q->ccp;
	27	void __iomem *cr_addr;
	28	u32 cr0, cmd;
	29	unsigned int i;
	30	int ret = 0;
	31
	32	/* We could read a status register to see how many free slots
	33	* are actually available, but reading that register resets it
	34	* and you could lose some error information.
	35	*/
	36	cmd_q->free_slots--;
	37
	38	cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
	39	\| (op->jobid << REQ0_JOBID_SHIFT)
	40	\| REQ0_WAIT_FOR_WRITE;
	41
	42	if (op->soc)
	43	cr0 \|= REQ0_STOP_ON_COMPLETE
	44	\| REQ0_INT_ON_COMPLETE;
	45
	46	if (op->ioc \|\| !cmd_q->free_slots)
	47	cr0 \|= REQ0_INT_ON_COMPLETE;
	48
	49	/* Start at CMD_REQ1 */
	50	cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
	51
	52	mutex_lock(&ccp->req_mutex);
	53
	54	/* Write CMD_REQ1 through CMD_REQx first */
	55	for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
	56	iowrite32(*(cr + i), cr_addr);
	57
	58	/* Tell the CCP to start */
	59	wmb();
	60	iowrite32(cr0, ccp->io_regs + CMD_REQ0);
	61
	62	mutex_unlock(&ccp->req_mutex);
	63
	64	if (cr0 & REQ0_INT_ON_COMPLETE) {
	65	/* Wait for the job to complete */
	66	ret = wait_event_interruptible(cmd_q->int_queue,
	67	cmd_q->int_rcvd);
	68	if (ret \|\| cmd_q->cmd_error) {
	69	/* On error delete all related jobs from the queue */
	70	cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
	71	\| op->jobid;
72
73	iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
74
75	if (!ret)
76	ret = -EIO;
77	} else if (op->soc) {
78	/* Delete just head job from the queue on SoC */
79	cmd = DEL_Q_ACTIVE
80	\| (cmd_q->id << DEL_Q_ID_SHIFT)
81	\| op->jobid;
82
83	iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
84	}
85
86	cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
87
88	cmd_q->int_rcvd = 0;
89	}
90
91	return ret;
92	}
93
94	static int ccp_perform_aes(struct ccp_op *op)
95	{
96	u32 cr[6];
97
98	/* Fill out the register contents for REQ1 through REQ6 */
99	cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
100	\| (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
101	\| (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
102	\| (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
103	\| (op->ksb_key << REQ1_KEY_KSB_SHIFT);
104	cr[1] = op->src.u.dma.length - 1;
105	cr[2] = ccp_addr_lo(&op->src.u.dma);
106	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
107	\| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
108	\| ccp_addr_hi(&op->src.u.dma);
109	cr[4] = ccp_addr_lo(&op->dst.u.dma);
110	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
111	\| ccp_addr_hi(&op->dst.u.dma);
112
113	if (op->u.aes.mode == CCP_AES_MODE_CFB)
114	cr[0] \|= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
115
116	if (op->eom)
117	cr[0] \|= REQ1_EOM;
118
119	if (op->init)
120	cr[0] \|= REQ1_INIT;
121
122	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
123	}
124
125	static int ccp_perform_xts_aes(struct ccp_op *op)
126	{
127	u32 cr[6];
128
129	/* Fill out the register contents for REQ1 through REQ6 */
130	cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
131	\| (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
132	\| (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
133	\| (op->ksb_key << REQ1_KEY_KSB_SHIFT);
134	cr[1] = op->src.u.dma.length - 1;
135	cr[2] = ccp_addr_lo(&op->src.u.dma);
136	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
137	\| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
138	\| ccp_addr_hi(&op->src.u.dma);
139	cr[4] = ccp_addr_lo(&op->dst.u.dma);
140	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
141	\| ccp_addr_hi(&op->dst.u.dma);
142
143	if (op->eom)
144	cr[0] \|= REQ1_EOM;
145
146	if (op->init)
147	cr[0] \|= REQ1_INIT;
148
149	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
150	}
151
152	static int ccp_perform_sha(struct ccp_op *op)
153	{
154	u32 cr[6];
155
156	/* Fill out the register contents for REQ1 through REQ6 */
157	cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
158	\| (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
159	\| REQ1_INIT;
160	cr[1] = op->src.u.dma.length - 1;
161	cr[2] = ccp_addr_lo(&op->src.u.dma);
162	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
163	\| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
164	\| ccp_addr_hi(&op->src.u.dma);
165
166	if (op->eom) {
167	cr[0] \|= REQ1_EOM;
168	cr[4] = lower_32_bits(op->u.sha.msg_bits);
169	cr[5] = upper_32_bits(op->u.sha.msg_bits);
170	} else {
171	cr[4] = 0;
172	cr[5] = 0;
173	}
174
175	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
176	}
177
178	static int ccp_perform_rsa(struct ccp_op *op)
179	{
180	u32 cr[6];
181
182	/* Fill out the register contents for REQ1 through REQ6 */
183	cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
184	\| (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
185	\| (op->ksb_key << REQ1_KEY_KSB_SHIFT)
186	\| REQ1_EOM;
187	cr[1] = op->u.rsa.input_len - 1;
188	cr[2] = ccp_addr_lo(&op->src.u.dma);
189	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
190	\| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
191	\| ccp_addr_hi(&op->src.u.dma);
192	cr[4] = ccp_addr_lo(&op->dst.u.dma);
193	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
194	\| ccp_addr_hi(&op->dst.u.dma);
195
196	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
197	}
198
199	static int ccp_perform_passthru(struct ccp_op *op)
200	{
201	u32 cr[6];
202
203	/* Fill out the register contents for REQ1 through REQ6 */
204	cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
205	\| (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
206	\| (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
207
208	if (op->src.type == CCP_MEMTYPE_SYSTEM)
209	cr[1] = op->src.u.dma.length - 1;
210	else
211	cr[1] = op->dst.u.dma.length - 1;
212
213	if (op->src.type == CCP_MEMTYPE_SYSTEM) {
214	cr[2] = ccp_addr_lo(&op->src.u.dma);
215	cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
216	\| ccp_addr_hi(&op->src.u.dma);
217
218	if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
219	cr[3] \|= (op->ksb_key << REQ4_KSB_SHIFT);
220	} else {
221	cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
222	cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
223	}
224
225	if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
226	cr[4] = ccp_addr_lo(&op->dst.u.dma);
227	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
228	\| ccp_addr_hi(&op->dst.u.dma);
229	} else {
230	cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
231	cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
232	}
233
234	if (op->eom)
235	cr[0] \|= REQ1_EOM;
236
237	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
238	}
239
240	static int ccp_perform_ecc(struct ccp_op *op)
241	{
242	u32 cr[6];
243
244	/* Fill out the register contents for REQ1 through REQ6 */
245	cr[0] = REQ1_ECC_AFFINE_CONVERT
246	\| (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
247	\| (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
248	\| REQ1_EOM;
249	cr[1] = op->src.u.dma.length - 1;
250	cr[2] = ccp_addr_lo(&op->src.u.dma);
251	cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
252	\| ccp_addr_hi(&op->src.u.dma);
253	cr[4] = ccp_addr_lo(&op->dst.u.dma);
254	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
255	\| ccp_addr_hi(&op->dst.u.dma);
256
257	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
258	}
259
260	static int ccp_trng_read(struct hwrng rng, void data, size_t max, bool wait)
261	{
262	struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng);
263	u32 trng_value;
264	int len = min_t(int, sizeof(trng_value), max);
265
266	/*
267	* Locking is provided by the caller so we can update device
268	* hwrng-related fields safely
269	*/
270	trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG);
271	if (!trng_value) {
272	/* Zero is returned if not data is available or if a
273	* bad-entropy error is present. Assume an error if
274	* we exceed TRNG_RETRIES reads of zero.
275	*/
276	if (ccp->hwrng_retries++ > TRNG_RETRIES)
277	return -EIO;
278
279	return 0;
280	}
281
282	/* Reset the counter and save the rng value */
283	ccp->hwrng_retries = 0;
284	memcpy(data, &trng_value, len);
285
286	return len;
287	}
288
289	static int ccp_init(struct ccp_device *ccp)
290	{
291	struct device *dev = ccp->dev;
292	struct ccp_cmd_queue *cmd_q;
293	struct dma_pool *dma_pool;
294	char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
295	unsigned int qmr, qim, i;
296	int ret;
297
298	/* Find available queues */
299	qim = 0;
300	qmr = ioread32(ccp->io_regs + Q_MASK_REG);
301	for (i = 0; i < MAX_HW_QUEUES; i++) {
302	if (!(qmr & (1 << i)))
303	continue;
304
305	/* Allocate a dma pool for this queue */
306	snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
307	ccp->name, i);
308	dma_pool = dma_pool_create(dma_pool_name, dev,
309	CCP_DMAPOOL_MAX_SIZE,
310	CCP_DMAPOOL_ALIGN, 0);
311	if (!dma_pool) {
312	dev_err(dev, "unable to allocate dma pool\n");
313	ret = -ENOMEM;
314	goto e_pool;
315	}
316
317	cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
318	ccp->cmd_q_count++;
319
320	cmd_q->ccp = ccp;
321	cmd_q->id = i;
322	cmd_q->dma_pool = dma_pool;
323
324	/* Reserve 2 KSB regions for the queue */
325	cmd_q->ksb_key = KSB_START + ccp->ksb_start++;
326	cmd_q->ksb_ctx = KSB_START + ccp->ksb_start++;
327	ccp->ksb_count -= 2;
328
329	/* Preset some register values and masks that are queue
330	* number dependent
331	*/
332	cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE +
333	(CMD_Q_STATUS_INCR * i);
334	cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +
335	(CMD_Q_STATUS_INCR * i);
336	cmd_q->int_ok = 1 << (i * 2);
337	cmd_q->int_err = 1 << ((i * 2) + 1);
338
339	cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
340
341	init_waitqueue_head(&cmd_q->int_queue);
342
343	/* Build queue interrupt mask (two interrupts per queue) */
344	qim \|= cmd_q->int_ok \| cmd_q->int_err;
345
346	#ifdef CONFIG_ARM64
347	/* For arm64 set the recommended queue cache settings */
348	iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE +
349	(CMD_Q_CACHE_INC * i));
350	#endif
351
352	dev_dbg(dev, "queue #%u available\n", i);
353	}
354	if (ccp->cmd_q_count == 0) {
355	dev_notice(dev, "no command queues available\n");
356	ret = -EIO;
357	goto e_pool;
358	}
359	dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
360
361	/* Disable and clear interrupts until ready */
362	iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
363	for (i = 0; i < ccp->cmd_q_count; i++) {
364	cmd_q = &ccp->cmd_q[i];
365
366	ioread32(cmd_q->reg_int_status);
367	ioread32(cmd_q->reg_status);
368	}
369	iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG);
370
371	/* Request an irq */
372	ret = ccp->get_irq(ccp);
373	if (ret) {
374	dev_err(dev, "unable to allocate an IRQ\n");
375	goto e_pool;
376	}
377
378	/* Initialize the queues used to wait for KSB space and suspend */
379	init_waitqueue_head(&ccp->ksb_queue);
380	init_waitqueue_head(&ccp->suspend_queue);
381
382	/* Create a kthread for each queue */
383	for (i = 0; i < ccp->cmd_q_count; i++) {
384	struct task_struct *kthread;
385
386	cmd_q = &ccp->cmd_q[i];
387
388	kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
389	"%s-q%u", ccp->name, cmd_q->id);
390	if (IS_ERR(kthread)) {
391	dev_err(dev, "error creating queue thread (%ld)\n",
392	PTR_ERR(kthread));
393	ret = PTR_ERR(kthread);
394	goto e_kthread;
395	}
396
397	cmd_q->kthread = kthread;
398	wake_up_process(kthread);
399	}
400
401	/* Register the RNG */
402	ccp->hwrng.name = ccp->rngname;
403	ccp->hwrng.read = ccp_trng_read;
404	ret = hwrng_register(&ccp->hwrng);
405	if (ret) {
406	dev_err(dev, "error registering hwrng (%d)\n", ret);
407	goto e_kthread;
408	}
409
58ea8abf GH	410	/* Register the DMA engine support */
	411	ret = ccp_dmaengine_register(ccp);
	412	if (ret)
	413	goto e_hwrng;
	414
ea0375af GH	415	ccp_add_device(ccp);
	416
	417	/* Enable interrupts */
	418	iowrite32(qim, ccp->io_regs + IRQ_MASK_REG);
	419
	420	return 0;
	421
58ea8abf GH	422	e_hwrng:
	423	hwrng_unregister(&ccp->hwrng);
	424
ea0375af GH	425	e_kthread:
	426	for (i = 0; i < ccp->cmd_q_count; i++)
	427	if (ccp->cmd_q[i].kthread)
	428	kthread_stop(ccp->cmd_q[i].kthread);
	429
	430	ccp->free_irq(ccp);
	431
	432	e_pool:
	433	for (i = 0; i < ccp->cmd_q_count; i++)
	434	dma_pool_destroy(ccp->cmd_q[i].dma_pool);
	435
	436	return ret;
	437	}
	438
	439	static void ccp_destroy(struct ccp_device *ccp)
	440	{
	441	struct ccp_cmd_queue *cmd_q;
	442	struct ccp_cmd *cmd;
	443	unsigned int qim, i;
	444
	445	/* Remove this device from the list of available units first */
	446	ccp_del_device(ccp);
	447
58ea8abf GH	448	/* Unregister the DMA engine */
	449	ccp_dmaengine_unregister(ccp);
	450
ea0375af GH	451	/* Unregister the RNG */
	452	hwrng_unregister(&ccp->hwrng);
	453
	454	/* Stop the queue kthreads */
	455	for (i = 0; i < ccp->cmd_q_count; i++)
	456	if (ccp->cmd_q[i].kthread)
	457	kthread_stop(ccp->cmd_q[i].kthread);
	458
	459	/* Build queue interrupt mask (two interrupt masks per queue) */
	460	qim = 0;
	461	for (i = 0; i < ccp->cmd_q_count; i++) {
	462	cmd_q = &ccp->cmd_q[i];
	463	qim \|= cmd_q->int_ok \| cmd_q->int_err;
	464	}
	465
	466	/* Disable and clear interrupts */
	467	iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
	468	for (i = 0; i < ccp->cmd_q_count; i++) {
	469	cmd_q = &ccp->cmd_q[i];
	470
	471	ioread32(cmd_q->reg_int_status);
	472	ioread32(cmd_q->reg_status);
	473	}
	474	iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG);
	475
	476	ccp->free_irq(ccp);
	477
	478	for (i = 0; i < ccp->cmd_q_count; i++)
	479	dma_pool_destroy(ccp->cmd_q[i].dma_pool);
	480
	481	/* Flush the cmd and backlog queue */
	482	while (!list_empty(&ccp->cmd)) {
	483	/* Invoke the callback directly with an error code */
	484	cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
	485	list_del(&cmd->entry);
	486	cmd->callback(cmd->data, -ENODEV);
	487	}
	488	while (!list_empty(&ccp->backlog)) {
	489	/* Invoke the callback directly with an error code */
	490	cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
	491	list_del(&cmd->entry);
	492	cmd->callback(cmd->data, -ENODEV);
	493	}
	494	}
	495
	496	static irqreturn_t ccp_irq_handler(int irq, void *data)
	497	{
	498	struct device *dev = data;
	499	struct ccp_device *ccp = dev_get_drvdata(dev);
	500	struct ccp_cmd_queue *cmd_q;
	501	u32 q_int, status;
	502	unsigned int i;
	503
	504	status = ioread32(ccp->io_regs + IRQ_STATUS_REG);
	505
	506	for (i = 0; i < ccp->cmd_q_count; i++) {
	507	cmd_q = &ccp->cmd_q[i];
	508
	509	q_int = status & (cmd_q->int_ok \| cmd_q->int_err);
	510	if (q_int) {
	511	cmd_q->int_status = status;
	512	cmd_q->q_status = ioread32(cmd_q->reg_status);
	513	cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
	514
515	/* On error, only save the first error value */
516	if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error)
517	cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
518
519	cmd_q->int_rcvd = 1;
520
521	/* Acknowledge the interrupt and wake the kthread */
522	iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);
523	wake_up_interruptible(&cmd_q->int_queue);
524	}
525	}
526
527	return IRQ_HANDLED;
528	}
529
bc197b2a	530	static const struct ccp_actions ccp3_actions = {
ea0375af GH	531	.perform_aes = ccp_perform_aes,
	532	.perform_xts_aes = ccp_perform_xts_aes,
	533	.perform_sha = ccp_perform_sha,
	534	.perform_rsa = ccp_perform_rsa,
	535	.perform_passthru = ccp_perform_passthru,
	536	.perform_ecc = ccp_perform_ecc,
	537	.init = ccp_init,
	538	.destroy = ccp_destroy,
	539	.irqhandler = ccp_irq_handler,
	540	};
	541
	542	struct ccp_vdata ccpv3 = {
	543	.version = CCP_VERSION(3, 0),
	544	.perform = &ccp3_actions,
fba8855c GH	545	.bar = 2,
fba8855c GH	546	.offset = 0x20000,
ea0375af	547	};